feat(ingestion): harden OFF ingestion for bulk seeding

- Add retry/backoff (429 + 5xx, Retry-After aware) to the OFF adapter so
  transient API errors no longer abort a run.
- Clamp bounded text fields (serving_size, net_content_unit,
  country_of_origin) to their column widths in transform; long OFF values
  previously raised StringDataRightTruncation and rolled back the batch.
- Load each record inside a savepoint (load_record_safe) so one malformed
  source record is skipped instead of aborting the whole import; jobs now
  report an errored count.
- Tests for retry behaviour, serving_size clamping, and per-record isolation.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

ingestion/tests/test_load_integration.py

@@ -11,7 +11,7 @@ from pathlib import Path
 
 import pytest
 
-from opengoods.etl.load import default_dsn, ensure_source, load_record
+from opengoods.etl.load import default_dsn, ensure_source, load_record, load_record_safe
 from opengoods.etl.transform import transform
 
 psycopg = pytest.importorskip("psycopg")
@@ -59,3 +59,25 @@ def test_load_record_roundtrip(conn):
     assert prov[0] >= 1
 
     conn.rollback()  # keep the test DB clean
+
+
+def test_load_record_safe_isolates_bad_record(conn):
+    source_id = ensure_source(conn)
+
+    # Unique gtin so the good record is a fresh INSERT, not an upsert/update.
+    good = transform(FIXTURE)
+    good["gtin"] = "4006381333931"
+    assert load_record_safe(conn, good, source_id, FIXTURE) is True
+    after_good = conn.execute("SELECT count(*) FROM product").fetchone()[0]
+
+    # A record whose name violates NOT NULL must not abort the batch.
+    bad = dict(good)
+    bad["gtin"] = "5000112637922"
+    bad["name"] = None
+    assert load_record_safe(conn, bad, source_id, {}) is False
+
+    # The good record survived the bad one's rollback-to-savepoint.
+    after_bad = conn.execute("SELECT count(*) FROM product").fetchone()[0]
+    assert after_bad == after_good
+
+    conn.rollback()  # keep the test DB clean

ingestion/tests/test_off_retry.py

@@ -0,0 +1,59 @@
+import httpx
+import pytest
+
+from opengoods.adapters.openfoodfacts import OpenFoodFactsAdapter
+
+
+def _adapter(handler, **kwargs):
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    return OpenFoodFactsAdapter(client=client, min_interval=0, backoff_base=0, **kwargs)
+
+
+def test_retries_transient_5xx_then_succeeds():
+    calls = {"n": 0}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls["n"] += 1
+        if calls["n"] < 3:
+            return httpx.Response(503)
+        return httpx.Response(200, json={"status": 1, "product": {"code": "x"}})
+
+    adapter = _adapter(handler, max_retries=4)
+    assert adapter.fetch_barcode("x") == {"code": "x"}
+    assert calls["n"] == 3  # two 503s retried, third succeeds
+
+
+def test_retries_exhausted_raises():
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(503)
+
+    adapter = _adapter(handler, max_retries=2)
+    with pytest.raises(httpx.HTTPStatusError):
+        adapter.fetch_barcode("x")
+
+
+def test_non_retryable_4xx_not_retried():
+    calls = {"n": 0}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls["n"] += 1
+        return httpx.Response(404)
+
+    adapter = _adapter(handler, max_retries=4)
+    with pytest.raises(httpx.HTTPStatusError):
+        adapter.fetch_barcode("x")
+    assert calls["n"] == 1  # 404 is not retried
+
+
+def test_retries_connection_error_then_succeeds():
+    calls = {"n": 0}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls["n"] += 1
+        if calls["n"] == 1:
+            raise httpx.ConnectError("boom")
+        return httpx.Response(200, json={"status": 1, "product": {"code": "y"}})
+
+    adapter = _adapter(handler, max_retries=4)
+    assert adapter.fetch_barcode("y") == {"code": "y"}
+    assert calls["n"] == 2

ingestion/tests/test_transform.py

@@ -65,3 +65,14 @@ def test_transform_full_record():
 
 def test_transform_drops_unnamed():
     assert transform({"code": "0000000000000"}) is None
+
+
+def test_transform_clamps_long_serving_size():
+    raw = {
+        "code": "3017624010701",
+        "product_name": "X",
+        "serving_size": "1 portion (30 g) / 1 portion (30 g) / extra long descriptive text here",
+    }
+    rec = transform(raw)
+    assert rec is not None
+    assert len(rec["food"]["serving_size"]) == 32