M4: ingestion management (incremental, GS1 supplement, dedup/conflict, quality, scheduler)

- OFF incremental fetch via search API + persistent watermark (ingest_state, migration 0004)
- GS1 barcode supplement adapter (offline mapping + GS1-style API) filling only gaps with field-level provenance
- Non-GTIN dedup with canonical selection + merge_log; field-level conflict resolution (source trust > recency)
- Quality scoring (0.4 completeness + 0.3 source trust + 0.2 multi-source + 0.1 freshness) wired into load/merge
- Jobs: update_off, dedup, schedule; docs/ingestion-management.md
- 19 new tests (pure + DB-integration), ruff clean

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

ingestion/opengoods/adapters/gs1.py

@@ -0,0 +1,124 @@
+"""GS1 barcode supplement adapter.
+
+GS1 (e.g. *Verified by GS1* / GS1 China) is the authoritative registry that maps
+a GTIN to its brand owner, product description and GPC category. We use it to
+*supplement* — fill gaps in — records gathered from crowd sources like Open Food
+Facts, never to overwrite existing values.
+
+Real GS1 access is credentialed and region-specific, so this adapter supports
+two modes:
+
+* **offline** (default): look barcodes up in a local JSON mapping file. This is
+  what tests and air-gapped runs use.
+* **online**: GET ``{base_url}/{gtin}`` with an API key header, then normalize
+  the response. Enabled by passing ``base_url`` + ``client``.
+
+Either way :meth:`fetch_barcode` returns a normalized *supplement* dict (or
+``None``); :mod:`opengoods.etl.supplement` applies it to the database.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from pathlib import Path
+
+import httpx
+
+SOURCE_NAME = "gs1"
+GS1_HOMEPAGE = "https://www.gs1.org"
+GS1_LICENSE = "proprietary"
+# GS1 is the authoritative barcode registry -> high trust.
+GS1_TRUST = 0.9
+
+# Keys of a normalized supplement record.
+_SUPPLEMENT_KEYS = (
+    "gtin",
+    "name",
+    "brand",
+    "manufacturer",
+    "gpc_brick_code",
+    "country_of_origin",
+    "net_content_value",
+    "net_content_unit",
+)
+
+
+def _normalize(code: str, data: dict) -> dict:
+    """Project a raw mapping/record onto the supplement schema (non-empty only)."""
+    rec: dict = {"gtin": code}
+    for key in _SUPPLEMENT_KEYS:
+        if key == "gtin":
+            continue
+        value = data.get(key)
+        if value not in (None, "", []):
+            rec[key] = value
+    return rec
+
+
+def _parse_api(code: str, payload: dict) -> dict:
+    """Best-effort mapping of a Verified-by-GS1 style payload to our schema."""
+    item = payload
+    if isinstance(payload.get("gtinRecords"), list) and payload["gtinRecords"]:
+        item = payload["gtinRecords"][0]
+    return _normalize(
+        code,
+        {
+            "name": item.get("productDescription") or item.get("description"),
+            "brand": item.get("brandName"),
+            "manufacturer": item.get("companyName") or item.get("licenseeName"),
+            "gpc_brick_code": item.get("gpcCategoryCode"),
+            "country_of_origin": item.get("countryOfSaleCode") or item.get("countryCode"),
+            "net_content_value": item.get("netContent"),
+            "net_content_unit": item.get("netContentUnit"),
+        },
+    )
+
+
+class GS1Adapter:
+    """Look up GTIN supplements from a local mapping or a GS1-style API."""
+
+    source_name = SOURCE_NAME
+
+    def __init__(
+        self,
+        mapping: dict | None = None,
+        *,
+        client: httpx.Client | None = None,
+        base_url: str | None = None,
+        api_key: str | None = None,
+    ) -> None:
+        self._mapping = mapping or {}
+        self._client = client
+        self._base_url = base_url.rstrip("/") if base_url else None
+        self._api_key = api_key
+
+    @classmethod
+    def from_file(cls, path: str | Path) -> GS1Adapter:
+        """Build an offline adapter from a JSON ``{gtin: {...}}`` mapping file."""
+        data = json.loads(Path(path).read_text(encoding="utf-8"))
+        return cls(mapping=data)
+
+    def fetch_barcode(self, code: str) -> dict | None:
+        """Return a normalized supplement dict for ``code`` (or ``None``)."""
+        if self._base_url and self._client is not None:
+            headers = {"apikey": self._api_key} if self._api_key else {}
+            resp = self._client.get(f"{self._base_url}/{code}", headers=headers)
+            if resp.status_code == 404:
+                return None
+            resp.raise_for_status()
+            rec = _parse_api(code, resp.json())
+        else:
+            data = self._mapping.get(code)
+            if not data:
+                return None
+            rec = _normalize(code, data)
+        # A record with only the GTIN carries no supplement.
+        return rec if len(rec) > 1 else None
+
+    def fetch(self, barcodes: list[str]) -> Iterator[dict]:
+        """Yield supplement records for the given barcodes."""
+        for code in barcodes:
+            rec = self.fetch_barcode(code)
+            if rec is not None:
+                yield rec

ingestion/opengoods/adapters/openfoodfacts.py

@@ -25,6 +25,16 @@ USER_AGENT = "OpenGoods/0.1 (+https://github.com/baicai2026-baicai/goods) public
 # Conservative client-side spacing between API calls (seconds).
 _DEFAULT_MIN_INTERVAL = 4.0
 _API_URL = "https://world.openfoodfacts.org/api/v2/product/{barcode}.json"
+_SEARCH_URL = "https://world.openfoodfacts.org/api/v2/search"
+
+# Fields requested from the search API so a returned product can be transformed
+# without an extra per-barcode round trip.
+_SEARCH_FIELDS = (
+    "code,product_name,product_name_en,product_name_zh,brands,quantity,"
+    "categories,categories_tags,countries,ingredients_text,allergens_tags,"
+    "additives_tags,nutriments,nutriscore_grade,serving_size,"
+    "image_front_url,image_url,last_modified_t"
+)
 
 
 class OpenFoodFactsAdapter:
@@ -65,6 +75,45 @@ class OpenFoodFactsAdapter:
             if record is not None:
                 yield record
 
+    def fetch_modified_since(
+        self,
+        since_t: int,
+        *,
+        page_size: int = 100,
+        max_pages: int = 10,
+    ) -> Iterator[dict]:
+        """Yield products modified after ``since_t`` (unix ``last_modified_t``).
+
+        Uses the OFF search API sorted by ``last_modified_t`` (most recent
+        first) and paginates until it reaches products at or before the
+        watermark, an empty/short page, or ``max_pages``. This is the
+        incremental ingestion path: callers persist the highest
+        ``last_modified_t`` they processed as the next watermark.
+        """
+        for page in range(1, max_pages + 1):
+            self._throttle()
+            resp = self._client.get(
+                _SEARCH_URL,
+                params={
+                    "fields": _SEARCH_FIELDS,
+                    "sort_by": "last_modified_t",
+                    "page": page,
+                    "page_size": page_size,
+                },
+            )
+            resp.raise_for_status()
+            products = resp.json().get("products") or []
+            if not products:
+                return
+            reached_old = False
+            for prod in products:
+                if int(prod.get("last_modified_t") or 0) <= since_t:
+                    reached_old = True
+                    break
+                yield prod
+            if reached_old or len(products) < page_size:
+                return
+
 
 def read_dump(path: str | Path) -> Iterator[dict]:
     """Yield raw product records from an OFF JSONL dump file.