M4: ingestion management (incremental, GS1 supplement, dedup/conflict, quality, scheduler)

- OFF incremental fetch via search API + persistent watermark (ingest_state, migration 0004)
- GS1 barcode supplement adapter (offline mapping + GS1-style API) filling only gaps with field-level provenance
- Non-GTIN dedup with canonical selection + merge_log; field-level conflict resolution (source trust > recency)
- Quality scoring (0.4 completeness + 0.3 source trust + 0.2 multi-source + 0.1 freshness) wired into load/merge
- Jobs: update_off, dedup, schedule; docs/ingestion-management.md
- 19 new tests (pure + DB-integration), ruff clean

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

ingestion/opengoods/adapters/gs1.py

@@ -0,0 +1,124 @@
+"""GS1 barcode supplement adapter.
+
+GS1 (e.g. *Verified by GS1* / GS1 China) is the authoritative registry that maps
+a GTIN to its brand owner, product description and GPC category. We use it to
+*supplement* — fill gaps in — records gathered from crowd sources like Open Food
+Facts, never to overwrite existing values.
+
+Real GS1 access is credentialed and region-specific, so this adapter supports
+two modes:
+
+* **offline** (default): look barcodes up in a local JSON mapping file. This is
+  what tests and air-gapped runs use.
+* **online**: GET ``{base_url}/{gtin}`` with an API key header, then normalize
+  the response. Enabled by passing ``base_url`` + ``client``.
+
+Either way :meth:`fetch_barcode` returns a normalized *supplement* dict (or
+``None``); :mod:`opengoods.etl.supplement` applies it to the database.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from pathlib import Path
+
+import httpx
+
+SOURCE_NAME = "gs1"
+GS1_HOMEPAGE = "https://www.gs1.org"
+GS1_LICENSE = "proprietary"
+# GS1 is the authoritative barcode registry -> high trust.
+GS1_TRUST = 0.9
+
+# Keys of a normalized supplement record.
+_SUPPLEMENT_KEYS = (
+    "gtin",
+    "name",
+    "brand",
+    "manufacturer",
+    "gpc_brick_code",
+    "country_of_origin",
+    "net_content_value",
+    "net_content_unit",
+)
+
+
+def _normalize(code: str, data: dict) -> dict:
+    """Project a raw mapping/record onto the supplement schema (non-empty only)."""
+    rec: dict = {"gtin": code}
+    for key in _SUPPLEMENT_KEYS:
+        if key == "gtin":
+            continue
+        value = data.get(key)
+        if value not in (None, "", []):
+            rec[key] = value
+    return rec
+
+
+def _parse_api(code: str, payload: dict) -> dict:
+    """Best-effort mapping of a Verified-by-GS1 style payload to our schema."""
+    item = payload
+    if isinstance(payload.get("gtinRecords"), list) and payload["gtinRecords"]:
+        item = payload["gtinRecords"][0]
+    return _normalize(
+        code,
+        {
+            "name": item.get("productDescription") or item.get("description"),
+            "brand": item.get("brandName"),
+            "manufacturer": item.get("companyName") or item.get("licenseeName"),
+            "gpc_brick_code": item.get("gpcCategoryCode"),
+            "country_of_origin": item.get("countryOfSaleCode") or item.get("countryCode"),
+            "net_content_value": item.get("netContent"),
+            "net_content_unit": item.get("netContentUnit"),
+        },
+    )
+
+
+class GS1Adapter:
+    """Look up GTIN supplements from a local mapping or a GS1-style API."""
+
+    source_name = SOURCE_NAME
+
+    def __init__(
+        self,
+        mapping: dict | None = None,
+        *,
+        client: httpx.Client | None = None,
+        base_url: str | None = None,
+        api_key: str | None = None,
+    ) -> None:
+        self._mapping = mapping or {}
+        self._client = client
+        self._base_url = base_url.rstrip("/") if base_url else None
+        self._api_key = api_key
+
+    @classmethod
+    def from_file(cls, path: str | Path) -> GS1Adapter:
+        """Build an offline adapter from a JSON ``{gtin: {...}}`` mapping file."""
+        data = json.loads(Path(path).read_text(encoding="utf-8"))
+        return cls(mapping=data)
+
+    def fetch_barcode(self, code: str) -> dict | None:
+        """Return a normalized supplement dict for ``code`` (or ``None``)."""
+        if self._base_url and self._client is not None:
+            headers = {"apikey": self._api_key} if self._api_key else {}
+            resp = self._client.get(f"{self._base_url}/{code}", headers=headers)
+            if resp.status_code == 404:
+                return None
+            resp.raise_for_status()
+            rec = _parse_api(code, resp.json())
+        else:
+            data = self._mapping.get(code)
+            if not data:
+                return None
+            rec = _normalize(code, data)
+        # A record with only the GTIN carries no supplement.
+        return rec if len(rec) > 1 else None
+
+    def fetch(self, barcodes: list[str]) -> Iterator[dict]:
+        """Yield supplement records for the given barcodes."""
+        for code in barcodes:
+            rec = self.fetch_barcode(code)
+            if rec is not None:
+                yield rec

ingestion/opengoods/adapters/openfoodfacts.py

@@ -25,6 +25,16 @@ USER_AGENT = "OpenGoods/0.1 (+https://github.com/baicai2026-baicai/goods) public
 # Conservative client-side spacing between API calls (seconds).
 _DEFAULT_MIN_INTERVAL = 4.0
 _API_URL = "https://world.openfoodfacts.org/api/v2/product/{barcode}.json"
+_SEARCH_URL = "https://world.openfoodfacts.org/api/v2/search"
+
+# Fields requested from the search API so a returned product can be transformed
+# without an extra per-barcode round trip.
+_SEARCH_FIELDS = (
+    "code,product_name,product_name_en,product_name_zh,brands,quantity,"
+    "categories,categories_tags,countries,ingredients_text,allergens_tags,"
+    "additives_tags,nutriments,nutriscore_grade,serving_size,"
+    "image_front_url,image_url,last_modified_t"
+)
 
 
 class OpenFoodFactsAdapter:
@@ -65,6 +75,45 @@ class OpenFoodFactsAdapter:
             if record is not None:
                 yield record
 
+    def fetch_modified_since(
+        self,
+        since_t: int,
+        *,
+        page_size: int = 100,
+        max_pages: int = 10,
+    ) -> Iterator[dict]:
+        """Yield products modified after ``since_t`` (unix ``last_modified_t``).
+
+        Uses the OFF search API sorted by ``last_modified_t`` (most recent
+        first) and paginates until it reaches products at or before the
+        watermark, an empty/short page, or ``max_pages``. This is the
+        incremental ingestion path: callers persist the highest
+        ``last_modified_t`` they processed as the next watermark.
+        """
+        for page in range(1, max_pages + 1):
+            self._throttle()
+            resp = self._client.get(
+                _SEARCH_URL,
+                params={
+                    "fields": _SEARCH_FIELDS,
+                    "sort_by": "last_modified_t",
+                    "page": page,
+                    "page_size": page_size,
+                },
+            )
+            resp.raise_for_status()
+            products = resp.json().get("products") or []
+            if not products:
+                return
+            reached_old = False
+            for prod in products:
+                if int(prod.get("last_modified_t") or 0) <= since_t:
+                    reached_old = True
+                    break
+                yield prod
+            if reached_old or len(products) < page_size:
+                return
+
 
 def read_dump(path: str | Path) -> Iterator[dict]:
     """Yield raw product records from an OFF JSONL dump file.

ingestion/opengoods/etl/dedup.py

@@ -0,0 +1,139 @@
+"""Duplicate detection and product merging.
+
+Barcodes (GTIN) are already unique at the schema level, so duplicates here are
+non-GTIN records that describe the same product (same normalized name + brand +
+net content). For each duplicate group we keep the highest-quality product as
+canonical and merge the rest into it: child rows (provenance, images, MSRP) are
+re-pointed to the canonical product, the merged product is marked ``merged``
+with ``canonical_id`` set, and a row is written to ``merge_log``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import psycopg
+
+from opengoods.etl.quality import update_quality
+
+
+def _norm(text: str | None) -> str:
+    return " ".join((text or "").lower().split())
+
+
+def product_signature(name: str | None, brand: str | None, net_canonical: Any | None) -> str | None:
+    """Stable signature for non-GTIN dedup, or ``None`` if too sparse to match."""
+    n = _norm(name)
+    if not n:
+        return None
+    net = "" if net_canonical is None else str(net_canonical)
+    return f"{n}|{_norm(brand)}|{net}"
+
+
+def choose_canonical(members: list[dict]) -> dict:
+    """Pick the canonical product: best quality, then oldest, then lowest id."""
+    return min(
+        members,
+        key=lambda m: (
+            -float(m.get("quality_score") or 0.0),
+            m.get("created_at"),
+            str(m.get("id")),
+        ),
+    )
+
+
+def find_duplicate_groups(conn: psycopg.Connection) -> list[list[dict]]:
+    """Return groups (size >= 2) of active products sharing a signature."""
+    rows = conn.execute(
+        """
+        SELECT p.id, p.name, b.normalized_name, p.net_content_canonical,
+               p.quality_score, p.created_at
+        FROM product p
+        LEFT JOIN brand b ON b.id = p.brand_id
+        WHERE p.status = 'active'
+        """
+    ).fetchall()
+
+    groups: dict[str, list[dict]] = {}
+    for r in rows:
+        sig = product_signature(r[1], r[2], r[3])
+        if sig is None:
+            continue
+        member = {
+            "id": r[0],
+            "name": r[1],
+            "quality_score": r[4],
+            "created_at": r[5],
+        }
+        groups.setdefault(sig, []).append(member)
+
+    return [m for m in groups.values() if len(m) >= 2]
+
+
+def merge_products(
+    conn: psycopg.Connection,
+    kept_id: str,
+    merged_id: str,
+    reason: str = "auto-dedup",
+    actor: str = "ingestion",
+) -> None:
+    """Merge ``merged_id`` into ``kept_id`` (re-point children, mark merged)."""
+    if kept_id == merged_id:
+        return
+
+    # Re-point provenance, images and MSRP to the canonical product.
+    conn.execute(
+        "UPDATE product_source SET product_id = %s WHERE product_id = %s",
+        (kept_id, merged_id),
+    )
+    conn.execute(
+        "UPDATE product_image SET product_id = %s WHERE product_id = %s",
+        (kept_id, merged_id),
+    )
+    conn.execute(
+        "UPDATE product_msrp SET product_id = %s WHERE product_id = %s",
+        (kept_id, merged_id),
+    )
+
+    # food_detail has product_id as PK, so it can only move if the canonical
+    # product does not already have one.
+    kept_has_food = conn.execute(
+        "SELECT 1 FROM food_detail WHERE product_id = %s", (kept_id,)
+    ).fetchone()
+    if not kept_has_food:
+        conn.execute(
+            "UPDATE food_detail SET product_id = %s WHERE product_id = %s",
+            (kept_id, merged_id),
+        )
+
+    conn.execute(
+        "UPDATE product SET status = 'merged', canonical_id = %s WHERE id = %s",
+        (kept_id, merged_id),
+    )
+    conn.execute(
+        """
+        INSERT INTO merge_log (kept_id, merged_id, reason, actor)
+        VALUES (%s, %s, %s, %s)
+        """,
+        (kept_id, merged_id, reason, actor),
+    )
+
+    # The canonical product gained sources, so its quality may have changed.
+    update_quality(conn, kept_id)
+
+
+def dedup_all(
+    conn: psycopg.Connection, actor: str = "ingestion", dry_run: bool = False
+) -> dict[str, int]:
+    """Merge every duplicate group. Returns counts of groups and merges."""
+    groups = find_duplicate_groups(conn)
+    merged = 0
+    for members in groups:
+        canonical = choose_canonical(members)
+        for m in members:
+            if m["id"] == canonical["id"]:
+                continue
+            if not dry_run:
+                merge_products(conn, canonical["id"], m["id"], actor=actor)
+            merged += 1
+    return {"groups": len(groups), "merged": merged}

ingestion/opengoods/etl/load.py

@@ -14,6 +14,7 @@ import psycopg
 from psycopg.types.json import Jsonb
 
 from opengoods.adapters.openfoodfacts import OFF_LICENSE, SOURCE_NAME
+from opengoods.etl.quality import update_quality
 
 OFF_HOMEPAGE = "https://world.openfoodfacts.org"
 
@@ -29,8 +30,14 @@ def _normalize_brand(name: str) -> str:
     return " ".join(name.lower().split())
 
 
-def ensure_source(conn: psycopg.Connection) -> str:
-    """Upsert the Open Food Facts source row and return its id."""
+def ensure_source_named(
+    conn: psycopg.Connection,
+    name: str,
+    homepage: str,
+    license: str,
+    trust_weight: float,
+) -> str:
+    """Upsert a source row by name and return its id."""
     row = conn.execute(
         """
         INSERT INTO source (name, homepage, license, trust_weight)
@@ -38,11 +45,16 @@ def ensure_source(conn: psycopg.Connection) -> str:
         ON CONFLICT (name) DO UPDATE SET homepage = EXCLUDED.homepage
         RETURNING id
         """,
-        (SOURCE_NAME, OFF_HOMEPAGE, OFF_LICENSE, 0.7),
+        (name, homepage, license, trust_weight),
     ).fetchone()
     return row[0]
 
 
+def ensure_source(conn: psycopg.Connection) -> str:
+    """Upsert the Open Food Facts source row and return its id."""
+    return ensure_source_named(conn, SOURCE_NAME, OFF_HOMEPAGE, OFF_LICENSE, 0.7)
+
+
 def _ensure_brand(conn: psycopg.Connection, name: str | None) -> str | None:
     if not name:
         return None
@@ -178,6 +190,9 @@ def load_record(conn: psycopg.Connection, rec: dict[str, Any], source_id: str, r
             Jsonb(_jsonable(raw)),
         ),
     )
+
+    # Recompute the data-quality score now that all facts + provenance exist.
+    update_quality(conn, product_id)
     return product_id
 
 

ingestion/opengoods/etl/merge.py

@@ -0,0 +1,104 @@
+"""Field-level conflict resolution for multi-source records.
+
+When more than one source describes the same product, each field may have
+several candidate values. We pick a winner per field by source trust first,
+then recency, ignoring empty values, and keep a provenance trail of which
+source won each field.
+
+These are pure functions (no DB / no network) so they are easy to unit-test;
+the DB-level record merge lives in :mod:`opengoods.etl.dedup`.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+
+
+@dataclass(frozen=True)
+class Candidate:
+    """One source's proposed value for a field."""
+
+    value: object
+    source: str
+    trust: float = 0.5
+    fetched_at: datetime | None = None
+
+
+@dataclass
+class FieldResolution:
+    """The winning value for a field plus the source it came from."""
+
+    value: object
+    source: str | None = None
+
+
+@dataclass
+class MergedRecord:
+    """A merged record with per-field provenance (field name -> source)."""
+
+    values: dict[str, object] = field(default_factory=dict)
+    provenance: dict[str, str] = field(default_factory=dict)
+
+
+def _is_empty(value: object) -> bool:
+    if value is None:
+        return True
+    if isinstance(value, str):
+        return value.strip() == ""
+    if isinstance(value, (list, dict, tuple, set)):
+        return len(value) == 0
+    return False
+
+
+def _sort_key(c: Candidate) -> tuple[float, float]:
+    ts = c.fetched_at.timestamp() if c.fetched_at is not None else float("-inf")
+    return (c.trust, ts)
+
+
+def resolve_field(candidates: list[Candidate]) -> FieldResolution | None:
+    """Pick the best non-empty candidate for one field.
+
+    Ranking: highest source trust, then most recent ``fetched_at``. Returns
+    ``None`` when there is no usable (non-empty) candidate.
+    """
+    usable = [c for c in candidates if not _is_empty(c.value)]
+    if not usable:
+        return None
+    winner = max(usable, key=_sort_key)
+    return FieldResolution(value=winner.value, source=winner.source)
+
+
+def merge_records(records: list[dict], *, fields: list[str] | None = None) -> MergedRecord:
+    """Merge several ``{field: Candidate|value}`` records into one.
+
+    Each input record maps field name -> :class:`Candidate` (preferred) or a
+    bare value (treated as trust 0.5, no timestamp). The result keeps, for each
+    field, the winning value and the name of the source that supplied it.
+    """
+    keys: list[str]
+    if fields is not None:
+        keys = list(fields)
+    else:
+        seen: dict[str, None] = {}
+        for rec in records:
+            for k in rec:
+                seen.setdefault(k, None)
+        keys = list(seen)
+
+    merged = MergedRecord()
+    for key in keys:
+        candidates: list[Candidate] = []
+        for rec in records:
+            if key not in rec:
+                continue
+            cand = rec[key]
+            if not isinstance(cand, Candidate):
+                cand = Candidate(value=cand, source="unknown")
+            candidates.append(cand)
+        resolution = resolve_field(candidates)
+        if resolution is not None:
+            merged.values[key] = resolution.value
+            if resolution.source is not None:
+                merged.provenance[key] = resolution.source
+    return merged

ingestion/opengoods/etl/quality.py

@@ -0,0 +1,176 @@
+"""Product data-quality scoring.
+
+The quality score is a 0..1 number combining four signals, per the locked
+project decision:
+
+    quality = 0.4 * completeness
+            + 0.3 * source_trust
+            + 0.2 * multi_source_agreement
+            + 0.1 * freshness
+
+Each component is itself normalized to 0..1. The pure helpers below are
+unit-testable; :func:`compute_quality` / :func:`update_quality` read the signals
+for a product out of the database and persist the result on ``product``.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import psycopg
+
+W_COMPLETENESS = 0.4
+W_SOURCE_TRUST = 0.3
+W_AGREEMENT = 0.2
+W_FRESHNESS = 0.1
+
+# Fields that count towards completeness (weighted equally).
+COMPLETENESS_FIELDS = (
+    "name",
+    "gtin",
+    "brand",
+    "category",
+    "net_content",
+    "country_of_origin",
+    "nutriments",
+    "ingredients",
+    "image",
+)
+
+
+def completeness(present: set[str]) -> float:
+    """Fraction of :data:`COMPLETENESS_FIELDS` that are present for a product."""
+    if not COMPLETENESS_FIELDS:
+        return 0.0
+    hits = sum(1 for f in COMPLETENESS_FIELDS if f in present)
+    return hits / len(COMPLETENESS_FIELDS)
+
+
+def agreement_from_sources(source_count: int) -> float:
+    """Multi-source corroboration proxy from the number of distinct sources.
+
+    A single source cannot be corroborated, so it scores a neutral 0.5; more
+    independent sources that describe the same product raise confidence.
+    """
+    if source_count <= 1:
+        return 0.5
+    if source_count == 2:
+        return 0.8
+    return 1.0
+
+
+def freshness_from_age(age_days: float | None) -> float:
+    """Recency score from the age (in days) of the most recent source fetch."""
+    if age_days is None:
+        return 0.5
+    if age_days <= 30:
+        return 1.0
+    if age_days <= 180:
+        return 0.8
+    if age_days <= 365:
+        return 0.6
+    if age_days <= 730:
+        return 0.4
+    return 0.2
+
+
+def score(
+    *,
+    completeness_score: float,
+    source_trust: float,
+    agreement: float,
+    freshness: float,
+) -> float:
+    """Combine the four normalized components into a 0..1 quality score."""
+    raw = (
+        W_COMPLETENESS * completeness_score
+        + W_SOURCE_TRUST * source_trust
+        + W_AGREEMENT * agreement
+        + W_FRESHNESS * freshness
+    )
+    return round(max(0.0, min(1.0, raw)), 3)
+
+
+def _present_fields(prod: dict, has_image: bool) -> set[str]:
+    present: set[str] = set()
+    if prod.get("name"):
+        present.add("name")
+    if prod.get("gtin"):
+        present.add("gtin")
+    if prod.get("brand_id"):
+        present.add("brand")
+    if prod.get("category_id"):
+        present.add("category")
+    if prod.get("net_content_canonical") is not None:
+        present.add("net_content")
+    if prod.get("country_of_origin"):
+        present.add("country_of_origin")
+    if prod.get("nutriments"):
+        present.add("nutriments")
+    if prod.get("ingredients_text"):
+        present.add("ingredients")
+    if has_image:
+        present.add("image")
+    return present
+
+
+def compute_quality(conn: psycopg.Connection, product_id: str) -> float:
+    """Compute (but do not persist) the quality score for one product."""
+    row = conn.execute(
+        """
+        SELECT p.name, p.gtin, p.brand_id, p.category_id, p.net_content_canonical,
+               p.country_of_origin, f.nutriments, f.ingredients_text,
+               EXISTS (SELECT 1 FROM product_image pi WHERE pi.product_id = p.id)
+        FROM product p
+        LEFT JOIN food_detail f ON f.product_id = p.id
+        WHERE p.id = %s
+        """,
+        (product_id,),
+    ).fetchone()
+    if row is None:
+        return 0.0
+    prod = {
+        "name": row[0],
+        "gtin": row[1],
+        "brand_id": row[2],
+        "category_id": row[3],
+        "net_content_canonical": row[4],
+        "country_of_origin": row[5],
+        "nutriments": row[6],
+        "ingredients_text": row[7],
+    }
+    has_image = bool(row[8])
+
+    src = conn.execute(
+        """
+        SELECT count(DISTINCT ps.source_id), COALESCE(max(s.trust_weight), 0), max(ps.fetched_at)
+        FROM product_source ps
+        LEFT JOIN source s ON s.id = ps.source_id
+        WHERE ps.product_id = %s
+        """,
+        (product_id,),
+    ).fetchone()
+    source_count = int(src[0] or 0)
+    source_trust = float(src[1] or 0.0)
+    last_fetched: datetime | None = src[2]
+
+    age_days: float | None = None
+    if last_fetched is not None:
+        now = datetime.now(UTC)
+        if last_fetched.tzinfo is None:
+            last_fetched = last_fetched.replace(tzinfo=UTC)
+        age_days = max(0.0, (now - last_fetched).total_seconds() / 86400.0)
+
+    return score(
+        completeness_score=completeness(_present_fields(prod, has_image)),
+        source_trust=source_trust,
+        agreement=agreement_from_sources(source_count),
+        freshness=freshness_from_age(age_days),
+    )
+
+
+def update_quality(conn: psycopg.Connection, product_id: str) -> float:
+    """Compute the quality score and write it to ``product.quality_score``."""
+    value = compute_quality(conn, product_id)
+    conn.execute("UPDATE product SET quality_score = %s WHERE id = %s", (value, product_id))
+    return value

ingestion/opengoods/etl/state.py

@@ -0,0 +1,45 @@
+"""Persistent ingestion watermark stored in the ``ingest_state`` table.
+
+The incremental updater uses this to remember how far it got for each source
+(e.g. Open Food Facts exposes a ``last_modified_t`` unix timestamp on every
+product) so repeated runs only fetch what changed.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import psycopg
+from psycopg.types.json import Jsonb
+
+
+def get_watermark(conn: psycopg.Connection, source: str) -> int:
+    """Return the last processed ``last_modified_t`` for *source* (0 if none)."""
+    row = conn.execute(
+        "SELECT last_modified_t FROM ingest_state WHERE source = %s", (source,)
+    ).fetchone()
+    return int(row[0]) if row else 0
+
+
+def set_watermark(
+    conn: psycopg.Connection,
+    source: str,
+    last_modified_t: int,
+    stats: dict[str, Any] | None = None,
+) -> None:
+    """Upsert the watermark and run metadata for *source*.
+
+    The watermark only ever moves forward: a lower ``last_modified_t`` is
+    ignored so an out-of-order or partial run cannot rewind progress.
+    """
+    conn.execute(
+        """
+        INSERT INTO ingest_state (source, last_modified_t, last_run_at, stats)
+        VALUES (%s, %s, now(), %s)
+        ON CONFLICT (source) DO UPDATE SET
+            last_modified_t = GREATEST(ingest_state.last_modified_t, EXCLUDED.last_modified_t),
+            last_run_at = now(),
+            stats = EXCLUDED.stats
+        """,
+        (source, int(last_modified_t), Jsonb(stats or {})),
+    )

ingestion/opengoods/etl/supplement.py

@@ -0,0 +1,133 @@
+"""Apply GS1 (or other authoritative) supplements to existing products.
+
+A supplement only fills *gaps*: a field is written only when the product does
+not already have a value. Each applied supplement records field-level provenance
+in ``product_source`` and refreshes the product's quality score.
+"""
+
+from __future__ import annotations
+
+from decimal import Decimal, InvalidOperation
+from typing import Any
+
+import psycopg
+from psycopg.types.json import Jsonb
+
+from opengoods import units
+from opengoods.adapters.gs1 import GS1_HOMEPAGE, GS1_LICENSE, GS1_TRUST, SOURCE_NAME
+from opengoods.etl.load import _ensure_brand, _normalize_brand, ensure_source_named
+from opengoods.etl.quality import update_quality
+
+
+def ensure_gs1_source(conn: psycopg.Connection) -> str:
+    """Upsert the GS1 source row and return its id."""
+    return ensure_source_named(conn, SOURCE_NAME, GS1_HOMEPAGE, GS1_LICENSE, GS1_TRUST)
+
+
+def _ensure_manufacturer(conn: psycopg.Connection, name: str | None) -> str | None:
+    if not name:
+        return None
+    row = conn.execute(
+        """
+        INSERT INTO manufacturer (name, normalized_name)
+        VALUES (%s, %s)
+        ON CONFLICT (normalized_name) DO UPDATE SET name = manufacturer.name
+        RETURNING id
+        """,
+        (name, _normalize_brand(name)),
+    ).fetchone()
+    return row[0]
+
+
+def _net_content(rec: dict) -> tuple[Decimal, str, Decimal | None] | None:
+    raw_value = rec.get("net_content_value")
+    unit = rec.get("net_content_unit")
+    if raw_value is None or not unit:
+        return None
+    try:
+        value = Decimal(str(raw_value))
+    except (InvalidOperation, ValueError):
+        return None
+    try:
+        canonical = units.normalize(value, unit).canonical_value
+    except units.UnitError:
+        canonical = None
+    return value, unit, canonical
+
+
+def apply_supplement(conn: psycopg.Connection, rec: dict[str, Any], source_id: str) -> list[str]:
+    """Fill missing fields of the GTIN-matched product from ``rec``.
+
+    Returns the list of field names actually filled (empty if the product is
+    unknown or already complete for the supplied fields).
+    """
+    gtin = rec.get("gtin")
+    if not gtin:
+        return []
+    prod = conn.execute(
+        """
+        SELECT id, brand_id, manufacturer_id, gpc_brick_code, country_of_origin,
+               net_content_value
+        FROM product
+        WHERE gtin = %s AND status = 'active'
+        """,
+        (gtin,),
+    ).fetchone()
+    if prod is None:
+        return []
+
+    product_id, brand_id, manufacturer_id, gpc, country, net_value = prod
+    sets: list[str] = []
+    params: list[Any] = []
+    filled: list[str] = []
+
+    if brand_id is None and rec.get("brand"):
+        new_brand_id = _ensure_brand(conn, rec["brand"])
+        if new_brand_id is not None:
+            sets.append("brand_id = %s")
+            params.append(new_brand_id)
+            filled.append("brand")
+
+    if manufacturer_id is None and rec.get("manufacturer"):
+        new_mfr_id = _ensure_manufacturer(conn, rec["manufacturer"])
+        if new_mfr_id is not None:
+            sets.append("manufacturer_id = %s")
+            params.append(new_mfr_id)
+            filled.append("manufacturer")
+
+    if gpc is None and rec.get("gpc_brick_code"):
+        sets.append("gpc_brick_code = %s")
+        params.append(rec["gpc_brick_code"])
+        filled.append("gpc_brick_code")
+
+    if country is None and rec.get("country_of_origin"):
+        sets.append("country_of_origin = %s")
+        params.append(rec["country_of_origin"])
+        filled.append("country_of_origin")
+
+    if net_value is None:
+        net = _net_content(rec)
+        if net is not None:
+            value, unit, canonical = net
+            sets += [
+                "net_content_value = %s",
+                "net_content_unit = %s",
+                "net_content_canonical = %s",
+            ]
+            params += [value, unit, canonical]
+            filled.append("net_content")
+
+    if not filled:
+        return []
+
+    params.append(product_id)
+    conn.execute(f"UPDATE product SET {', '.join(sets)} WHERE id = %s", params)
+    conn.execute(
+        """
+        INSERT INTO product_source (product_id, source_id, url, fields, fetched_at, raw)
+        VALUES (%s, %s, %s, %s, now(), %s)
+        """,
+        (product_id, source_id, GS1_HOMEPAGE, filled, Jsonb(rec)),
+    )
+    update_quality(conn, product_id)
+    return filled

ingestion/opengoods/jobs/dedup.py

@@ -0,0 +1,40 @@
+"""Deduplicate products: merge non-GTIN duplicates into a canonical record.
+
+Usage:
+    python -m opengoods.jobs.dedup --dry-run
+    python -m opengoods.jobs.dedup --actor nightly
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+import psycopg
+
+from opengoods.etl.dedup import dedup_all
+from opengoods.etl.load import default_dsn
+
+
+def run(args: argparse.Namespace) -> int:
+    with psycopg.connect(args.dsn, autocommit=False) as conn:
+        summary = dedup_all(conn, actor=args.actor, dry_run=args.dry_run)
+        if args.dry_run:
+            conn.rollback()
+        else:
+            conn.commit()
+    mode = "dry-run" if args.dry_run else "applied"
+    print(f"{mode} groups={summary['groups']} merged={summary['merged']}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Deduplicate OpenGoods products")
+    parser.add_argument("--actor", default="ingestion", help="merge_log actor label")
+    parser.add_argument("--dry-run", action="store_true", help="report only, do not write")
+    parser.add_argument("--dsn", default=default_dsn(), help="PostgreSQL DSN")
+    return run(parser.parse_args(argv))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ingestion/opengoods/jobs/schedule.py

@@ -0,0 +1,66 @@
+"""Lightweight recurring ingestion scheduler.
+
+Runs one ingestion cycle (incremental OFF update, then dedup) on a fixed
+interval. Dependency-free: a plain sleep loop rather than a cron/APScheduler
+dependency, so it is trivial to run in a container or under systemd/supervisor.
+
+Usage:
+    python -m opengoods.jobs.schedule --once            # single cycle, then exit
+    python -m opengoods.jobs.schedule --interval 3600   # every hour
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import time
+from datetime import UTC, datetime
+
+from opengoods.etl.load import default_dsn
+from opengoods.jobs import dedup as dedup_job
+from opengoods.jobs import update_off as update_job
+
+
+def _cycle(args: argparse.Namespace) -> None:
+    ts = datetime.now(UTC).isoformat(timespec="seconds")
+    print(f"[{ts}] cycle start")
+    update_job.run(
+        argparse.Namespace(
+            since=None,
+            page_size=args.page_size,
+            max_pages=args.max_pages,
+            min_interval=args.min_interval,
+            dsn=args.dsn,
+        )
+    )
+    if not args.skip_dedup:
+        dedup_job.run(argparse.Namespace(actor="scheduler", dry_run=False, dsn=args.dsn))
+
+
+def run(args: argparse.Namespace) -> int:
+    _cycle(args)
+    if args.once:
+        return 0
+    while True:
+        time.sleep(args.interval)
+        try:
+            _cycle(args)
+        except Exception as exc:  # noqa: BLE001 - keep the loop alive across failures
+            print(f"cycle error: {exc}", file=sys.stderr)
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Recurring OpenGoods ingestion")
+    parser.add_argument("--interval", type=int, default=3600, help="seconds between cycles")
+    parser.add_argument("--once", action="store_true", help="run a single cycle and exit")
+    parser.add_argument("--skip-dedup", action="store_true", help="run update only")
+    parser.add_argument("--page-size", type=int, default=100, help="search page size")
+    parser.add_argument("--max-pages", type=int, default=10, help="max pages to scan")
+    parser.add_argument("--min-interval", type=float, default=4.0, help="API throttle seconds")
+    parser.add_argument("--dsn", default=default_dsn(), help="PostgreSQL DSN")
+    return run(parser.parse_args(argv))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ingestion/opengoods/jobs/update_off.py

@@ -0,0 +1,65 @@
+"""Incremental Open Food Facts update.
+
+Fetches products modified since the persisted watermark, loads them, then
+advances the watermark to the newest ``last_modified_t`` processed so the next
+run only sees what changed.
+
+Usage:
+    python -m opengoods.jobs.update_off --max-pages 5
+    python -m opengoods.jobs.update_off --since 1700000000   # override watermark
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+import psycopg
+
+from opengoods.adapters.openfoodfacts import SOURCE_NAME, OpenFoodFactsAdapter
+from opengoods.etl.load import default_dsn, ensure_source, load_record
+from opengoods.etl.state import get_watermark, set_watermark
+from opengoods.etl.transform import transform
+
+
+def run(args: argparse.Namespace) -> int:
+    adapter = OpenFoodFactsAdapter(min_interval=args.min_interval)
+    loaded = skipped = 0
+    high_watermark = 0
+    with psycopg.connect(args.dsn, autocommit=False) as conn:
+        source_id = ensure_source(conn)
+        since = args.since if args.since is not None else get_watermark(conn, SOURCE_NAME)
+        high_watermark = since
+        for raw in adapter.fetch_modified_since(
+            since, page_size=args.page_size, max_pages=args.max_pages
+        ):
+            high_watermark = max(high_watermark, int(raw.get("last_modified_t") or 0))
+            rec = transform(raw)
+            if rec is None:
+                skipped += 1
+                continue
+            load_record(conn, rec, source_id, raw)
+            loaded += 1
+        set_watermark(
+            conn,
+            SOURCE_NAME,
+            high_watermark,
+            stats={"loaded": loaded, "skipped": skipped, "since": since},
+        )
+        conn.commit()
+    print(f"since={since} loaded={loaded} skipped={skipped} watermark={high_watermark}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Incremental OFF update")
+    parser.add_argument("--since", type=int, default=None, help="override watermark (unix ts)")
+    parser.add_argument("--page-size", type=int, default=100, help="search page size")
+    parser.add_argument("--max-pages", type=int, default=10, help="max pages to scan")
+    parser.add_argument("--min-interval", type=float, default=4.0, help="API throttle seconds")
+    parser.add_argument("--dsn", default=default_dsn(), help="PostgreSQL DSN")
+    return run(parser.parse_args(argv))
+
+
+if __name__ == "__main__":
+    sys.exit(main())