M4: ingestion management (incremental, GS1 supplement, dedup/conflict, quality, scheduler)

- OFF incremental fetch via search API + persistent watermark (ingest_state, migration 0004)
- GS1 barcode supplement adapter (offline mapping + GS1-style API) filling only gaps with field-level provenance
- Non-GTIN dedup with canonical selection + merge_log; field-level conflict resolution (source trust > recency)
- Quality scoring (0.4 completeness + 0.3 source trust + 0.2 multi-source + 0.1 freshness) wired into load/merge
- Jobs: update_off, dedup, schedule; docs/ingestion-management.md
- 19 new tests (pure + DB-integration), ruff clean

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

ingestion/tests/conftest.py

@@ -0,0 +1,31 @@
+"""Shared test fixtures.
+
+`db_conn` yields a psycopg connection inside a transaction that is rolled back
+after each test, so DB tests stay isolated and leave no residue. Tests are
+skipped automatically when no database is reachable or M4 migrations are not
+applied (e.g. local runs without docker).
+"""
+
+from __future__ import annotations
+
+import psycopg
+import pytest
+
+from opengoods.etl.load import default_dsn
+
+
+@pytest.fixture()
+def db_conn():
+    try:
+        conn = psycopg.connect(default_dsn(), connect_timeout=3)
+    except psycopg.OperationalError as exc:  # pragma: no cover - env dependent
+        pytest.skip(f"no database available: {exc}")
+    has_state = conn.execute("SELECT to_regclass('public.ingest_state') IS NOT NULL").fetchone()[0]
+    if not has_state:
+        conn.close()
+        pytest.skip("M4 migrations not applied")
+    try:
+        yield conn
+    finally:
+        conn.rollback()
+        conn.close()

ingestion/tests/fixtures/gs1_mapping.json

@@ -0,0 +1,14 @@
+{
+  "06901234567892": {
+    "name": "示例矿泉水 550ml",
+    "brand": "示例品牌",
+    "manufacturer": "示例饮品有限公司",
+    "gpc_brick_code": "10000224",
+    "country_of_origin": "China",
+    "net_content_value": 550,
+    "net_content_unit": "ml"
+  },
+  "00000000000000": {
+    "brand": ""
+  }
+}

ingestion/tests/test_dedup.py

@@ -0,0 +1,62 @@
+from opengoods.etl.dedup import choose_canonical, dedup_all, product_signature
+from opengoods.etl.load import _ensure_brand, ensure_source
+
+
+def test_product_signature_normalization():
+    a = product_signature("  Spring  Water ", "Acme", 500)
+    b = product_signature("spring water", "acme", 500)
+    assert a == b
+    assert product_signature("", "x", 1) is None
+
+
+def test_choose_canonical_prefers_quality():
+    members = [
+        {"id": "a", "quality_score": 0.2, "created_at": 1},
+        {"id": "b", "quality_score": 0.9, "created_at": 2},
+    ]
+    assert choose_canonical(members)["id"] == "b"
+
+
+def test_dedup_merges_duplicates(db_conn):
+    brand_id = _ensure_brand(db_conn, "DupBrand")
+
+    def mk(quality):
+        return db_conn.execute(
+            """
+            INSERT INTO product (name, brand_id, net_content_canonical, quality_score)
+            VALUES (%s, %s, %s, %s) RETURNING id
+            """,
+            ("Dup Snack", brand_id, 100, quality),
+        ).fetchone()[0]
+
+    keep = mk(0.9)
+    drop = mk(0.2)
+
+    src = ensure_source(db_conn)
+    db_conn.execute(
+        "INSERT INTO product_source (product_id, source_id, fields) VALUES (%s, %s, %s)",
+        (drop, src, ["name"]),
+    )
+
+    summary = dedup_all(db_conn)
+    assert summary == {"groups": 1, "merged": 1}
+
+    keep_status = db_conn.execute("SELECT status FROM product WHERE id = %s", (keep,)).fetchone()[0]
+    drop_status, canonical_id = db_conn.execute(
+        "SELECT status, canonical_id FROM product WHERE id = %s", (drop,)
+    ).fetchone()
+    assert keep_status == "active"
+    assert drop_status == "merged"
+    assert str(canonical_id) == str(keep)
+
+    # The merged product's source row was re-pointed to the canonical product.
+    reattached = db_conn.execute(
+        "SELECT count(*) FROM product_source WHERE product_id = %s", (keep,)
+    ).fetchone()[0]
+    assert reattached == 1
+
+    logged = db_conn.execute(
+        "SELECT count(*) FROM merge_log WHERE kept_id = %s AND merged_id = %s",
+        (keep, drop),
+    ).fetchone()[0]
+    assert logged == 1

ingestion/tests/test_gs1.py

@@ -0,0 +1,53 @@
+from pathlib import Path
+
+from opengoods.adapters.gs1 import GS1Adapter
+from opengoods.etl.supplement import apply_supplement, ensure_gs1_source
+
+MAPPING = Path(__file__).parent / "fixtures" / "gs1_mapping.json"
+GTIN = "06901234567892"
+
+
+def test_gs1_adapter_offline_lookup():
+    adapter = GS1Adapter.from_file(MAPPING)
+    rec = adapter.fetch_barcode(GTIN)
+    assert rec["brand"] == "示例品牌"
+    assert rec["net_content_value"] == 550
+    assert rec["net_content_unit"] == "ml"
+    # An entry that only has empty values yields no supplement.
+    assert adapter.fetch_barcode("00000000000000") is None
+    # Unknown barcode -> None.
+    assert adapter.fetch_barcode("99999999999999") is None
+
+
+def test_gs1_supplement_fills_only_gaps(db_conn):
+    pid = db_conn.execute(
+        "INSERT INTO product (gtin, name) VALUES (%s, %s) RETURNING id", (GTIN, "水")
+    ).fetchone()[0]
+
+    adapter = GS1Adapter.from_file(MAPPING)
+    rec = adapter.fetch_barcode(GTIN)
+    source_id = ensure_gs1_source(db_conn)
+
+    filled = apply_supplement(db_conn, rec, source_id)
+    assert {"brand", "country_of_origin", "net_content"} <= set(filled)
+
+    brand_id, country, net_value, net_unit = db_conn.execute(
+        """
+        SELECT brand_id, country_of_origin, net_content_value, net_content_unit
+        FROM product WHERE id = %s
+        """,
+        (pid,),
+    ).fetchone()
+    assert brand_id is not None
+    assert country == "China"
+    assert float(net_value) == 550.0
+    assert net_unit == "ml"
+
+    fields = db_conn.execute(
+        "SELECT fields FROM product_source WHERE product_id = %s AND source_id = %s",
+        (pid, source_id),
+    ).fetchone()[0]
+    assert "brand" in fields
+
+    # Re-applying does nothing because the gaps are now filled.
+    assert apply_supplement(db_conn, rec, source_id) == []

ingestion/tests/test_merge.py

@@ -0,0 +1,62 @@
+from datetime import UTC, datetime
+
+from opengoods.etl.merge import Candidate, merge_records, resolve_field
+
+
+def _ts(y, m, d):
+    return datetime(y, m, d, tzinfo=UTC)
+
+
+def test_resolve_field_prefers_trust_then_recency():
+    cands = [
+        Candidate(value="A", source="off", trust=0.7, fetched_at=_ts(2024, 1, 1)),
+        Candidate(value="B", source="gs1", trust=0.9, fetched_at=_ts(2023, 1, 1)),
+    ]
+    res = resolve_field(cands)
+    assert res is not None
+    assert res.value == "B"
+    assert res.source == "gs1"
+
+
+def test_resolve_field_recency_tiebreak_on_equal_trust():
+    cands = [
+        Candidate(value="old", source="a", trust=0.7, fetched_at=_ts(2023, 1, 1)),
+        Candidate(value="new", source="b", trust=0.7, fetched_at=_ts(2024, 6, 1)),
+    ]
+    assert resolve_field(cands).value == "new"
+
+
+def test_resolve_field_skips_empty():
+    cands = [
+        Candidate(value="", source="a", trust=0.99),
+        Candidate(value=None, source="b", trust=0.99),
+        Candidate(value="kept", source="c", trust=0.1),
+    ]
+    assert resolve_field(cands).value == "kept"
+    assert resolve_field([Candidate(value="", source="a")]) is None
+
+
+def test_merge_records_provenance():
+    records = [
+        {
+            "name": Candidate("Water", "off", 0.7, _ts(2024, 1, 1)),
+            "brand": Candidate("", "off", 0.7),
+        },
+        {
+            "brand": Candidate("Acme", "gs1", 0.9, _ts(2024, 2, 1)),
+            "gtin": Candidate("123", "gs1", 0.9),
+        },
+    ]
+    merged = merge_records(records)
+    assert merged.values["name"] == "Water"
+    assert merged.values["brand"] == "Acme"
+    assert merged.values["gtin"] == "123"
+    assert merged.provenance["brand"] == "gs1"
+    assert merged.provenance["name"] == "off"
+
+
+def test_merge_records_accepts_bare_values():
+    merged = merge_records([{"x": 1}, {"x": 2}])
+    # both bare -> trust tie, no timestamps -> first max() wins deterministically
+    assert merged.values["x"] in (1, 2)
+    assert merged.provenance["x"] == "unknown"

ingestion/tests/test_off_incremental.py

@@ -0,0 +1,44 @@
+import httpx
+
+from opengoods.adapters.openfoodfacts import OpenFoodFactsAdapter
+
+
+def _product(code, lm):
+    return {"code": code, "product_name": f"P{code}", "last_modified_t": lm}
+
+
+def _adapter(pages):
+    """Build an adapter whose search endpoint serves the given pages."""
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        page = int(request.url.params.get("page", "1"))
+        products = pages.get(page, [])
+        return httpx.Response(200, json={"products": products, "page": page})
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    return OpenFoodFactsAdapter(client=client, min_interval=0)
+
+
+def test_incremental_yields_only_newer_and_stops_at_watermark():
+    pages = {
+        1: [_product("1", 300), _product("2", 250), _product("3", 100)],
+    }
+    adapter = _adapter(pages)
+    got = list(adapter.fetch_modified_since(200, page_size=3, max_pages=5))
+    codes = [p["code"] for p in got]
+    assert codes == ["1", "2"]  # 100 <= 200 stops iteration
+
+
+def test_incremental_paginates_until_short_page():
+    pages = {
+        1: [_product("1", 900), _product("2", 800)],
+        2: [_product("3", 700)],  # short page -> stop after
+    }
+    adapter = _adapter(pages)
+    got = list(adapter.fetch_modified_since(0, page_size=2, max_pages=5))
+    assert [p["code"] for p in got] == ["1", "2", "3"]
+
+
+def test_incremental_empty_first_page():
+    adapter = _adapter({1: []})
+    assert list(adapter.fetch_modified_since(0, page_size=10, max_pages=3)) == []

ingestion/tests/test_quality.py

@@ -0,0 +1,38 @@
+from opengoods.etl.quality import (
+    COMPLETENESS_FIELDS,
+    agreement_from_sources,
+    completeness,
+    freshness_from_age,
+    score,
+)
+
+
+def test_completeness_bounds():
+    assert completeness(set()) == 0.0
+    assert completeness(set(COMPLETENESS_FIELDS)) == 1.0
+    half = set(list(COMPLETENESS_FIELDS)[: len(COMPLETENESS_FIELDS) // 2])
+    assert 0.0 < completeness(half) < 1.0
+
+
+def test_agreement_from_sources():
+    assert agreement_from_sources(0) == 0.5
+    assert agreement_from_sources(1) == 0.5
+    assert agreement_from_sources(2) == 0.8
+    assert agreement_from_sources(5) == 1.0
+
+
+def test_freshness_from_age():
+    assert freshness_from_age(None) == 0.5
+    assert freshness_from_age(1) == 1.0
+    assert freshness_from_age(100) == 0.8
+    assert freshness_from_age(300) == 0.6
+    assert freshness_from_age(700) == 0.4
+    assert freshness_from_age(5000) == 0.2
+
+
+def test_score_weighted_sum_and_bounds():
+    assert score(completeness_score=0, source_trust=0, agreement=0, freshness=0) == 0.0
+    assert score(completeness_score=1, source_trust=1, agreement=1, freshness=1) == 1.0
+    # 0.4*1 + 0.3*0.5 + 0.2*0.5 + 0.1*1 = 0.75
+    got = score(completeness_score=1.0, source_trust=0.5, agreement=0.5, freshness=1.0)
+    assert got == 0.75

ingestion/tests/test_quality_db.py

@@ -0,0 +1,21 @@
+import json
+from pathlib import Path
+
+from opengoods.etl.load import ensure_source, load_record
+from opengoods.etl.quality import compute_quality
+from opengoods.etl.transform import transform
+
+FIXTURE = json.loads((Path(__file__).parent / "fixtures" / "off_product.json").read_text())
+
+
+def test_quality_score_set_on_load(db_conn):
+    source_id = ensure_source(db_conn)
+    rec = transform(FIXTURE)
+    pid = load_record(db_conn, rec, source_id, FIXTURE)
+
+    stored = float(
+        db_conn.execute("SELECT quality_score FROM product WHERE id = %s", (pid,)).fetchone()[0]
+    )
+    assert 0.0 < stored <= 1.0
+    # The persisted value matches a fresh recomputation.
+    assert abs(stored - compute_quality(db_conn, pid)) < 1e-9

ingestion/tests/test_state.py

@@ -0,0 +1,16 @@
+from opengoods.etl.state import get_watermark, set_watermark
+
+
+def test_watermark_roundtrip_and_monotonic(db_conn):
+    src = "test-source"
+    assert get_watermark(db_conn, src) == 0
+
+    set_watermark(db_conn, src, 100, stats={"loaded": 1})
+    assert get_watermark(db_conn, src) == 100
+
+    # A lower watermark must not rewind progress.
+    set_watermark(db_conn, src, 50)
+    assert get_watermark(db_conn, src) == 100
+
+    set_watermark(db_conn, src, 150)
+    assert get_watermark(db_conn, src) == 150