M4: ingestion management (incremental, GS1 supplement, dedup/conflict, quality, scheduler)

- OFF incremental fetch via search API + persistent watermark (ingest_state, migration 0004)
- GS1 barcode supplement adapter (offline mapping + GS1-style API) filling only gaps with field-level provenance
- Non-GTIN dedup with canonical selection + merge_log; field-level conflict resolution (source trust > recency)
- Quality scoring (0.4 completeness + 0.3 source trust + 0.2 multi-source + 0.1 freshness) wired into load/merge
- Jobs: update_off, dedup, schedule; docs/ingestion-management.md
- 19 new tests (pure + DB-integration), ruff clean

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

ingestion/tests/test_dedup.py

@@ -0,0 +1,62 @@
+from opengoods.etl.dedup import choose_canonical, dedup_all, product_signature
+from opengoods.etl.load import _ensure_brand, ensure_source
+
+
+def test_product_signature_normalization():
+    a = product_signature("  Spring  Water ", "Acme", 500)
+    b = product_signature("spring water", "acme", 500)
+    assert a == b
+    assert product_signature("", "x", 1) is None
+
+
+def test_choose_canonical_prefers_quality():
+    members = [
+        {"id": "a", "quality_score": 0.2, "created_at": 1},
+        {"id": "b", "quality_score": 0.9, "created_at": 2},
+    ]
+    assert choose_canonical(members)["id"] == "b"
+
+
+def test_dedup_merges_duplicates(db_conn):
+    brand_id = _ensure_brand(db_conn, "DupBrand")
+
+    def mk(quality):
+        return db_conn.execute(
+            """
+            INSERT INTO product (name, brand_id, net_content_canonical, quality_score)
+            VALUES (%s, %s, %s, %s) RETURNING id
+            """,
+            ("Dup Snack", brand_id, 100, quality),
+        ).fetchone()[0]
+
+    keep = mk(0.9)
+    drop = mk(0.2)
+
+    src = ensure_source(db_conn)
+    db_conn.execute(
+        "INSERT INTO product_source (product_id, source_id, fields) VALUES (%s, %s, %s)",
+        (drop, src, ["name"]),
+    )
+
+    summary = dedup_all(db_conn)
+    assert summary == {"groups": 1, "merged": 1}
+
+    keep_status = db_conn.execute("SELECT status FROM product WHERE id = %s", (keep,)).fetchone()[0]
+    drop_status, canonical_id = db_conn.execute(
+        "SELECT status, canonical_id FROM product WHERE id = %s", (drop,)
+    ).fetchone()
+    assert keep_status == "active"
+    assert drop_status == "merged"
+    assert str(canonical_id) == str(keep)
+
+    # The merged product's source row was re-pointed to the canonical product.
+    reattached = db_conn.execute(
+        "SELECT count(*) FROM product_source WHERE product_id = %s", (keep,)
+    ).fetchone()[0]
+    assert reattached == 1
+
+    logged = db_conn.execute(
+        "SELECT count(*) FROM merge_log WHERE kept_id = %s AND merged_id = %s",
+        (keep, drop),
+    ).fetchone()[0]
+    assert logged == 1