M4: ingestion management (incremental, GS1 supplement, dedup/conflict, quality, scheduler)
- OFF incremental fetch via search API + persistent watermark (ingest_state, migration 0004) - GS1 barcode supplement adapter (offline mapping + GS1-style API) filling only gaps with field-level provenance - Non-GTIN dedup with canonical selection + merge_log; field-level conflict resolution (source trust > recency) - Quality scoring (0.4 completeness + 0.3 source trust + 0.2 multi-source + 0.1 freshness) wired into load/merge - Jobs: update_off, dedup, schedule; docs/ingestion-management.md - 19 new tests (pure + DB-integration), ruff clean Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
@@ -14,6 +14,7 @@ import psycopg
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from opengoods.adapters.openfoodfacts import OFF_LICENSE, SOURCE_NAME
|
||||
from opengoods.etl.quality import update_quality
|
||||
|
||||
OFF_HOMEPAGE = "https://world.openfoodfacts.org"
|
||||
|
||||
@@ -29,8 +30,14 @@ def _normalize_brand(name: str) -> str:
|
||||
return " ".join(name.lower().split())
|
||||
|
||||
|
||||
def ensure_source(conn: psycopg.Connection) -> str:
|
||||
"""Upsert the Open Food Facts source row and return its id."""
|
||||
def ensure_source_named(
|
||||
conn: psycopg.Connection,
|
||||
name: str,
|
||||
homepage: str,
|
||||
license: str,
|
||||
trust_weight: float,
|
||||
) -> str:
|
||||
"""Upsert a source row by name and return its id."""
|
||||
row = conn.execute(
|
||||
"""
|
||||
INSERT INTO source (name, homepage, license, trust_weight)
|
||||
@@ -38,11 +45,16 @@ def ensure_source(conn: psycopg.Connection) -> str:
|
||||
ON CONFLICT (name) DO UPDATE SET homepage = EXCLUDED.homepage
|
||||
RETURNING id
|
||||
""",
|
||||
(SOURCE_NAME, OFF_HOMEPAGE, OFF_LICENSE, 0.7),
|
||||
(name, homepage, license, trust_weight),
|
||||
).fetchone()
|
||||
return row[0]
|
||||
|
||||
|
||||
def ensure_source(conn: psycopg.Connection) -> str:
|
||||
"""Upsert the Open Food Facts source row and return its id."""
|
||||
return ensure_source_named(conn, SOURCE_NAME, OFF_HOMEPAGE, OFF_LICENSE, 0.7)
|
||||
|
||||
|
||||
def _ensure_brand(conn: psycopg.Connection, name: str | None) -> str | None:
|
||||
if not name:
|
||||
return None
|
||||
@@ -178,6 +190,9 @@ def load_record(conn: psycopg.Connection, rec: dict[str, Any], source_id: str, r
|
||||
Jsonb(_jsonable(raw)),
|
||||
),
|
||||
)
|
||||
|
||||
# Recompute the data-quality score now that all facts + provenance exist.
|
||||
update_quality(conn, product_id)
|
||||
return product_id
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user