John Doe
a35bcd6647
- OFF incremental fetch via search API + persistent watermark (ingest_state, migration 0004) - GS1 barcode supplement adapter (offline mapping + GS1-style API) filling only gaps with field-level provenance - Non-GTIN dedup with canonical selection + merge_log; field-level conflict resolution (source trust > recency) - Quality scoring (0.4 completeness + 0.3 source trust + 0.2 multi-source + 0.1 freshness) wired into load/merge - Jobs: update_off, dedup, schedule; docs/ingestion-management.md - 19 new tests (pure + DB-integration), ruff clean Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
66 lines
2.4 KiB
Python
66 lines
2.4 KiB
Python
"""Incremental Open Food Facts update.
|
|
|
|
Fetches products modified since the persisted watermark, loads them, then
|
|
advances the watermark to the newest ``last_modified_t`` processed so the next
|
|
run only sees what changed.
|
|
|
|
Usage:
|
|
python -m opengoods.jobs.update_off --max-pages 5
|
|
python -m opengoods.jobs.update_off --since 1700000000 # override watermark
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
|
|
import psycopg
|
|
|
|
from opengoods.adapters.openfoodfacts import SOURCE_NAME, OpenFoodFactsAdapter
|
|
from opengoods.etl.load import default_dsn, ensure_source, load_record
|
|
from opengoods.etl.state import get_watermark, set_watermark
|
|
from opengoods.etl.transform import transform
|
|
|
|
|
|
def run(args: argparse.Namespace) -> int:
|
|
adapter = OpenFoodFactsAdapter(min_interval=args.min_interval)
|
|
loaded = skipped = 0
|
|
high_watermark = 0
|
|
with psycopg.connect(args.dsn, autocommit=False) as conn:
|
|
source_id = ensure_source(conn)
|
|
since = args.since if args.since is not None else get_watermark(conn, SOURCE_NAME)
|
|
high_watermark = since
|
|
for raw in adapter.fetch_modified_since(
|
|
since, page_size=args.page_size, max_pages=args.max_pages
|
|
):
|
|
high_watermark = max(high_watermark, int(raw.get("last_modified_t") or 0))
|
|
rec = transform(raw)
|
|
if rec is None:
|
|
skipped += 1
|
|
continue
|
|
load_record(conn, rec, source_id, raw)
|
|
loaded += 1
|
|
set_watermark(
|
|
conn,
|
|
SOURCE_NAME,
|
|
high_watermark,
|
|
stats={"loaded": loaded, "skipped": skipped, "since": since},
|
|
)
|
|
conn.commit()
|
|
print(f"since={since} loaded={loaded} skipped={skipped} watermark={high_watermark}")
|
|
return 0
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="Incremental OFF update")
|
|
parser.add_argument("--since", type=int, default=None, help="override watermark (unix ts)")
|
|
parser.add_argument("--page-size", type=int, default=100, help="search page size")
|
|
parser.add_argument("--max-pages", type=int, default=10, help="max pages to scan")
|
|
parser.add_argument("--min-interval", type=float, default=4.0, help="API throttle seconds")
|
|
parser.add_argument("--dsn", default=default_dsn(), help="PostgreSQL DSN")
|
|
return run(parser.parse_args(argv))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|