"""Seed the database with Open Food Facts data. Usage: # from a list of barcodes via the OFF API python -m opengoods.jobs.seed_off --barcodes 3017624010701 5449000000996 # from a downloaded OFF JSONL dump (optionally .gz), limited to N records python -m opengoods.jobs.seed_off --dump products.jsonl.gz --limit 1000 # market-focused: the most-scanned products sold in China, restricted to # genuine GS1-China (69x) barcodes python -m opengoods.jobs.seed_off --country china --domestic-only \ --max-pages 20 --limit 1000 The OFF read API is rate-limited client-side; for large imports use a dump. """ from __future__ import annotations import argparse import sys from collections.abc import Iterator import psycopg from opengoods.adapters.openfoodfacts import ( OpenFoodFactsAdapter, is_cn_gs1, read_dump, ) from opengoods.cache import bump_cache_epoch from opengoods.etl.load import default_dsn, ensure_source, load_record_safe from opengoods.etl.transform import transform def _raw_records(args: argparse.Namespace) -> Iterator[dict]: if args.dump: records: Iterator[dict] = read_dump(args.dump) elif args.country: adapter = OpenFoodFactsAdapter(min_interval=args.min_interval) records = adapter.fetch_by_country( args.country, page_size=args.page_size, max_pages=args.max_pages ) else: adapter = OpenFoodFactsAdapter(min_interval=args.min_interval) records = adapter.fetch(args.barcodes) yielded = 0 for rec in records: if args.domestic_only and not is_cn_gs1(str(rec.get("code") or "")): continue if args.limit and yielded >= args.limit: break yielded += 1 yield rec def run(args: argparse.Namespace) -> int: loaded = skipped = errored = 0 with psycopg.connect(args.dsn, autocommit=False) as conn: source_id = ensure_source(conn) for raw in _raw_records(args): rec = transform(raw) if rec is None: skipped += 1 continue if load_record_safe(conn, rec, source_id, raw): loaded += 1 else: errored += 1 conn.commit() if loaded: bump_cache_epoch() print(f"loaded={loaded} skipped={skipped} errored={errored}") return 0 def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Seed OpenGoods from Open Food Facts") src = parser.add_mutually_exclusive_group(required=True) src.add_argument("--barcodes", nargs="+", help="barcodes to fetch via the OFF API") src.add_argument("--dump", help="path to an OFF JSONL dump (.jsonl or .jsonl.gz)") src.add_argument( "--country", help="OFF countries_tags_en slug to seed from, e.g. 'china' (most-scanned first)", ) parser.add_argument( "--domestic-only", action="store_true", help="keep only genuine GS1-China (69x) barcodes; drop imported goods", ) parser.add_argument("--limit", type=int, default=0, help="max records to load (0 = all)") parser.add_argument("--page-size", type=int, default=100, help="search page size") parser.add_argument("--max-pages", type=int, default=10, help="max search pages (country mode)") parser.add_argument("--min-interval", type=float, default=4.0, help="API throttle seconds") parser.add_argument("--dsn", default=default_dsn(), help="PostgreSQL DSN") return run(parser.parse_args(argv)) if __name__ == "__main__": sys.exit(main())