"""Seed the database with Open Food Facts data. Usage: # from a list of barcodes via the OFF API python -m opengoods.jobs.seed_off --barcodes 3017624010701 5449000000996 # from a downloaded OFF JSONL dump (optionally .gz), limited to N records python -m opengoods.jobs.seed_off --dump products.jsonl.gz --limit 1000 The OFF read API is rate-limited client-side; for large imports use a dump. """ from __future__ import annotations import argparse import sys from collections.abc import Iterator import psycopg from opengoods.adapters.openfoodfacts import OpenFoodFactsAdapter, read_dump from opengoods.etl.load import default_dsn, ensure_source, load_record_safe from opengoods.etl.transform import transform def _raw_records(args: argparse.Namespace) -> Iterator[dict]: if args.dump: records = read_dump(args.dump) else: adapter = OpenFoodFactsAdapter(min_interval=args.min_interval) records = adapter.fetch(args.barcodes) for i, rec in enumerate(records): if args.limit and i >= args.limit: break yield rec def run(args: argparse.Namespace) -> int: loaded = skipped = errored = 0 with psycopg.connect(args.dsn, autocommit=False) as conn: source_id = ensure_source(conn) for raw in _raw_records(args): rec = transform(raw) if rec is None: skipped += 1 continue if load_record_safe(conn, rec, source_id, raw): loaded += 1 else: errored += 1 conn.commit() print(f"loaded={loaded} skipped={skipped} errored={errored}") return 0 def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Seed OpenGoods from Open Food Facts") src = parser.add_mutually_exclusive_group(required=True) src.add_argument("--barcodes", nargs="+", help="barcodes to fetch via the OFF API") src.add_argument("--dump", help="path to an OFF JSONL dump (.jsonl or .jsonl.gz)") parser.add_argument("--limit", type=int, default=0, help="max records to load (0 = all)") parser.add_argument("--min-interval", type=float, default=4.0, help="API throttle seconds") parser.add_argument("--dsn", default=default_dsn(), help="PostgreSQL DSN") return run(parser.parse_args(argv)) if __name__ == "__main__": sys.exit(main())