rosemariejebbjtxbfp
a75318b811
Stage-0 caching from docs/scalability.md. The Go API now caches hot product-detail and search-result reads in Redis with a fail-open, epoch-versioned scheme; Python ingestion bumps the epoch after a write run to invalidate the cache globally in O(1). - api/internal/cache: fail-open Cache (GetJSON/SetJSON) namespaced by an epoch counter (og:cache:epoch); disabled when Redis is unconfigured. - store: ProductByID/ProductByGTIN (24h TTL) and SearchProducts (1h TTL) read-through the cache via WithCache. - ingestion: bump_cache_epoch() called after update_off/seed_off/ import_bypos/dedup commits when rows changed; best-effort, never fails a run. Adds redis dependency. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
101 lines
3.5 KiB
Python
101 lines
3.5 KiB
Python
"""Seed the database with Open Food Facts data.
|
|
|
|
Usage:
|
|
# from a list of barcodes via the OFF API
|
|
python -m opengoods.jobs.seed_off --barcodes 3017624010701 5449000000996
|
|
|
|
# from a downloaded OFF JSONL dump (optionally .gz), limited to N records
|
|
python -m opengoods.jobs.seed_off --dump products.jsonl.gz --limit 1000
|
|
|
|
# market-focused: the most-scanned products sold in China, restricted to
|
|
# genuine GS1-China (69x) barcodes
|
|
python -m opengoods.jobs.seed_off --country china --domestic-only \
|
|
--max-pages 20 --limit 1000
|
|
|
|
The OFF read API is rate-limited client-side; for large imports use a dump.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
from collections.abc import Iterator
|
|
|
|
import psycopg
|
|
|
|
from opengoods.adapters.openfoodfacts import (
|
|
OpenFoodFactsAdapter,
|
|
is_cn_gs1,
|
|
read_dump,
|
|
)
|
|
from opengoods.cache import bump_cache_epoch
|
|
from opengoods.etl.load import default_dsn, ensure_source, load_record_safe
|
|
from opengoods.etl.transform import transform
|
|
|
|
|
|
def _raw_records(args: argparse.Namespace) -> Iterator[dict]:
|
|
if args.dump:
|
|
records: Iterator[dict] = read_dump(args.dump)
|
|
elif args.country:
|
|
adapter = OpenFoodFactsAdapter(min_interval=args.min_interval)
|
|
records = adapter.fetch_by_country(
|
|
args.country, page_size=args.page_size, max_pages=args.max_pages
|
|
)
|
|
else:
|
|
adapter = OpenFoodFactsAdapter(min_interval=args.min_interval)
|
|
records = adapter.fetch(args.barcodes)
|
|
yielded = 0
|
|
for rec in records:
|
|
if args.domestic_only and not is_cn_gs1(str(rec.get("code") or "")):
|
|
continue
|
|
if args.limit and yielded >= args.limit:
|
|
break
|
|
yielded += 1
|
|
yield rec
|
|
|
|
|
|
def run(args: argparse.Namespace) -> int:
|
|
loaded = skipped = errored = 0
|
|
with psycopg.connect(args.dsn, autocommit=False) as conn:
|
|
source_id = ensure_source(conn)
|
|
for raw in _raw_records(args):
|
|
rec = transform(raw)
|
|
if rec is None:
|
|
skipped += 1
|
|
continue
|
|
if load_record_safe(conn, rec, source_id, raw):
|
|
loaded += 1
|
|
else:
|
|
errored += 1
|
|
conn.commit()
|
|
if loaded:
|
|
bump_cache_epoch()
|
|
print(f"loaded={loaded} skipped={skipped} errored={errored}")
|
|
return 0
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="Seed OpenGoods from Open Food Facts")
|
|
src = parser.add_mutually_exclusive_group(required=True)
|
|
src.add_argument("--barcodes", nargs="+", help="barcodes to fetch via the OFF API")
|
|
src.add_argument("--dump", help="path to an OFF JSONL dump (.jsonl or .jsonl.gz)")
|
|
src.add_argument(
|
|
"--country",
|
|
help="OFF countries_tags_en slug to seed from, e.g. 'china' (most-scanned first)",
|
|
)
|
|
parser.add_argument(
|
|
"--domestic-only",
|
|
action="store_true",
|
|
help="keep only genuine GS1-China (69x) barcodes; drop imported goods",
|
|
)
|
|
parser.add_argument("--limit", type=int, default=0, help="max records to load (0 = all)")
|
|
parser.add_argument("--page-size", type=int, default=100, help="search page size")
|
|
parser.add_argument("--max-pages", type=int, default=10, help="max search pages (country mode)")
|
|
parser.add_argument("--min-interval", type=float, default=4.0, help="API throttle seconds")
|
|
parser.add_argument("--dsn", default=default_dsn(), help="PostgreSQL DSN")
|
|
return run(parser.parse_args(argv))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|