feat(ingest): country-focused seeding (collect domestic CN products)
CI / Python (ingestion) (pull_request) Failing after 6s
CI / Migrations (postgres) (pull_request) Successful in 16s
CI / Go (api) (pull_request) Failing after 11m35s

Add a market-focused seeding path so the catalogue can be built from
domestic products rather than the English-heavy global default:

- adapter.fetch_by_country(country): OFF search filtered by
  countries_tags_en, sorted by unique_scans_n (most-scanned first),
  de-duplicated across pages since OFF popularity ordering is unstable.
- is_cn_gs1(code): True for GS1-China company prefixes (690-699),
  i.e. genuinely domestic items vs. imports merely sold in China.
- seed_off --country <slug> [--domestic-only] [--page-size/--max-pages]:
  e.g. 'seed_off --country china --domestic-only' loads only 69x
  barcodes.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
novaalphastrikeomegaz663
2026-06-20 07:14:45 +00:00
parent 044c870df7
commit 2255243081
3 changed files with 144 additions and 4 deletions
+33 -4
View File
@@ -7,6 +7,11 @@ Usage:
# from a downloaded OFF JSONL dump (optionally .gz), limited to N records
python -m opengoods.jobs.seed_off --dump products.jsonl.gz --limit 1000
# market-focused: the most-scanned products sold in China, restricted to
# genuine GS1-China (69x) barcodes
python -m opengoods.jobs.seed_off --country china --domestic-only \
--max-pages 20 --limit 1000
The OFF read API is rate-limited client-side; for large imports use a dump.
"""
@@ -18,20 +23,33 @@ from collections.abc import Iterator
import psycopg
from opengoods.adapters.openfoodfacts import OpenFoodFactsAdapter, read_dump
from opengoods.adapters.openfoodfacts import (
OpenFoodFactsAdapter,
is_cn_gs1,
read_dump,
)
from opengoods.etl.load import default_dsn, ensure_source, load_record_safe
from opengoods.etl.transform import transform
def _raw_records(args: argparse.Namespace) -> Iterator[dict]:
if args.dump:
records = read_dump(args.dump)
records: Iterator[dict] = read_dump(args.dump)
elif args.country:
adapter = OpenFoodFactsAdapter(min_interval=args.min_interval)
records = adapter.fetch_by_country(
args.country, page_size=args.page_size, max_pages=args.max_pages
)
else:
adapter = OpenFoodFactsAdapter(min_interval=args.min_interval)
records = adapter.fetch(args.barcodes)
for i, rec in enumerate(records):
if args.limit and i >= args.limit:
yielded = 0
for rec in records:
if args.domestic_only and not is_cn_gs1(str(rec.get("code") or "")):
continue
if args.limit and yielded >= args.limit:
break
yielded += 1
yield rec
@@ -58,7 +76,18 @@ def main(argv: list[str] | None = None) -> int:
src = parser.add_mutually_exclusive_group(required=True)
src.add_argument("--barcodes", nargs="+", help="barcodes to fetch via the OFF API")
src.add_argument("--dump", help="path to an OFF JSONL dump (.jsonl or .jsonl.gz)")
src.add_argument(
"--country",
help="OFF countries_tags_en slug to seed from, e.g. 'china' (most-scanned first)",
)
parser.add_argument(
"--domestic-only",
action="store_true",
help="keep only genuine GS1-China (69x) barcodes; drop imported goods",
)
parser.add_argument("--limit", type=int, default=0, help="max records to load (0 = all)")
parser.add_argument("--page-size", type=int, default=100, help="search page size")
parser.add_argument("--max-pages", type=int, default=10, help="max search pages (country mode)")
parser.add_argument("--min-interval", type=float, default=4.0, help="API throttle seconds")
parser.add_argument("--dsn", default=default_dsn(), help="PostgreSQL DSN")
return run(parser.parse_args(argv))