feat(ingest): country-focused seeding (collect domestic CN products)
Add a market-focused seeding path so the catalogue can be built from domestic products rather than the English-heavy global default: - adapter.fetch_by_country(country): OFF search filtered by countries_tags_en, sorted by unique_scans_n (most-scanned first), de-duplicated across pages since OFF popularity ordering is unstable. - is_cn_gs1(code): True for GS1-China company prefixes (690-699), i.e. genuinely domestic items vs. imports merely sold in China. - seed_off --country <slug> [--domestic-only] [--page-size/--max-pages]: e.g. 'seed_off --country china --domestic-only' loads only 69x barcodes. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
@@ -7,6 +7,11 @@ Usage:
|
||||
# from a downloaded OFF JSONL dump (optionally .gz), limited to N records
|
||||
python -m opengoods.jobs.seed_off --dump products.jsonl.gz --limit 1000
|
||||
|
||||
# market-focused: the most-scanned products sold in China, restricted to
|
||||
# genuine GS1-China (69x) barcodes
|
||||
python -m opengoods.jobs.seed_off --country china --domestic-only \
|
||||
--max-pages 20 --limit 1000
|
||||
|
||||
The OFF read API is rate-limited client-side; for large imports use a dump.
|
||||
"""
|
||||
|
||||
@@ -18,20 +23,33 @@ from collections.abc import Iterator
|
||||
|
||||
import psycopg
|
||||
|
||||
from opengoods.adapters.openfoodfacts import OpenFoodFactsAdapter, read_dump
|
||||
from opengoods.adapters.openfoodfacts import (
|
||||
OpenFoodFactsAdapter,
|
||||
is_cn_gs1,
|
||||
read_dump,
|
||||
)
|
||||
from opengoods.etl.load import default_dsn, ensure_source, load_record_safe
|
||||
from opengoods.etl.transform import transform
|
||||
|
||||
|
||||
def _raw_records(args: argparse.Namespace) -> Iterator[dict]:
|
||||
if args.dump:
|
||||
records = read_dump(args.dump)
|
||||
records: Iterator[dict] = read_dump(args.dump)
|
||||
elif args.country:
|
||||
adapter = OpenFoodFactsAdapter(min_interval=args.min_interval)
|
||||
records = adapter.fetch_by_country(
|
||||
args.country, page_size=args.page_size, max_pages=args.max_pages
|
||||
)
|
||||
else:
|
||||
adapter = OpenFoodFactsAdapter(min_interval=args.min_interval)
|
||||
records = adapter.fetch(args.barcodes)
|
||||
for i, rec in enumerate(records):
|
||||
if args.limit and i >= args.limit:
|
||||
yielded = 0
|
||||
for rec in records:
|
||||
if args.domestic_only and not is_cn_gs1(str(rec.get("code") or "")):
|
||||
continue
|
||||
if args.limit and yielded >= args.limit:
|
||||
break
|
||||
yielded += 1
|
||||
yield rec
|
||||
|
||||
|
||||
@@ -58,7 +76,18 @@ def main(argv: list[str] | None = None) -> int:
|
||||
src = parser.add_mutually_exclusive_group(required=True)
|
||||
src.add_argument("--barcodes", nargs="+", help="barcodes to fetch via the OFF API")
|
||||
src.add_argument("--dump", help="path to an OFF JSONL dump (.jsonl or .jsonl.gz)")
|
||||
src.add_argument(
|
||||
"--country",
|
||||
help="OFF countries_tags_en slug to seed from, e.g. 'china' (most-scanned first)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--domestic-only",
|
||||
action="store_true",
|
||||
help="keep only genuine GS1-China (69x) barcodes; drop imported goods",
|
||||
)
|
||||
parser.add_argument("--limit", type=int, default=0, help="max records to load (0 = all)")
|
||||
parser.add_argument("--page-size", type=int, default=100, help="search page size")
|
||||
parser.add_argument("--max-pages", type=int, default=10, help="max search pages (country mode)")
|
||||
parser.add_argument("--min-interval", type=float, default=4.0, help="API throttle seconds")
|
||||
parser.add_argument("--dsn", default=default_dsn(), help="PostgreSQL DSN")
|
||||
return run(parser.parse_args(argv))
|
||||
|
||||
Reference in New Issue
Block a user