feat(M2): Open Food Facts 采集导入 ETL
- adapters/openfoodfacts.py: OFF API 适配器(限速+User-Agent) + JSONL/.gz dump 读取 - etl/transform.py: 纯函数转换(GTIN 校验/净含量解析+归一/营养 per_100g 双能量/过敏原添加剂清洗/关键词分类映射) - etl/load.py: psycopg upsert(product/food_detail/product_image) + product_source 字段级溯源 - jobs/seed_off.py: CLI(--barcodes API / --dump 文件 / --limit) - 测试: 13 个离线 transform 单测(fixture) + 可跳过的 DB 集成测试 - 依赖: 增加 psycopg[binary] - 实跑: 从 OFF API 拉 5 个真实条码入本地 postgres 验证通过 - docs/etl-openfoodfacts.md: 流程/运行/字段映射 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,66 @@
|
||||
"""Seed the database with Open Food Facts data.
|
||||
|
||||
Usage:
|
||||
# from a list of barcodes via the OFF API
|
||||
python -m opengoods.jobs.seed_off --barcodes 3017624010701 5449000000996
|
||||
|
||||
# from a downloaded OFF JSONL dump (optionally .gz), limited to N records
|
||||
python -m opengoods.jobs.seed_off --dump products.jsonl.gz --limit 1000
|
||||
|
||||
The OFF read API is rate-limited client-side; for large imports use a dump.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
|
||||
import psycopg
|
||||
|
||||
from opengoods.adapters.openfoodfacts import OpenFoodFactsAdapter, read_dump
|
||||
from opengoods.etl.load import default_dsn, ensure_source, load_record
|
||||
from opengoods.etl.transform import transform
|
||||
|
||||
|
||||
def _raw_records(args: argparse.Namespace) -> Iterator[dict]:
|
||||
if args.dump:
|
||||
records = read_dump(args.dump)
|
||||
else:
|
||||
adapter = OpenFoodFactsAdapter(min_interval=args.min_interval)
|
||||
records = adapter.fetch(args.barcodes)
|
||||
for i, rec in enumerate(records):
|
||||
if args.limit and i >= args.limit:
|
||||
break
|
||||
yield rec
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> int:
|
||||
loaded = skipped = 0
|
||||
with psycopg.connect(args.dsn, autocommit=False) as conn:
|
||||
source_id = ensure_source(conn)
|
||||
for raw in _raw_records(args):
|
||||
rec = transform(raw)
|
||||
if rec is None:
|
||||
skipped += 1
|
||||
continue
|
||||
load_record(conn, rec, source_id, raw)
|
||||
loaded += 1
|
||||
conn.commit()
|
||||
print(f"loaded={loaded} skipped={skipped}")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Seed OpenGoods from Open Food Facts")
|
||||
src = parser.add_mutually_exclusive_group(required=True)
|
||||
src.add_argument("--barcodes", nargs="+", help="barcodes to fetch via the OFF API")
|
||||
src.add_argument("--dump", help="path to an OFF JSONL dump (.jsonl or .jsonl.gz)")
|
||||
parser.add_argument("--limit", type=int, default=0, help="max records to load (0 = all)")
|
||||
parser.add_argument("--min-interval", type=float, default=4.0, help="API throttle seconds")
|
||||
parser.add_argument("--dsn", default=default_dsn(), help="PostgreSQL DSN")
|
||||
return run(parser.parse_args(argv))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user