lixu
766a573989
- adapters/openfoodfacts.py: OFF API 适配器(限速+User-Agent) + JSONL/.gz dump 读取 - etl/transform.py: 纯函数转换(GTIN 校验/净含量解析+归一/营养 per_100g 双能量/过敏原添加剂清洗/关键词分类映射) - etl/load.py: psycopg upsert(product/food_detail/product_image) + product_source 字段级溯源 - jobs/seed_off.py: CLI(--barcodes API / --dump 文件 / --limit) - 测试: 13 个离线 transform 单测(fixture) + 可跳过的 DB 集成测试 - 依赖: 增加 psycopg[binary] - 实跑: 从 OFF API 拉 5 个真实条码入本地 postgres 验证通过 - docs/etl-openfoodfacts.md: 流程/运行/字段映射 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
33 lines
660 B
TOML
33 lines
660 B
TOML
[project]
|
|
name = "opengoods-ingestion"
|
|
version = "0.1.0"
|
|
description = "OpenGoods (天工·商品标签) ingestion & ETL: collect product data and load it into the OpenGoods database."
|
|
requires-python = ">=3.11"
|
|
dependencies = [
|
|
"httpx>=0.27",
|
|
"psycopg[binary]>=3.2",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
dev = [
|
|
"ruff>=0.6",
|
|
"pytest>=8.0",
|
|
]
|
|
|
|
[build-system]
|
|
requires = ["setuptools>=68"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[tool.setuptools.packages.find]
|
|
include = ["opengoods*"]
|
|
|
|
[tool.ruff]
|
|
line-length = 100
|
|
target-version = "py311"
|
|
|
|
[tool.ruff.lint]
|
|
select = ["E", "F", "I", "UP", "B"]
|
|
|
|
[tool.pytest.ini_options]
|
|
testpaths = ["tests"]
|