feat(M2): Open Food Facts 采集导入 ETL
- adapters/openfoodfacts.py: OFF API 适配器(限速+User-Agent) + JSONL/.gz dump 读取 - etl/transform.py: 纯函数转换(GTIN 校验/净含量解析+归一/营养 per_100g 双能量/过敏原添加剂清洗/关键词分类映射) - etl/load.py: psycopg upsert(product/food_detail/product_image) + product_source 字段级溯源 - jobs/seed_off.py: CLI(--barcodes API / --dump 文件 / --limit) - 测试: 13 个离线 transform 单测(fixture) + 可跳过的 DB 集成测试 - 依赖: 增加 psycopg[binary] - 实跑: 从 OFF API 拉 5 个真实条码入本地 postgres 验证通过 - docs/etl-openfoodfacts.md: 流程/运行/字段映射 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,159 @@
|
||||
"""Transform raw Open Food Facts records into the OpenGoods internal shape.
|
||||
|
||||
Pure functions (no DB, no network) so they are easy to unit-test against
|
||||
fixtures. The output dict mirrors the columns the loader writes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from decimal import Decimal
|
||||
|
||||
from opengoods.units import UnitError, normalize
|
||||
|
||||
# OFF nutriment key -> our attribute key. Energy handled separately.
|
||||
_NUTRIMENT_KEYS = {
|
||||
"proteins_100g": "proteins",
|
||||
"fat_100g": "fat",
|
||||
"saturated-fat_100g": "saturated_fat",
|
||||
"carbohydrates_100g": "carbohydrates",
|
||||
"sugars_100g": "sugars",
|
||||
"salt_100g": "salt",
|
||||
}
|
||||
|
||||
# Very small keyword -> category path map (starter; replaced by a proper
|
||||
# OFF taxonomy -> GPC mapping table later).
|
||||
_CATEGORY_KEYWORDS: list[tuple[tuple[str, ...], str]] = [
|
||||
(("water", "eau", "饮用水", "矿泉水"), "food.beverages.water"),
|
||||
(("soda", "carbonated", "汽水", "碳酸"), "food.beverages.carbonated"),
|
||||
(("juice", "jus", "果汁"), "food.beverages.juice"),
|
||||
(("milk", "lait", "牛奶"), "food.dairy.milk"),
|
||||
(("yogurt", "yoghurt", "yaourt", "酸奶"), "food.dairy.yogurt"),
|
||||
(("cheese", "fromage", "奶酪", "干酪"), "food.dairy.cheese"),
|
||||
(("bread", "pain", "面包"), "food.bakery.bread"),
|
||||
(("biscuit", "cookie", "饼干"), "food.bakery.biscuits"),
|
||||
(("chips", "crisps", "薯片", "膨化"), "food.snacks.chips"),
|
||||
(("chocolate", "chocolat", "巧克力"), "food.snacks.chocolate"),
|
||||
(("rice", "riz", "大米", "稻米"), "food.staple.rice"),
|
||||
(("noodle", "pasta", "面条", "挂面"), "food.staple.noodles"),
|
||||
(("oil", "huile", "食用油", "食油"), "food.staple.cooking_oil"),
|
||||
(("soy sauce", "酱油"), "food.condiments.soy_sauce"),
|
||||
(("salt", "sel", "食盐"), "food.condiments.salt"),
|
||||
]
|
||||
|
||||
_QTY_RE = re.compile(r"(?P<value>\d+(?:[.,]\d+)?)\s*(?P<unit>[a-zA-Z\u4e00-\u9fff%]+)")
|
||||
|
||||
|
||||
def is_valid_gtin(code: str) -> bool:
|
||||
"""Validate a GTIN-8/12/13/14 using the standard check digit."""
|
||||
if not code.isdigit() or len(code) not in (8, 12, 13, 14):
|
||||
return False
|
||||
digits = [int(c) for c in code]
|
||||
check = digits[-1]
|
||||
body = digits[:-1][::-1]
|
||||
total = sum(d * (3 if i % 2 == 0 else 1) for i, d in enumerate(body))
|
||||
return (10 - total % 10) % 10 == check
|
||||
|
||||
|
||||
def parse_quantity(text: str) -> tuple[Decimal, str] | None:
|
||||
"""Parse a free-text quantity like '500 g' or '1,5 L' -> (value, unit)."""
|
||||
if not text:
|
||||
return None
|
||||
m = _QTY_RE.search(text)
|
||||
if not m:
|
||||
return None
|
||||
value = Decimal(m.group("value").replace(",", "."))
|
||||
return value, m.group("unit")
|
||||
|
||||
|
||||
def map_category(raw: dict) -> str | None:
|
||||
"""Best-effort map OFF categories/name to a self-built category path."""
|
||||
haystack = " ".join(
|
||||
str(raw.get(k, ""))
|
||||
for k in ("categories", "categories_tags", "product_name", "product_name_en")
|
||||
).lower()
|
||||
for keywords, path in _CATEGORY_KEYWORDS:
|
||||
if any(kw.lower() in haystack for kw in keywords):
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def _clean_tags(tags: list[str] | None, prefix: str = "") -> list[str]:
|
||||
out: list[str] = []
|
||||
for t in tags or []:
|
||||
v = t.split(":", 1)[-1] if ":" in t else t
|
||||
v = v.strip().replace("-", " ")
|
||||
if v:
|
||||
out.append(v)
|
||||
return out
|
||||
|
||||
|
||||
def transform_nutriments(off_nutriments: dict) -> dict:
|
||||
"""Build a nutriments dict on a per_100g basis with dual energy units."""
|
||||
out: dict[str, object] = {}
|
||||
for off_key, our_key in _NUTRIMENT_KEYS.items():
|
||||
if off_key in off_nutriments and off_nutriments[off_key] is not None:
|
||||
out[our_key] = float(off_nutriments[off_key])
|
||||
|
||||
kj = off_nutriments.get("energy-kj_100g")
|
||||
kcal = off_nutriments.get("energy-kcal_100g")
|
||||
if kj is None and kcal is not None:
|
||||
kj = float(Decimal(str(kcal)) * Decimal("4.184"))
|
||||
if kcal is None and kj is not None:
|
||||
kcal = float(Decimal(str(kj)) / Decimal("4.184"))
|
||||
if kj is not None:
|
||||
out["energy_kj"] = round(float(kj), 3)
|
||||
if kcal is not None:
|
||||
out["energy_kcal"] = round(float(kcal), 3)
|
||||
return out
|
||||
|
||||
|
||||
def transform(raw: dict) -> dict | None:
|
||||
"""Transform one raw OFF product record into an internal product dict.
|
||||
|
||||
Returns None if the record lacks a usable name.
|
||||
"""
|
||||
name = raw.get("product_name_zh") or raw.get("product_name") or raw.get("product_name_en")
|
||||
if not name:
|
||||
return None
|
||||
|
||||
code = str(raw.get("code", "")).strip()
|
||||
gtin = code if code and is_valid_gtin(code) else None
|
||||
|
||||
brands = raw.get("brands") or ""
|
||||
brand = brands.split(",")[0].strip() or None
|
||||
|
||||
net_value = net_unit = net_canonical = None
|
||||
parsed = parse_quantity(raw.get("quantity", ""))
|
||||
if parsed:
|
||||
value, unit = parsed
|
||||
try:
|
||||
norm = normalize(value, unit)
|
||||
net_value, net_unit, net_canonical = (
|
||||
norm.value,
|
||||
norm.unit,
|
||||
norm.canonical_value,
|
||||
)
|
||||
except UnitError:
|
||||
net_value, net_unit = value, unit
|
||||
|
||||
return {
|
||||
"gtin": gtin,
|
||||
"name": str(name).strip(),
|
||||
"brand": brand,
|
||||
"category_path": map_category(raw),
|
||||
"net_content_value": net_value,
|
||||
"net_content_unit": net_unit,
|
||||
"net_content_canonical": net_canonical,
|
||||
"country_of_origin": (raw.get("countries") or "").split(",")[0].strip() or None,
|
||||
"food": {
|
||||
"ingredients_text": raw.get("ingredients_text") or None,
|
||||
"allergens": _clean_tags(raw.get("allergens_tags")),
|
||||
"additives": _clean_tags(raw.get("additives_tags")),
|
||||
"nutriments": transform_nutriments(raw.get("nutriments") or {}),
|
||||
"nutrition_basis": "per_100g",
|
||||
"serving_size": raw.get("serving_size") or None,
|
||||
"nutri_score": (raw.get("nutriscore_grade") or "").upper()[:1] or None,
|
||||
},
|
||||
"image_url": raw.get("image_front_url") or raw.get("image_url") or None,
|
||||
}
|
||||
Reference in New Issue
Block a user