feat(M2): Open Food Facts 采集导入 ETL

- adapters/openfoodfacts.py: OFF API 适配器(限速+User-Agent) + JSONL/.gz dump 读取 - etl/transform.py: 纯函数转换(GTIN 校验/净含量解析+归一/营养 per_100g 双能量/过敏原添加剂清洗/关键词分类映射) - etl/load.py: psycopg upsert(product/food_detail/product_image) + product_source 字段级溯源 - jobs/seed_off.py: CLI(--barcodes API / --dump 文件 / --limit) - 测试: 13 个离线 transform 单测(fixture) + 可跳过的 DB 集成测试 - 依赖: 增加 psycopg[binary] - 实跑: 从 OFF API 拉 5 个真实条码入本地 postgres 验证通过 - docs/etl-openfoodfacts.md: 流程/运行/字段映射 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-06-08 07:01:04 +00:00
parent b0b816b0ee
commit 766a573989
9 changed files with 702 additions and 0 deletions
@@ -0,0 +1,26 @@
+{
+  "code": "3017624010701",
+  "product_name": "Nutella",
+  "product_name_en": "Nutella hazelnut spread",
+  "brands": "Ferrero, Nutella",
+  "quantity": "400 g",
+  "countries": "France, China",
+  "categories": "Spreads, Hazelnut spreads, Chocolate spreads",
+  "categories_tags": ["en:spreads", "en:chocolate-spreads"],
+  "ingredients_text": "Sugar, palm oil, hazelnuts, cocoa, skimmed milk powder",
+  "allergens_tags": ["en:milk", "en:nuts"],
+  "additives_tags": ["en:e322"],
+  "serving_size": "15 g",
+  "nutriscore_grade": "e",
+  "image_front_url": "https://images.openfoodfacts.org/images/products/301/762/401/0701/front_en.jpg",
+  "nutriments": {
+    "energy-kj_100g": 2252,
+    "energy-kcal_100g": 539,
+    "fat_100g": 30.9,
+    "saturated-fat_100g": 10.6,
+    "carbohydrates_100g": 57.5,
+    "sugars_100g": 56.3,
+    "proteins_100g": 6.3,
+    "salt_100g": 0.107
+  }
+}
@@ -0,0 +1,61 @@
+"""Integration test for the DB loader.
+
+Skipped automatically when no database is reachable (e.g. local runs without
+docker, or CI jobs without a postgres service). Requires migrations applied.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from opengoods.etl.load import default_dsn, ensure_source, load_record
+from opengoods.etl.transform import transform
+
+psycopg = pytest.importorskip("psycopg")
+
+FIXTURE = json.loads((Path(__file__).parent / "fixtures" / "off_product.json").read_text())
+
+
+@pytest.fixture()
+def conn():
+    try:
+        c = psycopg.connect(default_dsn(), connect_timeout=3)
+    except psycopg.OperationalError as exc:  # pragma: no cover - env dependent
+        pytest.skip(f"no database available: {exc}")
+    # ensure schema present
+    has_product = c.execute("SELECT to_regclass('public.product') IS NOT NULL").fetchone()[0]
+    if not has_product:
+        c.close()
+        pytest.skip("migrations not applied")
+    yield c
+    c.rollback()
+    c.close()
+
+
+def test_load_record_roundtrip(conn):
+    source_id = ensure_source(conn)
+    rec = transform(FIXTURE)
+    product_id = load_record(conn, rec, source_id, FIXTURE)
+
+    row = conn.execute(
+        "SELECT name, gtin, net_content_unit FROM product WHERE id = %s", (product_id,)
+    ).fetchone()
+    assert row[0] == "Nutella"
+    assert row[1] == "3017624010701"
+    assert row[2] == "g"
+
+    nutri = conn.execute(
+        "SELECT nutriments ->> 'energy_kcal' FROM food_detail WHERE product_id = %s",
+        (product_id,),
+    ).fetchone()
+    assert nutri[0] == "539.0"
+
+    prov = conn.execute(
+        "SELECT count(*) FROM product_source WHERE product_id = %s", (product_id,)
+    ).fetchone()
+    assert prov[0] >= 1
+
+    conn.rollback()  # keep the test DB clean
@@ -0,0 +1,67 @@
+import json
+from decimal import Decimal
+from pathlib import Path
+
+import pytest
+
+from opengoods.etl.transform import (
+    is_valid_gtin,
+    map_category,
+    parse_quantity,
+    transform,
+    transform_nutriments,
+)
+
+FIXTURE = json.loads((Path(__file__).parent / "fixtures" / "off_product.json").read_text())
+
+
+def test_is_valid_gtin():
+    assert is_valid_gtin("3017624010701")  # real EAN-13
+    assert is_valid_gtin("5449000000996")  # Coca-Cola
+    assert not is_valid_gtin("3017624010700")  # bad check digit
+    assert not is_valid_gtin("123")
+    assert not is_valid_gtin("notanumber")
+
+
+def test_parse_quantity():
+    assert parse_quantity("400 g") == (Decimal("400"), "g")
+    assert parse_quantity("1,5 L") == (Decimal("1.5"), "L")
+    assert parse_quantity("") is None
+    assert parse_quantity("family size") is None
+
+
+def test_map_category():
+    assert map_category({"product_name": "Spring Water"}) == "food.beverages.water"
+    assert map_category({"categories": "Dark chocolate"}) == "food.snacks.chocolate"
+    assert map_category({"product_name": "Mystery"}) is None
+
+
+def test_transform_nutriments_dual_energy():
+    out = transform_nutriments(FIXTURE["nutriments"])
+    assert out["energy_kj"] == 2252.0
+    assert out["energy_kcal"] == 539.0
+    assert out["fat"] == 30.9
+    assert out["salt"] == 0.107
+
+
+def test_transform_nutriments_fills_missing_energy():
+    out = transform_nutriments({"energy-kcal_100g": 100})
+    assert out["energy_kj"] == pytest.approx(418.4)
+
+
+def test_transform_full_record():
+    rec = transform(FIXTURE)
+    assert rec is not None
+    assert rec["gtin"] == "3017624010701"
+    assert rec["name"] == "Nutella"
+    assert rec["brand"] == "Ferrero"
+    assert rec["net_content_unit"] == "g"
+    assert rec["net_content_canonical"] == Decimal("400")
+    assert rec["country_of_origin"] == "France"
+    assert rec["food"]["nutri_score"] == "E"
+    assert "milk" in rec["food"]["allergens"]
+    assert rec["image_url"].endswith(".jpg")
+
+
+def test_transform_drops_unnamed():
+    assert transform({"code": "0000000000000"}) is None