goods/ingestion/opengoods/etl/transform.py

"""Transform raw Open Food Facts records into the OpenGoods internal shape.

Pure functions (no DB, no network) so they are easy to unit-test against
fixtures. The output dict mirrors the columns the loader writes.
"""

from __future__ import annotations

import re
from decimal import Decimal

from opengoods.units import UnitError, normalize

# OFF nutriment key -> our attribute key. Energy handled separately.
_NUTRIMENT_KEYS = {
    "proteins_100g": "proteins",
    "fat_100g": "fat",
    "saturated-fat_100g": "saturated_fat",
    "carbohydrates_100g": "carbohydrates",
    "sugars_100g": "sugars",
    "salt_100g": "salt",
}

# Very small keyword -> category path map (starter; replaced by a proper
# OFF taxonomy -> GPC mapping table later).
_CATEGORY_KEYWORDS: list[tuple[tuple[str, ...], str]] = [
    (("water", "eau", "饮用水", "矿泉水"), "food.beverages.water"),
    (("soda", "carbonated", "汽水", "碳酸"), "food.beverages.carbonated"),
    (("juice", "jus", "果汁"), "food.beverages.juice"),
    (("milk", "lait", "牛奶"), "food.dairy.milk"),
    (("yogurt", "yoghurt", "yaourt", "酸奶"), "food.dairy.yogurt"),
    (("cheese", "fromage", "奶酪", "干酪"), "food.dairy.cheese"),
    (("bread", "pain", "面包"), "food.bakery.bread"),
    (("biscuit", "cookie", "饼干"), "food.bakery.biscuits"),
    (("chips", "crisps", "薯片", "膨化"), "food.snacks.chips"),
    (("chocolate", "chocolat", "巧克力"), "food.snacks.chocolate"),
    (("rice", "riz", "大米", "稻米"), "food.staple.rice"),
    (("noodle", "pasta", "面条", "挂面"), "food.staple.noodles"),
    (("oil", "huile", "食用油", "食油"), "food.staple.cooking_oil"),
    (("soy sauce", "酱油"), "food.condiments.soy_sauce"),
    (("salt", "sel", "食盐"), "food.condiments.salt"),
]

_QTY_RE = re.compile(r"(?P<value>\d+(?:[.,]\d+)?)\s*(?P<unit>[a-zA-Z\u4e00-\u9fff%]+)")


def is_valid_gtin(code: str) -> bool:
    """Validate a GTIN-8/12/13/14 using the standard check digit."""
    if not code.isdigit() or len(code) not in (8, 12, 13, 14):
        return False
    digits = [int(c) for c in code]
    check = digits[-1]
    body = digits[:-1][::-1]
    total = sum(d * (3 if i % 2 == 0 else 1) for i, d in enumerate(body))
    return (10 - total % 10) % 10 == check


def parse_quantity(text: str) -> tuple[Decimal, str] | None:
    """Parse a free-text quantity like '500 g' or '1,5 L' -> (value, unit)."""
    if not text:
        return None
    m = _QTY_RE.search(text)
    if not m:
        return None
    value = Decimal(m.group("value").replace(",", "."))
    return value, m.group("unit")


def map_category(raw: dict) -> str | None:
    """Best-effort map OFF categories/name to a self-built category path."""
    haystack = " ".join(
        str(raw.get(k, ""))
        for k in ("categories", "categories_tags", "product_name", "product_name_en")
    ).lower()
    for keywords, path in _CATEGORY_KEYWORDS:
        if any(kw.lower() in haystack for kw in keywords):
            return path
    return None


def _clamp(value: str | None, max_len: int) -> str | None:
    """Trim a string to fit a bounded DB column; external data length varies."""
    if value is None:
        return None
    value = value.strip()
    return value[:max_len] or None


def _clean_tags(tags: list[str] | None, prefix: str = "") -> list[str]:
    out: list[str] = []
    for t in tags or []:
        v = t.split(":", 1)[-1] if ":" in t else t
        v = v.strip().replace("-", " ")
        if v:
            out.append(v)
    return out


def transform_nutriments(off_nutriments: dict) -> dict:
    """Build a nutriments dict on a per_100g basis with dual energy units."""
    out: dict[str, object] = {}
    for off_key, our_key in _NUTRIMENT_KEYS.items():
        if off_key in off_nutriments and off_nutriments[off_key] is not None:
            out[our_key] = float(off_nutriments[off_key])

    kj = off_nutriments.get("energy-kj_100g")
    kcal = off_nutriments.get("energy-kcal_100g")
    if kj is None and kcal is not None:
        kj = float(Decimal(str(kcal)) * Decimal("4.184"))
    if kcal is None and kj is not None:
        kcal = float(Decimal(str(kj)) / Decimal("4.184"))
    if kj is not None:
        out["energy_kj"] = round(float(kj), 3)
    if kcal is not None:
        out["energy_kcal"] = round(float(kcal), 3)
    return out


def transform(raw: dict) -> dict | None:
    """Transform one raw OFF product record into an internal product dict.

    Returns None if the record lacks a usable name.
    """
    name = raw.get("product_name_zh") or raw.get("product_name") or raw.get("product_name_en")
    if not name:
        return None

    code = str(raw.get("code", "")).strip()
    gtin = code if code and is_valid_gtin(code) else None

    brands = raw.get("brands") or ""
    brand = brands.split(",")[0].strip() or None

    net_value = net_unit = net_canonical = None
    parsed = parse_quantity(raw.get("quantity", ""))
    if parsed:
        value, unit = parsed
        try:
            norm = normalize(value, unit)
            net_value, net_unit, net_canonical = (
                norm.value,
                norm.unit,
                norm.canonical_value,
            )
        except UnitError:
            net_value, net_unit = value, unit

    return {
        "gtin": gtin,
        "name": str(name).strip(),
        "brand": brand,
        "category_path": map_category(raw),
        "net_content_value": net_value,
        "net_content_unit": _clamp(net_unit, 16),
        "net_content_canonical": net_canonical,
        "country_of_origin": _clamp((raw.get("countries") or "").split(",")[0].strip() or None, 64),
        "food": {
            "ingredients_text": raw.get("ingredients_text") or None,
            "allergens": _clean_tags(raw.get("allergens_tags")),
            "additives": _clean_tags(raw.get("additives_tags")),
            "nutriments": transform_nutriments(raw.get("nutriments") or {}),
            "nutrition_basis": "per_100g",
            "serving_size": _clamp(raw.get("serving_size") or None, 32),
            "nutri_score": (raw.get("nutriscore_grade") or "").upper()[:1] or None,
        },
        "image_url": raw.get("image_front_url") or raw.get("image_url") or None,
    }