"""Transform raw Open Food Facts records into the OpenGoods internal shape. Pure functions (no DB, no network) so they are easy to unit-test against fixtures. The output dict mirrors the columns the loader writes. """ from __future__ import annotations import re from decimal import Decimal from opengoods.units import UnitError, normalize # OFF nutriment key -> our attribute key. Energy handled separately. _NUTRIMENT_KEYS = { "proteins_100g": "proteins", "fat_100g": "fat", "saturated-fat_100g": "saturated_fat", "carbohydrates_100g": "carbohydrates", "sugars_100g": "sugars", "salt_100g": "salt", } # Very small keyword -> category path map (starter; replaced by a proper # OFF taxonomy -> GPC mapping table later). _CATEGORY_KEYWORDS: list[tuple[tuple[str, ...], str]] = [ (("water", "eau", "饮用水", "矿泉水"), "food.beverages.water"), (("soda", "carbonated", "汽水", "碳酸"), "food.beverages.carbonated"), (("juice", "jus", "果汁"), "food.beverages.juice"), (("milk", "lait", "牛奶"), "food.dairy.milk"), (("yogurt", "yoghurt", "yaourt", "酸奶"), "food.dairy.yogurt"), (("cheese", "fromage", "奶酪", "干酪"), "food.dairy.cheese"), (("bread", "pain", "面包"), "food.bakery.bread"), (("biscuit", "cookie", "饼干"), "food.bakery.biscuits"), (("chips", "crisps", "薯片", "膨化"), "food.snacks.chips"), (("chocolate", "chocolat", "巧克力"), "food.snacks.chocolate"), (("rice", "riz", "大米", "稻米"), "food.staple.rice"), (("noodle", "pasta", "面条", "挂面"), "food.staple.noodles"), (("oil", "huile", "食用油", "食油"), "food.staple.cooking_oil"), (("soy sauce", "酱油"), "food.condiments.soy_sauce"), (("salt", "sel", "食盐"), "food.condiments.salt"), ] _QTY_RE = re.compile(r"(?P\d+(?:[.,]\d+)?)\s*(?P[a-zA-Z\u4e00-\u9fff%]+)") def is_valid_gtin(code: str) -> bool: """Validate a GTIN-8/12/13/14 using the standard check digit.""" if not code.isdigit() or len(code) not in (8, 12, 13, 14): return False digits = [int(c) for c in code] check = digits[-1] body = digits[:-1][::-1] total = sum(d * (3 if i % 2 == 0 else 1) for i, d in enumerate(body)) return (10 - total % 10) % 10 == check def parse_quantity(text: str) -> tuple[Decimal, str] | None: """Parse a free-text quantity like '500 g' or '1,5 L' -> (value, unit).""" if not text: return None m = _QTY_RE.search(text) if not m: return None value = Decimal(m.group("value").replace(",", ".")) return value, m.group("unit") def map_category(raw: dict) -> str | None: """Best-effort map OFF categories/name to a self-built category path.""" haystack = " ".join( str(raw.get(k, "")) for k in ("categories", "categories_tags", "product_name", "product_name_en") ).lower() for keywords, path in _CATEGORY_KEYWORDS: if any(kw.lower() in haystack for kw in keywords): return path return None def _clamp(value: str | None, max_len: int) -> str | None: """Trim a string to fit a bounded DB column; external data length varies.""" if value is None: return None value = value.strip() return value[:max_len] or None def _clean_tags(tags: list[str] | None, prefix: str = "") -> list[str]: out: list[str] = [] for t in tags or []: v = t.split(":", 1)[-1] if ":" in t else t v = v.strip().replace("-", " ") if v: out.append(v) return out def transform_nutriments(off_nutriments: dict) -> dict: """Build a nutriments dict on a per_100g basis with dual energy units.""" out: dict[str, object] = {} for off_key, our_key in _NUTRIMENT_KEYS.items(): if off_key in off_nutriments and off_nutriments[off_key] is not None: out[our_key] = float(off_nutriments[off_key]) kj = off_nutriments.get("energy-kj_100g") kcal = off_nutriments.get("energy-kcal_100g") if kj is None and kcal is not None: kj = float(Decimal(str(kcal)) * Decimal("4.184")) if kcal is None and kj is not None: kcal = float(Decimal(str(kj)) / Decimal("4.184")) if kj is not None: out["energy_kj"] = round(float(kj), 3) if kcal is not None: out["energy_kcal"] = round(float(kcal), 3) return out def transform(raw: dict) -> dict | None: """Transform one raw OFF product record into an internal product dict. Returns None if the record lacks a usable name. """ name = raw.get("product_name_zh") or raw.get("product_name") or raw.get("product_name_en") if not name: return None code = str(raw.get("code", "")).strip() gtin = code if code and is_valid_gtin(code) else None brands = raw.get("brands") or "" brand = brands.split(",")[0].strip() or None net_value = net_unit = net_canonical = None parsed = parse_quantity(raw.get("quantity", "")) if parsed: value, unit = parsed try: norm = normalize(value, unit) net_value, net_unit, net_canonical = ( norm.value, norm.unit, norm.canonical_value, ) except UnitError: net_value, net_unit = value, unit return { "gtin": gtin, "name": str(name).strip(), "brand": brand, "category_path": map_category(raw), "net_content_value": net_value, "net_content_unit": _clamp(net_unit, 16), "net_content_canonical": net_canonical, "country_of_origin": _clamp((raw.get("countries") or "").split(",")[0].strip() or None, 64), "food": { "ingredients_text": raw.get("ingredients_text") or None, "allergens": _clean_tags(raw.get("allergens_tags")), "additives": _clean_tags(raw.get("additives_tags")), "nutriments": transform_nutriments(raw.get("nutriments") or {}), "nutrition_basis": "per_100g", "serving_size": _clamp(raw.get("serving_size") or None, 32), "nutri_score": (raw.get("nutriscore_grade") or "").upper()[:1] or None, }, "image_url": raw.get("image_front_url") or raw.get("image_url") or None, }