feat(M0): 工程地基 - Go API + Python 采集骨架 + CI + 数据契约

- api/: Go(chi) 只读 API 骨架, /healthz + 版本化路由(占位), Dockerfile, 单测
- ingestion/: Python 采集/ETL 包骨架, units 单位归一化(纯函数+测试), adapter 协议
- docker-compose.yml: postgres + redis + minio + api
- .github/workflows/ci.yml: Go build/vet/test + Python ruff/pytest
- docs/data-contract.md(两端共享契约) + docs/disclaimer.md(不提供购买声明)
- migrations/ 占位(M1 起填充)

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
2026-06-08 06:18:31 +00:00
parent 729127661e
commit 786b7d3721
22 changed files with 572 additions and 0 deletions
+9
View File
@@ -0,0 +1,9 @@
"""OpenGoods (天工·商品标签) ingestion package.
Collects public product information from open data sources (e.g. Open Food
Facts) and normalizes it into the OpenGoods database. This package only
collects and processes product facts; it performs no purchase or commerce
actions.
"""
__version__ = "0.1.0"
+6
View File
@@ -0,0 +1,6 @@
"""Source adapters.
Each open data source (Open Food Facts, USDA FoodData Central, GS1, ...) gets
its own adapter that fetches raw records and yields them for the ETL layer.
Adapters must respect each source's robots.txt, rate limits and license.
"""
+17
View File
@@ -0,0 +1,17 @@
"""Base adapter protocol shared by all source adapters."""
from __future__ import annotations
from collections.abc import Iterator
from typing import Protocol
class SourceAdapter(Protocol):
"""A source adapter fetches raw product records from one data source."""
#: Stable identifier of the source, e.g. "openfoodfacts".
source_name: str
def fetch(self) -> Iterator[dict]:
"""Yield raw product records as dictionaries."""
...
+1
View File
@@ -0,0 +1 @@
"""ETL: clean, normalize, dedup and score raw records before loading."""
+1
View File
@@ -0,0 +1 @@
"""Jobs: seed import and scheduled incremental ingestion."""
+97
View File
@@ -0,0 +1,97 @@
"""Unit normalization for OpenGoods.
Product parameters arrive in many units (g/kg/ml/L, kcal/kJ, ...). To make
values comparable and searchable we store both the original value and a
normalized value expressed in a canonical unit per dimension.
This module is intentionally dependency-free and pure so it is easy to test.
"""
from __future__ import annotations
from dataclasses import dataclass
from decimal import Decimal
# Conversion factor maps each unit to its canonical unit within a dimension.
# canonical_value = value * factor
_FACTORS: dict[str, tuple[str, str, Decimal]] = {
# mass -> g
"mg": ("mass", "g", Decimal("0.001")),
"g": ("mass", "g", Decimal("1")),
"kg": ("mass", "g", Decimal("1000")),
# volume -> ml
"ml": ("volume", "ml", Decimal("1")),
"cl": ("volume", "ml", Decimal("10")),
"l": ("volume", "ml", Decimal("1000")),
# energy -> kJ
"kj": ("energy", "kJ", Decimal("1")),
"kcal": ("energy", "kJ", Decimal("4.184")),
}
# Alias map normalizes common spellings/locales to a canonical unit code.
_ALIASES: dict[str, str] = {
"kgs": "kg",
"千克": "kg",
"公斤": "kg",
"": "g",
"毫升": "ml",
"": "l",
"L": "l",
"litre": "l",
"liter": "l",
"kj": "kj",
"kJ": "kj",
"千焦": "kj",
"千卡": "kcal",
"大卡": "kcal",
}
class UnitError(ValueError):
"""Raised when a unit cannot be recognized."""
@dataclass(frozen=True)
class Normalized:
"""Result of normalizing a (value, unit) pair to its canonical unit."""
value: Decimal
unit: str
dimension: str
canonical_value: Decimal
canonical_unit: str
def canonical_unit_code(unit: str) -> str:
"""Resolve a raw unit string to a known canonical unit code."""
cleaned = unit.strip()
cleaned = _ALIASES.get(cleaned, cleaned).lower()
if cleaned not in _FACTORS:
raise UnitError(f"unknown unit: {unit!r}")
return cleaned
def normalize(value: Decimal | float | int | str, unit: str) -> Normalized:
"""Normalize a value+unit to its canonical unit within its dimension."""
code = canonical_unit_code(unit)
dimension, canonical, factor = _FACTORS[code]
dec = value if isinstance(value, Decimal) else Decimal(str(value))
return Normalized(
value=dec,
unit=code,
dimension=dimension,
canonical_value=dec * factor,
canonical_unit=canonical,
)
def kcal_to_kj(kcal: Decimal | float | int | str) -> Decimal:
"""Convert energy in kcal to kJ (1 kcal = 4.184 kJ)."""
dec = kcal if isinstance(kcal, Decimal) else Decimal(str(kcal))
return dec * Decimal("4.184")
def kj_to_kcal(kj: Decimal | float | int | str) -> Decimal:
"""Convert energy in kJ to kcal."""
dec = kj if isinstance(kj, Decimal) else Decimal(str(kj))
return dec / Decimal("4.184")