diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4da58d3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + go: + name: Go (api) + runs-on: ubuntu-latest + defaults: + run: + working-directory: api + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.23" + cache-dependency-path: api/go.sum + - name: Verify gofmt + run: test -z "$(gofmt -l .)" + - run: go vet ./... + - run: go build ./... + - run: go test ./... + + python: + name: Python (ingestion) + runs-on: ubuntu-latest + defaults: + run: + working-directory: ingestion + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install + run: pip install -e ".[dev]" + - name: Ruff lint + run: ruff check . + - name: Ruff format check + run: ruff format --check . + - name: Pytest + run: pytest -q diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..843f600 --- /dev/null +++ b/.gitignore @@ -0,0 +1,25 @@ +# Go +/api/server +*.test +*.out + +# Python +__pycache__/ +*.py[cod] +.venv/ +.pytest_cache/ +.ruff_cache/ +*.egg-info/ +build/ +dist/ + +# Env / local +.env +.env.* +!.env.example + +# OS / editors +.DS_Store +*.swp +.idea/ +.vscode/ diff --git a/api/Dockerfile b/api/Dockerfile new file mode 100644 index 0000000..b92be17 --- /dev/null +++ b/api/Dockerfile @@ -0,0 +1,13 @@ +# Build stage +FROM golang:1.23-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -o /out/server ./cmd/server + +# Runtime stage +FROM gcr.io/distroless/static-debian12 +COPY --from=build /out/server /server +EXPOSE 8080 +ENTRYPOINT ["/server"] diff --git a/api/cmd/server/main.go b/api/cmd/server/main.go new file mode 100644 index 0000000..fe1e0c9 --- /dev/null +++ b/api/cmd/server/main.go @@ -0,0 +1,26 @@ +// Command server starts the OpenGoods public read-only API. +package main + +import ( + "log" + "net/http" + "time" + + "github.com/baicai2026-baicai/goods/api/internal/config" + "github.com/baicai2026-baicai/goods/api/internal/handler" +) + +func main() { + cfg := config.Load() + + srv := &http.Server{ + Addr: cfg.Addr, + Handler: handler.Router(), + ReadHeaderTimeout: 10 * time.Second, + } + + log.Printf("OpenGoods API listening on %s", cfg.Addr) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("server error: %v", err) + } +} diff --git a/api/go.mod b/api/go.mod new file mode 100644 index 0000000..64268e5 --- /dev/null +++ b/api/go.mod @@ -0,0 +1,5 @@ +module github.com/baicai2026-baicai/goods/api + +go 1.23.4 + +require github.com/go-chi/chi/v5 v5.1.0 diff --git a/api/go.sum b/api/go.sum new file mode 100644 index 0000000..823cdbb --- /dev/null +++ b/api/go.sum @@ -0,0 +1,2 @@ +github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= +github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= diff --git a/api/internal/config/config.go b/api/internal/config/config.go new file mode 100644 index 0000000..6bfe038 --- /dev/null +++ b/api/internal/config/config.go @@ -0,0 +1,30 @@ +package config + +import ( + "os" +) + +// Config holds runtime configuration for the OpenGoods API server. +// Values are read from environment variables with sensible defaults so the +// server can boot in a local Docker Compose setup without extra configuration. +type Config struct { + Addr string + DatabaseURL string + RedisURL string +} + +// Load reads configuration from the environment. +func Load() Config { + return Config{ + Addr: getenv("OPENGOODS_ADDR", ":8080"), + DatabaseURL: getenv("OPENGOODS_DATABASE_URL", "postgres://opengoods:opengoods@localhost:5432/opengoods?sslmode=disable"), + RedisURL: getenv("OPENGOODS_REDIS_URL", "redis://localhost:6379/0"), + } +} + +func getenv(key, fallback string) string { + if v, ok := os.LookupEnv(key); ok && v != "" { + return v + } + return fallback +} diff --git a/api/internal/handler/handler.go b/api/internal/handler/handler.go new file mode 100644 index 0000000..96b255e --- /dev/null +++ b/api/internal/handler/handler.go @@ -0,0 +1,68 @@ +// Package handler wires up the public, read-only OpenGoods HTTP API. +// +// The OpenGoods service is a public-good product information API: it only +// collects and serves product facts. It exposes no purchase, checkout, or +// commerce endpoints by design. +package handler + +import ( + "encoding/json" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" +) + +// APIVersion is the current public API version prefix. +const APIVersion = "v1" + +// Router builds the top-level HTTP handler with middleware and routes mounted. +func Router() http.Handler { + r := chi.NewRouter() + r.Use(middleware.RequestID) + r.Use(middleware.RealIP) + r.Use(middleware.Recoverer) + + r.Get("/healthz", Healthz) + + r.Route("/api/"+APIVersion, func(r chi.Router) { + r.Route("/products", func(r chi.Router) { + r.Get("/barcode/{gtin}", notImplemented) + r.Get("/search", notImplemented) + r.Get("/{id}", notImplemented) + r.Get("/{id}/nutriments", notImplemented) + r.Get("/{id}/msrp", notImplemented) + }) + r.Get("/brands", notImplemented) + r.Get("/categories", notImplemented) + r.Get("/sources/{id}", notImplemented) + }) + + return r +} + +// Healthz reports liveness of the service. +func Healthz(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} + +// notImplemented is a placeholder for endpoints scoped to later milestones. +func notImplemented(w http.ResponseWriter, r *http.Request) { + writeError(w, r, http.StatusNotImplemented, "not_implemented", "endpoint not implemented yet") +} + +func writeJSON(w http.ResponseWriter, status int, body any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(body) +} + +func writeError(w http.ResponseWriter, r *http.Request, status int, code, message string) { + writeJSON(w, status, map[string]any{ + "error": map[string]string{ + "code": code, + "message": message, + "request_id": middleware.GetReqID(r.Context()), + }, + }) +} diff --git a/api/internal/handler/handler_test.go b/api/internal/handler/handler_test.go new file mode 100644 index 0000000..82aaae6 --- /dev/null +++ b/api/internal/handler/handler_test.go @@ -0,0 +1,38 @@ +package handler + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestHealthz(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/healthz", nil) + rec := httptest.NewRecorder() + + Router().ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + + var body map[string]string + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("failed to decode body: %v", err) + } + if body["status"] != "ok" { + t.Fatalf("expected status ok, got %q", body["status"]) + } +} + +func TestProductEndpointNotImplemented(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/api/"+APIVersion+"/products/barcode/3017624010701", nil) + rec := httptest.NewRecorder() + + Router().ServeHTTP(rec, req) + + if rec.Code != http.StatusNotImplemented { + t.Fatalf("expected status %d, got %d", http.StatusNotImplemented, rec.Code) + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..21edc51 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,61 @@ +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: opengoods + POSTGRES_PASSWORD: opengoods + POSTGRES_DB: opengoods + ports: + - "5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U opengoods"] + interval: 5s + timeout: 5s + retries: 5 + + redis: + image: redis:7-alpine + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 5 + + minio: + image: minio/minio:latest + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: opengoods + MINIO_ROOT_PASSWORD: opengoods123 + ports: + - "9000:9000" + - "9001:9001" + volumes: + - miniodata:/data + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + + api: + build: ./api + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + environment: + OPENGOODS_ADDR: ":8080" + OPENGOODS_DATABASE_URL: "postgres://opengoods:opengoods@postgres:5432/opengoods?sslmode=disable" + OPENGOODS_REDIS_URL: "redis://redis:6379/0" + ports: + - "8080:8080" + +volumes: + pgdata: + miniodata: diff --git a/docs/data-contract.md b/docs/data-contract.md new file mode 100644 index 0000000..2843c11 --- /dev/null +++ b/docs/data-contract.md @@ -0,0 +1,36 @@ +# 数据契约 (Data Contract) v0.1 + +本契约是 Go(API) 与 Python(ingestion) 两端共享的"事实约定",避免两端对字段含义理解不一致。 + +> 写入责任:**仅 Python (ingestion) 通过 ETL 写入数据库**;Go (API) **只读**。所有写入必须经过单位归一化与字段级溯源。 + +## 1. 边界原则 +- 系统只采集与提供**客观商品信息**;不包含任何购买/交易语义的字段或端点。 +- 价格仅收录**官方建议零售价 (MSRP)** 的静态快照,必须带 `currency`/`region`/`source`/`effective_date`。 + +## 2. 固定枚举 +| 字段 | 取值 | +|------|------| +| `product.status` | `active` / `merged` / `deprecated` | +| `food_detail.nutrition_basis` | `per_100g` / `per_100ml` / `per_serving` | +| `unit.dimension` | `mass` / `volume` / `energy` / `count` / `ratio` / `length` / `duration` | +| `source.license` | `ODbL` / `CC0` / `proprietary` / ... | +| `product_image.kind` | `front` / `ingredients` / `nutrition` / `other` | + +## 3. 单位规则 +- 数值字段同时保存**原始值 + 单位**与**归一化值 + 基准单位**(canonical)。 +- 质量 → `g`,体积 → `ml`,能量 → `kJ`(同时保留 `kcal`)。 +- 归一化逻辑由 `ingestion/opengoods/units.py` 提供(纯函数,含测试),换算因子是唯一事实来源。 +- 营养成分统一折算到品类模板规定的基准(`per_100g` / `per_100ml`)。 + +## 4. 标识与可空性 +- `product.gtin`:8/12/13/14 位数字,可空(无条码商品),非空时全局唯一。 +- `product.quality_score` ∈ [0, 1]。 +- 货币用 ISO 4217(`CNY` 等),国家/地区用简短代码(`CN` 等)。 + +## 5. 溯源 (Provenance) +- 每条数据通过 `product_source` 记录来源、URL、贡献字段、抓取时间与原始快照。 +- 对外 API 在 `sources` 中透明返回来源与其许可。 + +## 6. 版本 +- 本契约随 schema 演进版本化;任何 schema 变更需同步更新:迁移(SQL) + 本契约 + `docs/openapi.yaml`。 diff --git a/docs/disclaimer.md b/docs/disclaimer.md new file mode 100644 index 0000000..bd6d18f --- /dev/null +++ b/docs/disclaimer.md @@ -0,0 +1,11 @@ +# 免责声明 (Disclaimer) + +天工·商品标签 (OpenGoods) 是一个**公益信息平台**。 + +- 本站**仅提供商品参数信息,不提供任何购买、下单、比价或导购服务**,不包含任何购买入口或交易链接。 +- 商品参数(成分、营养、规格等)来自多个数据来源并标注出处,可能存在误差或滞后;**请以商品实物标签为准**。 +- 价格字段仅为**官方建议零售价 (MSRP) 的历史快照**,标注来源与时间,实际售价以零售商为准,**不构成消费或购买建议**。 +- 本站不提供医疗、健康或功效宣称。 +- 数据按各来源许可使用(详见各条数据的 `sources` 字段与来源说明);权利方可通过公开渠道申请更正或下架。 + +> The OpenGoods service only collects and serves product information for public benefit. It provides **no purchase, checkout, price-comparison, or shopping-guide functionality**. diff --git a/ingestion/opengoods/__init__.py b/ingestion/opengoods/__init__.py new file mode 100644 index 0000000..5ab8be6 --- /dev/null +++ b/ingestion/opengoods/__init__.py @@ -0,0 +1,9 @@ +"""OpenGoods (天工·商品标签) ingestion package. + +Collects public product information from open data sources (e.g. Open Food +Facts) and normalizes it into the OpenGoods database. This package only +collects and processes product facts; it performs no purchase or commerce +actions. +""" + +__version__ = "0.1.0" diff --git a/ingestion/opengoods/adapters/__init__.py b/ingestion/opengoods/adapters/__init__.py new file mode 100644 index 0000000..0bca320 --- /dev/null +++ b/ingestion/opengoods/adapters/__init__.py @@ -0,0 +1,6 @@ +"""Source adapters. + +Each open data source (Open Food Facts, USDA FoodData Central, GS1, ...) gets +its own adapter that fetches raw records and yields them for the ETL layer. +Adapters must respect each source's robots.txt, rate limits and license. +""" diff --git a/ingestion/opengoods/adapters/base.py b/ingestion/opengoods/adapters/base.py new file mode 100644 index 0000000..ac222b0 --- /dev/null +++ b/ingestion/opengoods/adapters/base.py @@ -0,0 +1,17 @@ +"""Base adapter protocol shared by all source adapters.""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Protocol + + +class SourceAdapter(Protocol): + """A source adapter fetches raw product records from one data source.""" + + #: Stable identifier of the source, e.g. "openfoodfacts". + source_name: str + + def fetch(self) -> Iterator[dict]: + """Yield raw product records as dictionaries.""" + ... diff --git a/ingestion/opengoods/etl/__init__.py b/ingestion/opengoods/etl/__init__.py new file mode 100644 index 0000000..56babb8 --- /dev/null +++ b/ingestion/opengoods/etl/__init__.py @@ -0,0 +1 @@ +"""ETL: clean, normalize, dedup and score raw records before loading.""" diff --git a/ingestion/opengoods/jobs/__init__.py b/ingestion/opengoods/jobs/__init__.py new file mode 100644 index 0000000..11886e5 --- /dev/null +++ b/ingestion/opengoods/jobs/__init__.py @@ -0,0 +1 @@ +"""Jobs: seed import and scheduled incremental ingestion.""" diff --git a/ingestion/opengoods/units.py b/ingestion/opengoods/units.py new file mode 100644 index 0000000..18f3591 --- /dev/null +++ b/ingestion/opengoods/units.py @@ -0,0 +1,97 @@ +"""Unit normalization for OpenGoods. + +Product parameters arrive in many units (g/kg/ml/L, kcal/kJ, ...). To make +values comparable and searchable we store both the original value and a +normalized value expressed in a canonical unit per dimension. + +This module is intentionally dependency-free and pure so it is easy to test. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal + +# Conversion factor maps each unit to its canonical unit within a dimension. +# canonical_value = value * factor +_FACTORS: dict[str, tuple[str, str, Decimal]] = { + # mass -> g + "mg": ("mass", "g", Decimal("0.001")), + "g": ("mass", "g", Decimal("1")), + "kg": ("mass", "g", Decimal("1000")), + # volume -> ml + "ml": ("volume", "ml", Decimal("1")), + "cl": ("volume", "ml", Decimal("10")), + "l": ("volume", "ml", Decimal("1000")), + # energy -> kJ + "kj": ("energy", "kJ", Decimal("1")), + "kcal": ("energy", "kJ", Decimal("4.184")), +} + +# Alias map normalizes common spellings/locales to a canonical unit code. +_ALIASES: dict[str, str] = { + "kgs": "kg", + "千克": "kg", + "公斤": "kg", + "克": "g", + "毫升": "ml", + "升": "l", + "L": "l", + "litre": "l", + "liter": "l", + "kj": "kj", + "kJ": "kj", + "千焦": "kj", + "千卡": "kcal", + "大卡": "kcal", +} + + +class UnitError(ValueError): + """Raised when a unit cannot be recognized.""" + + +@dataclass(frozen=True) +class Normalized: + """Result of normalizing a (value, unit) pair to its canonical unit.""" + + value: Decimal + unit: str + dimension: str + canonical_value: Decimal + canonical_unit: str + + +def canonical_unit_code(unit: str) -> str: + """Resolve a raw unit string to a known canonical unit code.""" + cleaned = unit.strip() + cleaned = _ALIASES.get(cleaned, cleaned).lower() + if cleaned not in _FACTORS: + raise UnitError(f"unknown unit: {unit!r}") + return cleaned + + +def normalize(value: Decimal | float | int | str, unit: str) -> Normalized: + """Normalize a value+unit to its canonical unit within its dimension.""" + code = canonical_unit_code(unit) + dimension, canonical, factor = _FACTORS[code] + dec = value if isinstance(value, Decimal) else Decimal(str(value)) + return Normalized( + value=dec, + unit=code, + dimension=dimension, + canonical_value=dec * factor, + canonical_unit=canonical, + ) + + +def kcal_to_kj(kcal: Decimal | float | int | str) -> Decimal: + """Convert energy in kcal to kJ (1 kcal = 4.184 kJ).""" + dec = kcal if isinstance(kcal, Decimal) else Decimal(str(kcal)) + return dec * Decimal("4.184") + + +def kj_to_kcal(kj: Decimal | float | int | str) -> Decimal: + """Convert energy in kJ to kcal.""" + dec = kj if isinstance(kj, Decimal) else Decimal(str(kj)) + return dec / Decimal("4.184") diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml new file mode 100644 index 0000000..8be7953 --- /dev/null +++ b/ingestion/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "opengoods-ingestion" +version = "0.1.0" +description = "OpenGoods (天工·商品标签) ingestion & ETL: collect product data and load it into the OpenGoods database." +requires-python = ">=3.11" +dependencies = [ + "httpx>=0.27", +] + +[project.optional-dependencies] +dev = [ + "ruff>=0.6", + "pytest>=8.0", +] + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["opengoods*"] + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/ingestion/tests/__init__.py b/ingestion/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ingestion/tests/test_units.py b/ingestion/tests/test_units.py new file mode 100644 index 0000000..cbaa012 --- /dev/null +++ b/ingestion/tests/test_units.py @@ -0,0 +1,47 @@ +from decimal import Decimal + +import pytest + +from opengoods.units import ( + UnitError, + canonical_unit_code, + kcal_to_kj, + kj_to_kcal, + normalize, +) + + +def test_normalize_mass_kg_to_g(): + result = normalize("1.5", "kg") + assert result.dimension == "mass" + assert result.canonical_unit == "g" + assert result.canonical_value == Decimal("1500.0") + + +def test_normalize_volume_litre_alias(): + result = normalize(2, "升") + assert result.dimension == "volume" + assert result.canonical_value == Decimal("2000") + assert result.canonical_unit == "ml" + + +def test_normalize_energy_kcal_to_kj(): + result = normalize("539", "kcal") + assert result.dimension == "energy" + assert result.canonical_unit == "kJ" + assert result.canonical_value == Decimal("539") * Decimal("4.184") + + +def test_canonical_unit_code_alias(): + assert canonical_unit_code("公斤") == "kg" + assert canonical_unit_code(" G ") == "g" + + +def test_unknown_unit_raises(): + with pytest.raises(UnitError): + normalize(1, "parsec") + + +def test_energy_roundtrip(): + assert kcal_to_kj(1) == Decimal("4.184") + assert kj_to_kcal(Decimal("4.184")) == Decimal("1") diff --git a/migrations/README.md b/migrations/README.md new file mode 100644 index 0000000..bf58d4e --- /dev/null +++ b/migrations/README.md @@ -0,0 +1,3 @@ +# Database migrations (golang-migrate) + +SQL migrations live here from milestone M1. Format: `NNNN_description.up.sql` / `.down.sql`.