Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 lixu
|
b0b816b0ee | ||
|
lixu
|
786b7d3721 |
@@ -0,0 +1,74 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
go:
|
||||
name: Go (api)
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: api
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.23"
|
||||
cache-dependency-path: api/go.sum
|
||||
- name: Verify gofmt
|
||||
run: test -z "$(gofmt -l .)"
|
||||
- run: go vet ./...
|
||||
- run: go build ./...
|
||||
- run: go test ./...
|
||||
|
||||
python:
|
||||
name: Python (ingestion)
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ingestion
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Install
|
||||
run: pip install -e ".[dev]"
|
||||
- name: Ruff lint
|
||||
run: ruff check .
|
||||
- name: Ruff format check
|
||||
run: ruff format --check .
|
||||
- name: Pytest
|
||||
run: pytest -q
|
||||
|
||||
migrations:
|
||||
name: Migrations (postgres)
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
POSTGRES_USER: opengoods
|
||||
POSTGRES_PASSWORD: opengoods
|
||||
POSTGRES_DB: opengoods
|
||||
ports:
|
||||
- "5432:5432"
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U opengoods"
|
||||
--health-interval 5s --health-timeout 5s --health-retries 10
|
||||
env:
|
||||
DBURL: postgres://opengoods:opengoods@localhost:5432/opengoods?sslmode=disable
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.23"
|
||||
- name: Install golang-migrate
|
||||
run: go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.18.1
|
||||
- name: Migrate up
|
||||
run: migrate -path migrations -database "$DBURL" up
|
||||
- name: Migrate down (reversibility)
|
||||
run: migrate -path migrations -database "$DBURL" down -all
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
# Go
|
||||
/api/server
|
||||
*.test
|
||||
*.out
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
.venv/
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
*.egg-info/
|
||||
build/
|
||||
dist/
|
||||
|
||||
# Env / local
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# OS / editors
|
||||
.DS_Store
|
||||
*.swp
|
||||
.idea/
|
||||
.vscode/
|
||||
@@ -0,0 +1,13 @@
|
||||
# Build stage
|
||||
FROM golang:1.23-alpine AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 go build -o /out/server ./cmd/server
|
||||
|
||||
# Runtime stage
|
||||
FROM gcr.io/distroless/static-debian12
|
||||
COPY --from=build /out/server /server
|
||||
EXPOSE 8080
|
||||
ENTRYPOINT ["/server"]
|
||||
@@ -0,0 +1,26 @@
|
||||
// Command server starts the OpenGoods public read-only API.
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/baicai2026-baicai/goods/api/internal/config"
|
||||
"github.com/baicai2026-baicai/goods/api/internal/handler"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := config.Load()
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: cfg.Addr,
|
||||
Handler: handler.Router(),
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
log.Printf("OpenGoods API listening on %s", cfg.Addr)
|
||||
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
log.Fatalf("server error: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
module github.com/baicai2026-baicai/goods/api
|
||||
|
||||
go 1.23.4
|
||||
|
||||
require github.com/go-chi/chi/v5 v5.1.0
|
||||
@@ -0,0 +1,2 @@
|
||||
github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
|
||||
github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
||||
@@ -0,0 +1,30 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// Config holds runtime configuration for the OpenGoods API server.
|
||||
// Values are read from environment variables with sensible defaults so the
|
||||
// server can boot in a local Docker Compose setup without extra configuration.
|
||||
type Config struct {
|
||||
Addr string
|
||||
DatabaseURL string
|
||||
RedisURL string
|
||||
}
|
||||
|
||||
// Load reads configuration from the environment.
|
||||
func Load() Config {
|
||||
return Config{
|
||||
Addr: getenv("OPENGOODS_ADDR", ":8080"),
|
||||
DatabaseURL: getenv("OPENGOODS_DATABASE_URL", "postgres://opengoods:opengoods@localhost:5432/opengoods?sslmode=disable"),
|
||||
RedisURL: getenv("OPENGOODS_REDIS_URL", "redis://localhost:6379/0"),
|
||||
}
|
||||
}
|
||||
|
||||
func getenv(key, fallback string) string {
|
||||
if v, ok := os.LookupEnv(key); ok && v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// Package handler wires up the public, read-only OpenGoods HTTP API.
|
||||
//
|
||||
// The OpenGoods service is a public-good product information API: it only
|
||||
// collects and serves product facts. It exposes no purchase, checkout, or
|
||||
// commerce endpoints by design.
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
)
|
||||
|
||||
// APIVersion is the current public API version prefix.
|
||||
const APIVersion = "v1"
|
||||
|
||||
// Router builds the top-level HTTP handler with middleware and routes mounted.
|
||||
func Router() http.Handler {
|
||||
r := chi.NewRouter()
|
||||
r.Use(middleware.RequestID)
|
||||
r.Use(middleware.RealIP)
|
||||
r.Use(middleware.Recoverer)
|
||||
|
||||
r.Get("/healthz", Healthz)
|
||||
|
||||
r.Route("/api/"+APIVersion, func(r chi.Router) {
|
||||
r.Route("/products", func(r chi.Router) {
|
||||
r.Get("/barcode/{gtin}", notImplemented)
|
||||
r.Get("/search", notImplemented)
|
||||
r.Get("/{id}", notImplemented)
|
||||
r.Get("/{id}/nutriments", notImplemented)
|
||||
r.Get("/{id}/msrp", notImplemented)
|
||||
})
|
||||
r.Get("/brands", notImplemented)
|
||||
r.Get("/categories", notImplemented)
|
||||
r.Get("/sources/{id}", notImplemented)
|
||||
})
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
// Healthz reports liveness of the service.
|
||||
func Healthz(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
// notImplemented is a placeholder for endpoints scoped to later milestones.
|
||||
func notImplemented(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, r, http.StatusNotImplemented, "not_implemented", "endpoint not implemented yet")
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, body any) {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(body)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, r *http.Request, status int, code, message string) {
|
||||
writeJSON(w, status, map[string]any{
|
||||
"error": map[string]string{
|
||||
"code": code,
|
||||
"message": message,
|
||||
"request_id": middleware.GetReqID(r.Context()),
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHealthz(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
Router().ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code)
|
||||
}
|
||||
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(rec.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("failed to decode body: %v", err)
|
||||
}
|
||||
if body["status"] != "ok" {
|
||||
t.Fatalf("expected status ok, got %q", body["status"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductEndpointNotImplemented(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/"+APIVersion+"/products/barcode/3017624010701", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
Router().ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNotImplemented {
|
||||
t.Fatalf("expected status %d, got %d", http.StatusNotImplemented, rec.Code)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
environment:
|
||||
POSTGRES_USER: opengoods
|
||||
POSTGRES_PASSWORD: opengoods
|
||||
POSTGRES_DB: opengoods
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U opengoods"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- "6379:6379"
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: opengoods
|
||||
MINIO_ROOT_PASSWORD: opengoods123
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
volumes:
|
||||
- miniodata:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
api:
|
||||
build: ./api
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
OPENGOODS_ADDR: ":8080"
|
||||
OPENGOODS_DATABASE_URL: "postgres://opengoods:opengoods@postgres:5432/opengoods?sslmode=disable"
|
||||
OPENGOODS_REDIS_URL: "redis://redis:6379/0"
|
||||
ports:
|
||||
- "8080:8080"
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
miniodata:
|
||||
@@ -0,0 +1,36 @@
|
||||
# 数据契约 (Data Contract) v0.1
|
||||
|
||||
本契约是 Go(API) 与 Python(ingestion) 两端共享的"事实约定"，避免两端对字段含义理解不一致。
|
||||
|
||||
> 写入责任：**仅 Python (ingestion) 通过 ETL 写入数据库**；Go (API) **只读**。所有写入必须经过单位归一化与字段级溯源。
|
||||
|
||||
## 1. 边界原则
|
||||
- 系统只采集与提供**客观商品信息**；不包含任何购买/交易语义的字段或端点。
|
||||
- 价格仅收录**官方建议零售价 (MSRP)** 的静态快照，必须带 `currency`/`region`/`source`/`effective_date`。
|
||||
|
||||
## 2. 固定枚举
|
||||
| 字段 | 取值 |
|
||||
|------|------|
|
||||
| `product.status` | `active` / `merged` / `deprecated` |
|
||||
| `food_detail.nutrition_basis` | `per_100g` / `per_100ml` / `per_serving` |
|
||||
| `unit.dimension` | `mass` / `volume` / `energy` / `count` / `ratio` / `length` / `duration` |
|
||||
| `source.license` | `ODbL` / `CC0` / `proprietary` / ... |
|
||||
| `product_image.kind` | `front` / `ingredients` / `nutrition` / `other` |
|
||||
|
||||
## 3. 单位规则
|
||||
- 数值字段同时保存**原始值 + 单位**与**归一化值 + 基准单位**（canonical）。
|
||||
- 质量 → `g`，体积 → `ml`，能量 → `kJ`（同时保留 `kcal`）。
|
||||
- 归一化逻辑由 `ingestion/opengoods/units.py` 提供（纯函数，含测试），换算因子是唯一事实来源。
|
||||
- 营养成分统一折算到品类模板规定的基准（`per_100g` / `per_100ml`）。
|
||||
|
||||
## 4. 标识与可空性
|
||||
- `product.gtin`：8/12/13/14 位数字，可空（无条码商品），非空时全局唯一。
|
||||
- `product.quality_score` ∈ [0, 1]。
|
||||
- 货币用 ISO 4217（`CNY` 等），国家/地区用简短代码（`CN` 等）。
|
||||
|
||||
## 5. 溯源 (Provenance)
|
||||
- 每条数据通过 `product_source` 记录来源、URL、贡献字段、抓取时间与原始快照。
|
||||
- 对外 API 在 `sources` 中透明返回来源与其许可。
|
||||
|
||||
## 6. 版本
|
||||
- 本契约随 schema 演进版本化；任何 schema 变更需同步更新：迁移(SQL) + 本契约 + `docs/openapi.yaml`。
|
||||
@@ -0,0 +1,11 @@
|
||||
# 免责声明 (Disclaimer)
|
||||
|
||||
天工·商品标签 (OpenGoods) 是一个**公益信息平台**。
|
||||
|
||||
- 本站**仅提供商品参数信息，不提供任何购买、下单、比价或导购服务**，不包含任何购买入口或交易链接。
|
||||
- 商品参数（成分、营养、规格等）来自多个数据来源并标注出处，可能存在误差或滞后；**请以商品实物标签为准**。
|
||||
- 价格字段仅为**官方建议零售价 (MSRP) 的历史快照**，标注来源与时间，实际售价以零售商为准，**不构成消费或购买建议**。
|
||||
- 本站不提供医疗、健康或功效宣称。
|
||||
- 数据按各来源许可使用（详见各条数据的 `sources` 字段与来源说明）；权利方可通过公开渠道申请更正或下架。
|
||||
|
||||
> The OpenGoods service only collects and serves product information for public benefit. It provides **no purchase, checkout, price-comparison, or shopping-guide functionality**.
|
||||
@@ -0,0 +1,9 @@
|
||||
"""OpenGoods (天工·商品标签) ingestion package.
|
||||
|
||||
Collects public product information from open data sources (e.g. Open Food
|
||||
Facts) and normalizes it into the OpenGoods database. This package only
|
||||
collects and processes product facts; it performs no purchase or commerce
|
||||
actions.
|
||||
"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Source adapters.
|
||||
|
||||
Each open data source (Open Food Facts, USDA FoodData Central, GS1, ...) gets
|
||||
its own adapter that fetches raw records and yields them for the ETL layer.
|
||||
Adapters must respect each source's robots.txt, rate limits and license.
|
||||
"""
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Base adapter protocol shared by all source adapters."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
class SourceAdapter(Protocol):
|
||||
"""A source adapter fetches raw product records from one data source."""
|
||||
|
||||
#: Stable identifier of the source, e.g. "openfoodfacts".
|
||||
source_name: str
|
||||
|
||||
def fetch(self) -> Iterator[dict]:
|
||||
"""Yield raw product records as dictionaries."""
|
||||
...
|
||||
@@ -0,0 +1 @@
|
||||
"""ETL: clean, normalize, dedup and score raw records before loading."""
|
||||
@@ -0,0 +1 @@
|
||||
"""Jobs: seed import and scheduled incremental ingestion."""
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Unit normalization for OpenGoods.
|
||||
|
||||
Product parameters arrive in many units (g/kg/ml/L, kcal/kJ, ...). To make
|
||||
values comparable and searchable we store both the original value and a
|
||||
normalized value expressed in a canonical unit per dimension.
|
||||
|
||||
This module is intentionally dependency-free and pure so it is easy to test.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from decimal import Decimal
|
||||
|
||||
# Conversion factor maps each unit to its canonical unit within a dimension.
|
||||
# canonical_value = value * factor
|
||||
_FACTORS: dict[str, tuple[str, str, Decimal]] = {
|
||||
# mass -> g
|
||||
"mg": ("mass", "g", Decimal("0.001")),
|
||||
"g": ("mass", "g", Decimal("1")),
|
||||
"kg": ("mass", "g", Decimal("1000")),
|
||||
# volume -> ml
|
||||
"ml": ("volume", "ml", Decimal("1")),
|
||||
"cl": ("volume", "ml", Decimal("10")),
|
||||
"l": ("volume", "ml", Decimal("1000")),
|
||||
# energy -> kJ
|
||||
"kj": ("energy", "kJ", Decimal("1")),
|
||||
"kcal": ("energy", "kJ", Decimal("4.184")),
|
||||
}
|
||||
|
||||
# Alias map normalizes common spellings/locales to a canonical unit code.
|
||||
_ALIASES: dict[str, str] = {
|
||||
"kgs": "kg",
|
||||
"千克": "kg",
|
||||
"公斤": "kg",
|
||||
"克": "g",
|
||||
"毫升": "ml",
|
||||
"升": "l",
|
||||
"L": "l",
|
||||
"litre": "l",
|
||||
"liter": "l",
|
||||
"kj": "kj",
|
||||
"kJ": "kj",
|
||||
"千焦": "kj",
|
||||
"千卡": "kcal",
|
||||
"大卡": "kcal",
|
||||
}
|
||||
|
||||
|
||||
class UnitError(ValueError):
|
||||
"""Raised when a unit cannot be recognized."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Normalized:
|
||||
"""Result of normalizing a (value, unit) pair to its canonical unit."""
|
||||
|
||||
value: Decimal
|
||||
unit: str
|
||||
dimension: str
|
||||
canonical_value: Decimal
|
||||
canonical_unit: str
|
||||
|
||||
|
||||
def canonical_unit_code(unit: str) -> str:
|
||||
"""Resolve a raw unit string to a known canonical unit code."""
|
||||
cleaned = unit.strip()
|
||||
cleaned = _ALIASES.get(cleaned, cleaned).lower()
|
||||
if cleaned not in _FACTORS:
|
||||
raise UnitError(f"unknown unit: {unit!r}")
|
||||
return cleaned
|
||||
|
||||
|
||||
def normalize(value: Decimal | float | int | str, unit: str) -> Normalized:
|
||||
"""Normalize a value+unit to its canonical unit within its dimension."""
|
||||
code = canonical_unit_code(unit)
|
||||
dimension, canonical, factor = _FACTORS[code]
|
||||
dec = value if isinstance(value, Decimal) else Decimal(str(value))
|
||||
return Normalized(
|
||||
value=dec,
|
||||
unit=code,
|
||||
dimension=dimension,
|
||||
canonical_value=dec * factor,
|
||||
canonical_unit=canonical,
|
||||
)
|
||||
|
||||
|
||||
def kcal_to_kj(kcal: Decimal | float | int | str) -> Decimal:
|
||||
"""Convert energy in kcal to kJ (1 kcal = 4.184 kJ)."""
|
||||
dec = kcal if isinstance(kcal, Decimal) else Decimal(str(kcal))
|
||||
return dec * Decimal("4.184")
|
||||
|
||||
|
||||
def kj_to_kcal(kj: Decimal | float | int | str) -> Decimal:
|
||||
"""Convert energy in kJ to kcal."""
|
||||
dec = kj if isinstance(kj, Decimal) else Decimal(str(kj))
|
||||
return dec / Decimal("4.184")
|
||||
@@ -0,0 +1,31 @@
|
||||
[project]
|
||||
name = "opengoods-ingestion"
|
||||
version = "0.1.0"
|
||||
description = "OpenGoods (天工·商品标签) ingestion & ETL: collect product data and load it into the OpenGoods database."
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"httpx>=0.27",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"ruff>=0.6",
|
||||
"pytest>=8.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=68"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
include = ["opengoods*"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py311"
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "UP", "B"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
@@ -0,0 +1,47 @@
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
|
||||
from opengoods.units import (
|
||||
UnitError,
|
||||
canonical_unit_code,
|
||||
kcal_to_kj,
|
||||
kj_to_kcal,
|
||||
normalize,
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_mass_kg_to_g():
|
||||
result = normalize("1.5", "kg")
|
||||
assert result.dimension == "mass"
|
||||
assert result.canonical_unit == "g"
|
||||
assert result.canonical_value == Decimal("1500.0")
|
||||
|
||||
|
||||
def test_normalize_volume_litre_alias():
|
||||
result = normalize(2, "升")
|
||||
assert result.dimension == "volume"
|
||||
assert result.canonical_value == Decimal("2000")
|
||||
assert result.canonical_unit == "ml"
|
||||
|
||||
|
||||
def test_normalize_energy_kcal_to_kj():
|
||||
result = normalize("539", "kcal")
|
||||
assert result.dimension == "energy"
|
||||
assert result.canonical_unit == "kJ"
|
||||
assert result.canonical_value == Decimal("539") * Decimal("4.184")
|
||||
|
||||
|
||||
def test_canonical_unit_code_alias():
|
||||
assert canonical_unit_code("公斤") == "kg"
|
||||
assert canonical_unit_code(" G ") == "g"
|
||||
|
||||
|
||||
def test_unknown_unit_raises():
|
||||
with pytest.raises(UnitError):
|
||||
normalize(1, "parsec")
|
||||
|
||||
|
||||
def test_energy_roundtrip():
|
||||
assert kcal_to_kj(1) == Decimal("4.184")
|
||||
assert kj_to_kcal(Decimal("4.184")) == Decimal("1")
|
||||
@@ -0,0 +1,21 @@
|
||||
DROP TRIGGER IF EXISTS trg_product_sync ON product;
|
||||
DROP FUNCTION IF EXISTS product_sync_tsv();
|
||||
|
||||
DROP TABLE IF EXISTS merge_log;
|
||||
DROP TABLE IF EXISTS product_source;
|
||||
DROP TABLE IF EXISTS product_image;
|
||||
DROP TABLE IF EXISTS product_msrp;
|
||||
DROP TABLE IF EXISTS food_detail;
|
||||
DROP TABLE IF EXISTS product;
|
||||
DROP TABLE IF EXISTS attribute_definition;
|
||||
DROP TABLE IF EXISTS unit;
|
||||
DROP TABLE IF EXISTS category_schema;
|
||||
DROP TABLE IF EXISTS category;
|
||||
DROP TABLE IF EXISTS manufacturer;
|
||||
DROP TABLE IF EXISTS brand;
|
||||
DROP TABLE IF EXISTS source;
|
||||
|
||||
DROP EXTENSION IF EXISTS ltree;
|
||||
DROP EXTENSION IF EXISTS pg_trgm;
|
||||
-- keep pgcrypto (commonly shared); drop only if you are sure:
|
||||
-- DROP EXTENSION IF EXISTS pgcrypto;
|
||||
@@ -0,0 +1,182 @@
|
||||
-- OpenGoods (天工·商品标签) initial schema.
|
||||
-- Public-good product information store: facts only, no commerce.
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS pgcrypto; -- gen_random_uuid()
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm; -- fuzzy name search
|
||||
CREATE EXTENSION IF NOT EXISTS ltree; -- category subtree queries
|
||||
|
||||
-- Data sources (Open Food Facts / USDA / GS1 ...) with trust + license.
|
||||
CREATE TABLE source (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name TEXT NOT NULL,
|
||||
homepage TEXT,
|
||||
license TEXT,
|
||||
trust_weight NUMERIC(3,2) NOT NULL DEFAULT 0.5,
|
||||
notes TEXT,
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE brand (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name TEXT NOT NULL,
|
||||
normalized_name TEXT NOT NULL,
|
||||
aliases TEXT[] NOT NULL DEFAULT '{}',
|
||||
UNIQUE (normalized_name)
|
||||
);
|
||||
|
||||
CREATE TABLE manufacturer (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name TEXT NOT NULL,
|
||||
normalized_name TEXT NOT NULL,
|
||||
country VARCHAR(64),
|
||||
UNIQUE (normalized_name)
|
||||
);
|
||||
|
||||
-- Self-built category tree, each node optionally mapped to a GS1 GPC brick.
|
||||
CREATE TABLE category (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name_zh TEXT NOT NULL,
|
||||
name_en TEXT,
|
||||
parent_id UUID REFERENCES category(id),
|
||||
path LTREE NOT NULL,
|
||||
gpc_brick_code VARCHAR(10),
|
||||
level INT NOT NULL DEFAULT 0,
|
||||
UNIQUE (path)
|
||||
);
|
||||
|
||||
-- Parameter template / constraints per category.
|
||||
CREATE TABLE category_schema (
|
||||
category_id UUID PRIMARY KEY REFERENCES category(id) ON DELETE CASCADE,
|
||||
required_attributes TEXT[] NOT NULL DEFAULT '{}',
|
||||
recommended_attributes TEXT[] NOT NULL DEFAULT '{}',
|
||||
nutriment_basis VARCHAR(16)
|
||||
);
|
||||
|
||||
-- Unit dictionary: each unit maps to a canonical unit within its dimension.
|
||||
CREATE TABLE unit (
|
||||
code VARCHAR(16) PRIMARY KEY,
|
||||
dimension VARCHAR(16) NOT NULL,
|
||||
canonical VARCHAR(16) NOT NULL,
|
||||
to_canonical_factor NUMERIC,
|
||||
aliases TEXT[] NOT NULL DEFAULT '{}',
|
||||
display TEXT
|
||||
);
|
||||
|
||||
-- Parameter dictionary: standard attribute keys with default unit.
|
||||
CREATE TABLE attribute_definition (
|
||||
key VARCHAR(64) PRIMARY KEY,
|
||||
label_zh TEXT,
|
||||
label_en TEXT,
|
||||
dimension VARCHAR(16),
|
||||
default_unit VARCHAR(16) REFERENCES unit(code),
|
||||
aliases TEXT[] NOT NULL DEFAULT '{}'
|
||||
);
|
||||
|
||||
CREATE TABLE product (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
gtin VARCHAR(14),
|
||||
name TEXT NOT NULL,
|
||||
brand_id UUID REFERENCES brand(id),
|
||||
manufacturer_id UUID REFERENCES manufacturer(id),
|
||||
category_id UUID REFERENCES category(id),
|
||||
gpc_brick_code VARCHAR(10),
|
||||
net_content_value NUMERIC,
|
||||
net_content_unit VARCHAR(16),
|
||||
net_content_canonical NUMERIC,
|
||||
country_of_origin VARCHAR(64),
|
||||
shelf_life_days INT,
|
||||
storage TEXT,
|
||||
attributes JSONB NOT NULL DEFAULT '{}',
|
||||
quality_score NUMERIC(4,3) NOT NULL DEFAULT 0,
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'active',
|
||||
canonical_id UUID REFERENCES product(id),
|
||||
search_tsv TSVECTOR,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT product_status_chk CHECK (status IN ('active','merged','deprecated')),
|
||||
CONSTRAINT product_quality_chk CHECK (quality_score >= 0 AND quality_score <= 1)
|
||||
);
|
||||
|
||||
CREATE TABLE food_detail (
|
||||
product_id UUID PRIMARY KEY REFERENCES product(id) ON DELETE CASCADE,
|
||||
ingredients_text TEXT,
|
||||
ingredients JSONB,
|
||||
allergens TEXT[] NOT NULL DEFAULT '{}',
|
||||
additives TEXT[] NOT NULL DEFAULT '{}',
|
||||
nutriments JSONB,
|
||||
nutrition_basis VARCHAR(16),
|
||||
serving_size VARCHAR(32),
|
||||
nutri_score CHAR(1),
|
||||
labels TEXT[] NOT NULL DEFAULT '{}',
|
||||
CONSTRAINT food_basis_chk CHECK (nutrition_basis IS NULL OR nutrition_basis IN ('per_100g','per_100ml','per_serving'))
|
||||
);
|
||||
|
||||
-- Official manufacturer-suggested retail price snapshot (no purchase link).
|
||||
CREATE TABLE product_msrp (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
product_id UUID NOT NULL REFERENCES product(id) ON DELETE CASCADE,
|
||||
amount NUMERIC(12,2) NOT NULL,
|
||||
currency CHAR(3) NOT NULL,
|
||||
region VARCHAR(8) NOT NULL DEFAULT 'CN',
|
||||
source_id UUID REFERENCES source(id),
|
||||
source_url TEXT,
|
||||
effective_date DATE,
|
||||
note TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE product_image (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
product_id UUID NOT NULL REFERENCES product(id) ON DELETE CASCADE,
|
||||
url TEXT NOT NULL,
|
||||
kind VARCHAR(16) NOT NULL DEFAULT 'other',
|
||||
license TEXT,
|
||||
source_id UUID REFERENCES source(id),
|
||||
CONSTRAINT image_kind_chk CHECK (kind IN ('front','ingredients','nutrition','other'))
|
||||
);
|
||||
|
||||
-- Field-level provenance: which source provided which fields.
|
||||
CREATE TABLE product_source (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
product_id UUID NOT NULL REFERENCES product(id) ON DELETE CASCADE,
|
||||
source_id UUID REFERENCES source(id),
|
||||
url TEXT,
|
||||
fields TEXT[] NOT NULL DEFAULT '{}',
|
||||
fetched_at TIMESTAMPTZ,
|
||||
raw JSONB
|
||||
);
|
||||
|
||||
CREATE TABLE merge_log (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
kept_id UUID,
|
||||
merged_id UUID,
|
||||
reason TEXT,
|
||||
actor TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Indexes
|
||||
CREATE UNIQUE INDEX idx_product_gtin ON product (gtin) WHERE gtin IS NOT NULL;
|
||||
CREATE INDEX idx_product_name_trgm ON product USING gin (name gin_trgm_ops);
|
||||
CREATE INDEX idx_product_attrs ON product USING gin (attributes);
|
||||
CREATE INDEX idx_product_tsv ON product USING gin (search_tsv);
|
||||
CREATE INDEX idx_product_category ON product (category_id);
|
||||
CREATE INDEX idx_product_brand ON product (brand_id);
|
||||
CREATE INDEX idx_product_updated ON product (updated_at);
|
||||
CREATE INDEX idx_food_nutriments ON food_detail USING gin (nutriments);
|
||||
CREATE INDEX idx_category_path ON category USING gist (path);
|
||||
CREATE INDEX idx_msrp_product ON product_msrp (product_id);
|
||||
CREATE INDEX idx_psource_product ON product_source (product_id);
|
||||
|
||||
-- Keep search_tsv and updated_at in sync.
|
||||
CREATE OR REPLACE FUNCTION product_sync_tsv() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.search_tsv := to_tsvector('simple', coalesce(NEW.name, ''));
|
||||
NEW.updated_at := now();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER trg_product_sync
|
||||
BEFORE INSERT OR UPDATE ON product
|
||||
FOR EACH ROW EXECUTE FUNCTION product_sync_tsv();
|
||||
@@ -0,0 +1,2 @@
|
||||
DELETE FROM attribute_definition;
|
||||
DELETE FROM unit;
|
||||
@@ -0,0 +1,29 @@
|
||||
-- Unit dictionary seed. Keep factors aligned with ingestion/opengoods/units.py.
|
||||
|
||||
INSERT INTO unit (code, dimension, canonical, to_canonical_factor, aliases, display) VALUES
|
||||
('mg', 'mass', 'g', 0.001, ARRAY['毫克'], 'mg'),
|
||||
('g', 'mass', 'g', 1, ARRAY['克','gram','grams'], 'g'),
|
||||
('kg', 'mass', 'g', 1000, ARRAY['kgs','千克','公斤'], 'kg'),
|
||||
('ml', 'volume', 'ml', 1, ARRAY['毫升','milliliter'], 'mL'),
|
||||
('cl', 'volume', 'ml', 10, ARRAY['厘升'], 'cL'),
|
||||
('l', 'volume', 'ml', 1000, ARRAY['L','升','litre','liter'], 'L'),
|
||||
('kj', 'energy', 'kJ', 1, ARRAY['kJ','千焦'], 'kJ'),
|
||||
('kcal', 'energy', 'kJ', 4.184, ARRAY['千卡','大卡'], 'kcal'),
|
||||
('pct', 'ratio', 'pct', 1, ARRAY['%','percent','百分比'], '%'),
|
||||
('unit', 'count', 'unit',1, ARRAY['个','件','pcs','piece'], '个'),
|
||||
('mm', 'length', 'mm', 1, ARRAY['毫米'], 'mm'),
|
||||
('cm', 'length', 'mm', 10, ARRAY['厘米'], 'cm'),
|
||||
('day', 'duration', 'day', 1, ARRAY['天','日','days'], 'day')
|
||||
ON CONFLICT (code) DO NOTHING;
|
||||
|
||||
-- A few common food attribute definitions referencing the unit dictionary.
|
||||
INSERT INTO attribute_definition (key, label_zh, label_en, dimension, default_unit, aliases) VALUES
|
||||
('energy', '能量', 'Energy', 'energy', 'kj', ARRAY['energy_kj']),
|
||||
('proteins', '蛋白质', 'Proteins', 'mass', 'g', ARRAY['protein']),
|
||||
('fat', '脂肪', 'Fat', 'mass', 'g', ARRAY['fats']),
|
||||
('saturated_fat', '饱和脂肪','Saturated fat','mass', 'g', ARRAY['saturated-fat']),
|
||||
('carbohydrates', '碳水化合物','Carbohydrates','mass', 'g', ARRAY['carbs']),
|
||||
('sugars', '糖', 'Sugars', 'mass', 'g', ARRAY['sugar']),
|
||||
('salt', '盐', 'Salt', 'mass', 'g', ARRAY['sodium_salt']),
|
||||
('net_content', '净含量', 'Net content', NULL, NULL, ARRAY['quantity'])
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,3 @@
|
||||
-- remove seeded categories (children first via path depth)
|
||||
DELETE FROM category_schema;
|
||||
DELETE FROM category;
|
||||
@@ -0,0 +1,53 @@
|
||||
-- Seed a FOOD-focused category skeleton.
|
||||
-- Structure = GS1 GPC backbone (segment/family/class) mapped to a self-built
|
||||
-- Chinese tree. ltree labels are english slugs (ltree forbids spaces/CJK);
|
||||
-- Chinese names live in name_zh. gpc_brick_code on leaves is a representative
|
||||
-- starter value to be replaced by a full official GPC import later.
|
||||
|
||||
-- Root segment: Food/Beverage/Tobacco (GPC segment 50000000)
|
||||
INSERT INTO category (name_zh, name_en, parent_id, path, gpc_brick_code, level)
|
||||
VALUES ('食品饮料', 'Food/Beverage', NULL, 'food', '50000000', 0);
|
||||
|
||||
-- Families (level 1)
|
||||
INSERT INTO category (name_zh, name_en, parent_id, path, gpc_brick_code, level)
|
||||
SELECT v.name_zh, v.name_en, c.id, v.path::ltree, v.code, 1
|
||||
FROM (VALUES
|
||||
('饮料', 'Beverages', 'food.beverages', '50130000'),
|
||||
('乳制品蛋类','Dairy/Eggs', 'food.dairy', '50180000'),
|
||||
('烘焙', 'Bakery', 'food.bakery', '50100000'),
|
||||
('零食', 'Snacks', 'food.snacks', '50190000'),
|
||||
('粮油', 'Staples/Oils', 'food.staple', '50160000'),
|
||||
('调味品', 'Condiments', 'food.condiments', '50170000')
|
||||
) AS v(name_zh, name_en, path, code)
|
||||
JOIN category c ON c.path = 'food';
|
||||
|
||||
-- Classes / leaves (level 2) with representative GPC brick codes
|
||||
INSERT INTO category (name_zh, name_en, parent_id, path, gpc_brick_code, level)
|
||||
SELECT v.name_zh, v.name_en, c.id, v.path::ltree, v.code, 2
|
||||
FROM (VALUES
|
||||
('包装饮用水', 'Bottled water', 'food.beverages.water', '10000224', 'food.beverages'),
|
||||
('碳酸饮料', 'Carbonated', 'food.beverages.carbonated', '10000225', 'food.beverages'),
|
||||
('果汁', 'Juice', 'food.beverages.juice', '10000226', 'food.beverages'),
|
||||
('牛奶', 'Milk', 'food.dairy.milk', '10000158', 'food.dairy'),
|
||||
('酸奶', 'Yogurt', 'food.dairy.yogurt', '10000159', 'food.dairy'),
|
||||
('奶酪', 'Cheese', 'food.dairy.cheese', '10000160', 'food.dairy'),
|
||||
('面包', 'Bread', 'food.bakery.bread', '10000040', 'food.bakery'),
|
||||
('饼干', 'Biscuits', 'food.bakery.biscuits', '10000041', 'food.bakery'),
|
||||
('薯片膨化', 'Chips/Snacks', 'food.snacks.chips', '10000310', 'food.snacks'),
|
||||
('巧克力', 'Chocolate', 'food.snacks.chocolate', '10000311', 'food.snacks'),
|
||||
('大米', 'Rice', 'food.staple.rice', '10000500', 'food.staple'),
|
||||
('面条', 'Noodles', 'food.staple.noodles', '10000501', 'food.staple'),
|
||||
('食用油', 'Cooking oil', 'food.staple.cooking_oil', '10000502', 'food.staple'),
|
||||
('酱油', 'Soy sauce', 'food.condiments.soy_sauce', '10000600', 'food.condiments'),
|
||||
('食盐', 'Table salt', 'food.condiments.salt', '10000601', 'food.condiments')
|
||||
) AS v(name_zh, name_en, path, code, parent_path)
|
||||
JOIN category c ON c.path = v.parent_path::ltree;
|
||||
|
||||
-- Parameter templates: leaf food categories use per_100g/ml nutrition basis.
|
||||
INSERT INTO category_schema (category_id, required_attributes, recommended_attributes, nutriment_basis)
|
||||
SELECT id,
|
||||
ARRAY['net_content'],
|
||||
ARRAY['energy','proteins','fat','carbohydrates','sugars','salt'],
|
||||
CASE WHEN path <@ 'food.beverages' THEN 'per_100ml' ELSE 'per_100g' END
|
||||
FROM category
|
||||
WHERE level = 2;
|
||||
@@ -0,0 +1,28 @@
|
||||
# Database migrations (golang-migrate)
|
||||
|
||||
SQL migrations for the OpenGoods database, applied with
|
||||
[golang-migrate](https://github.com/golang-migrate/migrate).
|
||||
Naming: `NNNN_description.up.sql` / `NNNN_description.down.sql`.
|
||||
|
||||
## Files
|
||||
| Version | Up | 内容 |
|
||||
|---------|----|------|
|
||||
| 0001 | `0001_init` | 扩展(pgcrypto/pg_trgm/ltree) + 全部核心表 + 索引 + tsvector 触发器 |
|
||||
| 0002 | `0002_seed_units` | 单位字典（与 `ingestion/opengoods/units.py` 一致）+ 常用营养参数定义 |
|
||||
| 0003 | `0003_seed_categories` | 食品品类骨架（GS1 GPC 映射 + 自建中文树）+ 品类参数模板 |
|
||||
|
||||
## 运行
|
||||
|
||||
先起本地依赖：`docker compose up -d postgres`
|
||||
|
||||
```bash
|
||||
export DBURL="postgres://opengoods:opengoods@localhost:5432/opengoods?sslmode=disable"
|
||||
migrate -path migrations -database "$DBURL" up # 升级到最新
|
||||
migrate -path migrations -database "$DBURL" down -all # 全部回滚
|
||||
migrate -path migrations -database "$DBURL" version # 查看当前版本
|
||||
```
|
||||
|
||||
安装 CLI：`go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.18.1`
|
||||
|
||||
> ltree 标签为英文 slug（不支持空格/中文），中文名存于 `category.name_zh`。
|
||||
> `gpc_brick_code` 为食品子集的代表值，后续用官方 GPC 全量导入替换。
|
||||
Reference in New Issue
Block a user