feat(M1): 数据模型迁移 + GS1 GPC 食品分类 + 单位字典

- migrations/0001_init: 全部核心表(product/food_detail/product_msrp/product_source/brand/manufacturer/category/category_schema/unit/attribute_definition/product_image/merge_log) + 索引(gtin唯一/name trigram/JSONB GIN/category ltree/tsvector) + tsvector/updated_at 触发器 - 0002_seed_units: 单位字典(与 units.py 一致, 含中文别名) + 常用营养参数定义 - 0003_seed_categories: 食品品类骨架(GS1 GPC 映射 + 自建中文树, ltree) + 品类参数模板(营养基准 per_100g/ml) - CI 增加 migrations job: 用 postgres service 跑 migrate up + down 验证可逆 - 本地实跑 up/down/re-up 全部通过 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-06-08 06:32:16 +00:00
8 changed files with 345 additions and 1 deletions
@@ -43,3 +43,32 @@ jobs:
        run: ruff format --check .
      - name: Pytest
        run: pytest -q
+
+  migrations:
+    name: Migrations (postgres)
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_USER: opengoods
+          POSTGRES_PASSWORD: opengoods
+          POSTGRES_DB: opengoods
+        ports:
+          - "5432:5432"
+        options: >-
+          --health-cmd "pg_isready -U opengoods"
+          --health-interval 5s --health-timeout 5s --health-retries 10
+    env:
+      DBURL: postgres://opengoods:opengoods@localhost:5432/opengoods?sslmode=disable
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.23"
+      - name: Install golang-migrate
+        run: go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.18.1
+      - name: Migrate up
+        run: migrate -path migrations -database "$DBURL" up
+      - name: Migrate down (reversibility)
+        run: migrate -path migrations -database "$DBURL" down -all
@@ -0,0 +1,21 @@
+DROP TRIGGER IF EXISTS trg_product_sync ON product;
+DROP FUNCTION IF EXISTS product_sync_tsv();
+
+DROP TABLE IF EXISTS merge_log;
+DROP TABLE IF EXISTS product_source;
+DROP TABLE IF EXISTS product_image;
+DROP TABLE IF EXISTS product_msrp;
+DROP TABLE IF EXISTS food_detail;
+DROP TABLE IF EXISTS product;
+DROP TABLE IF EXISTS attribute_definition;
+DROP TABLE IF EXISTS unit;
+DROP TABLE IF EXISTS category_schema;
+DROP TABLE IF EXISTS category;
+DROP TABLE IF EXISTS manufacturer;
+DROP TABLE IF EXISTS brand;
+DROP TABLE IF EXISTS source;
+
+DROP EXTENSION IF EXISTS ltree;
+DROP EXTENSION IF EXISTS pg_trgm;
+-- keep pgcrypto (commonly shared); drop only if you are sure:
+-- DROP EXTENSION IF EXISTS pgcrypto;
@@ -0,0 +1,182 @@
+-- OpenGoods (天工·商品标签) initial schema.
+-- Public-good product information store: facts only, no commerce.
+
+CREATE EXTENSION IF NOT EXISTS pgcrypto;   -- gen_random_uuid()
+CREATE EXTENSION IF NOT EXISTS pg_trgm;     -- fuzzy name search
+CREATE EXTENSION IF NOT EXISTS ltree;       -- category subtree queries
+
+-- Data sources (Open Food Facts / USDA / GS1 ...) with trust + license.
+CREATE TABLE source (
+    id           UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    name         TEXT NOT NULL,
+    homepage     TEXT,
+    license      TEXT,
+    trust_weight NUMERIC(3,2) NOT NULL DEFAULT 0.5,
+    notes        TEXT,
+    UNIQUE (name)
+);
+
+CREATE TABLE brand (
+    id              UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    name            TEXT NOT NULL,
+    normalized_name TEXT NOT NULL,
+    aliases         TEXT[] NOT NULL DEFAULT '{}',
+    UNIQUE (normalized_name)
+);
+
+CREATE TABLE manufacturer (
+    id              UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    name            TEXT NOT NULL,
+    normalized_name TEXT NOT NULL,
+    country         VARCHAR(64),
+    UNIQUE (normalized_name)
+);
+
+-- Self-built category tree, each node optionally mapped to a GS1 GPC brick.
+CREATE TABLE category (
+    id             UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    name_zh        TEXT NOT NULL,
+    name_en        TEXT,
+    parent_id      UUID REFERENCES category(id),
+    path           LTREE NOT NULL,
+    gpc_brick_code VARCHAR(10),
+    level          INT NOT NULL DEFAULT 0,
+    UNIQUE (path)
+);
+
+-- Parameter template / constraints per category.
+CREATE TABLE category_schema (
+    category_id           UUID PRIMARY KEY REFERENCES category(id) ON DELETE CASCADE,
+    required_attributes   TEXT[] NOT NULL DEFAULT '{}',
+    recommended_attributes TEXT[] NOT NULL DEFAULT '{}',
+    nutriment_basis       VARCHAR(16)
+);
+
+-- Unit dictionary: each unit maps to a canonical unit within its dimension.
+CREATE TABLE unit (
+    code                VARCHAR(16) PRIMARY KEY,
+    dimension           VARCHAR(16) NOT NULL,
+    canonical           VARCHAR(16) NOT NULL,
+    to_canonical_factor NUMERIC,
+    aliases             TEXT[] NOT NULL DEFAULT '{}',
+    display             TEXT
+);
+
+-- Parameter dictionary: standard attribute keys with default unit.
+CREATE TABLE attribute_definition (
+    key          VARCHAR(64) PRIMARY KEY,
+    label_zh     TEXT,
+    label_en     TEXT,
+    dimension    VARCHAR(16),
+    default_unit VARCHAR(16) REFERENCES unit(code),
+    aliases      TEXT[] NOT NULL DEFAULT '{}'
+);
+
+CREATE TABLE product (
+    id                    UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    gtin                  VARCHAR(14),
+    name                  TEXT NOT NULL,
+    brand_id              UUID REFERENCES brand(id),
+    manufacturer_id       UUID REFERENCES manufacturer(id),
+    category_id           UUID REFERENCES category(id),
+    gpc_brick_code        VARCHAR(10),
+    net_content_value     NUMERIC,
+    net_content_unit      VARCHAR(16),
+    net_content_canonical NUMERIC,
+    country_of_origin     VARCHAR(64),
+    shelf_life_days       INT,
+    storage               TEXT,
+    attributes            JSONB NOT NULL DEFAULT '{}',
+    quality_score         NUMERIC(4,3) NOT NULL DEFAULT 0,
+    status                VARCHAR(16) NOT NULL DEFAULT 'active',
+    canonical_id          UUID REFERENCES product(id),
+    search_tsv            TSVECTOR,
+    created_at            TIMESTAMPTZ NOT NULL DEFAULT now(),
+    updated_at            TIMESTAMPTZ NOT NULL DEFAULT now(),
+    CONSTRAINT product_status_chk CHECK (status IN ('active','merged','deprecated')),
+    CONSTRAINT product_quality_chk CHECK (quality_score >= 0 AND quality_score <= 1)
+);
+
+CREATE TABLE food_detail (
+    product_id       UUID PRIMARY KEY REFERENCES product(id) ON DELETE CASCADE,
+    ingredients_text TEXT,
+    ingredients      JSONB,
+    allergens        TEXT[] NOT NULL DEFAULT '{}',
+    additives        TEXT[] NOT NULL DEFAULT '{}',
+    nutriments       JSONB,
+    nutrition_basis  VARCHAR(16),
+    serving_size     VARCHAR(32),
+    nutri_score      CHAR(1),
+    labels           TEXT[] NOT NULL DEFAULT '{}',
+    CONSTRAINT food_basis_chk CHECK (nutrition_basis IS NULL OR nutrition_basis IN ('per_100g','per_100ml','per_serving'))
+);
+
+-- Official manufacturer-suggested retail price snapshot (no purchase link).
+CREATE TABLE product_msrp (
+    id             UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    product_id     UUID NOT NULL REFERENCES product(id) ON DELETE CASCADE,
+    amount         NUMERIC(12,2) NOT NULL,
+    currency       CHAR(3) NOT NULL,
+    region         VARCHAR(8) NOT NULL DEFAULT 'CN',
+    source_id      UUID REFERENCES source(id),
+    source_url     TEXT,
+    effective_date DATE,
+    note           TEXT,
+    created_at     TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE TABLE product_image (
+    id         UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    product_id UUID NOT NULL REFERENCES product(id) ON DELETE CASCADE,
+    url        TEXT NOT NULL,
+    kind       VARCHAR(16) NOT NULL DEFAULT 'other',
+    license    TEXT,
+    source_id  UUID REFERENCES source(id),
+    CONSTRAINT image_kind_chk CHECK (kind IN ('front','ingredients','nutrition','other'))
+);
+
+-- Field-level provenance: which source provided which fields.
+CREATE TABLE product_source (
+    id         UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    product_id UUID NOT NULL REFERENCES product(id) ON DELETE CASCADE,
+    source_id  UUID REFERENCES source(id),
+    url        TEXT,
+    fields     TEXT[] NOT NULL DEFAULT '{}',
+    fetched_at TIMESTAMPTZ,
+    raw        JSONB
+);
+
+CREATE TABLE merge_log (
+    id         UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    kept_id    UUID,
+    merged_id  UUID,
+    reason     TEXT,
+    actor      TEXT,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+-- Indexes
+CREATE UNIQUE INDEX idx_product_gtin ON product (gtin) WHERE gtin IS NOT NULL;
+CREATE INDEX idx_product_name_trgm ON product USING gin (name gin_trgm_ops);
+CREATE INDEX idx_product_attrs ON product USING gin (attributes);
+CREATE INDEX idx_product_tsv ON product USING gin (search_tsv);
+CREATE INDEX idx_product_category ON product (category_id);
+CREATE INDEX idx_product_brand ON product (brand_id);
+CREATE INDEX idx_product_updated ON product (updated_at);
+CREATE INDEX idx_food_nutriments ON food_detail USING gin (nutriments);
+CREATE INDEX idx_category_path ON category USING gist (path);
+CREATE INDEX idx_msrp_product ON product_msrp (product_id);
+CREATE INDEX idx_psource_product ON product_source (product_id);
+
+-- Keep search_tsv and updated_at in sync.
+CREATE OR REPLACE FUNCTION product_sync_tsv() RETURNS trigger AS $$
+BEGIN
+    NEW.search_tsv := to_tsvector('simple', coalesce(NEW.name, ''));
+    NEW.updated_at := now();
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER trg_product_sync
+    BEFORE INSERT OR UPDATE ON product
+    FOR EACH ROW EXECUTE FUNCTION product_sync_tsv();
@@ -0,0 +1,2 @@
+DELETE FROM attribute_definition;
+DELETE FROM unit;
@@ -0,0 +1,29 @@
+-- Unit dictionary seed. Keep factors aligned with ingestion/opengoods/units.py.
+
+INSERT INTO unit (code, dimension, canonical, to_canonical_factor, aliases, display) VALUES
+    ('mg',   'mass',     'g',   0.001,  ARRAY['毫克'],                 'mg'),
+    ('g',    'mass',     'g',   1,      ARRAY['克','gram','grams'],    'g'),
+    ('kg',   'mass',     'g',   1000,   ARRAY['kgs','千克','公斤'],     'kg'),
+    ('ml',   'volume',   'ml',  1,      ARRAY['毫升','milliliter'],     'mL'),
+    ('cl',   'volume',   'ml',  10,     ARRAY['厘升'],                 'cL'),
+    ('l',    'volume',   'ml',  1000,   ARRAY['L','升','litre','liter'], 'L'),
+    ('kj',   'energy',   'kJ',  1,      ARRAY['kJ','千焦'],            'kJ'),
+    ('kcal', 'energy',   'kJ',  4.184,  ARRAY['千卡','大卡'],          'kcal'),
+    ('pct',  'ratio',    'pct', 1,      ARRAY['%','percent','百分比'],  '%'),
+    ('unit', 'count',    'unit',1,      ARRAY['个','件','pcs','piece'], '个'),
+    ('mm',   'length',   'mm',  1,      ARRAY['毫米'],                 'mm'),
+    ('cm',   'length',   'mm',  10,     ARRAY['厘米'],                 'cm'),
+    ('day',  'duration', 'day', 1,      ARRAY['天','日','days'],        'day')
+ON CONFLICT (code) DO NOTHING;
+
+-- A few common food attribute definitions referencing the unit dictionary.
+INSERT INTO attribute_definition (key, label_zh, label_en, dimension, default_unit, aliases) VALUES
+    ('energy',        '能量',   'Energy',        'energy', 'kj',   ARRAY['energy_kj']),
+    ('proteins',      '蛋白质', 'Proteins',      'mass',   'g',    ARRAY['protein']),
+    ('fat',           '脂肪',   'Fat',           'mass',   'g',    ARRAY['fats']),
+    ('saturated_fat', '饱和脂肪','Saturated fat','mass',   'g',    ARRAY['saturated-fat']),
+    ('carbohydrates', '碳水化合物','Carbohydrates','mass', 'g',    ARRAY['carbs']),
+    ('sugars',        '糖',     'Sugars',        'mass',   'g',    ARRAY['sugar']),
+    ('salt',          '盐',     'Salt',          'mass',   'g',    ARRAY['sodium_salt']),
+    ('net_content',   '净含量', 'Net content',   NULL,     NULL,   ARRAY['quantity'])
+ON CONFLICT (key) DO NOTHING;
@@ -0,0 +1,3 @@
+-- remove seeded categories (children first via path depth)
+DELETE FROM category_schema;
+DELETE FROM category;
@@ -0,0 +1,53 @@
+-- Seed a FOOD-focused category skeleton.
+-- Structure = GS1 GPC backbone (segment/family/class) mapped to a self-built
+-- Chinese tree. ltree labels are english slugs (ltree forbids spaces/CJK);
+-- Chinese names live in name_zh. gpc_brick_code on leaves is a representative
+-- starter value to be replaced by a full official GPC import later.
+
+-- Root segment: Food/Beverage/Tobacco (GPC segment 50000000)
+INSERT INTO category (name_zh, name_en, parent_id, path, gpc_brick_code, level)
+VALUES ('食品饮料', 'Food/Beverage', NULL, 'food', '50000000', 0);
+
+-- Families (level 1)
+INSERT INTO category (name_zh, name_en, parent_id, path, gpc_brick_code, level)
+SELECT v.name_zh, v.name_en, c.id, v.path::ltree, v.code, 1
+FROM (VALUES
+    ('饮料',     'Beverages',        'food.beverages',  '50130000'),
+    ('乳制品蛋类','Dairy/Eggs',       'food.dairy',      '50180000'),
+    ('烘焙',     'Bakery',           'food.bakery',     '50100000'),
+    ('零食',     'Snacks',           'food.snacks',     '50190000'),
+    ('粮油',     'Staples/Oils',     'food.staple',     '50160000'),
+    ('调味品',   'Condiments',       'food.condiments', '50170000')
+) AS v(name_zh, name_en, path, code)
+JOIN category c ON c.path = 'food';
+
+-- Classes / leaves (level 2) with representative GPC brick codes
+INSERT INTO category (name_zh, name_en, parent_id, path, gpc_brick_code, level)
+SELECT v.name_zh, v.name_en, c.id, v.path::ltree, v.code, 2
+FROM (VALUES
+    ('包装饮用水', 'Bottled water',   'food.beverages.water',      '10000224', 'food.beverages'),
+    ('碳酸饮料',   'Carbonated',      'food.beverages.carbonated', '10000225', 'food.beverages'),
+    ('果汁',       'Juice',           'food.beverages.juice',      '10000226', 'food.beverages'),
+    ('牛奶',       'Milk',            'food.dairy.milk',           '10000158', 'food.dairy'),
+    ('酸奶',       'Yogurt',          'food.dairy.yogurt',         '10000159', 'food.dairy'),
+    ('奶酪',       'Cheese',          'food.dairy.cheese',         '10000160', 'food.dairy'),
+    ('面包',       'Bread',           'food.bakery.bread',         '10000040', 'food.bakery'),
+    ('饼干',       'Biscuits',        'food.bakery.biscuits',      '10000041', 'food.bakery'),
+    ('薯片膨化',   'Chips/Snacks',    'food.snacks.chips',         '10000310', 'food.snacks'),
+    ('巧克力',     'Chocolate',       'food.snacks.chocolate',     '10000311', 'food.snacks'),
+    ('大米',       'Rice',            'food.staple.rice',          '10000500', 'food.staple'),
+    ('面条',       'Noodles',         'food.staple.noodles',       '10000501', 'food.staple'),
+    ('食用油',     'Cooking oil',     'food.staple.cooking_oil',   '10000502', 'food.staple'),
+    ('酱油',       'Soy sauce',       'food.condiments.soy_sauce', '10000600', 'food.condiments'),
+    ('食盐',       'Table salt',      'food.condiments.salt',      '10000601', 'food.condiments')
+) AS v(name_zh, name_en, path, code, parent_path)
+JOIN category c ON c.path = v.parent_path::ltree;
+
+-- Parameter templates: leaf food categories use per_100g/ml nutrition basis.
+INSERT INTO category_schema (category_id, required_attributes, recommended_attributes, nutriment_basis)
+SELECT id,
+       ARRAY['net_content'],
+       ARRAY['energy','proteins','fat','carbohydrates','sugars','salt'],
+       CASE WHEN path <@ 'food.beverages' THEN 'per_100ml' ELSE 'per_100g' END
+FROM category
+WHERE level = 2;
@@ -1,3 +1,28 @@
 # Database migrations (golang-migrate)

-SQL migrations live here from milestone M1. Format: `NNNN_description.up.sql` / `.down.sql`.
+SQL migrations for the OpenGoods database, applied with
+[golang-migrate](https://github.com/golang-migrate/migrate).
+Naming: `NNNN_description.up.sql` / `NNNN_description.down.sql`.
+
+## Files
+| Version | Up | 内容 |
+|---------|----|------|
+| 0001 | `0001_init` | 扩展(pgcrypto/pg_trgm/ltree) + 全部核心表 + 索引 + tsvector 触发器 |
+| 0002 | `0002_seed_units` | 单位字典（与 `ingestion/opengoods/units.py` 一致）+ 常用营养参数定义 |
+| 0003 | `0003_seed_categories` | 食品品类骨架（GS1 GPC 映射 + 自建中文树）+ 品类参数模板 |
+
+## 运行
+
+先起本地依赖：`docker compose up -d postgres`
+
+```bash
+export DBURL="postgres://opengoods:opengoods@localhost:5432/opengoods?sslmode=disable"
+migrate -path migrations -database "$DBURL" up        # 升级到最新
+migrate -path migrations -database "$DBURL" down -all # 全部回滚
+migrate -path migrations -database "$DBURL" version   # 查看当前版本
+```
+
+安装 CLI：`go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.18.1`
+
+> ltree 标签为英文 slug（不支持空格/中文），中文名存于 `category.name_zh`。
+> `gpc_brick_code` 为食品子集的代表值，后续用官方 GPC 全量导入替换。