feat(M1): 数据模型迁移 + GS1 GPC 食品分类 + 单位字典
- migrations/0001_init: 全部核心表(product/food_detail/product_msrp/product_source/brand/manufacturer/category/category_schema/unit/attribute_definition/product_image/merge_log) + 索引(gtin唯一/name trigram/JSONB GIN/category ltree/tsvector) + tsvector/updated_at 触发器 - 0002_seed_units: 单位字典(与 units.py 一致, 含中文别名) + 常用营养参数定义 - 0003_seed_categories: 食品品类骨架(GS1 GPC 映射 + 自建中文树, ltree) + 品类参数模板(营养基准 per_100g/ml) - CI 增加 migrations job: 用 postgres service 跑 migrate up + down 验证可逆 - 本地实跑 up/down/re-up 全部通过 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
@@ -43,3 +43,32 @@ jobs:
|
||||
run: ruff format --check .
|
||||
- name: Pytest
|
||||
run: pytest -q
|
||||
|
||||
migrations:
|
||||
name: Migrations (postgres)
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
POSTGRES_USER: opengoods
|
||||
POSTGRES_PASSWORD: opengoods
|
||||
POSTGRES_DB: opengoods
|
||||
ports:
|
||||
- "5432:5432"
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U opengoods"
|
||||
--health-interval 5s --health-timeout 5s --health-retries 10
|
||||
env:
|
||||
DBURL: postgres://opengoods:opengoods@localhost:5432/opengoods?sslmode=disable
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.23"
|
||||
- name: Install golang-migrate
|
||||
run: go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.18.1
|
||||
- name: Migrate up
|
||||
run: migrate -path migrations -database "$DBURL" up
|
||||
- name: Migrate down (reversibility)
|
||||
run: migrate -path migrations -database "$DBURL" down -all
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
DROP TRIGGER IF EXISTS trg_product_sync ON product;
|
||||
DROP FUNCTION IF EXISTS product_sync_tsv();
|
||||
|
||||
DROP TABLE IF EXISTS merge_log;
|
||||
DROP TABLE IF EXISTS product_source;
|
||||
DROP TABLE IF EXISTS product_image;
|
||||
DROP TABLE IF EXISTS product_msrp;
|
||||
DROP TABLE IF EXISTS food_detail;
|
||||
DROP TABLE IF EXISTS product;
|
||||
DROP TABLE IF EXISTS attribute_definition;
|
||||
DROP TABLE IF EXISTS unit;
|
||||
DROP TABLE IF EXISTS category_schema;
|
||||
DROP TABLE IF EXISTS category;
|
||||
DROP TABLE IF EXISTS manufacturer;
|
||||
DROP TABLE IF EXISTS brand;
|
||||
DROP TABLE IF EXISTS source;
|
||||
|
||||
DROP EXTENSION IF EXISTS ltree;
|
||||
DROP EXTENSION IF EXISTS pg_trgm;
|
||||
-- keep pgcrypto (commonly shared); drop only if you are sure:
|
||||
-- DROP EXTENSION IF EXISTS pgcrypto;
|
||||
@@ -0,0 +1,182 @@
|
||||
-- OpenGoods (天工·商品标签) initial schema.
|
||||
-- Public-good product information store: facts only, no commerce.
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS pgcrypto; -- gen_random_uuid()
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm; -- fuzzy name search
|
||||
CREATE EXTENSION IF NOT EXISTS ltree; -- category subtree queries
|
||||
|
||||
-- Data sources (Open Food Facts / USDA / GS1 ...) with trust + license.
|
||||
CREATE TABLE source (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name TEXT NOT NULL,
|
||||
homepage TEXT,
|
||||
license TEXT,
|
||||
trust_weight NUMERIC(3,2) NOT NULL DEFAULT 0.5,
|
||||
notes TEXT,
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE brand (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name TEXT NOT NULL,
|
||||
normalized_name TEXT NOT NULL,
|
||||
aliases TEXT[] NOT NULL DEFAULT '{}',
|
||||
UNIQUE (normalized_name)
|
||||
);
|
||||
|
||||
CREATE TABLE manufacturer (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name TEXT NOT NULL,
|
||||
normalized_name TEXT NOT NULL,
|
||||
country VARCHAR(64),
|
||||
UNIQUE (normalized_name)
|
||||
);
|
||||
|
||||
-- Self-built category tree, each node optionally mapped to a GS1 GPC brick.
|
||||
CREATE TABLE category (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name_zh TEXT NOT NULL,
|
||||
name_en TEXT,
|
||||
parent_id UUID REFERENCES category(id),
|
||||
path LTREE NOT NULL,
|
||||
gpc_brick_code VARCHAR(10),
|
||||
level INT NOT NULL DEFAULT 0,
|
||||
UNIQUE (path)
|
||||
);
|
||||
|
||||
-- Parameter template / constraints per category.
|
||||
CREATE TABLE category_schema (
|
||||
category_id UUID PRIMARY KEY REFERENCES category(id) ON DELETE CASCADE,
|
||||
required_attributes TEXT[] NOT NULL DEFAULT '{}',
|
||||
recommended_attributes TEXT[] NOT NULL DEFAULT '{}',
|
||||
nutriment_basis VARCHAR(16)
|
||||
);
|
||||
|
||||
-- Unit dictionary: each unit maps to a canonical unit within its dimension.
|
||||
CREATE TABLE unit (
|
||||
code VARCHAR(16) PRIMARY KEY,
|
||||
dimension VARCHAR(16) NOT NULL,
|
||||
canonical VARCHAR(16) NOT NULL,
|
||||
to_canonical_factor NUMERIC,
|
||||
aliases TEXT[] NOT NULL DEFAULT '{}',
|
||||
display TEXT
|
||||
);
|
||||
|
||||
-- Parameter dictionary: standard attribute keys with default unit.
|
||||
CREATE TABLE attribute_definition (
|
||||
key VARCHAR(64) PRIMARY KEY,
|
||||
label_zh TEXT,
|
||||
label_en TEXT,
|
||||
dimension VARCHAR(16),
|
||||
default_unit VARCHAR(16) REFERENCES unit(code),
|
||||
aliases TEXT[] NOT NULL DEFAULT '{}'
|
||||
);
|
||||
|
||||
CREATE TABLE product (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
gtin VARCHAR(14),
|
||||
name TEXT NOT NULL,
|
||||
brand_id UUID REFERENCES brand(id),
|
||||
manufacturer_id UUID REFERENCES manufacturer(id),
|
||||
category_id UUID REFERENCES category(id),
|
||||
gpc_brick_code VARCHAR(10),
|
||||
net_content_value NUMERIC,
|
||||
net_content_unit VARCHAR(16),
|
||||
net_content_canonical NUMERIC,
|
||||
country_of_origin VARCHAR(64),
|
||||
shelf_life_days INT,
|
||||
storage TEXT,
|
||||
attributes JSONB NOT NULL DEFAULT '{}',
|
||||
quality_score NUMERIC(4,3) NOT NULL DEFAULT 0,
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'active',
|
||||
canonical_id UUID REFERENCES product(id),
|
||||
search_tsv TSVECTOR,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT product_status_chk CHECK (status IN ('active','merged','deprecated')),
|
||||
CONSTRAINT product_quality_chk CHECK (quality_score >= 0 AND quality_score <= 1)
|
||||
);
|
||||
|
||||
CREATE TABLE food_detail (
|
||||
product_id UUID PRIMARY KEY REFERENCES product(id) ON DELETE CASCADE,
|
||||
ingredients_text TEXT,
|
||||
ingredients JSONB,
|
||||
allergens TEXT[] NOT NULL DEFAULT '{}',
|
||||
additives TEXT[] NOT NULL DEFAULT '{}',
|
||||
nutriments JSONB,
|
||||
nutrition_basis VARCHAR(16),
|
||||
serving_size VARCHAR(32),
|
||||
nutri_score CHAR(1),
|
||||
labels TEXT[] NOT NULL DEFAULT '{}',
|
||||
CONSTRAINT food_basis_chk CHECK (nutrition_basis IS NULL OR nutrition_basis IN ('per_100g','per_100ml','per_serving'))
|
||||
);
|
||||
|
||||
-- Official manufacturer-suggested retail price snapshot (no purchase link).
|
||||
CREATE TABLE product_msrp (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
product_id UUID NOT NULL REFERENCES product(id) ON DELETE CASCADE,
|
||||
amount NUMERIC(12,2) NOT NULL,
|
||||
currency CHAR(3) NOT NULL,
|
||||
region VARCHAR(8) NOT NULL DEFAULT 'CN',
|
||||
source_id UUID REFERENCES source(id),
|
||||
source_url TEXT,
|
||||
effective_date DATE,
|
||||
note TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE product_image (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
product_id UUID NOT NULL REFERENCES product(id) ON DELETE CASCADE,
|
||||
url TEXT NOT NULL,
|
||||
kind VARCHAR(16) NOT NULL DEFAULT 'other',
|
||||
license TEXT,
|
||||
source_id UUID REFERENCES source(id),
|
||||
CONSTRAINT image_kind_chk CHECK (kind IN ('front','ingredients','nutrition','other'))
|
||||
);
|
||||
|
||||
-- Field-level provenance: which source provided which fields.
|
||||
CREATE TABLE product_source (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
product_id UUID NOT NULL REFERENCES product(id) ON DELETE CASCADE,
|
||||
source_id UUID REFERENCES source(id),
|
||||
url TEXT,
|
||||
fields TEXT[] NOT NULL DEFAULT '{}',
|
||||
fetched_at TIMESTAMPTZ,
|
||||
raw JSONB
|
||||
);
|
||||
|
||||
CREATE TABLE merge_log (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
kept_id UUID,
|
||||
merged_id UUID,
|
||||
reason TEXT,
|
||||
actor TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Indexes
|
||||
CREATE UNIQUE INDEX idx_product_gtin ON product (gtin) WHERE gtin IS NOT NULL;
|
||||
CREATE INDEX idx_product_name_trgm ON product USING gin (name gin_trgm_ops);
|
||||
CREATE INDEX idx_product_attrs ON product USING gin (attributes);
|
||||
CREATE INDEX idx_product_tsv ON product USING gin (search_tsv);
|
||||
CREATE INDEX idx_product_category ON product (category_id);
|
||||
CREATE INDEX idx_product_brand ON product (brand_id);
|
||||
CREATE INDEX idx_product_updated ON product (updated_at);
|
||||
CREATE INDEX idx_food_nutriments ON food_detail USING gin (nutriments);
|
||||
CREATE INDEX idx_category_path ON category USING gist (path);
|
||||
CREATE INDEX idx_msrp_product ON product_msrp (product_id);
|
||||
CREATE INDEX idx_psource_product ON product_source (product_id);
|
||||
|
||||
-- Keep search_tsv and updated_at in sync.
|
||||
CREATE OR REPLACE FUNCTION product_sync_tsv() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.search_tsv := to_tsvector('simple', coalesce(NEW.name, ''));
|
||||
NEW.updated_at := now();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER trg_product_sync
|
||||
BEFORE INSERT OR UPDATE ON product
|
||||
FOR EACH ROW EXECUTE FUNCTION product_sync_tsv();
|
||||
@@ -0,0 +1,2 @@
|
||||
DELETE FROM attribute_definition;
|
||||
DELETE FROM unit;
|
||||
@@ -0,0 +1,29 @@
|
||||
-- Unit dictionary seed. Keep factors aligned with ingestion/opengoods/units.py.
|
||||
|
||||
INSERT INTO unit (code, dimension, canonical, to_canonical_factor, aliases, display) VALUES
|
||||
('mg', 'mass', 'g', 0.001, ARRAY['毫克'], 'mg'),
|
||||
('g', 'mass', 'g', 1, ARRAY['克','gram','grams'], 'g'),
|
||||
('kg', 'mass', 'g', 1000, ARRAY['kgs','千克','公斤'], 'kg'),
|
||||
('ml', 'volume', 'ml', 1, ARRAY['毫升','milliliter'], 'mL'),
|
||||
('cl', 'volume', 'ml', 10, ARRAY['厘升'], 'cL'),
|
||||
('l', 'volume', 'ml', 1000, ARRAY['L','升','litre','liter'], 'L'),
|
||||
('kj', 'energy', 'kJ', 1, ARRAY['kJ','千焦'], 'kJ'),
|
||||
('kcal', 'energy', 'kJ', 4.184, ARRAY['千卡','大卡'], 'kcal'),
|
||||
('pct', 'ratio', 'pct', 1, ARRAY['%','percent','百分比'], '%'),
|
||||
('unit', 'count', 'unit',1, ARRAY['个','件','pcs','piece'], '个'),
|
||||
('mm', 'length', 'mm', 1, ARRAY['毫米'], 'mm'),
|
||||
('cm', 'length', 'mm', 10, ARRAY['厘米'], 'cm'),
|
||||
('day', 'duration', 'day', 1, ARRAY['天','日','days'], 'day')
|
||||
ON CONFLICT (code) DO NOTHING;
|
||||
|
||||
-- A few common food attribute definitions referencing the unit dictionary.
|
||||
INSERT INTO attribute_definition (key, label_zh, label_en, dimension, default_unit, aliases) VALUES
|
||||
('energy', '能量', 'Energy', 'energy', 'kj', ARRAY['energy_kj']),
|
||||
('proteins', '蛋白质', 'Proteins', 'mass', 'g', ARRAY['protein']),
|
||||
('fat', '脂肪', 'Fat', 'mass', 'g', ARRAY['fats']),
|
||||
('saturated_fat', '饱和脂肪','Saturated fat','mass', 'g', ARRAY['saturated-fat']),
|
||||
('carbohydrates', '碳水化合物','Carbohydrates','mass', 'g', ARRAY['carbs']),
|
||||
('sugars', '糖', 'Sugars', 'mass', 'g', ARRAY['sugar']),
|
||||
('salt', '盐', 'Salt', 'mass', 'g', ARRAY['sodium_salt']),
|
||||
('net_content', '净含量', 'Net content', NULL, NULL, ARRAY['quantity'])
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
@@ -0,0 +1,3 @@
|
||||
-- remove seeded categories (children first via path depth)
|
||||
DELETE FROM category_schema;
|
||||
DELETE FROM category;
|
||||
@@ -0,0 +1,53 @@
|
||||
-- Seed a FOOD-focused category skeleton.
|
||||
-- Structure = GS1 GPC backbone (segment/family/class) mapped to a self-built
|
||||
-- Chinese tree. ltree labels are english slugs (ltree forbids spaces/CJK);
|
||||
-- Chinese names live in name_zh. gpc_brick_code on leaves is a representative
|
||||
-- starter value to be replaced by a full official GPC import later.
|
||||
|
||||
-- Root segment: Food/Beverage/Tobacco (GPC segment 50000000)
|
||||
INSERT INTO category (name_zh, name_en, parent_id, path, gpc_brick_code, level)
|
||||
VALUES ('食品饮料', 'Food/Beverage', NULL, 'food', '50000000', 0);
|
||||
|
||||
-- Families (level 1)
|
||||
INSERT INTO category (name_zh, name_en, parent_id, path, gpc_brick_code, level)
|
||||
SELECT v.name_zh, v.name_en, c.id, v.path::ltree, v.code, 1
|
||||
FROM (VALUES
|
||||
('饮料', 'Beverages', 'food.beverages', '50130000'),
|
||||
('乳制品蛋类','Dairy/Eggs', 'food.dairy', '50180000'),
|
||||
('烘焙', 'Bakery', 'food.bakery', '50100000'),
|
||||
('零食', 'Snacks', 'food.snacks', '50190000'),
|
||||
('粮油', 'Staples/Oils', 'food.staple', '50160000'),
|
||||
('调味品', 'Condiments', 'food.condiments', '50170000')
|
||||
) AS v(name_zh, name_en, path, code)
|
||||
JOIN category c ON c.path = 'food';
|
||||
|
||||
-- Classes / leaves (level 2) with representative GPC brick codes
|
||||
INSERT INTO category (name_zh, name_en, parent_id, path, gpc_brick_code, level)
|
||||
SELECT v.name_zh, v.name_en, c.id, v.path::ltree, v.code, 2
|
||||
FROM (VALUES
|
||||
('包装饮用水', 'Bottled water', 'food.beverages.water', '10000224', 'food.beverages'),
|
||||
('碳酸饮料', 'Carbonated', 'food.beverages.carbonated', '10000225', 'food.beverages'),
|
||||
('果汁', 'Juice', 'food.beverages.juice', '10000226', 'food.beverages'),
|
||||
('牛奶', 'Milk', 'food.dairy.milk', '10000158', 'food.dairy'),
|
||||
('酸奶', 'Yogurt', 'food.dairy.yogurt', '10000159', 'food.dairy'),
|
||||
('奶酪', 'Cheese', 'food.dairy.cheese', '10000160', 'food.dairy'),
|
||||
('面包', 'Bread', 'food.bakery.bread', '10000040', 'food.bakery'),
|
||||
('饼干', 'Biscuits', 'food.bakery.biscuits', '10000041', 'food.bakery'),
|
||||
('薯片膨化', 'Chips/Snacks', 'food.snacks.chips', '10000310', 'food.snacks'),
|
||||
('巧克力', 'Chocolate', 'food.snacks.chocolate', '10000311', 'food.snacks'),
|
||||
('大米', 'Rice', 'food.staple.rice', '10000500', 'food.staple'),
|
||||
('面条', 'Noodles', 'food.staple.noodles', '10000501', 'food.staple'),
|
||||
('食用油', 'Cooking oil', 'food.staple.cooking_oil', '10000502', 'food.staple'),
|
||||
('酱油', 'Soy sauce', 'food.condiments.soy_sauce', '10000600', 'food.condiments'),
|
||||
('食盐', 'Table salt', 'food.condiments.salt', '10000601', 'food.condiments')
|
||||
) AS v(name_zh, name_en, path, code, parent_path)
|
||||
JOIN category c ON c.path = v.parent_path::ltree;
|
||||
|
||||
-- Parameter templates: leaf food categories use per_100g/ml nutrition basis.
|
||||
INSERT INTO category_schema (category_id, required_attributes, recommended_attributes, nutriment_basis)
|
||||
SELECT id,
|
||||
ARRAY['net_content'],
|
||||
ARRAY['energy','proteins','fat','carbohydrates','sugars','salt'],
|
||||
CASE WHEN path <@ 'food.beverages' THEN 'per_100ml' ELSE 'per_100g' END
|
||||
FROM category
|
||||
WHERE level = 2;
|
||||
+26
-1
@@ -1,3 +1,28 @@
|
||||
# Database migrations (golang-migrate)
|
||||
|
||||
SQL migrations live here from milestone M1. Format: `NNNN_description.up.sql` / `.down.sql`.
|
||||
SQL migrations for the OpenGoods database, applied with
|
||||
[golang-migrate](https://github.com/golang-migrate/migrate).
|
||||
Naming: `NNNN_description.up.sql` / `NNNN_description.down.sql`.
|
||||
|
||||
## Files
|
||||
| Version | Up | 内容 |
|
||||
|---------|----|------|
|
||||
| 0001 | `0001_init` | 扩展(pgcrypto/pg_trgm/ltree) + 全部核心表 + 索引 + tsvector 触发器 |
|
||||
| 0002 | `0002_seed_units` | 单位字典（与 `ingestion/opengoods/units.py` 一致）+ 常用营养参数定义 |
|
||||
| 0003 | `0003_seed_categories` | 食品品类骨架（GS1 GPC 映射 + 自建中文树）+ 品类参数模板 |
|
||||
|
||||
## 运行
|
||||
|
||||
先起本地依赖：`docker compose up -d postgres`
|
||||
|
||||
```bash
|
||||
export DBURL="postgres://opengoods:opengoods@localhost:5432/opengoods?sslmode=disable"
|
||||
migrate -path migrations -database "$DBURL" up # 升级到最新
|
||||
migrate -path migrations -database "$DBURL" down -all # 全部回滚
|
||||
migrate -path migrations -database "$DBURL" version # 查看当前版本
|
||||
```
|
||||
|
||||
安装 CLI：`go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.18.1`
|
||||
|
||||
> ltree 标签为英文 slug（不支持空格/中文），中文名存于 `category.name_zh`。
|
||||
> `gpc_brick_code` 为食品子集的代表值，后续用官方 GPC 全量导入替换。
|
||||
|
||||
Reference in New Issue
Block a user