From b0b816b0ee81cb1a0d266feac0bba2725f49db83 Mon Sep 17 00:00:00 2001 From: John Doe <1115084741@qq.com> Date: Mon, 8 Jun 2026 06:32:16 +0000 Subject: [PATCH] =?UTF-8?q?feat(M1):=20=E6=95=B0=E6=8D=AE=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E8=BF=81=E7=A7=BB=20+=20GS1=20GPC=20=E9=A3=9F?= =?UTF-8?q?=E5=93=81=E5=88=86=E7=B1=BB=20+=20=E5=8D=95=E4=BD=8D=E5=AD=97?= =?UTF-8?q?=E5=85=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - migrations/0001_init: 全部核心表(product/food_detail/product_msrp/product_source/brand/manufacturer/category/category_schema/unit/attribute_definition/product_image/merge_log) + 索引(gtin唯一/name trigram/JSONB GIN/category ltree/tsvector) + tsvector/updated_at 触发器 - 0002_seed_units: 单位字典(与 units.py 一致, 含中文别名) + 常用营养参数定义 - 0003_seed_categories: 食品品类骨架(GS1 GPC 映射 + 自建中文树, ltree) + 品类参数模板(营养基准 per_100g/ml) - CI 增加 migrations job: 用 postgres service 跑 migrate up + down 验证可逆 - 本地实跑 up/down/re-up 全部通过 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 29 ++++ migrations/0001_init.down.sql | 21 +++ migrations/0001_init.up.sql | 182 +++++++++++++++++++++++ migrations/0002_seed_units.down.sql | 2 + migrations/0002_seed_units.up.sql | 29 ++++ migrations/0003_seed_categories.down.sql | 3 + migrations/0003_seed_categories.up.sql | 53 +++++++ migrations/README.md | 27 +++- 8 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 migrations/0001_init.down.sql create mode 100644 migrations/0001_init.up.sql create mode 100644 migrations/0002_seed_units.down.sql create mode 100644 migrations/0002_seed_units.up.sql create mode 100644 migrations/0003_seed_categories.down.sql create mode 100644 migrations/0003_seed_categories.up.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4da58d3..e5d0d9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,3 +43,32 @@ jobs: run: ruff format --check . - name: Pytest run: pytest -q + + migrations: + name: Migrations (postgres) + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: opengoods + POSTGRES_PASSWORD: opengoods + POSTGRES_DB: opengoods + ports: + - "5432:5432" + options: >- + --health-cmd "pg_isready -U opengoods" + --health-interval 5s --health-timeout 5s --health-retries 10 + env: + DBURL: postgres://opengoods:opengoods@localhost:5432/opengoods?sslmode=disable + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.23" + - name: Install golang-migrate + run: go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.18.1 + - name: Migrate up + run: migrate -path migrations -database "$DBURL" up + - name: Migrate down (reversibility) + run: migrate -path migrations -database "$DBURL" down -all diff --git a/migrations/0001_init.down.sql b/migrations/0001_init.down.sql new file mode 100644 index 0000000..c159768 --- /dev/null +++ b/migrations/0001_init.down.sql @@ -0,0 +1,21 @@ +DROP TRIGGER IF EXISTS trg_product_sync ON product; +DROP FUNCTION IF EXISTS product_sync_tsv(); + +DROP TABLE IF EXISTS merge_log; +DROP TABLE IF EXISTS product_source; +DROP TABLE IF EXISTS product_image; +DROP TABLE IF EXISTS product_msrp; +DROP TABLE IF EXISTS food_detail; +DROP TABLE IF EXISTS product; +DROP TABLE IF EXISTS attribute_definition; +DROP TABLE IF EXISTS unit; +DROP TABLE IF EXISTS category_schema; +DROP TABLE IF EXISTS category; +DROP TABLE IF EXISTS manufacturer; +DROP TABLE IF EXISTS brand; +DROP TABLE IF EXISTS source; + +DROP EXTENSION IF EXISTS ltree; +DROP EXTENSION IF EXISTS pg_trgm; +-- keep pgcrypto (commonly shared); drop only if you are sure: +-- DROP EXTENSION IF EXISTS pgcrypto; diff --git a/migrations/0001_init.up.sql b/migrations/0001_init.up.sql new file mode 100644 index 0000000..f122d0d --- /dev/null +++ b/migrations/0001_init.up.sql @@ -0,0 +1,182 @@ +-- OpenGoods (天工·商品标签) initial schema. +-- Public-good product information store: facts only, no commerce. + +CREATE EXTENSION IF NOT EXISTS pgcrypto; -- gen_random_uuid() +CREATE EXTENSION IF NOT EXISTS pg_trgm; -- fuzzy name search +CREATE EXTENSION IF NOT EXISTS ltree; -- category subtree queries + +-- Data sources (Open Food Facts / USDA / GS1 ...) with trust + license. +CREATE TABLE source ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + homepage TEXT, + license TEXT, + trust_weight NUMERIC(3,2) NOT NULL DEFAULT 0.5, + notes TEXT, + UNIQUE (name) +); + +CREATE TABLE brand ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + normalized_name TEXT NOT NULL, + aliases TEXT[] NOT NULL DEFAULT '{}', + UNIQUE (normalized_name) +); + +CREATE TABLE manufacturer ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + normalized_name TEXT NOT NULL, + country VARCHAR(64), + UNIQUE (normalized_name) +); + +-- Self-built category tree, each node optionally mapped to a GS1 GPC brick. +CREATE TABLE category ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name_zh TEXT NOT NULL, + name_en TEXT, + parent_id UUID REFERENCES category(id), + path LTREE NOT NULL, + gpc_brick_code VARCHAR(10), + level INT NOT NULL DEFAULT 0, + UNIQUE (path) +); + +-- Parameter template / constraints per category. +CREATE TABLE category_schema ( + category_id UUID PRIMARY KEY REFERENCES category(id) ON DELETE CASCADE, + required_attributes TEXT[] NOT NULL DEFAULT '{}', + recommended_attributes TEXT[] NOT NULL DEFAULT '{}', + nutriment_basis VARCHAR(16) +); + +-- Unit dictionary: each unit maps to a canonical unit within its dimension. +CREATE TABLE unit ( + code VARCHAR(16) PRIMARY KEY, + dimension VARCHAR(16) NOT NULL, + canonical VARCHAR(16) NOT NULL, + to_canonical_factor NUMERIC, + aliases TEXT[] NOT NULL DEFAULT '{}', + display TEXT +); + +-- Parameter dictionary: standard attribute keys with default unit. +CREATE TABLE attribute_definition ( + key VARCHAR(64) PRIMARY KEY, + label_zh TEXT, + label_en TEXT, + dimension VARCHAR(16), + default_unit VARCHAR(16) REFERENCES unit(code), + aliases TEXT[] NOT NULL DEFAULT '{}' +); + +CREATE TABLE product ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + gtin VARCHAR(14), + name TEXT NOT NULL, + brand_id UUID REFERENCES brand(id), + manufacturer_id UUID REFERENCES manufacturer(id), + category_id UUID REFERENCES category(id), + gpc_brick_code VARCHAR(10), + net_content_value NUMERIC, + net_content_unit VARCHAR(16), + net_content_canonical NUMERIC, + country_of_origin VARCHAR(64), + shelf_life_days INT, + storage TEXT, + attributes JSONB NOT NULL DEFAULT '{}', + quality_score NUMERIC(4,3) NOT NULL DEFAULT 0, + status VARCHAR(16) NOT NULL DEFAULT 'active', + canonical_id UUID REFERENCES product(id), + search_tsv TSVECTOR, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT product_status_chk CHECK (status IN ('active','merged','deprecated')), + CONSTRAINT product_quality_chk CHECK (quality_score >= 0 AND quality_score <= 1) +); + +CREATE TABLE food_detail ( + product_id UUID PRIMARY KEY REFERENCES product(id) ON DELETE CASCADE, + ingredients_text TEXT, + ingredients JSONB, + allergens TEXT[] NOT NULL DEFAULT '{}', + additives TEXT[] NOT NULL DEFAULT '{}', + nutriments JSONB, + nutrition_basis VARCHAR(16), + serving_size VARCHAR(32), + nutri_score CHAR(1), + labels TEXT[] NOT NULL DEFAULT '{}', + CONSTRAINT food_basis_chk CHECK (nutrition_basis IS NULL OR nutrition_basis IN ('per_100g','per_100ml','per_serving')) +); + +-- Official manufacturer-suggested retail price snapshot (no purchase link). +CREATE TABLE product_msrp ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + product_id UUID NOT NULL REFERENCES product(id) ON DELETE CASCADE, + amount NUMERIC(12,2) NOT NULL, + currency CHAR(3) NOT NULL, + region VARCHAR(8) NOT NULL DEFAULT 'CN', + source_id UUID REFERENCES source(id), + source_url TEXT, + effective_date DATE, + note TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE product_image ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + product_id UUID NOT NULL REFERENCES product(id) ON DELETE CASCADE, + url TEXT NOT NULL, + kind VARCHAR(16) NOT NULL DEFAULT 'other', + license TEXT, + source_id UUID REFERENCES source(id), + CONSTRAINT image_kind_chk CHECK (kind IN ('front','ingredients','nutrition','other')) +); + +-- Field-level provenance: which source provided which fields. +CREATE TABLE product_source ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + product_id UUID NOT NULL REFERENCES product(id) ON DELETE CASCADE, + source_id UUID REFERENCES source(id), + url TEXT, + fields TEXT[] NOT NULL DEFAULT '{}', + fetched_at TIMESTAMPTZ, + raw JSONB +); + +CREATE TABLE merge_log ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + kept_id UUID, + merged_id UUID, + reason TEXT, + actor TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Indexes +CREATE UNIQUE INDEX idx_product_gtin ON product (gtin) WHERE gtin IS NOT NULL; +CREATE INDEX idx_product_name_trgm ON product USING gin (name gin_trgm_ops); +CREATE INDEX idx_product_attrs ON product USING gin (attributes); +CREATE INDEX idx_product_tsv ON product USING gin (search_tsv); +CREATE INDEX idx_product_category ON product (category_id); +CREATE INDEX idx_product_brand ON product (brand_id); +CREATE INDEX idx_product_updated ON product (updated_at); +CREATE INDEX idx_food_nutriments ON food_detail USING gin (nutriments); +CREATE INDEX idx_category_path ON category USING gist (path); +CREATE INDEX idx_msrp_product ON product_msrp (product_id); +CREATE INDEX idx_psource_product ON product_source (product_id); + +-- Keep search_tsv and updated_at in sync. +CREATE OR REPLACE FUNCTION product_sync_tsv() RETURNS trigger AS $$ +BEGIN + NEW.search_tsv := to_tsvector('simple', coalesce(NEW.name, '')); + NEW.updated_at := now(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_product_sync + BEFORE INSERT OR UPDATE ON product + FOR EACH ROW EXECUTE FUNCTION product_sync_tsv(); diff --git a/migrations/0002_seed_units.down.sql b/migrations/0002_seed_units.down.sql new file mode 100644 index 0000000..f8d5bb5 --- /dev/null +++ b/migrations/0002_seed_units.down.sql @@ -0,0 +1,2 @@ +DELETE FROM attribute_definition; +DELETE FROM unit; diff --git a/migrations/0002_seed_units.up.sql b/migrations/0002_seed_units.up.sql new file mode 100644 index 0000000..20c5224 --- /dev/null +++ b/migrations/0002_seed_units.up.sql @@ -0,0 +1,29 @@ +-- Unit dictionary seed. Keep factors aligned with ingestion/opengoods/units.py. + +INSERT INTO unit (code, dimension, canonical, to_canonical_factor, aliases, display) VALUES + ('mg', 'mass', 'g', 0.001, ARRAY['毫克'], 'mg'), + ('g', 'mass', 'g', 1, ARRAY['克','gram','grams'], 'g'), + ('kg', 'mass', 'g', 1000, ARRAY['kgs','千克','公斤'], 'kg'), + ('ml', 'volume', 'ml', 1, ARRAY['毫升','milliliter'], 'mL'), + ('cl', 'volume', 'ml', 10, ARRAY['厘升'], 'cL'), + ('l', 'volume', 'ml', 1000, ARRAY['L','升','litre','liter'], 'L'), + ('kj', 'energy', 'kJ', 1, ARRAY['kJ','千焦'], 'kJ'), + ('kcal', 'energy', 'kJ', 4.184, ARRAY['千卡','大卡'], 'kcal'), + ('pct', 'ratio', 'pct', 1, ARRAY['%','percent','百分比'], '%'), + ('unit', 'count', 'unit',1, ARRAY['个','件','pcs','piece'], '个'), + ('mm', 'length', 'mm', 1, ARRAY['毫米'], 'mm'), + ('cm', 'length', 'mm', 10, ARRAY['厘米'], 'cm'), + ('day', 'duration', 'day', 1, ARRAY['天','日','days'], 'day') +ON CONFLICT (code) DO NOTHING; + +-- A few common food attribute definitions referencing the unit dictionary. +INSERT INTO attribute_definition (key, label_zh, label_en, dimension, default_unit, aliases) VALUES + ('energy', '能量', 'Energy', 'energy', 'kj', ARRAY['energy_kj']), + ('proteins', '蛋白质', 'Proteins', 'mass', 'g', ARRAY['protein']), + ('fat', '脂肪', 'Fat', 'mass', 'g', ARRAY['fats']), + ('saturated_fat', '饱和脂肪','Saturated fat','mass', 'g', ARRAY['saturated-fat']), + ('carbohydrates', '碳水化合物','Carbohydrates','mass', 'g', ARRAY['carbs']), + ('sugars', '糖', 'Sugars', 'mass', 'g', ARRAY['sugar']), + ('salt', '盐', 'Salt', 'mass', 'g', ARRAY['sodium_salt']), + ('net_content', '净含量', 'Net content', NULL, NULL, ARRAY['quantity']) +ON CONFLICT (key) DO NOTHING; diff --git a/migrations/0003_seed_categories.down.sql b/migrations/0003_seed_categories.down.sql new file mode 100644 index 0000000..a486b5e --- /dev/null +++ b/migrations/0003_seed_categories.down.sql @@ -0,0 +1,3 @@ +-- remove seeded categories (children first via path depth) +DELETE FROM category_schema; +DELETE FROM category; diff --git a/migrations/0003_seed_categories.up.sql b/migrations/0003_seed_categories.up.sql new file mode 100644 index 0000000..5eabdb4 --- /dev/null +++ b/migrations/0003_seed_categories.up.sql @@ -0,0 +1,53 @@ +-- Seed a FOOD-focused category skeleton. +-- Structure = GS1 GPC backbone (segment/family/class) mapped to a self-built +-- Chinese tree. ltree labels are english slugs (ltree forbids spaces/CJK); +-- Chinese names live in name_zh. gpc_brick_code on leaves is a representative +-- starter value to be replaced by a full official GPC import later. + +-- Root segment: Food/Beverage/Tobacco (GPC segment 50000000) +INSERT INTO category (name_zh, name_en, parent_id, path, gpc_brick_code, level) +VALUES ('食品饮料', 'Food/Beverage', NULL, 'food', '50000000', 0); + +-- Families (level 1) +INSERT INTO category (name_zh, name_en, parent_id, path, gpc_brick_code, level) +SELECT v.name_zh, v.name_en, c.id, v.path::ltree, v.code, 1 +FROM (VALUES + ('饮料', 'Beverages', 'food.beverages', '50130000'), + ('乳制品蛋类','Dairy/Eggs', 'food.dairy', '50180000'), + ('烘焙', 'Bakery', 'food.bakery', '50100000'), + ('零食', 'Snacks', 'food.snacks', '50190000'), + ('粮油', 'Staples/Oils', 'food.staple', '50160000'), + ('调味品', 'Condiments', 'food.condiments', '50170000') +) AS v(name_zh, name_en, path, code) +JOIN category c ON c.path = 'food'; + +-- Classes / leaves (level 2) with representative GPC brick codes +INSERT INTO category (name_zh, name_en, parent_id, path, gpc_brick_code, level) +SELECT v.name_zh, v.name_en, c.id, v.path::ltree, v.code, 2 +FROM (VALUES + ('包装饮用水', 'Bottled water', 'food.beverages.water', '10000224', 'food.beverages'), + ('碳酸饮料', 'Carbonated', 'food.beverages.carbonated', '10000225', 'food.beverages'), + ('果汁', 'Juice', 'food.beverages.juice', '10000226', 'food.beverages'), + ('牛奶', 'Milk', 'food.dairy.milk', '10000158', 'food.dairy'), + ('酸奶', 'Yogurt', 'food.dairy.yogurt', '10000159', 'food.dairy'), + ('奶酪', 'Cheese', 'food.dairy.cheese', '10000160', 'food.dairy'), + ('面包', 'Bread', 'food.bakery.bread', '10000040', 'food.bakery'), + ('饼干', 'Biscuits', 'food.bakery.biscuits', '10000041', 'food.bakery'), + ('薯片膨化', 'Chips/Snacks', 'food.snacks.chips', '10000310', 'food.snacks'), + ('巧克力', 'Chocolate', 'food.snacks.chocolate', '10000311', 'food.snacks'), + ('大米', 'Rice', 'food.staple.rice', '10000500', 'food.staple'), + ('面条', 'Noodles', 'food.staple.noodles', '10000501', 'food.staple'), + ('食用油', 'Cooking oil', 'food.staple.cooking_oil', '10000502', 'food.staple'), + ('酱油', 'Soy sauce', 'food.condiments.soy_sauce', '10000600', 'food.condiments'), + ('食盐', 'Table salt', 'food.condiments.salt', '10000601', 'food.condiments') +) AS v(name_zh, name_en, path, code, parent_path) +JOIN category c ON c.path = v.parent_path::ltree; + +-- Parameter templates: leaf food categories use per_100g/ml nutrition basis. +INSERT INTO category_schema (category_id, required_attributes, recommended_attributes, nutriment_basis) +SELECT id, + ARRAY['net_content'], + ARRAY['energy','proteins','fat','carbohydrates','sugars','salt'], + CASE WHEN path <@ 'food.beverages' THEN 'per_100ml' ELSE 'per_100g' END +FROM category +WHERE level = 2; diff --git a/migrations/README.md b/migrations/README.md index bf58d4e..d5f6711 100644 --- a/migrations/README.md +++ b/migrations/README.md @@ -1,3 +1,28 @@ # Database migrations (golang-migrate) -SQL migrations live here from milestone M1. Format: `NNNN_description.up.sql` / `.down.sql`. +SQL migrations for the OpenGoods database, applied with +[golang-migrate](https://github.com/golang-migrate/migrate). +Naming: `NNNN_description.up.sql` / `NNNN_description.down.sql`. + +## Files +| Version | Up | 内容 | +|---------|----|------| +| 0001 | `0001_init` | 扩展(pgcrypto/pg_trgm/ltree) + 全部核心表 + 索引 + tsvector 触发器 | +| 0002 | `0002_seed_units` | 单位字典（与 `ingestion/opengoods/units.py` 一致）+ 常用营养参数定义 | +| 0003 | `0003_seed_categories` | 食品品类骨架（GS1 GPC 映射 + 自建中文树）+ 品类参数模板 | + +## 运行 + +先起本地依赖：`docker compose up -d postgres` + +```bash +export DBURL="postgres://opengoods:opengoods@localhost:5432/opengoods?sslmode=disable" +migrate -path migrations -database "$DBURL" up # 升级到最新 +migrate -path migrations -database "$DBURL" down -all # 全部回滚 +migrate -path migrations -database "$DBURL" version # 查看当前版本 +``` + +安装 CLI：`go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.18.1` + +> ltree 标签为英文 slug（不支持空格/中文），中文名存于 `category.name_zh`。 +> `gpc_brick_code` 为食品子集的代表值，后续用官方 GPC 全量导入替换。