Compare commits

...

13 Commits

Author SHA1 Message Date
lixu 20b9eb0fc9 Merge pull request 'ci: 修复 main 上 Python CI 历史失败(在 python3.12+node 容器内运行)' (#28) from devin/1782280936-fix-python-ci into main
CI / Go (api) (push) Successful in 54s
CI / Python (ingestion) (push) Successful in 23s
CI / Migrations (postgres) (push) Successful in 27s
2026-06-24 14:32:51 +08:00
rosemariejebbjtxbfp 17bc0ed680 ci: remove GitHub-hosted actions, manual checkout + Go from CN mirror
CI / Go (api) (pull_request) Successful in 51s
CI / Python (ingestion) (pull_request) Successful in 21s
CI / Migrations (postgres) (pull_request) Successful in 26s
Self-hosted Gitea runner has flaky/blocked access to github.com, causing
actions/checkout and actions/setup-go to time out intermittently across all
jobs. Replace them with:
- manual git checkout against $GITHUB_SERVER_URL (the Gitea host, reachable
  from job containers)
- Go installed from mirrors.aliyun.com

Python job stays on the python3.12-nodejs20 container (fixes the original
PEP 660 editable-install failure). No more github.com network dependency.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-06-24 06:30:03 +00:00
rosemariejebbjtxbfp 5020fdcc19 ci: re-trigger with public gitea actions url
CI / Go (api) (pull_request) Failing after 11s
CI / Python (ingestion) (pull_request) Failing after 31s
CI / Migrations (postgres) (pull_request) Failing after 11s
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-06-24 06:24:52 +00:00
rosemariejebbjtxbfp 885f6c2b01 ci: re-trigger with self-hosted actions
CI / Go (api) (pull_request) Failing after 12s
CI / Python (ingestion) (pull_request) Failing after 1m31s
CI / Migrations (postgres) (pull_request) Failing after 32s
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-06-24 06:21:01 +00:00
rosemariejebbjtxbfp 2aff6287d2 ci: re-trigger after configuring action proxy
CI / Go (api) (pull_request) Failing after 13s
CI / Python (ingestion) (pull_request) Failing after 31s
CI / Migrations (postgres) (pull_request) Failing after 1m32s
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-06-24 06:10:22 +00:00
rosemariejebbjtxbfp 3364a50c68 ci: run Python job in python3.12+node container
CI / Go (api) (pull_request) Failing after 1m32s
CI / Python (ingestion) (pull_request) Failing after 31s
CI / Migrations (postgres) (pull_request) Failing after 19m16s
setup-python@v5 can't fetch CPython 3.12 on the self-hosted Gitea runner
(it queries the Gitea API for actions/python-versions, which 404s), so the
job silently fell back to the image's system Python 3.10 + pip 22.0.2. That
old pip lacks PEP 660 editable support, so 'pip install -e' failed with
'build backend is missing the build_editable hook', and 3.10 is below the
project's requires-python>=3.11 (code uses datetime.UTC).

Run the job inside nikolaik/python-nodejs:python3.12-nodejs20 which bundles
Python 3.12, Node 20 (for actions/checkout) and modern pip, removing the
GitHub download dependency entirely.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-06-24 06:02:28 +00:00
lixu 936828abae Merge pull request 'perf(search): 浏览路径部分复合索引 (0012)' (#27) from devin/1782279211-search-indexes into main
CI / Go (api) (push) Successful in 49s
CI / Python (ingestion) (push) Failing after 15s
CI / Migrations (postgres) (push) Successful in 31s
2026-06-24 13:35:50 +08:00
rosemariejebbjtxbfp ab7a964934 perf(search): 为浏览路径补部分复合索引 (0012)
CI / Go (api) (pull_request) Successful in 58s
CI / Python (ingestion) (pull_request) Failing after 21s
CI / Migrations (postgres) (pull_request) Successful in 32s
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-06-24 05:33:31 +00:00
lixu 649fcc711c Merge pull request 'docs: 可扩展性设计与路线图 (scalability roadmap)' (#26) from devin/1782278890-scalability-doc into main
CI / Go (api) (push) Successful in 52s
CI / Python (ingestion) (push) Failing after 16s
CI / Migrations (postgres) (push) Successful in 26s
2026-06-24 13:31:21 +08:00
rosemariejebbjtxbfp 3b62f61288 docs: 新增可扩展性设计与路线图 (scalability roadmap)
CI / Go (api) (pull_request) Successful in 59s
CI / Python (ingestion) (pull_request) Failing after 20s
CI / Migrations (postgres) (pull_request) Successful in 34s
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-06-24 05:29:08 +00:00
lixu ba36e3ff4c Merge pull request '新增联系我们页面' (#25) from devin/1782277231-contact-us into main
CI / Go (api) (push) Failing after 32s
CI / Python (ingestion) (push) Failing after 31s
CI / Migrations (postgres) (push) Successful in 35s
2026-06-24 13:02:07 +08:00
rosemariejebbjtxbfp d837dd38bf feat(public-frontend): 新增联系我们页面
CI / Go (api) (pull_request) Successful in 56s
CI / Python (ingestion) (pull_request) Failing after 1m31s
CI / Migrations (postgres) (pull_request) Failing after 32s
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-06-24 05:00:31 +00:00
lixu 53ce705572 Merge pull request '移除首页 API 调用说明入口' (#23) from devin/1782276245-home-remove-api-cta into main
CI / Go (api) (push) Failing after 16m4s
CI / Python (ingestion) (push) Failing after 17s
CI / Migrations (postgres) (push) Successful in 29s
2026-06-24 12:46:03 +08:00
6 changed files with 299 additions and 15 deletions
+45 -13
View File
@@ -5,6 +5,10 @@ on:
branches: [main]
pull_request:
env:
GO_VERSION: "1.23.12"
GOPROXY: "https://goproxy.cn,direct"
jobs:
go:
name: Go (api)
@@ -25,11 +29,22 @@ jobs:
env:
OPENGOODS_DATABASE_URL: postgres://opengoods:opengoods@postgres:5432/opengoods?sslmode=disable
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: "1.23"
cache-dependency-path: api/go.sum
- name: Checkout
working-directory: ${{ github.workspace }}
run: |
git config --global --add safe.directory '*'
git init -q .
git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
git -c protocol.version=2 fetch -q --no-tags --depth 1 origin "${GITHUB_REF}"
git checkout -q --force FETCH_HEAD
- name: Setup Go
working-directory: ${{ github.workspace }}
run: |
curl -fsSL -o /tmp/go.tgz "https://mirrors.aliyun.com/golang/go${GO_VERSION}.linux-amd64.tar.gz"
rm -rf /usr/local/go
tar -C /usr/local -xzf /tmp/go.tgz
echo "/usr/local/go/bin" >> "$GITHUB_PATH"
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Apply migrations
working-directory: .
run: |
@@ -44,14 +59,20 @@ jobs:
python:
name: Python (ingestion)
runs-on: ubuntu-latest
container:
image: nikolaik/python-nodejs:python3.12-nodejs20
defaults:
run:
working-directory: ingestion
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Checkout
working-directory: ${{ github.workspace }}
run: |
git config --global --add safe.directory '*'
git init -q .
git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
git -c protocol.version=2 fetch -q --no-tags --depth 1 origin "${GITHUB_REF}"
git checkout -q --force FETCH_HEAD
- name: Install
run: pip install -e ".[dev]"
- name: Ruff lint
@@ -77,10 +98,21 @@ jobs:
env:
DBURL: postgres://opengoods:opengoods@postgres:5432/opengoods?sslmode=disable
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: "1.23"
- name: Checkout
working-directory: ${{ github.workspace }}
run: |
git config --global --add safe.directory '*'
git init -q .
git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
git -c protocol.version=2 fetch -q --no-tags --depth 1 origin "${GITHUB_REF}"
git checkout -q --force FETCH_HEAD
- name: Setup Go
run: |
curl -fsSL -o /tmp/go.tgz "https://mirrors.aliyun.com/golang/go${GO_VERSION}.linux-amd64.tar.gz"
rm -rf /usr/local/go
tar -C /usr/local -xzf /tmp/go.tgz
echo "/usr/local/go/bin" >> "$GITHUB_PATH"
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Install golang-migrate
run: go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.18.1
- name: Migrate up
+112
View File
@@ -0,0 +1,112 @@
# 可扩展性设计与路线图 (Scalability Roadmap)
本文档回答一个长期问题:随着商品越来越多、品类越来越杂(食品 / 电子 3C / 药品 /
……),**检索与新增会不会压垮数据库?要不要按品类「分表」?**
> 结论先行:**现阶段不要按品类手动分表。** 现有「单表 + JSONB + archive_kind 框架」
> 的设计方向是对的。扩展应当靠 **分区(非分表) + 读副本 + 缓存 + 专用搜索引擎**,
> 按数据量分阶段推进,避免提前过度设计。
---
## 1. 现状盘点
### 1.1 数据模型
- 所有商品落在**一张 `product` 表**;非食品的领域字段存 `product.attributes`(JSONB)。
- 食品有独立明细表 `food_detail`(配料 / 营养 / 过敏原等结构化字段)。
- `0010_archive_kinds` 引入 **archive_kind 框架**:每个品类带一个 `archive_kind`
(`food` / `electronics` / `generic`),`kind_field` 表按 kind 定义字段模板,
驱动后台动态表单与合格度评分。
- **加新品类(如药品)不需要新建表**:只要新增一组 `kind_field` 行 + 一棵品类子树;
仅当某品类有大量需被独立筛选/排序的结构化字段时,才考虑像 `food_detail` 那样补一张
明细表。
### 1.2 已有索引(检索性能的基础)
| 对象 | 索引 | 用途 |
|------|------|------|
| `product.gtin` | 唯一索引 | 条码精确查 |
| `product.name` | trigram GIN (`pg_trgm`) | 名称模糊/相似匹配 |
| `product.search_tsv` | 全文 GIN (`tsvector`) | 全文检索 |
| `product.attributes` | JSONB GIN | 属性过滤 |
| `brand.name` | trigram GIN | 品牌模糊匹配 |
| `product.country_of_origin` | btree | 产地精确/前缀过滤 |
| category / brand / updated_at | btree | 关联与增量 |
### 1.3 检索方式
- 有关键词时:`name ILIKE` `word_similarity ≥ 阈值(~0.42)` 条码匹配,
排序按 `相似度 × (0.5 + quality_score)`
- 无关键词时:按 `quality_score` 排序。
- 翻页:`LIMIT / OFFSET`
### 1.4 写入特征
- 写库**只有** Python ingestion 一条路径(批量 ETL),不是高并发 OLTP。
- 插入瓶颈极低;`search_tsv` 由触发器逐行重算,正常增量下开销可忽略。
---
## 2. 为什么不建议按品类「分表」
1. **核心场景是全局检索**:用户通常不知道商品属于哪个品类,搜索要跨所有品类。
按品类拆成多表后,一次搜索得 `UNION ALL` 所有表,**更慢、代码更复杂、排序更难统一**。
2. **单表足够能打**:Postgres 单表配好索引,**几千万行**量级的检索完全可承载。
"表大"很少是真正瓶颈,"搜索方式"和"读并发"才是。
3. **分表会侵蚀框架优势**:archive_kind 框架的价值就是"加品类零建表";手动分表等于
把这套通用能力又拆碎。
> 区分两个概念:**分表(sharding,应用层拆多张表)** ≠ **分区(Postgres 原生
> declarative partitioning,对上层透明的一张逻辑表)**。后者在超大规模时才有意义,
> 见第 3 阶段。
---
## 3. 分阶段路线图(按数据量触发,不提前做)
### 阶段 0 — 现在 ~ 数百万条:维持现状 + 低成本优化
触发:当前规模。改动小、收益稳,建议尽早做:
- **深翻页改 keyset 分页**:`OFFSET` 越翻越慢(需扫描并丢弃前 N 行);改用
`WHERE (score, id) < (:last_score, :last_id)` 形式的游标分页。
- **部分索引**:绝大多数查询限定 `status='active'`,可建
`CREATE INDEX ... WHERE status='active'` 缩小索引、提速。
- **常用筛选复合索引**:如 `(category_id, quality_score DESC)`
`(archive_kind, quality_score DESC)` 配合域内列表。
- **Redis 缓存**(已在技术栈内):缓存热门搜索结果与商品详情,挡住重复读。
### 阶段 1 — 千万级以上:读扩展 + 调优
触发:单库读 QPS 升高、P99 变慢。
- **只读副本(read replica)**:本服务是**只读公益 API**,天然适合一主多从,
把检索/详情读流量分到副本,主库只承接 ingestion 写入。
- **索引与查询调优**:按慢查询日志补/删索引,`EXPLAIN ANALYZE` 校核计划。
- **(可选)Postgres 原生分区**:若多数检索"限定单一域"(只搜药品 / 只搜食品),
可按 `archive_kind`**LIST 分区**(对上层透明,仍是一张逻辑表)。主要利好
**维护**(分区级 vacuum / 归档)与**域内查询裁剪**;对真正的全局搜索帮助有限。
### 阶段 2 — 搜索相关性/规模成为痛点:引入专用搜索引擎
触发:`pg_trgm`/`tsvector` 在相关性排序、跨字段检索、规模上吃力。
- 把**检索**迁到专用倒排引擎:**OpenSearch / Meilisearch / Typesense**,
或 Postgres 内的 **ParadeDB(`pg_search`,BM25)**
- **Postgres 仍是唯一事实来源**;搜索引擎只做索引,由 ingestion 在写库后同步。
- 这才是"搜索量大"的正解,**比分表有效得多**。
---
## 4. 大批量导入的建议
- 海量初始化/回填用 `COPY` 而非逐行 `INSERT`
- 超大批量时可"先停建二级索引 → COPY → 重建索引",比边插边维护索引快得多。
- ETL 控制并发与批大小,避免与在线读争抢。
---
## 5. 药品档案怎么落地(回到最初的问题)
在上述设计下,加"药品"属于**阶段 0 的常规扩展**,不触动架构:
1. 新增 `drug``kind_field` 模板(批准文号 / 通用名 / 商品名 / 剂型 / 规格 /
生产企业 / OTC 分类 / 适应症 / 用法用量 / 不良反应 / 禁忌 / 注意事项 / 贮藏 /
有效期 等),`qualified` 标记关键字段参与合格度评分。
2. 加一棵药品品类子树,并把这些品类的 `archive_kind` 置为 `drug`
3. 仅当药品需要**被独立筛选/排序的强结构化字段**(如按批准文号精确查、按 OTC 分类
过滤)时,才考虑补一张 `drug_detail` 明细表;否则继续走 `attributes` JSONB。
---
## 6. 版本
- 本路线图随规模演进更新;任何落地改动需同步:迁移(SQL) + 本文档 +(涉及对外字段时)
`docs/data-contract.md` / `docs/openapi.yaml`
+2
View File
@@ -0,0 +1,2 @@
DROP INDEX IF EXISTS idx_product_active_category;
DROP INDEX IF EXISTS idx_product_active_quality;
+15
View File
@@ -0,0 +1,15 @@
-- 检索优化(可扩展性路线图 阶段0):为最常见的"浏览"路径补部分/复合索引。
-- 搜索查询恒带 WHERE status = 'active';无关键词时按 quality_score DESC, name 排序。
-- 现有索引无法同时满足"过滤 active + 按 quality_score/name 排序",深翻页时需要对全部
-- active 行排序。下面的部分复合索引让规划器直接走索引顺序扫描,省掉排序、加速深翻页与
-- count(*)。索引只覆盖 active 行,体积更小。
-- 默认浏览(无关键词、无品类):ORDER BY quality_score DESC, name
CREATE INDEX IF NOT EXISTS idx_product_active_quality
ON product (quality_score DESC, name)
WHERE status = 'active';
-- 品类内浏览:先按 category_id 收敛,再按 quality_score 排序
CREATE INDEX IF NOT EXISTS idx_product_active_category
ON product (category_id, quality_score DESC)
WHERE status = 'active';
+12 -2
View File
@@ -1,10 +1,11 @@
import { useEffect, useState } from "react";
import { Boxes, Search, PlusCircle, Code2, KeyRound } from "lucide-react";
import { Boxes, Search, PlusCircle, Code2, KeyRound, Headset } from "lucide-react";
import Home from "./components/Home";
import ProductView from "./components/ProductView";
import Contribute from "./components/Contribute";
import ApiDocs from "./components/ApiDocs";
import Account from "./components/Account";
import Contact from "./components/Contact";
import { api } from "./api";
type View =
@@ -12,13 +13,15 @@ type View =
| { name: "product"; id: string }
| { name: "contribute" }
| { name: "api" }
| { name: "account" };
| { name: "account" }
| { name: "contact" };
const NAV: { key: View["name"]; label: string; icon: typeof Search }[] = [
{ key: "home", label: "检索", icon: Search },
{ key: "contribute", label: "贡献档案", icon: PlusCircle },
{ key: "api", label: "API", icon: Code2 },
{ key: "account", label: "API 密钥", icon: KeyRound },
{ key: "contact", label: "联系我们", icon: Headset },
];
export default function App() {
@@ -88,6 +91,7 @@ export default function App() {
)}
{view.name === "api" && <ApiDocs onRegister={() => setView({ name: "account" })} />}
{view.name === "account" && <Account />}
{view.name === "contact" && <Contact />}
</main>
<footer className="border-t border-gray-200/70 bg-white/60">
@@ -109,6 +113,12 @@ export default function App() {
>
API
</button>
<button
onClick={() => setView({ name: "contact" })}
className="ml-1 text-brand-600 font-medium hover:underline"
>
</button>
</p>
<div className="mt-3">
<a
+113
View File
@@ -0,0 +1,113 @@
import { useState } from "react";
import { Phone, Mail, Globe, MessageSquare, Copy, Check, Headset } from "lucide-react";
type Channel = {
key: string;
icon: typeof Phone;
label: string;
value: string;
href?: string;
copy: string;
};
const CHANNELS: Channel[] = [
{
key: "phone",
icon: Phone,
label: "电话",
value: "188 6595 7520",
href: "tel:18865957520",
copy: "18865957520",
},
{
key: "email",
icon: Mail,
label: "邮箱",
value: "1115084741@qq.com",
href: "mailto:1115084741@qq.com",
copy: "1115084741@qq.com",
},
{
key: "site",
icon: Globe,
label: "官网",
value: "www.wenyaoyu.com",
href: "https://www.wenyaoyu.com",
copy: "https://www.wenyaoyu.com",
},
{
key: "wechat",
icon: MessageSquare,
label: "微信",
value: "s-b-m-y",
copy: "s-b-m-y",
},
];
export default function Contact() {
const [copied, setCopied] = useState<string | null>(null);
async function copy(channel: Channel) {
try {
await navigator.clipboard.writeText(channel.copy);
setCopied(channel.key);
setTimeout(() => setCopied((k) => (k === channel.key ? null : k)), 1500);
} catch {
/* clipboard unavailable */
}
}
return (
<div className="max-w-3xl mx-auto animate-fade-up">
<div className="text-center">
<span className="mx-auto grid h-14 w-14 place-items-center rounded-2xl bg-gradient-to-br from-brand-500 to-brand-700 text-white shadow-glow">
<Headset className="w-7 h-7" />
</span>
<h1 className="mt-4 text-2xl font-semibold tracking-tight text-gray-900"></h1>
<p className="mx-auto mt-2 max-w-xl text-sm text-gray-500">
</p>
</div>
<div className="mt-8 grid grid-cols-1 sm:grid-cols-2 gap-4">
{CHANNELS.map((c) => {
const Icon = c.icon;
return (
<div key={c.key} className="card p-5 flex items-center gap-4">
<span className="grid h-11 w-11 shrink-0 place-items-center rounded-xl bg-brand-50 text-brand-600">
<Icon className="w-5 h-5" />
</span>
<div className="min-w-0 flex-1">
<div className="text-xs font-medium text-gray-400">{c.label}</div>
{c.href ? (
<a
href={c.href}
target={c.key === "site" ? "_blank" : undefined}
rel={c.key === "site" ? "noreferrer" : undefined}
className="block truncate font-medium text-gray-800 hover:text-brand-600 hover:underline"
>
{c.value}
</a>
) : (
<div className="truncate font-medium text-gray-800">{c.value}</div>
)}
</div>
<button
type="button"
onClick={() => copy(c)}
title="复制"
className="shrink-0 grid h-9 w-9 place-items-center rounded-lg border border-gray-200 text-gray-400 transition hover:bg-gray-50 hover:text-brand-600"
>
{copied === c.key ? (
<Check className="w-4 h-4 text-brand-600" />
) : (
<Copy className="w-4 h-4" />
)}
</button>
</div>
);
})}
</div>
</div>
);
}