Compare commits
14 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a75318b811 | |||
| 20b9eb0fc9 | |||
| 17bc0ed680 | |||
| 5020fdcc19 | |||
| 885f6c2b01 | |||
| 2aff6287d2 | |||
| 3364a50c68 | |||
| 936828abae | |||
| ab7a964934 | |||
| 649fcc711c | |||
| 3b62f61288 | |||
| ba36e3ff4c | |||
| d837dd38bf | |||
| 53ce705572 |
+45
-13
@@ -5,6 +5,10 @@ on:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
|
||||
env:
|
||||
GO_VERSION: "1.23.12"
|
||||
GOPROXY: "https://goproxy.cn,direct"
|
||||
|
||||
jobs:
|
||||
go:
|
||||
name: Go (api)
|
||||
@@ -25,11 +29,22 @@ jobs:
|
||||
env:
|
||||
OPENGOODS_DATABASE_URL: postgres://opengoods:opengoods@postgres:5432/opengoods?sslmode=disable
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.23"
|
||||
cache-dependency-path: api/go.sum
|
||||
- name: Checkout
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
git config --global --add safe.directory '*'
|
||||
git init -q .
|
||||
git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
|
||||
git -c protocol.version=2 fetch -q --no-tags --depth 1 origin "${GITHUB_REF}"
|
||||
git checkout -q --force FETCH_HEAD
|
||||
- name: Setup Go
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
curl -fsSL -o /tmp/go.tgz "https://mirrors.aliyun.com/golang/go${GO_VERSION}.linux-amd64.tar.gz"
|
||||
rm -rf /usr/local/go
|
||||
tar -C /usr/local -xzf /tmp/go.tgz
|
||||
echo "/usr/local/go/bin" >> "$GITHUB_PATH"
|
||||
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
||||
- name: Apply migrations
|
||||
working-directory: .
|
||||
run: |
|
||||
@@ -44,14 +59,20 @@ jobs:
|
||||
python:
|
||||
name: Python (ingestion)
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: nikolaik/python-nodejs:python3.12-nodejs20
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ingestion
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Checkout
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
git config --global --add safe.directory '*'
|
||||
git init -q .
|
||||
git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
|
||||
git -c protocol.version=2 fetch -q --no-tags --depth 1 origin "${GITHUB_REF}"
|
||||
git checkout -q --force FETCH_HEAD
|
||||
- name: Install
|
||||
run: pip install -e ".[dev]"
|
||||
- name: Ruff lint
|
||||
@@ -77,10 +98,21 @@ jobs:
|
||||
env:
|
||||
DBURL: postgres://opengoods:opengoods@postgres:5432/opengoods?sslmode=disable
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.23"
|
||||
- name: Checkout
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
git config --global --add safe.directory '*'
|
||||
git init -q .
|
||||
git remote add origin "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
|
||||
git -c protocol.version=2 fetch -q --no-tags --depth 1 origin "${GITHUB_REF}"
|
||||
git checkout -q --force FETCH_HEAD
|
||||
- name: Setup Go
|
||||
run: |
|
||||
curl -fsSL -o /tmp/go.tgz "https://mirrors.aliyun.com/golang/go${GO_VERSION}.linux-amd64.tar.gz"
|
||||
rm -rf /usr/local/go
|
||||
tar -C /usr/local -xzf /tmp/go.tgz
|
||||
echo "/usr/local/go/bin" >> "$GITHUB_PATH"
|
||||
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
||||
- name: Install golang-migrate
|
||||
run: go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.18.1
|
||||
- name: Migrate up
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/baicai2026-baicai/goods/api/internal/cache"
|
||||
"github.com/baicai2026-baicai/goods/api/internal/config"
|
||||
"github.com/baicai2026-baicai/goods/api/internal/handler"
|
||||
"github.com/baicai2026-baicai/goods/api/internal/publicweb"
|
||||
@@ -36,7 +37,12 @@ func main() {
|
||||
if !limiter.Enabled() {
|
||||
log.Print("warning: Redis not configured; public API rate limiting disabled")
|
||||
}
|
||||
h := handler.New(store.New(pool), publicweb.Dist()).
|
||||
|
||||
readCache := cache.New(cfg.RedisURL)
|
||||
if !readCache.Enabled() {
|
||||
log.Print("warning: Redis not configured; public API read cache disabled")
|
||||
}
|
||||
h := handler.New(store.New(pool).WithCache(readCache), publicweb.Dist()).
|
||||
WithRateLimit(limiter, cfg.AnonRateLimitPerMin).
|
||||
WithQuotas(cfg.AnonTotalQuota, cfg.RegisteredRateLimitPerMin, cfg.RegisteredQuotaTotal)
|
||||
|
||||
|
||||
Vendored
+126
@@ -0,0 +1,126 @@
|
||||
// Package cache is a Redis-backed, fail-open read cache for the public API.
|
||||
//
|
||||
// It caches hot product details and search results so repeated reads avoid
|
||||
// PostgreSQL. Like the ratelimit package, every operation fails open: if Redis
|
||||
// is unavailable or misconfigured the caller simply falls back to the database,
|
||||
// so the cache can never take the API down or serve stale data after Redis loss.
|
||||
//
|
||||
// Invalidation is global and O(1): keys are namespaced by an epoch counter
|
||||
// stored in Redis (og:cache:epoch). The Python ingestion bumps that counter
|
||||
// after a write run, which logically invalidates every cached entry at once
|
||||
// while old keys age out via their TTL. The epoch is read at most once per
|
||||
// refresh interval per process, so it adds no per-request round trip.
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// epochKey is the Redis key holding the global cache generation counter.
|
||||
const epochKey = "og:cache:epoch"
|
||||
|
||||
// epochRefresh bounds how often a process re-reads the epoch from Redis.
|
||||
const epochRefresh = 10 * time.Second
|
||||
|
||||
// opTimeout caps any single Redis operation so a slow backend never blocks a
|
||||
// request beyond this; on timeout the cache fails open.
|
||||
const opTimeout = 150 * time.Millisecond
|
||||
|
||||
// Cache wraps a Redis client. A nil-backed Cache (Redis unconfigured) disables
|
||||
// caching: every Get misses and every Set is a no-op.
|
||||
type Cache struct {
|
||||
rdb *redis.Client
|
||||
|
||||
mu sync.RWMutex
|
||||
epoch int64
|
||||
epochSetAt time.Time
|
||||
epochOK bool
|
||||
}
|
||||
|
||||
// New builds a Cache from a redis:// URL. On a parse error it logs and returns a
|
||||
// disabled (fail-open) cache so the server still boots.
|
||||
func New(redisURL string) *Cache {
|
||||
opt, err := redis.ParseURL(redisURL)
|
||||
if err != nil {
|
||||
log.Printf("cache: invalid redis url %q: %v (caching disabled)", redisURL, err)
|
||||
return &Cache{}
|
||||
}
|
||||
return &Cache{rdb: redis.NewClient(opt)}
|
||||
}
|
||||
|
||||
// Enabled reports whether a Redis backend is configured.
|
||||
func (c *Cache) Enabled() bool { return c != nil && c.rdb != nil }
|
||||
|
||||
// epochNow returns the current cache generation, reading it from Redis at most
|
||||
// once per epochRefresh. On any Redis error it keeps the last known value and
|
||||
// throttles re-reads so a down backend cannot slow the hot path.
|
||||
func (c *Cache) epochNow(ctx context.Context) int64 {
|
||||
c.mu.RLock()
|
||||
if c.epochOK && time.Since(c.epochSetAt) < epochRefresh {
|
||||
e := c.epoch
|
||||
c.mu.RUnlock()
|
||||
return e
|
||||
}
|
||||
c.mu.RUnlock()
|
||||
|
||||
cctx, cancel := context.WithTimeout(ctx, opTimeout)
|
||||
defer cancel()
|
||||
n, err := c.rdb.Get(cctx, epochKey).Int64()
|
||||
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
switch {
|
||||
case err == nil:
|
||||
c.epoch = n
|
||||
case errors.Is(err, redis.Nil):
|
||||
c.epoch = 0
|
||||
}
|
||||
c.epochSetAt = time.Now()
|
||||
c.epochOK = true
|
||||
return c.epoch
|
||||
}
|
||||
|
||||
// key namespaces a logical suffix under the current epoch.
|
||||
func (c *Cache) key(ctx context.Context, suffix string) string {
|
||||
return "og:v" + strconv.FormatInt(c.epochNow(ctx), 10) + ":" + suffix
|
||||
}
|
||||
|
||||
// GetJSON unmarshals the cached value for suffix into dest and reports a hit.
|
||||
// Any miss, decode error, or Redis error returns false (fail-open).
|
||||
func (c *Cache) GetJSON(ctx context.Context, suffix string, dest any) bool {
|
||||
if !c.Enabled() {
|
||||
return false
|
||||
}
|
||||
k := c.key(ctx, suffix)
|
||||
cctx, cancel := context.WithTimeout(ctx, opTimeout)
|
||||
defer cancel()
|
||||
b, err := c.rdb.Get(cctx, k).Bytes()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return json.Unmarshal(b, dest) == nil
|
||||
}
|
||||
|
||||
// SetJSON stores val (JSON-encoded) for suffix with the given TTL. Best effort:
|
||||
// marshal or Redis errors are ignored.
|
||||
func (c *Cache) SetJSON(ctx context.Context, suffix string, val any, ttl time.Duration) {
|
||||
if !c.Enabled() {
|
||||
return
|
||||
}
|
||||
b, err := json.Marshal(val)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
k := c.key(ctx, suffix)
|
||||
cctx, cancel := context.WithTimeout(ctx, opTimeout)
|
||||
defer cancel()
|
||||
_ = c.rdb.Set(cctx, k, b, ttl).Err()
|
||||
}
|
||||
Vendored
+84
@@ -0,0 +1,84 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestDisabledFailsOpen verifies a Cache without a Redis backend never panics,
|
||||
// always misses, and silently drops writes.
|
||||
func TestDisabledFailsOpen(t *testing.T) {
|
||||
c := New("not-a-valid-url") // parse error => disabled
|
||||
if c.Enabled() {
|
||||
t.Fatal("expected cache to be disabled for invalid url")
|
||||
}
|
||||
c.SetJSON(context.Background(), "k", map[string]int{"a": 1}, time.Minute)
|
||||
var dst map[string]int
|
||||
if c.GetJSON(context.Background(), "k", &dst) {
|
||||
t.Fatalf("disabled cache must always miss, got %+v", dst)
|
||||
}
|
||||
}
|
||||
|
||||
func testCache(t *testing.T) *Cache {
|
||||
t.Helper()
|
||||
url := os.Getenv("OPENGOODS_REDIS_URL")
|
||||
if url == "" {
|
||||
url = "redis://localhost:6379/0"
|
||||
}
|
||||
c := New(url)
|
||||
if !c.Enabled() {
|
||||
t.Skip("redis not configured")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if err := c.rdb.Ping(ctx).Err(); err != nil {
|
||||
t.Skipf("redis not reachable: %v", err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// TestRoundTrip stores then reads a value back.
|
||||
func TestRoundTrip(t *testing.T) {
|
||||
c := testCache(t)
|
||||
ctx := context.Background()
|
||||
suffix := fmt.Sprintf("test:rt:%d", time.Now().UnixNano())
|
||||
|
||||
c.SetJSON(ctx, suffix, map[string]any{"name": "foo", "n": float64(3)}, time.Minute)
|
||||
got := map[string]any{}
|
||||
if !c.GetJSON(ctx, suffix, &got) {
|
||||
t.Fatal("expected cache hit after set")
|
||||
}
|
||||
if got["name"] != "foo" || got["n"] != float64(3) {
|
||||
t.Fatalf("unexpected payload: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEpochInvalidation verifies that bumping the epoch counter logically drops
|
||||
// every previously cached entry.
|
||||
func TestEpochInvalidation(t *testing.T) {
|
||||
c := testCache(t)
|
||||
ctx := context.Background()
|
||||
suffix := fmt.Sprintf("test:epoch:%d", time.Now().UnixNano())
|
||||
|
||||
c.SetJSON(ctx, suffix, map[string]int{"v": 1}, time.Minute)
|
||||
var dst map[string]int
|
||||
if !c.GetJSON(ctx, suffix, &dst) {
|
||||
t.Fatal("expected hit before epoch bump")
|
||||
}
|
||||
|
||||
// Simulate an ingestion write bumping the global epoch.
|
||||
if err := c.rdb.Incr(ctx, epochKey).Err(); err != nil {
|
||||
t.Fatalf("incr epoch: %v", err)
|
||||
}
|
||||
// Force the process to re-read the epoch rather than use its cached value.
|
||||
c.mu.Lock()
|
||||
c.epochOK = false
|
||||
c.mu.Unlock()
|
||||
|
||||
if c.GetJSON(ctx, suffix, &dst) {
|
||||
t.Fatal("entry should be invisible after epoch bump")
|
||||
}
|
||||
}
|
||||
@@ -4,21 +4,35 @@ package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/baicai2026-baicai/goods/api/internal/cache"
|
||||
)
|
||||
|
||||
// Cache TTLs for the public read cache. Product details are far less volatile
|
||||
// than search result sets, so they live longer; both are also invalidated
|
||||
// wholesale whenever ingestion bumps the cache epoch.
|
||||
const (
|
||||
productCacheTTL = 24 * time.Hour
|
||||
searchCacheTTL = time.Hour
|
||||
)
|
||||
|
||||
// ErrNotFound is returned when a requested row does not exist.
|
||||
var ErrNotFound = errors.New("not found")
|
||||
|
||||
// Store wraps a PostgreSQL connection pool.
|
||||
// Store wraps a PostgreSQL connection pool and an optional read cache.
|
||||
type Store struct {
|
||||
pool *pgxpool.Pool
|
||||
pool *pgxpool.Pool
|
||||
cache *cache.Cache
|
||||
}
|
||||
|
||||
// New constructs a Store from an existing pgx pool.
|
||||
@@ -26,6 +40,30 @@ func New(pool *pgxpool.Pool) *Store {
|
||||
return &Store{pool: pool}
|
||||
}
|
||||
|
||||
// WithCache attaches a Redis-backed read cache. A nil or disabled cache leaves
|
||||
// the Store reading straight from PostgreSQL.
|
||||
func (s *Store) WithCache(c *cache.Cache) *Store {
|
||||
s.cache = c
|
||||
return s
|
||||
}
|
||||
|
||||
// cacheGet reads a cached JSON value into dest, reporting a hit. It is a no-op
|
||||
// miss when no cache is attached.
|
||||
func (s *Store) cacheGet(ctx context.Context, suffix string, dest any) bool {
|
||||
if s.cache == nil {
|
||||
return false
|
||||
}
|
||||
return s.cache.GetJSON(ctx, suffix, dest)
|
||||
}
|
||||
|
||||
// cacheSet stores a JSON value when a cache is attached.
|
||||
func (s *Store) cacheSet(ctx context.Context, suffix string, val any, ttl time.Duration) {
|
||||
if s.cache == nil {
|
||||
return
|
||||
}
|
||||
s.cache.SetJSON(ctx, suffix, val, ttl)
|
||||
}
|
||||
|
||||
// Ping verifies database connectivity.
|
||||
func (s *Store) Ping(ctx context.Context) error {
|
||||
return s.pool.Ping(ctx)
|
||||
@@ -222,6 +260,10 @@ func stringifyAttr(v any) string {
|
||||
|
||||
// ProductByGTIN looks up an active product by any of its barcodes.
|
||||
func (s *Store) ProductByGTIN(ctx context.Context, gtin string) (*Product, error) {
|
||||
const suffix = "prod:gtin:"
|
||||
if cached := new(Product); s.cacheGet(ctx, suffix+gtin, cached) {
|
||||
return cached, nil
|
||||
}
|
||||
row := s.pool.QueryRow(ctx, productSelect+`
|
||||
WHERE p.status = 'active'
|
||||
AND (p.gtin = $1 OR EXISTS (
|
||||
@@ -238,11 +280,16 @@ func (s *Store) ProductByGTIN(ctx context.Context, gtin string) (*Product, error
|
||||
if p.Specs, err = s.buildSpecs(ctx, p.ArchiveKind, attrs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.cacheSet(ctx, suffix+gtin, p, productCacheTTL)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// ProductByID looks up a product by its UUID.
|
||||
func (s *Store) ProductByID(ctx context.Context, id string) (*Product, error) {
|
||||
const suffix = "prod:id:"
|
||||
if cached := new(Product); s.cacheGet(ctx, suffix+id, cached) {
|
||||
return cached, nil
|
||||
}
|
||||
row := s.pool.QueryRow(ctx, productSelect+" WHERE p.id = $1", id)
|
||||
p, attrs, err := scanProduct(row)
|
||||
if err != nil {
|
||||
@@ -254,6 +301,7 @@ func (s *Store) ProductByID(ctx context.Context, id string) (*Product, error) {
|
||||
if p.Specs, err = s.buildSpecs(ctx, p.ArchiveKind, attrs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.cacheSet(ctx, suffix+id, p, productCacheTTL)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
@@ -268,6 +316,11 @@ const fuzzyThreshold = "0.42"
|
||||
// similarity blended with quality_score so the best, most-complete records
|
||||
// surface first. Without a query, results are ordered by quality_score.
|
||||
func (s *Store) SearchProducts(ctx context.Context, f SearchFilters, limit, offset int) ([]ProductSummary, int, error) {
|
||||
suffix := searchCacheSuffix(f, limit, offset)
|
||||
if entry := new(searchCacheEntry); s.cacheGet(ctx, suffix, entry) {
|
||||
return entry.Items, entry.Total, nil
|
||||
}
|
||||
|
||||
args := []any{}
|
||||
where := "WHERE p.status = 'active'"
|
||||
|
||||
@@ -334,7 +387,28 @@ LEFT JOIN category c ON c.id = p.category_id `
|
||||
}
|
||||
out = append(out, ps)
|
||||
}
|
||||
return out, total, rows.Err()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
s.cacheSet(ctx, suffix, searchCacheEntry{Items: out, Total: total}, searchCacheTTL)
|
||||
return out, total, nil
|
||||
}
|
||||
|
||||
// searchCacheEntry is the cached payload for a SearchProducts call.
|
||||
type searchCacheEntry struct {
|
||||
Items []ProductSummary `json:"items"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
// searchCacheSuffix derives a stable cache key from the full filter set and
|
||||
// paging window so distinct queries never collide.
|
||||
func searchCacheSuffix(f SearchFilters, limit, offset int) string {
|
||||
raw := strings.Join([]string{
|
||||
f.Query, f.Category, f.Brand, f.Country,
|
||||
strconv.Itoa(limit), strconv.Itoa(offset),
|
||||
}, "\x1f")
|
||||
sum := sha1.Sum([]byte(raw))
|
||||
return "search:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Nutriments returns just the nutrition payload for a product.
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
# 可扩展性设计与路线图 (Scalability Roadmap)
|
||||
|
||||
本文档回答一个长期问题:随着商品越来越多、品类越来越杂(食品 / 电子 3C / 药品 /
|
||||
……),**检索与新增会不会压垮数据库?要不要按品类「分表」?**
|
||||
|
||||
> 结论先行:**现阶段不要按品类手动分表。** 现有「单表 + JSONB + archive_kind 框架」
|
||||
> 的设计方向是对的。扩展应当靠 **分区(非分表) + 读副本 + 缓存 + 专用搜索引擎**,
|
||||
> 按数据量分阶段推进,避免提前过度设计。
|
||||
|
||||
---
|
||||
|
||||
## 1. 现状盘点
|
||||
|
||||
### 1.1 数据模型
|
||||
- 所有商品落在**一张 `product` 表**;非食品的领域字段存 `product.attributes`(JSONB)。
|
||||
- 食品有独立明细表 `food_detail`(配料 / 营养 / 过敏原等结构化字段)。
|
||||
- `0010_archive_kinds` 引入 **archive_kind 框架**:每个品类带一个 `archive_kind`
|
||||
(`food` / `electronics` / `generic`),`kind_field` 表按 kind 定义字段模板,
|
||||
驱动后台动态表单与合格度评分。
|
||||
- **加新品类(如药品)不需要新建表**:只要新增一组 `kind_field` 行 + 一棵品类子树;
|
||||
仅当某品类有大量需被独立筛选/排序的结构化字段时,才考虑像 `food_detail` 那样补一张
|
||||
明细表。
|
||||
|
||||
### 1.2 已有索引(检索性能的基础)
|
||||
| 对象 | 索引 | 用途 |
|
||||
|------|------|------|
|
||||
| `product.gtin` | 唯一索引 | 条码精确查 |
|
||||
| `product.name` | trigram GIN (`pg_trgm`) | 名称模糊/相似匹配 |
|
||||
| `product.search_tsv` | 全文 GIN (`tsvector`) | 全文检索 |
|
||||
| `product.attributes` | JSONB GIN | 属性过滤 |
|
||||
| `brand.name` | trigram GIN | 品牌模糊匹配 |
|
||||
| `product.country_of_origin` | btree | 产地精确/前缀过滤 |
|
||||
| category / brand / updated_at | btree | 关联与增量 |
|
||||
|
||||
### 1.3 检索方式
|
||||
- 有关键词时:`name ILIKE` ∪ `word_similarity ≥ 阈值(~0.42)` ∪ 条码匹配,
|
||||
排序按 `相似度 × (0.5 + quality_score)`。
|
||||
- 无关键词时:按 `quality_score` 排序。
|
||||
- 翻页:`LIMIT / OFFSET`。
|
||||
|
||||
### 1.4 写入特征
|
||||
- 写库**只有** Python ingestion 一条路径(批量 ETL),不是高并发 OLTP。
|
||||
- 插入瓶颈极低;`search_tsv` 由触发器逐行重算,正常增量下开销可忽略。
|
||||
|
||||
---
|
||||
|
||||
## 2. 为什么不建议按品类「分表」
|
||||
|
||||
1. **核心场景是全局检索**:用户通常不知道商品属于哪个品类,搜索要跨所有品类。
|
||||
按品类拆成多表后,一次搜索得 `UNION ALL` 所有表,**更慢、代码更复杂、排序更难统一**。
|
||||
2. **单表足够能打**:Postgres 单表配好索引,**几千万行**量级的检索完全可承载。
|
||||
"表大"很少是真正瓶颈,"搜索方式"和"读并发"才是。
|
||||
3. **分表会侵蚀框架优势**:archive_kind 框架的价值就是"加品类零建表";手动分表等于
|
||||
把这套通用能力又拆碎。
|
||||
|
||||
> 区分两个概念:**分表(sharding,应用层拆多张表)** ≠ **分区(Postgres 原生
|
||||
> declarative partitioning,对上层透明的一张逻辑表)**。后者在超大规模时才有意义,
|
||||
> 见第 3 阶段。
|
||||
|
||||
---
|
||||
|
||||
## 3. 分阶段路线图(按数据量触发,不提前做)
|
||||
|
||||
### 阶段 0 — 现在 ~ 数百万条:维持现状 + 低成本优化
|
||||
触发:当前规模。改动小、收益稳,建议尽早做:
|
||||
- **深翻页改 keyset 分页**:`OFFSET` 越翻越慢(需扫描并丢弃前 N 行);改用
|
||||
`WHERE (score, id) < (:last_score, :last_id)` 形式的游标分页。
|
||||
- **部分索引**:绝大多数查询限定 `status='active'`,可建
|
||||
`CREATE INDEX ... WHERE status='active'` 缩小索引、提速。
|
||||
- **常用筛选复合索引**:如 `(category_id, quality_score DESC)`、
|
||||
`(archive_kind, quality_score DESC)` 配合域内列表。
|
||||
- **Redis 缓存**(已在技术栈内):缓存热门搜索结果与商品详情,挡住重复读。
|
||||
|
||||
### 阶段 1 — 千万级以上:读扩展 + 调优
|
||||
触发:单库读 QPS 升高、P99 变慢。
|
||||
- **只读副本(read replica)**:本服务是**只读公益 API**,天然适合一主多从,
|
||||
把检索/详情读流量分到副本,主库只承接 ingestion 写入。
|
||||
- **索引与查询调优**:按慢查询日志补/删索引,`EXPLAIN ANALYZE` 校核计划。
|
||||
- **(可选)Postgres 原生分区**:若多数检索"限定单一域"(只搜药品 / 只搜食品),
|
||||
可按 `archive_kind` 做 **LIST 分区**(对上层透明,仍是一张逻辑表)。主要利好
|
||||
**维护**(分区级 vacuum / 归档)与**域内查询裁剪**;对真正的全局搜索帮助有限。
|
||||
|
||||
### 阶段 2 — 搜索相关性/规模成为痛点:引入专用搜索引擎
|
||||
触发:`pg_trgm`/`tsvector` 在相关性排序、跨字段检索、规模上吃力。
|
||||
- 把**检索**迁到专用倒排引擎:**OpenSearch / Meilisearch / Typesense**,
|
||||
或 Postgres 内的 **ParadeDB(`pg_search`,BM25)**。
|
||||
- **Postgres 仍是唯一事实来源**;搜索引擎只做索引,由 ingestion 在写库后同步。
|
||||
- 这才是"搜索量大"的正解,**比分表有效得多**。
|
||||
|
||||
---
|
||||
|
||||
## 4. 大批量导入的建议
|
||||
- 海量初始化/回填用 `COPY` 而非逐行 `INSERT`。
|
||||
- 超大批量时可"先停建二级索引 → COPY → 重建索引",比边插边维护索引快得多。
|
||||
- ETL 控制并发与批大小,避免与在线读争抢。
|
||||
|
||||
---
|
||||
|
||||
## 5. 药品档案怎么落地(回到最初的问题)
|
||||
在上述设计下,加"药品"属于**阶段 0 的常规扩展**,不触动架构:
|
||||
1. 新增 `drug` 的 `kind_field` 模板(批准文号 / 通用名 / 商品名 / 剂型 / 规格 /
|
||||
生产企业 / OTC 分类 / 适应症 / 用法用量 / 不良反应 / 禁忌 / 注意事项 / 贮藏 /
|
||||
有效期 等),`qualified` 标记关键字段参与合格度评分。
|
||||
2. 加一棵药品品类子树,并把这些品类的 `archive_kind` 置为 `drug`。
|
||||
3. 仅当药品需要**被独立筛选/排序的强结构化字段**(如按批准文号精确查、按 OTC 分类
|
||||
过滤)时,才考虑补一张 `drug_detail` 明细表;否则继续走 `attributes` JSONB。
|
||||
|
||||
---
|
||||
|
||||
## 6. 版本
|
||||
- 本路线图随规模演进更新;任何落地改动需同步:迁移(SQL) + 本文档 +(涉及对外字段时)
|
||||
`docs/data-contract.md` / `docs/openapi.yaml`。
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Read-cache invalidation signal for the public API.
|
||||
|
||||
The Go API caches hot product details and search results in Redis, namespacing
|
||||
every key by a global generation counter (``og:cache:epoch``). Bumping that
|
||||
counter logically invalidates the entire cache in one O(1) operation while the
|
||||
old keys age out via their TTL.
|
||||
|
||||
Ingestion is the only writer to the database, so after a run that changed data
|
||||
it calls :func:`bump_cache_epoch` to make those changes visible immediately
|
||||
instead of waiting for per-key TTLs to expire.
|
||||
|
||||
Like the API's cache, this is strictly best effort and fails open: if Redis is
|
||||
unconfigured or unreachable the ingestion run still succeeds, and stale entries
|
||||
simply expire on their own.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
EPOCH_KEY = "og:cache:epoch"
|
||||
|
||||
|
||||
def default_redis_url() -> str | None:
|
||||
"""Return the configured Redis URL, or ``None`` when caching is disabled."""
|
||||
return os.environ.get("OPENGOODS_REDIS_URL") or None
|
||||
|
||||
|
||||
def bump_cache_epoch(url: str | None = None) -> bool:
|
||||
"""Increment the API cache generation counter.
|
||||
|
||||
Returns ``True`` if the counter was bumped, ``False`` if caching is disabled
|
||||
or Redis was unreachable. Never raises: invalidation failures must not fail
|
||||
an ingestion run.
|
||||
"""
|
||||
url = url or default_redis_url()
|
||||
if not url:
|
||||
return False
|
||||
try:
|
||||
import redis # imported lazily so the dependency is optional at runtime
|
||||
|
||||
client = redis.Redis.from_url(url, socket_timeout=2, socket_connect_timeout=2)
|
||||
new_epoch = client.incr(EPOCH_KEY)
|
||||
client.close()
|
||||
logger.info("bumped API cache epoch to %s", new_epoch)
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001 - invalidation is best effort
|
||||
logger.warning("cache epoch bump skipped: %s", exc)
|
||||
return False
|
||||
@@ -12,6 +12,7 @@ import sys
|
||||
|
||||
import psycopg
|
||||
|
||||
from opengoods.cache import bump_cache_epoch
|
||||
from opengoods.etl.dedup import dedup_all
|
||||
from opengoods.etl.load import default_dsn
|
||||
|
||||
@@ -23,6 +24,8 @@ def run(args: argparse.Namespace) -> int:
|
||||
conn.rollback()
|
||||
else:
|
||||
conn.commit()
|
||||
if not args.dry_run and summary["merged"]:
|
||||
bump_cache_epoch()
|
||||
mode = "dry-run" if args.dry_run else "applied"
|
||||
print(f"{mode} groups={summary['groups']} merged={summary['merged']}")
|
||||
return 0
|
||||
|
||||
@@ -25,6 +25,7 @@ from collections.abc import Iterator
|
||||
import psycopg
|
||||
|
||||
from opengoods.adapters.bypos import transform_bypos
|
||||
from opengoods.cache import bump_cache_epoch
|
||||
from opengoods.etl.load import default_dsn, ensure_bypos_source, load_bypos_record_safe
|
||||
|
||||
|
||||
@@ -59,6 +60,8 @@ def run(args: argparse.Namespace) -> int:
|
||||
else:
|
||||
errored += 1
|
||||
conn.commit()
|
||||
if loaded:
|
||||
bump_cache_epoch()
|
||||
print(f"loaded={loaded} skipped={skipped} errored={errored}")
|
||||
return 0
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ from opengoods.adapters.openfoodfacts import (
|
||||
is_cn_gs1,
|
||||
read_dump,
|
||||
)
|
||||
from opengoods.cache import bump_cache_epoch
|
||||
from opengoods.etl.load import default_dsn, ensure_source, load_record_safe
|
||||
from opengoods.etl.transform import transform
|
||||
|
||||
@@ -67,6 +68,8 @@ def run(args: argparse.Namespace) -> int:
|
||||
else:
|
||||
errored += 1
|
||||
conn.commit()
|
||||
if loaded:
|
||||
bump_cache_epoch()
|
||||
print(f"loaded={loaded} skipped={skipped} errored={errored}")
|
||||
return 0
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ import sys
|
||||
import psycopg
|
||||
|
||||
from opengoods.adapters.openfoodfacts import SOURCE_NAME, OpenFoodFactsAdapter
|
||||
from opengoods.cache import bump_cache_epoch
|
||||
from opengoods.etl.load import default_dsn, ensure_source, load_record_safe
|
||||
from opengoods.etl.state import get_watermark, set_watermark
|
||||
from opengoods.etl.transform import transform
|
||||
@@ -49,6 +50,8 @@ def run(args: argparse.Namespace) -> int:
|
||||
stats={"loaded": loaded, "skipped": skipped, "errored": errored, "since": since},
|
||||
)
|
||||
conn.commit()
|
||||
if loaded:
|
||||
bump_cache_epoch()
|
||||
print(
|
||||
f"since={since} loaded={loaded} skipped={skipped} "
|
||||
f"errored={errored} watermark={high_watermark}"
|
||||
|
||||
@@ -6,6 +6,7 @@ requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"httpx>=0.27",
|
||||
"psycopg[binary]>=3.2",
|
||||
"redis>=5.0,<6",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Tests for the best-effort API cache invalidation signal."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
import opengoods.cache as cache
|
||||
|
||||
|
||||
def test_disabled_when_no_url(monkeypatch):
|
||||
monkeypatch.delenv("OPENGOODS_REDIS_URL", raising=False)
|
||||
assert cache.default_redis_url() is None
|
||||
assert cache.bump_cache_epoch() is False
|
||||
|
||||
|
||||
def test_bump_fails_open_when_unreachable(monkeypatch):
|
||||
# An unroutable URL must not raise; the run still succeeds.
|
||||
monkeypatch.setenv("OPENGOODS_REDIS_URL", "redis://127.0.0.1:1/0")
|
||||
assert cache.bump_cache_epoch() is False
|
||||
|
||||
|
||||
def test_bump_increments_epoch(monkeypatch):
|
||||
calls = {}
|
||||
|
||||
class FakeClient:
|
||||
def incr(self, key):
|
||||
calls["key"] = key
|
||||
return 7
|
||||
|
||||
def close(self):
|
||||
calls["closed"] = True
|
||||
|
||||
class FakeRedis:
|
||||
@staticmethod
|
||||
def from_url(url, **kwargs):
|
||||
calls["url"] = url
|
||||
return FakeClient()
|
||||
|
||||
monkeypatch.setitem(sys.modules, "redis", type("M", (), {"Redis": FakeRedis}))
|
||||
|
||||
assert cache.bump_cache_epoch("redis://example:6379/0") is True
|
||||
assert calls["key"] == "og:cache:epoch"
|
||||
assert calls["url"] == "redis://example:6379/0"
|
||||
assert calls["closed"] is True
|
||||
@@ -0,0 +1,2 @@
|
||||
DROP INDEX IF EXISTS idx_product_active_category;
|
||||
DROP INDEX IF EXISTS idx_product_active_quality;
|
||||
@@ -0,0 +1,15 @@
|
||||
-- 检索优化(可扩展性路线图 阶段0):为最常见的"浏览"路径补部分/复合索引。
|
||||
-- 搜索查询恒带 WHERE status = 'active';无关键词时按 quality_score DESC, name 排序。
|
||||
-- 现有索引无法同时满足"过滤 active + 按 quality_score/name 排序",深翻页时需要对全部
|
||||
-- active 行排序。下面的部分复合索引让规划器直接走索引顺序扫描,省掉排序、加速深翻页与
|
||||
-- count(*)。索引只覆盖 active 行,体积更小。
|
||||
|
||||
-- 默认浏览(无关键词、无品类):ORDER BY quality_score DESC, name
|
||||
CREATE INDEX IF NOT EXISTS idx_product_active_quality
|
||||
ON product (quality_score DESC, name)
|
||||
WHERE status = 'active';
|
||||
|
||||
-- 品类内浏览:先按 category_id 收敛,再按 quality_score 排序
|
||||
CREATE INDEX IF NOT EXISTS idx_product_active_category
|
||||
ON product (category_id, quality_score DESC)
|
||||
WHERE status = 'active';
|
||||
@@ -1,10 +1,11 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Boxes, Search, PlusCircle, Code2, KeyRound } from "lucide-react";
|
||||
import { Boxes, Search, PlusCircle, Code2, KeyRound, Headset } from "lucide-react";
|
||||
import Home from "./components/Home";
|
||||
import ProductView from "./components/ProductView";
|
||||
import Contribute from "./components/Contribute";
|
||||
import ApiDocs from "./components/ApiDocs";
|
||||
import Account from "./components/Account";
|
||||
import Contact from "./components/Contact";
|
||||
import { api } from "./api";
|
||||
|
||||
type View =
|
||||
@@ -12,13 +13,15 @@ type View =
|
||||
| { name: "product"; id: string }
|
||||
| { name: "contribute" }
|
||||
| { name: "api" }
|
||||
| { name: "account" };
|
||||
| { name: "account" }
|
||||
| { name: "contact" };
|
||||
|
||||
const NAV: { key: View["name"]; label: string; icon: typeof Search }[] = [
|
||||
{ key: "home", label: "检索", icon: Search },
|
||||
{ key: "contribute", label: "贡献档案", icon: PlusCircle },
|
||||
{ key: "api", label: "API", icon: Code2 },
|
||||
{ key: "account", label: "API 密钥", icon: KeyRound },
|
||||
{ key: "contact", label: "联系我们", icon: Headset },
|
||||
];
|
||||
|
||||
export default function App() {
|
||||
@@ -88,6 +91,7 @@ export default function App() {
|
||||
)}
|
||||
{view.name === "api" && <ApiDocs onRegister={() => setView({ name: "account" })} />}
|
||||
{view.name === "account" && <Account />}
|
||||
{view.name === "contact" && <Contact />}
|
||||
</main>
|
||||
|
||||
<footer className="border-t border-gray-200/70 bg-white/60">
|
||||
@@ -109,6 +113,12 @@ export default function App() {
|
||||
>
|
||||
API 调用说明
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setView({ name: "contact" })}
|
||||
className="ml-1 text-brand-600 font-medium hover:underline"
|
||||
>
|
||||
联系我们
|
||||
</button>
|
||||
</p>
|
||||
<div className="mt-3">
|
||||
<a
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import { useState } from "react";
|
||||
import { Phone, Mail, Globe, MessageSquare, Copy, Check, Headset } from "lucide-react";
|
||||
|
||||
type Channel = {
|
||||
key: string;
|
||||
icon: typeof Phone;
|
||||
label: string;
|
||||
value: string;
|
||||
href?: string;
|
||||
copy: string;
|
||||
};
|
||||
|
||||
const CHANNELS: Channel[] = [
|
||||
{
|
||||
key: "phone",
|
||||
icon: Phone,
|
||||
label: "电话",
|
||||
value: "188 6595 7520",
|
||||
href: "tel:18865957520",
|
||||
copy: "18865957520",
|
||||
},
|
||||
{
|
||||
key: "email",
|
||||
icon: Mail,
|
||||
label: "邮箱",
|
||||
value: "1115084741@qq.com",
|
||||
href: "mailto:1115084741@qq.com",
|
||||
copy: "1115084741@qq.com",
|
||||
},
|
||||
{
|
||||
key: "site",
|
||||
icon: Globe,
|
||||
label: "官网",
|
||||
value: "www.wenyaoyu.com",
|
||||
href: "https://www.wenyaoyu.com",
|
||||
copy: "https://www.wenyaoyu.com",
|
||||
},
|
||||
{
|
||||
key: "wechat",
|
||||
icon: MessageSquare,
|
||||
label: "微信",
|
||||
value: "s-b-m-y",
|
||||
copy: "s-b-m-y",
|
||||
},
|
||||
];
|
||||
|
||||
export default function Contact() {
|
||||
const [copied, setCopied] = useState<string | null>(null);
|
||||
|
||||
async function copy(channel: Channel) {
|
||||
try {
|
||||
await navigator.clipboard.writeText(channel.copy);
|
||||
setCopied(channel.key);
|
||||
setTimeout(() => setCopied((k) => (k === channel.key ? null : k)), 1500);
|
||||
} catch {
|
||||
/* clipboard unavailable */
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="max-w-3xl mx-auto animate-fade-up">
|
||||
<div className="text-center">
|
||||
<span className="mx-auto grid h-14 w-14 place-items-center rounded-2xl bg-gradient-to-br from-brand-500 to-brand-700 text-white shadow-glow">
|
||||
<Headset className="w-7 h-7" />
|
||||
</span>
|
||||
<h1 className="mt-4 text-2xl font-semibold tracking-tight text-gray-900">联系我们</h1>
|
||||
<p className="mx-auto mt-2 max-w-xl text-sm text-gray-500">
|
||||
欢迎交流与对接合作。无论是数据接入、商务合作还是问题反馈,都可以通过以下任一方式联系我们。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="mt-8 grid grid-cols-1 sm:grid-cols-2 gap-4">
|
||||
{CHANNELS.map((c) => {
|
||||
const Icon = c.icon;
|
||||
return (
|
||||
<div key={c.key} className="card p-5 flex items-center gap-4">
|
||||
<span className="grid h-11 w-11 shrink-0 place-items-center rounded-xl bg-brand-50 text-brand-600">
|
||||
<Icon className="w-5 h-5" />
|
||||
</span>
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="text-xs font-medium text-gray-400">{c.label}</div>
|
||||
{c.href ? (
|
||||
<a
|
||||
href={c.href}
|
||||
target={c.key === "site" ? "_blank" : undefined}
|
||||
rel={c.key === "site" ? "noreferrer" : undefined}
|
||||
className="block truncate font-medium text-gray-800 hover:text-brand-600 hover:underline"
|
||||
>
|
||||
{c.value}
|
||||
</a>
|
||||
) : (
|
||||
<div className="truncate font-medium text-gray-800">{c.value}</div>
|
||||
)}
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => copy(c)}
|
||||
title="复制"
|
||||
className="shrink-0 grid h-9 w-9 place-items-center rounded-lg border border-gray-200 text-gray-400 transition hover:bg-gray-50 hover:text-brand-600"
|
||||
>
|
||||
{copied === c.key ? (
|
||||
<Check className="w-4 h-4 text-brand-600" />
|
||||
) : (
|
||||
<Copy className="w-4 h-4" />
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user