feat(api): Redis read cache for product details and search
Stage-0 caching from docs/scalability.md. The Go API now caches hot product-detail and search-result reads in Redis with a fail-open, epoch-versioned scheme; Python ingestion bumps the epoch after a write run to invalidate the cache globally in O(1). - api/internal/cache: fail-open Cache (GetJSON/SetJSON) namespaced by an epoch counter (og:cache:epoch); disabled when Redis is unconfigured. - store: ProductByID/ProductByGTIN (24h TTL) and SearchProducts (1h TTL) read-through the cache via WithCache. - ingestion: bump_cache_epoch() called after update_off/seed_off/ import_bypos/dedup commits when rows changed; best-effort, never fails a run. Adds redis dependency. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
@@ -12,6 +12,7 @@ import sys
|
||||
|
||||
import psycopg
|
||||
|
||||
from opengoods.cache import bump_cache_epoch
|
||||
from opengoods.etl.dedup import dedup_all
|
||||
from opengoods.etl.load import default_dsn
|
||||
|
||||
@@ -23,6 +24,8 @@ def run(args: argparse.Namespace) -> int:
|
||||
conn.rollback()
|
||||
else:
|
||||
conn.commit()
|
||||
if not args.dry_run and summary["merged"]:
|
||||
bump_cache_epoch()
|
||||
mode = "dry-run" if args.dry_run else "applied"
|
||||
print(f"{mode} groups={summary['groups']} merged={summary['merged']}")
|
||||
return 0
|
||||
|
||||
@@ -25,6 +25,7 @@ from collections.abc import Iterator
|
||||
import psycopg
|
||||
|
||||
from opengoods.adapters.bypos import transform_bypos
|
||||
from opengoods.cache import bump_cache_epoch
|
||||
from opengoods.etl.load import default_dsn, ensure_bypos_source, load_bypos_record_safe
|
||||
|
||||
|
||||
@@ -59,6 +60,8 @@ def run(args: argparse.Namespace) -> int:
|
||||
else:
|
||||
errored += 1
|
||||
conn.commit()
|
||||
if loaded:
|
||||
bump_cache_epoch()
|
||||
print(f"loaded={loaded} skipped={skipped} errored={errored}")
|
||||
return 0
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ from opengoods.adapters.openfoodfacts import (
|
||||
is_cn_gs1,
|
||||
read_dump,
|
||||
)
|
||||
from opengoods.cache import bump_cache_epoch
|
||||
from opengoods.etl.load import default_dsn, ensure_source, load_record_safe
|
||||
from opengoods.etl.transform import transform
|
||||
|
||||
@@ -67,6 +68,8 @@ def run(args: argparse.Namespace) -> int:
|
||||
else:
|
||||
errored += 1
|
||||
conn.commit()
|
||||
if loaded:
|
||||
bump_cache_epoch()
|
||||
print(f"loaded={loaded} skipped={skipped} errored={errored}")
|
||||
return 0
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ import sys
|
||||
import psycopg
|
||||
|
||||
from opengoods.adapters.openfoodfacts import SOURCE_NAME, OpenFoodFactsAdapter
|
||||
from opengoods.cache import bump_cache_epoch
|
||||
from opengoods.etl.load import default_dsn, ensure_source, load_record_safe
|
||||
from opengoods.etl.state import get_watermark, set_watermark
|
||||
from opengoods.etl.transform import transform
|
||||
@@ -49,6 +50,8 @@ def run(args: argparse.Namespace) -> int:
|
||||
stats={"loaded": loaded, "skipped": skipped, "errored": errored, "since": since},
|
||||
)
|
||||
conn.commit()
|
||||
if loaded:
|
||||
bump_cache_epoch()
|
||||
print(
|
||||
f"since={since} loaded={loaded} skipped={skipped} "
|
||||
f"errored={errored} watermark={high_watermark}"
|
||||
|
||||
Reference in New Issue
Block a user