rosemariejebbjtxbfp
a75318b811
Stage-0 caching from docs/scalability.md. The Go API now caches hot product-detail and search-result reads in Redis with a fail-open, epoch-versioned scheme; Python ingestion bumps the epoch after a write run to invalidate the cache globally in O(1). - api/internal/cache: fail-open Cache (GetJSON/SetJSON) namespaced by an epoch counter (og:cache:epoch); disabled when Redis is unconfigured. - store: ProductByID/ProductByGTIN (24h TTL) and SearchProducts (1h TTL) read-through the cache via WithCache. - ingestion: bump_cache_epoch() called after update_off/seed_off/ import_bypos/dedup commits when rows changed; best-effort, never fails a run. Adds redis dependency. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
44 lines
1.3 KiB
Python
44 lines
1.3 KiB
Python
"""Deduplicate products: merge non-GTIN duplicates into a canonical record.
|
|
|
|
Usage:
|
|
python -m opengoods.jobs.dedup --dry-run
|
|
python -m opengoods.jobs.dedup --actor nightly
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
|
|
import psycopg
|
|
|
|
from opengoods.cache import bump_cache_epoch
|
|
from opengoods.etl.dedup import dedup_all
|
|
from opengoods.etl.load import default_dsn
|
|
|
|
|
|
def run(args: argparse.Namespace) -> int:
|
|
with psycopg.connect(args.dsn, autocommit=False) as conn:
|
|
summary = dedup_all(conn, actor=args.actor, dry_run=args.dry_run)
|
|
if args.dry_run:
|
|
conn.rollback()
|
|
else:
|
|
conn.commit()
|
|
if not args.dry_run and summary["merged"]:
|
|
bump_cache_epoch()
|
|
mode = "dry-run" if args.dry_run else "applied"
|
|
print(f"{mode} groups={summary['groups']} merged={summary['merged']}")
|
|
return 0
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="Deduplicate OpenGoods products")
|
|
parser.add_argument("--actor", default="ingestion", help="merge_log actor label")
|
|
parser.add_argument("--dry-run", action="store_true", help="report only, do not write")
|
|
parser.add_argument("--dsn", default=default_dsn(), help="PostgreSQL DSN")
|
|
return run(parser.parse_args(argv))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|