rosemariejebbjtxbfp
a75318b811
Stage-0 caching from docs/scalability.md. The Go API now caches hot product-detail and search-result reads in Redis with a fail-open, epoch-versioned scheme; Python ingestion bumps the epoch after a write run to invalidate the cache globally in O(1). - api/internal/cache: fail-open Cache (GetJSON/SetJSON) namespaced by an epoch counter (og:cache:epoch); disabled when Redis is unconfigured. - store: ProductByID/ProductByGTIN (24h TTL) and SearchProducts (1h TTL) read-through the cache via WithCache. - ingestion: bump_cache_epoch() called after update_off/seed_off/ import_bypos/dedup commits when rows changed; best-effort, never fails a run. Adds redis dependency. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
81 lines
2.6 KiB
Python
81 lines
2.6 KiB
Python
"""Import products collected by the bypos-collector tool into OpenGoods.
|
|
|
|
The ``tools/bypos-collector`` program writes one product per line (JSONL). This
|
|
job reads such a file, transforms each ``hit`` record into the internal product
|
|
shape, and upserts it into the database under the ``bypos中心库`` source with
|
|
field-level provenance.
|
|
|
|
Usage::
|
|
|
|
python -m opengoods.jobs.import_bypos --input products.jsonl
|
|
python -m opengoods.jobs.import_bypos --input products.jsonl --limit 500
|
|
|
|
Re-running is safe: products are upserted by GTIN and the source's MSRP/
|
|
provenance rows are refreshed rather than duplicated.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import gzip
|
|
import json
|
|
import sys
|
|
from collections.abc import Iterator
|
|
|
|
import psycopg
|
|
|
|
from opengoods.adapters.bypos import transform_bypos
|
|
from opengoods.cache import bump_cache_epoch
|
|
from opengoods.etl.load import default_dsn, ensure_bypos_source, load_bypos_record_safe
|
|
|
|
|
|
def _read_jsonl(path: str) -> Iterator[dict]:
|
|
opener = gzip.open if path.endswith(".gz") else open
|
|
with opener(path, "rt", encoding="utf-8") as fh:
|
|
for line in fh:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
yield json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
|
|
|
|
def run(args: argparse.Namespace) -> int:
|
|
loaded = skipped = errored = 0
|
|
with psycopg.connect(args.dsn, autocommit=False) as conn:
|
|
source_id = ensure_bypos_source(conn)
|
|
yielded = 0
|
|
for raw in _read_jsonl(args.input):
|
|
if args.limit and yielded >= args.limit:
|
|
break
|
|
rec = transform_bypos(raw)
|
|
if rec is None:
|
|
skipped += 1
|
|
continue
|
|
yielded += 1
|
|
if load_bypos_record_safe(conn, rec, source_id, raw):
|
|
loaded += 1
|
|
else:
|
|
errored += 1
|
|
conn.commit()
|
|
if loaded:
|
|
bump_cache_epoch()
|
|
print(f"loaded={loaded} skipped={skipped} errored={errored}")
|
|
return 0
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="Import bypos-collector JSONL into OpenGoods")
|
|
parser.add_argument(
|
|
"--input", required=True, help="path to a bypos-collector JSONL (.jsonl or .jsonl.gz)"
|
|
)
|
|
parser.add_argument("--limit", type=int, default=0, help="max hit records to load (0 = all)")
|
|
parser.add_argument("--dsn", default=default_dsn(), help="PostgreSQL DSN")
|
|
return run(parser.parse_args(argv))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|