feat(ingestion): import bypos-collector JSONL into goods
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,77 @@
|
||||
"""Import products collected by the bypos-collector tool into OpenGoods.
|
||||
|
||||
The ``tools/bypos-collector`` program writes one product per line (JSONL). This
|
||||
job reads such a file, transforms each ``hit`` record into the internal product
|
||||
shape, and upserts it into the database under the ``bypos中心库`` source with
|
||||
field-level provenance.
|
||||
|
||||
Usage::
|
||||
|
||||
python -m opengoods.jobs.import_bypos --input products.jsonl
|
||||
python -m opengoods.jobs.import_bypos --input products.jsonl --limit 500
|
||||
|
||||
Re-running is safe: products are upserted by GTIN and the source's MSRP/
|
||||
provenance rows are refreshed rather than duplicated.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import json
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
|
||||
import psycopg
|
||||
|
||||
from opengoods.adapters.bypos import transform_bypos
|
||||
from opengoods.etl.load import default_dsn, ensure_bypos_source, load_bypos_record_safe
|
||||
|
||||
|
||||
def _read_jsonl(path: str) -> Iterator[dict]:
|
||||
opener = gzip.open if path.endswith(".gz") else open
|
||||
with opener(path, "rt", encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
yield json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> int:
|
||||
loaded = skipped = errored = 0
|
||||
with psycopg.connect(args.dsn, autocommit=False) as conn:
|
||||
source_id = ensure_bypos_source(conn)
|
||||
yielded = 0
|
||||
for raw in _read_jsonl(args.input):
|
||||
if args.limit and yielded >= args.limit:
|
||||
break
|
||||
rec = transform_bypos(raw)
|
||||
if rec is None:
|
||||
skipped += 1
|
||||
continue
|
||||
yielded += 1
|
||||
if load_bypos_record_safe(conn, rec, source_id, raw):
|
||||
loaded += 1
|
||||
else:
|
||||
errored += 1
|
||||
conn.commit()
|
||||
print(f"loaded={loaded} skipped={skipped} errored={errored}")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Import bypos-collector JSONL into OpenGoods")
|
||||
parser.add_argument(
|
||||
"--input", required=True, help="path to a bypos-collector JSONL (.jsonl or .jsonl.gz)"
|
||||
)
|
||||
parser.add_argument("--limit", type=int, default=0, help="max hit records to load (0 = all)")
|
||||
parser.add_argument("--dsn", default=default_dsn(), help="PostgreSQL DSN")
|
||||
return run(parser.parse_args(argv))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user