M4: ingestion management (incremental, GS1 supplement, dedup/conflict, quality, scheduler)
- OFF incremental fetch via search API + persistent watermark (ingest_state, migration 0004) - GS1 barcode supplement adapter (offline mapping + GS1-style API) filling only gaps with field-level provenance - Non-GTIN dedup with canonical selection + merge_log; field-level conflict resolution (source trust > recency) - Quality scoring (0.4 completeness + 0.3 source trust + 0.2 multi-source + 0.1 freshness) wired into load/merge - Jobs: update_off, dedup, schedule; docs/ingestion-management.md - 19 new tests (pure + DB-integration), ruff clean Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,40 @@
|
||||
"""Deduplicate products: merge non-GTIN duplicates into a canonical record.
|
||||
|
||||
Usage:
|
||||
python -m opengoods.jobs.dedup --dry-run
|
||||
python -m opengoods.jobs.dedup --actor nightly
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
import psycopg
|
||||
|
||||
from opengoods.etl.dedup import dedup_all
|
||||
from opengoods.etl.load import default_dsn
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> int:
|
||||
with psycopg.connect(args.dsn, autocommit=False) as conn:
|
||||
summary = dedup_all(conn, actor=args.actor, dry_run=args.dry_run)
|
||||
if args.dry_run:
|
||||
conn.rollback()
|
||||
else:
|
||||
conn.commit()
|
||||
mode = "dry-run" if args.dry_run else "applied"
|
||||
print(f"{mode} groups={summary['groups']} merged={summary['merged']}")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Deduplicate OpenGoods products")
|
||||
parser.add_argument("--actor", default="ingestion", help="merge_log actor label")
|
||||
parser.add_argument("--dry-run", action="store_true", help="report only, do not write")
|
||||
parser.add_argument("--dsn", default=default_dsn(), help="PostgreSQL DSN")
|
||||
return run(parser.parse_args(argv))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user