John Doe
a35bcd6647
- OFF incremental fetch via search API + persistent watermark (ingest_state, migration 0004) - GS1 barcode supplement adapter (offline mapping + GS1-style API) filling only gaps with field-level provenance - Non-GTIN dedup with canonical selection + merge_log; field-level conflict resolution (source trust > recency) - Quality scoring (0.4 completeness + 0.3 source trust + 0.2 multi-source + 0.1 freshness) wired into load/merge - Jobs: update_off, dedup, schedule; docs/ingestion-management.md - 19 new tests (pure + DB-integration), ruff clean Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
63 lines
2.0 KiB
Python
63 lines
2.0 KiB
Python
from opengoods.etl.dedup import choose_canonical, dedup_all, product_signature
|
|
from opengoods.etl.load import _ensure_brand, ensure_source
|
|
|
|
|
|
def test_product_signature_normalization():
|
|
a = product_signature(" Spring Water ", "Acme", 500)
|
|
b = product_signature("spring water", "acme", 500)
|
|
assert a == b
|
|
assert product_signature("", "x", 1) is None
|
|
|
|
|
|
def test_choose_canonical_prefers_quality():
|
|
members = [
|
|
{"id": "a", "quality_score": 0.2, "created_at": 1},
|
|
{"id": "b", "quality_score": 0.9, "created_at": 2},
|
|
]
|
|
assert choose_canonical(members)["id"] == "b"
|
|
|
|
|
|
def test_dedup_merges_duplicates(db_conn):
|
|
brand_id = _ensure_brand(db_conn, "DupBrand")
|
|
|
|
def mk(quality):
|
|
return db_conn.execute(
|
|
"""
|
|
INSERT INTO product (name, brand_id, net_content_canonical, quality_score)
|
|
VALUES (%s, %s, %s, %s) RETURNING id
|
|
""",
|
|
("Dup Snack", brand_id, 100, quality),
|
|
).fetchone()[0]
|
|
|
|
keep = mk(0.9)
|
|
drop = mk(0.2)
|
|
|
|
src = ensure_source(db_conn)
|
|
db_conn.execute(
|
|
"INSERT INTO product_source (product_id, source_id, fields) VALUES (%s, %s, %s)",
|
|
(drop, src, ["name"]),
|
|
)
|
|
|
|
summary = dedup_all(db_conn)
|
|
assert summary == {"groups": 1, "merged": 1}
|
|
|
|
keep_status = db_conn.execute("SELECT status FROM product WHERE id = %s", (keep,)).fetchone()[0]
|
|
drop_status, canonical_id = db_conn.execute(
|
|
"SELECT status, canonical_id FROM product WHERE id = %s", (drop,)
|
|
).fetchone()
|
|
assert keep_status == "active"
|
|
assert drop_status == "merged"
|
|
assert str(canonical_id) == str(keep)
|
|
|
|
# The merged product's source row was re-pointed to the canonical product.
|
|
reattached = db_conn.execute(
|
|
"SELECT count(*) FROM product_source WHERE product_id = %s", (keep,)
|
|
).fetchone()[0]
|
|
assert reattached == 1
|
|
|
|
logged = db_conn.execute(
|
|
"SELECT count(*) FROM merge_log WHERE kept_id = %s AND merged_id = %s",
|
|
(keep, drop),
|
|
).fetchone()[0]
|
|
assert logged == 1
|