from opengoods.etl.dedup import choose_canonical, dedup_all, product_signature from opengoods.etl.load import _ensure_brand, ensure_source def test_product_signature_normalization(): a = product_signature(" Spring Water ", "Acme", 500) b = product_signature("spring water", "acme", 500) assert a == b assert product_signature("", "x", 1) is None def test_choose_canonical_prefers_quality(): members = [ {"id": "a", "quality_score": 0.2, "created_at": 1}, {"id": "b", "quality_score": 0.9, "created_at": 2}, ] assert choose_canonical(members)["id"] == "b" def test_dedup_merges_duplicates(db_conn): brand_id = _ensure_brand(db_conn, "DupBrand") def mk(quality): return db_conn.execute( """ INSERT INTO product (name, brand_id, net_content_canonical, quality_score) VALUES (%s, %s, %s, %s) RETURNING id """, ("Dup Snack", brand_id, 100, quality), ).fetchone()[0] keep = mk(0.9) drop = mk(0.2) src = ensure_source(db_conn) db_conn.execute( "INSERT INTO product_source (product_id, source_id, fields) VALUES (%s, %s, %s)", (drop, src, ["name"]), ) summary = dedup_all(db_conn) assert summary == {"groups": 1, "merged": 1} keep_status = db_conn.execute("SELECT status FROM product WHERE id = %s", (keep,)).fetchone()[0] drop_status, canonical_id = db_conn.execute( "SELECT status, canonical_id FROM product WHERE id = %s", (drop,) ).fetchone() assert keep_status == "active" assert drop_status == "merged" assert str(canonical_id) == str(keep) # The merged product's source row was re-pointed to the canonical product. reattached = db_conn.execute( "SELECT count(*) FROM product_source WHERE product_id = %s", (keep,) ).fetchone()[0] assert reattached == 1 logged = db_conn.execute( "SELECT count(*) FROM merge_log WHERE kept_id = %s AND merged_id = %s", (keep, drop), ).fetchone()[0] assert logged == 1