John Doe
a35bcd6647
- OFF incremental fetch via search API + persistent watermark (ingest_state, migration 0004) - GS1 barcode supplement adapter (offline mapping + GS1-style API) filling only gaps with field-level provenance - Non-GTIN dedup with canonical selection + merge_log; field-level conflict resolution (source trust > recency) - Quality scoring (0.4 completeness + 0.3 source trust + 0.2 multi-source + 0.1 freshness) wired into load/merge - Jobs: update_off, dedup, schedule; docs/ingestion-management.md - 19 new tests (pure + DB-integration), ruff clean Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
63 lines
2.0 KiB
Python
63 lines
2.0 KiB
Python
from datetime import UTC, datetime
|
|
|
|
from opengoods.etl.merge import Candidate, merge_records, resolve_field
|
|
|
|
|
|
def _ts(y, m, d):
|
|
return datetime(y, m, d, tzinfo=UTC)
|
|
|
|
|
|
def test_resolve_field_prefers_trust_then_recency():
|
|
cands = [
|
|
Candidate(value="A", source="off", trust=0.7, fetched_at=_ts(2024, 1, 1)),
|
|
Candidate(value="B", source="gs1", trust=0.9, fetched_at=_ts(2023, 1, 1)),
|
|
]
|
|
res = resolve_field(cands)
|
|
assert res is not None
|
|
assert res.value == "B"
|
|
assert res.source == "gs1"
|
|
|
|
|
|
def test_resolve_field_recency_tiebreak_on_equal_trust():
|
|
cands = [
|
|
Candidate(value="old", source="a", trust=0.7, fetched_at=_ts(2023, 1, 1)),
|
|
Candidate(value="new", source="b", trust=0.7, fetched_at=_ts(2024, 6, 1)),
|
|
]
|
|
assert resolve_field(cands).value == "new"
|
|
|
|
|
|
def test_resolve_field_skips_empty():
|
|
cands = [
|
|
Candidate(value="", source="a", trust=0.99),
|
|
Candidate(value=None, source="b", trust=0.99),
|
|
Candidate(value="kept", source="c", trust=0.1),
|
|
]
|
|
assert resolve_field(cands).value == "kept"
|
|
assert resolve_field([Candidate(value="", source="a")]) is None
|
|
|
|
|
|
def test_merge_records_provenance():
|
|
records = [
|
|
{
|
|
"name": Candidate("Water", "off", 0.7, _ts(2024, 1, 1)),
|
|
"brand": Candidate("", "off", 0.7),
|
|
},
|
|
{
|
|
"brand": Candidate("Acme", "gs1", 0.9, _ts(2024, 2, 1)),
|
|
"gtin": Candidate("123", "gs1", 0.9),
|
|
},
|
|
]
|
|
merged = merge_records(records)
|
|
assert merged.values["name"] == "Water"
|
|
assert merged.values["brand"] == "Acme"
|
|
assert merged.values["gtin"] == "123"
|
|
assert merged.provenance["brand"] == "gs1"
|
|
assert merged.provenance["name"] == "off"
|
|
|
|
|
|
def test_merge_records_accepts_bare_values():
|
|
merged = merge_records([{"x": 1}, {"x": 2}])
|
|
# both bare -> trust tie, no timestamps -> first max() wins deterministically
|
|
assert merged.values["x"] in (1, 2)
|
|
assert merged.provenance["x"] == "unknown"
|