John Doe
a35bcd6647
- OFF incremental fetch via search API + persistent watermark (ingest_state, migration 0004) - GS1 barcode supplement adapter (offline mapping + GS1-style API) filling only gaps with field-level provenance - Non-GTIN dedup with canonical selection + merge_log; field-level conflict resolution (source trust > recency) - Quality scoring (0.4 completeness + 0.3 source trust + 0.2 multi-source + 0.1 freshness) wired into load/merge - Jobs: update_off, dedup, schedule; docs/ingestion-management.md - 19 new tests (pure + DB-integration), ruff clean Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
from opengoods.etl.quality import (
|
|
COMPLETENESS_FIELDS,
|
|
agreement_from_sources,
|
|
completeness,
|
|
freshness_from_age,
|
|
score,
|
|
)
|
|
|
|
|
|
def test_completeness_bounds():
|
|
assert completeness(set()) == 0.0
|
|
assert completeness(set(COMPLETENESS_FIELDS)) == 1.0
|
|
half = set(list(COMPLETENESS_FIELDS)[: len(COMPLETENESS_FIELDS) // 2])
|
|
assert 0.0 < completeness(half) < 1.0
|
|
|
|
|
|
def test_agreement_from_sources():
|
|
assert agreement_from_sources(0) == 0.5
|
|
assert agreement_from_sources(1) == 0.5
|
|
assert agreement_from_sources(2) == 0.8
|
|
assert agreement_from_sources(5) == 1.0
|
|
|
|
|
|
def test_freshness_from_age():
|
|
assert freshness_from_age(None) == 0.5
|
|
assert freshness_from_age(1) == 1.0
|
|
assert freshness_from_age(100) == 0.8
|
|
assert freshness_from_age(300) == 0.6
|
|
assert freshness_from_age(700) == 0.4
|
|
assert freshness_from_age(5000) == 0.2
|
|
|
|
|
|
def test_score_weighted_sum_and_bounds():
|
|
assert score(completeness_score=0, source_trust=0, agreement=0, freshness=0) == 0.0
|
|
assert score(completeness_score=1, source_trust=1, agreement=1, freshness=1) == 1.0
|
|
# 0.4*1 + 0.3*0.5 + 0.2*0.5 + 0.1*1 = 0.75
|
|
got = score(completeness_score=1.0, source_trust=0.5, agreement=0.5, freshness=1.0)
|
|
assert got == 0.75
|