novaalphastrikeomegaz663
044c870df7
- Add retry/backoff (429 + 5xx, Retry-After aware) to the OFF adapter so transient API errors no longer abort a run. - Clamp bounded text fields (serving_size, net_content_unit, country_of_origin) to their column widths in transform; long OFF values previously raised StringDataRightTruncation and rolled back the batch. - Load each record inside a savepoint (load_record_safe) so one malformed source record is skipped instead of aborting the whole import; jobs now report an errored count. - Tests for retry behaviour, serving_size clamping, and per-record isolation. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
79 lines
2.4 KiB
Python
79 lines
2.4 KiB
Python
import json
|
|
from decimal import Decimal
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from opengoods.etl.transform import (
|
|
is_valid_gtin,
|
|
map_category,
|
|
parse_quantity,
|
|
transform,
|
|
transform_nutriments,
|
|
)
|
|
|
|
FIXTURE = json.loads((Path(__file__).parent / "fixtures" / "off_product.json").read_text())
|
|
|
|
|
|
def test_is_valid_gtin():
|
|
assert is_valid_gtin("3017624010701") # real EAN-13
|
|
assert is_valid_gtin("5449000000996") # Coca-Cola
|
|
assert not is_valid_gtin("3017624010700") # bad check digit
|
|
assert not is_valid_gtin("123")
|
|
assert not is_valid_gtin("notanumber")
|
|
|
|
|
|
def test_parse_quantity():
|
|
assert parse_quantity("400 g") == (Decimal("400"), "g")
|
|
assert parse_quantity("1,5 L") == (Decimal("1.5"), "L")
|
|
assert parse_quantity("") is None
|
|
assert parse_quantity("family size") is None
|
|
|
|
|
|
def test_map_category():
|
|
assert map_category({"product_name": "Spring Water"}) == "food.beverages.water"
|
|
assert map_category({"categories": "Dark chocolate"}) == "food.snacks.chocolate"
|
|
assert map_category({"product_name": "Mystery"}) is None
|
|
|
|
|
|
def test_transform_nutriments_dual_energy():
|
|
out = transform_nutriments(FIXTURE["nutriments"])
|
|
assert out["energy_kj"] == 2252.0
|
|
assert out["energy_kcal"] == 539.0
|
|
assert out["fat"] == 30.9
|
|
assert out["salt"] == 0.107
|
|
|
|
|
|
def test_transform_nutriments_fills_missing_energy():
|
|
out = transform_nutriments({"energy-kcal_100g": 100})
|
|
assert out["energy_kj"] == pytest.approx(418.4)
|
|
|
|
|
|
def test_transform_full_record():
|
|
rec = transform(FIXTURE)
|
|
assert rec is not None
|
|
assert rec["gtin"] == "3017624010701"
|
|
assert rec["name"] == "Nutella"
|
|
assert rec["brand"] == "Ferrero"
|
|
assert rec["net_content_unit"] == "g"
|
|
assert rec["net_content_canonical"] == Decimal("400")
|
|
assert rec["country_of_origin"] == "France"
|
|
assert rec["food"]["nutri_score"] == "E"
|
|
assert "milk" in rec["food"]["allergens"]
|
|
assert rec["image_url"].endswith(".jpg")
|
|
|
|
|
|
def test_transform_drops_unnamed():
|
|
assert transform({"code": "0000000000000"}) is None
|
|
|
|
|
|
def test_transform_clamps_long_serving_size():
|
|
raw = {
|
|
"code": "3017624010701",
|
|
"product_name": "X",
|
|
"serving_size": "1 portion (30 g) / 1 portion (30 g) / extra long descriptive text here",
|
|
}
|
|
rec = transform(raw)
|
|
assert rec is not None
|
|
assert len(rec["food"]["serving_size"]) == 32
|