novaalphastrikeomegaz663
044c870df7
- Add retry/backoff (429 + 5xx, Retry-After aware) to the OFF adapter so transient API errors no longer abort a run. - Clamp bounded text fields (serving_size, net_content_unit, country_of_origin) to their column widths in transform; long OFF values previously raised StringDataRightTruncation and rolled back the batch. - Load each record inside a savepoint (load_record_safe) so one malformed source record is skipped instead of aborting the whole import; jobs now report an errored count. - Tests for retry behaviour, serving_size clamping, and per-record isolation. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
60 lines
1.8 KiB
Python
60 lines
1.8 KiB
Python
import httpx
|
|
import pytest
|
|
|
|
from opengoods.adapters.openfoodfacts import OpenFoodFactsAdapter
|
|
|
|
|
|
def _adapter(handler, **kwargs):
|
|
client = httpx.Client(transport=httpx.MockTransport(handler))
|
|
return OpenFoodFactsAdapter(client=client, min_interval=0, backoff_base=0, **kwargs)
|
|
|
|
|
|
def test_retries_transient_5xx_then_succeeds():
|
|
calls = {"n": 0}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
calls["n"] += 1
|
|
if calls["n"] < 3:
|
|
return httpx.Response(503)
|
|
return httpx.Response(200, json={"status": 1, "product": {"code": "x"}})
|
|
|
|
adapter = _adapter(handler, max_retries=4)
|
|
assert adapter.fetch_barcode("x") == {"code": "x"}
|
|
assert calls["n"] == 3 # two 503s retried, third succeeds
|
|
|
|
|
|
def test_retries_exhausted_raises():
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(503)
|
|
|
|
adapter = _adapter(handler, max_retries=2)
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
adapter.fetch_barcode("x")
|
|
|
|
|
|
def test_non_retryable_4xx_not_retried():
|
|
calls = {"n": 0}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
calls["n"] += 1
|
|
return httpx.Response(404)
|
|
|
|
adapter = _adapter(handler, max_retries=4)
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
adapter.fetch_barcode("x")
|
|
assert calls["n"] == 1 # 404 is not retried
|
|
|
|
|
|
def test_retries_connection_error_then_succeeds():
|
|
calls = {"n": 0}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
calls["n"] += 1
|
|
if calls["n"] == 1:
|
|
raise httpx.ConnectError("boom")
|
|
return httpx.Response(200, json={"status": 1, "product": {"code": "y"}})
|
|
|
|
adapter = _adapter(handler, max_retries=4)
|
|
assert adapter.fetch_barcode("y") == {"code": "y"}
|
|
assert calls["n"] == 2
|