feat(ingestion): harden OFF ingestion for bulk seeding
- Add retry/backoff (429 + 5xx, Retry-After aware) to the OFF adapter so transient API errors no longer abort a run. - Clamp bounded text fields (serving_size, net_content_unit, country_of_origin) to their column widths in transform; long OFF values previously raised StringDataRightTruncation and rolled back the batch. - Load each record inside a savepoint (load_record_safe) so one malformed source record is skipped instead of aborting the whole import; jobs now report an errored count. - Tests for retry behaviour, serving_size clamping, and per-record isolation. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
@@ -11,7 +11,7 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from opengoods.etl.load import default_dsn, ensure_source, load_record
|
||||
from opengoods.etl.load import default_dsn, ensure_source, load_record, load_record_safe
|
||||
from opengoods.etl.transform import transform
|
||||
|
||||
psycopg = pytest.importorskip("psycopg")
|
||||
@@ -59,3 +59,25 @@ def test_load_record_roundtrip(conn):
|
||||
assert prov[0] >= 1
|
||||
|
||||
conn.rollback() # keep the test DB clean
|
||||
|
||||
|
||||
def test_load_record_safe_isolates_bad_record(conn):
|
||||
source_id = ensure_source(conn)
|
||||
|
||||
# Unique gtin so the good record is a fresh INSERT, not an upsert/update.
|
||||
good = transform(FIXTURE)
|
||||
good["gtin"] = "4006381333931"
|
||||
assert load_record_safe(conn, good, source_id, FIXTURE) is True
|
||||
after_good = conn.execute("SELECT count(*) FROM product").fetchone()[0]
|
||||
|
||||
# A record whose name violates NOT NULL must not abort the batch.
|
||||
bad = dict(good)
|
||||
bad["gtin"] = "5000112637922"
|
||||
bad["name"] = None
|
||||
assert load_record_safe(conn, bad, source_id, {}) is False
|
||||
|
||||
# The good record survived the bad one's rollback-to-savepoint.
|
||||
after_bad = conn.execute("SELECT count(*) FROM product").fetchone()[0]
|
||||
assert after_bad == after_good
|
||||
|
||||
conn.rollback() # keep the test DB clean
|
||||
|
||||
Reference in New Issue
Block a user