feat(ingestion): harden OFF ingestion for bulk seeding
- Add retry/backoff (429 + 5xx, Retry-After aware) to the OFF adapter so transient API errors no longer abort a run. - Clamp bounded text fields (serving_size, net_content_unit, country_of_origin) to their column widths in transform; long OFF values previously raised StringDataRightTruncation and rolled back the batch. - Load each record inside a savepoint (load_record_safe) so one malformed source record is skipped instead of aborting the whole import; jobs now report an errored count. - Tests for retry behaviour, serving_size clamping, and per-record isolation. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
@@ -7,6 +7,7 @@ source with field-level provenance in `product_source`.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
@@ -18,6 +19,8 @@ from opengoods.etl.quality import update_quality
|
||||
|
||||
OFF_HOMEPAGE = "https://world.openfoodfacts.org"
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def default_dsn() -> str:
|
||||
return os.environ.get(
|
||||
@@ -196,6 +199,25 @@ def load_record(conn: psycopg.Connection, rec: dict[str, Any], source_id: str, r
|
||||
return product_id
|
||||
|
||||
|
||||
def load_record_safe(
|
||||
conn: psycopg.Connection, rec: dict[str, Any], source_id: str, raw: dict
|
||||
) -> bool:
|
||||
"""Load one record inside a savepoint.
|
||||
|
||||
On success the record's writes stay in the surrounding transaction. On any
|
||||
error, only this record's writes are rolled back (to the savepoint) and the
|
||||
batch continues, so a single malformed source record cannot abort a large
|
||||
import. Returns True if loaded, False if skipped due to an error.
|
||||
"""
|
||||
try:
|
||||
with conn.transaction():
|
||||
load_record(conn, rec, source_id, raw)
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001 - per-record isolation is intentional
|
||||
logger.warning("skipping record gtin=%s: %s", rec.get("gtin"), exc)
|
||||
return False
|
||||
|
||||
|
||||
def _jsonable(raw: dict) -> dict:
|
||||
"""Drop values that are not JSON-serializable from a raw record."""
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user