feat(ingestion): harden OFF ingestion for bulk seeding
- Add retry/backoff (429 + 5xx, Retry-After aware) to the OFF adapter so transient API errors no longer abort a run. - Clamp bounded text fields (serving_size, net_content_unit, country_of_origin) to their column widths in transform; long OFF values previously raised StringDataRightTruncation and rolled back the batch. - Load each record inside a savepoint (load_record_safe) so one malformed source record is skipped instead of aborting the whole import; jobs now report an errored count. - Tests for retry behaviour, serving_size clamping, and per-record isolation. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
@@ -78,6 +78,14 @@ def map_category(raw: dict) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _clamp(value: str | None, max_len: int) -> str | None:
|
||||
"""Trim a string to fit a bounded DB column; external data length varies."""
|
||||
if value is None:
|
||||
return None
|
||||
value = value.strip()
|
||||
return value[:max_len] or None
|
||||
|
||||
|
||||
def _clean_tags(tags: list[str] | None, prefix: str = "") -> list[str]:
|
||||
out: list[str] = []
|
||||
for t in tags or []:
|
||||
@@ -143,16 +151,16 @@ def transform(raw: dict) -> dict | None:
|
||||
"brand": brand,
|
||||
"category_path": map_category(raw),
|
||||
"net_content_value": net_value,
|
||||
"net_content_unit": net_unit,
|
||||
"net_content_unit": _clamp(net_unit, 16),
|
||||
"net_content_canonical": net_canonical,
|
||||
"country_of_origin": (raw.get("countries") or "").split(",")[0].strip() or None,
|
||||
"country_of_origin": _clamp((raw.get("countries") or "").split(",")[0].strip() or None, 64),
|
||||
"food": {
|
||||
"ingredients_text": raw.get("ingredients_text") or None,
|
||||
"allergens": _clean_tags(raw.get("allergens_tags")),
|
||||
"additives": _clean_tags(raw.get("additives_tags")),
|
||||
"nutriments": transform_nutriments(raw.get("nutriments") or {}),
|
||||
"nutrition_basis": "per_100g",
|
||||
"serving_size": raw.get("serving_size") or None,
|
||||
"serving_size": _clamp(raw.get("serving_size") or None, 32),
|
||||
"nutri_score": (raw.get("nutriscore_grade") or "").upper()[:1] or None,
|
||||
},
|
||||
"image_url": raw.get("image_front_url") or raw.get("image_url") or None,
|
||||
|
||||
Reference in New Issue
Block a user