Files
goods/ingestion/opengoods/adapters/gs1.py
T
John Doe a35bcd6647
CI / Go (api) (pull_request) Has been cancelled
CI / Python (ingestion) (pull_request) Has been cancelled
CI / Migrations (postgres) (pull_request) Has been cancelled
M4: ingestion management (incremental, GS1 supplement, dedup/conflict, quality, scheduler)
- OFF incremental fetch via search API + persistent watermark (ingest_state, migration 0004)
- GS1 barcode supplement adapter (offline mapping + GS1-style API) filling only gaps with field-level provenance
- Non-GTIN dedup with canonical selection + merge_log; field-level conflict resolution (source trust > recency)
- Quality scoring (0.4 completeness + 0.3 source trust + 0.2 multi-source + 0.1 freshness) wired into load/merge
- Jobs: update_off, dedup, schedule; docs/ingestion-management.md
- 19 new tests (pure + DB-integration), ruff clean

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-06-08 09:27:42 +00:00

125 lines
4.2 KiB
Python

"""GS1 barcode supplement adapter.
GS1 (e.g. *Verified by GS1* / GS1 China) is the authoritative registry that maps
a GTIN to its brand owner, product description and GPC category. We use it to
*supplement* — fill gaps in — records gathered from crowd sources like Open Food
Facts, never to overwrite existing values.
Real GS1 access is credentialed and region-specific, so this adapter supports
two modes:
* **offline** (default): look barcodes up in a local JSON mapping file. This is
what tests and air-gapped runs use.
* **online**: GET ``{base_url}/{gtin}`` with an API key header, then normalize
the response. Enabled by passing ``base_url`` + ``client``.
Either way :meth:`fetch_barcode` returns a normalized *supplement* dict (or
``None``); :mod:`opengoods.etl.supplement` applies it to the database.
"""
from __future__ import annotations
import json
from collections.abc import Iterator
from pathlib import Path
import httpx
SOURCE_NAME = "gs1"
GS1_HOMEPAGE = "https://www.gs1.org"
GS1_LICENSE = "proprietary"
# GS1 is the authoritative barcode registry -> high trust.
GS1_TRUST = 0.9
# Keys of a normalized supplement record.
_SUPPLEMENT_KEYS = (
"gtin",
"name",
"brand",
"manufacturer",
"gpc_brick_code",
"country_of_origin",
"net_content_value",
"net_content_unit",
)
def _normalize(code: str, data: dict) -> dict:
"""Project a raw mapping/record onto the supplement schema (non-empty only)."""
rec: dict = {"gtin": code}
for key in _SUPPLEMENT_KEYS:
if key == "gtin":
continue
value = data.get(key)
if value not in (None, "", []):
rec[key] = value
return rec
def _parse_api(code: str, payload: dict) -> dict:
"""Best-effort mapping of a Verified-by-GS1 style payload to our schema."""
item = payload
if isinstance(payload.get("gtinRecords"), list) and payload["gtinRecords"]:
item = payload["gtinRecords"][0]
return _normalize(
code,
{
"name": item.get("productDescription") or item.get("description"),
"brand": item.get("brandName"),
"manufacturer": item.get("companyName") or item.get("licenseeName"),
"gpc_brick_code": item.get("gpcCategoryCode"),
"country_of_origin": item.get("countryOfSaleCode") or item.get("countryCode"),
"net_content_value": item.get("netContent"),
"net_content_unit": item.get("netContentUnit"),
},
)
class GS1Adapter:
"""Look up GTIN supplements from a local mapping or a GS1-style API."""
source_name = SOURCE_NAME
def __init__(
self,
mapping: dict | None = None,
*,
client: httpx.Client | None = None,
base_url: str | None = None,
api_key: str | None = None,
) -> None:
self._mapping = mapping or {}
self._client = client
self._base_url = base_url.rstrip("/") if base_url else None
self._api_key = api_key
@classmethod
def from_file(cls, path: str | Path) -> GS1Adapter:
"""Build an offline adapter from a JSON ``{gtin: {...}}`` mapping file."""
data = json.loads(Path(path).read_text(encoding="utf-8"))
return cls(mapping=data)
def fetch_barcode(self, code: str) -> dict | None:
"""Return a normalized supplement dict for ``code`` (or ``None``)."""
if self._base_url and self._client is not None:
headers = {"apikey": self._api_key} if self._api_key else {}
resp = self._client.get(f"{self._base_url}/{code}", headers=headers)
if resp.status_code == 404:
return None
resp.raise_for_status()
rec = _parse_api(code, resp.json())
else:
data = self._mapping.get(code)
if not data:
return None
rec = _normalize(code, data)
# A record with only the GTIN carries no supplement.
return rec if len(rec) > 1 else None
def fetch(self, barcodes: list[str]) -> Iterator[dict]:
"""Yield supplement records for the given barcodes."""
for code in barcodes:
rec = self.fetch_barcode(code)
if rec is not None:
yield rec