feat(ingestion): import bypos-collector JSONL into goods

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

ingestion/opengoods/etl/load.py

@@ -14,6 +14,10 @@ from typing import Any
 import psycopg
 from psycopg.types.json import Jsonb
 
+from opengoods.adapters.bypos import SOURCE_HOMEPAGE as SOURCE_HOMEPAGE_BYPOS
+from opengoods.adapters.bypos import SOURCE_LICENSE as SOURCE_LICENSE_BYPOS
+from opengoods.adapters.bypos import SOURCE_NAME as SOURCE_NAME_BYPOS
+from opengoods.adapters.bypos import SOURCE_TRUST as SOURCE_TRUST_BYPOS
 from opengoods.adapters.openfoodfacts import OFF_LICENSE, SOURCE_NAME
 from opengoods.etl.quality import update_quality
 
@@ -73,6 +77,23 @@ def _ensure_brand(conn: psycopg.Connection, name: str | None) -> str | None:
     return row[0]
 
 
+def _ensure_manufacturer(
+    conn: psycopg.Connection, name: str | None, country: str | None = None
+) -> str | None:
+    if not name:
+        return None
+    row = conn.execute(
+        """
+        INSERT INTO manufacturer (name, normalized_name, country)
+        VALUES (%s, %s, %s)
+        ON CONFLICT (normalized_name) DO UPDATE SET name = manufacturer.name
+        RETURNING id
+        """,
+        (name, _normalize_brand(name), country),
+    ).fetchone()
+    return row[0]
+
+
 def _category_id(conn: psycopg.Connection, path: str | None) -> tuple[str | None, str | None]:
     if not path:
         return None, None
@@ -218,6 +239,140 @@ def load_record_safe(
         return False
 
 
+def ensure_bypos_source(conn: psycopg.Connection) -> str:
+    """Upsert the bypos central-library source row and return its id."""
+    return ensure_source_named(
+        conn,
+        SOURCE_NAME_BYPOS,
+        SOURCE_HOMEPAGE_BYPOS,
+        SOURCE_LICENSE_BYPOS,
+        SOURCE_TRUST_BYPOS,
+    )
+
+
+def load_bypos_record(
+    conn: psycopg.Connection, rec: dict[str, Any], source_id: str, raw: dict
+) -> str:
+    """Upsert one transformed bypos record; return the product id.
+
+    Unlike OFF records these have no ingredients/nutrition, so no ``food_detail``
+    row is written. The suggested retail price (if any) is stored as a CNY MSRP
+    snapshot, and provenance/MSRP rows are keyed by source so a re-import
+    refreshes rather than duplicates them.
+    """
+    manufacturer_id = _ensure_manufacturer(
+        conn, rec.get("manufacturer"), rec.get("country_of_origin")
+    )
+    attrs = rec.get("attributes") or {}
+    fields = ["name", "net_content", "country_of_origin"]
+    if manufacturer_id:
+        fields.append("manufacturer")
+    if attrs:
+        fields.append("attributes")
+
+    if rec.get("gtin"):
+        fields.append("gtin")
+        prod = conn.execute(
+            """
+            INSERT INTO product (gtin, name, manufacturer_id,
+                net_content_value, net_content_unit, net_content_canonical,
+                country_of_origin, attributes)
+            VALUES (%s,%s,%s,%s,%s,%s,%s,%s)
+            ON CONFLICT (gtin) WHERE gtin IS NOT NULL DO UPDATE SET
+                name = EXCLUDED.name,
+                manufacturer_id = COALESCE(EXCLUDED.manufacturer_id, product.manufacturer_id),
+                net_content_value = COALESCE(EXCLUDED.net_content_value, product.net_content_value),
+                net_content_unit = COALESCE(EXCLUDED.net_content_unit, product.net_content_unit),
+                net_content_canonical = COALESCE(
+                    EXCLUDED.net_content_canonical, product.net_content_canonical),
+                country_of_origin = COALESCE(EXCLUDED.country_of_origin, product.country_of_origin),
+                attributes = product.attributes || EXCLUDED.attributes
+            RETURNING id
+            """,
+            (
+                rec["gtin"],
+                rec["name"],
+                manufacturer_id,
+                rec.get("net_content_value"),
+                rec.get("net_content_unit"),
+                rec.get("net_content_canonical"),
+                rec.get("country_of_origin"),
+                Jsonb(attrs),
+            ),
+        ).fetchone()
+    else:
+        prod = conn.execute(
+            """
+            INSERT INTO product (name, manufacturer_id,
+                net_content_value, net_content_unit, net_content_canonical,
+                country_of_origin, attributes)
+            VALUES (%s,%s,%s,%s,%s,%s,%s)
+            RETURNING id
+            """,
+            (
+                rec["name"],
+                manufacturer_id,
+                rec.get("net_content_value"),
+                rec.get("net_content_unit"),
+                rec.get("net_content_canonical"),
+                rec.get("country_of_origin"),
+                Jsonb(attrs),
+            ),
+        ).fetchone()
+    product_id = prod[0]
+
+    # Refresh this source's MSRP snapshot (suggested retail price, CNY).
+    conn.execute(
+        "DELETE FROM product_msrp WHERE product_id = %s AND source_id = %s",
+        (product_id, source_id),
+    )
+    if rec.get("msrp") is not None:
+        conn.execute(
+            """
+            INSERT INTO product_msrp (product_id, amount, currency, region, source_id, source_url)
+            VALUES (%s,%s,'CNY','CN',%s,%s)
+            """,
+            (product_id, rec["msrp"], source_id, SOURCE_HOMEPAGE_BYPOS),
+        )
+        fields.append("msrp")
+
+    # Refresh this source's provenance row (one per source for idempotency).
+    conn.execute(
+        "DELETE FROM product_source WHERE product_id = %s AND source_id = %s",
+        (product_id, source_id),
+    )
+    conn.execute(
+        """
+        INSERT INTO product_source (product_id, source_id, url, fields, fetched_at, raw)
+        VALUES (%s,%s,%s,%s, COALESCE(%s::timestamptz, now()), %s)
+        """,
+        (
+            product_id,
+            source_id,
+            SOURCE_HOMEPAGE_BYPOS,
+            fields,
+            rec.get("fetched_at"),
+            Jsonb(_jsonable(raw)),
+        ),
+    )
+
+    update_quality(conn, product_id)
+    return product_id
+
+
+def load_bypos_record_safe(
+    conn: psycopg.Connection, rec: dict[str, Any], source_id: str, raw: dict
+) -> bool:
+    """Load one bypos record inside a savepoint (see :func:`load_record_safe`)."""
+    try:
+        with conn.transaction():
+            load_bypos_record(conn, rec, source_id, raw)
+        return True
+    except Exception as exc:  # noqa: BLE001 - per-record isolation is intentional
+        logger.warning("skipping bypos record gtin=%s: %s", rec.get("gtin"), exc)
+        return False
+
+
 def _jsonable(raw: dict) -> dict:
     """Drop values that are not JSON-serializable from a raw record."""
     try: