feat(ingest): country-focused seeding (collect domestic CN products)
Add a market-focused seeding path so the catalogue can be built from domestic products rather than the English-heavy global default: - adapter.fetch_by_country(country): OFF search filtered by countries_tags_en, sorted by unique_scans_n (most-scanned first), de-duplicated across pages since OFF popularity ordering is unstable. - is_cn_gs1(code): True for GS1-China company prefixes (690-699), i.e. genuinely domestic items vs. imports merely sold in China. - seed_off --country <slug> [--domestic-only] [--page-size/--max-pages]: e.g. 'seed_off --country china --domestic-only' loads only 69x barcodes. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
@@ -157,6 +157,60 @@ class OpenFoodFactsAdapter:
|
||||
if reached_old or len(products) < page_size:
|
||||
return
|
||||
|
||||
def fetch_by_country(
|
||||
self,
|
||||
country: str,
|
||||
*,
|
||||
page_size: int = 100,
|
||||
max_pages: int = 10,
|
||||
sort_by: str = "unique_scans_n",
|
||||
) -> Iterator[dict]:
|
||||
"""Yield products sold in ``country`` (an OFF ``countries_tags_en`` slug).
|
||||
|
||||
Used to seed a market-specific catalogue (e.g. ``china``). Results are
|
||||
sorted by ``sort_by`` (default ``unique_scans_n`` so the most-scanned,
|
||||
best-known products come first) and de-duplicated across pages, since
|
||||
OFF's popularity ordering is not stable between page requests.
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
for page in range(1, max_pages + 1):
|
||||
resp = self._get(
|
||||
_SEARCH_URL,
|
||||
params={
|
||||
"fields": _SEARCH_FIELDS,
|
||||
"countries_tags_en": country,
|
||||
"sort_by": sort_by,
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
},
|
||||
)
|
||||
products = resp.json().get("products") or []
|
||||
if not products:
|
||||
return
|
||||
new_on_page = 0
|
||||
for prod in products:
|
||||
code = str(prod.get("code") or "")
|
||||
if code and code in seen:
|
||||
continue
|
||||
if code:
|
||||
seen.add(code)
|
||||
new_on_page += 1
|
||||
yield prod
|
||||
if len(products) < page_size or new_on_page == 0:
|
||||
return
|
||||
|
||||
|
||||
def is_cn_gs1(code: str | None) -> bool:
|
||||
"""Return True for a GS1 China company prefix (barcodes starting 690-699).
|
||||
|
||||
These identify products registered with GS1 China, i.e. genuinely domestic
|
||||
items, as opposed to imported goods merely tagged as sold in China.
|
||||
"""
|
||||
if not code:
|
||||
return False
|
||||
code = code.strip()
|
||||
return len(code) >= 3 and code[:2] == "69" and code[2].isdigit()
|
||||
|
||||
|
||||
def read_dump(path: str | Path) -> Iterator[dict]:
|
||||
"""Yield raw product records from an OFF JSONL dump file.
|
||||
|
||||
Reference in New Issue
Block a user