John Doe
a35bcd6647
- OFF incremental fetch via search API + persistent watermark (ingest_state, migration 0004) - GS1 barcode supplement adapter (offline mapping + GS1-style API) filling only gaps with field-level provenance - Non-GTIN dedup with canonical selection + merge_log; field-level conflict resolution (source trust > recency) - Quality scoring (0.4 completeness + 0.3 source trust + 0.2 multi-source + 0.1 freshness) wired into load/merge - Jobs: update_off, dedup, schedule; docs/ingestion-management.md - 19 new tests (pure + DB-integration), ruff clean Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
67 lines
2.3 KiB
Python
67 lines
2.3 KiB
Python
"""Lightweight recurring ingestion scheduler.
|
|
|
|
Runs one ingestion cycle (incremental OFF update, then dedup) on a fixed
|
|
interval. Dependency-free: a plain sleep loop rather than a cron/APScheduler
|
|
dependency, so it is trivial to run in a container or under systemd/supervisor.
|
|
|
|
Usage:
|
|
python -m opengoods.jobs.schedule --once # single cycle, then exit
|
|
python -m opengoods.jobs.schedule --interval 3600 # every hour
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
import time
|
|
from datetime import UTC, datetime
|
|
|
|
from opengoods.etl.load import default_dsn
|
|
from opengoods.jobs import dedup as dedup_job
|
|
from opengoods.jobs import update_off as update_job
|
|
|
|
|
|
def _cycle(args: argparse.Namespace) -> None:
|
|
ts = datetime.now(UTC).isoformat(timespec="seconds")
|
|
print(f"[{ts}] cycle start")
|
|
update_job.run(
|
|
argparse.Namespace(
|
|
since=None,
|
|
page_size=args.page_size,
|
|
max_pages=args.max_pages,
|
|
min_interval=args.min_interval,
|
|
dsn=args.dsn,
|
|
)
|
|
)
|
|
if not args.skip_dedup:
|
|
dedup_job.run(argparse.Namespace(actor="scheduler", dry_run=False, dsn=args.dsn))
|
|
|
|
|
|
def run(args: argparse.Namespace) -> int:
|
|
_cycle(args)
|
|
if args.once:
|
|
return 0
|
|
while True:
|
|
time.sleep(args.interval)
|
|
try:
|
|
_cycle(args)
|
|
except Exception as exc: # noqa: BLE001 - keep the loop alive across failures
|
|
print(f"cycle error: {exc}", file=sys.stderr)
|
|
return 0
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="Recurring OpenGoods ingestion")
|
|
parser.add_argument("--interval", type=int, default=3600, help="seconds between cycles")
|
|
parser.add_argument("--once", action="store_true", help="run a single cycle and exit")
|
|
parser.add_argument("--skip-dedup", action="store_true", help="run update only")
|
|
parser.add_argument("--page-size", type=int, default=100, help="search page size")
|
|
parser.add_argument("--max-pages", type=int, default=10, help="max pages to scan")
|
|
parser.add_argument("--min-interval", type=float, default=4.0, help="API throttle seconds")
|
|
parser.add_argument("--dsn", default=default_dsn(), help="PostgreSQL DSN")
|
|
return run(parser.parse_args(argv))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|