package main import ( "context" "crypto/md5" "encoding/hex" "encoding/json" "fmt" "io" "net/http" "net/url" "os" "path/filepath" "regexp" "strconv" "strings" "sync" "sync/atomic" "time" ) // ---- upstream config ---- // sdogid is the 云店 account license id observed in the live request. It is // configurable so the tool is not tied to a single account. const defaultSdogID = "137966" const endpoint = "http://zc.bypos.net/byGoodsService/byMessage.asmx/GetGoodsinfo" var stringTagRe = regexp.MustCompile(`(?s)]*>(.*)`) // Product is the normalized record we persist (one JSON object per line). type Product struct { Barcode string `json:"barcode"` Name string `json:"name"` // item_name 品名 Spec string `json:"spec"` // item_size 规格 Unit string `json:"unit"` // unit_no 单位 Area string `json:"area"` // item_area 产地/地区 Manufacturer string `json:"manufacturer"` // birth_com 生产企业 License string `json:"license"` // birth_doc 生产许可 InPrice string `json:"in_price"` // 建议进价 SellPrice string `json:"sell_price"` // 建议零售价 Status string `json:"status"` // hit / miss / invalid / error RetMsg string `json:"retmsg"` // 原始返回信息 FetchedAt string `json:"fetched_at"` // RFC3339 Source string `json:"source"` // zc.bypos.net } // upstream raw fields type rawResp struct { RetCode string `json:"retcode"` RetMsg string `json:"retmsg"` Barcode string `json:"barcode"` ItemName string `json:"item_name"` UnitNo string `json:"unit_no"` ItemSize string `json:"item_size"` ItemArea string `json:"item_area"` BirthCom string `json:"birth_com"` BirthDoc string `json:"birth_doc"` InPrice string `json:"inprice"` SellPrice string `json:"sellprice"` } func md5hex(s string) string { h := md5.Sum([]byte(s)) return hex.EncodeToString(h[:]) } // ean13Check computes the EAN-13 check digit for a 12-digit body. func ean13Check(body string) (string, bool) { if len(body) != 12 { return "", false } sum := 0 for i := 0; i < 12; i++ { c := body[i] if c < '0' || c > '9' { return "", false } d := int(c - '0') if i%2 == 0 { sum += d } else { sum += d * 3 } } chk := (10 - (sum % 10)) % 10 return body + strconv.Itoa(chk), true } // lookup queries the upstream central library for one barcode. func (c *Collector) lookup(ctx context.Context, barcode string) (*Product, error) { tsMs := strconv.FormatInt(time.Now().Unix()*1000, 10) // always ends in 000 sparm1 := md5hex(c.sdogID) sparm2 := md5hex(barcode + tsMs) sparm3 := tsMs[:10] q := url.Values{} q.Set("sdogid", c.sdogID) q.Set("regnum", "1") q.Set("barcode", barcode) q.Set("sparm1", sparm1) q.Set("sparm2", sparm2) q.Set("sparm3", sparm3) q.Set("sparm4", "") q.Set("barcodetype", "yunpos") reqURL := endpoint + "?" + q.Encode() req, _ := http.NewRequestWithContext(ctx, "GET", reqURL, nil) req.Header.Set("User-Agent", "Mozilla/5.0") resp, err := c.client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() b, err := io.ReadAll(resp.Body) if err != nil { return nil, err } inner := b if m := stringTagRe.FindSubmatch(b); m != nil { inner = m[1] } var r rawResp if err := json.Unmarshal(inner, &r); err != nil { return nil, fmt.Errorf("parse: %v (body=%.120s)", err, string(b)) } p := &Product{ Barcode: barcode, RetMsg: r.RetMsg, FetchedAt: time.Now().Format(time.RFC3339), Source: "zc.bypos.net", } if r.RetCode == "1" { p.Status = "hit" p.Name = strings.TrimSpace(r.ItemName) p.Spec = strings.TrimSpace(r.ItemSize) p.Unit = strings.TrimSpace(r.UnitNo) p.Area = strings.TrimSpace(r.ItemArea) p.Manufacturer = strings.TrimSpace(r.BirthCom) p.License = strings.TrimSpace(r.BirthDoc) p.InPrice = strings.TrimSpace(r.InPrice) p.SellPrice = strings.TrimSpace(r.SellPrice) } else if strings.Contains(r.RetMsg, "非国标") || strings.Contains(r.RetMsg, "参数异常") { p.Status = "invalid" } else { p.Status = "miss" } return p, nil } // ---- job / collector state ---- type Stats struct { Running bool `json:"running"` Total int64 `json:"total"` Done int64 `json:"done"` Hits int64 `json:"hits"` Miss int64 `json:"miss"` Invalid int64 `json:"invalid"` Errors int64 `json:"errors"` Skipped int64 `json:"skipped"` Current string `json:"current"` OutFile string `json:"out_file"` StartedAt string `json:"started_at"` Message string `json:"message"` } type Collector struct { mu sync.Mutex client *http.Client sdogID string outPath string outFile *os.File cancel context.CancelFunc wg sync.WaitGroup // atomic counters total, done, hits, miss, invalid, errors, skipped int64 running int32 current atomic.Value // string startedAt string message string seen map[string]struct{} // barcodes already in output (dedupe / resume) recent []Product // ring of last results for UI } func NewCollector(sdogID string) *Collector { if sdogID == "" { sdogID = defaultSdogID } c := &Collector{ client: &http.Client{Timeout: 25 * time.Second}, sdogID: sdogID, seen: map[string]struct{}{}, } c.current.Store("") return c } func (c *Collector) isRunning() bool { return atomic.LoadInt32(&c.running) == 1 } func (c *Collector) snapshot() Stats { cur, _ := c.current.Load().(string) c.mu.Lock() msg := c.message out := c.outPath started := c.startedAt c.mu.Unlock() return Stats{ Running: c.isRunning(), Total: atomic.LoadInt64(&c.total), Done: atomic.LoadInt64(&c.done), Hits: atomic.LoadInt64(&c.hits), Miss: atomic.LoadInt64(&c.miss), Invalid: atomic.LoadInt64(&c.invalid), Errors: atomic.LoadInt64(&c.errors), Skipped: atomic.LoadInt64(&c.skipped), Current: cur, OutFile: out, StartedAt: started, Message: msg, } } func (c *Collector) recentResults() []Product { c.mu.Lock() defer c.mu.Unlock() out := make([]Product, len(c.recent)) copy(out, c.recent) return out } func (c *Collector) pushRecent(p Product) { c.mu.Lock() c.recent = append(c.recent, p) if len(c.recent) > 60 { c.recent = c.recent[len(c.recent)-60:] } c.mu.Unlock() } // loadSeen reads an existing output file to build the dedupe set (for resume). func (c *Collector) loadSeen(path string) error { c.seen = map[string]struct{}{} f, err := os.Open(path) if err != nil { if os.IsNotExist(err) { return nil } return err } defer f.Close() dec := json.NewDecoder(f) for { var p Product if err := dec.Decode(&p); err != nil { break } if p.Barcode != "" { c.seen[p.Barcode] = struct{}{} } } return nil } type JobReq struct { Mode string `json:"mode"` // "range" | "list" StartBody string `json:"start_body"` // 12-digit body (range mode) EndBody string `json:"end_body"` // 12-digit body (range mode) List string `json:"list"` // newline/space separated barcodes (list mode) Concurrency int `json:"concurrency"` // parallel requests DelayMs int `json:"delay_ms"` // min interval between request starts LogMiss bool `json:"log_miss"` // also write miss/invalid lines OutFile string `json:"out_file"` SdogID string `json:"sdog_id"` } func sanitizeBarcodes(s string) []string { fields := regexp.MustCompile(`[^0-9]+`).Split(s, -1) var out []string for _, f := range fields { if f != "" { out = append(out, f) } } return out } // Start launches a collection job. Returns error if validation fails or busy. func (c *Collector) Start(req JobReq) error { if c.isRunning() { return fmt.Errorf("已有任务在运行") } if req.Concurrency <= 0 { req.Concurrency = 3 } if req.Concurrency > 20 { req.Concurrency = 20 } if req.DelayMs < 0 { req.DelayMs = 0 } if req.OutFile == "" { req.OutFile = "products.jsonl" } if req.SdogID != "" { c.sdogID = req.SdogID } // Build the list of barcodes to query. var barcodes []string switch req.Mode { case "list": barcodes = sanitizeBarcodes(req.List) if len(barcodes) == 0 { return fmt.Errorf("条码清单为空") } case "range": start, err := strconv.ParseInt(req.StartBody, 10, 64) if err != nil || len(req.StartBody) != 12 { return fmt.Errorf("起始码必须是 12 位数字(不含校验位)") } end, err := strconv.ParseInt(req.EndBody, 10, 64) if err != nil || len(req.EndBody) != 12 { return fmt.Errorf("结束码必须是 12 位数字(不含校验位)") } if end < start { return fmt.Errorf("结束码不能小于起始码") } if end-start+1 > 5_000_000 { return fmt.Errorf("单次范围过大(>500万),请缩小区间分批采集") } for v := start; v <= end; v++ { body := fmt.Sprintf("%012d", v) full, ok := ean13Check(body) if ok { barcodes = append(barcodes, full) } } default: return fmt.Errorf("未知模式: %s", req.Mode) } abs, _ := filepath.Abs(req.OutFile) if err := c.loadSeen(abs); err != nil { return fmt.Errorf("读取已有文件失败: %v", err) } f, err := os.OpenFile(abs, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) if err != nil { return fmt.Errorf("打开输出文件失败: %v", err) } c.outFile = f c.outPath = abs // reset counters atomic.StoreInt64(&c.total, int64(len(barcodes))) atomic.StoreInt64(&c.done, 0) atomic.StoreInt64(&c.hits, 0) atomic.StoreInt64(&c.miss, 0) atomic.StoreInt64(&c.invalid, 0) atomic.StoreInt64(&c.errors, 0) atomic.StoreInt64(&c.skipped, 0) c.mu.Lock() c.recent = nil c.startedAt = time.Now().Format(time.RFC3339) c.message = "" c.mu.Unlock() atomic.StoreInt32(&c.running, 1) ctx, cancel := context.WithCancel(context.Background()) c.cancel = cancel go c.run(ctx, barcodes, req) return nil } func (c *Collector) Stop() { if c.cancel != nil { c.cancel() } } func (c *Collector) run(ctx context.Context, barcodes []string, req JobReq) { defer func() { atomic.StoreInt32(&c.running, 0) if c.outFile != nil { c.outFile.Sync() c.outFile.Close() c.outFile = nil } c.current.Store("") }() jobs := make(chan string, req.Concurrency*2) var writeMu sync.Mutex // global rate limiter: one token every DelayMs var ticker *time.Ticker if req.DelayMs > 0 { ticker = time.NewTicker(time.Duration(req.DelayMs) * time.Millisecond) defer ticker.Stop() } worker := func() { defer c.wg.Done() for bc := range jobs { if ctx.Err() != nil { return } if ticker != nil { select { case <-ticker.C: case <-ctx.Done(): return } } c.current.Store(bc) p, err := c.lookup(ctx, bc) if err != nil { if ctx.Err() != nil { return } atomic.AddInt64(&c.errors, 1) atomic.AddInt64(&c.done, 1) ep := Product{Barcode: bc, Status: "error", RetMsg: err.Error(), FetchedAt: time.Now().Format(time.RFC3339), Source: "zc.bypos.net"} c.pushRecent(ep) continue } switch p.Status { case "hit": atomic.AddInt64(&c.hits, 1) case "miss": atomic.AddInt64(&c.miss, 1) case "invalid": atomic.AddInt64(&c.invalid, 1) } atomic.AddInt64(&c.done, 1) c.pushRecent(*p) if p.Status == "hit" || req.LogMiss { line, _ := json.Marshal(p) writeMu.Lock() c.outFile.Write(line) c.outFile.Write([]byte("\n")) c.seen[bc] = struct{}{} writeMu.Unlock() } } } for i := 0; i < req.Concurrency; i++ { c.wg.Add(1) go worker() } for _, bc := range barcodes { if ctx.Err() != nil { break } if _, ok := c.seen[bc]; ok { atomic.AddInt64(&c.skipped, 1) atomic.AddInt64(&c.done, 1) continue } select { case jobs <- bc: case <-ctx.Done(): } } close(jobs) c.wg.Wait() c.mu.Lock() if ctx.Err() != nil { c.message = "已停止" } else { c.message = "采集完成" } c.mu.Unlock() }