chore(tools): add bypos-collector (条码批量采集器源码备份)
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
novaalphastrikeomegaz663
@@ -0,0 +1,481 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ---- upstream config ----
|
||||
// sdogid is the 云店 account license id observed in the live request. It is
|
||||
// configurable so the tool is not tied to a single account.
|
||||
const defaultSdogID = "137966"
|
||||
|
||||
const endpoint = "http://zc.bypos.net/byGoodsService/byMessage.asmx/GetGoodsinfo"
|
||||
|
||||
var stringTagRe = regexp.MustCompile(`(?s)<string[^>]*>(.*)</string>`)
|
||||
|
||||
// Product is the normalized record we persist (one JSON object per line).
|
||||
type Product struct {
|
||||
Barcode string `json:"barcode"`
|
||||
Name string `json:"name"` // item_name 品名
|
||||
Spec string `json:"spec"` // item_size 规格
|
||||
Unit string `json:"unit"` // unit_no 单位
|
||||
Area string `json:"area"` // item_area 产地/地区
|
||||
Manufacturer string `json:"manufacturer"` // birth_com 生产企业
|
||||
License string `json:"license"` // birth_doc 生产许可
|
||||
InPrice string `json:"in_price"` // 建议进价
|
||||
SellPrice string `json:"sell_price"` // 建议零售价
|
||||
Status string `json:"status"` // hit / miss / invalid / error
|
||||
RetMsg string `json:"retmsg"` // 原始返回信息
|
||||
FetchedAt string `json:"fetched_at"` // RFC3339
|
||||
Source string `json:"source"` // zc.bypos.net
|
||||
}
|
||||
|
||||
// upstream raw fields
|
||||
type rawResp struct {
|
||||
RetCode string `json:"retcode"`
|
||||
RetMsg string `json:"retmsg"`
|
||||
Barcode string `json:"barcode"`
|
||||
ItemName string `json:"item_name"`
|
||||
UnitNo string `json:"unit_no"`
|
||||
ItemSize string `json:"item_size"`
|
||||
ItemArea string `json:"item_area"`
|
||||
BirthCom string `json:"birth_com"`
|
||||
BirthDoc string `json:"birth_doc"`
|
||||
InPrice string `json:"inprice"`
|
||||
SellPrice string `json:"sellprice"`
|
||||
}
|
||||
|
||||
func md5hex(s string) string {
|
||||
h := md5.Sum([]byte(s))
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
// ean13Check computes the EAN-13 check digit for a 12-digit body.
|
||||
func ean13Check(body string) (string, bool) {
|
||||
if len(body) != 12 {
|
||||
return "", false
|
||||
}
|
||||
sum := 0
|
||||
for i := 0; i < 12; i++ {
|
||||
c := body[i]
|
||||
if c < '0' || c > '9' {
|
||||
return "", false
|
||||
}
|
||||
d := int(c - '0')
|
||||
if i%2 == 0 {
|
||||
sum += d
|
||||
} else {
|
||||
sum += d * 3
|
||||
}
|
||||
}
|
||||
chk := (10 - (sum % 10)) % 10
|
||||
return body + strconv.Itoa(chk), true
|
||||
}
|
||||
|
||||
// lookup queries the upstream central library for one barcode.
|
||||
func (c *Collector) lookup(ctx context.Context, barcode string) (*Product, error) {
|
||||
tsMs := strconv.FormatInt(time.Now().Unix()*1000, 10) // always ends in 000
|
||||
sparm1 := md5hex(c.sdogID)
|
||||
sparm2 := md5hex(barcode + tsMs)
|
||||
sparm3 := tsMs[:10]
|
||||
q := url.Values{}
|
||||
q.Set("sdogid", c.sdogID)
|
||||
q.Set("regnum", "1")
|
||||
q.Set("barcode", barcode)
|
||||
q.Set("sparm1", sparm1)
|
||||
q.Set("sparm2", sparm2)
|
||||
q.Set("sparm3", sparm3)
|
||||
q.Set("sparm4", "")
|
||||
q.Set("barcodetype", "yunpos")
|
||||
reqURL := endpoint + "?" + q.Encode()
|
||||
|
||||
req, _ := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0")
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
inner := b
|
||||
if m := stringTagRe.FindSubmatch(b); m != nil {
|
||||
inner = m[1]
|
||||
}
|
||||
var r rawResp
|
||||
if err := json.Unmarshal(inner, &r); err != nil {
|
||||
return nil, fmt.Errorf("parse: %v (body=%.120s)", err, string(b))
|
||||
}
|
||||
p := &Product{
|
||||
Barcode: barcode,
|
||||
RetMsg: r.RetMsg,
|
||||
FetchedAt: time.Now().Format(time.RFC3339),
|
||||
Source: "zc.bypos.net",
|
||||
}
|
||||
if r.RetCode == "1" {
|
||||
p.Status = "hit"
|
||||
p.Name = strings.TrimSpace(r.ItemName)
|
||||
p.Spec = strings.TrimSpace(r.ItemSize)
|
||||
p.Unit = strings.TrimSpace(r.UnitNo)
|
||||
p.Area = strings.TrimSpace(r.ItemArea)
|
||||
p.Manufacturer = strings.TrimSpace(r.BirthCom)
|
||||
p.License = strings.TrimSpace(r.BirthDoc)
|
||||
p.InPrice = strings.TrimSpace(r.InPrice)
|
||||
p.SellPrice = strings.TrimSpace(r.SellPrice)
|
||||
} else if strings.Contains(r.RetMsg, "非国标") || strings.Contains(r.RetMsg, "参数异常") {
|
||||
p.Status = "invalid"
|
||||
} else {
|
||||
p.Status = "miss"
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// ---- job / collector state ----
|
||||
|
||||
type Stats struct {
|
||||
Running bool `json:"running"`
|
||||
Total int64 `json:"total"`
|
||||
Done int64 `json:"done"`
|
||||
Hits int64 `json:"hits"`
|
||||
Miss int64 `json:"miss"`
|
||||
Invalid int64 `json:"invalid"`
|
||||
Errors int64 `json:"errors"`
|
||||
Skipped int64 `json:"skipped"`
|
||||
Current string `json:"current"`
|
||||
OutFile string `json:"out_file"`
|
||||
StartedAt string `json:"started_at"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type Collector struct {
|
||||
mu sync.Mutex
|
||||
client *http.Client
|
||||
sdogID string
|
||||
outPath string
|
||||
outFile *os.File
|
||||
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
|
||||
// atomic counters
|
||||
total, done, hits, miss, invalid, errors, skipped int64
|
||||
running int32
|
||||
|
||||
current atomic.Value // string
|
||||
startedAt string
|
||||
message string
|
||||
|
||||
seen map[string]struct{} // barcodes already in output (dedupe / resume)
|
||||
recent []Product // ring of last results for UI
|
||||
}
|
||||
|
||||
func NewCollector(sdogID string) *Collector {
|
||||
if sdogID == "" {
|
||||
sdogID = defaultSdogID
|
||||
}
|
||||
c := &Collector{
|
||||
client: &http.Client{Timeout: 25 * time.Second},
|
||||
sdogID: sdogID,
|
||||
seen: map[string]struct{}{},
|
||||
}
|
||||
c.current.Store("")
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Collector) isRunning() bool { return atomic.LoadInt32(&c.running) == 1 }
|
||||
|
||||
func (c *Collector) snapshot() Stats {
|
||||
cur, _ := c.current.Load().(string)
|
||||
c.mu.Lock()
|
||||
msg := c.message
|
||||
out := c.outPath
|
||||
started := c.startedAt
|
||||
c.mu.Unlock()
|
||||
return Stats{
|
||||
Running: c.isRunning(),
|
||||
Total: atomic.LoadInt64(&c.total),
|
||||
Done: atomic.LoadInt64(&c.done),
|
||||
Hits: atomic.LoadInt64(&c.hits),
|
||||
Miss: atomic.LoadInt64(&c.miss),
|
||||
Invalid: atomic.LoadInt64(&c.invalid),
|
||||
Errors: atomic.LoadInt64(&c.errors),
|
||||
Skipped: atomic.LoadInt64(&c.skipped),
|
||||
Current: cur,
|
||||
OutFile: out,
|
||||
StartedAt: started,
|
||||
Message: msg,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Collector) recentResults() []Product {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
out := make([]Product, len(c.recent))
|
||||
copy(out, c.recent)
|
||||
return out
|
||||
}
|
||||
|
||||
func (c *Collector) pushRecent(p Product) {
|
||||
c.mu.Lock()
|
||||
c.recent = append(c.recent, p)
|
||||
if len(c.recent) > 60 {
|
||||
c.recent = c.recent[len(c.recent)-60:]
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// loadSeen reads an existing output file to build the dedupe set (for resume).
|
||||
func (c *Collector) loadSeen(path string) error {
|
||||
c.seen = map[string]struct{}{}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
dec := json.NewDecoder(f)
|
||||
for {
|
||||
var p Product
|
||||
if err := dec.Decode(&p); err != nil {
|
||||
break
|
||||
}
|
||||
if p.Barcode != "" {
|
||||
c.seen[p.Barcode] = struct{}{}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type JobReq struct {
|
||||
Mode string `json:"mode"` // "range" | "list"
|
||||
StartBody string `json:"start_body"` // 12-digit body (range mode)
|
||||
EndBody string `json:"end_body"` // 12-digit body (range mode)
|
||||
List string `json:"list"` // newline/space separated barcodes (list mode)
|
||||
Concurrency int `json:"concurrency"` // parallel requests
|
||||
DelayMs int `json:"delay_ms"` // min interval between request starts
|
||||
LogMiss bool `json:"log_miss"` // also write miss/invalid lines
|
||||
OutFile string `json:"out_file"`
|
||||
SdogID string `json:"sdog_id"`
|
||||
}
|
||||
|
||||
func sanitizeBarcodes(s string) []string {
|
||||
fields := regexp.MustCompile(`[^0-9]+`).Split(s, -1)
|
||||
var out []string
|
||||
for _, f := range fields {
|
||||
if f != "" {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Start launches a collection job. Returns error if validation fails or busy.
|
||||
func (c *Collector) Start(req JobReq) error {
|
||||
if c.isRunning() {
|
||||
return fmt.Errorf("已有任务在运行")
|
||||
}
|
||||
if req.Concurrency <= 0 {
|
||||
req.Concurrency = 3
|
||||
}
|
||||
if req.Concurrency > 20 {
|
||||
req.Concurrency = 20
|
||||
}
|
||||
if req.DelayMs < 0 {
|
||||
req.DelayMs = 0
|
||||
}
|
||||
if req.OutFile == "" {
|
||||
req.OutFile = "products.jsonl"
|
||||
}
|
||||
if req.SdogID != "" {
|
||||
c.sdogID = req.SdogID
|
||||
}
|
||||
|
||||
// Build the list of barcodes to query.
|
||||
var barcodes []string
|
||||
switch req.Mode {
|
||||
case "list":
|
||||
barcodes = sanitizeBarcodes(req.List)
|
||||
if len(barcodes) == 0 {
|
||||
return fmt.Errorf("条码清单为空")
|
||||
}
|
||||
case "range":
|
||||
start, err := strconv.ParseInt(req.StartBody, 10, 64)
|
||||
if err != nil || len(req.StartBody) != 12 {
|
||||
return fmt.Errorf("起始码必须是 12 位数字(不含校验位)")
|
||||
}
|
||||
end, err := strconv.ParseInt(req.EndBody, 10, 64)
|
||||
if err != nil || len(req.EndBody) != 12 {
|
||||
return fmt.Errorf("结束码必须是 12 位数字(不含校验位)")
|
||||
}
|
||||
if end < start {
|
||||
return fmt.Errorf("结束码不能小于起始码")
|
||||
}
|
||||
if end-start+1 > 5_000_000 {
|
||||
return fmt.Errorf("单次范围过大(>500万),请缩小区间分批采集")
|
||||
}
|
||||
for v := start; v <= end; v++ {
|
||||
body := fmt.Sprintf("%012d", v)
|
||||
full, ok := ean13Check(body)
|
||||
if ok {
|
||||
barcodes = append(barcodes, full)
|
||||
}
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("未知模式: %s", req.Mode)
|
||||
}
|
||||
|
||||
abs, _ := filepath.Abs(req.OutFile)
|
||||
if err := c.loadSeen(abs); err != nil {
|
||||
return fmt.Errorf("读取已有文件失败: %v", err)
|
||||
}
|
||||
f, err := os.OpenFile(abs, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("打开输出文件失败: %v", err)
|
||||
}
|
||||
c.outFile = f
|
||||
c.outPath = abs
|
||||
|
||||
// reset counters
|
||||
atomic.StoreInt64(&c.total, int64(len(barcodes)))
|
||||
atomic.StoreInt64(&c.done, 0)
|
||||
atomic.StoreInt64(&c.hits, 0)
|
||||
atomic.StoreInt64(&c.miss, 0)
|
||||
atomic.StoreInt64(&c.invalid, 0)
|
||||
atomic.StoreInt64(&c.errors, 0)
|
||||
atomic.StoreInt64(&c.skipped, 0)
|
||||
c.mu.Lock()
|
||||
c.recent = nil
|
||||
c.startedAt = time.Now().Format(time.RFC3339)
|
||||
c.message = ""
|
||||
c.mu.Unlock()
|
||||
atomic.StoreInt32(&c.running, 1)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
c.cancel = cancel
|
||||
|
||||
go c.run(ctx, barcodes, req)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Collector) Stop() {
|
||||
if c.cancel != nil {
|
||||
c.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Collector) run(ctx context.Context, barcodes []string, req JobReq) {
|
||||
defer func() {
|
||||
atomic.StoreInt32(&c.running, 0)
|
||||
if c.outFile != nil {
|
||||
c.outFile.Sync()
|
||||
c.outFile.Close()
|
||||
c.outFile = nil
|
||||
}
|
||||
c.current.Store("")
|
||||
}()
|
||||
|
||||
jobs := make(chan string, req.Concurrency*2)
|
||||
var writeMu sync.Mutex
|
||||
|
||||
// global rate limiter: one token every DelayMs
|
||||
var ticker *time.Ticker
|
||||
if req.DelayMs > 0 {
|
||||
ticker = time.NewTicker(time.Duration(req.DelayMs) * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
}
|
||||
|
||||
worker := func() {
|
||||
defer c.wg.Done()
|
||||
for bc := range jobs {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if ticker != nil {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
c.current.Store(bc)
|
||||
p, err := c.lookup(ctx, bc)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
atomic.AddInt64(&c.errors, 1)
|
||||
atomic.AddInt64(&c.done, 1)
|
||||
ep := Product{Barcode: bc, Status: "error", RetMsg: err.Error(), FetchedAt: time.Now().Format(time.RFC3339), Source: "zc.bypos.net"}
|
||||
c.pushRecent(ep)
|
||||
continue
|
||||
}
|
||||
switch p.Status {
|
||||
case "hit":
|
||||
atomic.AddInt64(&c.hits, 1)
|
||||
case "miss":
|
||||
atomic.AddInt64(&c.miss, 1)
|
||||
case "invalid":
|
||||
atomic.AddInt64(&c.invalid, 1)
|
||||
}
|
||||
atomic.AddInt64(&c.done, 1)
|
||||
c.pushRecent(*p)
|
||||
if p.Status == "hit" || req.LogMiss {
|
||||
line, _ := json.Marshal(p)
|
||||
writeMu.Lock()
|
||||
c.outFile.Write(line)
|
||||
c.outFile.Write([]byte("\n"))
|
||||
c.seen[bc] = struct{}{}
|
||||
writeMu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < req.Concurrency; i++ {
|
||||
c.wg.Add(1)
|
||||
go worker()
|
||||
}
|
||||
|
||||
for _, bc := range barcodes {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
if _, ok := c.seen[bc]; ok {
|
||||
atomic.AddInt64(&c.skipped, 1)
|
||||
atomic.AddInt64(&c.done, 1)
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case jobs <- bc:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}
|
||||
close(jobs)
|
||||
c.wg.Wait()
|
||||
|
||||
c.mu.Lock()
|
||||
if ctx.Err() != nil {
|
||||
c.message = "已停止"
|
||||
} else {
|
||||
c.message = "采集完成"
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
Reference in New Issue
Block a user