novaalphastrikeomegaz663
e746b9cd31
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
482 lines
12 KiB
Go
482 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/md5"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
)
|
|
|
|
// ---- upstream config ----
|
|
// sdogid is the 云店 account license id observed in the live request. It is
|
|
// configurable so the tool is not tied to a single account.
|
|
const defaultSdogID = "137966"
|
|
|
|
const endpoint = "http://zc.bypos.net/byGoodsService/byMessage.asmx/GetGoodsinfo"
|
|
|
|
var stringTagRe = regexp.MustCompile(`(?s)<string[^>]*>(.*)</string>`)
|
|
|
|
// Product is the normalized record we persist (one JSON object per line).
|
|
type Product struct {
|
|
Barcode string `json:"barcode"`
|
|
Name string `json:"name"` // item_name 品名
|
|
Spec string `json:"spec"` // item_size 规格
|
|
Unit string `json:"unit"` // unit_no 单位
|
|
Area string `json:"area"` // item_area 产地/地区
|
|
Manufacturer string `json:"manufacturer"` // birth_com 生产企业
|
|
License string `json:"license"` // birth_doc 生产许可
|
|
InPrice string `json:"in_price"` // 建议进价
|
|
SellPrice string `json:"sell_price"` // 建议零售价
|
|
Status string `json:"status"` // hit / miss / invalid / error
|
|
RetMsg string `json:"retmsg"` // 原始返回信息
|
|
FetchedAt string `json:"fetched_at"` // RFC3339
|
|
Source string `json:"source"` // zc.bypos.net
|
|
}
|
|
|
|
// upstream raw fields
|
|
type rawResp struct {
|
|
RetCode string `json:"retcode"`
|
|
RetMsg string `json:"retmsg"`
|
|
Barcode string `json:"barcode"`
|
|
ItemName string `json:"item_name"`
|
|
UnitNo string `json:"unit_no"`
|
|
ItemSize string `json:"item_size"`
|
|
ItemArea string `json:"item_area"`
|
|
BirthCom string `json:"birth_com"`
|
|
BirthDoc string `json:"birth_doc"`
|
|
InPrice string `json:"inprice"`
|
|
SellPrice string `json:"sellprice"`
|
|
}
|
|
|
|
func md5hex(s string) string {
|
|
h := md5.Sum([]byte(s))
|
|
return hex.EncodeToString(h[:])
|
|
}
|
|
|
|
// ean13Check computes the EAN-13 check digit for a 12-digit body.
|
|
func ean13Check(body string) (string, bool) {
|
|
if len(body) != 12 {
|
|
return "", false
|
|
}
|
|
sum := 0
|
|
for i := 0; i < 12; i++ {
|
|
c := body[i]
|
|
if c < '0' || c > '9' {
|
|
return "", false
|
|
}
|
|
d := int(c - '0')
|
|
if i%2 == 0 {
|
|
sum += d
|
|
} else {
|
|
sum += d * 3
|
|
}
|
|
}
|
|
chk := (10 - (sum % 10)) % 10
|
|
return body + strconv.Itoa(chk), true
|
|
}
|
|
|
|
// lookup queries the upstream central library for one barcode.
|
|
func (c *Collector) lookup(ctx context.Context, barcode string) (*Product, error) {
|
|
tsMs := strconv.FormatInt(time.Now().Unix()*1000, 10) // always ends in 000
|
|
sparm1 := md5hex(c.sdogID)
|
|
sparm2 := md5hex(barcode + tsMs)
|
|
sparm3 := tsMs[:10]
|
|
q := url.Values{}
|
|
q.Set("sdogid", c.sdogID)
|
|
q.Set("regnum", "1")
|
|
q.Set("barcode", barcode)
|
|
q.Set("sparm1", sparm1)
|
|
q.Set("sparm2", sparm2)
|
|
q.Set("sparm3", sparm3)
|
|
q.Set("sparm4", "")
|
|
q.Set("barcodetype", "yunpos")
|
|
reqURL := endpoint + "?" + q.Encode()
|
|
|
|
req, _ := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
|
|
req.Header.Set("User-Agent", "Mozilla/5.0")
|
|
resp, err := c.client.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
b, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
inner := b
|
|
if m := stringTagRe.FindSubmatch(b); m != nil {
|
|
inner = m[1]
|
|
}
|
|
var r rawResp
|
|
if err := json.Unmarshal(inner, &r); err != nil {
|
|
return nil, fmt.Errorf("parse: %v (body=%.120s)", err, string(b))
|
|
}
|
|
p := &Product{
|
|
Barcode: barcode,
|
|
RetMsg: r.RetMsg,
|
|
FetchedAt: time.Now().Format(time.RFC3339),
|
|
Source: "zc.bypos.net",
|
|
}
|
|
if r.RetCode == "1" {
|
|
p.Status = "hit"
|
|
p.Name = strings.TrimSpace(r.ItemName)
|
|
p.Spec = strings.TrimSpace(r.ItemSize)
|
|
p.Unit = strings.TrimSpace(r.UnitNo)
|
|
p.Area = strings.TrimSpace(r.ItemArea)
|
|
p.Manufacturer = strings.TrimSpace(r.BirthCom)
|
|
p.License = strings.TrimSpace(r.BirthDoc)
|
|
p.InPrice = strings.TrimSpace(r.InPrice)
|
|
p.SellPrice = strings.TrimSpace(r.SellPrice)
|
|
} else if strings.Contains(r.RetMsg, "非国标") || strings.Contains(r.RetMsg, "参数异常") {
|
|
p.Status = "invalid"
|
|
} else {
|
|
p.Status = "miss"
|
|
}
|
|
return p, nil
|
|
}
|
|
|
|
// ---- job / collector state ----
|
|
|
|
type Stats struct {
|
|
Running bool `json:"running"`
|
|
Total int64 `json:"total"`
|
|
Done int64 `json:"done"`
|
|
Hits int64 `json:"hits"`
|
|
Miss int64 `json:"miss"`
|
|
Invalid int64 `json:"invalid"`
|
|
Errors int64 `json:"errors"`
|
|
Skipped int64 `json:"skipped"`
|
|
Current string `json:"current"`
|
|
OutFile string `json:"out_file"`
|
|
StartedAt string `json:"started_at"`
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
type Collector struct {
|
|
mu sync.Mutex
|
|
client *http.Client
|
|
sdogID string
|
|
outPath string
|
|
outFile *os.File
|
|
|
|
cancel context.CancelFunc
|
|
wg sync.WaitGroup
|
|
|
|
// atomic counters
|
|
total, done, hits, miss, invalid, errors, skipped int64
|
|
running int32
|
|
|
|
current atomic.Value // string
|
|
startedAt string
|
|
message string
|
|
|
|
seen map[string]struct{} // barcodes already in output (dedupe / resume)
|
|
recent []Product // ring of last results for UI
|
|
}
|
|
|
|
func NewCollector(sdogID string) *Collector {
|
|
if sdogID == "" {
|
|
sdogID = defaultSdogID
|
|
}
|
|
c := &Collector{
|
|
client: &http.Client{Timeout: 25 * time.Second},
|
|
sdogID: sdogID,
|
|
seen: map[string]struct{}{},
|
|
}
|
|
c.current.Store("")
|
|
return c
|
|
}
|
|
|
|
func (c *Collector) isRunning() bool { return atomic.LoadInt32(&c.running) == 1 }
|
|
|
|
func (c *Collector) snapshot() Stats {
|
|
cur, _ := c.current.Load().(string)
|
|
c.mu.Lock()
|
|
msg := c.message
|
|
out := c.outPath
|
|
started := c.startedAt
|
|
c.mu.Unlock()
|
|
return Stats{
|
|
Running: c.isRunning(),
|
|
Total: atomic.LoadInt64(&c.total),
|
|
Done: atomic.LoadInt64(&c.done),
|
|
Hits: atomic.LoadInt64(&c.hits),
|
|
Miss: atomic.LoadInt64(&c.miss),
|
|
Invalid: atomic.LoadInt64(&c.invalid),
|
|
Errors: atomic.LoadInt64(&c.errors),
|
|
Skipped: atomic.LoadInt64(&c.skipped),
|
|
Current: cur,
|
|
OutFile: out,
|
|
StartedAt: started,
|
|
Message: msg,
|
|
}
|
|
}
|
|
|
|
func (c *Collector) recentResults() []Product {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
out := make([]Product, len(c.recent))
|
|
copy(out, c.recent)
|
|
return out
|
|
}
|
|
|
|
func (c *Collector) pushRecent(p Product) {
|
|
c.mu.Lock()
|
|
c.recent = append(c.recent, p)
|
|
if len(c.recent) > 60 {
|
|
c.recent = c.recent[len(c.recent)-60:]
|
|
}
|
|
c.mu.Unlock()
|
|
}
|
|
|
|
// loadSeen reads an existing output file to build the dedupe set (for resume).
|
|
func (c *Collector) loadSeen(path string) error {
|
|
c.seen = map[string]struct{}{}
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
dec := json.NewDecoder(f)
|
|
for {
|
|
var p Product
|
|
if err := dec.Decode(&p); err != nil {
|
|
break
|
|
}
|
|
if p.Barcode != "" {
|
|
c.seen[p.Barcode] = struct{}{}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type JobReq struct {
|
|
Mode string `json:"mode"` // "range" | "list"
|
|
StartBody string `json:"start_body"` // 12-digit body (range mode)
|
|
EndBody string `json:"end_body"` // 12-digit body (range mode)
|
|
List string `json:"list"` // newline/space separated barcodes (list mode)
|
|
Concurrency int `json:"concurrency"` // parallel requests
|
|
DelayMs int `json:"delay_ms"` // min interval between request starts
|
|
LogMiss bool `json:"log_miss"` // also write miss/invalid lines
|
|
OutFile string `json:"out_file"`
|
|
SdogID string `json:"sdog_id"`
|
|
}
|
|
|
|
func sanitizeBarcodes(s string) []string {
|
|
fields := regexp.MustCompile(`[^0-9]+`).Split(s, -1)
|
|
var out []string
|
|
for _, f := range fields {
|
|
if f != "" {
|
|
out = append(out, f)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Start launches a collection job. Returns error if validation fails or busy.
|
|
func (c *Collector) Start(req JobReq) error {
|
|
if c.isRunning() {
|
|
return fmt.Errorf("已有任务在运行")
|
|
}
|
|
if req.Concurrency <= 0 {
|
|
req.Concurrency = 3
|
|
}
|
|
if req.Concurrency > 20 {
|
|
req.Concurrency = 20
|
|
}
|
|
if req.DelayMs < 0 {
|
|
req.DelayMs = 0
|
|
}
|
|
if req.OutFile == "" {
|
|
req.OutFile = "products.jsonl"
|
|
}
|
|
if req.SdogID != "" {
|
|
c.sdogID = req.SdogID
|
|
}
|
|
|
|
// Build the list of barcodes to query.
|
|
var barcodes []string
|
|
switch req.Mode {
|
|
case "list":
|
|
barcodes = sanitizeBarcodes(req.List)
|
|
if len(barcodes) == 0 {
|
|
return fmt.Errorf("条码清单为空")
|
|
}
|
|
case "range":
|
|
start, err := strconv.ParseInt(req.StartBody, 10, 64)
|
|
if err != nil || len(req.StartBody) != 12 {
|
|
return fmt.Errorf("起始码必须是 12 位数字(不含校验位)")
|
|
}
|
|
end, err := strconv.ParseInt(req.EndBody, 10, 64)
|
|
if err != nil || len(req.EndBody) != 12 {
|
|
return fmt.Errorf("结束码必须是 12 位数字(不含校验位)")
|
|
}
|
|
if end < start {
|
|
return fmt.Errorf("结束码不能小于起始码")
|
|
}
|
|
if end-start+1 > 5_000_000 {
|
|
return fmt.Errorf("单次范围过大(>500万),请缩小区间分批采集")
|
|
}
|
|
for v := start; v <= end; v++ {
|
|
body := fmt.Sprintf("%012d", v)
|
|
full, ok := ean13Check(body)
|
|
if ok {
|
|
barcodes = append(barcodes, full)
|
|
}
|
|
}
|
|
default:
|
|
return fmt.Errorf("未知模式: %s", req.Mode)
|
|
}
|
|
|
|
abs, _ := filepath.Abs(req.OutFile)
|
|
if err := c.loadSeen(abs); err != nil {
|
|
return fmt.Errorf("读取已有文件失败: %v", err)
|
|
}
|
|
f, err := os.OpenFile(abs, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
|
if err != nil {
|
|
return fmt.Errorf("打开输出文件失败: %v", err)
|
|
}
|
|
c.outFile = f
|
|
c.outPath = abs
|
|
|
|
// reset counters
|
|
atomic.StoreInt64(&c.total, int64(len(barcodes)))
|
|
atomic.StoreInt64(&c.done, 0)
|
|
atomic.StoreInt64(&c.hits, 0)
|
|
atomic.StoreInt64(&c.miss, 0)
|
|
atomic.StoreInt64(&c.invalid, 0)
|
|
atomic.StoreInt64(&c.errors, 0)
|
|
atomic.StoreInt64(&c.skipped, 0)
|
|
c.mu.Lock()
|
|
c.recent = nil
|
|
c.startedAt = time.Now().Format(time.RFC3339)
|
|
c.message = ""
|
|
c.mu.Unlock()
|
|
atomic.StoreInt32(&c.running, 1)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
c.cancel = cancel
|
|
|
|
go c.run(ctx, barcodes, req)
|
|
return nil
|
|
}
|
|
|
|
func (c *Collector) Stop() {
|
|
if c.cancel != nil {
|
|
c.cancel()
|
|
}
|
|
}
|
|
|
|
func (c *Collector) run(ctx context.Context, barcodes []string, req JobReq) {
|
|
defer func() {
|
|
atomic.StoreInt32(&c.running, 0)
|
|
if c.outFile != nil {
|
|
c.outFile.Sync()
|
|
c.outFile.Close()
|
|
c.outFile = nil
|
|
}
|
|
c.current.Store("")
|
|
}()
|
|
|
|
jobs := make(chan string, req.Concurrency*2)
|
|
var writeMu sync.Mutex
|
|
|
|
// global rate limiter: one token every DelayMs
|
|
var ticker *time.Ticker
|
|
if req.DelayMs > 0 {
|
|
ticker = time.NewTicker(time.Duration(req.DelayMs) * time.Millisecond)
|
|
defer ticker.Stop()
|
|
}
|
|
|
|
worker := func() {
|
|
defer c.wg.Done()
|
|
for bc := range jobs {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
if ticker != nil {
|
|
select {
|
|
case <-ticker.C:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
c.current.Store(bc)
|
|
p, err := c.lookup(ctx, bc)
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
atomic.AddInt64(&c.errors, 1)
|
|
atomic.AddInt64(&c.done, 1)
|
|
ep := Product{Barcode: bc, Status: "error", RetMsg: err.Error(), FetchedAt: time.Now().Format(time.RFC3339), Source: "zc.bypos.net"}
|
|
c.pushRecent(ep)
|
|
continue
|
|
}
|
|
switch p.Status {
|
|
case "hit":
|
|
atomic.AddInt64(&c.hits, 1)
|
|
case "miss":
|
|
atomic.AddInt64(&c.miss, 1)
|
|
case "invalid":
|
|
atomic.AddInt64(&c.invalid, 1)
|
|
}
|
|
atomic.AddInt64(&c.done, 1)
|
|
c.pushRecent(*p)
|
|
if p.Status == "hit" || req.LogMiss {
|
|
line, _ := json.Marshal(p)
|
|
writeMu.Lock()
|
|
c.outFile.Write(line)
|
|
c.outFile.Write([]byte("\n"))
|
|
c.seen[bc] = struct{}{}
|
|
writeMu.Unlock()
|
|
}
|
|
}
|
|
}
|
|
|
|
for i := 0; i < req.Concurrency; i++ {
|
|
c.wg.Add(1)
|
|
go worker()
|
|
}
|
|
|
|
for _, bc := range barcodes {
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
if _, ok := c.seen[bc]; ok {
|
|
atomic.AddInt64(&c.skipped, 1)
|
|
atomic.AddInt64(&c.done, 1)
|
|
continue
|
|
}
|
|
select {
|
|
case jobs <- bc:
|
|
case <-ctx.Done():
|
|
}
|
|
}
|
|
close(jobs)
|
|
c.wg.Wait()
|
|
|
|
c.mu.Lock()
|
|
if ctx.Err() != nil {
|
|
c.message = "已停止"
|
|
} else {
|
|
c.message = "采集完成"
|
|
}
|
|
c.mu.Unlock()
|
|
}
|