bypos-collector: 移除硬编码 sdogid、添加重试、补全 CSV 导出、增加测试
CI / Go (api) (pull_request) Failing after 1s
CI / Python (ingestion) (pull_request) Failing after 0s
CI / Migrations (postgres) (pull_request) Failing after 2s

改进内容:
- sdogid 不再硬编码,优先从 $BYPOS_SDOGID 环境变量读取,其次 -sdogid 参数,
  也可在 Web 控制台输入框填写;三者都未设时启动警告、采集报错
- 网络错误自动重试(最多 3 次,指数退避 500ms/1s/2s)
- Web 控制台采集参数区增加 sdogid 输入框
- CSV 导出补全 retmsg 和 source 两列
- 新增 Go 单元测试(ean13Check、md5hex、sanitizeBarcodes、resolveSdogID 等)
- README 更新配置说明

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
ceyhandagdas51272
2026-06-24 03:10:09 +00:00
parent 8f9a03a929
commit 21c895a008
5 changed files with 171 additions and 17 deletions
+42 -8
View File
@@ -20,12 +20,21 @@ import (
)
// ---- upstream config ----
// sdogid is the 云店 account license id observed in the live request. It is
// configurable so the tool is not tied to a single account.
const defaultSdogID = "137966"
const endpoint = "http://zc.bypos.net/byGoodsService/byMessage.asmx/GetGoodsinfo"
// maxRetries is the number of retry attempts for transient network errors.
const maxRetries = 3
// resolveSdogID reads the account id from $BYPOS_SDOGID, falling back to the
// explicit argument (CLI flag or Web UI input). Returns empty if neither set.
func resolveSdogID(explicit string) string {
if v := os.Getenv("BYPOS_SDOGID"); v != "" {
return v
}
return explicit
}
var stringTagRe = regexp.MustCompile(`(?s)<string[^>]*>(.*)</string>`)
// Product is the normalized record we persist (one JSON object per line).
@@ -87,8 +96,8 @@ func ean13Check(body string) (string, bool) {
return body + strconv.Itoa(chk), true
}
// lookup queries the upstream central library for one barcode.
func (c *Collector) lookup(ctx context.Context, barcode string) (*Product, error) {
// lookupOnce performs a single HTTP request to the upstream central library.
func (c *Collector) lookupOnce(ctx context.Context, barcode string) (*Product, error) {
tsMs := strconv.FormatInt(time.Now().Unix()*1000, 10) // always ends in 000
sparm1 := md5hex(c.sdogID)
sparm2 := md5hex(barcode + tsMs)
@@ -147,6 +156,30 @@ func (c *Collector) lookup(ctx context.Context, barcode string) (*Product, error
return p, nil
}
// lookup queries the upstream with up to maxRetries retries on transient errors.
func (c *Collector) lookup(ctx context.Context, barcode string) (*Product, error) {
var lastErr error
for attempt := 0; attempt <= maxRetries; attempt++ {
if ctx.Err() != nil {
return nil, ctx.Err()
}
p, err := c.lookupOnce(ctx, barcode)
if err == nil {
return p, nil
}
lastErr = err
if attempt < maxRetries {
backoff := time.Duration(1<<uint(attempt)) * 500 * time.Millisecond
select {
case <-time.After(backoff):
case <-ctx.Done():
return nil, ctx.Err()
}
}
}
return nil, lastErr
}
// ---- job / collector state ----
type Stats struct {
@@ -187,9 +220,7 @@ type Collector struct {
}
func NewCollector(sdogID string) *Collector {
if sdogID == "" {
sdogID = defaultSdogID
}
sdogID = resolveSdogID(sdogID)
c := &Collector{
client: &http.Client{Timeout: 25 * time.Second},
sdogID: sdogID,
@@ -308,6 +339,9 @@ func (c *Collector) Start(req JobReq) error {
if req.SdogID != "" {
c.sdogID = req.SdogID
}
if c.sdogID == "" {
return fmt.Errorf("未指定 sdogid,请通过控制台输入框、环境变量 $BYPOS_SDOGID 或 -sdogid 参数配置")
}
// Build the list of barcodes to query.
var barcodes []string