public/saasapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加命令功能

Browse Source
This commit is contained in:
algotao
2025-04-07 10:42:08 +08:00
parent 0344c09ce7
commit 74b0033e45
18 changed files with 1844 additions and 494 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
cmd/saastool/write.go
View File

@@ -19,7 +19,6 @@ type writeParams struct {
sourcePath string
appid string
batchSize uint
async bool
clear bool
saasHttp *saashttp.SaasClient
}
@@ -30,7 +29,6 @@ func RunWrite(args ...string) error {
sourcePath := paramSourcePath(fs)
appid := paramAppid(fs)
batchSize := paramBatchSize(fs)
async := paramAsync(fs)
clear := paramClear(fs)
if err := fs.Parse(args); err != nil {
@@ -53,7 +51,6 @@ func RunWrite(args ...string) error {
sourcePath: *sourcePath,
appid: *appid,
batchSize: *batchSize,
async: *async,
clear: *clear,
saasHttp: &saashttp.SaasClient{
Client: http.Client{},
@@ -105,7 +102,7 @@ func doLoadFileToWrite(writeParams writeParams) error {
scaner := bufio.NewScanner(file)
saasWriteCmds := []*saasapi.WriteCmd{}
saasWriteItems := []*saasapi.WriteItem{}
succ := uint32(0)
succTotal := uint32(0)
@@ -115,29 +112,29 @@ func doLoadFileToWrite(writeParams writeParams) error {
if line == "" {
continue
}
saasWriteCmd := &saasapi.WriteCmd{}
if err = protojson.Unmarshal([]byte(line), saasWriteCmd); err != nil {
saasWriteItem := &saasapi.WriteItem{}
if err = protojson.Unmarshal([]byte(line), saasWriteItem); err != nil {
return err
}
saasWriteCmds = append(saasWriteCmds, saasWriteCmd)
saasWriteItems = append(saasWriteItems, saasWriteItem)
total++
if len(saasWriteCmds) == int(writeParams.batchSize) {
if succ, _, err = submitWrite(writeParams, saasWriteCmds); err != nil {
if len(saasWriteItems) == int(writeParams.batchSize) {
if succ, _, err = submitWrite(writeParams, saasWriteItems); err != nil {
return err
}
succTotal += succ
fmt.Printf("[%v] batch_succ = %v, succ_total = %v, total_processed = %v\n", writeParams.sourcePath, succ, succTotal, total)
saasWriteCmds = saasWriteCmds[:0]
saasWriteItems = saasWriteItems[:0]
}
}
if len(saasWriteCmds) > 0 {
if succ, _, err = submitWrite(writeParams, saasWriteCmds); err != nil {
if len(saasWriteItems) > 0 {
if succ, _, err = submitWrite(writeParams, saasWriteItems); err != nil {
return err
}
succTotal += succ
@@ -147,12 +144,11 @@ func doLoadFileToWrite(writeParams writeParams) error {
return nil
}
func submitWrite(writeParams writeParams, saasWriteCmds []*saasapi.WriteCmd) (succ, total uint32, err error) {
func submitWrite(writeParams writeParams, saasWriteCmds []*saasapi.WriteItem) (succ, total uint32, err error) {
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_Write{
Write: &saasapi.Write{
IsClearAllFirst: writeParams.clear,
Async: writeParams.async,
},
},
}
@@ -162,10 +158,15 @@ func submitWrite(writeParams writeParams, saasWriteCmds []*saasapi.WriteCmd) (su
saasReq.Appid = writeParams.appid
}
saasReq.Cmd.(*saasapi.SaasReq_Write).Write.WriteCmds = saasWriteCmds
saasReq.Cmd.(*saasapi.SaasReq_Write).Write.WriteItems = saasWriteCmds
total = uint32(len(saasWriteCmds))
succ, err = writeParams.saasHttp.Write(saasReq)
res, err := writeParams.saasHttp.Write(saasReq)
return
if err != nil {
slog.Error("submitWrite error", "err", err)
return
}
return res.GetWriteRes().GetSuccCmdCount(), total, nil
}