增加命令功能

cmd.proto

@@ -10,25 +10,36 @@ message SaasReq {
     string appid                                 = 2;   // 小程序/小游戏/公众号/视频号的appid
 
     oneof cmd {
-        Write write                             = 10;   // 批量写入
+        Read read                                = 10;  // 批量读取
-        Read read                               = 11;   // 批量读取
+        Write write                              = 11;  // 批量写入
         ColumnWrite column_write                 = 12;  // 全量列式写入
 
-        TaskList task_list                      = 20;   // 任务列表
+        Task task_create                         = 20;  // 任务创建
-        TaskCancel task_cancel                  = 21;   // 取消任务
+        TaskList task_list                       = 21;  // 列出任务
-        TaskDetail task_detail                  = 22;   // 任务详情
+        TaskRun task_run                         = 22;  // 执行任务
+        TaskDelete task_delete                   = 23;  // 删除任务
+        TaskInfo task_info                       = 24;  // 任务详情
     }
 }
 
+// Read 批量读取命令
+message Read {
+    repeated ReadItem read_items                 = 1;   // 批量获取命令
+}
+
+// ReadItem 读取命令
+message ReadItem {
+    string userid                                = 1;   // 用户ID
+}
+
 // Write 批量写入命令
 message Write {
-    bool async                                  = 1;   // 是否异步执行
+    bool is_clear_all_first                      = 1;   // 是否先清空该用户所有数据
-    bool is_clear_all_first                     = 2;   // 是否先执行清空
+    repeated WriteItem write_items               = 2;   // 批量写入命令
-    repeated WriteCmd write_cmds                = 3;   // 批量写入命令
 }
 
-// WriteCmd 写入命令
+// WriteItem 写入命令
-message WriteCmd {
+message WriteItem {
     string userid                                = 1;   // 用户ID
     Bytes write_bytes                            = 2;   // byte区域
     Uint32s write_uint32s                        = 3;   // uint32区域
@@ -66,16 +77,7 @@ message FlagWithExpire {
 enum UserIdType {
     DEVICEID                                     = 0;   // 设备号
     OPENID                                       = 1;   // OpenId
-}
+    INNERID1                                     = 10;   // 内部ID1
-
-// Write 批量读取命令
-message Read {
-    repeated ReadCmd read_cmds                   = 1;  // 批量获取命令
-}
-
-// WriteCmd 读取命令
-message ReadCmd {
-    string userid                                = 1;  // 用户ID
 }
 
 // ColumnWrite 全量列式写入命令
@@ -86,32 +88,82 @@ message ColumnWrite {
     bool is_clear_all_first                      = 5;   // 是否先执行清空
 }
 
+message Task {
+    string task_sha256                           = 1;   // 任务sha256
+    string task_description                      = 2;   // 任务描述
+    repeated FileInfo task_file_infos            = 3;   // 文件列表
+    uint64 task_block_size                       = 4;   // 文件块字节大小（推荐50M）
+
+    // 以下字段只在返回时填写，用于提供服务端的任务状态。在请求时填写会被忽略
+    string create_time                           = 10;   // 创建时间
+    string run_time                              = 11;   // 运行时间
+    string finish_time                           = 12;   // 完成时间
+
+    TaskStatus status                            = 15;   // 任务状态
+}
+
 // TaskList 任务列表
 message TaskList {
-
+    TaskStatus status_filter                     = 1;   // 只显示指定状态的任务
 }
 
-// TaskCancel 取消任务
+// TaskRun 任务运行
-message TaskCancel {
+message TaskRun {
-
+    string task_sha256                           = 1;   // 任务sha256
 }
 
-// TaskDetail 任务详情
+// TaskDelete 取消任务
-message TaskDetail {
+message TaskDelete {
+    string task_sha256                           = 1;   // 任务sha256
+}
 
+// TaskInfo 任务详情
+message TaskInfo {
+    string task_sha256                           = 1;   // 任务sha256
+}
+
+message FileInfo {
+    string file_name                             = 1;   // 文件名
+    uint64 file_size                             = 2;   // 文件大小
+    repeated FileBlock file_blocks               = 3;   // 文件块列表
+}
+
+message FileBlock {
+    string block_sha256                          = 1;   // 块的sha256
+    uint64 block_length                          = 2;   // 块的字节长度
+    bool uploaded                                = 3;   // 是否已上传（在TaskInfo请求返回）
 }
 
 // SaasRes 命令返回
 message SaasRes {
     ErrorCode code                               = 1;  // 返回码
     string status                                = 2;  // 返回信息的文本提示
-    uint32 succ_cmd_count                        = 3;  // 成功的命令数量
+    oneof res {
-    uint32 fail_cmd_count                        = 4;  // 失败的命令数量
+        ReadRes read_res                         = 10;  // 读取命令返回
-    repeated CmdsResItem cmd_res                 = 5;  // 返回的命令
+        WriteRes write_res                       = 11;  // 写入命令返回
+
+        Task task_create_res                     = 20;  // 创建任务返回状态
+        TaskListRes task_list_res                = 21;  // 任务列表返回状态
+        Task task_run_res                        = 22;  // 运行任务返回状态
+        Task task_info_res                       = 23;  // 任务详情返回状态
+        Task task_delete_res                     = 24;  // 删除任务返回状态
+    }
 }
 
-// CmdsResItem 读取命令返回内容
+message ReadRes {
-message CmdsResItem {
+    uint32 succ_cmd_count                        = 1;  // 成功的命令数量
+    uint32 fail_cmd_count                        = 2;  // 失败的命令数量
+    repeated ValueItem cmd_res                   = 3;  // 返回的命令
+}
+
+message WriteRes {
+    uint32 succ_cmd_count                        = 1;  // 成功的命令数量
+    uint32 fail_cmd_count                        = 2;  // 失败的命令数量
+    repeated ValueItem cmd_res                   = 3;  // 返回的失败命令，仅填写cmd_index和cmd_code
+}
+
+// ValueItem 读取命令返回内容
+message ValueItem {
     uint32 cmd_index                             = 1;  // 命令索引
     CmdErrorCode cmd_code                        = 2;  // 状态
     bytes bytes                                  = 3;  // byte区域
@@ -120,11 +172,15 @@ message CmdsResItem {
     uint32 last_modify_time                      = 6;  // 最后修改时间
 }
 
+message TaskListRes {
+    repeated Task tasks                          = 1;  // 任务列表
+}
+
 // ErrorCode 返回码
 enum ErrorCode {
     SUCC                                         = 0;   // 成功
     INVALID_ACCOUNT                              = 101; // Account不合法
-	INVALID_TIMESTAMP                            = 102; // 头信息缺少时间戳
+    INVALID_TIMESTAMP                            = 102; // 头信息缺少时间戳或不正确
     INVALID_SIGNATURE                            = 103; // 头信息缺少签名
     AUTH_FAIL                                    = 104; // 签名较验失败
     DISABLED_ACCOUNT                             = 105; // 账号已禁用
@@ -134,10 +190,28 @@ enum ErrorCode {
     QPS_LIMIT                                    = 113; // 并发请求量超限
     CMDS_LIMIT                                   = 114; // 命令数量超限
     CMDS_NULL                                    = 115; // 命令为空
+
+    TASK_EXISTS                                  = 120; // 任务已存在
+    TASK_IS_NOT_EXISTS                           = 121; // 任务不存在
+    TASK_NUM_LIMIT                               = 122; // 任务数达到上限
+    TASK_BLOCK_SIZE                              = 123; // 块大小超限
+    TASK_TOTAL_SIZE                              = 124; // 总文件大小超限
+    TASK_MARSHAL                                 = 125; // 序列化
+
+    DATA_ERROR                                   = 201; // 数据错误
 }
 
 enum CmdErrorCode {
     OK                                           = 0;   // 成功
 }
 
+enum TaskStatus {
+    ALL                                          = 0;   // 全部
+    WAITING                                      = 1;   // 等待中
+    RUNNING                                      = 2;   // 运行中
+    SUCCESS                                      = 3;   // 成功
+    FAIL                                         = 4;   // 失败
+
+    DELETED                                      = 5;   // 已删除，仅在执行删除成功时返回
+}
 

cmd/saastool/convert.go

@@ -2,27 +2,38 @@ package main
 
 import (
 	"bufio"
+	"bytes"
 	"flag"
 	"fmt"
 	"os"
 	"path"
+	"runtime"
 	"strings"
+	"sync"
 
 	"e.coding.net/rta/public/saasapi"
 	"google.golang.org/protobuf/encoding/protojson"
 )
 
-// TODO 转换加速
+const (
+	convertBatchSize = 100000
+	convertedExt     = ".converted"
+)
 
 type convertParams struct {
-	targetCfg  *TargetConfig
+	mapCfg     *MapConfig
 	sourcePath string
 	destPath   string
 }
 
+type convertResult struct {
+	resultBuf      bytes.Buffer
+	convertedLines int
+}
+
 func RunConvert(args ...string) error {
 	fs := flag.NewFlagSet("convert", flag.ExitOnError)
-	targetCfgFile := paramTargets(fs)
+	mapCfgFile := paramMap(fs)
 	sourcePath := paramSourcePath(fs)
 	destPath := paramDestPath(fs)
 
@@ -31,19 +42,19 @@ func RunConvert(args ...string) error {
 		return err
 	}
 
-	if fs.NArg() > 0 || *targetCfgFile == "" || len(*sourcePath) == 0 || len(*destPath) == 0 {
+	if fs.NArg() > 0 || *mapCfgFile == "" || len(*sourcePath) == 0 || len(*destPath) == 0 {
 		fs.PrintDefaults()
 		return nil
 	}
 
-	targetCfg, err := LoadTargetFile(*targetCfgFile)
+	mapCfg, err := LoadMapFile(*mapCfgFile)
 	if err != nil {
 		fmt.Println("LoadConfigFile error", "err", err)
 		return err
 	}
 
 	convertParams := convertParams{
-		targetCfg:  targetCfg,
+		mapCfg:     mapCfg,
 		sourcePath: *sourcePath,
 		destPath:   *destPath,
 	}
@@ -97,7 +108,7 @@ func doFileConvert(convertParams convertParams) error {
 		os.MkdirAll(convertParams.destPath, os.ModePerm)
 	}
 
-	destName := path.Join(convertParams.destPath, path.Base(convertParams.sourcePath)+".converted")
+	destName := path.Join(convertParams.destPath, path.Base(convertParams.sourcePath)+convertedExt)
 	destFile, err := os.Create(destName)
 	if err != nil {
 		return err
@@ -108,15 +119,92 @@ func doFileConvert(convertParams convertParams) error {
 	destWriter := bufio.NewWriter(destFile)
 	defer destWriter.Flush()
 
-	jasonMarshal := protojson.MarshalOptions{Multiline: false, Indent: ""}
+	// 启动处理协程
-
+	workers := []chan []string{}
+	results := []chan convertResult{}
 	processedLine := 0
+
+	wg := sync.WaitGroup{}
+	convertMaxWorkers := runtime.GOMAXPROCS(0)
+	for range convertMaxWorkers {
+		workerChan := make(chan []string)
+		workers = append(workers, workerChan)
+		resultChan := make(chan convertResult)
+		results = append(results, resultChan)
+
+		go func(workerChan <-chan []string, resultChan chan<- convertResult) {
+			for lines := range workerChan {
+				convertBatch(lines, convertParams, resultChan)
+			}
+		}(workerChan, resultChan)
+	}
+
+	// 启动写入协程
+	go func() {
+		i := 0
+
+		// TIP: 不要改成for range
+		for {
+			select {
+			case result, ok := <-results[i%convertMaxWorkers]:
+				if !ok {
+					return
+				}
+				destWriter.Write(result.resultBuf.Bytes())
+				destWriter.Flush()
+				processedLine += result.convertedLines
+				fmt.Printf("\rconverted records: %v [%v]", processedLine, destName)
+				i++
+				wg.Done()
+			}
+
+		}
+	}()
+
+	// 读取文件并塞给协程处理
+	batch := []string{}
+	batchCount := 0
 	for scaner.Scan() {
 		line := scaner.Text()
 		if line == "" {
 			continue
 		}
 
+		batch = append(batch, line)
+		if len(batch) == convertBatchSize {
+			// 将batch写入协程
+			wg.Add(1)
+			workers[batchCount%convertMaxWorkers] <- batch
+			batch = nil
+			batchCount++
+		}
+	}
+
+	if len(batch) > 0 {
+		wg.Add(1)
+		workers[batchCount%convertMaxWorkers] <- batch
+	}
+
+	wg.Wait()
+	// 关闭所有工作协程的通道
+	for _, workerChan := range workers {
+		close(workerChan)
+	}
+	for _, resultChan := range results {
+		close(resultChan)
+	}
+	fmt.Println("")
+
+	return nil
+}
+
+func convertBatch(lines []string, convertParams convertParams, resultChan chan<- convertResult) {
+	byteBuf := bytes.Buffer{}
+	byteBuf.Grow(1024 * 1024 * 10)
+
+	jasonMarshal := protojson.MarshalOptions{Multiline: false, Indent: ""}
+
+	for _, line := range lines {
 		// 按\t分割为两列
 		parts := strings.Split(line, "\t")
 		if len(parts) != 2 {
@@ -125,65 +213,63 @@ func doFileConvert(convertParams convertParams) error {
 
 		// 读取userid
 		userid := parts[0]
+		if len(userid) == 0 {
+			continue
+		}
+
 		value := parts[1]
 		value = strings.ReplaceAll(value, "[", "")
 		value = strings.ReplaceAll(value, "]", "")
 		// 第二列解析为string数组
 		targets := strings.Split(value, " ")
 
-		saasWriteCmd := &saasapi.WriteCmd{
+		saasWriteItem := &saasapi.WriteItem{
 			Userid: userid,
 		}
-		if len(userid) == 0 || len(targets) == 0 {
+
-			continue
+		// 遍历targets，转换成saasapi.WriteCmd
-		}
 		for _, target := range targets {
-			if targetinfo, ok := convertParams.targetCfg.Targets[target]; ok {
+			if targetinfo, ok := convertParams.mapCfg.Targets[target]; ok {
 				if targetinfo.WriteByte != nil {
-					if saasWriteCmd.WriteBytes == nil {
+					// 转换byte区
-						saasWriteCmd.WriteBytes = &saasapi.Bytes{}
+					if saasWriteItem.WriteBytes == nil {
+						saasWriteItem.WriteBytes = &saasapi.Bytes{}
 					}
-					saasWriteCmd.WriteBytes.Bytes = append(saasWriteCmd.WriteBytes.Bytes, *targetinfo.WriteByte)
+					saasWriteItem.WriteBytes.Bytes = append(saasWriteItem.WriteBytes.Bytes, *targetinfo.WriteByte)
 					if targetinfo.WriteBytePos < 64 {
-						saasWriteCmd.WriteBytes.Index_1 |= 1 << targetinfo.WriteBytePos
+						saasWriteItem.WriteBytes.Index_1 |= 1 << targetinfo.WriteBytePos
 					} else if targetinfo.WriteBytePos < 128 {
-						saasWriteCmd.WriteBytes.Index_2 |= 1 << (targetinfo.WriteBytePos - 64)
+						saasWriteItem.WriteBytes.Index_2 |= 1 << (targetinfo.WriteBytePos - 64)
 					}
 				}
 
 				if targetinfo.WriteUint32 != nil {
-					if saasWriteCmd.WriteUint32S == nil {
+					// 转换uint32区
-						saasWriteCmd.WriteUint32S = &saasapi.Uint32S{}
+					if saasWriteItem.WriteUint32S == nil {
+						saasWriteItem.WriteUint32S = &saasapi.Uint32S{}
 					}
-					saasWriteCmd.WriteUint32S.Uint32S = append(saasWriteCmd.WriteUint32S.Uint32S, *targetinfo.WriteUint32)
+					saasWriteItem.WriteUint32S.Uint32S = append(saasWriteItem.WriteUint32S.Uint32S, *targetinfo.WriteUint32)
-					saasWriteCmd.WriteUint32S.Index_1 |= 1 << targetinfo.WriteUint32Pos
+					saasWriteItem.WriteUint32S.Index_1 |= 1 << targetinfo.WriteUint32Pos
 				}
 
 				if targetinfo.WriteFlag != nil && targetinfo.WriteExpire != nil {
-					if saasWriteCmd.WriteFlagsWithExpire == nil {
+					// 转换flag区
-						saasWriteCmd.WriteFlagsWithExpire = &saasapi.FlagsWithExpire{}
+					if saasWriteItem.WriteFlagsWithExpire == nil {
+						saasWriteItem.WriteFlagsWithExpire = &saasapi.FlagsWithExpire{}
 					}
-					saasWriteCmd.WriteFlagsWithExpire.FlagsWithExpire = append(
+					saasWriteItem.WriteFlagsWithExpire.FlagsWithExpire = append(
-						saasWriteCmd.WriteFlagsWithExpire.FlagsWithExpire, &saasapi.FlagWithExpire{
+						saasWriteItem.WriteFlagsWithExpire.FlagsWithExpire, &saasapi.FlagWithExpire{
 							Flag:   *targetinfo.WriteFlag,
 							Expire: *targetinfo.WriteExpire,
 						})
-					saasWriteCmd.WriteFlagsWithExpire.Index_1 |= 1 << targetinfo.WriteFlagWithExpirePos
+					saasWriteItem.WriteFlagsWithExpire.Index_1 |= 1 << targetinfo.WriteFlagWithExpirePos
 				}
 			}
 		}
 
-		// 写入文件
+		byteBuf.WriteString(jasonMarshal.Format(saasWriteItem))
-		destWriter.WriteString(jasonMarshal.Format(saasWriteCmd))
+		byteBuf.WriteString("\n")
-		destWriter.WriteString("\n")
-
-		processedLine++
-		if processedLine%100000 == 0 {
-			fmt.Printf("\rconverted records: %v [%v]", processedLine, destName)
 	}
 
-	}
+	resultChan <- convertResult{byteBuf, len(lines)}
-
-	fmt.Printf("\rconverted records: %v [%v]\n", processedLine, destName)
-	return nil
 }

cmd/saastool/help.go

@@ -12,21 +12,19 @@ func RunHelp(args ...string) error {
 }
 
 const usage = `
-Usage:  [[command] [arguments]]
+Usage:  saastool COMMAND [OPTIONS]
-
-The commands are:
 
+Commands:
     write              Write user's 'bytes / uint32s / flags'
     read               Read user's 'bytes / uint32s / flags'
     columnwrite        Write columns for 'deviceid / openid' users
 
-	tasklist           List tasks
+    convert            Convert data to write format
-	taskcancel         Cancel task
+    makehash           Make file hash for upload task
-	taskdetail         Show task detail
+
+    task               Task commands
 
 "help" is the default command.
 
-Use "saastool [command] -help" for more information about a command.
+Use "saastool COMMAND -help" for more information about a command.
 `
-
-//    strip            Stripping redundant data from redis

cmd/saastool/main.go

@@ -29,14 +29,13 @@ func Run(args ...string) error {
 		return RunColumnWrite(args...)
 	case "convert":
 		return RunConvert(args...)
-	case "tasklist":
+	case "makehash":
-		return RunTaskList(args...)
+		return RunMakeHash(args...)
-	case "taskcancel":
-		return RunTaskCancel(args...)
-	case "taskdetail":
-		return RunTaskDetail(args...)
 	case "verify":
 		return RunVerify(args...)
+	case "task":
+		return RunTask(args...)
+
 	default:
 		err := fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'saastool help' for usage`, name)
 		slog.Warn(err.Error())

cmd/saastool/make_hash.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"flag"
+	"fmt"
+	"os"
+	"path"
+	"runtime"
+	"sort"
+	"sync"
+
+	"e.coding.net/rta/public/saasapi"
+	"google.golang.org/protobuf/encoding/protojson"
+)
+
+const (
+	blockSizeMin = 10 * 1024 * 1024
+	blockSizeMax = 200 * 1024 * 1024
+)
+
+type makeHashParams struct {
+	sourcePath string
+	destPath   string
+	task       *saasapi.Task
+}
+
+// 计算任务
+type hashTask struct {
+	chunk []byte
+	index int
+}
+
+// 计算结果
+type hashResult struct {
+	hash      string
+	blockSize uint64
+	index     int
+}
+
+func RunMakeHash(args ...string) error {
+	fs := flag.NewFlagSet("tasklocalmake", flag.ExitOnError)
+	sourcePath := paramSourcePath(fs)
+	destPath := paramDestPath(fs)
+	blockSize := paramBlockSize(fs)
+
+	if err := fs.Parse(args); err != nil {
+		fmt.Println("command line parse error", "err", err)
+		return err
+	}
+
+	if fs.NArg() > 0 || len(*sourcePath) == 0 || len(*destPath) == 0 {
+		fs.PrintDefaults()
+		return nil
+	}
+
+	if blockSize < blockSizeMin || blockSize > blockSizeMax {
+		fmt.Println("block size error", "min", blockSizeMin, "max", blockSizeMax)
+		return nil
+	}
+
+	makeHashParams := makeHashParams{
+		sourcePath: *sourcePath,
+		destPath:   *destPath,
+		task: &saasapi.Task{
+			TaskBlockSize: blockSize,
+		},
+	}
+	return doMakeHash(makeHashParams)
+}
+
+func doMakeHash(makeHashParams makeHashParams) error {
+	fsInfo, err := os.Stat(makeHashParams.sourcePath)
+	if err != nil {
+		return err
+	}
+
+	if !fsInfo.IsDir() {
+		// 如果是文件，直接计算
+		return doFileHash(makeHashParams)
+	}
+
+	// 读取目录下信息
+	dirEntry, err := os.ReadDir(makeHashParams.sourcePath)
+	if err != nil {
+		return err
+	}
+
+	// 遍历目录
+	for _, dir := range dirEntry {
+		newParam := makeHashParams
+		newParam.sourcePath = path.Join(makeHashParams.sourcePath, dir.Name())
+
+		if dir.IsDir() {
+			newParam.destPath = path.Join(makeHashParams.destPath, dir.Name())
+		}
+
+		if err = doMakeHash(newParam); err != nil {
+			return err
+		}
+	}
+
+	return saveTaskFile(makeHashParams)
+}
+
+func doFileHash(makeHashParams makeHashParams) error {
+	sourceFile, err := os.Open(makeHashParams.sourcePath)
+	if err != nil {
+		return err
+	}
+	defer sourceFile.Close()
+
+	fi, err := sourceFile.Stat()
+	if err != nil {
+		return err
+	}
+
+	tasks := make(chan hashTask)
+	results := make(chan hashResult)
+
+	// 启动工作协程
+	hashMaxWorker := runtime.GOMAXPROCS(0)
+	for range hashMaxWorker {
+		go hashWorker(tasks, results)
+	}
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		index := 0
+		buffer := make([]byte, makeHashParams.task.TaskBlockSize)
+		for {
+			n, err := sourceFile.Read(buffer)
+			if n > 0 {
+				wg.Add(1)
+				fmt.Printf("\rhashing file [%v], block [%v]", makeHashParams.sourcePath, index)
+				tasks <- hashTask{chunk: buffer[:n], index: index}
+				index++
+			}
+			if err != nil {
+				break
+			}
+		}
+		close(tasks)
+		wg.Done()
+	}()
+
+	var allResults []hashResult
+	go func() {
+		for r := range results {
+			allResults = append(allResults, r)
+			wg.Done()
+		}
+	}()
+
+	wg.Wait()
+	close(results)
+
+	// 按索引排序结果
+	sort.Slice(allResults, func(i, j int) bool {
+		return allResults[i].index < allResults[j].index
+	})
+
+	// 输出结果
+	fileInfo := &saasapi.FileInfo{
+		FileName: makeHashParams.sourcePath,
+		FileSize: uint64(fi.Size()),
+	}
+	for _, r := range allResults {
+		fileInfo.FileBlocks = append(fileInfo.FileBlocks, &saasapi.FileBlock{
+			BlockSha256: r.hash,
+			BlockLength: r.blockSize,
+		})
+	}
+	makeHashParams.task.TaskFileInfos = append(makeHashParams.task.TaskFileInfos, fileInfo)
+
+	fmt.Println("")
+	return nil
+}
+
+// hash计算协程
+func hashWorker(tasks <-chan hashTask, results chan<- hashResult) {
+	for t := range tasks {
+		h := sha256.New()
+		h.Write(t.chunk)
+		hash := hex.EncodeToString(h.Sum(nil))
+		results <- hashResult{hash: hash, index: t.index, blockSize: uint64(len(t.chunk))}
+	}
+}
+
+func saveTaskFile(makeHashParams makeHashParams) error {
+	taskFile, err := os.Create(makeHashParams.destPath)
+	if err != nil {
+		return err
+	}
+	defer taskFile.Close()
+
+	h := sha256.New()
+
+	for _, fileInfo := range makeHashParams.task.TaskFileInfos {
+		for _, fileBlock := range fileInfo.FileBlocks {
+			h.Write([]byte(fileBlock.BlockSha256))
+		}
+	}
+
+	makeHashParams.task.TaskSha256 = hex.EncodeToString(h.Sum(nil))
+
+	_, err = taskFile.WriteString(protojson.Format(makeHashParams.task))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/saastool/params.go

@@ -2,31 +2,50 @@ package main
 
 import (
 	"flag"
+	"fmt"
+	"strconv"
+	"strings"
 )
 
 func paramConfig(fs *flag.FlagSet) *string {
 	return fs.String("config", "cfg.toml", "Config file.")
 }
-func paramTargets(fs *flag.FlagSet) *string {
+func paramMap(fs *flag.FlagSet) *string {
-	return fs.String("targets", "", "target setting")
+	return fs.String("map", "", "target map setting")
 }
 
 func paramSourcePath(fs *flag.FlagSet) *string {
-	return fs.String("source", "", "Data path source for write command.")
+	return fs.String("source", "", "Source path or filename")
 }
 
 func paramDestPath(fs *flag.FlagSet) *string {
-	return fs.String("dest", "", "Data path destination for write command.")
+	return fs.String("dest", "", "Destination path or filename")
 }
 
 func paramAppid(fs *flag.FlagSet) *string {
 	return fs.String("appid", "", "Wechat appid")
 }
 
+func paramUserids(fs *flag.FlagSet) *string {
+	return fs.String("userids", "", "Device ID or Wechat UserID, separated by comma")
+}
+
 func paramBatchSize(fs *flag.FlagSet) *uint {
 	return fs.Uint("batchsize", 10000, "Batch size to sync")
 }
 
+func paramBlockSize(fs *flag.FlagSet) uint64 {
+	bsize := fs.String("blocksize", "50M", "Block size to make hash. using size mode K, M, G, T")
+	num, err := ParseByteSize(*bsize)
+	if err != nil {
+		fmt.Println("Error parsing block size", "err", err)
+		fmt.Println("Using default 50M")
+		num = 50 * 1024 * 1024
+
+	}
+	return num
+}
+
 func paramAsync(fs *flag.FlagSet) *bool {
 	return fs.Bool("async", false, "Async mode")
 }
@@ -34,3 +53,55 @@ func paramAsync(fs *flag.FlagSet) *bool {
 func paramClear(fs *flag.FlagSet) *bool {
 	return fs.Bool("clear", false, "Clear all data before write")
 }
+
+// ParseByteSize 解析字节大小字符串为字节数
+func ParseByteSize(sizeStr string) (uint64, error) {
+	sizeStr = strings.TrimSpace(sizeStr)
+	unit := ""
+	numStr := sizeStr
+	// 提取单位
+	if len(sizeStr) > 1 && (sizeStr[len(sizeStr)-1] == 'B' || sizeStr[len(sizeStr)-1] == 'b') {
+		unit = string(sizeStr[len(sizeStr)-2:])
+		numStr = sizeStr[:len(sizeStr)-2]
+	} else if len(sizeStr) > 0 && (sizeStr[len(sizeStr)-1] >= 'A' && sizeStr[len(sizeStr)-1] <= 'Z' ||
+		sizeStr[len(sizeStr)-1] >= 'a' && sizeStr[len(sizeStr)-1] <= 'z') {
+		unit = string(sizeStr[len(sizeStr)-1])
+		numStr = sizeStr[:len(sizeStr)-1]
+	}
+
+	// 解析数字部分
+	num, err := strconv.ParseFloat(numStr, 64)
+	if err != nil {
+		return 0, err
+	}
+
+	// 根据单位计算字节数
+	switch strings.ToUpper(unit) {
+	case "":
+		return uint64(num), nil
+	case "K", "KB":
+		return uint64(num * 1024), nil
+	case "M", "MB":
+		return uint64(num * 1024 * 1024), nil
+	case "G", "GB":
+		return uint64(num * 1024 * 1024 * 1024), nil
+	case "T", "TB":
+		return uint64(num * 1024 * 1024 * 1024 * 1024), nil
+	default:
+		return 0, fmt.Errorf("unknown unit: %s", unit)
+	}
+}
+
+/*
+func main() {
+    sizes := []string{"1K", "2M", "3G", "4T", "5"}
+    for _, sizeStr := range sizes {
+        size, err := ParseByteSize(sizeStr)
+        if err != nil {
+            fmt.Printf("Error parsing %s: %v\n", sizeStr, err)
+        } else {
+            fmt.Printf("%s = %d bytes\n", sizeStr, size)
+        }
+    }
+}
+*/

cmd/saastool/read.go

@@ -1,5 +1,95 @@
 package main
 
+import (
+	"flag"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"strings"
+
+	"e.coding.net/rta/public/saasapi"
+	"e.coding.net/rta/public/saasapi/pkg/saashttp"
+	"google.golang.org/protobuf/encoding/protojson"
+)
+
+const (
+	getIdsMax = 100
+)
+
+type readParams struct {
+	cfg      *Config
+	appid    string
+	userids  []string
+	saasHttp *saashttp.SaasClient
+}
+
 func RunRead(args ...string) error {
+	fs := flag.NewFlagSet("read", flag.ExitOnError)
+	cfgFile := paramConfig(fs)
+	appid := paramAppid(fs)
+	userids := paramUserids(fs)
+
+	if err := fs.Parse(args); err != nil {
+		fmt.Println("command line parse error", "err", err)
+		return err
+	}
+
+	// 切割字符串
+	idsSlice := strings.Split(*userids, ",")
+
+	if fs.NArg() > 0 || len(idsSlice) == 0 || (len(idsSlice) == 1 && idsSlice[0] == "") || len(idsSlice) > getIdsMax {
+		fs.PrintDefaults()
+		return nil
+	}
+
+	cfg, err := LoadConfigFile(*cfgFile)
+	if err != nil {
+		slog.Error("LoadConfigFile error", "err", err)
+		return err
+	}
+
+	readParams := readParams{
+		cfg:     cfg,
+		userids: idsSlice,
+		appid:   *appid,
+		saasHttp: &saashttp.SaasClient{
+			Client:  http.Client{},
+			ApiUrls: cfg.ApiUrls,
+			Auth:    cfg.Auth,
+		},
+	}
+
+	return doRead(readParams)
+}
+
+func doRead(readParams readParams) error {
+	saasReq := &saasapi.SaasReq{
+		Cmd: &saasapi.SaasReq_Read{
+			Read: &saasapi.Read{},
+		},
+	}
+
+	if readParams.appid != "" {
+		saasReq.UseridType = saasapi.UserIdType_OPENID
+		saasReq.Appid = readParams.appid
+	}
+	saasReadItems := []*saasapi.ReadItem{}
+	for _, userid := range readParams.userids {
+		saasReadItems = append(saasReadItems, &saasapi.ReadItem{
+			Userid: userid,
+		})
+	}
+
+	saasReq.Cmd.(*saasapi.SaasReq_Read).Read.ReadItems = saasReadItems
+
+	res, err := readParams.saasHttp.Read(saasReq)
+
+	if err != nil {
+		slog.Error("submitRead error", "err", err)
+		return err
+	}
+
+	fmt.Println(protojson.Format(res))
+
 	return nil
 }

cmd/saastool/target_cfg.go

@@ -5,8 +5,8 @@ import (
 	"os"
 )
 
-// TargetConfig 配置
+// MapConfig 配置
-type TargetConfig struct {
+type MapConfig struct {
 	Targets map[string]*Target `json:"targets"`
 }
 
@@ -23,7 +23,7 @@ type Target struct {
 }
 
 // LoadConfigFile 加载配置文件
-func LoadTargetFile(filename string) (*TargetConfig, error) {
+func LoadMapFile(filename string) (*MapConfig, error) {
 	// 打开文件
 	f, err := os.Open(filename)
 	if err != nil {
@@ -31,7 +31,7 @@ func LoadTargetFile(filename string) (*TargetConfig, error) {
 	}
 	defer f.Close()
 
-	sc := &TargetConfig{}
+	sc := &MapConfig{}
 
 	err = json.NewDecoder(f).Decode(sc)
 	return sc, err

cmd/saastool/task.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"fmt"
+	"log/slog"
+	"strings"
+)
+
+func RunTask(args ...string) error {
+	name, args := ParseCommandName(args)
+
+	// 从参数中解析出命令
+	switch name {
+	case "", "help":
+		return RunTaskHelp(args...)
+	case "create":
+		return RunTaskCreate(args...)
+	case "list":
+		return RunTaskList(args...)
+	case "delete":
+		return RunTaskDelete(args...)
+	case "info":
+		return RunTaskInfo(args...)
+	default:
+		err := fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'saastool task help' for usage`, name)
+		slog.Warn(err.Error())
+		return err
+	}
+}
+
+func RunTaskHelp(args ...string) error {
+	fmt.Println(strings.TrimSpace(taskUsage))
+	return nil
+}
+
+const taskUsage = `
+Usage:  saastoola task COMMAND [OPTIONS]
+
+Commands:
+    create             
+    upload               Read user's 'bytes / uint32s / flags'
+    run 
+    delete
+    info
+
+"help" is the default command.
+
+Use "saastool task COMMAND -help" for more information about a command.
+`

cmd/saastool/task_cancel.go

@@ -1,5 +0,0 @@
-package main
-
-func RunTaskCancel(args ...string) error {
-	return nil
-}

cmd/saastool/task_create.go

@@ -0,0 +1,5 @@
+package main
+
+func RunTaskCreate(args ...string) error {
+	return nil
+}

cmd/saastool/task_delete.go

@@ -0,0 +1,5 @@
+package main
+
+func RunTaskDelete(args ...string) error {
+	return nil
+}

cmd/saastool/task_detail.go

@@ -1,5 +0,0 @@
-package main
-
-func RunTaskDetail(args ...string) error {
-	return nil
-}

cmd/saastool/task_info.go

@@ -0,0 +1,5 @@
+package main
+
+func RunTaskInfo(args ...string) error {
+	return nil
+}

cmd/saastool/task_run.go

@@ -0,0 +1,5 @@
+package main
+
+func RunTaskRun(args ...string) error {
+	return nil
+}

cmd/saastool/write.go

@@ -19,7 +19,6 @@ type writeParams struct {
 	sourcePath string
 	appid      string
 	batchSize  uint
-	async      bool
 	clear      bool
 	saasHttp   *saashttp.SaasClient
 }
@@ -30,7 +29,6 @@ func RunWrite(args ...string) error {
 	sourcePath := paramSourcePath(fs)
 	appid := paramAppid(fs)
 	batchSize := paramBatchSize(fs)
-	async := paramAsync(fs)
 	clear := paramClear(fs)
 
 	if err := fs.Parse(args); err != nil {
@@ -53,7 +51,6 @@ func RunWrite(args ...string) error {
 		sourcePath: *sourcePath,
 		appid:      *appid,
 		batchSize:  *batchSize,
-		async:      *async,
 		clear:      *clear,
 		saasHttp: &saashttp.SaasClient{
 			Client:  http.Client{},
@@ -105,7 +102,7 @@ func doLoadFileToWrite(writeParams writeParams) error {
 
 	scaner := bufio.NewScanner(file)
 
-	saasWriteCmds := []*saasapi.WriteCmd{}
+	saasWriteItems := []*saasapi.WriteItem{}
 
 	succ := uint32(0)
 	succTotal := uint32(0)
@@ -115,29 +112,29 @@ func doLoadFileToWrite(writeParams writeParams) error {
 		if line == "" {
 			continue
 		}
-		saasWriteCmd := &saasapi.WriteCmd{}
+		saasWriteItem := &saasapi.WriteItem{}
-		if err = protojson.Unmarshal([]byte(line), saasWriteCmd); err != nil {
+		if err = protojson.Unmarshal([]byte(line), saasWriteItem); err != nil {
 			return err
 		}
 
-		saasWriteCmds = append(saasWriteCmds, saasWriteCmd)
+		saasWriteItems = append(saasWriteItems, saasWriteItem)
 		total++
 
-		if len(saasWriteCmds) == int(writeParams.batchSize) {
+		if len(saasWriteItems) == int(writeParams.batchSize) {
-			if succ, _, err = submitWrite(writeParams, saasWriteCmds); err != nil {
+			if succ, _, err = submitWrite(writeParams, saasWriteItems); err != nil {
 				return err
 			}
 
 			succTotal += succ
 			fmt.Printf("[%v]  batch_succ = %v, succ_total = %v, total_processed = %v\n", writeParams.sourcePath, succ, succTotal, total)
 
-			saasWriteCmds = saasWriteCmds[:0]
+			saasWriteItems = saasWriteItems[:0]
 		}
 
 	}
 
-	if len(saasWriteCmds) > 0 {
+	if len(saasWriteItems) > 0 {
-		if succ, _, err = submitWrite(writeParams, saasWriteCmds); err != nil {
+		if succ, _, err = submitWrite(writeParams, saasWriteItems); err != nil {
 			return err
 		}
 		succTotal += succ
@@ -147,12 +144,11 @@ func doLoadFileToWrite(writeParams writeParams) error {
 	return nil
 }
 
-func submitWrite(writeParams writeParams, saasWriteCmds []*saasapi.WriteCmd) (succ, total uint32, err error) {
+func submitWrite(writeParams writeParams, saasWriteCmds []*saasapi.WriteItem) (succ, total uint32, err error) {
 	saasReq := &saasapi.SaasReq{
 		Cmd: &saasapi.SaasReq_Write{
 			Write: &saasapi.Write{
 				IsClearAllFirst: writeParams.clear,
-				Async:           writeParams.async,
 			},
 		},
 	}
@@ -162,10 +158,15 @@ func submitWrite(writeParams writeParams, saasWriteCmds []*saasapi.WriteCmd) (su
 		saasReq.Appid = writeParams.appid
 	}
 
-	saasReq.Cmd.(*saasapi.SaasReq_Write).Write.WriteCmds = saasWriteCmds
+	saasReq.Cmd.(*saasapi.SaasReq_Write).Write.WriteItems = saasWriteCmds
 
 	total = uint32(len(saasWriteCmds))
-	succ, err = writeParams.saasHttp.Write(saasReq)
+	res, err := writeParams.saasHttp.Write(saasReq)
 
+	if err != nil {
+		slog.Error("submitWrite error", "err", err)
 		return
 	}
+
+	return res.GetWriteRes().GetSuccCmdCount(), total, nil
+}

pkg/saashttp/httpcli.go

@@ -30,32 +30,32 @@ type SaasClient struct {
 	ResponseEncoder ResponseEncoder
 }
 
-func (c *SaasClient) Write(saasReq *saasapi.SaasReq) (succ uint32, err error) {
+func (c *SaasClient) Write(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
 	postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.WritePath)
 	return c.post(postUrl, saasReq)
 }
 
-func (c *SaasClient) Read(saasReq *saasapi.SaasReq) (succ uint32, err error) {
+func (c *SaasClient) Read(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
 	postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.ReadPath)
 	return c.post(postUrl, saasReq)
 }
 
-func (c *SaasClient) ColumnWrite(saasReq *saasapi.SaasReq) (succ uint32, err error) {
+func (c *SaasClient) ColumnWrite(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
 	postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.ColumnWritePath)
 	return c.post(postUrl, saasReq)
 }
 
-func (c *SaasClient) TaskList(saasReq *saasapi.SaasReq) (succ uint32, err error) {
+func (c *SaasClient) TaskList(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
 	postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskListPath)
 	return c.post(postUrl, saasReq)
 }
 
-func (c *SaasClient) TaskCancel(saasReq *saasapi.SaasReq) (succ uint32, err error) {
+func (c *SaasClient) TaskCancel(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
 	postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskCancelPath)
 	return c.post(postUrl, saasReq)
 }
 
-func (c *SaasClient) TaskDetail(saasReq *saasapi.SaasReq) (succ uint32, err error) {
+func (c *SaasClient) TaskDetail(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
 
 	postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskDetailPath)
 	return c.post(postUrl, saasReq)
@@ -82,17 +82,17 @@ func (c *SaasClient) makeUrl(baseUrl, path string) string {
 	return url.String()
 }
 
-func (c *SaasClient) post(url string, saasReq *saasapi.SaasReq) (succ uint32, err error) {
+func (c *SaasClient) post(url string, saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
 	postBuf, err := proto.Marshal(saasReq)
 	if err != nil {
 		fmt.Println("marshal saas req error", err)
-		return 0, err
+		return nil, err
 	}
 
 	req, err := http.NewRequest("POST", url, bytes.NewBuffer(postBuf))
 	if err != nil {
 		fmt.Println("http new request error", err)
-		return 0, err
+		return nil, err
 	}
 
 	timeStamp := strconv.FormatInt(time.Now().Unix(), 10)
@@ -106,7 +106,7 @@ func (c *SaasClient) post(url string, saasReq *saasapi.SaasReq) (succ uint32, er
 	res, err := c.Client.Do(req)
 	if err != nil {
 		fmt.Println("http send error", err)
-		return 0, err
+		return nil, err
 	}
 
 	defer res.Body.Close()
@@ -114,27 +114,23 @@ func (c *SaasClient) post(url string, saasReq *saasapi.SaasReq) (succ uint32, er
 	resBody, err := io.ReadAll(res.Body)
 	if err != nil {
 		fmt.Println("http read body error", err)
-		return 0, err
+		return nil, err
 	}
 
-	saasRes := &saasapi.SaasRes{}
+	saasRes = &saasapi.SaasRes{}
 	if c.ResponseEncoder == RESPONSE_ENCODER_PROTOBUF {
 		err = proto.Unmarshal(resBody, saasRes)
 		if err != nil {
 			fmt.Println("unmarshal response body to protobuf error", err)
-			return 0, err
+			return nil, err
 		}
 	} else {
 		err = json.Unmarshal(resBody, saasRes)
 		if err != nil {
 			fmt.Println("unmarshal response body to json error", err)
-			return 0, err
+			return nil, err
 		}
 	}
-	if saasRes.Code != saasapi.ErrorCode(saasapi.CmdErrorCode_OK) {
-		fmt.Println("saas api error", saasRes.Code, saasRes.Status)
-		return 0, err
-	}
-	return saasRes.GetSuccCmdCount(), nil
 
+	return saasRes, nil
 }