public/saasapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加命令功能

Browse Source
This commit is contained in:
algotao
2025-04-07 10:42:08 +08:00
parent 0344c09ce7
commit 74b0033e45
18 changed files with 1844 additions and 494 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1397
cmd.pb.go
View File

File diff suppressed because it is too large Load Diff

138
cmd.proto
View File

@@ -10,25 +10,36 @@ message SaasReq {
string appid = 2; // 小程序/小游戏/公众号/视频号的appid
oneof cmd {
Write write = 10; // 批量写入
Read read = 11; // 批量读取
Read read = 10; // 批量读取
Write write = 11; // 批量写入
ColumnWrite column_write = 12; // 全量列式写入
TaskList task_list = 20; // 任务列表
TaskCancel task_cancel = 21; // 取消任务
TaskDetail task_detail = 22; // 任务详情
Task task_create = 20; // 任务创建
TaskList task_list = 21; // 列出任务
TaskRun task_run = 22; // 执行任务
TaskDelete task_delete = 23; // 删除任务
TaskInfo task_info = 24; // 任务详情
}
}
// Read 批量读取命令
message Read {
repeated ReadItem read_items = 1; // 批量获取命令
}
// ReadItem 读取命令
message ReadItem {
string userid = 1; // 用户ID
}
// Write 批量写入命令
message Write {
bool async = 1; // 是否异步执行
bool is_clear_all_first = 2; // 是否先执行清空
repeated WriteCmd write_cmds = 3; // 批量写入命令
bool is_clear_all_first = 1; // 是否先清空该用户所有数据
repeated WriteItem write_items = 2; // 批量写入命令
}
// WriteCmd 写入命令
message WriteCmd {
// WriteItem 写入命令
message WriteItem {
string userid = 1; // 用户ID
Bytes write_bytes = 2; // byte区域
Uint32s write_uint32s = 3; // uint32区域
@@ -66,16 +77,7 @@ message FlagWithExpire {
enum UserIdType {
DEVICEID = 0; // 设备号
OPENID = 1; // OpenId
}
// Write 批量读取命令
message Read {
repeated ReadCmd read_cmds = 1; // 批量获取命令
}
// WriteCmd 读取命令
message ReadCmd {
string userid = 1; // 用户ID
INNERID1 = 10; // 内部ID1
}
// ColumnWrite 全量列式写入命令
@@ -86,32 +88,82 @@ message ColumnWrite {
bool is_clear_all_first = 5; // 是否先执行清空
}
message Task {
string task_sha256 = 1; // 任务sha256
string task_description = 2; // 任务描述
repeated FileInfo task_file_infos = 3; // 文件列表
uint64 task_block_size = 4; // 文件块字节大小（推荐50M）
// 以下字段只在返回时填写，用于提供服务端的任务状态。在请求时填写会被忽略
string create_time = 10; // 创建时间
string run_time = 11; // 运行时间
string finish_time = 12; // 完成时间
TaskStatus status = 15; // 任务状态
}
// TaskList 任务列表
message TaskList {
TaskStatus status_filter = 1; // 只显示指定状态的任务
}
// TaskCancel 取消任务
message TaskCancel {
// TaskRun 任务运行
message TaskRun {
string task_sha256 = 1; // 任务sha256
}
// TaskDetail 任务详情
message TaskDetail {
// TaskDelete 取消任务
message TaskDelete {
string task_sha256 = 1; // 任务sha256
}
// TaskInfo 任务详情
message TaskInfo {
string task_sha256 = 1; // 任务sha256
}
message FileInfo {
string file_name = 1; // 文件名
uint64 file_size = 2; // 文件大小
repeated FileBlock file_blocks = 3; // 文件块列表
}
message FileBlock {
string block_sha256 = 1; // 块的sha256
uint64 block_length = 2; // 块的字节长度
bool uploaded = 3; // 是否已上传（在TaskInfo请求返回）
}
// SaasRes 命令返回
message SaasRes {
ErrorCode code = 1; // 返回码
string status = 2; // 返回信息的文本提示
uint32 succ_cmd_count = 3; // 成功的命令数量
uint32 fail_cmd_count = 4; // 失败的命令数量
repeated CmdsResItem cmd_res = 5; // 返回的命令
oneof res {
ReadRes read_res = 10; // 读取命令返回
WriteRes write_res = 11; // 写入命令返回
Task task_create_res = 20; // 创建任务返回状态
TaskListRes task_list_res = 21; // 任务列表返回状态
Task task_run_res = 22; // 运行任务返回状态
Task task_info_res = 23; // 任务详情返回状态
Task task_delete_res = 24; // 删除任务返回状态
}
}
// CmdsResItem 读取命令返回内容
message CmdsResItem {
message ReadRes {
uint32 succ_cmd_count = 1; // 成功的命令数量
uint32 fail_cmd_count = 2; // 失败的命令数量
repeated ValueItem cmd_res = 3; // 返回的命令
}
message WriteRes {
uint32 succ_cmd_count = 1; // 成功的命令数量
uint32 fail_cmd_count = 2; // 失败的命令数量
repeated ValueItem cmd_res = 3; // 返回的失败命令，仅填写cmd_index和cmd_code
}
// ValueItem 读取命令返回内容
message ValueItem {
uint32 cmd_index = 1; // 命令索引
CmdErrorCode cmd_code = 2; // 状态
bytes bytes = 3; // byte区域
@@ -120,11 +172,15 @@ message CmdsResItem {
uint32 last_modify_time = 6; // 最后修改时间
}
message TaskListRes {
repeated Task tasks = 1; // 任务列表
}
// ErrorCode 返回码
enum ErrorCode {
SUCC = 0; // 成功
INVALID_ACCOUNT = 101; // Account不合法
INVALID_TIMESTAMP = 102; // 头信息缺少时间戳
INVALID_TIMESTAMP = 102; // 头信息缺少时间戳或不正确
INVALID_SIGNATURE = 103; // 头信息缺少签名
AUTH_FAIL = 104; // 签名较验失败
DISABLED_ACCOUNT = 105; // 账号已禁用
@@ -134,10 +190,28 @@ enum ErrorCode {
QPS_LIMIT = 113; // 并发请求量超限
CMDS_LIMIT = 114; // 命令数量超限
CMDS_NULL = 115; // 命令为空
TASK_EXISTS = 120; // 任务已存在
TASK_IS_NOT_EXISTS = 121; // 任务不存在
TASK_NUM_LIMIT = 122; // 任务数达到上限
TASK_BLOCK_SIZE = 123; // 块大小超限
TASK_TOTAL_SIZE = 124; // 总文件大小超限
TASK_MARSHAL = 125; // 序列化
DATA_ERROR = 201; // 数据错误
}
enum CmdErrorCode {
OK = 0; // 成功
}
enum TaskStatus {
ALL = 0; // 全部
WAITING = 1; // 等待中
RUNNING = 2; // 运行中
SUCCESS = 3; // 成功
FAIL = 4; // 失败
DELETED = 5; // 已删除，仅在执行删除成功时返回
}

164
cmd/saastool/convert.go
View File

@@ -2,27 +2,38 @@ package main
import (
"bufio"
"bytes"
"flag"
"fmt"
"os"
"path"
"runtime"
"strings"
"sync"
"e.coding.net/rta/public/saasapi"
"google.golang.org/protobuf/encoding/protojson"
)
// TODO 转换加速
const (
convertBatchSize = 100000
convertedExt = ".converted"
)
type convertParams struct {
targetCfg *TargetConfig
mapCfg *MapConfig
sourcePath string
destPath string
}
type convertResult struct {
resultBuf bytes.Buffer
convertedLines int
}
func RunConvert(args ...string) error {
fs := flag.NewFlagSet("convert", flag.ExitOnError)
targetCfgFile := paramTargets(fs)
mapCfgFile := paramMap(fs)
sourcePath := paramSourcePath(fs)
destPath := paramDestPath(fs)
@@ -31,19 +42,19 @@ func RunConvert(args ...string) error {
return err
}
if fs.NArg() > 0 || *targetCfgFile == "" || len(*sourcePath) == 0 || len(*destPath) == 0 {
if fs.NArg() > 0 || *mapCfgFile == "" || len(*sourcePath) == 0 || len(*destPath) == 0 {
fs.PrintDefaults()
return nil
}
targetCfg, err := LoadTargetFile(*targetCfgFile)
mapCfg, err := LoadMapFile(*mapCfgFile)
if err != nil {
fmt.Println("LoadConfigFile error", "err", err)
return err
}
convertParams := convertParams{
targetCfg: targetCfg,
mapCfg: mapCfg,
sourcePath: *sourcePath,
destPath: *destPath,
}
@@ -97,7 +108,7 @@ func doFileConvert(convertParams convertParams) error {
os.MkdirAll(convertParams.destPath, os.ModePerm)
}
destName := path.Join(convertParams.destPath, path.Base(convertParams.sourcePath)+".converted")
destName := path.Join(convertParams.destPath, path.Base(convertParams.sourcePath)+convertedExt)
destFile, err := os.Create(destName)
if err != nil {
return err
@@ -108,15 +119,92 @@ func doFileConvert(convertParams convertParams) error {
destWriter := bufio.NewWriter(destFile)
defer destWriter.Flush()
jasonMarshal := protojson.MarshalOptions{Multiline: false, Indent: ""}
// 启动处理协程
workers := []chan []string{}
results := []chan convertResult{}
processedLine := 0
wg := sync.WaitGroup{}
convertMaxWorkers := runtime.GOMAXPROCS(0)
for range convertMaxWorkers {
workerChan := make(chan []string)
workers = append(workers, workerChan)
resultChan := make(chan convertResult)
results = append(results, resultChan)
go func(workerChan <-chan []string, resultChan chan<- convertResult) {
for lines := range workerChan {
convertBatch(lines, convertParams, resultChan)
}
}(workerChan, resultChan)
}
// 启动写入协程
go func() {
i := 0
// TIP: 不要改成for range
for {
select {
case result, ok := <-results[i%convertMaxWorkers]:
if !ok {
return
}
destWriter.Write(result.resultBuf.Bytes())
destWriter.Flush()
processedLine += result.convertedLines
fmt.Printf("\rconverted records: %v [%v]", processedLine, destName)
i++
wg.Done()
}
}
}()
// 读取文件并塞给协程处理
batch := []string{}
batchCount := 0
for scaner.Scan() {
line := scaner.Text()
if line == "" {
continue
}
batch = append(batch, line)
if len(batch) == convertBatchSize {
// 将batch写入协程
wg.Add(1)
workers[batchCount%convertMaxWorkers] <- batch
batch = nil
batchCount++
}
}
if len(batch) > 0 {
wg.Add(1)
workers[batchCount%convertMaxWorkers] <- batch
}
wg.Wait()
// 关闭所有工作协程的通道
for _, workerChan := range workers {
close(workerChan)
}
for _, resultChan := range results {
close(resultChan)
}
fmt.Println("")
return nil
}
func convertBatch(lines []string, convertParams convertParams, resultChan chan<- convertResult) {
byteBuf := bytes.Buffer{}
byteBuf.Grow(1024 * 1024 * 10)
jasonMarshal := protojson.MarshalOptions{Multiline: false, Indent: ""}
for _, line := range lines {
// 按\t分割为两列
parts := strings.Split(line, "\t")
if len(parts) != 2 {
@@ -125,65 +213,63 @@ func doFileConvert(convertParams convertParams) error {
// 读取userid
userid := parts[0]
if len(userid) == 0 {
continue
}
value := parts[1]
value = strings.ReplaceAll(value, "[", "")
value = strings.ReplaceAll(value, "]", "")
// 第二列解析为string数组
targets := strings.Split(value, " ")
saasWriteCmd := &saasapi.WriteCmd{
saasWriteItem := &saasapi.WriteItem{
Userid: userid,
}
if len(userid) == 0 || len(targets) == 0 {
continue
}
// 遍历targets，转换成saasapi.WriteCmd
for _, target := range targets {
if targetinfo, ok := convertParams.targetCfg.Targets[target]; ok {
if targetinfo, ok := convertParams.mapCfg.Targets[target]; ok {
if targetinfo.WriteByte != nil {
if saasWriteCmd.WriteBytes == nil {
saasWriteCmd.WriteBytes = &saasapi.Bytes{}
// 转换byte区
if saasWriteItem.WriteBytes == nil {
saasWriteItem.WriteBytes = &saasapi.Bytes{}
}
saasWriteCmd.WriteBytes.Bytes = append(saasWriteCmd.WriteBytes.Bytes, *targetinfo.WriteByte)
saasWriteItem.WriteBytes.Bytes = append(saasWriteItem.WriteBytes.Bytes, *targetinfo.WriteByte)
if targetinfo.WriteBytePos < 64 {
saasWriteCmd.WriteBytes.Index_1 |= 1 << targetinfo.WriteBytePos
saasWriteItem.WriteBytes.Index_1 |= 1 << targetinfo.WriteBytePos
} else if targetinfo.WriteBytePos < 128 {
saasWriteCmd.WriteBytes.Index_2 |= 1 << (targetinfo.WriteBytePos - 64)
saasWriteItem.WriteBytes.Index_2 |= 1 << (targetinfo.WriteBytePos - 64)
}
}
if targetinfo.WriteUint32 != nil {
if saasWriteCmd.WriteUint32S == nil {
saasWriteCmd.WriteUint32S = &saasapi.Uint32S{}
// 转换uint32区
if saasWriteItem.WriteUint32S == nil {
saasWriteItem.WriteUint32S = &saasapi.Uint32S{}
}
saasWriteCmd.WriteUint32S.Uint32S = append(saasWriteCmd.WriteUint32S.Uint32S, *targetinfo.WriteUint32)
saasWriteCmd.WriteUint32S.Index_1 |= 1 << targetinfo.WriteUint32Pos
saasWriteItem.WriteUint32S.Uint32S = append(saasWriteItem.WriteUint32S.Uint32S, *targetinfo.WriteUint32)
saasWriteItem.WriteUint32S.Index_1 |= 1 << targetinfo.WriteUint32Pos
}
if targetinfo.WriteFlag != nil && targetinfo.WriteExpire != nil {
if saasWriteCmd.WriteFlagsWithExpire == nil {
saasWriteCmd.WriteFlagsWithExpire = &saasapi.FlagsWithExpire{}
// 转换flag区
if saasWriteItem.WriteFlagsWithExpire == nil {
saasWriteItem.WriteFlagsWithExpire = &saasapi.FlagsWithExpire{}
}
saasWriteCmd.WriteFlagsWithExpire.FlagsWithExpire = append(
saasWriteCmd.WriteFlagsWithExpire.FlagsWithExpire, &saasapi.FlagWithExpire{
saasWriteItem.WriteFlagsWithExpire.FlagsWithExpire = append(
saasWriteItem.WriteFlagsWithExpire.FlagsWithExpire, &saasapi.FlagWithExpire{
Flag: *targetinfo.WriteFlag,
Expire: *targetinfo.WriteExpire,
})
saasWriteCmd.WriteFlagsWithExpire.Index_1 |= 1 << targetinfo.WriteFlagWithExpirePos
saasWriteItem.WriteFlagsWithExpire.Index_1 |= 1 << targetinfo.WriteFlagWithExpirePos
}
}
}
// 写入文件
destWriter.WriteString(jasonMarshal.Format(saasWriteCmd))
destWriter.WriteString("\n")
processedLine++
if processedLine%100000 == 0 {
fmt.Printf("\rconverted records: %v [%v]", processedLine, destName)
byteBuf.WriteString(jasonMarshal.Format(saasWriteItem))
byteBuf.WriteString("\n")
}
}
fmt.Printf("\rconverted records: %v [%v]\n", processedLine, destName)
return nil
resultChan <- convertResult{byteBuf, len(lines)}
}

16
cmd/saastool/help.go
View File

@@ -12,21 +12,19 @@ func RunHelp(args ...string) error {
}
const usage = `
Usage: [[command] [arguments]]
The commands are:
Usage: saastool COMMAND [OPTIONS]
Commands:
write Write user's 'bytes / uint32s / flags'
read Read user's 'bytes / uint32s / flags'
columnwrite Write columns for 'deviceid / openid' users
tasklist List tasks
taskcancel Cancel task
taskdetail Show task detail
convert Convert data to write format
makehash Make file hash for upload task
task Task commands
"help" is the default command.
Use "saastool [command] -help" for more information about a command.
Use "saastool COMMAND -help" for more information about a command.
`
// strip Stripping redundant data from redis

11
cmd/saastool/main.go
View File

@@ -29,14 +29,13 @@ func Run(args ...string) error {
return RunColumnWrite(args...)
case "convert":
return RunConvert(args...)
case "tasklist":
return RunTaskList(args...)
case "taskcancel":
return RunTaskCancel(args...)
case "taskdetail":
return RunTaskDetail(args...)
case "makehash":
return RunMakeHash(args...)
case "verify":
return RunVerify(args...)
case "task":
return RunTask(args...)
default:
err := fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'saastool help' for usage`, name)
slog.Warn(err.Error())

215
cmd/saastool/make_hash.go Normal file
View File

@@ -0,0 +1,215 @@
package main
import (
"crypto/sha256"
"encoding/hex"
"flag"
"fmt"
"os"
"path"
"runtime"
"sort"
"sync"
"e.coding.net/rta/public/saasapi"
"google.golang.org/protobuf/encoding/protojson"
)
const (
blockSizeMin = 10 * 1024 * 1024
blockSizeMax = 200 * 1024 * 1024
)
type makeHashParams struct {
sourcePath string
destPath string
task *saasapi.Task
}
// 计算任务
type hashTask struct {
chunk []byte
index int
}
// 计算结果
type hashResult struct {
hash string
blockSize uint64
index int
}
func RunMakeHash(args ...string) error {
fs := flag.NewFlagSet("tasklocalmake", flag.ExitOnError)
sourcePath := paramSourcePath(fs)
destPath := paramDestPath(fs)
blockSize := paramBlockSize(fs)
if err := fs.Parse(args); err != nil {
fmt.Println("command line parse error", "err", err)
return err
}
if fs.NArg() > 0 || len(*sourcePath) == 0 || len(*destPath) == 0 {
fs.PrintDefaults()
return nil
}
if blockSize < blockSizeMin || blockSize > blockSizeMax {
fmt.Println("block size error", "min", blockSizeMin, "max", blockSizeMax)
return nil
}
makeHashParams := makeHashParams{
sourcePath: *sourcePath,
destPath: *destPath,
task: &saasapi.Task{
TaskBlockSize: blockSize,
},
}
return doMakeHash(makeHashParams)
}
func doMakeHash(makeHashParams makeHashParams) error {
fsInfo, err := os.Stat(makeHashParams.sourcePath)
if err != nil {
return err
}
if !fsInfo.IsDir() {
// 如果是文件，直接计算
return doFileHash(makeHashParams)
}
// 读取目录下信息
dirEntry, err := os.ReadDir(makeHashParams.sourcePath)
if err != nil {
return err
}
// 遍历目录
for _, dir := range dirEntry {
newParam := makeHashParams
newParam.sourcePath = path.Join(makeHashParams.sourcePath, dir.Name())
if dir.IsDir() {
newParam.destPath = path.Join(makeHashParams.destPath, dir.Name())
}
if err = doMakeHash(newParam); err != nil {
return err
}
}
return saveTaskFile(makeHashParams)
}
func doFileHash(makeHashParams makeHashParams) error {
sourceFile, err := os.Open(makeHashParams.sourcePath)
if err != nil {
return err
}
defer sourceFile.Close()
fi, err := sourceFile.Stat()
if err != nil {
return err
}
tasks := make(chan hashTask)
results := make(chan hashResult)
// 启动工作协程
hashMaxWorker := runtime.GOMAXPROCS(0)
for range hashMaxWorker {
go hashWorker(tasks, results)
}
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
index := 0
buffer := make([]byte, makeHashParams.task.TaskBlockSize)
for {
n, err := sourceFile.Read(buffer)
if n > 0 {
wg.Add(1)
fmt.Printf("\rhashing file [%v], block [%v]", makeHashParams.sourcePath, index)
tasks <- hashTask{chunk: buffer[:n], index: index}
index++
}
if err != nil {
break
}
}
close(tasks)
wg.Done()
}()
var allResults []hashResult
go func() {
for r := range results {
allResults = append(allResults, r)
wg.Done()
}
}()
wg.Wait()
close(results)
// 按索引排序结果
sort.Slice(allResults, func(i, j int) bool {
return allResults[i].index < allResults[j].index
})
// 输出结果
fileInfo := &saasapi.FileInfo{
FileName: makeHashParams.sourcePath,
FileSize: uint64(fi.Size()),
}
for _, r := range allResults {
fileInfo.FileBlocks = append(fileInfo.FileBlocks, &saasapi.FileBlock{
BlockSha256: r.hash,
BlockLength: r.blockSize,
})
}
makeHashParams.task.TaskFileInfos = append(makeHashParams.task.TaskFileInfos, fileInfo)
fmt.Println("")
return nil
}
// hash计算协程
func hashWorker(tasks <-chan hashTask, results chan<- hashResult) {
for t := range tasks {
h := sha256.New()
h.Write(t.chunk)
hash := hex.EncodeToString(h.Sum(nil))
results <- hashResult{hash: hash, index: t.index, blockSize: uint64(len(t.chunk))}
}
}
func saveTaskFile(makeHashParams makeHashParams) error {
taskFile, err := os.Create(makeHashParams.destPath)
if err != nil {
return err
}
defer taskFile.Close()
h := sha256.New()
for _, fileInfo := range makeHashParams.task.TaskFileInfos {
for _, fileBlock := range fileInfo.FileBlocks {
h.Write([]byte(fileBlock.BlockSha256))
}
}
makeHashParams.task.TaskSha256 = hex.EncodeToString(h.Sum(nil))
_, err = taskFile.WriteString(protojson.Format(makeHashParams.task))
if err != nil {
return err
}
return nil
}

79
cmd/saastool/params.go
View File

@@ -2,31 +2,50 @@ package main
import (
"flag"
"fmt"
"strconv"
"strings"
)
func paramConfig(fs *flag.FlagSet) *string {
return fs.String("config", "cfg.toml", "Config file.")
}
func paramTargets(fs *flag.FlagSet) *string {
return fs.String("targets", "", "target setting")
func paramMap(fs *flag.FlagSet) *string {
return fs.String("map", "", "target map setting")
}
func paramSourcePath(fs *flag.FlagSet) *string {
return fs.String("source", "", "Data path source for write command.")
return fs.String("source", "", "Source path or filename")
}
func paramDestPath(fs *flag.FlagSet) *string {
return fs.String("dest", "", "Data path destination for write command.")
return fs.String("dest", "", "Destination path or filename")
}
func paramAppid(fs *flag.FlagSet) *string {
return fs.String("appid", "", "Wechat appid")
}
func paramUserids(fs *flag.FlagSet) *string {
return fs.String("userids", "", "Device ID or Wechat UserID, separated by comma")
}
func paramBatchSize(fs *flag.FlagSet) *uint {
return fs.Uint("batchsize", 10000, "Batch size to sync")
}
func paramBlockSize(fs *flag.FlagSet) uint64 {
bsize := fs.String("blocksize", "50M", "Block size to make hash. using size mode K, M, G, T")
num, err := ParseByteSize(*bsize)
if err != nil {
fmt.Println("Error parsing block size", "err", err)
fmt.Println("Using default 50M")
num = 50 * 1024 * 1024
}
return num
}
func paramAsync(fs *flag.FlagSet) *bool {
return fs.Bool("async", false, "Async mode")
}
@@ -34,3 +53,55 @@ func paramAsync(fs *flag.FlagSet) *bool {
func paramClear(fs *flag.FlagSet) *bool {
return fs.Bool("clear", false, "Clear all data before write")
}
// ParseByteSize 解析字节大小字符串为字节数
func ParseByteSize(sizeStr string) (uint64, error) {
sizeStr = strings.TrimSpace(sizeStr)
unit := ""
numStr := sizeStr
// 提取单位
if len(sizeStr) > 1 && (sizeStr[len(sizeStr)-1] == 'B' || sizeStr[len(sizeStr)-1] == 'b') {
unit = string(sizeStr[len(sizeStr)-2:])
numStr = sizeStr[:len(sizeStr)-2]
} else if len(sizeStr) > 0 && (sizeStr[len(sizeStr)-1] >= 'A' && sizeStr[len(sizeStr)-1] <= 'Z' ||
sizeStr[len(sizeStr)-1] >= 'a' && sizeStr[len(sizeStr)-1] <= 'z') {
unit = string(sizeStr[len(sizeStr)-1])
numStr = sizeStr[:len(sizeStr)-1]
}
// 解析数字部分
num, err := strconv.ParseFloat(numStr, 64)
if err != nil {
return 0, err
}
// 根据单位计算字节数
switch strings.ToUpper(unit) {
case "":
return uint64(num), nil
case "K", "KB":
return uint64(num * 1024), nil
case "M", "MB":
return uint64(num * 1024 * 1024), nil
case "G", "GB":
return uint64(num * 1024 * 1024 * 1024), nil
case "T", "TB":
return uint64(num * 1024 * 1024 * 1024 * 1024), nil
default:
return 0, fmt.Errorf("unknown unit: %s", unit)
}
}
/*
func main() {
sizes := []string{"1K", "2M", "3G", "4T", "5"}
for _, sizeStr := range sizes {
size, err := ParseByteSize(sizeStr)
if err != nil {
fmt.Printf("Error parsing %s: %v\n", sizeStr, err)
} else {
fmt.Printf("%s = %d bytes\n", sizeStr, size)
}
}
}
*/

90
cmd/saastool/read.go
View File

@@ -1,5 +1,95 @@
package main
import (
"flag"
"fmt"
"log/slog"
"net/http"
"strings"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
const (
getIdsMax = 100
)
type readParams struct {
cfg *Config
appid string
userids []string
saasHttp *saashttp.SaasClient
}
func RunRead(args ...string) error {
fs := flag.NewFlagSet("read", flag.ExitOnError)
cfgFile := paramConfig(fs)
appid := paramAppid(fs)
userids := paramUserids(fs)
if err := fs.Parse(args); err != nil {
fmt.Println("command line parse error", "err", err)
return err
}
// 切割字符串
idsSlice := strings.Split(*userids, ",")
if fs.NArg() > 0 || len(idsSlice) == 0 || (len(idsSlice) == 1 && idsSlice[0] == "") || len(idsSlice) > getIdsMax {
fs.PrintDefaults()
return nil
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
slog.Error("LoadConfigFile error", "err", err)
return err
}
readParams := readParams{
cfg: cfg,
userids: idsSlice,
appid: *appid,
saasHttp: &saashttp.SaasClient{
Client: http.Client{},
ApiUrls: cfg.ApiUrls,
Auth: cfg.Auth,
},
}
return doRead(readParams)
}
func doRead(readParams readParams) error {
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_Read{
Read: &saasapi.Read{},
},
}
if readParams.appid != "" {
saasReq.UseridType = saasapi.UserIdType_OPENID
saasReq.Appid = readParams.appid
}
saasReadItems := []*saasapi.ReadItem{}
for _, userid := range readParams.userids {
saasReadItems = append(saasReadItems, &saasapi.ReadItem{
Userid: userid,
})
}
saasReq.Cmd.(*saasapi.SaasReq_Read).Read.ReadItems = saasReadItems
res, err := readParams.saasHttp.Read(saasReq)
if err != nil {
slog.Error("submitRead error", "err", err)
return err
}
fmt.Println(protojson.Format(res))
return nil
}

8
cmd/saastool/target_cfg.go
View File

@@ -5,8 +5,8 @@ import (
"os"
)
// TargetConfig 配置
type TargetConfig struct {
// MapConfig 配置
type MapConfig struct {
Targets map[string]*Target `json:"targets"`
}
@@ -23,7 +23,7 @@ type Target struct {
}
// LoadConfigFile 加载配置文件
func LoadTargetFile(filename string) (*TargetConfig, error) {
func LoadMapFile(filename string) (*MapConfig, error) {
// 打开文件
f, err := os.Open(filename)
if err != nil {
@@ -31,7 +31,7 @@ func LoadTargetFile(filename string) (*TargetConfig, error) {
}
defer f.Close()
sc := &TargetConfig{}
sc := &MapConfig{}
err = json.NewDecoder(f).Decode(sc)
return sc, err

49
cmd/saastool/task.go Normal file
View File

@@ -0,0 +1,49 @@
package main
import (
"fmt"
"log/slog"
"strings"
)
func RunTask(args ...string) error {
name, args := ParseCommandName(args)
// 从参数中解析出命令
switch name {
case "", "help":
return RunTaskHelp(args...)
case "create":
return RunTaskCreate(args...)
case "list":
return RunTaskList(args...)
case "delete":
return RunTaskDelete(args...)
case "info":
return RunTaskInfo(args...)
default:
err := fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'saastool task help' for usage`, name)
slog.Warn(err.Error())
return err
}
}
func RunTaskHelp(args ...string) error {
fmt.Println(strings.TrimSpace(taskUsage))
return nil
}
const taskUsage = `
Usage: saastoola task COMMAND [OPTIONS]
Commands:
create
upload Read user's 'bytes / uint32s / flags'
run
delete
info
"help" is the default command.
Use "saastool task COMMAND -help" for more information about a command.
`

5
cmd/saastool/task_cancel.go
View File

@@ -1,5 +0,0 @@
package main
func RunTaskCancel(args ...string) error {
return nil
}

5
cmd/saastool/task_create.go Normal file
View File

@@ -0,0 +1,5 @@
package main
func RunTaskCreate(args ...string) error {
return nil
}

5
cmd/saastool/task_delete.go Normal file
View File

@@ -0,0 +1,5 @@
package main
func RunTaskDelete(args ...string) error {
return nil
}

5
cmd/saastool/task_detail.go
View File

@@ -1,5 +0,0 @@
package main
func RunTaskDetail(args ...string) error {
return nil
}

5
cmd/saastool/task_info.go Normal file
View File

@@ -0,0 +1,5 @@
package main
func RunTaskInfo(args ...string) error {
return nil
}

5
cmd/saastool/task_run.go Normal file
View File

@@ -0,0 +1,5 @@
package main
func RunTaskRun(args ...string) error {
return nil
}

33
cmd/saastool/write.go
View File

@@ -19,7 +19,6 @@ type writeParams struct {
sourcePath string
appid string
batchSize uint
async bool
clear bool
saasHttp *saashttp.SaasClient
}
@@ -30,7 +29,6 @@ func RunWrite(args ...string) error {
sourcePath := paramSourcePath(fs)
appid := paramAppid(fs)
batchSize := paramBatchSize(fs)
async := paramAsync(fs)
clear := paramClear(fs)
if err := fs.Parse(args); err != nil {
@@ -53,7 +51,6 @@ func RunWrite(args ...string) error {
sourcePath: *sourcePath,
appid: *appid,
batchSize: *batchSize,
async: *async,
clear: *clear,
saasHttp: &saashttp.SaasClient{
Client: http.Client{},
@@ -105,7 +102,7 @@ func doLoadFileToWrite(writeParams writeParams) error {
scaner := bufio.NewScanner(file)
saasWriteCmds := []*saasapi.WriteCmd{}
saasWriteItems := []*saasapi.WriteItem{}
succ := uint32(0)
succTotal := uint32(0)
@@ -115,29 +112,29 @@ func doLoadFileToWrite(writeParams writeParams) error {
if line == "" {
continue
}
saasWriteCmd := &saasapi.WriteCmd{}
if err = protojson.Unmarshal([]byte(line), saasWriteCmd); err != nil {
saasWriteItem := &saasapi.WriteItem{}
if err = protojson.Unmarshal([]byte(line), saasWriteItem); err != nil {
return err
}
saasWriteCmds = append(saasWriteCmds, saasWriteCmd)
saasWriteItems = append(saasWriteItems, saasWriteItem)
total++
if len(saasWriteCmds) == int(writeParams.batchSize) {
if succ, _, err = submitWrite(writeParams, saasWriteCmds); err != nil {
if len(saasWriteItems) == int(writeParams.batchSize) {
if succ, _, err = submitWrite(writeParams, saasWriteItems); err != nil {
return err
}
succTotal += succ
fmt.Printf("[%v] batch_succ = %v, succ_total = %v, total_processed = %v\n", writeParams.sourcePath, succ, succTotal, total)
saasWriteCmds = saasWriteCmds[:0]
saasWriteItems = saasWriteItems[:0]
}
}
if len(saasWriteCmds) > 0 {
if succ, _, err = submitWrite(writeParams, saasWriteCmds); err != nil {
if len(saasWriteItems) > 0 {
if succ, _, err = submitWrite(writeParams, saasWriteItems); err != nil {
return err
}
succTotal += succ
@@ -147,12 +144,11 @@ func doLoadFileToWrite(writeParams writeParams) error {
return nil
}
func submitWrite(writeParams writeParams, saasWriteCmds []*saasapi.WriteCmd) (succ, total uint32, err error) {
func submitWrite(writeParams writeParams, saasWriteCmds []*saasapi.WriteItem) (succ, total uint32, err error) {
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_Write{
Write: &saasapi.Write{
IsClearAllFirst: writeParams.clear,
Async: writeParams.async,
},
},
}
@@ -162,10 +158,15 @@ func submitWrite(writeParams writeParams, saasWriteCmds []*saasapi.WriteCmd) (su
saasReq.Appid = writeParams.appid
}
saasReq.Cmd.(*saasapi.SaasReq_Write).Write.WriteCmds = saasWriteCmds
saasReq.Cmd.(*saasapi.SaasReq_Write).Write.WriteItems = saasWriteCmds
total = uint32(len(saasWriteCmds))
succ, err = writeParams.saasHttp.Write(saasReq)
res, err := writeParams.saasHttp.Write(saasReq)
if err != nil {
slog.Error("submitWrite error", "err", err)
return
}
return res.GetWriteRes().GetSuccCmdCount(), total, nil
}

34
pkg/saashttp/httpcli.go
View File

@@ -30,32 +30,32 @@ type SaasClient struct {
ResponseEncoder ResponseEncoder
}
func (c *SaasClient) Write(saasReq *saasapi.SaasReq) (succ uint32, err error) {
func (c *SaasClient) Write(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.WritePath)
return c.post(postUrl, saasReq)
}
func (c *SaasClient) Read(saasReq *saasapi.SaasReq) (succ uint32, err error) {
func (c *SaasClient) Read(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.ReadPath)
return c.post(postUrl, saasReq)
}
func (c *SaasClient) ColumnWrite(saasReq *saasapi.SaasReq) (succ uint32, err error) {
func (c *SaasClient) ColumnWrite(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.ColumnWritePath)
return c.post(postUrl, saasReq)
}
func (c *SaasClient) TaskList(saasReq *saasapi.SaasReq) (succ uint32, err error) {
func (c *SaasClient) TaskList(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskListPath)
return c.post(postUrl, saasReq)
}
func (c *SaasClient) TaskCancel(saasReq *saasapi.SaasReq) (succ uint32, err error) {
func (c *SaasClient) TaskCancel(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskCancelPath)
return c.post(postUrl, saasReq)
}
func (c *SaasClient) TaskDetail(saasReq *saasapi.SaasReq) (succ uint32, err error) {
func (c *SaasClient) TaskDetail(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskDetailPath)
return c.post(postUrl, saasReq)
@@ -82,17 +82,17 @@ func (c *SaasClient) makeUrl(baseUrl, path string) string {
return url.String()
}
func (c *SaasClient) post(url string, saasReq *saasapi.SaasReq) (succ uint32, err error) {
func (c *SaasClient) post(url string, saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postBuf, err := proto.Marshal(saasReq)
if err != nil {
fmt.Println("marshal saas req error", err)
return 0, err
return nil, err
}
req, err := http.NewRequest("POST", url, bytes.NewBuffer(postBuf))
if err != nil {
fmt.Println("http new request error", err)
return 0, err
return nil, err
}
timeStamp := strconv.FormatInt(time.Now().Unix(), 10)
@@ -106,7 +106,7 @@ func (c *SaasClient) post(url string, saasReq *saasapi.SaasReq) (succ uint32, er
res, err := c.Client.Do(req)
if err != nil {
fmt.Println("http send error", err)
return 0, err
return nil, err
}
defer res.Body.Close()
@@ -114,27 +114,23 @@ func (c *SaasClient) post(url string, saasReq *saasapi.SaasReq) (succ uint32, er
resBody, err := io.ReadAll(res.Body)
if err != nil {
fmt.Println("http read body error", err)
return 0, err
return nil, err
}
saasRes := &saasapi.SaasRes{}
saasRes = &saasapi.SaasRes{}
if c.ResponseEncoder == RESPONSE_ENCODER_PROTOBUF {
err = proto.Unmarshal(resBody, saasRes)
if err != nil {
fmt.Println("unmarshal response body to protobuf error", err)
return 0, err
return nil, err
}
} else {
err = json.Unmarshal(resBody, saasRes)
if err != nil {
fmt.Println("unmarshal response body to json error", err)
return 0, err
return nil, err
}
}
if saasRes.Code != saasapi.ErrorCode(saasapi.CmdErrorCode_OK) {
fmt.Println("saas api error", saasRes.Code, saasRes.Status)
return 0, err
}
return saasRes.GetSuccCmdCount(), nil
return saasRes, nil
}