public/saasapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tasklist taskinfo

Browse Source
This commit is contained in:
algotao
2025-04-12 21:53:56 +08:00
parent 4fc2aeb9c4
commit 8da742fea1
14 changed files with 484 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

208
cmd.pb.go
View File

@@ -71,6 +71,53 @@ func (UserIdType) EnumDescriptor() ([]byte, []int) {
return file_cmd_proto_rawDescGZIP(), []int{0} return file_cmd_proto_rawDescGZIP(), []int{0}
} }
// NameSpaceId 用户 ID 类型
type NameSpaceId int32
const (
NameSpaceId_DID NameSpaceId = 0 // 默认设备号命名空间
NameSpaceId_WUID NameSpaceId = 1 // 默认WUID命名空间
)
// Enum value maps for NameSpaceId.
var (
NameSpaceId_name = map[int32]string{
0: "DID",
1: "WUID",
}
NameSpaceId_value = map[string]int32{
"DID": 0,
"WUID": 1,
}
)
func (x NameSpaceId) Enum() *NameSpaceId {
p := new(NameSpaceId)
*p = x
return p
}
func (x NameSpaceId) String() string {
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
}
func (NameSpaceId) Descriptor() protoreflect.EnumDescriptor {
return file_cmd_proto_enumTypes[1].Descriptor()
}
func (NameSpaceId) Type() protoreflect.EnumType {
return &file_cmd_proto_enumTypes[1]
}
func (x NameSpaceId) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
// Deprecated: Use NameSpaceId.Descriptor instead.
func (NameSpaceId) EnumDescriptor() ([]byte, []int) {
return file_cmd_proto_rawDescGZIP(), []int{1}
}
// ErrorCode 返回码 // ErrorCode 返回码
type ErrorCode int32 type ErrorCode int32
@@ -153,11 +200,11 @@ func (x ErrorCode) String() string {
} }
func (ErrorCode) Descriptor() protoreflect.EnumDescriptor { func (ErrorCode) Descriptor() protoreflect.EnumDescriptor {
return file_cmd_proto_enumTypes[1].Descriptor() return file_cmd_proto_enumTypes[2].Descriptor()
} }
func (ErrorCode) Type() protoreflect.EnumType { func (ErrorCode) Type() protoreflect.EnumType {
return &file_cmd_proto_enumTypes[1] return &file_cmd_proto_enumTypes[2]
} }
func (x ErrorCode) Number() protoreflect.EnumNumber { func (x ErrorCode) Number() protoreflect.EnumNumber {
@@ -166,7 +213,7 @@ func (x ErrorCode) Number() protoreflect.EnumNumber {
// Deprecated: Use ErrorCode.Descriptor instead. // Deprecated: Use ErrorCode.Descriptor instead.
func (ErrorCode) EnumDescriptor() ([]byte, []int) { func (ErrorCode) EnumDescriptor() ([]byte, []int) {
return file_cmd_proto_rawDescGZIP(), []int{1} return file_cmd_proto_rawDescGZIP(), []int{2}
} }
type CmdErrorCode int32 type CmdErrorCode int32
@@ -196,11 +243,11 @@ func (x CmdErrorCode) String() string {
} }
func (CmdErrorCode) Descriptor() protoreflect.EnumDescriptor { func (CmdErrorCode) Descriptor() protoreflect.EnumDescriptor {
return file_cmd_proto_enumTypes[2].Descriptor() return file_cmd_proto_enumTypes[3].Descriptor()
} }
func (CmdErrorCode) Type() protoreflect.EnumType { func (CmdErrorCode) Type() protoreflect.EnumType {
return &file_cmd_proto_enumTypes[2] return &file_cmd_proto_enumTypes[3]
} }
func (x CmdErrorCode) Number() protoreflect.EnumNumber { func (x CmdErrorCode) Number() protoreflect.EnumNumber {
@@ -209,7 +256,7 @@ func (x CmdErrorCode) Number() protoreflect.EnumNumber {
// Deprecated: Use CmdErrorCode.Descriptor instead. // Deprecated: Use CmdErrorCode.Descriptor instead.
func (CmdErrorCode) EnumDescriptor() ([]byte, []int) { func (CmdErrorCode) EnumDescriptor() ([]byte, []int) {
return file_cmd_proto_rawDescGZIP(), []int{2} return file_cmd_proto_rawDescGZIP(), []int{3}
} }
type TaskStatus int32 type TaskStatus int32
@@ -254,11 +301,11 @@ func (x TaskStatus) String() string {
} }
func (TaskStatus) Descriptor() protoreflect.EnumDescriptor { func (TaskStatus) Descriptor() protoreflect.EnumDescriptor {
return file_cmd_proto_enumTypes[3].Descriptor() return file_cmd_proto_enumTypes[4].Descriptor()
} }
func (TaskStatus) Type() protoreflect.EnumType { func (TaskStatus) Type() protoreflect.EnumType {
return &file_cmd_proto_enumTypes[3] return &file_cmd_proto_enumTypes[4]
} }
func (x TaskStatus) Number() protoreflect.EnumNumber { func (x TaskStatus) Number() protoreflect.EnumNumber {
@@ -267,7 +314,7 @@ func (x TaskStatus) Number() protoreflect.EnumNumber {
// Deprecated: Use TaskStatus.Descriptor instead. // Deprecated: Use TaskStatus.Descriptor instead.
func (TaskStatus) EnumDescriptor() ([]byte, []int) { func (TaskStatus) EnumDescriptor() ([]byte, []int) {
return file_cmd_proto_rawDescGZIP(), []int{3} return file_cmd_proto_rawDescGZIP(), []int{4}
} }
// SaasReq 命令请求 // SaasReq 命令请求
@@ -275,6 +322,7 @@ type SaasReq struct {
state protoimpl.MessageState `protogen:"open.v1"` state protoimpl.MessageState `protogen:"open.v1"`
UseridType UserIdType `protobuf:"varint,1,opt,name=userid_type,json=useridType,proto3,enum=saasapi.UserIdType" json:"userid_type,omitempty"` // 用户ID类型 UseridType UserIdType `protobuf:"varint,1,opt,name=userid_type,json=useridType,proto3,enum=saasapi.UserIdType" json:"userid_type,omitempty"` // 用户ID类型
Appid string `protobuf:"bytes,2,opt,name=appid,proto3" json:"appid,omitempty"` // 小程序/小游戏/公众号/视频号的appid Appid string `protobuf:"bytes,2,opt,name=appid,proto3" json:"appid,omitempty"` // 小程序/小游戏/公众号/视频号的appid
NamespaceId uint32 `protobuf:"varint,3,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` // 命名空间ID
// Types that are valid to be assigned to Cmd: // Types that are valid to be assigned to Cmd:
// //
// *SaasReq_Read // *SaasReq_Read
@@ -334,6 +382,13 @@ func (x *SaasReq) GetAppid() string {
return "" return ""
} }
func (x *SaasReq) GetNamespaceId() uint32 {
if x != nil {
return x.NamespaceId
}
return 0
}
func (x *SaasReq) GetCmd() isSaasReq_Cmd { func (x *SaasReq) GetCmd() isSaasReq_Cmd {
if x != nil { if x != nil {
return x.Cmd return x.Cmd
@@ -1319,7 +1374,7 @@ type FileBlock struct {
state protoimpl.MessageState `protogen:"open.v1"` state protoimpl.MessageState `protogen:"open.v1"`
BlockSha256 string `protobuf:"bytes,1,opt,name=block_sha256,json=blockSha256,proto3" json:"block_sha256,omitempty"` // 块的sha256 BlockSha256 string `protobuf:"bytes,1,opt,name=block_sha256,json=blockSha256,proto3" json:"block_sha256,omitempty"` // 块的sha256
BlockLength uint64 `protobuf:"varint,2,opt,name=block_length,json=blockLength,proto3" json:"block_length,omitempty"` // 块的字节长度 BlockLength uint64 `protobuf:"varint,2,opt,name=block_length,json=blockLength,proto3" json:"block_length,omitempty"` // 块的字节长度
Uploaded bool `protobuf:"varint,3,opt,name=uploaded,proto3" json:"uploaded,omitempty"` // 是否已上传（在TaskInfo请求返回） Uploaded bool `protobuf:"varint,3,opt,name=uploaded,proto3" json:"uploaded,omitempty"` // 是否已上传（在TaskCreate/TaskInfo请求返回）
unknownFields protoimpl.UnknownFields unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
} }
@@ -1807,11 +1862,12 @@ var File_cmd_proto protoreflect.FileDescriptor
const file_cmd_proto_rawDesc = "" + const file_cmd_proto_rawDesc = "" +
"\n" + "\n" +
"\tcmd.proto\x12\asaasapi\"\xe1\x03\n" + "\tcmd.proto\x12\asaasapi\"\x84\x04\n" +
"\aSaasReq\x124\n" + "\aSaasReq\x124\n" +
"\vuserid_type\x18\x01 \x01(\x0e2\x13.saasapi.UserIdTypeR\n" + "\vuserid_type\x18\x01 \x01(\x0e2\x13.saasapi.UserIdTypeR\n" +
"useridType\x12\x14\n" + "useridType\x12\x14\n" +
"\x05appid\x18\x02 \x01(\tR\x05appid\x12#\n" + "\x05appid\x18\x02 \x01(\tR\x05appid\x12!\n" +
"\fnamespace_id\x18\x03 \x01(\rR\vnamespaceId\x12#\n" +
"\x04read\x18\n" + "\x04read\x18\n" +
" \x01(\v2\r.saasapi.ReadH\x00R\x04read\x12&\n" + " \x01(\v2\r.saasapi.ReadH\x00R\x04read\x12&\n" +
"\x05write\x18\v \x01(\v2\x0e.saasapi.WriteH\x00R\x05write\x129\n" + "\x05write\x18\v \x01(\v2\x0e.saasapi.WriteH\x00R\x05write\x129\n" +
@@ -1929,7 +1985,10 @@ const file_cmd_proto_rawDesc = "" +
"\n" + "\n" +
"\x06OPENID\x10\x01\x12\f\n" + "\x06OPENID\x10\x01\x12\f\n" +
"\bINNERID1\x10\n" + "\bINNERID1\x10\n" +
"*\xef\x02\n" + "* \n" +
"\vNameSpaceId\x12\a\n" +
"\x03DID\x10\x00\x12\b\n" +
"\x04WUID\x10\x01*\xef\x02\n" +
"\tErrorCode\x12\b\n" + "\tErrorCode\x12\b\n" +
"\x04SUCC\x10\x00\x12\x13\n" + "\x04SUCC\x10\x00\x12\x13\n" +
"\x0fINVALID_ACCOUNT\x10e\x12\x15\n" + "\x0fINVALID_ACCOUNT\x10e\x12\x15\n" +
@@ -1975,72 +2034,73 @@ func file_cmd_proto_rawDescGZIP() []byte {
return file_cmd_proto_rawDescData return file_cmd_proto_rawDescData
} }
var file_cmd_proto_enumTypes = make([]protoimpl.EnumInfo, 4) var file_cmd_proto_enumTypes = make([]protoimpl.EnumInfo, 5)
var file_cmd_proto_msgTypes = make([]protoimpl.MessageInfo, 22) var file_cmd_proto_msgTypes = make([]protoimpl.MessageInfo, 22)
var file_cmd_proto_goTypes = []any{ var file_cmd_proto_goTypes = []any{
(UserIdType)(0), // 0: saasapi.UserIdType (UserIdType)(0), // 0: saasapi.UserIdType
(ErrorCode)(0), // 1: saasapi.ErrorCode (NameSpaceId)(0), // 1: saasapi.NameSpaceId
(CmdErrorCode)(0), // 2: saasapi.CmdErrorCode (ErrorCode)(0), // 2: saasapi.ErrorCode
(TaskStatus)(0), // 3: saasapi.TaskStatus (CmdErrorCode)(0), // 3: saasapi.CmdErrorCode
(*SaasReq)(nil), // 4: saasapi.SaasReq (TaskStatus)(0), // 4: saasapi.TaskStatus
(*Read)(nil), // 5: saasapi.Read (*SaasReq)(nil), // 5: saasapi.SaasReq
(*ReadItem)(nil), // 6: saasapi.ReadItem (*Read)(nil), // 6: saasapi.Read
(*Write)(nil), // 7: saasapi.Write (*ReadItem)(nil), // 7: saasapi.ReadItem
(*WriteItem)(nil), // 8: saasapi.WriteItem (*Write)(nil), // 8: saasapi.Write
(*Bytes)(nil), // 9: saasapi.Bytes (*WriteItem)(nil), // 9: saasapi.WriteItem
(*Uint32S)(nil), // 10: saasapi.Uint32s (*Bytes)(nil), // 10: saasapi.Bytes
(*FlagsWithExpire)(nil), // 11: saasapi.FlagsWithExpire (*Uint32S)(nil), // 11: saasapi.Uint32s
(*FlagWithExpire)(nil), // 12: saasapi.FlagWithExpire (*FlagsWithExpire)(nil), // 12: saasapi.FlagsWithExpire
(*ColumnWrite)(nil), // 13: saasapi.ColumnWrite (*FlagWithExpire)(nil), // 13: saasapi.FlagWithExpire
(*Task)(nil), // 14: saasapi.Task (*ColumnWrite)(nil), // 14: saasapi.ColumnWrite
(*TaskList)(nil), // 15: saasapi.TaskList (*Task)(nil), // 15: saasapi.Task
(*TaskRun)(nil), // 16: saasapi.TaskRun (*TaskList)(nil), // 16: saasapi.TaskList
(*TaskDelete)(nil), // 17: saasapi.TaskDelete (*TaskRun)(nil), // 17: saasapi.TaskRun
(*TaskInfo)(nil), // 18: saasapi.TaskInfo (*TaskDelete)(nil), // 18: saasapi.TaskDelete
(*FileInfo)(nil), // 19: saasapi.FileInfo (*TaskInfo)(nil), // 19: saasapi.TaskInfo
(*FileBlock)(nil), // 20: saasapi.FileBlock (*FileInfo)(nil), // 20: saasapi.FileInfo
(*SaasRes)(nil), // 21: saasapi.SaasRes (*FileBlock)(nil), // 21: saasapi.FileBlock
(*ReadRes)(nil), // 22: saasapi.ReadRes (*SaasRes)(nil), // 22: saasapi.SaasRes
(*WriteRes)(nil), // 23: saasapi.WriteRes (*ReadRes)(nil), // 23: saasapi.ReadRes
(*ValueItem)(nil), // 24: saasapi.ValueItem (*WriteRes)(nil), // 24: saasapi.WriteRes
(*TaskListRes)(nil), // 25: saasapi.TaskListRes (*ValueItem)(nil), // 25: saasapi.ValueItem
(*TaskListRes)(nil), // 26: saasapi.TaskListRes
} }
var file_cmd_proto_depIdxs = []int32{ var file_cmd_proto_depIdxs = []int32{
0, // 0: saasapi.SaasReq.userid_type:type_name -> saasapi.UserIdType 0, // 0: saasapi.SaasReq.userid_type:type_name -> saasapi.UserIdType
5, // 1: saasapi.SaasReq.read:type_name -> saasapi.Read 6, // 1: saasapi.SaasReq.read:type_name -> saasapi.Read
7, // 2: saasapi.SaasReq.write:type_name -> saasapi.Write 8, // 2: saasapi.SaasReq.write:type_name -> saasapi.Write
13, // 3: saasapi.SaasReq.column_write:type_name -> saasapi.ColumnWrite 14, // 3: saasapi.SaasReq.column_write:type_name -> saasapi.ColumnWrite
14, // 4: saasapi.SaasReq.task_create:type_name -> saasapi.Task 15, // 4: saasapi.SaasReq.task_create:type_name -> saasapi.Task
15, // 5: saasapi.SaasReq.task_list:type_name -> saasapi.TaskList 16, // 5: saasapi.SaasReq.task_list:type_name -> saasapi.TaskList
16, // 6: saasapi.SaasReq.task_run:type_name -> saasapi.TaskRun 17, // 6: saasapi.SaasReq.task_run:type_name -> saasapi.TaskRun
17, // 7: saasapi.SaasReq.task_delete:type_name -> saasapi.TaskDelete 18, // 7: saasapi.SaasReq.task_delete:type_name -> saasapi.TaskDelete
18, // 8: saasapi.SaasReq.task_info:type_name -> saasapi.TaskInfo 19, // 8: saasapi.SaasReq.task_info:type_name -> saasapi.TaskInfo
6, // 9: saasapi.Read.read_items:type_name -> saasapi.ReadItem 7, // 9: saasapi.Read.read_items:type_name -> saasapi.ReadItem
8, // 10: saasapi.Write.write_items:type_name -> saasapi.WriteItem 9, // 10: saasapi.Write.write_items:type_name -> saasapi.WriteItem
9, // 11: saasapi.WriteItem.write_bytes:type_name -> saasapi.Bytes 10, // 11: saasapi.WriteItem.write_bytes:type_name -> saasapi.Bytes
10, // 12: saasapi.WriteItem.write_uint32s:type_name -> saasapi.Uint32s 11, // 12: saasapi.WriteItem.write_uint32s:type_name -> saasapi.Uint32s
11, // 13: saasapi.WriteItem.write_flags_with_expire:type_name -> saasapi.FlagsWithExpire 12, // 13: saasapi.WriteItem.write_flags_with_expire:type_name -> saasapi.FlagsWithExpire
12, // 14: saasapi.FlagsWithExpire.flags_with_expire:type_name -> saasapi.FlagWithExpire 13, // 14: saasapi.FlagsWithExpire.flags_with_expire:type_name -> saasapi.FlagWithExpire
9, // 15: saasapi.ColumnWrite.write_bytes:type_name -> saasapi.Bytes 10, // 15: saasapi.ColumnWrite.write_bytes:type_name -> saasapi.Bytes
10, // 16: saasapi.ColumnWrite.write_uint32s:type_name -> saasapi.Uint32s 11, // 16: saasapi.ColumnWrite.write_uint32s:type_name -> saasapi.Uint32s
11, // 17: saasapi.ColumnWrite.write_flags_with_expire:type_name -> saasapi.FlagsWithExpire 12, // 17: saasapi.ColumnWrite.write_flags_with_expire:type_name -> saasapi.FlagsWithExpire
19, // 18: saasapi.Task.task_file_infos:type_name -> saasapi.FileInfo 20, // 18: saasapi.Task.task_file_infos:type_name -> saasapi.FileInfo
3, // 19: saasapi.Task.status:type_name -> saasapi.TaskStatus 4, // 19: saasapi.Task.status:type_name -> saasapi.TaskStatus
3, // 20: saasapi.TaskList.status_filter:type_name -> saasapi.TaskStatus 4, // 20: saasapi.TaskList.status_filter:type_name -> saasapi.TaskStatus
20, // 21: saasapi.FileInfo.file_blocks:type_name -> saasapi.FileBlock 21, // 21: saasapi.FileInfo.file_blocks:type_name -> saasapi.FileBlock
1, // 22: saasapi.SaasRes.code:type_name -> saasapi.ErrorCode 2, // 22: saasapi.SaasRes.code:type_name -> saasapi.ErrorCode
22, // 23: saasapi.SaasRes.read_res:type_name -> saasapi.ReadRes 23, // 23: saasapi.SaasRes.read_res:type_name -> saasapi.ReadRes
23, // 24: saasapi.SaasRes.write_res:type_name -> saasapi.WriteRes 24, // 24: saasapi.SaasRes.write_res:type_name -> saasapi.WriteRes
14, // 25: saasapi.SaasRes.task_create_res:type_name -> saasapi.Task 15, // 25: saasapi.SaasRes.task_create_res:type_name -> saasapi.Task
25, // 26: saasapi.SaasRes.task_list_res:type_name -> saasapi.TaskListRes 26, // 26: saasapi.SaasRes.task_list_res:type_name -> saasapi.TaskListRes
14, // 27: saasapi.SaasRes.task_run_res:type_name -> saasapi.Task 15, // 27: saasapi.SaasRes.task_run_res:type_name -> saasapi.Task
14, // 28: saasapi.SaasRes.task_delete_res:type_name -> saasapi.Task 15, // 28: saasapi.SaasRes.task_delete_res:type_name -> saasapi.Task
14, // 29: saasapi.SaasRes.task_info_res:type_name -> saasapi.Task 15, // 29: saasapi.SaasRes.task_info_res:type_name -> saasapi.Task
24, // 30: saasapi.ReadRes.cmd_res:type_name -> saasapi.ValueItem 25, // 30: saasapi.ReadRes.cmd_res:type_name -> saasapi.ValueItem
24, // 31: saasapi.WriteRes.cmd_res:type_name -> saasapi.ValueItem 25, // 31: saasapi.WriteRes.cmd_res:type_name -> saasapi.ValueItem
2, // 32: saasapi.ValueItem.cmd_code:type_name -> saasapi.CmdErrorCode 3, // 32: saasapi.ValueItem.cmd_code:type_name -> saasapi.CmdErrorCode
12, // 33: saasapi.ValueItem.flags_with_expire:type_name -> saasapi.FlagWithExpire 13, // 33: saasapi.ValueItem.flags_with_expire:type_name -> saasapi.FlagWithExpire
14, // 34: saasapi.TaskListRes.tasks:type_name -> saasapi.Task 15, // 34: saasapi.TaskListRes.tasks:type_name -> saasapi.Task
35, // [35:35] is the sub-list for method output_type 35, // [35:35] is the sub-list for method output_type
35, // [35:35] is the sub-list for method input_type 35, // [35:35] is the sub-list for method input_type
35, // [35:35] is the sub-list for extension type_name 35, // [35:35] is the sub-list for extension type_name
@@ -2077,7 +2137,7 @@ func file_cmd_proto_init() {
File: protoimpl.DescBuilder{ File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(), GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_cmd_proto_rawDesc), len(file_cmd_proto_rawDesc)), RawDescriptor: unsafe.Slice(unsafe.StringData(file_cmd_proto_rawDesc), len(file_cmd_proto_rawDesc)),
NumEnums: 4, NumEnums: 5,
NumMessages: 22, NumMessages: 22,
NumExtensions: 0, NumExtensions: 0,
NumServices: 0, NumServices: 0,

7
cmd.proto
View File

@@ -8,6 +8,7 @@ option go_package = "e.coding.net/rta/public/saasapi";
message SaasReq { message SaasReq {
UserIdType userid_type = 1; // 用户ID类型 UserIdType userid_type = 1; // 用户ID类型
string appid = 2; // 小程序/小游戏/公众号/视频号的appid string appid = 2; // 小程序/小游戏/公众号/视频号的appid
uint32 namespace_id = 3; // 命名空间ID
oneof cmd { oneof cmd {
Read read = 10; // 批量读取 Read read = 10; // 批量读取
@@ -80,6 +81,12 @@ enum UserIdType {
INNERID1 = 10; // 内部ID1 INNERID1 = 10; // 内部ID1
} }
// NameSpaceId 用户 ID 类型
enum NameSpaceId {
DID = 0; // 默认设备号命名空间
WUID = 1; // 默认WUID命名空间
}
// ColumnWrite 全量列式写入命令 // ColumnWrite 全量列式写入命令
message ColumnWrite { message ColumnWrite {
bool is_clear_all_first = 1; // 是否先执行清空 bool is_clear_all_first = 1; // 是否先执行清空

1
cmd/saastool/help.go
View File

@@ -20,7 +20,6 @@ Commands:
columnwrite Write columns for 'deviceid / openid' users columnwrite Write columns for 'deviceid / openid' users
convert Convert data to write format convert Convert data to write format
makehash Make file hash for upload task
task Task commands task Task commands

2
cmd/saastool/main.go
View File

@@ -29,8 +29,6 @@ func Run(args ...string) error {
return RunColumnWrite(args...) return RunColumnWrite(args...)
case "convert": case "convert":
return RunConvert(args...) return RunConvert(args...)
case "makehash":
return RunMakeHash(args...)
case "verify": case "verify":
return RunVerify(args...) return RunVerify(args...)
case "task": case "task":

31
cmd/saastool/params.go
View File

@@ -7,6 +7,9 @@ import (
"strings" "strings"
) )
// paramConfig 设置并返回配置文件路径的命令行参数。
// 该函数接收一个 flag.FlagSet 指针作为参数，用于注册 "-config" 标志，
// 默认值为 "cfg.toml"，返回存储配置文件路径的字符串指针。
func paramConfig(fs *flag.FlagSet) *string { func paramConfig(fs *flag.FlagSet) *string {
return fs.String("config", "cfg.toml", "Config file.") return fs.String("config", "cfg.toml", "Config file.")
} }
@@ -18,10 +21,30 @@ func paramSourcePath(fs *flag.FlagSet) *string {
return fs.String("source", "", "Source path or filename") return fs.String("source", "", "Source path or filename")
} }
func paramSourceConvertedPath(fs *flag.FlagSet) *string {
return fs.String("source", "", "Source path of the converted files")
}
func paramTaskDesc(fs *flag.FlagSet) *string {
return fs.String("desc", "", "Task description")
}
func paramDestPath(fs *flag.FlagSet) *string { func paramDestPath(fs *flag.FlagSet) *string {
return fs.String("dest", "", "Destination path or filename") return fs.String("dest", "", "Destination path or filename")
} }
func paramOutputHashFile(fs *flag.FlagSet) *string {
return fs.String("hashfile", "", "Output hash file")
}
func paramInputHashFile(fs *flag.FlagSet) *string {
return fs.String("hashfile", "", "Input hash file")
}
func paramFilterStatus(fs *flag.FlagSet) *string {
return fs.String("status", "", "Filter status. enums 'all', 'waiting', 'running', 'success', 'fail', 'deleted'")
}
func paramAppid(fs *flag.FlagSet) *string { func paramAppid(fs *flag.FlagSet) *string {
return fs.String("appid", "", "Wechat appid") return fs.String("appid", "", "Wechat appid")
} }
@@ -30,6 +53,10 @@ func paramUserids(fs *flag.FlagSet) *string {
return fs.String("userids", "", "Device ID or Wechat UserID, separated by comma") return fs.String("userids", "", "Device ID or Wechat UserID, separated by comma")
} }
func paramSha256(fs *flag.FlagSet) *string {
return fs.String("sha256", "", "Task SHA256 hash")
}
func paramBatchSize(fs *flag.FlagSet) *uint { func paramBatchSize(fs *flag.FlagSet) *uint {
return fs.Uint("batchsize", 10000, "Batch size to sync") return fs.Uint("batchsize", 10000, "Batch size to sync")
} }
@@ -46,10 +73,6 @@ func paramBlockSize(fs *flag.FlagSet) uint64 {
return num return num
} }
func paramAsync(fs *flag.FlagSet) *bool {
return fs.Bool("async", false, "Async mode")
}
func paramClear(fs *flag.FlagSet) *bool { func paramClear(fs *flag.FlagSet) *bool {
return fs.Bool("clear", false, "Clear all data before write") return fs.Bool("clear", false, "Clear all data before write")
} }

6
cmd/saastool/read.go
View File

@@ -53,9 +53,9 @@ func RunRead(args ...string) error {
userids: idsSlice, userids: idsSlice,
appid: *appid, appid: *appid,
saasHttp: &saashttp.SaasClient{ saasHttp: &saashttp.SaasClient{
Client: http.Client{}, Client: &http.Client{},
ApiUrls: cfg.ApiUrls, ApiUrls: &cfg.ApiUrls,
Auth: cfg.Auth, Auth: &cfg.Auth,
}, },
} }

13
cmd/saastool/task.go
View File

@@ -13,6 +13,8 @@ func RunTask(args ...string) error {
switch name { switch name {
case "", "help": case "", "help":
return RunTaskHelp(args...) return RunTaskHelp(args...)
case "make":
return RunTaskMake(args...)
case "create": case "create":
return RunTaskCreate(args...) return RunTaskCreate(args...)
case "list": case "list":
@@ -37,12 +39,13 @@ const taskUsage = `
Usage: saastoola task COMMAND [OPTIONS] Usage: saastoola task COMMAND [OPTIONS]
Commands: Commands:
create Create data file to task make Make file hash for upload task
create Create a task on server
list List tasks on server list List tasks on server
run Run task on server run Run a task on server
delete Delete task on server delete Delete a task on server
info Get task info on server info Get a task info on server
upload Upload file block to server upload Upload task's file block to server
"help" is the default command. "help" is the default command.

91
cmd/saastool/task_create.go
View File

@@ -1,5 +1,96 @@
package main package main
import (
"flag"
"fmt"
"net/http"
"os"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type createTaskParams struct {
hashFile string
appid string
task *saasapi.Task
saasHttp *saashttp.SaasClient
}
func RunTaskCreate(args ...string) error { func RunTaskCreate(args ...string) error {
fs := flag.NewFlagSet("create", flag.ExitOnError)
cfgFile := paramConfig(fs)
// sourcePath := paramSourceConvertedPath(fs)
hashFile := paramInputHashFile(fs)
appid := paramAppid(fs)
if err := fs.Parse(args); err != nil {
fmt.Println("command line parse error", "err", err)
return err
}
if fs.NArg() > 0 || len(*hashFile) == 0 {
fs.PrintDefaults()
return nil
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Println("LoadConfigFile error", "err", err)
return err
}
createTaskParams := createTaskParams{
hashFile: *hashFile,
appid: *appid,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
Auth: &cfg.Auth,
},
task: &saasapi.Task{},
}
taskBuf, err := os.ReadFile(createTaskParams.hashFile)
if err != nil {
fmt.Println("open task file error", "err", err)
return err
}
if err = protojson.Unmarshal(taskBuf, createTaskParams.task); err != nil {
fmt.Println("parse task file error", "err", err)
}
return doTaskCreate(createTaskParams)
}
func doTaskCreate(createTaskParams createTaskParams) error {
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_TaskCreate{
TaskCreate: createTaskParams.task,
},
}
if createTaskParams.appid != "" {
saasReq.UseridType = saasapi.UserIdType_OPENID
saasReq.Appid = createTaskParams.appid
}
res, err := createTaskParams.saasHttp.TaskCreate(saasReq)
if err != nil {
fmt.Println("submit Create Task error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Println("task create failed", "code", res.Code, "status", res.Status)
return nil
}
taskRes := res.GetTaskCreateRes()
fmt.Printf("task res: %v\n", protojson.Format(taskRes))
return nil return nil
} }

71
cmd/saastool/task_info.go
View File

@@ -1,5 +1,76 @@
package main package main
import (
"flag"
"fmt"
"net/http"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type infoTaskParams struct {
taskSha256 string
saasHttp *saashttp.SaasClient
}
func RunTaskInfo(args ...string) error { func RunTaskInfo(args ...string) error {
fs := flag.NewFlagSet("create", flag.ExitOnError)
cfgFile := paramConfig(fs)
sha256 := paramSha256(fs)
if err := fs.Parse(args); err != nil {
fmt.Println("command line parse error", "err", err)
return err
}
if fs.NArg() > 0 || len(*sha256) == 0 {
fs.PrintDefaults()
return nil
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Println("LoadConfigFile error", "err", err)
return err
}
infoTaskParams := infoTaskParams{
taskSha256: *sha256,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
Auth: &cfg.Auth,
},
}
return doTaskInfo(infoTaskParams)
}
func doTaskInfo(infoTaskParams infoTaskParams) error {
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_TaskInfo{
TaskInfo: &saasapi.TaskInfo{
TaskSha256: infoTaskParams.taskSha256,
},
},
}
res, err := infoTaskParams.saasHttp.TaskInfo(saasReq)
if err != nil {
fmt.Println("submit Task info error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Println("task info failed", "code", res.Code, "status", res.Status)
return nil
}
taskRes := res.GetTaskInfoRes()
fmt.Printf("task res: %v\n", protojson.Format(taskRes))
return nil return nil
} }

83
cmd/saastool/task_list.go
View File

@@ -1,5 +1,88 @@
package main package main
import (
"flag"
"fmt"
"net/http"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type listTaskParams struct {
filterStatus saasapi.TaskStatus
saasHttp *saashttp.SaasClient
}
func RunTaskList(args ...string) error { func RunTaskList(args ...string) error {
fs := flag.NewFlagSet("create", flag.ExitOnError)
cfgFile := paramConfig(fs)
filter := paramFilterStatus(fs)
if err := fs.Parse(args); err != nil {
fmt.Println("command line parse error", "err", err)
return err
}
if fs.NArg() > 0 {
fs.PrintDefaults()
return nil
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Println("LoadConfigFile error", "err", err)
return err
}
listTaskParams := listTaskParams{
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
Auth: &cfg.Auth,
},
}
switch *filter {
case "all":
listTaskParams.filterStatus = saasapi.TaskStatus_ALL
case "waiting":
listTaskParams.filterStatus = saasapi.TaskStatus_WAITING
case "running":
listTaskParams.filterStatus = saasapi.TaskStatus_RUNNING
case "success":
listTaskParams.filterStatus = saasapi.TaskStatus_SUCCESS
case "fail":
listTaskParams.filterStatus = saasapi.TaskStatus_FAIL
}
return doTaskList(listTaskParams)
}
func doTaskList(listTaskParams listTaskParams) error {
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_TaskList{
TaskList: &saasapi.TaskList{
StatusFilter: listTaskParams.filterStatus,
},
},
}
res, err := listTaskParams.saasHttp.TaskList(saasReq)
if err != nil {
fmt.Println("submit List Task error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Println("task list failed", "code", res.Code, "status", res.Status)
return nil
}
taskRes := res.GetTaskListRes()
fmt.Printf("task res: %v\n", protojson.Format(taskRes))
return nil return nil
} }

62
cmd/saastool/make_hash.go → cmd/saastool/task_make.go
View File

@@ -20,9 +20,9 @@ const (
blockSizeMax = 200 * 1024 * 1024 blockSizeMax = 200 * 1024 * 1024
) )
type makeHashParams struct { type makeTaskParams struct {
sourcePath string sourcePath string
destPath string hashFile string
task *saasapi.Task task *saasapi.Task
} }
@@ -39,18 +39,19 @@ type hashResult struct {
index int index int
} }
func RunMakeHash(args ...string) error { func RunTaskMake(args ...string) error {
fs := flag.NewFlagSet("tasklocalmake", flag.ExitOnError) fs := flag.NewFlagSet("make", flag.ExitOnError)
sourcePath := paramSourcePath(fs) sourcePath := paramSourceConvertedPath(fs)
destPath := paramDestPath(fs) hashFile := paramOutputHashFile(fs)
blockSize := paramBlockSize(fs) blockSize := paramBlockSize(fs)
desc := paramTaskDesc(fs)
if err := fs.Parse(args); err != nil { if err := fs.Parse(args); err != nil {
fmt.Println("command line parse error", "err", err) fmt.Println("command line parse error", "err", err)
return err return err
} }
if fs.NArg() > 0 || len(*sourcePath) == 0 || len(*destPath) == 0 { if fs.NArg() > 0 || len(*sourcePath) == 0 || len(*hashFile) == 0 {
fs.PrintDefaults() fs.PrintDefaults()
return nil return nil
} }
@@ -60,52 +61,49 @@ func RunMakeHash(args ...string) error {
return nil return nil
} }
makeHashParams := makeHashParams{ makeTaskParams := makeTaskParams{
sourcePath: *sourcePath, sourcePath: *sourcePath,
destPath: *destPath, hashFile: *hashFile,
task: &saasapi.Task{ task: &saasapi.Task{
TaskBlockSize: blockSize, TaskBlockSize: blockSize,
TaskDescription: *desc,
}, },
} }
return doMakeHash(makeHashParams) return doMakeHash(makeTaskParams)
} }
func doMakeHash(makeHashParams makeHashParams) error { func doMakeHash(makeTaskParams makeTaskParams) error {
fsInfo, err := os.Stat(makeHashParams.sourcePath) fsInfo, err := os.Stat(makeTaskParams.sourcePath)
if err != nil { if err != nil {
return err return err
} }
if !fsInfo.IsDir() { if !fsInfo.IsDir() {
// 如果是文件，直接计算 // 如果是文件，直接计算
return doFileHash(makeHashParams) return doTaskMake(makeTaskParams)
} }
// 读取目录下信息 // 读取目录下信息
dirEntry, err := os.ReadDir(makeHashParams.sourcePath) dirEntry, err := os.ReadDir(makeTaskParams.sourcePath)
if err != nil { if err != nil {
return err return err
} }
// 遍历目录 // 遍历目录
for _, dir := range dirEntry { for _, dir := range dirEntry {
newParam := makeHashParams newParam := makeTaskParams
newParam.sourcePath = path.Join(makeHashParams.sourcePath, dir.Name()) newParam.sourcePath = path.Join(makeTaskParams.sourcePath, dir.Name())
if dir.IsDir() {
newParam.destPath = path.Join(makeHashParams.destPath, dir.Name())
}
if err = doMakeHash(newParam); err != nil { if err = doMakeHash(newParam); err != nil {
return err return err
} }
} }
return saveTaskFile(makeHashParams) return saveTaskFile(makeTaskParams)
} }
func doFileHash(makeHashParams makeHashParams) error { func doTaskMake(makeTaskParams makeTaskParams) error {
sourceFile, err := os.Open(makeHashParams.sourcePath) sourceFile, err := os.Open(makeTaskParams.sourcePath)
if err != nil { if err != nil {
return err return err
} }
@@ -129,12 +127,12 @@ func doFileHash(makeHashParams makeHashParams) error {
wg.Add(1) wg.Add(1)
go func() { go func() {
index := 0 index := 0
buffer := make([]byte, makeHashParams.task.TaskBlockSize) buffer := make([]byte, makeTaskParams.task.TaskBlockSize)
for { for {
n, err := sourceFile.Read(buffer) n, err := sourceFile.Read(buffer)
if n > 0 { if n > 0 {
wg.Add(1) wg.Add(1)
fmt.Printf("\rhashing file [%v], block [%v]", makeHashParams.sourcePath, index) fmt.Printf("\rhashing file [%v], block [%v]", makeTaskParams.sourcePath, index)
tasks <- hashTask{chunk: buffer[:n], index: index} tasks <- hashTask{chunk: buffer[:n], index: index}
index++ index++
} }
@@ -164,7 +162,7 @@ func doFileHash(makeHashParams makeHashParams) error {
// 输出结果 // 输出结果
fileInfo := &saasapi.FileInfo{ fileInfo := &saasapi.FileInfo{
FileName: makeHashParams.sourcePath, FileName: makeTaskParams.sourcePath,
FileSize: uint64(fi.Size()), FileSize: uint64(fi.Size()),
} }
for _, r := range allResults { for _, r := range allResults {
@@ -173,7 +171,7 @@ func doFileHash(makeHashParams makeHashParams) error {
BlockLength: r.blockSize, BlockLength: r.blockSize,
}) })
} }
makeHashParams.task.TaskFileInfos = append(makeHashParams.task.TaskFileInfos, fileInfo) makeTaskParams.task.TaskFileInfos = append(makeTaskParams.task.TaskFileInfos, fileInfo)
fmt.Println("") fmt.Println("")
return nil return nil
@@ -189,8 +187,8 @@ func hashWorker(tasks <-chan hashTask, results chan<- hashResult) {
} }
} }
func saveTaskFile(makeHashParams makeHashParams) error { func saveTaskFile(makeTaskParams makeTaskParams) error {
taskFile, err := os.Create(makeHashParams.destPath) taskFile, err := os.Create(makeTaskParams.hashFile)
if err != nil { if err != nil {
return err return err
} }
@@ -198,15 +196,15 @@ func saveTaskFile(makeHashParams makeHashParams) error {
h := sha256.New() h := sha256.New()
for _, fileInfo := range makeHashParams.task.TaskFileInfos { for _, fileInfo := range makeTaskParams.task.TaskFileInfos {
for _, fileBlock := range fileInfo.FileBlocks { for _, fileBlock := range fileInfo.FileBlocks {
h.Write([]byte(fileBlock.BlockSha256)) h.Write([]byte(fileBlock.BlockSha256))
} }
} }
makeHashParams.task.TaskSha256 = hex.EncodeToString(h.Sum(nil)) makeTaskParams.task.TaskSha256 = hex.EncodeToString(h.Sum(nil))
_, err = taskFile.WriteString(protojson.Format(makeHashParams.task)) _, err = taskFile.WriteString(protojson.Format(makeTaskParams.task))
if err != nil { if err != nil {
return err return err
} }

6
cmd/saastool/write.go
View File

@@ -53,9 +53,9 @@ func RunWrite(args ...string) error {
batchSize: *batchSize, batchSize: *batchSize,
clear: *clear, clear: *clear,
saasHttp: &saashttp.SaasClient{ saasHttp: &saashttp.SaasClient{
Client: http.Client{}, Client: &http.Client{},
ApiUrls: cfg.ApiUrls, ApiUrls: &cfg.ApiUrls,
Auth: cfg.Auth, Auth: &cfg.Auth,
}, },
} }

7
pkg/saashttp/cfg.go
View File

@@ -5,9 +5,12 @@ type ApiUrls struct {
WritePath string WritePath string
ReadPath string ReadPath string
ColumnWritePath string ColumnWritePath string
TaskCreatePath string
TaskListPath string TaskListPath string
TaskCancelPath string TaskInfoPath string
TaskDetailPath string TaskDeletePath string
TaskRunPath string
TaskUploadPath string
} }
type Auth struct { type Auth struct {

23
pkg/saashttp/httpcli.go
View File

@@ -24,9 +24,9 @@ const (
) )
type SaasClient struct { type SaasClient struct {
Client http.Client Client *http.Client
ApiUrls ApiUrls ApiUrls *ApiUrls
Auth Auth Auth *Auth
ResponseEncoder ResponseEncoder ResponseEncoder ResponseEncoder
} }
@@ -45,19 +45,28 @@ func (c *SaasClient) ColumnWrite(saasReq *saasapi.SaasReq) (saasRes *saasapi.Saa
return c.post(postUrl, saasReq) return c.post(postUrl, saasReq)
} }
func (c *SaasClient) TaskCreate(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskCreatePath)
return c.post(postUrl, saasReq)
}
func (c *SaasClient) TaskList(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) { func (c *SaasClient) TaskList(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskListPath) postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskListPath)
return c.post(postUrl, saasReq) return c.post(postUrl, saasReq)
} }
func (c *SaasClient) TaskCancel(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) { func (c *SaasClient) TaskDelete(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskCancelPath) postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskDeletePath)
return c.post(postUrl, saasReq) return c.post(postUrl, saasReq)
} }
func (c *SaasClient) TaskDetail(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) { func (c *SaasClient) TaskInfo(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskInfoPath)
return c.post(postUrl, saasReq)
}
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskDetailPath) func (c *SaasClient) TaskRun(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskRunPath)
return c.post(postUrl, saasReq) return c.post(postUrl, saasReq)
} }