public/saasapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tasklist taskinfo

Browse Source
This commit is contained in:
algotao
2025-04-12 21:53:56 +08:00
parent 4fc2aeb9c4
commit 8da742fea1
14 changed files with 484 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

214
cmd.pb.go
View File

@@ -71,6 +71,53 @@ func (UserIdType) EnumDescriptor() ([]byte, []int) {
return file_cmd_proto_rawDescGZIP(), []int{0} return file_cmd_proto_rawDescGZIP(), []int{0}
} }
// NameSpaceId 用户 ID 类型
type NameSpaceId int32
const (
NameSpaceId_DID NameSpaceId = 0 // 默认设备号命名空间
NameSpaceId_WUID NameSpaceId = 1 // 默认WUID命名空间
)
// Enum value maps for NameSpaceId.
var (
NameSpaceId_name = map[int32]string{
0: "DID",
1: "WUID",
}
NameSpaceId_value = map[string]int32{
"DID": 0,
"WUID": 1,
}
)
func (x NameSpaceId) Enum() *NameSpaceId {
p := new(NameSpaceId)
*p = x
return p
}
func (x NameSpaceId) String() string {
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
}
func (NameSpaceId) Descriptor() protoreflect.EnumDescriptor {
return file_cmd_proto_enumTypes[1].Descriptor()
}
func (NameSpaceId) Type() protoreflect.EnumType {
return &file_cmd_proto_enumTypes[1]
}
func (x NameSpaceId) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
// Deprecated: Use NameSpaceId.Descriptor instead.
func (NameSpaceId) EnumDescriptor() ([]byte, []int) {
return file_cmd_proto_rawDescGZIP(), []int{1}
}
// ErrorCode 返回码 // ErrorCode 返回码
type ErrorCode int32 type ErrorCode int32
@@ -153,11 +200,11 @@ func (x ErrorCode) String() string {
} }
func (ErrorCode) Descriptor() protoreflect.EnumDescriptor { func (ErrorCode) Descriptor() protoreflect.EnumDescriptor {
return file_cmd_proto_enumTypes[1].Descriptor() return file_cmd_proto_enumTypes[2].Descriptor()
} }
func (ErrorCode) Type() protoreflect.EnumType { func (ErrorCode) Type() protoreflect.EnumType {
return &file_cmd_proto_enumTypes[1] return &file_cmd_proto_enumTypes[2]
} }
func (x ErrorCode) Number() protoreflect.EnumNumber { func (x ErrorCode) Number() protoreflect.EnumNumber {
@@ -166,7 +213,7 @@ func (x ErrorCode) Number() protoreflect.EnumNumber {
// Deprecated: Use ErrorCode.Descriptor instead. // Deprecated: Use ErrorCode.Descriptor instead.
func (ErrorCode) EnumDescriptor() ([]byte, []int) { func (ErrorCode) EnumDescriptor() ([]byte, []int) {
return file_cmd_proto_rawDescGZIP(), []int{1} return file_cmd_proto_rawDescGZIP(), []int{2}
} }
type CmdErrorCode int32 type CmdErrorCode int32
@@ -196,11 +243,11 @@ func (x CmdErrorCode) String() string {
} }
func (CmdErrorCode) Descriptor() protoreflect.EnumDescriptor { func (CmdErrorCode) Descriptor() protoreflect.EnumDescriptor {
return file_cmd_proto_enumTypes[2].Descriptor() return file_cmd_proto_enumTypes[3].Descriptor()
} }
func (CmdErrorCode) Type() protoreflect.EnumType { func (CmdErrorCode) Type() protoreflect.EnumType {
return &file_cmd_proto_enumTypes[2] return &file_cmd_proto_enumTypes[3]
} }
func (x CmdErrorCode) Number() protoreflect.EnumNumber { func (x CmdErrorCode) Number() protoreflect.EnumNumber {
@@ -209,7 +256,7 @@ func (x CmdErrorCode) Number() protoreflect.EnumNumber {
// Deprecated: Use CmdErrorCode.Descriptor instead. // Deprecated: Use CmdErrorCode.Descriptor instead.
func (CmdErrorCode) EnumDescriptor() ([]byte, []int) { func (CmdErrorCode) EnumDescriptor() ([]byte, []int) {
return file_cmd_proto_rawDescGZIP(), []int{2} return file_cmd_proto_rawDescGZIP(), []int{3}
} }
type TaskStatus int32 type TaskStatus int32
@@ -254,11 +301,11 @@ func (x TaskStatus) String() string {
} }
func (TaskStatus) Descriptor() protoreflect.EnumDescriptor { func (TaskStatus) Descriptor() protoreflect.EnumDescriptor {
return file_cmd_proto_enumTypes[3].Descriptor() return file_cmd_proto_enumTypes[4].Descriptor()
} }
func (TaskStatus) Type() protoreflect.EnumType { func (TaskStatus) Type() protoreflect.EnumType {
return &file_cmd_proto_enumTypes[3] return &file_cmd_proto_enumTypes[4]
} }
func (x TaskStatus) Number() protoreflect.EnumNumber { func (x TaskStatus) Number() protoreflect.EnumNumber {
@@ -267,14 +314,15 @@ func (x TaskStatus) Number() protoreflect.EnumNumber {
// Deprecated: Use TaskStatus.Descriptor instead. // Deprecated: Use TaskStatus.Descriptor instead.
func (TaskStatus) EnumDescriptor() ([]byte, []int) { func (TaskStatus) EnumDescriptor() ([]byte, []int) {
return file_cmd_proto_rawDescGZIP(), []int{3} return file_cmd_proto_rawDescGZIP(), []int{4}
} }
// SaasReq 命令请求 // SaasReq 命令请求
type SaasReq struct { type SaasReq struct {
state protoimpl.MessageState `protogen:"open.v1"` state protoimpl.MessageState `protogen:"open.v1"`
UseridType UserIdType `protobuf:"varint,1,opt,name=userid_type,json=useridType,proto3,enum=saasapi.UserIdType" json:"userid_type,omitempty"` // 用户ID类型 UseridType UserIdType `protobuf:"varint,1,opt,name=userid_type,json=useridType,proto3,enum=saasapi.UserIdType" json:"userid_type,omitempty"` // 用户ID类型
Appid string `protobuf:"bytes,2,opt,name=appid,proto3" json:"appid,omitempty"` // 小程序/小游戏/公众号/视频号的appid Appid string `protobuf:"bytes,2,opt,name=appid,proto3" json:"appid,omitempty"` // 小程序/小游戏/公众号/视频号的appid
NamespaceId uint32 `protobuf:"varint,3,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` // 命名空间ID
// Types that are valid to be assigned to Cmd: // Types that are valid to be assigned to Cmd:
// //
// *SaasReq_Read // *SaasReq_Read
@@ -334,6 +382,13 @@ func (x *SaasReq) GetAppid() string {
return "" return ""
} }
func (x *SaasReq) GetNamespaceId() uint32 {
if x != nil {
return x.NamespaceId
}
return 0
}
func (x *SaasReq) GetCmd() isSaasReq_Cmd { func (x *SaasReq) GetCmd() isSaasReq_Cmd {
if x != nil { if x != nil {
return x.Cmd return x.Cmd
@@ -1319,7 +1374,7 @@ type FileBlock struct {
state protoimpl.MessageState `protogen:"open.v1"` state protoimpl.MessageState `protogen:"open.v1"`
BlockSha256 string `protobuf:"bytes,1,opt,name=block_sha256,json=blockSha256,proto3" json:"block_sha256,omitempty"` // 块的sha256 BlockSha256 string `protobuf:"bytes,1,opt,name=block_sha256,json=blockSha256,proto3" json:"block_sha256,omitempty"` // 块的sha256
BlockLength uint64 `protobuf:"varint,2,opt,name=block_length,json=blockLength,proto3" json:"block_length,omitempty"` // 块的字节长度 BlockLength uint64 `protobuf:"varint,2,opt,name=block_length,json=blockLength,proto3" json:"block_length,omitempty"` // 块的字节长度
Uploaded bool `protobuf:"varint,3,opt,name=uploaded,proto3" json:"uploaded,omitempty"` // 是否已上传（在TaskInfo请求返回） Uploaded bool `protobuf:"varint,3,opt,name=uploaded,proto3" json:"uploaded,omitempty"` // 是否已上传（在TaskCreate/TaskInfo请求返回）
unknownFields protoimpl.UnknownFields unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
} }
@@ -1807,11 +1862,12 @@ var File_cmd_proto protoreflect.FileDescriptor
const file_cmd_proto_rawDesc = "" + const file_cmd_proto_rawDesc = "" +
"\n" + "\n" +
"\tcmd.proto\x12\asaasapi\"\xe1\x03\n" + "\tcmd.proto\x12\asaasapi\"\x84\x04\n" +
"\aSaasReq\x124\n" + "\aSaasReq\x124\n" +
"\vuserid_type\x18\x01 \x01(\x0e2\x13.saasapi.UserIdTypeR\n" + "\vuserid_type\x18\x01 \x01(\x0e2\x13.saasapi.UserIdTypeR\n" +
"useridType\x12\x14\n" + "useridType\x12\x14\n" +
"\x05appid\x18\x02 \x01(\tR\x05appid\x12#\n" + "\x05appid\x18\x02 \x01(\tR\x05appid\x12!\n" +
"\fnamespace_id\x18\x03 \x01(\rR\vnamespaceId\x12#\n" +
"\x04read\x18\n" + "\x04read\x18\n" +
" \x01(\v2\r.saasapi.ReadH\x00R\x04read\x12&\n" + " \x01(\v2\r.saasapi.ReadH\x00R\x04read\x12&\n" +
"\x05write\x18\v \x01(\v2\x0e.saasapi.WriteH\x00R\x05write\x129\n" + "\x05write\x18\v \x01(\v2\x0e.saasapi.WriteH\x00R\x05write\x129\n" +
@@ -1929,7 +1985,10 @@ const file_cmd_proto_rawDesc = "" +
"\n" + "\n" +
"\x06OPENID\x10\x01\x12\f\n" + "\x06OPENID\x10\x01\x12\f\n" +
"\bINNERID1\x10\n" + "\bINNERID1\x10\n" +
"*\xef\x02\n" + "* \n" +
"\vNameSpaceId\x12\a\n" +
"\x03DID\x10\x00\x12\b\n" +
"\x04WUID\x10\x01*\xef\x02\n" +
"\tErrorCode\x12\b\n" + "\tErrorCode\x12\b\n" +
"\x04SUCC\x10\x00\x12\x13\n" + "\x04SUCC\x10\x00\x12\x13\n" +
"\x0fINVALID_ACCOUNT\x10e\x12\x15\n" + "\x0fINVALID_ACCOUNT\x10e\x12\x15\n" +
@@ -1975,72 +2034,73 @@ func file_cmd_proto_rawDescGZIP() []byte {
return file_cmd_proto_rawDescData return file_cmd_proto_rawDescData
} }
var file_cmd_proto_enumTypes = make([]protoimpl.EnumInfo, 4) var file_cmd_proto_enumTypes = make([]protoimpl.EnumInfo, 5)
var file_cmd_proto_msgTypes = make([]protoimpl.MessageInfo, 22) var file_cmd_proto_msgTypes = make([]protoimpl.MessageInfo, 22)
var file_cmd_proto_goTypes = []any{ var file_cmd_proto_goTypes = []any{
(UserIdType)(0), // 0: saasapi.UserIdType (UserIdType)(0), // 0: saasapi.UserIdType
(ErrorCode)(0), // 1: saasapi.ErrorCode (NameSpaceId)(0), // 1: saasapi.NameSpaceId
(CmdErrorCode)(0), // 2: saasapi.CmdErrorCode (ErrorCode)(0), // 2: saasapi.ErrorCode
(TaskStatus)(0), // 3: saasapi.TaskStatus (CmdErrorCode)(0), // 3: saasapi.CmdErrorCode
(*SaasReq)(nil), // 4: saasapi.SaasReq (TaskStatus)(0), // 4: saasapi.TaskStatus
(*Read)(nil), // 5: saasapi.Read (*SaasReq)(nil), // 5: saasapi.SaasReq
(*ReadItem)(nil), // 6: saasapi.ReadItem (*Read)(nil), // 6: saasapi.Read
(*Write)(nil), // 7: saasapi.Write (*ReadItem)(nil), // 7: saasapi.ReadItem
(*WriteItem)(nil), // 8: saasapi.WriteItem (*Write)(nil), // 8: saasapi.Write
(*Bytes)(nil), // 9: saasapi.Bytes (*WriteItem)(nil), // 9: saasapi.WriteItem
(*Uint32S)(nil), // 10: saasapi.Uint32s (*Bytes)(nil), // 10: saasapi.Bytes
(*FlagsWithExpire)(nil), // 11: saasapi.FlagsWithExpire (*Uint32S)(nil), // 11: saasapi.Uint32s
(*FlagWithExpire)(nil), // 12: saasapi.FlagWithExpire (*FlagsWithExpire)(nil), // 12: saasapi.FlagsWithExpire
(*ColumnWrite)(nil), // 13: saasapi.ColumnWrite (*FlagWithExpire)(nil), // 13: saasapi.FlagWithExpire
(*Task)(nil), // 14: saasapi.Task (*ColumnWrite)(nil), // 14: saasapi.ColumnWrite
(*TaskList)(nil), // 15: saasapi.TaskList (*Task)(nil), // 15: saasapi.Task
(*TaskRun)(nil), // 16: saasapi.TaskRun (*TaskList)(nil), // 16: saasapi.TaskList
(*TaskDelete)(nil), // 17: saasapi.TaskDelete (*TaskRun)(nil), // 17: saasapi.TaskRun
(*TaskInfo)(nil), // 18: saasapi.TaskInfo (*TaskDelete)(nil), // 18: saasapi.TaskDelete
(*FileInfo)(nil), // 19: saasapi.FileInfo (*TaskInfo)(nil), // 19: saasapi.TaskInfo
(*FileBlock)(nil), // 20: saasapi.FileBlock (*FileInfo)(nil), // 20: saasapi.FileInfo
(*SaasRes)(nil), // 21: saasapi.SaasRes (*FileBlock)(nil), // 21: saasapi.FileBlock
(*ReadRes)(nil), // 22: saasapi.ReadRes (*SaasRes)(nil), // 22: saasapi.SaasRes
(*WriteRes)(nil), // 23: saasapi.WriteRes (*ReadRes)(nil), // 23: saasapi.ReadRes
(*ValueItem)(nil), // 24: saasapi.ValueItem (*WriteRes)(nil), // 24: saasapi.WriteRes
(*TaskListRes)(nil), // 25: saasapi.TaskListRes (*ValueItem)(nil), // 25: saasapi.ValueItem
(*TaskListRes)(nil), // 26: saasapi.TaskListRes
} }
var file_cmd_proto_depIdxs = []int32{ var file_cmd_proto_depIdxs = []int32{
0, // 0: saasapi.SaasReq.userid_type:type_name -> saasapi.UserIdType 0, // 0: saasapi.SaasReq.userid_type:type_name -> saasapi.UserIdType
5, // 1: saasapi.SaasReq.read:type_name -> saasapi.Read 6, // 1: saasapi.SaasReq.read:type_name -> saasapi.Read
7, // 2: saasapi.SaasReq.write:type_name -> saasapi.Write 8, // 2: saasapi.SaasReq.write:type_name -> saasapi.Write
13, // 3: saasapi.SaasReq.column_write:type_name -> saasapi.ColumnWrite 14, // 3: saasapi.SaasReq.column_write:type_name -> saasapi.ColumnWrite
14, // 4: saasapi.SaasReq.task_create:type_name -> saasapi.Task 15, // 4: saasapi.SaasReq.task_create:type_name -> saasapi.Task
15, // 5: saasapi.SaasReq.task_list:type_name -> saasapi.TaskList 16, // 5: saasapi.SaasReq.task_list:type_name -> saasapi.TaskList
16, // 6: saasapi.SaasReq.task_run:type_name -> saasapi.TaskRun 17, // 6: saasapi.SaasReq.task_run:type_name -> saasapi.TaskRun
17, // 7: saasapi.SaasReq.task_delete:type_name -> saasapi.TaskDelete 18, // 7: saasapi.SaasReq.task_delete:type_name -> saasapi.TaskDelete
18, // 8: saasapi.SaasReq.task_info:type_name -> saasapi.TaskInfo 19, // 8: saasapi.SaasReq.task_info:type_name -> saasapi.TaskInfo
6, // 9: saasapi.Read.read_items:type_name -> saasapi.ReadItem 7, // 9: saasapi.Read.read_items:type_name -> saasapi.ReadItem
8, // 10: saasapi.Write.write_items:type_name -> saasapi.WriteItem 9, // 10: saasapi.Write.write_items:type_name -> saasapi.WriteItem
9, // 11: saasapi.WriteItem.write_bytes:type_name -> saasapi.Bytes 10, // 11: saasapi.WriteItem.write_bytes:type_name -> saasapi.Bytes
10, // 12: saasapi.WriteItem.write_uint32s:type_name -> saasapi.Uint32s 11, // 12: saasapi.WriteItem.write_uint32s:type_name -> saasapi.Uint32s
11, // 13: saasapi.WriteItem.write_flags_with_expire:type_name -> saasapi.FlagsWithExpire 12, // 13: saasapi.WriteItem.write_flags_with_expire:type_name -> saasapi.FlagsWithExpire
12, // 14: saasapi.FlagsWithExpire.flags_with_expire:type_name -> saasapi.FlagWithExpire 13, // 14: saasapi.FlagsWithExpire.flags_with_expire:type_name -> saasapi.FlagWithExpire
9, // 15: saasapi.ColumnWrite.write_bytes:type_name -> saasapi.Bytes 10, // 15: saasapi.ColumnWrite.write_bytes:type_name -> saasapi.Bytes
10, // 16: saasapi.ColumnWrite.write_uint32s:type_name -> saasapi.Uint32s 11, // 16: saasapi.ColumnWrite.write_uint32s:type_name -> saasapi.Uint32s
11, // 17: saasapi.ColumnWrite.write_flags_with_expire:type_name -> saasapi.FlagsWithExpire 12, // 17: saasapi.ColumnWrite.write_flags_with_expire:type_name -> saasapi.FlagsWithExpire
19, // 18: saasapi.Task.task_file_infos:type_name -> saasapi.FileInfo 20, // 18: saasapi.Task.task_file_infos:type_name -> saasapi.FileInfo
3, // 19: saasapi.Task.status:type_name -> saasapi.TaskStatus 4, // 19: saasapi.Task.status:type_name -> saasapi.TaskStatus
3, // 20: saasapi.TaskList.status_filter:type_name -> saasapi.TaskStatus 4, // 20: saasapi.TaskList.status_filter:type_name -> saasapi.TaskStatus
20, // 21: saasapi.FileInfo.file_blocks:type_name -> saasapi.FileBlock 21, // 21: saasapi.FileInfo.file_blocks:type_name -> saasapi.FileBlock
1, // 22: saasapi.SaasRes.code:type_name -> saasapi.ErrorCode 2, // 22: saasapi.SaasRes.code:type_name -> saasapi.ErrorCode
22, // 23: saasapi.SaasRes.read_res:type_name -> saasapi.ReadRes 23, // 23: saasapi.SaasRes.read_res:type_name -> saasapi.ReadRes
23, // 24: saasapi.SaasRes.write_res:type_name -> saasapi.WriteRes 24, // 24: saasapi.SaasRes.write_res:type_name -> saasapi.WriteRes
14, // 25: saasapi.SaasRes.task_create_res:type_name -> saasapi.Task 15, // 25: saasapi.SaasRes.task_create_res:type_name -> saasapi.Task
25, // 26: saasapi.SaasRes.task_list_res:type_name -> saasapi.TaskListRes 26, // 26: saasapi.SaasRes.task_list_res:type_name -> saasapi.TaskListRes
14, // 27: saasapi.SaasRes.task_run_res:type_name -> saasapi.Task 15, // 27: saasapi.SaasRes.task_run_res:type_name -> saasapi.Task
14, // 28: saasapi.SaasRes.task_delete_res:type_name -> saasapi.Task 15, // 28: saasapi.SaasRes.task_delete_res:type_name -> saasapi.Task
14, // 29: saasapi.SaasRes.task_info_res:type_name -> saasapi.Task 15, // 29: saasapi.SaasRes.task_info_res:type_name -> saasapi.Task
24, // 30: saasapi.ReadRes.cmd_res:type_name -> saasapi.ValueItem 25, // 30: saasapi.ReadRes.cmd_res:type_name -> saasapi.ValueItem
24, // 31: saasapi.WriteRes.cmd_res:type_name -> saasapi.ValueItem 25, // 31: saasapi.WriteRes.cmd_res:type_name -> saasapi.ValueItem
2, // 32: saasapi.ValueItem.cmd_code:type_name -> saasapi.CmdErrorCode 3, // 32: saasapi.ValueItem.cmd_code:type_name -> saasapi.CmdErrorCode
12, // 33: saasapi.ValueItem.flags_with_expire:type_name -> saasapi.FlagWithExpire 13, // 33: saasapi.ValueItem.flags_with_expire:type_name -> saasapi.FlagWithExpire
14, // 34: saasapi.TaskListRes.tasks:type_name -> saasapi.Task 15, // 34: saasapi.TaskListRes.tasks:type_name -> saasapi.Task
35, // [35:35] is the sub-list for method output_type 35, // [35:35] is the sub-list for method output_type
35, // [35:35] is the sub-list for method input_type 35, // [35:35] is the sub-list for method input_type
35, // [35:35] is the sub-list for extension type_name 35, // [35:35] is the sub-list for extension type_name
@@ -2077,7 +2137,7 @@ func file_cmd_proto_init() {
File: protoimpl.DescBuilder{ File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(), GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_cmd_proto_rawDesc), len(file_cmd_proto_rawDesc)), RawDescriptor: unsafe.Slice(unsafe.StringData(file_cmd_proto_rawDesc), len(file_cmd_proto_rawDesc)),
NumEnums: 4, NumEnums: 5,
NumMessages: 22, NumMessages: 22,
NumExtensions: 0, NumExtensions: 0,
NumServices: 0, NumServices: 0,

9
cmd.proto
View File

@@ -8,7 +8,8 @@ option go_package = "e.coding.net/rta/public/saasapi";
message SaasReq { message SaasReq {
UserIdType userid_type = 1; // 用户ID类型 UserIdType userid_type = 1; // 用户ID类型
string appid = 2; // 小程序/小游戏/公众号/视频号的appid string appid = 2; // 小程序/小游戏/公众号/视频号的appid
uint32 namespace_id = 3; // 命名空间ID
oneof cmd { oneof cmd {
Read read = 10; // 批量读取 Read read = 10; // 批量读取
Write write = 11; // 批量写入 Write write = 11; // 批量写入
@@ -80,6 +81,12 @@ enum UserIdType {
INNERID1 = 10; // 内部ID1 INNERID1 = 10; // 内部ID1
} }
// NameSpaceId 用户 ID 类型
enum NameSpaceId {
DID = 0; // 默认设备号命名空间
WUID = 1; // 默认WUID命名空间
}
// ColumnWrite 全量列式写入命令 // ColumnWrite 全量列式写入命令
message ColumnWrite { message ColumnWrite {
bool is_clear_all_first = 1; // 是否先执行清空 bool is_clear_all_first = 1; // 是否先执行清空

3
cmd/saastool/help.go
View File

@@ -20,8 +20,7 @@ Commands:
columnwrite Write columns for 'deviceid / openid' users columnwrite Write columns for 'deviceid / openid' users
convert Convert data to write format convert Convert data to write format
makehash Make file hash for upload task
task Task commands task Task commands
"help" is the default command. "help" is the default command.

2
cmd/saastool/main.go
View File

@@ -29,8 +29,6 @@ func Run(args ...string) error {
return RunColumnWrite(args...) return RunColumnWrite(args...)
case "convert": case "convert":
return RunConvert(args...) return RunConvert(args...)
case "makehash":
return RunMakeHash(args...)
case "verify": case "verify":
return RunVerify(args...) return RunVerify(args...)
case "task": case "task":

31
cmd/saastool/params.go
View File

@@ -7,6 +7,9 @@ import (
"strings" "strings"
) )
// paramConfig 设置并返回配置文件路径的命令行参数。
// 该函数接收一个 flag.FlagSet 指针作为参数，用于注册 "-config" 标志，
// 默认值为 "cfg.toml"，返回存储配置文件路径的字符串指针。
func paramConfig(fs *flag.FlagSet) *string { func paramConfig(fs *flag.FlagSet) *string {
return fs.String("config", "cfg.toml", "Config file.") return fs.String("config", "cfg.toml", "Config file.")
} }
@@ -18,10 +21,30 @@ func paramSourcePath(fs *flag.FlagSet) *string {
return fs.String("source", "", "Source path or filename") return fs.String("source", "", "Source path or filename")
} }
func paramSourceConvertedPath(fs *flag.FlagSet) *string {
return fs.String("source", "", "Source path of the converted files")
}
func paramTaskDesc(fs *flag.FlagSet) *string {
return fs.String("desc", "", "Task description")
}
func paramDestPath(fs *flag.FlagSet) *string { func paramDestPath(fs *flag.FlagSet) *string {
return fs.String("dest", "", "Destination path or filename") return fs.String("dest", "", "Destination path or filename")
} }
func paramOutputHashFile(fs *flag.FlagSet) *string {
return fs.String("hashfile", "", "Output hash file")
}
func paramInputHashFile(fs *flag.FlagSet) *string {
return fs.String("hashfile", "", "Input hash file")
}
func paramFilterStatus(fs *flag.FlagSet) *string {
return fs.String("status", "", "Filter status. enums 'all', 'waiting', 'running', 'success', 'fail', 'deleted'")
}
func paramAppid(fs *flag.FlagSet) *string { func paramAppid(fs *flag.FlagSet) *string {
return fs.String("appid", "", "Wechat appid") return fs.String("appid", "", "Wechat appid")
} }
@@ -30,6 +53,10 @@ func paramUserids(fs *flag.FlagSet) *string {
return fs.String("userids", "", "Device ID or Wechat UserID, separated by comma") return fs.String("userids", "", "Device ID or Wechat UserID, separated by comma")
} }
func paramSha256(fs *flag.FlagSet) *string {
return fs.String("sha256", "", "Task SHA256 hash")
}
func paramBatchSize(fs *flag.FlagSet) *uint { func paramBatchSize(fs *flag.FlagSet) *uint {
return fs.Uint("batchsize", 10000, "Batch size to sync") return fs.Uint("batchsize", 10000, "Batch size to sync")
} }
@@ -46,10 +73,6 @@ func paramBlockSize(fs *flag.FlagSet) uint64 {
return num return num
} }
func paramAsync(fs *flag.FlagSet) *bool {
return fs.Bool("async", false, "Async mode")
}
func paramClear(fs *flag.FlagSet) *bool { func paramClear(fs *flag.FlagSet) *bool {
return fs.Bool("clear", false, "Clear all data before write") return fs.Bool("clear", false, "Clear all data before write")
} }

6
cmd/saastool/read.go
View File

@@ -53,9 +53,9 @@ func RunRead(args ...string) error {
userids: idsSlice, userids: idsSlice,
appid: *appid, appid: *appid,
saasHttp: &saashttp.SaasClient{ saasHttp: &saashttp.SaasClient{
Client: http.Client{}, Client: &http.Client{},
ApiUrls: cfg.ApiUrls, ApiUrls: &cfg.ApiUrls,
Auth: cfg.Auth, Auth: &cfg.Auth,
}, },
} }

13
cmd/saastool/task.go
View File

@@ -13,6 +13,8 @@ func RunTask(args ...string) error {
switch name { switch name {
case "", "help": case "", "help":
return RunTaskHelp(args...) return RunTaskHelp(args...)
case "make":
return RunTaskMake(args...)
case "create": case "create":
return RunTaskCreate(args...) return RunTaskCreate(args...)
case "list": case "list":
@@ -37,12 +39,13 @@ const taskUsage = `
Usage: saastoola task COMMAND [OPTIONS] Usage: saastoola task COMMAND [OPTIONS]
Commands: Commands:
create Create data file to task make Make file hash for upload task
create Create a task on server
list List tasks on server list List tasks on server
run Run task on server run Run a task on server
delete Delete task on server delete Delete a task on server
info Get task info on server info Get a task info on server
upload Upload file block to server upload Upload task's file block to server
"help" is the default command. "help" is the default command.

91
cmd/saastool/task_create.go
View File

@@ -1,5 +1,96 @@
package main package main
import (
"flag"
"fmt"
"net/http"
"os"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type createTaskParams struct {
hashFile string
appid string
task *saasapi.Task
saasHttp *saashttp.SaasClient
}
func RunTaskCreate(args ...string) error { func RunTaskCreate(args ...string) error {
fs := flag.NewFlagSet("create", flag.ExitOnError)
cfgFile := paramConfig(fs)
// sourcePath := paramSourceConvertedPath(fs)
hashFile := paramInputHashFile(fs)
appid := paramAppid(fs)
if err := fs.Parse(args); err != nil {
fmt.Println("command line parse error", "err", err)
return err
}
if fs.NArg() > 0 || len(*hashFile) == 0 {
fs.PrintDefaults()
return nil
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Println("LoadConfigFile error", "err", err)
return err
}
createTaskParams := createTaskParams{
hashFile: *hashFile,
appid: *appid,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
Auth: &cfg.Auth,
},
task: &saasapi.Task{},
}
taskBuf, err := os.ReadFile(createTaskParams.hashFile)
if err != nil {
fmt.Println("open task file error", "err", err)
return err
}
if err = protojson.Unmarshal(taskBuf, createTaskParams.task); err != nil {
fmt.Println("parse task file error", "err", err)
}
return doTaskCreate(createTaskParams)
}
func doTaskCreate(createTaskParams createTaskParams) error {
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_TaskCreate{
TaskCreate: createTaskParams.task,
},
}
if createTaskParams.appid != "" {
saasReq.UseridType = saasapi.UserIdType_OPENID
saasReq.Appid = createTaskParams.appid
}
res, err := createTaskParams.saasHttp.TaskCreate(saasReq)
if err != nil {
fmt.Println("submit Create Task error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Println("task create failed", "code", res.Code, "status", res.Status)
return nil
}
taskRes := res.GetTaskCreateRes()
fmt.Printf("task res: %v\n", protojson.Format(taskRes))
return nil return nil
} }

71
cmd/saastool/task_info.go
View File

@@ -1,5 +1,76 @@
package main package main
import (
"flag"
"fmt"
"net/http"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type infoTaskParams struct {
taskSha256 string
saasHttp *saashttp.SaasClient
}
func RunTaskInfo(args ...string) error { func RunTaskInfo(args ...string) error {
fs := flag.NewFlagSet("create", flag.ExitOnError)
cfgFile := paramConfig(fs)
sha256 := paramSha256(fs)
if err := fs.Parse(args); err != nil {
fmt.Println("command line parse error", "err", err)
return err
}
if fs.NArg() > 0 || len(*sha256) == 0 {
fs.PrintDefaults()
return nil
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Println("LoadConfigFile error", "err", err)
return err
}
infoTaskParams := infoTaskParams{
taskSha256: *sha256,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
Auth: &cfg.Auth,
},
}
return doTaskInfo(infoTaskParams)
}
func doTaskInfo(infoTaskParams infoTaskParams) error {
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_TaskInfo{
TaskInfo: &saasapi.TaskInfo{
TaskSha256: infoTaskParams.taskSha256,
},
},
}
res, err := infoTaskParams.saasHttp.TaskInfo(saasReq)
if err != nil {
fmt.Println("submit Task info error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Println("task info failed", "code", res.Code, "status", res.Status)
return nil
}
taskRes := res.GetTaskInfoRes()
fmt.Printf("task res: %v\n", protojson.Format(taskRes))
return nil return nil
} }

83
cmd/saastool/task_list.go
View File

@@ -1,5 +1,88 @@
package main package main
import (
"flag"
"fmt"
"net/http"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type listTaskParams struct {
filterStatus saasapi.TaskStatus
saasHttp *saashttp.SaasClient
}
func RunTaskList(args ...string) error { func RunTaskList(args ...string) error {
fs := flag.NewFlagSet("create", flag.ExitOnError)
cfgFile := paramConfig(fs)
filter := paramFilterStatus(fs)
if err := fs.Parse(args); err != nil {
fmt.Println("command line parse error", "err", err)
return err
}
if fs.NArg() > 0 {
fs.PrintDefaults()
return nil
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Println("LoadConfigFile error", "err", err)
return err
}
listTaskParams := listTaskParams{
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
Auth: &cfg.Auth,
},
}
switch *filter {
case "all":
listTaskParams.filterStatus = saasapi.TaskStatus_ALL
case "waiting":
listTaskParams.filterStatus = saasapi.TaskStatus_WAITING
case "running":
listTaskParams.filterStatus = saasapi.TaskStatus_RUNNING
case "success":
listTaskParams.filterStatus = saasapi.TaskStatus_SUCCESS
case "fail":
listTaskParams.filterStatus = saasapi.TaskStatus_FAIL
}
return doTaskList(listTaskParams)
}
func doTaskList(listTaskParams listTaskParams) error {
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_TaskList{
TaskList: &saasapi.TaskList{
StatusFilter: listTaskParams.filterStatus,
},
},
}
res, err := listTaskParams.saasHttp.TaskList(saasReq)
if err != nil {
fmt.Println("submit List Task error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Println("task list failed", "code", res.Code, "status", res.Status)
return nil
}
taskRes := res.GetTaskListRes()
fmt.Printf("task res: %v\n", protojson.Format(taskRes))
return nil return nil
} }

64
cmd/saastool/make_hash.go → cmd/saastool/task_make.go
View File

@@ -20,9 +20,9 @@ const (
blockSizeMax = 200 * 1024 * 1024 blockSizeMax = 200 * 1024 * 1024
) )
type makeHashParams struct { type makeTaskParams struct {
sourcePath string sourcePath string
destPath string hashFile string
task *saasapi.Task task *saasapi.Task
} }
@@ -39,18 +39,19 @@ type hashResult struct {
index int index int
} }
func RunMakeHash(args ...string) error { func RunTaskMake(args ...string) error {
fs := flag.NewFlagSet("tasklocalmake", flag.ExitOnError) fs := flag.NewFlagSet("make", flag.ExitOnError)
sourcePath := paramSourcePath(fs) sourcePath := paramSourceConvertedPath(fs)
destPath := paramDestPath(fs) hashFile := paramOutputHashFile(fs)
blockSize := paramBlockSize(fs) blockSize := paramBlockSize(fs)
desc := paramTaskDesc(fs)
if err := fs.Parse(args); err != nil { if err := fs.Parse(args); err != nil {
fmt.Println("command line parse error", "err", err) fmt.Println("command line parse error", "err", err)
return err return err
} }
if fs.NArg() > 0 || len(*sourcePath) == 0 || len(*destPath) == 0 { if fs.NArg() > 0 || len(*sourcePath) == 0 || len(*hashFile) == 0 {
fs.PrintDefaults() fs.PrintDefaults()
return nil return nil
} }
@@ -60,52 +61,49 @@ func RunMakeHash(args ...string) error {
return nil return nil
} }
makeHashParams := makeHashParams{ makeTaskParams := makeTaskParams{
sourcePath: *sourcePath, sourcePath: *sourcePath,
destPath: *destPath, hashFile: *hashFile,
task: &saasapi.Task{ task: &saasapi.Task{
TaskBlockSize: blockSize, TaskBlockSize: blockSize,
TaskDescription: *desc,
}, },
} }
return doMakeHash(makeHashParams) return doMakeHash(makeTaskParams)
} }
func doMakeHash(makeHashParams makeHashParams) error { func doMakeHash(makeTaskParams makeTaskParams) error {
fsInfo, err := os.Stat(makeHashParams.sourcePath) fsInfo, err := os.Stat(makeTaskParams.sourcePath)
if err != nil { if err != nil {
return err return err
} }
if !fsInfo.IsDir() { if !fsInfo.IsDir() {
// 如果是文件，直接计算 // 如果是文件，直接计算
return doFileHash(makeHashParams) return doTaskMake(makeTaskParams)
} }
// 读取目录下信息 // 读取目录下信息
dirEntry, err := os.ReadDir(makeHashParams.sourcePath) dirEntry, err := os.ReadDir(makeTaskParams.sourcePath)
if err != nil { if err != nil {
return err return err
} }
// 遍历目录 // 遍历目录
for _, dir := range dirEntry { for _, dir := range dirEntry {
newParam := makeHashParams newParam := makeTaskParams
newParam.sourcePath = path.Join(makeHashParams.sourcePath, dir.Name()) newParam.sourcePath = path.Join(makeTaskParams.sourcePath, dir.Name())
if dir.IsDir() {
newParam.destPath = path.Join(makeHashParams.destPath, dir.Name())
}
if err = doMakeHash(newParam); err != nil { if err = doMakeHash(newParam); err != nil {
return err return err
} }
} }
return saveTaskFile(makeHashParams) return saveTaskFile(makeTaskParams)
} }
func doFileHash(makeHashParams makeHashParams) error { func doTaskMake(makeTaskParams makeTaskParams) error {
sourceFile, err := os.Open(makeHashParams.sourcePath) sourceFile, err := os.Open(makeTaskParams.sourcePath)
if err != nil { if err != nil {
return err return err
} }
@@ -129,12 +127,12 @@ func doFileHash(makeHashParams makeHashParams) error {
wg.Add(1) wg.Add(1)
go func() { go func() {
index := 0 index := 0
buffer := make([]byte, makeHashParams.task.TaskBlockSize) buffer := make([]byte, makeTaskParams.task.TaskBlockSize)
for { for {
n, err := sourceFile.Read(buffer) n, err := sourceFile.Read(buffer)
if n > 0 { if n > 0 {
wg.Add(1) wg.Add(1)
fmt.Printf("\rhashing file [%v], block [%v]", makeHashParams.sourcePath, index) fmt.Printf("\rhashing file [%v], block [%v]", makeTaskParams.sourcePath, index)
tasks <- hashTask{chunk: buffer[:n], index: index} tasks <- hashTask{chunk: buffer[:n], index: index}
index++ index++
} }
@@ -164,7 +162,7 @@ func doFileHash(makeHashParams makeHashParams) error {
// 输出结果 // 输出结果
fileInfo := &saasapi.FileInfo{ fileInfo := &saasapi.FileInfo{
FileName: makeHashParams.sourcePath, FileName: makeTaskParams.sourcePath,
FileSize: uint64(fi.Size()), FileSize: uint64(fi.Size()),
} }
for _, r := range allResults { for _, r := range allResults {
@@ -173,7 +171,7 @@ func doFileHash(makeHashParams makeHashParams) error {
BlockLength: r.blockSize, BlockLength: r.blockSize,
}) })
} }
makeHashParams.task.TaskFileInfos = append(makeHashParams.task.TaskFileInfos, fileInfo) makeTaskParams.task.TaskFileInfos = append(makeTaskParams.task.TaskFileInfos, fileInfo)
fmt.Println("") fmt.Println("")
return nil return nil
@@ -189,8 +187,8 @@ func hashWorker(tasks <-chan hashTask, results chan<- hashResult) {
} }
} }
func saveTaskFile(makeHashParams makeHashParams) error { func saveTaskFile(makeTaskParams makeTaskParams) error {
taskFile, err := os.Create(makeHashParams.destPath) taskFile, err := os.Create(makeTaskParams.hashFile)
if err != nil { if err != nil {
return err return err
} }
@@ -198,15 +196,15 @@ func saveTaskFile(makeHashParams makeHashParams) error {
h := sha256.New() h := sha256.New()
for _, fileInfo := range makeHashParams.task.TaskFileInfos { for _, fileInfo := range makeTaskParams.task.TaskFileInfos {
for _, fileBlock := range fileInfo.FileBlocks { for _, fileBlock := range fileInfo.FileBlocks {
h.Write([]byte(fileBlock.BlockSha256)) h.Write([]byte(fileBlock.BlockSha256))
} }
} }
makeHashParams.task.TaskSha256 = hex.EncodeToString(h.Sum(nil)) makeTaskParams.task.TaskSha256 = hex.EncodeToString(h.Sum(nil))
_, err = taskFile.WriteString(protojson.Format(makeHashParams.task)) _, err = taskFile.WriteString(protojson.Format(makeTaskParams.task))
if err != nil { if err != nil {
return err return err
} }

6
cmd/saastool/write.go
View File

@@ -53,9 +53,9 @@ func RunWrite(args ...string) error {
batchSize: *batchSize, batchSize: *batchSize,
clear: *clear, clear: *clear,
saasHttp: &saashttp.SaasClient{ saasHttp: &saashttp.SaasClient{
Client: http.Client{}, Client: &http.Client{},
ApiUrls: cfg.ApiUrls, ApiUrls: &cfg.ApiUrls,
Auth: cfg.Auth, Auth: &cfg.Auth,
}, },
} }

7
pkg/saashttp/cfg.go
View File

@@ -5,9 +5,12 @@ type ApiUrls struct {
WritePath string WritePath string
ReadPath string ReadPath string
ColumnWritePath string ColumnWritePath string
TaskCreatePath string
TaskListPath string TaskListPath string
TaskCancelPath string TaskInfoPath string
TaskDetailPath string TaskDeletePath string
TaskRunPath string
TaskUploadPath string
} }
type Auth struct { type Auth struct {

23
pkg/saashttp/httpcli.go
View File

@@ -24,9 +24,9 @@ const (
) )
type SaasClient struct { type SaasClient struct {
Client http.Client Client *http.Client
ApiUrls ApiUrls ApiUrls *ApiUrls
Auth Auth Auth *Auth
ResponseEncoder ResponseEncoder ResponseEncoder ResponseEncoder
} }
@@ -45,19 +45,28 @@ func (c *SaasClient) ColumnWrite(saasReq *saasapi.SaasReq) (saasRes *saasapi.Saa
return c.post(postUrl, saasReq) return c.post(postUrl, saasReq)
} }
func (c *SaasClient) TaskCreate(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskCreatePath)
return c.post(postUrl, saasReq)
}
func (c *SaasClient) TaskList(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) { func (c *SaasClient) TaskList(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskListPath) postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskListPath)
return c.post(postUrl, saasReq) return c.post(postUrl, saasReq)
} }
func (c *SaasClient) TaskCancel(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) { func (c *SaasClient) TaskDelete(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskCancelPath) postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskDeletePath)
return c.post(postUrl, saasReq) return c.post(postUrl, saasReq)
} }
func (c *SaasClient) TaskDetail(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) { func (c *SaasClient) TaskInfo(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskInfoPath)
return c.post(postUrl, saasReq)
}
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskDetailPath) func (c *SaasClient) TaskRun(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskRunPath)
return c.post(postUrl, saasReq) return c.post(postUrl, saasReq)
} }