From b8c2a3a7d44ed2735c4a5a8d86dfb884844a8283 Mon Sep 17 00:00:00 2001 From: algotao Date: Wed, 21 May 2025 17:06:32 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81dataspace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd.pb.go | 456 +++++++++++++++--------------------- cmd.proto | 53 ++--- cmd/saastool/params.go | 4 + cmd/saastool/read.go | 23 +- cmd/saastool/task_create.go | 9 - cmd/saastool/task_make.go | 12 +- cmd/saastool/write.go | 50 ++-- 7 files changed, 265 insertions(+), 342 deletions(-) diff --git a/cmd.pb.go b/cmd.pb.go index 71fd139..f43a083 100644 --- a/cmd.pb.go +++ b/cmd.pb.go @@ -21,103 +21,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// UserIdType 用户 ID 类型 -type UserIdType int32 - -const ( - UserIdType_DEVICEID UserIdType = 0 // 设备号 - UserIdType_OPENID UserIdType = 1 // OpenId - UserIdType_INNERID1 UserIdType = 10 // 内部ID1 -) - -// Enum value maps for UserIdType. -var ( - UserIdType_name = map[int32]string{ - 0: "DEVICEID", - 1: "OPENID", - 10: "INNERID1", - } - UserIdType_value = map[string]int32{ - "DEVICEID": 0, - "OPENID": 1, - "INNERID1": 10, - } -) - -func (x UserIdType) Enum() *UserIdType { - p := new(UserIdType) - *p = x - return p -} - -func (x UserIdType) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (UserIdType) Descriptor() protoreflect.EnumDescriptor { - return file_cmd_proto_enumTypes[0].Descriptor() -} - -func (UserIdType) Type() protoreflect.EnumType { - return &file_cmd_proto_enumTypes[0] -} - -func (x UserIdType) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use UserIdType.Descriptor instead. -func (UserIdType) EnumDescriptor() ([]byte, []int) { - return file_cmd_proto_rawDescGZIP(), []int{0} -} - -// NameSpaceId 用户 ID 类型 -type NameSpaceId int32 - -const ( - NameSpaceId_DID NameSpaceId = 0 // 默认设备号命名空间 - NameSpaceId_WUID NameSpaceId = 1 // 默认WUID命名空间 -) - -// Enum value maps for NameSpaceId. -var ( - NameSpaceId_name = map[int32]string{ - 0: "DID", - 1: "WUID", - } - NameSpaceId_value = map[string]int32{ - "DID": 0, - "WUID": 1, - } -) - -func (x NameSpaceId) Enum() *NameSpaceId { - p := new(NameSpaceId) - *p = x - return p -} - -func (x NameSpaceId) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (NameSpaceId) Descriptor() protoreflect.EnumDescriptor { - return file_cmd_proto_enumTypes[1].Descriptor() -} - -func (NameSpaceId) Type() protoreflect.EnumType { - return &file_cmd_proto_enumTypes[1] -} - -func (x NameSpaceId) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use NameSpaceId.Descriptor instead. -func (NameSpaceId) EnumDescriptor() ([]byte, []int) { - return file_cmd_proto_rawDescGZIP(), []int{1} -} - // ErrorCode 返回码 type ErrorCode int32 @@ -134,6 +37,7 @@ const ( ErrorCode_QPS_LIMIT ErrorCode = 113 // 并发请求量超限 ErrorCode_CMDS_LIMIT ErrorCode = 114 // 命令数量超限 ErrorCode_CMDS_NULL ErrorCode = 115 // 命令为空 + ErrorCode_DATASPACE_NOT_EXISTS ErrorCode = 116 // 数据空间不存在 ErrorCode_TASK_EXISTS ErrorCode = 120 // 任务已存在 ErrorCode_TASK_IS_NOT_EXISTS ErrorCode = 121 // 任务不存在 ErrorCode_TASK_NUM_LIMIT ErrorCode = 122 // 任务数达到上限 @@ -163,6 +67,7 @@ var ( 113: "QPS_LIMIT", 114: "CMDS_LIMIT", 115: "CMDS_NULL", + 116: "DATASPACE_NOT_EXISTS", 120: "TASK_EXISTS", 121: "TASK_IS_NOT_EXISTS", 122: "TASK_NUM_LIMIT", @@ -189,6 +94,7 @@ var ( "QPS_LIMIT": 113, "CMDS_LIMIT": 114, "CMDS_NULL": 115, + "DATASPACE_NOT_EXISTS": 116, "TASK_EXISTS": 120, "TASK_IS_NOT_EXISTS": 121, "TASK_NUM_LIMIT": 122, @@ -215,11 +121,11 @@ func (x ErrorCode) String() string { } func (ErrorCode) Descriptor() protoreflect.EnumDescriptor { - return file_cmd_proto_enumTypes[2].Descriptor() + return file_cmd_proto_enumTypes[0].Descriptor() } func (ErrorCode) Type() protoreflect.EnumType { - return &file_cmd_proto_enumTypes[2] + return &file_cmd_proto_enumTypes[0] } func (x ErrorCode) Number() protoreflect.EnumNumber { @@ -228,7 +134,7 @@ func (x ErrorCode) Number() protoreflect.EnumNumber { // Deprecated: Use ErrorCode.Descriptor instead. func (ErrorCode) EnumDescriptor() ([]byte, []int) { - return file_cmd_proto_rawDescGZIP(), []int{2} + return file_cmd_proto_rawDescGZIP(), []int{0} } type CmdErrorCode int32 @@ -258,11 +164,11 @@ func (x CmdErrorCode) String() string { } func (CmdErrorCode) Descriptor() protoreflect.EnumDescriptor { - return file_cmd_proto_enumTypes[3].Descriptor() + return file_cmd_proto_enumTypes[1].Descriptor() } func (CmdErrorCode) Type() protoreflect.EnumType { - return &file_cmd_proto_enumTypes[3] + return &file_cmd_proto_enumTypes[1] } func (x CmdErrorCode) Number() protoreflect.EnumNumber { @@ -271,7 +177,7 @@ func (x CmdErrorCode) Number() protoreflect.EnumNumber { // Deprecated: Use CmdErrorCode.Descriptor instead. func (CmdErrorCode) EnumDescriptor() ([]byte, []int) { - return file_cmd_proto_rawDescGZIP(), []int{3} + return file_cmd_proto_rawDescGZIP(), []int{1} } type TaskStatus int32 @@ -319,11 +225,11 @@ func (x TaskStatus) String() string { } func (TaskStatus) Descriptor() protoreflect.EnumDescriptor { - return file_cmd_proto_enumTypes[4].Descriptor() + return file_cmd_proto_enumTypes[2].Descriptor() } func (TaskStatus) Type() protoreflect.EnumType { - return &file_cmd_proto_enumTypes[4] + return &file_cmd_proto_enumTypes[2] } func (x TaskStatus) Number() protoreflect.EnumNumber { @@ -332,15 +238,12 @@ func (x TaskStatus) Number() protoreflect.EnumNumber { // Deprecated: Use TaskStatus.Descriptor instead. func (TaskStatus) EnumDescriptor() ([]byte, []int) { - return file_cmd_proto_rawDescGZIP(), []int{4} + return file_cmd_proto_rawDescGZIP(), []int{2} } // SaasReq 命令请求 type SaasReq struct { - state protoimpl.MessageState `protogen:"open.v1"` - UseridType UserIdType `protobuf:"varint,1,opt,name=userid_type,json=useridType,proto3,enum=saasapi.UserIdType" json:"userid_type,omitempty"` // 用户ID类型 - Appid string `protobuf:"bytes,2,opt,name=appid,proto3" json:"appid,omitempty"` // 小程序/小游戏/公众号/视频号的appid - NamespaceId uint32 `protobuf:"varint,3,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` // 命名空间ID + state protoimpl.MessageState `protogen:"open.v1"` // Types that are valid to be assigned to Cmd: // // *SaasReq_Read @@ -386,27 +289,6 @@ func (*SaasReq) Descriptor() ([]byte, []int) { return file_cmd_proto_rawDescGZIP(), []int{0} } -func (x *SaasReq) GetUseridType() UserIdType { - if x != nil { - return x.UseridType - } - return UserIdType_DEVICEID -} - -func (x *SaasReq) GetAppid() string { - if x != nil { - return x.Appid - } - return "" -} - -func (x *SaasReq) GetNamespaceId() uint32 { - if x != nil { - return x.NamespaceId - } - return 0 -} - func (x *SaasReq) GetCmd() isSaasReq_Cmd { if x != nil { return x.Cmd @@ -541,7 +423,9 @@ func (*SaasReq_TaskInfo) isSaasReq_Cmd() {} // Read 批量读取命令 type Read struct { state protoimpl.MessageState `protogen:"open.v1"` - ReadItems []*ReadItem `protobuf:"bytes,1,rep,name=read_items,json=readItems,proto3" json:"read_items,omitempty"` // 批量获取命令 + DataspaceId string `protobuf:"bytes,1,opt,name=dataspace_id,json=dataspaceId,proto3" json:"dataspace_id,omitempty"` // 数据空间ID + Appid string `protobuf:"bytes,2,opt,name=appid,proto3" json:"appid,omitempty"` // 小程序/小游戏/公众号/视频号的appid + ReadItems []*ReadItem `protobuf:"bytes,3,rep,name=read_items,json=readItems,proto3" json:"read_items,omitempty"` // 批量获取命令 unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -576,6 +460,20 @@ func (*Read) Descriptor() ([]byte, []int) { return file_cmd_proto_rawDescGZIP(), []int{1} } +func (x *Read) GetDataspaceId() string { + if x != nil { + return x.DataspaceId + } + return "" +} + +func (x *Read) GetAppid() string { + if x != nil { + return x.Appid + } + return "" +} + func (x *Read) GetReadItems() []*ReadItem { if x != nil { return x.ReadItems @@ -631,8 +529,10 @@ func (x *ReadItem) GetUserid() string { // Write 批量写入命令 type Write struct { state protoimpl.MessageState `protogen:"open.v1"` - IsClearAllFirst bool `protobuf:"varint,1,opt,name=is_clear_all_first,json=isClearAllFirst,proto3" json:"is_clear_all_first,omitempty"` // 是否先清空该用户所有数据 - WriteItems []*WriteItem `protobuf:"bytes,2,rep,name=write_items,json=writeItems,proto3" json:"write_items,omitempty"` // 批量写入命令 + DataspaceId string `protobuf:"bytes,1,opt,name=dataspace_id,json=dataspaceId,proto3" json:"dataspace_id,omitempty"` // 数据空间ID + Appid string `protobuf:"bytes,2,opt,name=appid,proto3" json:"appid,omitempty"` // 小程序/小游戏/公众号/视频号的appid + IsClearAllFirst bool `protobuf:"varint,3,opt,name=is_clear_all_first,json=isClearAllFirst,proto3" json:"is_clear_all_first,omitempty"` // 是否先清空该用户所有数据 + WriteItems []*WriteItem `protobuf:"bytes,4,rep,name=write_items,json=writeItems,proto3" json:"write_items,omitempty"` // 批量写入命令 unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -667,6 +567,20 @@ func (*Write) Descriptor() ([]byte, []int) { return file_cmd_proto_rawDescGZIP(), []int{3} } +func (x *Write) GetDataspaceId() string { + if x != nil { + return x.DataspaceId + } + return "" +} + +func (x *Write) GetAppid() string { + if x != nil { + return x.Appid + } + return "" +} + func (x *Write) GetIsClearAllFirst() bool { if x != nil { return x.IsClearAllFirst @@ -981,10 +895,11 @@ func (x *FlagWithExpire) GetExpire() uint32 { // ColumnWrite 全量列式写入命令 type ColumnWrite struct { state protoimpl.MessageState `protogen:"open.v1"` - IsClearAllFirst bool `protobuf:"varint,1,opt,name=is_clear_all_first,json=isClearAllFirst,proto3" json:"is_clear_all_first,omitempty"` // 是否先执行清空 - WriteBytes *Bytes `protobuf:"bytes,2,opt,name=write_bytes,json=writeBytes,proto3" json:"write_bytes,omitempty"` // byte区域 - WriteUint32S *Uint32S `protobuf:"bytes,3,opt,name=write_uint32s,json=writeUint32s,proto3" json:"write_uint32s,omitempty"` // uint32区域 - WriteFlagsWithExpire *FlagsWithExpire `protobuf:"bytes,4,opt,name=write_flags_with_expire,json=writeFlagsWithExpire,proto3" json:"write_flags_with_expire,omitempty"` // 标志位区域 + DataspaceId string `protobuf:"bytes,1,opt,name=dataspace_id,json=dataspaceId,proto3" json:"dataspace_id,omitempty"` // 数据空间ID + IsClearAllFirst bool `protobuf:"varint,2,opt,name=is_clear_all_first,json=isClearAllFirst,proto3" json:"is_clear_all_first,omitempty"` // 是否先执行清空 + WriteBytes *Bytes `protobuf:"bytes,3,opt,name=write_bytes,json=writeBytes,proto3" json:"write_bytes,omitempty"` // byte区域 + WriteUint32S *Uint32S `protobuf:"bytes,4,opt,name=write_uint32s,json=writeUint32s,proto3" json:"write_uint32s,omitempty"` // uint32区域 + WriteFlagsWithExpire *FlagsWithExpire `protobuf:"bytes,5,opt,name=write_flags_with_expire,json=writeFlagsWithExpire,proto3" json:"write_flags_with_expire,omitempty"` // 标志位区域 unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1019,6 +934,13 @@ func (*ColumnWrite) Descriptor() ([]byte, []int) { return file_cmd_proto_rawDescGZIP(), []int{9} } +func (x *ColumnWrite) GetDataspaceId() string { + if x != nil { + return x.DataspaceId + } + return "" +} + func (x *ColumnWrite) GetIsClearAllFirst() bool { if x != nil { return x.IsClearAllFirst @@ -1049,10 +971,12 @@ func (x *ColumnWrite) GetWriteFlagsWithExpire() *FlagsWithExpire { type Task struct { state protoimpl.MessageState `protogen:"open.v1"` - TaskSha256 string `protobuf:"bytes,1,opt,name=task_sha256,json=taskSha256,proto3" json:"task_sha256,omitempty"` // 任务sha256 - TaskDescription string `protobuf:"bytes,2,opt,name=task_description,json=taskDescription,proto3" json:"task_description,omitempty"` // 任务描述 - TaskFileInfos []*FileInfo `protobuf:"bytes,3,rep,name=task_file_infos,json=taskFileInfos,proto3" json:"task_file_infos,omitempty"` // 文件列表 - TaskBlockSize uint64 `protobuf:"varint,4,opt,name=task_block_size,json=taskBlockSize,proto3" json:"task_block_size,omitempty"` // 文件块字节大小（推荐200M） + DataspaceId string `protobuf:"bytes,1,opt,name=dataspace_id,json=dataspaceId,proto3" json:"dataspace_id,omitempty"` // 数据空间ID + Appid string `protobuf:"bytes,2,opt,name=appid,proto3" json:"appid,omitempty"` // 小程序/小游戏/公众号/视频号的appid + TaskSha256 string `protobuf:"bytes,3,opt,name=task_sha256,json=taskSha256,proto3" json:"task_sha256,omitempty"` // 任务sha256 + TaskDescription string `protobuf:"bytes,4,opt,name=task_description,json=taskDescription,proto3" json:"task_description,omitempty"` // 任务描述 + TaskFileInfos []*FileInfo `protobuf:"bytes,5,rep,name=task_file_infos,json=taskFileInfos,proto3" json:"task_file_infos,omitempty"` // 文件列表 + TaskBlockSize uint64 `protobuf:"varint,6,opt,name=task_block_size,json=taskBlockSize,proto3" json:"task_block_size,omitempty"` // 文件块字节大小（推荐200M） // 以下字段只在返回时填写，用于提供服务端的任务状态。在请求时填写会被忽略 CreateTime string `protobuf:"bytes,10,opt,name=create_time,json=createTime,proto3" json:"create_time,omitempty"` // 创建时间 RunTime string `protobuf:"bytes,11,opt,name=run_time,json=runTime,proto3" json:"run_time,omitempty"` // 运行时间 @@ -1092,6 +1016,20 @@ func (*Task) Descriptor() ([]byte, []int) { return file_cmd_proto_rawDescGZIP(), []int{10} } +func (x *Task) GetDataspaceId() string { + if x != nil { + return x.DataspaceId + } + return "" +} + +func (x *Task) GetAppid() string { + if x != nil { + return x.Appid + } + return "" +} + func (x *Task) GetTaskSha256() string { if x != nil { return x.TaskSha256 @@ -1688,10 +1626,10 @@ func (x *ReadRes) GetCmdRes() []*ValueItem { } type WriteRes struct { - state protoimpl.MessageState `protogen:"open.v1"` - SuccCmdCount uint32 `protobuf:"varint,1,opt,name=succ_cmd_count,json=succCmdCount,proto3" json:"succ_cmd_count,omitempty"` // 成功的命令数量 - FailCmdCount uint32 `protobuf:"varint,2,opt,name=fail_cmd_count,json=failCmdCount,proto3" json:"fail_cmd_count,omitempty"` // 失败的命令数量 - CmdRes []*ValueItem `protobuf:"bytes,3,rep,name=cmd_res,json=cmdRes,proto3" json:"cmd_res,omitempty"` // 返回的失败命令，仅填写cmd_index和cmd_code + state protoimpl.MessageState `protogen:"open.v1"` + // uint32 succ_cmd_count = 1; // 成功的命令数量 + // uint32 fail_cmd_count = 2; // 失败的命令数量 + FailedUserid []string `protobuf:"bytes,3,rep,name=failed_userid,json=failedUserid,proto3" json:"failed_userid,omitempty"` // 返回的失败的用户ID unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1726,23 +1664,9 @@ func (*WriteRes) Descriptor() ([]byte, []int) { return file_cmd_proto_rawDescGZIP(), []int{19} } -func (x *WriteRes) GetSuccCmdCount() uint32 { +func (x *WriteRes) GetFailedUserid() []string { if x != nil { - return x.SuccCmdCount - } - return 0 -} - -func (x *WriteRes) GetFailCmdCount() uint32 { - if x != nil { - return x.FailCmdCount - } - return 0 -} - -func (x *WriteRes) GetCmdRes() []*ValueItem { - if x != nil { - return x.CmdRes + return x.FailedUserid } return nil } @@ -1880,12 +1804,8 @@ var File_cmd_proto protoreflect.FileDescriptor const file_cmd_proto_rawDesc = "" + "\n" + - "\tcmd.proto\x12\asaasapi\"\x84\x04\n" + - "\aSaasReq\x124\n" + - "\vuserid_type\x18\x01 \x01(\x0e2\x13.saasapi.UserIdTypeR\n" + - "useridType\x12\x14\n" + - "\x05appid\x18\x02 \x01(\tR\x05appid\x12!\n" + - "\fnamespace_id\x18\x03 \x01(\rR\vnamespaceId\x12#\n" + + "\tcmd.proto\x12\asaasapi\"\x95\x03\n" + + "\aSaasReq\x12#\n" + "\x04read\x18\n" + " \x01(\v2\r.saasapi.ReadH\x00R\x04read\x12&\n" + "\x05write\x18\v \x01(\v2\x0e.saasapi.WriteH\x00R\x05write\x129\n" + @@ -1897,15 +1817,19 @@ const file_cmd_proto_rawDesc = "" + "\vtask_delete\x18\x17 \x01(\v2\x13.saasapi.TaskDeleteH\x00R\n" + "taskDelete\x120\n" + "\ttask_info\x18\x18 \x01(\v2\x11.saasapi.TaskInfoH\x00R\btaskInfoB\x05\n" + - "\x03cmd\"8\n" + - "\x04Read\x120\n" + + "\x03cmd\"q\n" + + "\x04Read\x12!\n" + + "\fdataspace_id\x18\x01 \x01(\tR\vdataspaceId\x12\x14\n" + + "\x05appid\x18\x02 \x01(\tR\x05appid\x120\n" + "\n" + - "read_items\x18\x01 \x03(\v2\x11.saasapi.ReadItemR\treadItems\"\"\n" + + "read_items\x18\x03 \x03(\v2\x11.saasapi.ReadItemR\treadItems\"\"\n" + "\bReadItem\x12\x16\n" + - "\x06userid\x18\x01 \x01(\tR\x06userid\"i\n" + - "\x05Write\x12+\n" + - "\x12is_clear_all_first\x18\x01 \x01(\bR\x0fisClearAllFirst\x123\n" + - "\vwrite_items\x18\x02 \x03(\v2\x12.saasapi.WriteItemR\n" + + "\x06userid\x18\x01 \x01(\tR\x06userid\"\xa2\x01\n" + + "\x05Write\x12!\n" + + "\fdataspace_id\x18\x01 \x01(\tR\vdataspaceId\x12\x14\n" + + "\x05appid\x18\x02 \x01(\tR\x05appid\x12+\n" + + "\x12is_clear_all_first\x18\x03 \x01(\bR\x0fisClearAllFirst\x123\n" + + "\vwrite_items\x18\x04 \x03(\v2\x12.saasapi.WriteItemR\n" + "writeItems\"\xdc\x01\n" + "\tWriteItem\x12\x16\n" + "\x06userid\x18\x01 \x01(\tR\x06userid\x12/\n" + @@ -1926,19 +1850,22 @@ const file_cmd_proto_rawDesc = "" + "\x0eFlagWithExpire\x12\x12\n" + "\x04flag\x18\x01 \x01(\bR\x04flag\x12!\n" + "\fdefault_flag\x18\x02 \x01(\bR\vdefaultFlag\x12\x16\n" + - "\x06expire\x18\x03 \x01(\rR\x06expire\"\xf3\x01\n" + - "\vColumnWrite\x12+\n" + - "\x12is_clear_all_first\x18\x01 \x01(\bR\x0fisClearAllFirst\x12/\n" + - "\vwrite_bytes\x18\x02 \x01(\v2\x0e.saasapi.BytesR\n" + + "\x06expire\x18\x03 \x01(\rR\x06expire\"\x96\x02\n" + + "\vColumnWrite\x12!\n" + + "\fdataspace_id\x18\x01 \x01(\tR\vdataspaceId\x12+\n" + + "\x12is_clear_all_first\x18\x02 \x01(\bR\x0fisClearAllFirst\x12/\n" + + "\vwrite_bytes\x18\x03 \x01(\v2\x0e.saasapi.BytesR\n" + "writeBytes\x125\n" + - "\rwrite_uint32s\x18\x03 \x01(\v2\x10.saasapi.Uint32sR\fwriteUint32s\x12O\n" + - "\x17write_flags_with_expire\x18\x04 \x01(\v2\x18.saasapi.FlagsWithExpireR\x14writeFlagsWithExpire\"\xbf\x02\n" + - "\x04Task\x12\x1f\n" + - "\vtask_sha256\x18\x01 \x01(\tR\n" + + "\rwrite_uint32s\x18\x04 \x01(\v2\x10.saasapi.Uint32sR\fwriteUint32s\x12O\n" + + "\x17write_flags_with_expire\x18\x05 \x01(\v2\x18.saasapi.FlagsWithExpireR\x14writeFlagsWithExpire\"\xf8\x02\n" + + "\x04Task\x12!\n" + + "\fdataspace_id\x18\x01 \x01(\tR\vdataspaceId\x12\x14\n" + + "\x05appid\x18\x02 \x01(\tR\x05appid\x12\x1f\n" + + "\vtask_sha256\x18\x03 \x01(\tR\n" + "taskSha256\x12)\n" + - "\x10task_description\x18\x02 \x01(\tR\x0ftaskDescription\x129\n" + - "\x0ftask_file_infos\x18\x03 \x03(\v2\x11.saasapi.FileInfoR\rtaskFileInfos\x12&\n" + - "\x0ftask_block_size\x18\x04 \x01(\x04R\rtaskBlockSize\x12\x1f\n" + + "\x10task_description\x18\x04 \x01(\tR\x0ftaskDescription\x129\n" + + "\x0ftask_file_infos\x18\x05 \x03(\v2\x11.saasapi.FileInfoR\rtaskFileInfos\x12&\n" + + "\x0ftask_block_size\x18\x06 \x01(\x04R\rtaskBlockSize\x12\x1f\n" + "\vcreate_time\x18\n" + " \x01(\tR\n" + "createTime\x12\x19\n" + @@ -1983,11 +1910,9 @@ const file_cmd_proto_rawDesc = "" + "\aReadRes\x12$\n" + "\x0esucc_cmd_count\x18\x01 \x01(\rR\fsuccCmdCount\x12$\n" + "\x0efail_cmd_count\x18\x02 \x01(\rR\ffailCmdCount\x12+\n" + - "\acmd_res\x18\x03 \x03(\v2\x12.saasapi.ValueItemR\x06cmdRes\"\x83\x01\n" + - "\bWriteRes\x12$\n" + - "\x0esucc_cmd_count\x18\x01 \x01(\rR\fsuccCmdCount\x12$\n" + - "\x0efail_cmd_count\x18\x02 \x01(\rR\ffailCmdCount\x12+\n" + - "\acmd_res\x18\x03 \x03(\v2\x12.saasapi.ValueItemR\x06cmdRes\"\xf9\x01\n" + + "\acmd_res\x18\x03 \x03(\v2\x12.saasapi.ValueItemR\x06cmdRes\"/\n" + + "\bWriteRes\x12#\n" + + "\rfailed_userid\x18\x03 \x03(\tR\ffailedUserid\"\xf9\x01\n" + "\tValueItem\x12\x1b\n" + "\tcmd_index\x18\x01 \x01(\rR\bcmdIndex\x120\n" + "\bcmd_code\x18\x02 \x01(\x0e2\x15.saasapi.CmdErrorCodeR\acmdCode\x12\x14\n" + @@ -1996,17 +1921,7 @@ const file_cmd_proto_rawDesc = "" + "\x11flags_with_expire\x18\x05 \x03(\v2\x17.saasapi.FlagWithExpireR\x0fflagsWithExpire\x12(\n" + "\x10last_modify_time\x18\x06 \x01(\rR\x0elastModifyTime\"2\n" + "\vTaskListRes\x12#\n" + - "\x05tasks\x18\x01 \x03(\v2\r.saasapi.TaskR\x05tasks*4\n" + - "\n" + - "UserIdType\x12\f\n" + - "\bDEVICEID\x10\x00\x12\n" + - "\n" + - "\x06OPENID\x10\x01\x12\f\n" + - "\bINNERID1\x10\n" + - "* \n" + - "\vNameSpaceId\x12\a\n" + - "\x03DID\x10\x00\x12\b\n" + - "\x04WUID\x10\x01*\xd0\x03\n" + + "\x05tasks\x18\x01 \x03(\v2\r.saasapi.TaskR\x05tasks*\xea\x03\n" + "\tErrorCode\x12\b\n" + "\x04SUCC\x10\x00\x12\x13\n" + "\x0fINVALID_ACCOUNT\x10e\x12\x15\n" + @@ -2020,7 +1935,8 @@ const file_cmd_proto_rawDesc = "" + "\tQPS_LIMIT\x10q\x12\x0e\n" + "\n" + "CMDS_LIMIT\x10r\x12\r\n" + - "\tCMDS_NULL\x10s\x12\x0f\n" + + "\tCMDS_NULL\x10s\x12\x18\n" + + "\x14DATASPACE_NOT_EXISTS\x10t\x12\x0f\n" + "\vTASK_EXISTS\x10x\x12\x16\n" + "\x12TASK_IS_NOT_EXISTS\x10y\x12\x12\n" + "\x0eTASK_NUM_LIMIT\x10z\x12\x13\n" + @@ -2059,78 +1975,74 @@ func file_cmd_proto_rawDescGZIP() []byte { return file_cmd_proto_rawDescData } -var file_cmd_proto_enumTypes = make([]protoimpl.EnumInfo, 5) +var file_cmd_proto_enumTypes = make([]protoimpl.EnumInfo, 3) var file_cmd_proto_msgTypes = make([]protoimpl.MessageInfo, 22) var file_cmd_proto_goTypes = []any{ - (UserIdType)(0), // 0: saasapi.UserIdType - (NameSpaceId)(0), // 1: saasapi.NameSpaceId - (ErrorCode)(0), // 2: saasapi.ErrorCode - (CmdErrorCode)(0), // 3: saasapi.CmdErrorCode - (TaskStatus)(0), // 4: saasapi.TaskStatus - (*SaasReq)(nil), // 5: saasapi.SaasReq - (*Read)(nil), // 6: saasapi.Read - (*ReadItem)(nil), // 7: saasapi.ReadItem - (*Write)(nil), // 8: saasapi.Write - (*WriteItem)(nil), // 9: saasapi.WriteItem - (*Bytes)(nil), // 10: saasapi.Bytes - (*Uint32S)(nil), // 11: saasapi.Uint32s - (*FlagsWithExpire)(nil), // 12: saasapi.FlagsWithExpire - (*FlagWithExpire)(nil), // 13: saasapi.FlagWithExpire - (*ColumnWrite)(nil), // 14: saasapi.ColumnWrite - (*Task)(nil), // 15: saasapi.Task - (*TaskList)(nil), // 16: saasapi.TaskList - (*TaskRun)(nil), // 17: saasapi.TaskRun - (*TaskDelete)(nil), // 18: saasapi.TaskDelete - (*TaskInfo)(nil), // 19: saasapi.TaskInfo - (*FileInfo)(nil), // 20: saasapi.FileInfo - (*FileBlock)(nil), // 21: saasapi.FileBlock - (*SaasRes)(nil), // 22: saasapi.SaasRes - (*ReadRes)(nil), // 23: saasapi.ReadRes - (*WriteRes)(nil), // 24: saasapi.WriteRes - (*ValueItem)(nil), // 25: saasapi.ValueItem - (*TaskListRes)(nil), // 26: saasapi.TaskListRes + (ErrorCode)(0), // 0: saasapi.ErrorCode + (CmdErrorCode)(0), // 1: saasapi.CmdErrorCode + (TaskStatus)(0), // 2: saasapi.TaskStatus + (*SaasReq)(nil), // 3: saasapi.SaasReq + (*Read)(nil), // 4: saasapi.Read + (*ReadItem)(nil), // 5: saasapi.ReadItem + (*Write)(nil), // 6: saasapi.Write + (*WriteItem)(nil), // 7: saasapi.WriteItem + (*Bytes)(nil), // 8: saasapi.Bytes + (*Uint32S)(nil), // 9: saasapi.Uint32s + (*FlagsWithExpire)(nil), // 10: saasapi.FlagsWithExpire + (*FlagWithExpire)(nil), // 11: saasapi.FlagWithExpire + (*ColumnWrite)(nil), // 12: saasapi.ColumnWrite + (*Task)(nil), // 13: saasapi.Task + (*TaskList)(nil), // 14: saasapi.TaskList + (*TaskRun)(nil), // 15: saasapi.TaskRun + (*TaskDelete)(nil), // 16: saasapi.TaskDelete + (*TaskInfo)(nil), // 17: saasapi.TaskInfo + (*FileInfo)(nil), // 18: saasapi.FileInfo + (*FileBlock)(nil), // 19: saasapi.FileBlock + (*SaasRes)(nil), // 20: saasapi.SaasRes + (*ReadRes)(nil), // 21: saasapi.ReadRes + (*WriteRes)(nil), // 22: saasapi.WriteRes + (*ValueItem)(nil), // 23: saasapi.ValueItem + (*TaskListRes)(nil), // 24: saasapi.TaskListRes } var file_cmd_proto_depIdxs = []int32{ - 0, // 0: saasapi.SaasReq.userid_type:type_name -> saasapi.UserIdType - 6, // 1: saasapi.SaasReq.read:type_name -> saasapi.Read - 8, // 2: saasapi.SaasReq.write:type_name -> saasapi.Write - 14, // 3: saasapi.SaasReq.column_write:type_name -> saasapi.ColumnWrite - 15, // 4: saasapi.SaasReq.task_create:type_name -> saasapi.Task - 16, // 5: saasapi.SaasReq.task_list:type_name -> saasapi.TaskList - 17, // 6: saasapi.SaasReq.task_run:type_name -> saasapi.TaskRun - 18, // 7: saasapi.SaasReq.task_delete:type_name -> saasapi.TaskDelete - 19, // 8: saasapi.SaasReq.task_info:type_name -> saasapi.TaskInfo - 7, // 9: saasapi.Read.read_items:type_name -> saasapi.ReadItem - 9, // 10: saasapi.Write.write_items:type_name -> saasapi.WriteItem - 10, // 11: saasapi.WriteItem.write_bytes:type_name -> saasapi.Bytes - 11, // 12: saasapi.WriteItem.write_uint32s:type_name -> saasapi.Uint32s - 12, // 13: saasapi.WriteItem.write_flags_with_expire:type_name -> saasapi.FlagsWithExpire - 13, // 14: saasapi.FlagsWithExpire.flags_with_expire:type_name -> saasapi.FlagWithExpire - 10, // 15: saasapi.ColumnWrite.write_bytes:type_name -> saasapi.Bytes - 11, // 16: saasapi.ColumnWrite.write_uint32s:type_name -> saasapi.Uint32s - 12, // 17: saasapi.ColumnWrite.write_flags_with_expire:type_name -> saasapi.FlagsWithExpire - 20, // 18: saasapi.Task.task_file_infos:type_name -> saasapi.FileInfo - 4, // 19: saasapi.Task.status:type_name -> saasapi.TaskStatus - 4, // 20: saasapi.TaskList.status_filter:type_name -> saasapi.TaskStatus - 21, // 21: saasapi.FileInfo.file_blocks:type_name -> saasapi.FileBlock - 2, // 22: saasapi.SaasRes.code:type_name -> saasapi.ErrorCode - 23, // 23: saasapi.SaasRes.read_res:type_name -> saasapi.ReadRes - 24, // 24: saasapi.SaasRes.write_res:type_name -> saasapi.WriteRes - 15, // 25: saasapi.SaasRes.task_create_res:type_name -> saasapi.Task - 26, // 26: saasapi.SaasRes.task_list_res:type_name -> saasapi.TaskListRes - 15, // 27: saasapi.SaasRes.task_run_res:type_name -> saasapi.Task - 15, // 28: saasapi.SaasRes.task_delete_res:type_name -> saasapi.Task - 15, // 29: saasapi.SaasRes.task_info_res:type_name -> saasapi.Task - 25, // 30: saasapi.ReadRes.cmd_res:type_name -> saasapi.ValueItem - 25, // 31: saasapi.WriteRes.cmd_res:type_name -> saasapi.ValueItem - 3, // 32: saasapi.ValueItem.cmd_code:type_name -> saasapi.CmdErrorCode - 13, // 33: saasapi.ValueItem.flags_with_expire:type_name -> saasapi.FlagWithExpire - 15, // 34: saasapi.TaskListRes.tasks:type_name -> saasapi.Task - 35, // [35:35] is the sub-list for method output_type - 35, // [35:35] is the sub-list for method input_type - 35, // [35:35] is the sub-list for extension type_name - 35, // [35:35] is the sub-list for extension extendee - 0, // [0:35] is the sub-list for field type_name + 4, // 0: saasapi.SaasReq.read:type_name -> saasapi.Read + 6, // 1: saasapi.SaasReq.write:type_name -> saasapi.Write + 12, // 2: saasapi.SaasReq.column_write:type_name -> saasapi.ColumnWrite + 13, // 3: saasapi.SaasReq.task_create:type_name -> saasapi.Task + 14, // 4: saasapi.SaasReq.task_list:type_name -> saasapi.TaskList + 15, // 5: saasapi.SaasReq.task_run:type_name -> saasapi.TaskRun + 16, // 6: saasapi.SaasReq.task_delete:type_name -> saasapi.TaskDelete + 17, // 7: saasapi.SaasReq.task_info:type_name -> saasapi.TaskInfo + 5, // 8: saasapi.Read.read_items:type_name -> saasapi.ReadItem + 7, // 9: saasapi.Write.write_items:type_name -> saasapi.WriteItem + 8, // 10: saasapi.WriteItem.write_bytes:type_name -> saasapi.Bytes + 9, // 11: saasapi.WriteItem.write_uint32s:type_name -> saasapi.Uint32s + 10, // 12: saasapi.WriteItem.write_flags_with_expire:type_name -> saasapi.FlagsWithExpire + 11, // 13: saasapi.FlagsWithExpire.flags_with_expire:type_name -> saasapi.FlagWithExpire + 8, // 14: saasapi.ColumnWrite.write_bytes:type_name -> saasapi.Bytes + 9, // 15: saasapi.ColumnWrite.write_uint32s:type_name -> saasapi.Uint32s + 10, // 16: saasapi.ColumnWrite.write_flags_with_expire:type_name -> saasapi.FlagsWithExpire + 18, // 17: saasapi.Task.task_file_infos:type_name -> saasapi.FileInfo + 2, // 18: saasapi.Task.status:type_name -> saasapi.TaskStatus + 2, // 19: saasapi.TaskList.status_filter:type_name -> saasapi.TaskStatus + 19, // 20: saasapi.FileInfo.file_blocks:type_name -> saasapi.FileBlock + 0, // 21: saasapi.SaasRes.code:type_name -> saasapi.ErrorCode + 21, // 22: saasapi.SaasRes.read_res:type_name -> saasapi.ReadRes + 22, // 23: saasapi.SaasRes.write_res:type_name -> saasapi.WriteRes + 13, // 24: saasapi.SaasRes.task_create_res:type_name -> saasapi.Task + 24, // 25: saasapi.SaasRes.task_list_res:type_name -> saasapi.TaskListRes + 13, // 26: saasapi.SaasRes.task_run_res:type_name -> saasapi.Task + 13, // 27: saasapi.SaasRes.task_delete_res:type_name -> saasapi.Task + 13, // 28: saasapi.SaasRes.task_info_res:type_name -> saasapi.Task + 23, // 29: saasapi.ReadRes.cmd_res:type_name -> saasapi.ValueItem + 1, // 30: saasapi.ValueItem.cmd_code:type_name -> saasapi.CmdErrorCode + 11, // 31: saasapi.ValueItem.flags_with_expire:type_name -> saasapi.FlagWithExpire + 13, // 32: saasapi.TaskListRes.tasks:type_name -> saasapi.Task + 33, // [33:33] is the sub-list for method output_type + 33, // [33:33] is the sub-list for method input_type + 33, // [33:33] is the sub-list for extension type_name + 33, // [33:33] is the sub-list for extension extendee + 0, // [0:33] is the sub-list for field type_name } func init() { file_cmd_proto_init() } @@ -2162,7 +2074,7 @@ func file_cmd_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_cmd_proto_rawDesc), len(file_cmd_proto_rawDesc)), - NumEnums: 5, + NumEnums: 3, NumMessages: 22, NumExtensions: 0, NumServices: 0, diff --git a/cmd.proto b/cmd.proto index b1051a5..1b882e7 100644 --- a/cmd.proto +++ b/cmd.proto @@ -6,10 +6,6 @@ option go_package = "e.coding.net/rta/public/saasapi"; // SaasReq 命令请求 message SaasReq { - UserIdType userid_type = 1; // 用户ID类型 - string appid = 2; // 小程序/小游戏/公众号/视频号的appid - uint32 namespace_id = 3; // 命名空间ID - oneof cmd { Read read = 10; // 批量读取 Write write = 11; // 批量写入 @@ -25,7 +21,9 @@ message SaasReq { // Read 批量读取命令 message Read { - repeated ReadItem read_items = 1; // 批量获取命令 + string dataspace_id = 1; // 数据空间ID + string appid = 2; // 小程序/小游戏/公众号/视频号的appid + repeated ReadItem read_items = 3; // 批量获取命令 } // ReadItem 读取命令 @@ -35,8 +33,10 @@ message ReadItem { // Write 批量写入命令 message Write { - bool is_clear_all_first = 1; // 是否先清空该用户所有数据 - repeated WriteItem write_items = 2; // 批量写入命令 + string dataspace_id = 1; // 数据空间ID + string appid = 2; // 小程序/小游戏/公众号/视频号的appid + bool is_clear_all_first = 3; // 是否先清空该用户所有数据 + repeated WriteItem write_items = 4; // 批量写入命令 } // WriteItem 写入命令 @@ -74,32 +74,22 @@ message FlagWithExpire { uint32 expire = 3; // 过期时间，为 0 则永不过期 } -// UserIdType 用户 ID 类型 -enum UserIdType { - DEVICEID = 0; // 设备号 - OPENID = 1; // OpenId - INNERID1 = 10; // 内部ID1 -} - -// NameSpaceId 用户 ID 类型 -enum NameSpaceId { - DID = 0; // 默认设备号命名空间 - WUID = 1; // 默认WUID命名空间 -} - // ColumnWrite 全量列式写入命令 message ColumnWrite { - bool is_clear_all_first = 1; // 是否先执行清空 - Bytes write_bytes = 2; // byte区域 - Uint32s write_uint32s = 3; // uint32区域 - FlagsWithExpire write_flags_with_expire = 4; // 标志位区域 + string dataspace_id = 1; // 数据空间ID + bool is_clear_all_first = 2; // 是否先执行清空 + Bytes write_bytes = 3; // byte区域 + Uint32s write_uint32s = 4; // uint32区域 + FlagsWithExpire write_flags_with_expire = 5; // 标志位区域 } message Task { - string task_sha256 = 1; // 任务sha256 - string task_description = 2; // 任务描述 - repeated FileInfo task_file_infos = 3; // 文件列表 - uint64 task_block_size = 4; // 文件块字节大小（推荐200M） + string dataspace_id = 1; // 数据空间ID + string appid = 2; // 小程序/小游戏/公众号/视频号的appid + string task_sha256 = 3; // 任务sha256 + string task_description = 4; // 任务描述 + repeated FileInfo task_file_infos = 5; // 文件列表 + uint64 task_block_size = 6; // 文件块字节大小（推荐200M） // 以下字段只在返回时填写，用于提供服务端的任务状态。在请求时填写会被忽略 string create_time = 10; // 创建时间 @@ -164,9 +154,9 @@ message ReadRes { } message WriteRes { - uint32 succ_cmd_count = 1; // 成功的命令数量 - uint32 fail_cmd_count = 2; // 失败的命令数量 - repeated ValueItem cmd_res = 3; // 返回的失败命令，仅填写cmd_index和cmd_code + //uint32 succ_cmd_count = 1; // 成功的命令数量 + //uint32 fail_cmd_count = 2; // 失败的命令数量 + repeated string failed_userid = 3; // 返回的失败的用户ID } // ValueItem 读取命令返回内容 @@ -197,6 +187,7 @@ enum ErrorCode { QPS_LIMIT = 113; // 并发请求量超限 CMDS_LIMIT = 114; // 命令数量超限 CMDS_NULL = 115; // 命令为空 + DATASPACE_NOT_EXISTS = 116; // 数据空间不存在 TASK_EXISTS = 120; // 任务已存在 TASK_IS_NOT_EXISTS = 121; // 任务不存在 diff --git a/cmd/saastool/params.go b/cmd/saastool/params.go index 00f74bd..596ef26 100644 --- a/cmd/saastool/params.go +++ b/cmd/saastool/params.go @@ -69,6 +69,10 @@ func paramClear(fs *flag.FlagSet) *bool { return fs.Bool("clear", false, "Clear all data before write") } +func paramDataSpaceId(fs *flag.FlagSet) *string { + return fs.String("ds", "", "Data space ID") +} + // ParseByteSize 解析字节大小字符串为字节数 func ParseByteSize(sizeStr string) (uint64, error) { sizeStr = strings.TrimSpace(sizeStr) diff --git a/cmd/saastool/read.go b/cmd/saastool/read.go index 19ea527..bb3650d 100644 --- a/cmd/saastool/read.go +++ b/cmd/saastool/read.go @@ -19,6 +19,7 @@ const ( type readParams struct { cfg *Config appid string + ds string userids []string saasHttp *saashttp.SaasClient } @@ -27,6 +28,7 @@ func RunRead(args ...string) error { fs := flag.NewFlagSet("read", flag.ExitOnError) cfgFile := paramConfig(fs) appid := paramAppid(fs) + ds := paramDataSpaceId(fs) userids := paramUserids(fs) if err := fs.Parse(args); err != nil { @@ -37,11 +39,16 @@ func RunRead(args ...string) error { // 切割字符串 idsSlice := strings.Split(*userids, ",") - if fs.NArg() > 0 || len(idsSlice) == 0 || (len(idsSlice) == 1 && idsSlice[0] == "") || len(idsSlice) > getIdsMax { + if fs.NArg() > 0 || len(idsSlice) == 0 || (len(idsSlice) == 1 && idsSlice[0] == "") || len(idsSlice) > getIdsMax || len(*ds) == 0 { fs.PrintDefaults() return nil } + if strings.ToLower(*ds) == "openid" && len(*appid) == 0 { + fmt.Fprintln(os.Stderr, "appid must be set when data space is openid") + return nil + } + cfg, err := LoadConfigFile(*cfgFile) if err != nil { fmt.Fprintln(os.Stderr, "load config file error", "err", err) @@ -52,6 +59,7 @@ func RunRead(args ...string) error { cfg: cfg, userids: idsSlice, appid: *appid, + ds: *ds, saasHttp: &saashttp.SaasClient{ Client: &http.Client{}, ApiUrls: &cfg.ApiUrls, @@ -63,16 +71,17 @@ func RunRead(args ...string) error { } func doRead(readParams readParams) error { + read := &saasapi.Read{ + DataspaceId: readParams.ds, + Appid: readParams.appid, + } + saasReq := &saasapi.SaasReq{ Cmd: &saasapi.SaasReq_Read{ - Read: &saasapi.Read{}, + Read: read, }, } - if readParams.appid != "" { - saasReq.UseridType = saasapi.UserIdType_OPENID - saasReq.Appid = readParams.appid - } saasReadItems := []*saasapi.ReadItem{} for _, userid := range readParams.userids { saasReadItems = append(saasReadItems, &saasapi.ReadItem{ @@ -80,7 +89,7 @@ func doRead(readParams readParams) error { }) } - saasReq.Cmd.(*saasapi.SaasReq_Read).Read.ReadItems = saasReadItems + read.ReadItems = saasReadItems res, err := readParams.saasHttp.Read(saasReq) diff --git a/cmd/saastool/task_create.go b/cmd/saastool/task_create.go index b11893a..1afaf08 100644 --- a/cmd/saastool/task_create.go +++ b/cmd/saastool/task_create.go @@ -13,7 +13,6 @@ import ( type createTaskParams struct { hashFile string - appid string task *saasapi.Task saasHttp *saashttp.SaasClient } @@ -21,9 +20,7 @@ type createTaskParams struct { func RunTaskCreate(args ...string) error { fs := flag.NewFlagSet("create", flag.ExitOnError) cfgFile := paramConfig(fs) - // sourcePath := paramSourceConvertedPath(fs) hashFile := paramInputHashFile(fs) - appid := paramAppid(fs) if err := fs.Parse(args); err != nil { fmt.Fprintln(os.Stderr, "command line parse error", "err", err) @@ -43,7 +40,6 @@ func RunTaskCreate(args ...string) error { createTaskParams := createTaskParams{ hashFile: *hashFile, - appid: *appid, saasHttp: &saashttp.SaasClient{ Client: &http.Client{}, ApiUrls: &cfg.ApiUrls, @@ -72,11 +68,6 @@ func doTaskCreate(createTaskParams createTaskParams) error { }, } - if createTaskParams.appid != "" { - saasReq.UseridType = saasapi.UserIdType_OPENID - saasReq.Appid = createTaskParams.appid - } - res, err := createTaskParams.saasHttp.TaskCreate(saasReq) if err != nil { diff --git a/cmd/saastool/task_make.go b/cmd/saastool/task_make.go index 49c43cf..d483350 100644 --- a/cmd/saastool/task_make.go +++ b/cmd/saastool/task_make.go @@ -9,6 +9,7 @@ import ( "path" "runtime" "sort" + "strings" "sync" "e.coding.net/rta/public/saasapi" @@ -49,17 +50,24 @@ func RunTaskMake(args ...string) error { hashFile := paramOutputHashFile(fs) blockSize := paramBlockSize(fs) desc := paramTaskDesc(fs) + appid := paramAppid(fs) + ds := paramDataSpaceId(fs) if err := fs.Parse(args); err != nil { fmt.Fprintln(os.Stderr, "command line parse error", "err", err) return err } - if fs.NArg() > 0 || len(*sourcePath) == 0 || len(*hashFile) == 0 { + if fs.NArg() > 0 || len(*sourcePath) == 0 || len(*hashFile) == 0 || len(*ds) == 0 { fs.PrintDefaults() return nil } + if strings.ToLower(*ds) == "openid" && len(*appid) == 0 { + fmt.Fprintln(os.Stderr, "appid must be set when data space is openid") + return nil + } + blockSizeNum, err := ParseByteSize(*blockSize) if err != nil { fmt.Fprintln(os.Stderr, "Error parsing block size", "err", err) @@ -79,6 +87,8 @@ func RunTaskMake(args ...string) error { task: &saasapi.Task{ TaskBlockSize: blockSizeNum, TaskDescription: *desc, + Appid: *appid, + DataspaceId: *ds, }, } return doMakeHash(makeTaskParams) diff --git a/cmd/saastool/write.go b/cmd/saastool/write.go index 3a2d322..b6ce77b 100644 --- a/cmd/saastool/write.go +++ b/cmd/saastool/write.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "path" + "strings" "e.coding.net/rta/public/saasapi" "e.coding.net/rta/public/saasapi/pkg/saashttp" @@ -17,6 +18,7 @@ type writeParams struct { cfg *Config sourcePath string appid string + ds string batchSize uint clear bool saasHttp *saashttp.SaasClient @@ -27,6 +29,7 @@ func RunWrite(args ...string) error { cfgFile := paramConfig(fs) sourcePath := paramSourcePath(fs) appid := paramAppid(fs) + ds := paramDataSpaceId(fs) batchSize := paramBatchSize(fs) clear := paramClear(fs) @@ -35,11 +38,16 @@ func RunWrite(args ...string) error { return err } - if fs.NArg() > 0 || len(*sourcePath) == 0 { + if fs.NArg() > 0 || len(*sourcePath) == 0 || len(*ds) == 0 { fs.PrintDefaults() return nil } + if strings.ToLower(*ds) == "openid" && len(*appid) == 0 { + fmt.Fprintln(os.Stderr, "appid must be set when data space is openid") + return nil + } + cfg, err := LoadConfigFile(*cfgFile) if err != nil { fmt.Fprintln(os.Stderr, "load config file error", "err", err) @@ -50,6 +58,7 @@ func RunWrite(args ...string) error { cfg: cfg, sourcePath: *sourcePath, appid: *appid, + ds: *ds, batchSize: *batchSize, clear: *clear, saasHttp: &saashttp.SaasClient{ @@ -107,9 +116,9 @@ func doLoadFileToWrite(writeParams writeParams) error { saasWriteItems := []*saasapi.WriteItem{} - succ := uint32(0) - succTotal := uint32(0) - total := uint32(0) + errCount := 0 + errTotal := 0 + total := 0 for scaner.Scan() { total++ line := scaner.Text() @@ -125,45 +134,42 @@ func doLoadFileToWrite(writeParams writeParams) error { saasWriteItems = append(saasWriteItems, saasWriteItem) if len(saasWriteItems) == int(writeParams.batchSize) { - if succ, _, err = submitWrite(writeParams, saasWriteItems); err != nil { + if errCount, err = submitWrite(writeParams, saasWriteItems); err != nil { return err } - succTotal += succ - fmt.Printf("[%v] batch_succ = %v, succ_total = %v, total_processed = %v\n", writeParams.sourcePath, succ, succTotal, total) + errTotal += errCount + fmt.Printf("[%v] err_batch = %v, err_total = %v, total_processed = %v\n", writeParams.sourcePath, errCount, errTotal, total) saasWriteItems = saasWriteItems[:0] } } if len(saasWriteItems) > 0 { - if succ, _, err = submitWrite(writeParams, saasWriteItems); err != nil { + if errCount, err = submitWrite(writeParams, saasWriteItems); err != nil { return err } - succTotal += succ - fmt.Printf("[%v] batch_succ = %v, succ_total = %v, total_processed = %v\n", writeParams.sourcePath, succ, succTotal, total) + errTotal += errCount + fmt.Printf("[%v] err_batch = %v, err_total = %v, total_processed = %v\n", writeParams.sourcePath, errCount, errTotal, total) } return nil } -func submitWrite(writeParams writeParams, saasWriteCmds []*saasapi.WriteItem) (succ, total uint32, err error) { +func submitWrite(writeParams writeParams, saasWriteCmds []*saasapi.WriteItem) (errcount int, err error) { + write := &saasapi.Write{ + DataspaceId: writeParams.ds, + Appid: writeParams.appid, + IsClearAllFirst: writeParams.clear, + } saasReq := &saasapi.SaasReq{ Cmd: &saasapi.SaasReq_Write{ - Write: &saasapi.Write{ - IsClearAllFirst: writeParams.clear, - }, + Write: write, }, } - if writeParams.appid != "" { - saasReq.UseridType = saasapi.UserIdType_OPENID - saasReq.Appid = writeParams.appid - } + write.WriteItems = saasWriteCmds - saasReq.Cmd.(*saasapi.SaasReq_Write).Write.WriteItems = saasWriteCmds - - total = uint32(len(saasWriteCmds)) res, err := writeParams.saasHttp.Write(saasReq) if err != nil { @@ -177,5 +183,5 @@ func submitWrite(writeParams writeParams, saasWriteCmds []*saasapi.WriteItem) (s return } - return res.GetWriteRes().GetSuccCmdCount(), total, nil + return len(res.GetWriteRes().GetFailedUserid()), nil }