Compare commits

...

7 Commits

Author SHA1 Message Date
algotao
095a0b9b01 更新 2025-08-30 14:22:05 +08:00
algotao
80a758f1e3 增加info获取接口 2025-08-09 22:09:10 +08:00
algotao
edb12c3b1f 增加策略列表、绑定、解绑 2025-08-09 15:54:57 +08:00
algotao
6ca9fe7a02 增加任务取回功能 2025-07-27 18:16:08 +08:00
algotao
a78c16d301 优化task make 2025-07-25 17:25:17 +08:00
algotao
a2bf3c853e 为task增加sourcepath,便于处理 2025-07-25 12:04:16 +08:00
algotao
d1ad148725 兼容单文件处理 2025-06-07 19:15:23 +08:00
30 changed files with 2673 additions and 205 deletions

1
.gitignore vendored
View File

@@ -1,2 +1,3 @@
vendor/
*.out
build/

1621
cmd.pb.go

File diff suppressed because it is too large Load Diff

152
cmd.proto
View File

@@ -7,6 +7,8 @@ option go_package = "e.coding.net/rta/public/saasapi";
// SaasReq 命令请求
message SaasReq {
oneof cmd {
Info info = 5; // 获取账号设置
Read read = 10; // 批量读取
Write write = 11; // 批量写入
ColumnWrite column_write = 12; // 全量列式写入
@@ -16,9 +18,22 @@ message SaasReq {
TaskRun task_run = 22; // 执行任务
TaskDelete task_delete = 23; // 删除任务
TaskInfo task_info = 24; // 任务详情
TargetList target_list = 50; // 列出策略及绑定
BindSet bind_set = 61; // 设置绑定
BindDelete bind_delete = 62; // 解除绑定
ScriptRun script_run = 90; // 运行脚本
ScriptUpdate script_update = 91; // 脚本升级
}
}
// Info 获取账号信息
message Info {
}
// Read 批量读取命令
message Read {
string dataspace_id = 1; // 数据空间ID
@@ -89,7 +104,9 @@ message Task {
string task_sha256 = 3; // 任务sha256
string task_description = 4; // 任务描述
repeated FileInfo task_file_infos = 5; // 文件列表
uint64 task_block_size = 6; // 文件块字节大小(推荐200M
uint64 task_block_size = 6; // 文件块字节大小(推荐50M
string source_path = 7; // 任务数据源路径
uint64 task_size = 8; // 任务所有文件的总大小
// 以下字段只在返回时填写,用于提供服务端的任务状态。在请求时填写会被忽略
string create_time = 10; // 创建时间
@@ -119,23 +136,56 @@ message TaskInfo {
string task_sha256 = 1; // 任务sha256
}
// FileInfo 任务文件信息
message FileInfo {
string file_name = 1; // 文件名
uint64 file_size = 2; // 文件大小
repeated FileBlock file_blocks = 3; // 文件块列表
}
// FileBlock 文件块信息
message FileBlock {
string block_sha256 = 1; // 块的sha256
uint64 block_length = 2; // 块的字节长度
bool uploaded = 3; // 是否已上传在TaskCreate/TaskInfo请求返回
}
// TargetList 列出策略
message TargetList {
repeated string targets = 1; // 指定要列出的绑定的策略列表,如不指定则返回全部
bool list_bind = 2; // 是否同时列出绑定信息
}
// BindSet 设置绑定
message BindSet {
repeated Bind binds = 2; // 设置绑定内容
}
// BindDelete 删除绑定
message BindDelete {
repeated Bind binds = 2; // 解除绑定内容
}
// ScriptRun 运行脚本
message ScriptRun {
string lua_script = 1; // 要调试的lua脚本
string server_did = 2; // 将从服务端读取该DID下的数据
string appid = 3; // 小程序/小游戏/公众号/视频号的appid
string server_openid = 4; // 将从服务端读取该openid下的数据需与appid配对使用
OS os = 5; // 操作系统
}
// ScriptUpdate 升级脚本
message ScriptUpdate {
}
// SaasRes 命令返回
message SaasRes {
ErrorCode code = 1; // 返回码
string status = 2; // 返回信息的文本提示
oneof res {
InfoRes info_res = 5; // 账号信息返回
ReadRes read_res = 10; // 读取命令返回
WriteRes write_res = 11; // 写入命令返回
@@ -144,18 +194,35 @@ message SaasRes {
Task task_run_res = 22; // 运行任务返回状态
Task task_delete_res = 23; // 删除任务返回状态
Task task_info_res = 24; // 任务详情返回状态
TargetListRes target_list_res = 50; // 列出策略及绑定返回状态
BindSetRes bind_set_res = 61; // 设置绑定返回状态
BindDeleteRes bind_delete_res = 62; // 删除绑定返回状态
ScriptRunRes script_run_res = 90; // 运行脚本返回
ScriptUpdateRes script_update_res = 91; // 升级脚本返回
}
}
// InfoRes 账号信息返回
message InfoRes {
repeated string dataspace_id = 1; // 可用数据区列表
repeated string target_id = 2; // 策略ID列表
}
// ReadRes 读记录返回
message ReadRes {
uint32 succ_cmd_count = 1; // 成功的命令数量
uint32 fail_cmd_count = 2; // 失败的命令数量
repeated ValueItem cmd_res = 3; // 返回的命令
}
// WriteRes 写记录返回
message WriteRes {
//uint32 succ_cmd_count = 1; // 成功的命令数量
//uint32 fail_cmd_count = 2; // 失败的命令数量
//uint32 succ_cmd_count = 1; // 成功的命令数量
//uint32 fail_cmd_count = 2; // 失败的命令数量
repeated string failed_userid = 3; // 返回的失败的用户ID
}
@@ -169,10 +236,79 @@ message ValueItem {
uint32 last_modify_time = 6; // 最后修改时间
}
// TaskListRes 任务列表返回
message TaskListRes {
repeated Task tasks = 1; // 任务列表
}
// TargetListRes 策略列表返回
message TargetListRes {
map<string, Binds> target_list = 1; // 绑定列表
}
message Binds {
repeated Bind binds = 1;
}
// Bind 绑定信息
message Bind {
int64 bind_id = 1; //绑定的ID
BindType bind_type = 2; //绑定类型
string target_id = 3; //策略ID
int64 account_id = 4; //广告主ID
BindSourceType bind_source = 5; //绑定操作来源
}
// BindType 绑定类型
enum BindType {
UnknownBindType = 0;
AdgroupId = 1; //广告
AccountId = 3; //广告主
}
// BindSourceType 绑定操作来源
enum BindSourceType {
DefaultBindSourceType = 0; //广告主或未填写
ThirdPartyApi = 1; //第三方API
ADQ = 2; //ADQ平台
MP = 3; //MP平台
MktApi = 4; //MarketingAPI
}
// BindSetRes 设置绑定返回
message BindSetRes {
int32 success_num = 1; //成功数
int32 error_num = 2; //错误数
repeated BindError errors = 3; //绑定错误的记录
}
// BindDeleteRes 删除绑定返回
message BindDeleteRes {
int32 success_num = 1; //成功数
int32 error_num = 2; //错误数
repeated BindError errors = 3; //绑定错误的记录
}
// BindError 绑定错误信息
message BindError {
int64 bind_id = 1; //错误绑定的绑定ID
int32 bind_type = 2; //绑定类型
string reason = 3; //错误绑定原因
}
// ScriptRunRes 运行脚本返回
message ScriptRunRes {
string print_output = 1; // print输出
string error = 2; // 错误信息
string targets_output = 3; // 策略输出
}
// ScriptUpdateRes 升级脚本返回
message ScriptUpdateRes {
}
// ErrorCode 返回码
enum ErrorCode {
SUCC = 0; // 成功
@@ -204,6 +340,8 @@ enum ErrorCode {
DATA_ERROR = 201; // 数据错误
CMD_ERROR = 202; // 命令行执行错误
API_ERROR = 301; // 调用内部API错误
}
enum CmdErrorCode {
@@ -218,6 +356,12 @@ enum TaskStatus {
SUCCESS = 4; // 成功
FAIL = 5; // 失败
DELETED = 10; // 已删除,仅在执行删除成功时返回
DELETED = 10; // 已删除,仅在执行删除成功时返回
}
enum OS {
UNKNOWN = 0;
IOS = 1;
ANDROID = 2;
}

View File

@@ -1,4 +1,5 @@
debug/
saastool
saastool_linux
cfg.toml
*.toml
test/

46
cmd/saastool/bind.go Normal file
View File

@@ -0,0 +1,46 @@
package main
import (
"fmt"
"os"
"strings"
)
func RunBind(args ...string) error {
name, args := ParseCommandName(args)
// 从参数中解析出命令
switch name {
case "", "help":
return RunBindHelp(args...)
case "setaccount":
return RunBindSetAccount(args...)
case "setad":
return RunBindSetAd(args...)
case "delete":
return RunBindDelete(args...)
default:
err := fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'saastool bind help' for usage`, name)
fmt.Fprintln(os.Stderr, err)
return err
}
}
func RunBindHelp(args ...string) error {
fmt.Println(strings.TrimSpace(bindUsage))
return nil
}
const bindUsage = `
Usage: saastoola bind COMMAND [OPTIONS]
Commands:
setaccount Set Account binds
setad Set AdGroup binds
delete Delete binds
"help" is the default command.
Use "saastool bind COMMAND -help" for more information about a command.
`

105
cmd/saastool/bind_delete.go Normal file
View File

@@ -0,0 +1,105 @@
package main
import (
"flag"
"fmt"
"net/http"
"os"
"strconv"
"strings"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type bindDeleteParams struct {
idType int
ids []int64
saasHttp *saashttp.SaasClient
}
func RunBindDelete(args ...string) error {
fs := flag.NewFlagSet("delete", flag.ExitOnError)
cfgFile := paramConfig(fs)
idtype := paramIdType(fs)
ids := paramIDs(fs)
if err := fs.Parse(args); err != nil {
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
return err
}
// 切割字符串
idsSlice := strings.Split(*ids, ",")
if len(idsSlice) == 1 && idsSlice[0] == "" {
idsSlice = []string{}
}
if fs.NArg() > 0 || len(idsSlice) == 0 || (len(idsSlice) == 1 && idsSlice[0] == "") {
fs.PrintDefaults()
return nil
}
numIds := []int64{}
for _, v := range idsSlice {
n, err := strconv.ParseInt(v, 10, 64)
if err != nil {
fmt.Fprintln(os.Stderr, "id parse error", "value", v, "err", err)
return err
}
numIds = append(numIds, n)
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
return err
}
bindDeleteParams := bindDeleteParams{
idType: *idtype,
ids: numIds,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}
return doBindDeleteAccount(bindDeleteParams)
}
func doBindDeleteAccount(p bindDeleteParams) error {
bindDelete := &saasapi.BindDelete{}
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_BindDelete{
BindDelete: bindDelete,
},
}
for _, v := range p.ids {
bindDelete.Binds = append(bindDelete.Binds, &saasapi.Bind{
BindId: v,
BindType: saasapi.BindType(p.idType),
})
}
res, err := p.saasHttp.BindDelete(saasReq)
if err != nil {
fmt.Fprintln(os.Stderr, "submit Bind Delete error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Fprintln(os.Stderr, "Bind Delete failed", "code", res.Code, "status", res.Status)
return nil
}
bindRes := res.GetBindDeleteRes()
fmt.Printf("bind res: %v\n", protojson.Format(bindRes))
return nil
}

View File

@@ -0,0 +1,107 @@
package main
import (
"flag"
"fmt"
"net/http"
"os"
"strconv"
"strings"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type bindSetAccountParams struct {
target string
accounts []int64
saasHttp *saashttp.SaasClient
}
func RunBindSetAccount(args ...string) error {
fs := flag.NewFlagSet("setaccount", flag.ExitOnError)
cfgFile := paramConfig(fs)
target := paramTarget(fs)
accounts := paramAccounts(fs)
if err := fs.Parse(args); err != nil {
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
return err
}
// 切割字符串
idsSlice := strings.Split(*accounts, ",")
if len(idsSlice) == 1 && idsSlice[0] == "" {
idsSlice = []string{}
}
if fs.NArg() > 0 || len(idsSlice) == 0 || (len(idsSlice) == 1 && idsSlice[0] == "") {
fs.PrintDefaults()
return nil
}
numAccounts := []int64{}
for _, v := range idsSlice {
n, err := strconv.ParseInt(v, 10, 64)
if err != nil {
fmt.Fprintln(os.Stderr, "account parse error", "value", v, "err", err)
return err
}
numAccounts = append(numAccounts, n)
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
return err
}
bindSetAccountParams := bindSetAccountParams{
target: *target,
accounts: numAccounts,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}
return doBindSetAccount(bindSetAccountParams)
}
func doBindSetAccount(p bindSetAccountParams) error {
bindSet := &saasapi.BindSet{}
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_BindSet{
BindSet: bindSet,
},
}
for _, v := range p.accounts {
bindSet.Binds = append(bindSet.Binds, &saasapi.Bind{
BindId: v,
BindType: saasapi.BindType_AccountId,
TargetId: p.target,
AccountId: v,
})
}
res, err := p.saasHttp.BindSet(saasReq)
if err != nil {
fmt.Fprintln(os.Stderr, "submit Bind Set error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Fprintln(os.Stderr, "Bind Set failed", "code", res.Code, "status", res.Status)
return nil
}
bindRes := res.GetBindSetRes()
fmt.Printf("bind res: %v\n", protojson.Format(bindRes))
return nil
}

110
cmd/saastool/bind_setad.go Normal file
View File

@@ -0,0 +1,110 @@
package main
import (
"flag"
"fmt"
"net/http"
"os"
"strconv"
"strings"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type bindSetAdParams struct {
target string
account int64
ads []int64
saasHttp *saashttp.SaasClient
}
func RunBindSetAd(args ...string) error {
fs := flag.NewFlagSet("setad", flag.ExitOnError)
cfgFile := paramConfig(fs)
target := paramTarget(fs)
account := paramAccount(fs)
ads := paramAds(fs)
if err := fs.Parse(args); err != nil {
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
return err
}
// 切割字符串
idsSlice := strings.Split(*ads, ",")
if len(idsSlice) == 1 && idsSlice[0] == "" {
idsSlice = []string{}
}
if fs.NArg() > 0 || len(idsSlice) == 0 || (len(idsSlice) == 1 && idsSlice[0] == "") {
fs.PrintDefaults()
return nil
}
numAds := []int64{}
for _, v := range idsSlice {
n, err := strconv.ParseInt(v, 10, 64)
if err != nil {
fmt.Fprintln(os.Stderr, "account parse error", "value", v, "err", err)
return err
}
numAds = append(numAds, n)
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
return err
}
bindSetAdParams := bindSetAdParams{
target: *target,
account: *account,
ads: numAds,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}
return doBindSetAd(bindSetAdParams)
}
func doBindSetAd(p bindSetAdParams) error {
bindSet := &saasapi.BindSet{}
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_BindSet{
BindSet: bindSet,
},
}
for _, v := range p.ads {
bindSet.Binds = append(bindSet.Binds, &saasapi.Bind{
BindId: v,
BindType: saasapi.BindType_AdgroupId,
TargetId: p.target,
AccountId: p.account,
})
}
res, err := p.saasHttp.BindSet(saasReq)
if err != nil {
fmt.Fprintln(os.Stderr, "submit Bind Set error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Fprintln(os.Stderr, "Bind Set failed", "code", res.Code, "status", res.Status)
return nil
}
bindRes := res.GetBindSetRes()
fmt.Printf("bind res: %v\n", protojson.Format(bindRes))
return nil
}

View File

@@ -15,13 +15,16 @@ const usage = `
Usage: saastool COMMAND [OPTIONS]
Commands:
info Saas Info
write Write user's 'bytes / uint32s / flags'
read Read user's 'bytes / uint32s / flags'
columnwrite Write columns for 'deviceid / openid' users
convert Convert data to write format
task Task commands
task Task commands
target Target commands
bind Bind commands
"help" is the default command.

71
cmd/saastool/info.go Normal file
View File

@@ -0,0 +1,71 @@
package main
import (
"flag"
"fmt"
"net/http"
"os"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type infoParams struct {
saasHttp *saashttp.SaasClient
}
func RunInfo(args ...string) error {
fs := flag.NewFlagSet("info", flag.ExitOnError)
cfgFile := paramConfig(fs)
if err := fs.Parse(args); err != nil {
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
return err
}
if fs.NArg() > 0 {
fs.PrintDefaults()
return nil
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
return err
}
infoParams := infoParams{
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}
return doInfo(infoParams)
}
func doInfo(infoParams infoParams) error {
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_Info{
Info: &saasapi.Info{},
},
}
res, err := infoParams.saasHttp.Info(saasReq)
if err != nil {
fmt.Fprintln(os.Stderr, "submit Info error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Fprintln(os.Stderr, "info failed", "code", res.Code, "status", res.Status)
return nil
}
infoRes := res.GetInfoRes()
fmt.Printf("info res: %v\n", protojson.Format(infoRes))
return nil
}

View File

@@ -18,6 +18,8 @@ func Run(args ...string) error {
switch name {
case "", "help":
return RunHelp(args...)
case "info":
return RunInfo(args...)
case "write":
return RunWrite(args...)
case "read":
@@ -30,6 +32,10 @@ func Run(args ...string) error {
return RunVerify(args...)
case "task":
return RunTask(args...)
case "target":
return RunTarget(args...)
case "bind":
return RunBind(args...)
default:
err := fmt.Errorf(`unknown command "%v"`+"\n"+`Run 'saastool help' for usage`, name)
fmt.Fprintln(os.Stderr, err.Error())

View File

@@ -62,7 +62,7 @@ func paramBatchSize(fs *flag.FlagSet) *uint {
}
func paramBlockSize(fs *flag.FlagSet) *string {
return fs.String("blocksize", "200M", "Block size to make hash. using size mode K, M, G, T")
return fs.String("blocksize", "50M", "Block size to make hash. using size mode K, M, G, T")
}
func paramClear(fs *flag.FlagSet) *bool {
@@ -73,6 +73,38 @@ func paramDataSpaceId(fs *flag.FlagSet) *string {
return fs.String("ds", "", "Data space ID")
}
func paramTargets(fs *flag.FlagSet) *string {
return fs.String("targets", "", "Target IDs. Use commas to separate multiple IDs")
}
func paramListBinds(fs *flag.FlagSet) *bool {
return fs.Bool("b", false, "List Binds")
}
func paramTarget(fs *flag.FlagSet) *string {
return fs.String("target", "", "Target ID")
}
func paramAccount(fs *flag.FlagSet) *int64 {
return fs.Int64("account", 0, "Advertiser ID")
}
func paramAccounts(fs *flag.FlagSet) *string {
return fs.String("accounts", "", "Advertiser IDs. Use commas to separate multiple IDs")
}
func paramAds(fs *flag.FlagSet) *string {
return fs.String("ads", "", "AdGroup IDs. Use commas to separate multiple IDs")
}
func paramIdType(fs *flag.FlagSet) *int {
return fs.Int("idtype", 0, "ID Type. empty is Automatic matching, 1=AdGroup, 3=Account")
}
func paramIDs(fs *flag.FlagSet) *string {
return fs.String("ids", "", "IDs for delete. Use commas to separate multiple IDs")
}
// ParseByteSize 解析字节大小字符串为字节数
func ParseByteSize(sizeStr string) (uint64, error) {
sizeStr = strings.TrimSpace(sizeStr)

View File

@@ -62,7 +62,7 @@ func RunRead(args ...string) error {
ds: *ds,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}

40
cmd/saastool/target.go Normal file
View File

@@ -0,0 +1,40 @@
package main
import (
"fmt"
"os"
"strings"
)
func RunTarget(args ...string) error {
name, args := ParseCommandName(args)
// 从参数中解析出命令
switch name {
case "", "help":
return RunTargetHelp(args...)
case "list":
return RunTargetList(args...)
default:
err := fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'saastool target help' for usage`, name)
fmt.Fprintln(os.Stderr, err)
return err
}
}
func RunTargetHelp(args ...string) error {
fmt.Println(strings.TrimSpace(targetUsage))
return nil
}
const targetUsage = `
Usage: saastoola target COMMAND [OPTIONS]
Commands:
list List targets
"help" is the default command.
Use "saastool target COMMAND -help" for more information about a command.
`

View File

@@ -0,0 +1,88 @@
package main
import (
"flag"
"fmt"
"net/http"
"os"
"strings"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type listTargetParams struct {
targets []string
listBinds bool
saasHttp *saashttp.SaasClient
}
func RunTargetList(args ...string) error {
fs := flag.NewFlagSet("list", flag.ExitOnError)
cfgFile := paramConfig(fs)
targets := paramTargets(fs)
listBinds := paramListBinds(fs)
if err := fs.Parse(args); err != nil {
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
return err
}
if fs.NArg() > 0 {
fs.PrintDefaults()
return nil
}
// 切割字符串
idsSlice := strings.Split(*targets, ",")
if len(idsSlice) == 1 && idsSlice[0] == "" {
idsSlice = []string{}
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
return err
}
listTargetParams := listTargetParams{
listBinds: *listBinds,
targets: idsSlice,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}
return doTargetList(listTargetParams)
}
func doTargetList(listTargetParams listTargetParams) error {
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_TargetList{
TargetList: &saasapi.TargetList{
Targets: listTargetParams.targets,
ListBind: listTargetParams.listBinds,
},
},
}
res, err := listTargetParams.saasHttp.TargetList(saasReq)
if err != nil {
fmt.Fprintln(os.Stderr, "submit List Target error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Fprintln(os.Stderr, "Target list failed", "code", res.Code, "status", res.Status)
return nil
}
targetRes := res.GetTargetListRes()
fmt.Printf("target res: %v\n", protojson.Format(targetRes))
return nil
}

View File

@@ -25,6 +25,8 @@ func RunTask(args ...string) error {
return RunTaskInfo(args...)
case "upload":
return RunTaskUpload(args...)
case "download":
return RunTaskDownload(args...)
case "run":
return RunTaskRun(args...)
default:
@@ -51,6 +53,7 @@ Commands:
delete Delete a task on server
info Get a task info on server
upload Upload task's file block to server
download Download task's file block to local
"help" is the default command.

View File

@@ -42,7 +42,7 @@ func RunTaskCreate(args ...string) error {
hashFile: *hashFile,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
task: &saasapi.Task{},

View File

@@ -41,7 +41,7 @@ func RunTaskDelete(args ...string) error {
taskSha256: *sha256,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}

View File

@@ -0,0 +1,134 @@
package main
import (
"flag"
"fmt"
"net/http"
"os"
"path/filepath"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
)
type downloadTaskParams struct {
taskSha256 string
destPath string
saasHttp *saashttp.SaasClient
}
func RunTaskDownload(args ...string) error {
fs := flag.NewFlagSet("download", flag.ExitOnError)
cfgFile := paramConfig(fs)
sha256 := paramSha256(fs)
destPath := paramDestPath(fs)
if err := fs.Parse(args); err != nil {
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
return err
}
if fs.NArg() > 0 || len(*sha256) == 0 || len(*destPath) == 0 {
fs.PrintDefaults()
return nil
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
return err
}
downloadTaskParams := downloadTaskParams{
taskSha256: *sha256,
destPath: *destPath,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}
return doTaskDownload(downloadTaskParams)
}
func doTaskDownload(downloadTaskParams downloadTaskParams) error {
infoTaskParams := infoTaskParams{
taskSha256: downloadTaskParams.taskSha256,
saasHttp: downloadTaskParams.saasHttp,
}
taskInfo, err := doTaskInfo(infoTaskParams)
if err != nil {
return err
}
if len(taskInfo.GetSourcePath()) == 0 {
err = fmt.Errorf("task download failed. task info source path is empty")
fmt.Fprintln(os.Stderr, err)
return err
}
totalFiles := len(taskInfo.GetTaskFileInfos())
fi := 0
for _, finfo := range taskInfo.GetTaskFileInfos() {
fi++
var f *os.File
offset := int64(0)
totalBlocks := len(finfo.GetFileBlocks())
bi := 0
for _, binfo := range finfo.GetFileBlocks() {
bi++
if binfo.GetUploaded() {
if f == nil {
fname := finfo.GetFileName()
if len(downloadTaskParams.destPath) > 0 {
fname = filepath.Join(downloadTaskParams.destPath, fname)
}
os.MkdirAll(filepath.Dir(fname), os.ModePerm)
f, err = os.Create(fname)
if err != nil {
return err
}
}
blockRes, err := downloadTaskParams.saasHttp.TaskDownload(
binfo.GetBlockSha256(),
f,
offset,
int(binfo.GetBlockLength()),
)
if err != nil {
return err
}
if blockRes.GetCode() != saasapi.ErrorCode_SUCC {
err = fmt.Errorf("download block error, code %d, msg %s", blockRes.GetCode(), blockRes.GetStatus())
fmt.Fprintln(os.Stderr, err)
return err
} else {
fmt.Printf("download block success. file: %v, sha256 %v. block %v/%v, file %v/%v\n",
finfo.GetFileName(), binfo.GetBlockSha256(),
bi, totalBlocks, fi, totalFiles,
)
}
} else {
fmt.Printf("download block. file: %v, sha256 %v. block %v/%v, file %v/%v\n",
finfo.GetFileName(), binfo.GetBlockSha256(),
bi, totalBlocks, fi, totalFiles,
)
}
offset += int64(binfo.GetBlockLength())
}
if f != nil {
f.Close()
}
}
return nil
}

View File

@@ -41,7 +41,7 @@ func RunTaskInfo(args ...string) error {
taskSha256: *sha256,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}

View File

@@ -40,7 +40,7 @@ func RunTaskList(args ...string) error {
listTaskParams := listTaskParams{
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}

View File

@@ -7,6 +7,7 @@ import (
"fmt"
"os"
"path"
"path/filepath"
"runtime"
"sort"
"strings"
@@ -81,7 +82,7 @@ func RunTaskMake(args ...string) error {
return nil
}
makeTaskParams := makeTaskParams{
makeTaskParams := &makeTaskParams{
sourcePath: *sourcePath,
hashFile: *hashFile,
task: &saasapi.Task{
@@ -89,12 +90,13 @@ func RunTaskMake(args ...string) error {
TaskDescription: *desc,
Appid: *appid,
DataspaceId: *ds,
SourcePath: *sourcePath,
},
}
return doMakeHash(makeTaskParams)
return doMakeHash(makeTaskParams, true)
}
func doMakeHash(makeTaskParams makeTaskParams) error {
func doMakeHash(makeTaskParams *makeTaskParams, firstLevel bool) error {
fsInfo, err := os.Stat(makeTaskParams.sourcePath)
if err != nil {
return err
@@ -102,29 +104,39 @@ func doMakeHash(makeTaskParams makeTaskParams) error {
if !fsInfo.IsDir() {
// 如果是文件,直接计算
return doTaskMake(makeTaskParams)
}
// 读取目录下信息
dirEntry, err := os.ReadDir(makeTaskParams.sourcePath)
if err != nil {
return err
}
// 遍历目录
for _, dir := range dirEntry {
newParam := makeTaskParams
newParam.sourcePath = path.Join(makeTaskParams.sourcePath, dir.Name())
if err = doMakeHash(newParam); err != nil {
if firstLevel {
makeTaskParams.task.SourcePath = filepath.Dir(makeTaskParams.sourcePath)
}
err = doTaskMake(makeTaskParams)
if err != nil {
return err
}
} else {
// 读取目录下信息
dirEntry, err := os.ReadDir(makeTaskParams.sourcePath)
if err != nil {
return err
}
// 遍历目录
for _, dir := range dirEntry {
oldSourcePath := makeTaskParams.sourcePath
makeTaskParams.sourcePath = path.Join(makeTaskParams.sourcePath, dir.Name())
if err = doMakeHash(makeTaskParams, false); err != nil {
return err
}
makeTaskParams.sourcePath = oldSourcePath
}
}
return saveTaskFile(makeTaskParams)
if firstLevel {
return saveTaskFile(makeTaskParams)
}
return nil
}
func doTaskMake(makeTaskParams makeTaskParams) error {
func doTaskMake(makeTaskParams *makeTaskParams) error {
sourceFile, err := os.Open(makeTaskParams.sourcePath)
if err != nil {
return err
@@ -200,10 +212,12 @@ func doTaskMake(makeTaskParams makeTaskParams) error {
})
// 输出结果
relPath, _ := filepath.Rel(makeTaskParams.task.GetSourcePath(), makeTaskParams.sourcePath)
fileInfo := &saasapi.FileInfo{
FileName: makeTaskParams.sourcePath,
FileName: relPath,
FileSize: uint64(fi.Size()),
}
for _, r := range allResults {
fileInfo.FileBlocks = append(fileInfo.FileBlocks, &saasapi.FileBlock{
BlockSha256: r.hash,
@@ -212,6 +226,8 @@ func doTaskMake(makeTaskParams makeTaskParams) error {
}
makeTaskParams.task.TaskFileInfos = append(makeTaskParams.task.TaskFileInfos, fileInfo)
makeTaskParams.task.TaskSize += uint64(fi.Size())
fmt.Println("")
return nil
}
@@ -227,7 +243,7 @@ func hashWorker(tasks <-chan *hashTask, results chan<- *hashTask) {
}
}
func saveTaskFile(makeTaskParams makeTaskParams) error {
func saveTaskFile(makeTaskParams *makeTaskParams) error {
taskFile, err := os.Create(makeTaskParams.hashFile)
if err != nil {
return err

View File

@@ -41,7 +41,7 @@ func RunTaskRun(args ...string) error {
taskSha256: *sha256,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"net/http"
"os"
"path/filepath"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
@@ -40,7 +41,7 @@ func RunTaskUpload(args ...string) error {
taskSha256: *sha256,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}
@@ -58,10 +59,10 @@ func doTaskUpload(uploadTaskParams uploadTaskParams) error {
return err
}
sourcePath := taskInfo.GetSourcePath()
totalFiles := len(taskInfo.GetTaskFileInfos())
fi := 0
for _, finfo := range taskInfo.GetTaskFileInfos() {
fi++
var f *os.File
offset := int64(0)
@@ -71,7 +72,11 @@ func doTaskUpload(uploadTaskParams uploadTaskParams) error {
bi++
if !binfo.GetUploaded() {
if f == nil {
f, err = os.Open(finfo.GetFileName())
fname := finfo.GetFileName()
if len(sourcePath) > 0 {
fname = filepath.Join(sourcePath, fname)
}
f, err = os.Open(fname)
if err != nil {
return err
}
@@ -89,7 +94,9 @@ func doTaskUpload(uploadTaskParams uploadTaskParams) error {
}
if blockRes.GetCode() != saasapi.ErrorCode_SUCC {
return fmt.Errorf("upload block error, code %d, msg %s", blockRes.GetCode(), blockRes.GetStatus())
err = fmt.Errorf("upload block error, code %d, msg %s", blockRes.GetCode(), blockRes.GetStatus())
fmt.Fprintln(os.Stderr, err)
return err
} else {
fmt.Printf("upload block success. file: %v, sha256 %v. block %v/%v, file %v/%v\n",
finfo.GetFileName(), binfo.GetBlockSha256(),

View File

@@ -63,7 +63,7 @@ func RunWrite(args ...string) error {
clear: *clear,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}

View File

@@ -0,0 +1,12 @@
FROM rta-docker.pkg.coding.net/public/docker/entre_dev:latest AS builder
COPY . /tmp/saasapi/
WORKDIR /tmp/saasapi/
RUN --mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/go/pkg \
./make.sh
FROM rta-docker.pkg.coding.net/public/docker/alpine:latest AS product
COPY --from=builder /tmp/saasapi/build/saastool /bin/saastool

3
make.sh Executable file
View File

@@ -0,0 +1,3 @@
cd cmd/saastool
go build -o ../../build/saastool
cd ../../

View File

@@ -1,16 +1,135 @@
package saashttp
const (
baseUrl = "https://api.rta.qq.com"
infoPath = "/saas/info"
writePath = "/saas/write"
readPath = "/saas/read"
columnWritePath = "/saas/column_write"
taskCreatePath = "/saas/task/create"
taskListPath = "/saas/task/list"
taskCancelPath = "/saas/task/delete"
taskInfoPath = "/saas/task/info"
taskUploadPath = "/saas/task/upload"
taskDownloadPath = "/saas/task/download"
taskDeletePath = "/saas/task/delete"
taskRunPath = "/saas/task/run"
targetListPath = "/saas/target/list"
bindSetPath = "/saas/bind/set"
bindDeletePath = "/saas/bind/delete"
)
type ApiUrls struct {
BaseUrl string
WritePath string
ReadPath string
ColumnWritePath string
TaskCreatePath string
TaskListPath string
TaskInfoPath string
TaskDeletePath string
TaskRunPath string
TaskUploadPath string
BaseUrl string
InfoPath string
WritePath string
ReadPath string
ColumnWritePath string
TaskCreatePath string
TaskListPath string
TaskInfoPath string
TaskDeletePath string
TaskRunPath string
TaskUploadPath string
TaskDownloadPath string
TargetListPath string
BindSetPath string
BindDeletePath string
}
func InitAPIUrl(c *ApiUrls) *ApiUrls {
r := &ApiUrls{}
if c.BaseUrl != "" {
r.BaseUrl = c.BaseUrl
} else {
r.BaseUrl = baseUrl
}
if c.InfoPath != "" {
r.InfoPath = c.InfoPath
} else {
r.InfoPath = infoPath
}
if c.WritePath != "" {
r.WritePath = c.WritePath
} else {
r.WritePath = writePath
}
if c.ReadPath != "" {
r.ReadPath = c.ReadPath
} else {
r.ReadPath = readPath
}
if c.ColumnWritePath != "" {
r.ColumnWritePath = c.ColumnWritePath
} else {
r.ColumnWritePath = columnWritePath
}
if c.TaskCreatePath != "" {
r.TaskCreatePath = c.TaskCreatePath
} else {
r.TaskCreatePath = taskCreatePath
}
if c.TaskListPath != "" {
r.TaskListPath = c.TaskListPath
} else {
r.TaskListPath = taskListPath
}
if c.TaskInfoPath != "" {
r.TaskInfoPath = c.TaskInfoPath
} else {
r.TaskInfoPath = taskInfoPath
}
if c.TaskDeletePath != "" {
r.TaskDeletePath = c.TaskDeletePath
} else {
r.TaskDeletePath = taskDeletePath
}
if c.TaskRunPath != "" {
r.TaskRunPath = c.TaskRunPath
} else {
r.TaskRunPath = taskRunPath
}
if c.TaskUploadPath != "" {
r.TaskUploadPath = c.TaskUploadPath
} else {
r.TaskUploadPath = taskUploadPath
}
if c.TaskDownloadPath != "" {
r.TaskDownloadPath = c.TaskDownloadPath
} else {
r.TaskDownloadPath = taskDownloadPath
}
if c.TargetListPath != "" {
r.TargetListPath = c.TargetListPath
} else {
r.TargetListPath = targetListPath
}
if c.BindSetPath != "" {
r.BindSetPath = c.BindSetPath
} else {
r.BindSetPath = bindSetPath
}
if c.BindDeletePath != "" {
r.BindDeletePath = c.BindDeletePath
} else {
r.BindDeletePath = bindDeletePath
}
return r
}
type Auth struct {

View File

@@ -12,6 +12,7 @@ import (
"net/url"
"os"
"strconv"
"strings"
"time"
"e.coding.net/rta/public/saasapi"
@@ -32,6 +33,11 @@ type SaasClient struct {
ResponseEncoder ResponseEncoder
}
func (c *SaasClient) Info(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.InfoPath)
return c.post(postUrl, saasReq)
}
func (c *SaasClient) Write(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.WritePath)
return c.post(postUrl, saasReq)
@@ -77,6 +83,26 @@ func (c *SaasClient) TaskUpload(sha256 string, file *os.File, offset int64, size
return c.upload(postUrl, file, offset, size)
}
func (c *SaasClient) TaskDownload(sha256 string, file *os.File, offset int64, size int) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskDownloadPath, "sha256", sha256)
return c.download(postUrl, file, offset, size)
}
func (c *SaasClient) TargetList(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TargetListPath)
return c.post(postUrl, saasReq)
}
func (c *SaasClient) BindSet(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.BindSetPath)
return c.post(postUrl, saasReq)
}
func (c *SaasClient) BindDelete(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.BindDeletePath)
return c.post(postUrl, saasReq)
}
func (c *SaasClient) makeUrl(baseUrl, path string, params ...string) string {
url, err := url.Parse(baseUrl)
if err != nil {
@@ -232,3 +258,86 @@ func (c *SaasClient) upload(url string, file *os.File, offset int64, size int) (
return saasRes, nil
}
func (c *SaasClient) download(url string, file *os.File, offset int64, size int) (saasRes *saasapi.SaasRes, err error) {
if file == nil {
return nil, fmt.Errorf("file is nil")
}
if size <= 0 {
return nil, fmt.Errorf("size is invalid")
}
req, err := http.NewRequest("GET", url, bytes.NewBuffer([]byte{}))
if err != nil {
fmt.Println("http new request error", err)
return nil, err
}
timeStamp := strconv.FormatInt(time.Now().Unix(), 10)
md5byte := md5.Sum([]byte(c.Auth.Account + c.Auth.Token + timeStamp))
authorization := hex.EncodeToString(md5byte[:])
req.Header.Add("Account", c.Auth.Account)
req.Header.Add("Time", timeStamp)
req.Header.Add("Authorization", authorization)
req.Header.Add("Accept-Encoding", "gzip")
res, err := c.Client.Do(req)
if err != nil {
fmt.Println("http send error", err)
return nil, err
}
defer res.Body.Close()
if res.StatusCode != 200 {
err = fmt.Errorf("NOT 200. %v", res.StatusCode)
fmt.Println("http state error", err)
return nil, err
}
bodyReader := res.Body
if strings.Contains(res.Header.Get("Content-Encoding"), "gzip") {
gz, err := gzip.NewReader(res.Body)
if err != nil {
fmt.Println("gzip newreader error", err)
return nil, err
}
defer gz.Close()
bodyReader = gz
}
resBody, err := io.ReadAll(bodyReader)
if err != nil {
fmt.Println("read body error", err)
return nil, err
}
saasRes = &saasapi.SaasRes{}
if strings.Contains(res.Header.Get("Content-Type"), "application/octet-stream") {
if len(resBody) == size {
file.WriteAt(resBody, offset)
} else {
err = fmt.Errorf("body size error. body:%v, want:%v", len(resBody), size)
fmt.Println("http read body error", err)
return nil, err
}
} else {
if c.ResponseEncoder == RESPONSE_ENCODER_PROTOBUF {
err = proto.Unmarshal(resBody, saasRes)
if err != nil {
fmt.Println("unmarshal response body to protobuf error", err)
return nil, err
}
} else {
err = json.Unmarshal(resBody, saasRes)
if err != nil {
fmt.Println("unmarshal response body to json error", err)
return nil, err
}
}
}
return saasRes, nil
}