Compare commits

...

7 Commits

Author SHA1 Message Date
algotao
095a0b9b01 更新 2025-08-30 14:22:05 +08:00
algotao
80a758f1e3 增加info获取接口 2025-08-09 22:09:10 +08:00
algotao
edb12c3b1f 增加策略列表、绑定、解绑 2025-08-09 15:54:57 +08:00
algotao
6ca9fe7a02 增加任务取回功能 2025-07-27 18:16:08 +08:00
algotao
a78c16d301 优化task make 2025-07-25 17:25:17 +08:00
algotao
a2bf3c853e 为task增加sourcepath,便于处理 2025-07-25 12:04:16 +08:00
algotao
d1ad148725 兼容单文件处理 2025-06-07 19:15:23 +08:00
30 changed files with 2673 additions and 205 deletions

1
.gitignore vendored
View File

@@ -1,2 +1,3 @@
vendor/
*.out
build/

1617
cmd.pb.go

File diff suppressed because it is too large Load Diff

146
cmd.proto
View File

@@ -7,6 +7,8 @@ option go_package = "e.coding.net/rta/public/saasapi";
// SaasReq 命令请求
message SaasReq {
oneof cmd {
Info info = 5; // 获取账号设置
Read read = 10; // 批量读取
Write write = 11; // 批量写入
ColumnWrite column_write = 12; // 全量列式写入
@@ -16,9 +18,22 @@ message SaasReq {
TaskRun task_run = 22; // 执行任务
TaskDelete task_delete = 23; // 删除任务
TaskInfo task_info = 24; // 任务详情
TargetList target_list = 50; // 列出策略及绑定
BindSet bind_set = 61; // 设置绑定
BindDelete bind_delete = 62; // 解除绑定
ScriptRun script_run = 90; // 运行脚本
ScriptUpdate script_update = 91; // 脚本升级
}
}
// Info 获取账号信息
message Info {
}
// Read 批量读取命令
message Read {
string dataspace_id = 1; // 数据空间ID
@@ -89,7 +104,9 @@ message Task {
string task_sha256 = 3; // 任务sha256
string task_description = 4; // 任务描述
repeated FileInfo task_file_infos = 5; // 文件列表
uint64 task_block_size = 6; // 文件块字节大小(推荐200M
uint64 task_block_size = 6; // 文件块字节大小(推荐50M
string source_path = 7; // 任务数据源路径
uint64 task_size = 8; // 任务所有文件的总大小
// 以下字段只在返回时填写,用于提供服务端的任务状态。在请求时填写会被忽略
string create_time = 10; // 创建时间
@@ -119,23 +136,56 @@ message TaskInfo {
string task_sha256 = 1; // 任务sha256
}
// FileInfo 任务文件信息
message FileInfo {
string file_name = 1; // 文件名
uint64 file_size = 2; // 文件大小
repeated FileBlock file_blocks = 3; // 文件块列表
}
// FileBlock 文件块信息
message FileBlock {
string block_sha256 = 1; // 块的sha256
uint64 block_length = 2; // 块的字节长度
bool uploaded = 3; // 是否已上传在TaskCreate/TaskInfo请求返回
}
// TargetList 列出策略
message TargetList {
repeated string targets = 1; // 指定要列出的绑定的策略列表,如不指定则返回全部
bool list_bind = 2; // 是否同时列出绑定信息
}
// BindSet 设置绑定
message BindSet {
repeated Bind binds = 2; // 设置绑定内容
}
// BindDelete 删除绑定
message BindDelete {
repeated Bind binds = 2; // 解除绑定内容
}
// ScriptRun 运行脚本
message ScriptRun {
string lua_script = 1; // 要调试的lua脚本
string server_did = 2; // 将从服务端读取该DID下的数据
string appid = 3; // 小程序/小游戏/公众号/视频号的appid
string server_openid = 4; // 将从服务端读取该openid下的数据需与appid配对使用
OS os = 5; // 操作系统
}
// ScriptUpdate 升级脚本
message ScriptUpdate {
}
// SaasRes 命令返回
message SaasRes {
ErrorCode code = 1; // 返回码
string status = 2; // 返回信息的文本提示
oneof res {
InfoRes info_res = 5; // 账号信息返回
ReadRes read_res = 10; // 读取命令返回
WriteRes write_res = 11; // 写入命令返回
@@ -144,15 +194,32 @@ message SaasRes {
Task task_run_res = 22; // 运行任务返回状态
Task task_delete_res = 23; // 删除任务返回状态
Task task_info_res = 24; // 任务详情返回状态
TargetListRes target_list_res = 50; // 列出策略及绑定返回状态
BindSetRes bind_set_res = 61; // 设置绑定返回状态
BindDeleteRes bind_delete_res = 62; // 删除绑定返回状态
ScriptRunRes script_run_res = 90; // 运行脚本返回
ScriptUpdateRes script_update_res = 91; // 升级脚本返回
}
}
// InfoRes 账号信息返回
message InfoRes {
repeated string dataspace_id = 1; // 可用数据区列表
repeated string target_id = 2; // 策略ID列表
}
// ReadRes 读记录返回
message ReadRes {
uint32 succ_cmd_count = 1; // 成功的命令数量
uint32 fail_cmd_count = 2; // 失败的命令数量
repeated ValueItem cmd_res = 3; // 返回的命令
}
// WriteRes 写记录返回
message WriteRes {
//uint32 succ_cmd_count = 1; // 成功的命令数量
//uint32 fail_cmd_count = 2; // 失败的命令数量
@@ -169,10 +236,79 @@ message ValueItem {
uint32 last_modify_time = 6; // 最后修改时间
}
// TaskListRes 任务列表返回
message TaskListRes {
repeated Task tasks = 1; // 任务列表
}
// TargetListRes 策略列表返回
message TargetListRes {
map<string, Binds> target_list = 1; // 绑定列表
}
message Binds {
repeated Bind binds = 1;
}
// Bind 绑定信息
message Bind {
int64 bind_id = 1; //绑定的ID
BindType bind_type = 2; //绑定类型
string target_id = 3; //策略ID
int64 account_id = 4; //广告主ID
BindSourceType bind_source = 5; //绑定操作来源
}
// BindType 绑定类型
enum BindType {
UnknownBindType = 0;
AdgroupId = 1; //广告
AccountId = 3; //广告主
}
// BindSourceType 绑定操作来源
enum BindSourceType {
DefaultBindSourceType = 0; //广告主或未填写
ThirdPartyApi = 1; //第三方API
ADQ = 2; //ADQ平台
MP = 3; //MP平台
MktApi = 4; //MarketingAPI
}
// BindSetRes 设置绑定返回
message BindSetRes {
int32 success_num = 1; //成功数
int32 error_num = 2; //错误数
repeated BindError errors = 3; //绑定错误的记录
}
// BindDeleteRes 删除绑定返回
message BindDeleteRes {
int32 success_num = 1; //成功数
int32 error_num = 2; //错误数
repeated BindError errors = 3; //绑定错误的记录
}
// BindError 绑定错误信息
message BindError {
int64 bind_id = 1; //错误绑定的绑定ID
int32 bind_type = 2; //绑定类型
string reason = 3; //错误绑定原因
}
// ScriptRunRes 运行脚本返回
message ScriptRunRes {
string print_output = 1; // print输出
string error = 2; // 错误信息
string targets_output = 3; // 策略输出
}
// ScriptUpdateRes 升级脚本返回
message ScriptUpdateRes {
}
// ErrorCode 返回码
enum ErrorCode {
SUCC = 0; // 成功
@@ -204,6 +340,8 @@ enum ErrorCode {
DATA_ERROR = 201; // 数据错误
CMD_ERROR = 202; // 命令行执行错误
API_ERROR = 301; // 调用内部API错误
}
enum CmdErrorCode {
@@ -221,3 +359,9 @@ enum TaskStatus {
DELETED = 10; // 已删除,仅在执行删除成功时返回
}
enum OS {
UNKNOWN = 0;
IOS = 1;
ANDROID = 2;
}

View File

@@ -1,4 +1,5 @@
debug/
saastool
saastool_linux
cfg.toml
*.toml
test/

46
cmd/saastool/bind.go Normal file
View File

@@ -0,0 +1,46 @@
package main
import (
"fmt"
"os"
"strings"
)
func RunBind(args ...string) error {
name, args := ParseCommandName(args)
// 从参数中解析出命令
switch name {
case "", "help":
return RunBindHelp(args...)
case "setaccount":
return RunBindSetAccount(args...)
case "setad":
return RunBindSetAd(args...)
case "delete":
return RunBindDelete(args...)
default:
err := fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'saastool bind help' for usage`, name)
fmt.Fprintln(os.Stderr, err)
return err
}
}
func RunBindHelp(args ...string) error {
fmt.Println(strings.TrimSpace(bindUsage))
return nil
}
const bindUsage = `
Usage: saastoola bind COMMAND [OPTIONS]
Commands:
setaccount Set Account binds
setad Set AdGroup binds
delete Delete binds
"help" is the default command.
Use "saastool bind COMMAND -help" for more information about a command.
`

105
cmd/saastool/bind_delete.go Normal file
View File

@@ -0,0 +1,105 @@
package main
import (
"flag"
"fmt"
"net/http"
"os"
"strconv"
"strings"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type bindDeleteParams struct {
idType int
ids []int64
saasHttp *saashttp.SaasClient
}
func RunBindDelete(args ...string) error {
fs := flag.NewFlagSet("delete", flag.ExitOnError)
cfgFile := paramConfig(fs)
idtype := paramIdType(fs)
ids := paramIDs(fs)
if err := fs.Parse(args); err != nil {
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
return err
}
// 切割字符串
idsSlice := strings.Split(*ids, ",")
if len(idsSlice) == 1 && idsSlice[0] == "" {
idsSlice = []string{}
}
if fs.NArg() > 0 || len(idsSlice) == 0 || (len(idsSlice) == 1 && idsSlice[0] == "") {
fs.PrintDefaults()
return nil
}
numIds := []int64{}
for _, v := range idsSlice {
n, err := strconv.ParseInt(v, 10, 64)
if err != nil {
fmt.Fprintln(os.Stderr, "id parse error", "value", v, "err", err)
return err
}
numIds = append(numIds, n)
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
return err
}
bindDeleteParams := bindDeleteParams{
idType: *idtype,
ids: numIds,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}
return doBindDeleteAccount(bindDeleteParams)
}
func doBindDeleteAccount(p bindDeleteParams) error {
bindDelete := &saasapi.BindDelete{}
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_BindDelete{
BindDelete: bindDelete,
},
}
for _, v := range p.ids {
bindDelete.Binds = append(bindDelete.Binds, &saasapi.Bind{
BindId: v,
BindType: saasapi.BindType(p.idType),
})
}
res, err := p.saasHttp.BindDelete(saasReq)
if err != nil {
fmt.Fprintln(os.Stderr, "submit Bind Delete error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Fprintln(os.Stderr, "Bind Delete failed", "code", res.Code, "status", res.Status)
return nil
}
bindRes := res.GetBindDeleteRes()
fmt.Printf("bind res: %v\n", protojson.Format(bindRes))
return nil
}

View File

@@ -0,0 +1,107 @@
package main
import (
"flag"
"fmt"
"net/http"
"os"
"strconv"
"strings"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type bindSetAccountParams struct {
target string
accounts []int64
saasHttp *saashttp.SaasClient
}
func RunBindSetAccount(args ...string) error {
fs := flag.NewFlagSet("setaccount", flag.ExitOnError)
cfgFile := paramConfig(fs)
target := paramTarget(fs)
accounts := paramAccounts(fs)
if err := fs.Parse(args); err != nil {
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
return err
}
// 切割字符串
idsSlice := strings.Split(*accounts, ",")
if len(idsSlice) == 1 && idsSlice[0] == "" {
idsSlice = []string{}
}
if fs.NArg() > 0 || len(idsSlice) == 0 || (len(idsSlice) == 1 && idsSlice[0] == "") {
fs.PrintDefaults()
return nil
}
numAccounts := []int64{}
for _, v := range idsSlice {
n, err := strconv.ParseInt(v, 10, 64)
if err != nil {
fmt.Fprintln(os.Stderr, "account parse error", "value", v, "err", err)
return err
}
numAccounts = append(numAccounts, n)
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
return err
}
bindSetAccountParams := bindSetAccountParams{
target: *target,
accounts: numAccounts,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}
return doBindSetAccount(bindSetAccountParams)
}
func doBindSetAccount(p bindSetAccountParams) error {
bindSet := &saasapi.BindSet{}
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_BindSet{
BindSet: bindSet,
},
}
for _, v := range p.accounts {
bindSet.Binds = append(bindSet.Binds, &saasapi.Bind{
BindId: v,
BindType: saasapi.BindType_AccountId,
TargetId: p.target,
AccountId: v,
})
}
res, err := p.saasHttp.BindSet(saasReq)
if err != nil {
fmt.Fprintln(os.Stderr, "submit Bind Set error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Fprintln(os.Stderr, "Bind Set failed", "code", res.Code, "status", res.Status)
return nil
}
bindRes := res.GetBindSetRes()
fmt.Printf("bind res: %v\n", protojson.Format(bindRes))
return nil
}

110
cmd/saastool/bind_setad.go Normal file
View File

@@ -0,0 +1,110 @@
package main
import (
"flag"
"fmt"
"net/http"
"os"
"strconv"
"strings"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type bindSetAdParams struct {
target string
account int64
ads []int64
saasHttp *saashttp.SaasClient
}
func RunBindSetAd(args ...string) error {
fs := flag.NewFlagSet("setad", flag.ExitOnError)
cfgFile := paramConfig(fs)
target := paramTarget(fs)
account := paramAccount(fs)
ads := paramAds(fs)
if err := fs.Parse(args); err != nil {
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
return err
}
// 切割字符串
idsSlice := strings.Split(*ads, ",")
if len(idsSlice) == 1 && idsSlice[0] == "" {
idsSlice = []string{}
}
if fs.NArg() > 0 || len(idsSlice) == 0 || (len(idsSlice) == 1 && idsSlice[0] == "") {
fs.PrintDefaults()
return nil
}
numAds := []int64{}
for _, v := range idsSlice {
n, err := strconv.ParseInt(v, 10, 64)
if err != nil {
fmt.Fprintln(os.Stderr, "account parse error", "value", v, "err", err)
return err
}
numAds = append(numAds, n)
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
return err
}
bindSetAdParams := bindSetAdParams{
target: *target,
account: *account,
ads: numAds,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}
return doBindSetAd(bindSetAdParams)
}
func doBindSetAd(p bindSetAdParams) error {
bindSet := &saasapi.BindSet{}
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_BindSet{
BindSet: bindSet,
},
}
for _, v := range p.ads {
bindSet.Binds = append(bindSet.Binds, &saasapi.Bind{
BindId: v,
BindType: saasapi.BindType_AdgroupId,
TargetId: p.target,
AccountId: p.account,
})
}
res, err := p.saasHttp.BindSet(saasReq)
if err != nil {
fmt.Fprintln(os.Stderr, "submit Bind Set error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Fprintln(os.Stderr, "Bind Set failed", "code", res.Code, "status", res.Status)
return nil
}
bindRes := res.GetBindSetRes()
fmt.Printf("bind res: %v\n", protojson.Format(bindRes))
return nil
}

View File

@@ -15,6 +15,7 @@ const usage = `
Usage: saastool COMMAND [OPTIONS]
Commands:
info Saas Info
write Write user's 'bytes / uint32s / flags'
read Read user's 'bytes / uint32s / flags'
columnwrite Write columns for 'deviceid / openid' users
@@ -22,6 +23,8 @@ Commands:
convert Convert data to write format
task Task commands
target Target commands
bind Bind commands
"help" is the default command.

71
cmd/saastool/info.go Normal file
View File

@@ -0,0 +1,71 @@
package main
import (
"flag"
"fmt"
"net/http"
"os"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type infoParams struct {
saasHttp *saashttp.SaasClient
}
func RunInfo(args ...string) error {
fs := flag.NewFlagSet("info", flag.ExitOnError)
cfgFile := paramConfig(fs)
if err := fs.Parse(args); err != nil {
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
return err
}
if fs.NArg() > 0 {
fs.PrintDefaults()
return nil
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
return err
}
infoParams := infoParams{
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}
return doInfo(infoParams)
}
func doInfo(infoParams infoParams) error {
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_Info{
Info: &saasapi.Info{},
},
}
res, err := infoParams.saasHttp.Info(saasReq)
if err != nil {
fmt.Fprintln(os.Stderr, "submit Info error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Fprintln(os.Stderr, "info failed", "code", res.Code, "status", res.Status)
return nil
}
infoRes := res.GetInfoRes()
fmt.Printf("info res: %v\n", protojson.Format(infoRes))
return nil
}

View File

@@ -18,6 +18,8 @@ func Run(args ...string) error {
switch name {
case "", "help":
return RunHelp(args...)
case "info":
return RunInfo(args...)
case "write":
return RunWrite(args...)
case "read":
@@ -30,6 +32,10 @@ func Run(args ...string) error {
return RunVerify(args...)
case "task":
return RunTask(args...)
case "target":
return RunTarget(args...)
case "bind":
return RunBind(args...)
default:
err := fmt.Errorf(`unknown command "%v"`+"\n"+`Run 'saastool help' for usage`, name)
fmt.Fprintln(os.Stderr, err.Error())

View File

@@ -62,7 +62,7 @@ func paramBatchSize(fs *flag.FlagSet) *uint {
}
func paramBlockSize(fs *flag.FlagSet) *string {
return fs.String("blocksize", "200M", "Block size to make hash. using size mode K, M, G, T")
return fs.String("blocksize", "50M", "Block size to make hash. using size mode K, M, G, T")
}
func paramClear(fs *flag.FlagSet) *bool {
@@ -73,6 +73,38 @@ func paramDataSpaceId(fs *flag.FlagSet) *string {
return fs.String("ds", "", "Data space ID")
}
func paramTargets(fs *flag.FlagSet) *string {
return fs.String("targets", "", "Target IDs. Use commas to separate multiple IDs")
}
func paramListBinds(fs *flag.FlagSet) *bool {
return fs.Bool("b", false, "List Binds")
}
func paramTarget(fs *flag.FlagSet) *string {
return fs.String("target", "", "Target ID")
}
func paramAccount(fs *flag.FlagSet) *int64 {
return fs.Int64("account", 0, "Advertiser ID")
}
func paramAccounts(fs *flag.FlagSet) *string {
return fs.String("accounts", "", "Advertiser IDs. Use commas to separate multiple IDs")
}
func paramAds(fs *flag.FlagSet) *string {
return fs.String("ads", "", "AdGroup IDs. Use commas to separate multiple IDs")
}
func paramIdType(fs *flag.FlagSet) *int {
return fs.Int("idtype", 0, "ID Type. empty is Automatic matching, 1=AdGroup, 3=Account")
}
func paramIDs(fs *flag.FlagSet) *string {
return fs.String("ids", "", "IDs for delete. Use commas to separate multiple IDs")
}
// ParseByteSize 解析字节大小字符串为字节数
func ParseByteSize(sizeStr string) (uint64, error) {
sizeStr = strings.TrimSpace(sizeStr)

View File

@@ -62,7 +62,7 @@ func RunRead(args ...string) error {
ds: *ds,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}

40
cmd/saastool/target.go Normal file
View File

@@ -0,0 +1,40 @@
package main
import (
"fmt"
"os"
"strings"
)
func RunTarget(args ...string) error {
name, args := ParseCommandName(args)
// 从参数中解析出命令
switch name {
case "", "help":
return RunTargetHelp(args...)
case "list":
return RunTargetList(args...)
default:
err := fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'saastool target help' for usage`, name)
fmt.Fprintln(os.Stderr, err)
return err
}
}
func RunTargetHelp(args ...string) error {
fmt.Println(strings.TrimSpace(targetUsage))
return nil
}
const targetUsage = `
Usage: saastoola target COMMAND [OPTIONS]
Commands:
list List targets
"help" is the default command.
Use "saastool target COMMAND -help" for more information about a command.
`

View File

@@ -0,0 +1,88 @@
package main
import (
"flag"
"fmt"
"net/http"
"os"
"strings"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
"google.golang.org/protobuf/encoding/protojson"
)
type listTargetParams struct {
targets []string
listBinds bool
saasHttp *saashttp.SaasClient
}
func RunTargetList(args ...string) error {
fs := flag.NewFlagSet("list", flag.ExitOnError)
cfgFile := paramConfig(fs)
targets := paramTargets(fs)
listBinds := paramListBinds(fs)
if err := fs.Parse(args); err != nil {
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
return err
}
if fs.NArg() > 0 {
fs.PrintDefaults()
return nil
}
// 切割字符串
idsSlice := strings.Split(*targets, ",")
if len(idsSlice) == 1 && idsSlice[0] == "" {
idsSlice = []string{}
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
return err
}
listTargetParams := listTargetParams{
listBinds: *listBinds,
targets: idsSlice,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}
return doTargetList(listTargetParams)
}
func doTargetList(listTargetParams listTargetParams) error {
saasReq := &saasapi.SaasReq{
Cmd: &saasapi.SaasReq_TargetList{
TargetList: &saasapi.TargetList{
Targets: listTargetParams.targets,
ListBind: listTargetParams.listBinds,
},
},
}
res, err := listTargetParams.saasHttp.TargetList(saasReq)
if err != nil {
fmt.Fprintln(os.Stderr, "submit List Target error", "err", err)
return err
}
if res.Code != saasapi.ErrorCode_SUCC {
fmt.Fprintln(os.Stderr, "Target list failed", "code", res.Code, "status", res.Status)
return nil
}
targetRes := res.GetTargetListRes()
fmt.Printf("target res: %v\n", protojson.Format(targetRes))
return nil
}

View File

@@ -25,6 +25,8 @@ func RunTask(args ...string) error {
return RunTaskInfo(args...)
case "upload":
return RunTaskUpload(args...)
case "download":
return RunTaskDownload(args...)
case "run":
return RunTaskRun(args...)
default:
@@ -51,6 +53,7 @@ Commands:
delete Delete a task on server
info Get a task info on server
upload Upload task's file block to server
download Download task's file block to local
"help" is the default command.

View File

@@ -42,7 +42,7 @@ func RunTaskCreate(args ...string) error {
hashFile: *hashFile,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
task: &saasapi.Task{},

View File

@@ -41,7 +41,7 @@ func RunTaskDelete(args ...string) error {
taskSha256: *sha256,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}

View File

@@ -0,0 +1,134 @@
package main
import (
"flag"
"fmt"
"net/http"
"os"
"path/filepath"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
)
type downloadTaskParams struct {
taskSha256 string
destPath string
saasHttp *saashttp.SaasClient
}
func RunTaskDownload(args ...string) error {
fs := flag.NewFlagSet("download", flag.ExitOnError)
cfgFile := paramConfig(fs)
sha256 := paramSha256(fs)
destPath := paramDestPath(fs)
if err := fs.Parse(args); err != nil {
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
return err
}
if fs.NArg() > 0 || len(*sha256) == 0 || len(*destPath) == 0 {
fs.PrintDefaults()
return nil
}
cfg, err := LoadConfigFile(*cfgFile)
if err != nil {
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
return err
}
downloadTaskParams := downloadTaskParams{
taskSha256: *sha256,
destPath: *destPath,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}
return doTaskDownload(downloadTaskParams)
}
func doTaskDownload(downloadTaskParams downloadTaskParams) error {
infoTaskParams := infoTaskParams{
taskSha256: downloadTaskParams.taskSha256,
saasHttp: downloadTaskParams.saasHttp,
}
taskInfo, err := doTaskInfo(infoTaskParams)
if err != nil {
return err
}
if len(taskInfo.GetSourcePath()) == 0 {
err = fmt.Errorf("task download failed. task info source path is empty")
fmt.Fprintln(os.Stderr, err)
return err
}
totalFiles := len(taskInfo.GetTaskFileInfos())
fi := 0
for _, finfo := range taskInfo.GetTaskFileInfos() {
fi++
var f *os.File
offset := int64(0)
totalBlocks := len(finfo.GetFileBlocks())
bi := 0
for _, binfo := range finfo.GetFileBlocks() {
bi++
if binfo.GetUploaded() {
if f == nil {
fname := finfo.GetFileName()
if len(downloadTaskParams.destPath) > 0 {
fname = filepath.Join(downloadTaskParams.destPath, fname)
}
os.MkdirAll(filepath.Dir(fname), os.ModePerm)
f, err = os.Create(fname)
if err != nil {
return err
}
}
blockRes, err := downloadTaskParams.saasHttp.TaskDownload(
binfo.GetBlockSha256(),
f,
offset,
int(binfo.GetBlockLength()),
)
if err != nil {
return err
}
if blockRes.GetCode() != saasapi.ErrorCode_SUCC {
err = fmt.Errorf("download block error, code %d, msg %s", blockRes.GetCode(), blockRes.GetStatus())
fmt.Fprintln(os.Stderr, err)
return err
} else {
fmt.Printf("download block success. file: %v, sha256 %v. block %v/%v, file %v/%v\n",
finfo.GetFileName(), binfo.GetBlockSha256(),
bi, totalBlocks, fi, totalFiles,
)
}
} else {
fmt.Printf("download block. file: %v, sha256 %v. block %v/%v, file %v/%v\n",
finfo.GetFileName(), binfo.GetBlockSha256(),
bi, totalBlocks, fi, totalFiles,
)
}
offset += int64(binfo.GetBlockLength())
}
if f != nil {
f.Close()
}
}
return nil
}

View File

@@ -41,7 +41,7 @@ func RunTaskInfo(args ...string) error {
taskSha256: *sha256,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}

View File

@@ -40,7 +40,7 @@ func RunTaskList(args ...string) error {
listTaskParams := listTaskParams{
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}

View File

@@ -7,6 +7,7 @@ import (
"fmt"
"os"
"path"
"path/filepath"
"runtime"
"sort"
"strings"
@@ -81,7 +82,7 @@ func RunTaskMake(args ...string) error {
return nil
}
makeTaskParams := makeTaskParams{
makeTaskParams := &makeTaskParams{
sourcePath: *sourcePath,
hashFile: *hashFile,
task: &saasapi.Task{
@@ -89,12 +90,13 @@ func RunTaskMake(args ...string) error {
TaskDescription: *desc,
Appid: *appid,
DataspaceId: *ds,
SourcePath: *sourcePath,
},
}
return doMakeHash(makeTaskParams)
return doMakeHash(makeTaskParams, true)
}
func doMakeHash(makeTaskParams makeTaskParams) error {
func doMakeHash(makeTaskParams *makeTaskParams, firstLevel bool) error {
fsInfo, err := os.Stat(makeTaskParams.sourcePath)
if err != nil {
return err
@@ -102,9 +104,14 @@ func doMakeHash(makeTaskParams makeTaskParams) error {
if !fsInfo.IsDir() {
// 如果是文件,直接计算
return doTaskMake(makeTaskParams)
if firstLevel {
makeTaskParams.task.SourcePath = filepath.Dir(makeTaskParams.sourcePath)
}
err = doTaskMake(makeTaskParams)
if err != nil {
return err
}
} else {
// 读取目录下信息
dirEntry, err := os.ReadDir(makeTaskParams.sourcePath)
if err != nil {
@@ -113,18 +120,23 @@ func doMakeHash(makeTaskParams makeTaskParams) error {
// 遍历目录
for _, dir := range dirEntry {
newParam := makeTaskParams
newParam.sourcePath = path.Join(makeTaskParams.sourcePath, dir.Name())
oldSourcePath := makeTaskParams.sourcePath
makeTaskParams.sourcePath = path.Join(makeTaskParams.sourcePath, dir.Name())
if err = doMakeHash(newParam); err != nil {
if err = doMakeHash(makeTaskParams, false); err != nil {
return err
}
makeTaskParams.sourcePath = oldSourcePath
}
}
if firstLevel {
return saveTaskFile(makeTaskParams)
}
return nil
}
func doTaskMake(makeTaskParams makeTaskParams) error {
func doTaskMake(makeTaskParams *makeTaskParams) error {
sourceFile, err := os.Open(makeTaskParams.sourcePath)
if err != nil {
return err
@@ -200,10 +212,12 @@ func doTaskMake(makeTaskParams makeTaskParams) error {
})
// 输出结果
relPath, _ := filepath.Rel(makeTaskParams.task.GetSourcePath(), makeTaskParams.sourcePath)
fileInfo := &saasapi.FileInfo{
FileName: makeTaskParams.sourcePath,
FileName: relPath,
FileSize: uint64(fi.Size()),
}
for _, r := range allResults {
fileInfo.FileBlocks = append(fileInfo.FileBlocks, &saasapi.FileBlock{
BlockSha256: r.hash,
@@ -212,6 +226,8 @@ func doTaskMake(makeTaskParams makeTaskParams) error {
}
makeTaskParams.task.TaskFileInfos = append(makeTaskParams.task.TaskFileInfos, fileInfo)
makeTaskParams.task.TaskSize += uint64(fi.Size())
fmt.Println("")
return nil
}
@@ -227,7 +243,7 @@ func hashWorker(tasks <-chan *hashTask, results chan<- *hashTask) {
}
}
func saveTaskFile(makeTaskParams makeTaskParams) error {
func saveTaskFile(makeTaskParams *makeTaskParams) error {
taskFile, err := os.Create(makeTaskParams.hashFile)
if err != nil {
return err

View File

@@ -41,7 +41,7 @@ func RunTaskRun(args ...string) error {
taskSha256: *sha256,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"net/http"
"os"
"path/filepath"
"e.coding.net/rta/public/saasapi"
"e.coding.net/rta/public/saasapi/pkg/saashttp"
@@ -40,7 +41,7 @@ func RunTaskUpload(args ...string) error {
taskSha256: *sha256,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}
@@ -58,10 +59,10 @@ func doTaskUpload(uploadTaskParams uploadTaskParams) error {
return err
}
sourcePath := taskInfo.GetSourcePath()
totalFiles := len(taskInfo.GetTaskFileInfos())
fi := 0
for _, finfo := range taskInfo.GetTaskFileInfos() {
fi++
var f *os.File
offset := int64(0)
@@ -71,7 +72,11 @@ func doTaskUpload(uploadTaskParams uploadTaskParams) error {
bi++
if !binfo.GetUploaded() {
if f == nil {
f, err = os.Open(finfo.GetFileName())
fname := finfo.GetFileName()
if len(sourcePath) > 0 {
fname = filepath.Join(sourcePath, fname)
}
f, err = os.Open(fname)
if err != nil {
return err
}
@@ -89,7 +94,9 @@ func doTaskUpload(uploadTaskParams uploadTaskParams) error {
}
if blockRes.GetCode() != saasapi.ErrorCode_SUCC {
return fmt.Errorf("upload block error, code %d, msg %s", blockRes.GetCode(), blockRes.GetStatus())
err = fmt.Errorf("upload block error, code %d, msg %s", blockRes.GetCode(), blockRes.GetStatus())
fmt.Fprintln(os.Stderr, err)
return err
} else {
fmt.Printf("upload block success. file: %v, sha256 %v. block %v/%v, file %v/%v\n",
finfo.GetFileName(), binfo.GetBlockSha256(),

View File

@@ -63,7 +63,7 @@ func RunWrite(args ...string) error {
clear: *clear,
saasHttp: &saashttp.SaasClient{
Client: &http.Client{},
ApiUrls: &cfg.ApiUrls,
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
Auth: &cfg.Auth,
},
}

View File

@@ -0,0 +1,12 @@
FROM rta-docker.pkg.coding.net/public/docker/entre_dev:latest AS builder
COPY . /tmp/saasapi/
WORKDIR /tmp/saasapi/
RUN --mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/go/pkg \
./make.sh
FROM rta-docker.pkg.coding.net/public/docker/alpine:latest AS product
COPY --from=builder /tmp/saasapi/build/saastool /bin/saastool

3
make.sh Executable file
View File

@@ -0,0 +1,3 @@
cd cmd/saastool
go build -o ../../build/saastool
cd ../../

View File

@@ -1,7 +1,27 @@
package saashttp
const (
baseUrl = "https://api.rta.qq.com"
infoPath = "/saas/info"
writePath = "/saas/write"
readPath = "/saas/read"
columnWritePath = "/saas/column_write"
taskCreatePath = "/saas/task/create"
taskListPath = "/saas/task/list"
taskCancelPath = "/saas/task/delete"
taskInfoPath = "/saas/task/info"
taskUploadPath = "/saas/task/upload"
taskDownloadPath = "/saas/task/download"
taskDeletePath = "/saas/task/delete"
taskRunPath = "/saas/task/run"
targetListPath = "/saas/target/list"
bindSetPath = "/saas/bind/set"
bindDeletePath = "/saas/bind/delete"
)
type ApiUrls struct {
BaseUrl string
InfoPath string
WritePath string
ReadPath string
ColumnWritePath string
@@ -11,6 +31,105 @@ type ApiUrls struct {
TaskDeletePath string
TaskRunPath string
TaskUploadPath string
TaskDownloadPath string
TargetListPath string
BindSetPath string
BindDeletePath string
}
func InitAPIUrl(c *ApiUrls) *ApiUrls {
r := &ApiUrls{}
if c.BaseUrl != "" {
r.BaseUrl = c.BaseUrl
} else {
r.BaseUrl = baseUrl
}
if c.InfoPath != "" {
r.InfoPath = c.InfoPath
} else {
r.InfoPath = infoPath
}
if c.WritePath != "" {
r.WritePath = c.WritePath
} else {
r.WritePath = writePath
}
if c.ReadPath != "" {
r.ReadPath = c.ReadPath
} else {
r.ReadPath = readPath
}
if c.ColumnWritePath != "" {
r.ColumnWritePath = c.ColumnWritePath
} else {
r.ColumnWritePath = columnWritePath
}
if c.TaskCreatePath != "" {
r.TaskCreatePath = c.TaskCreatePath
} else {
r.TaskCreatePath = taskCreatePath
}
if c.TaskListPath != "" {
r.TaskListPath = c.TaskListPath
} else {
r.TaskListPath = taskListPath
}
if c.TaskInfoPath != "" {
r.TaskInfoPath = c.TaskInfoPath
} else {
r.TaskInfoPath = taskInfoPath
}
if c.TaskDeletePath != "" {
r.TaskDeletePath = c.TaskDeletePath
} else {
r.TaskDeletePath = taskDeletePath
}
if c.TaskRunPath != "" {
r.TaskRunPath = c.TaskRunPath
} else {
r.TaskRunPath = taskRunPath
}
if c.TaskUploadPath != "" {
r.TaskUploadPath = c.TaskUploadPath
} else {
r.TaskUploadPath = taskUploadPath
}
if c.TaskDownloadPath != "" {
r.TaskDownloadPath = c.TaskDownloadPath
} else {
r.TaskDownloadPath = taskDownloadPath
}
if c.TargetListPath != "" {
r.TargetListPath = c.TargetListPath
} else {
r.TargetListPath = targetListPath
}
if c.BindSetPath != "" {
r.BindSetPath = c.BindSetPath
} else {
r.BindSetPath = bindSetPath
}
if c.BindDeletePath != "" {
r.BindDeletePath = c.BindDeletePath
} else {
r.BindDeletePath = bindDeletePath
}
return r
}
type Auth struct {

View File

@@ -12,6 +12,7 @@ import (
"net/url"
"os"
"strconv"
"strings"
"time"
"e.coding.net/rta/public/saasapi"
@@ -32,6 +33,11 @@ type SaasClient struct {
ResponseEncoder ResponseEncoder
}
func (c *SaasClient) Info(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.InfoPath)
return c.post(postUrl, saasReq)
}
func (c *SaasClient) Write(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.WritePath)
return c.post(postUrl, saasReq)
@@ -77,6 +83,26 @@ func (c *SaasClient) TaskUpload(sha256 string, file *os.File, offset int64, size
return c.upload(postUrl, file, offset, size)
}
func (c *SaasClient) TaskDownload(sha256 string, file *os.File, offset int64, size int) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskDownloadPath, "sha256", sha256)
return c.download(postUrl, file, offset, size)
}
func (c *SaasClient) TargetList(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TargetListPath)
return c.post(postUrl, saasReq)
}
func (c *SaasClient) BindSet(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.BindSetPath)
return c.post(postUrl, saasReq)
}
func (c *SaasClient) BindDelete(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.BindDeletePath)
return c.post(postUrl, saasReq)
}
func (c *SaasClient) makeUrl(baseUrl, path string, params ...string) string {
url, err := url.Parse(baseUrl)
if err != nil {
@@ -232,3 +258,86 @@ func (c *SaasClient) upload(url string, file *os.File, offset int64, size int) (
return saasRes, nil
}
func (c *SaasClient) download(url string, file *os.File, offset int64, size int) (saasRes *saasapi.SaasRes, err error) {
if file == nil {
return nil, fmt.Errorf("file is nil")
}
if size <= 0 {
return nil, fmt.Errorf("size is invalid")
}
req, err := http.NewRequest("GET", url, bytes.NewBuffer([]byte{}))
if err != nil {
fmt.Println("http new request error", err)
return nil, err
}
timeStamp := strconv.FormatInt(time.Now().Unix(), 10)
md5byte := md5.Sum([]byte(c.Auth.Account + c.Auth.Token + timeStamp))
authorization := hex.EncodeToString(md5byte[:])
req.Header.Add("Account", c.Auth.Account)
req.Header.Add("Time", timeStamp)
req.Header.Add("Authorization", authorization)
req.Header.Add("Accept-Encoding", "gzip")
res, err := c.Client.Do(req)
if err != nil {
fmt.Println("http send error", err)
return nil, err
}
defer res.Body.Close()
if res.StatusCode != 200 {
err = fmt.Errorf("NOT 200. %v", res.StatusCode)
fmt.Println("http state error", err)
return nil, err
}
bodyReader := res.Body
if strings.Contains(res.Header.Get("Content-Encoding"), "gzip") {
gz, err := gzip.NewReader(res.Body)
if err != nil {
fmt.Println("gzip newreader error", err)
return nil, err
}
defer gz.Close()
bodyReader = gz
}
resBody, err := io.ReadAll(bodyReader)
if err != nil {
fmt.Println("read body error", err)
return nil, err
}
saasRes = &saasapi.SaasRes{}
if strings.Contains(res.Header.Get("Content-Type"), "application/octet-stream") {
if len(resBody) == size {
file.WriteAt(resBody, offset)
} else {
err = fmt.Errorf("body size error. body:%v, want:%v", len(resBody), size)
fmt.Println("http read body error", err)
return nil, err
}
} else {
if c.ResponseEncoder == RESPONSE_ENCODER_PROTOBUF {
err = proto.Unmarshal(resBody, saasRes)
if err != nil {
fmt.Println("unmarshal response body to protobuf error", err)
return nil, err
}
} else {
err = json.Unmarshal(resBody, saasRes)
if err != nil {
fmt.Println("unmarshal response body to json error", err)
return nil, err
}
}
}
return saasRes, nil
}