Compare commits
7 Commits
b8c2a3a7d4
...
095a0b9b01
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
095a0b9b01 | ||
|
|
80a758f1e3 | ||
|
|
edb12c3b1f | ||
|
|
6ca9fe7a02 | ||
|
|
a78c16d301 | ||
|
|
a2bf3c853e | ||
|
|
d1ad148725 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,2 +1,3 @@
|
||||
vendor/
|
||||
*.out
|
||||
build/
|
||||
152
cmd.proto
152
cmd.proto
@@ -7,6 +7,8 @@ option go_package = "e.coding.net/rta/public/saasapi";
|
||||
// SaasReq 命令请求
|
||||
message SaasReq {
|
||||
oneof cmd {
|
||||
Info info = 5; // 获取账号设置
|
||||
|
||||
Read read = 10; // 批量读取
|
||||
Write write = 11; // 批量写入
|
||||
ColumnWrite column_write = 12; // 全量列式写入
|
||||
@@ -16,9 +18,22 @@ message SaasReq {
|
||||
TaskRun task_run = 22; // 执行任务
|
||||
TaskDelete task_delete = 23; // 删除任务
|
||||
TaskInfo task_info = 24; // 任务详情
|
||||
|
||||
TargetList target_list = 50; // 列出策略及绑定
|
||||
|
||||
BindSet bind_set = 61; // 设置绑定
|
||||
BindDelete bind_delete = 62; // 解除绑定
|
||||
|
||||
ScriptRun script_run = 90; // 运行脚本
|
||||
ScriptUpdate script_update = 91; // 脚本升级
|
||||
}
|
||||
}
|
||||
|
||||
// Info 获取账号信息
|
||||
message Info {
|
||||
|
||||
}
|
||||
|
||||
// Read 批量读取命令
|
||||
message Read {
|
||||
string dataspace_id = 1; // 数据空间ID
|
||||
@@ -89,7 +104,9 @@ message Task {
|
||||
string task_sha256 = 3; // 任务sha256
|
||||
string task_description = 4; // 任务描述
|
||||
repeated FileInfo task_file_infos = 5; // 文件列表
|
||||
uint64 task_block_size = 6; // 文件块字节大小(推荐200M)
|
||||
uint64 task_block_size = 6; // 文件块字节大小(推荐50M)
|
||||
string source_path = 7; // 任务数据源路径
|
||||
uint64 task_size = 8; // 任务所有文件的总大小
|
||||
|
||||
// 以下字段只在返回时填写,用于提供服务端的任务状态。在请求时填写会被忽略
|
||||
string create_time = 10; // 创建时间
|
||||
@@ -119,23 +136,56 @@ message TaskInfo {
|
||||
string task_sha256 = 1; // 任务sha256
|
||||
}
|
||||
|
||||
// FileInfo 任务文件信息
|
||||
message FileInfo {
|
||||
string file_name = 1; // 文件名
|
||||
uint64 file_size = 2; // 文件大小
|
||||
repeated FileBlock file_blocks = 3; // 文件块列表
|
||||
}
|
||||
|
||||
// FileBlock 文件块信息
|
||||
message FileBlock {
|
||||
string block_sha256 = 1; // 块的sha256
|
||||
uint64 block_length = 2; // 块的字节长度
|
||||
bool uploaded = 3; // 是否已上传(在TaskCreate/TaskInfo请求返回)
|
||||
}
|
||||
|
||||
// TargetList 列出策略
|
||||
message TargetList {
|
||||
repeated string targets = 1; // 指定要列出的绑定的策略列表,如不指定则返回全部
|
||||
bool list_bind = 2; // 是否同时列出绑定信息
|
||||
}
|
||||
|
||||
// BindSet 设置绑定
|
||||
message BindSet {
|
||||
repeated Bind binds = 2; // 设置绑定内容
|
||||
}
|
||||
|
||||
// BindDelete 删除绑定
|
||||
message BindDelete {
|
||||
repeated Bind binds = 2; // 解除绑定内容
|
||||
}
|
||||
|
||||
// ScriptRun 运行脚本
|
||||
message ScriptRun {
|
||||
string lua_script = 1; // 要调试的lua脚本
|
||||
string server_did = 2; // 将从服务端读取该DID下的数据
|
||||
string appid = 3; // 小程序/小游戏/公众号/视频号的appid
|
||||
string server_openid = 4; // 将从服务端读取该openid下的数据,需与appid配对使用
|
||||
OS os = 5; // 操作系统
|
||||
}
|
||||
|
||||
// ScriptUpdate 升级脚本
|
||||
message ScriptUpdate {
|
||||
}
|
||||
|
||||
// SaasRes 命令返回
|
||||
message SaasRes {
|
||||
ErrorCode code = 1; // 返回码
|
||||
string status = 2; // 返回信息的文本提示
|
||||
oneof res {
|
||||
InfoRes info_res = 5; // 账号信息返回
|
||||
|
||||
ReadRes read_res = 10; // 读取命令返回
|
||||
WriteRes write_res = 11; // 写入命令返回
|
||||
|
||||
@@ -144,18 +194,35 @@ message SaasRes {
|
||||
Task task_run_res = 22; // 运行任务返回状态
|
||||
Task task_delete_res = 23; // 删除任务返回状态
|
||||
Task task_info_res = 24; // 任务详情返回状态
|
||||
|
||||
TargetListRes target_list_res = 50; // 列出策略及绑定返回状态
|
||||
|
||||
BindSetRes bind_set_res = 61; // 设置绑定返回状态
|
||||
BindDeleteRes bind_delete_res = 62; // 删除绑定返回状态
|
||||
|
||||
ScriptRunRes script_run_res = 90; // 运行脚本返回
|
||||
ScriptUpdateRes script_update_res = 91; // 升级脚本返回
|
||||
}
|
||||
}
|
||||
|
||||
// InfoRes 账号信息返回
|
||||
message InfoRes {
|
||||
repeated string dataspace_id = 1; // 可用数据区列表
|
||||
repeated string target_id = 2; // 策略ID列表
|
||||
|
||||
}
|
||||
|
||||
// ReadRes 读记录返回
|
||||
message ReadRes {
|
||||
uint32 succ_cmd_count = 1; // 成功的命令数量
|
||||
uint32 fail_cmd_count = 2; // 失败的命令数量
|
||||
repeated ValueItem cmd_res = 3; // 返回的命令
|
||||
}
|
||||
|
||||
// WriteRes 写记录返回
|
||||
message WriteRes {
|
||||
//uint32 succ_cmd_count = 1; // 成功的命令数量
|
||||
//uint32 fail_cmd_count = 2; // 失败的命令数量
|
||||
//uint32 succ_cmd_count = 1; // 成功的命令数量
|
||||
//uint32 fail_cmd_count = 2; // 失败的命令数量
|
||||
repeated string failed_userid = 3; // 返回的失败的用户ID
|
||||
}
|
||||
|
||||
@@ -169,10 +236,79 @@ message ValueItem {
|
||||
uint32 last_modify_time = 6; // 最后修改时间
|
||||
}
|
||||
|
||||
// TaskListRes 任务列表返回
|
||||
message TaskListRes {
|
||||
repeated Task tasks = 1; // 任务列表
|
||||
}
|
||||
|
||||
// TargetListRes 策略列表返回
|
||||
message TargetListRes {
|
||||
map<string, Binds> target_list = 1; // 绑定列表
|
||||
}
|
||||
|
||||
message Binds {
|
||||
repeated Bind binds = 1;
|
||||
}
|
||||
|
||||
// Bind 绑定信息
|
||||
message Bind {
|
||||
int64 bind_id = 1; //绑定的ID
|
||||
BindType bind_type = 2; //绑定类型
|
||||
string target_id = 3; //策略ID
|
||||
int64 account_id = 4; //广告主ID
|
||||
BindSourceType bind_source = 5; //绑定操作来源
|
||||
}
|
||||
|
||||
|
||||
// BindType 绑定类型
|
||||
enum BindType {
|
||||
UnknownBindType = 0;
|
||||
AdgroupId = 1; //广告
|
||||
AccountId = 3; //广告主
|
||||
}
|
||||
|
||||
// BindSourceType 绑定操作来源
|
||||
enum BindSourceType {
|
||||
DefaultBindSourceType = 0; //广告主或未填写
|
||||
ThirdPartyApi = 1; //第三方API
|
||||
ADQ = 2; //ADQ平台
|
||||
MP = 3; //MP平台
|
||||
MktApi = 4; //MarketingAPI
|
||||
}
|
||||
|
||||
// BindSetRes 设置绑定返回
|
||||
message BindSetRes {
|
||||
int32 success_num = 1; //成功数
|
||||
int32 error_num = 2; //错误数
|
||||
repeated BindError errors = 3; //绑定错误的记录
|
||||
}
|
||||
|
||||
// BindDeleteRes 删除绑定返回
|
||||
message BindDeleteRes {
|
||||
int32 success_num = 1; //成功数
|
||||
int32 error_num = 2; //错误数
|
||||
repeated BindError errors = 3; //绑定错误的记录
|
||||
}
|
||||
|
||||
// BindError 绑定错误信息
|
||||
message BindError {
|
||||
int64 bind_id = 1; //错误绑定的绑定ID
|
||||
int32 bind_type = 2; //绑定类型
|
||||
string reason = 3; //错误绑定原因
|
||||
}
|
||||
|
||||
|
||||
// ScriptRunRes 运行脚本返回
|
||||
message ScriptRunRes {
|
||||
string print_output = 1; // print输出
|
||||
string error = 2; // 错误信息
|
||||
string targets_output = 3; // 策略输出
|
||||
}
|
||||
|
||||
// ScriptUpdateRes 升级脚本返回
|
||||
message ScriptUpdateRes {
|
||||
}
|
||||
|
||||
// ErrorCode 返回码
|
||||
enum ErrorCode {
|
||||
SUCC = 0; // 成功
|
||||
@@ -204,6 +340,8 @@ enum ErrorCode {
|
||||
|
||||
DATA_ERROR = 201; // 数据错误
|
||||
CMD_ERROR = 202; // 命令行执行错误
|
||||
|
||||
API_ERROR = 301; // 调用内部API错误
|
||||
}
|
||||
|
||||
enum CmdErrorCode {
|
||||
@@ -218,6 +356,12 @@ enum TaskStatus {
|
||||
SUCCESS = 4; // 成功
|
||||
FAIL = 5; // 失败
|
||||
|
||||
DELETED = 10; // 已删除,仅在执行删除成功时返回
|
||||
DELETED = 10; // 已删除,仅在执行删除成功时返回
|
||||
}
|
||||
|
||||
enum OS {
|
||||
UNKNOWN = 0;
|
||||
IOS = 1;
|
||||
ANDROID = 2;
|
||||
}
|
||||
|
||||
|
||||
3
cmd/saastool/.gitignore
vendored
3
cmd/saastool/.gitignore
vendored
@@ -1,4 +1,5 @@
|
||||
debug/
|
||||
saastool
|
||||
saastool_linux
|
||||
cfg.toml
|
||||
*.toml
|
||||
test/
|
||||
46
cmd/saastool/bind.go
Normal file
46
cmd/saastool/bind.go
Normal file
@@ -0,0 +1,46 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func RunBind(args ...string) error {
|
||||
name, args := ParseCommandName(args)
|
||||
|
||||
// 从参数中解析出命令
|
||||
switch name {
|
||||
case "", "help":
|
||||
return RunBindHelp(args...)
|
||||
case "setaccount":
|
||||
return RunBindSetAccount(args...)
|
||||
case "setad":
|
||||
return RunBindSetAd(args...)
|
||||
case "delete":
|
||||
return RunBindDelete(args...)
|
||||
default:
|
||||
err := fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'saastool bind help' for usage`, name)
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func RunBindHelp(args ...string) error {
|
||||
fmt.Println(strings.TrimSpace(bindUsage))
|
||||
return nil
|
||||
}
|
||||
|
||||
const bindUsage = `
|
||||
Usage: saastoola bind COMMAND [OPTIONS]
|
||||
|
||||
Commands:
|
||||
setaccount Set Account binds
|
||||
setad Set AdGroup binds
|
||||
delete Delete binds
|
||||
|
||||
"help" is the default command.
|
||||
|
||||
Use "saastool bind COMMAND -help" for more information about a command.
|
||||
`
|
||||
105
cmd/saastool/bind_delete.go
Normal file
105
cmd/saastool/bind_delete.go
Normal file
@@ -0,0 +1,105 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"e.coding.net/rta/public/saasapi"
|
||||
"e.coding.net/rta/public/saasapi/pkg/saashttp"
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
)
|
||||
|
||||
type bindDeleteParams struct {
|
||||
idType int
|
||||
ids []int64
|
||||
saasHttp *saashttp.SaasClient
|
||||
}
|
||||
|
||||
func RunBindDelete(args ...string) error {
|
||||
fs := flag.NewFlagSet("delete", flag.ExitOnError)
|
||||
cfgFile := paramConfig(fs)
|
||||
idtype := paramIdType(fs)
|
||||
ids := paramIDs(fs)
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// 切割字符串
|
||||
idsSlice := strings.Split(*ids, ",")
|
||||
if len(idsSlice) == 1 && idsSlice[0] == "" {
|
||||
idsSlice = []string{}
|
||||
}
|
||||
|
||||
if fs.NArg() > 0 || len(idsSlice) == 0 || (len(idsSlice) == 1 && idsSlice[0] == "") {
|
||||
fs.PrintDefaults()
|
||||
return nil
|
||||
}
|
||||
|
||||
numIds := []int64{}
|
||||
for _, v := range idsSlice {
|
||||
n, err := strconv.ParseInt(v, 10, 64)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "id parse error", "value", v, "err", err)
|
||||
return err
|
||||
}
|
||||
numIds = append(numIds, n)
|
||||
}
|
||||
|
||||
cfg, err := LoadConfigFile(*cfgFile)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
bindDeleteParams := bindDeleteParams{
|
||||
idType: *idtype,
|
||||
ids: numIds,
|
||||
saasHttp: &saashttp.SaasClient{
|
||||
Client: &http.Client{},
|
||||
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
|
||||
Auth: &cfg.Auth,
|
||||
},
|
||||
}
|
||||
|
||||
return doBindDeleteAccount(bindDeleteParams)
|
||||
}
|
||||
|
||||
func doBindDeleteAccount(p bindDeleteParams) error {
|
||||
bindDelete := &saasapi.BindDelete{}
|
||||
|
||||
saasReq := &saasapi.SaasReq{
|
||||
Cmd: &saasapi.SaasReq_BindDelete{
|
||||
BindDelete: bindDelete,
|
||||
},
|
||||
}
|
||||
|
||||
for _, v := range p.ids {
|
||||
bindDelete.Binds = append(bindDelete.Binds, &saasapi.Bind{
|
||||
BindId: v,
|
||||
BindType: saasapi.BindType(p.idType),
|
||||
})
|
||||
}
|
||||
|
||||
res, err := p.saasHttp.BindDelete(saasReq)
|
||||
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "submit Bind Delete error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if res.Code != saasapi.ErrorCode_SUCC {
|
||||
fmt.Fprintln(os.Stderr, "Bind Delete failed", "code", res.Code, "status", res.Status)
|
||||
return nil
|
||||
}
|
||||
|
||||
bindRes := res.GetBindDeleteRes()
|
||||
|
||||
fmt.Printf("bind res: %v\n", protojson.Format(bindRes))
|
||||
return nil
|
||||
}
|
||||
107
cmd/saastool/bind_setaccount.go
Normal file
107
cmd/saastool/bind_setaccount.go
Normal file
@@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"e.coding.net/rta/public/saasapi"
|
||||
"e.coding.net/rta/public/saasapi/pkg/saashttp"
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
)
|
||||
|
||||
type bindSetAccountParams struct {
|
||||
target string
|
||||
accounts []int64
|
||||
saasHttp *saashttp.SaasClient
|
||||
}
|
||||
|
||||
func RunBindSetAccount(args ...string) error {
|
||||
fs := flag.NewFlagSet("setaccount", flag.ExitOnError)
|
||||
cfgFile := paramConfig(fs)
|
||||
target := paramTarget(fs)
|
||||
accounts := paramAccounts(fs)
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// 切割字符串
|
||||
idsSlice := strings.Split(*accounts, ",")
|
||||
if len(idsSlice) == 1 && idsSlice[0] == "" {
|
||||
idsSlice = []string{}
|
||||
}
|
||||
|
||||
if fs.NArg() > 0 || len(idsSlice) == 0 || (len(idsSlice) == 1 && idsSlice[0] == "") {
|
||||
fs.PrintDefaults()
|
||||
return nil
|
||||
}
|
||||
|
||||
numAccounts := []int64{}
|
||||
for _, v := range idsSlice {
|
||||
n, err := strconv.ParseInt(v, 10, 64)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "account parse error", "value", v, "err", err)
|
||||
return err
|
||||
}
|
||||
numAccounts = append(numAccounts, n)
|
||||
}
|
||||
|
||||
cfg, err := LoadConfigFile(*cfgFile)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
bindSetAccountParams := bindSetAccountParams{
|
||||
target: *target,
|
||||
accounts: numAccounts,
|
||||
saasHttp: &saashttp.SaasClient{
|
||||
Client: &http.Client{},
|
||||
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
|
||||
Auth: &cfg.Auth,
|
||||
},
|
||||
}
|
||||
|
||||
return doBindSetAccount(bindSetAccountParams)
|
||||
}
|
||||
|
||||
func doBindSetAccount(p bindSetAccountParams) error {
|
||||
bindSet := &saasapi.BindSet{}
|
||||
|
||||
saasReq := &saasapi.SaasReq{
|
||||
Cmd: &saasapi.SaasReq_BindSet{
|
||||
BindSet: bindSet,
|
||||
},
|
||||
}
|
||||
|
||||
for _, v := range p.accounts {
|
||||
bindSet.Binds = append(bindSet.Binds, &saasapi.Bind{
|
||||
BindId: v,
|
||||
BindType: saasapi.BindType_AccountId,
|
||||
TargetId: p.target,
|
||||
AccountId: v,
|
||||
})
|
||||
}
|
||||
|
||||
res, err := p.saasHttp.BindSet(saasReq)
|
||||
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "submit Bind Set error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if res.Code != saasapi.ErrorCode_SUCC {
|
||||
fmt.Fprintln(os.Stderr, "Bind Set failed", "code", res.Code, "status", res.Status)
|
||||
return nil
|
||||
}
|
||||
|
||||
bindRes := res.GetBindSetRes()
|
||||
|
||||
fmt.Printf("bind res: %v\n", protojson.Format(bindRes))
|
||||
return nil
|
||||
}
|
||||
110
cmd/saastool/bind_setad.go
Normal file
110
cmd/saastool/bind_setad.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"e.coding.net/rta/public/saasapi"
|
||||
"e.coding.net/rta/public/saasapi/pkg/saashttp"
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
)
|
||||
|
||||
type bindSetAdParams struct {
|
||||
target string
|
||||
account int64
|
||||
ads []int64
|
||||
saasHttp *saashttp.SaasClient
|
||||
}
|
||||
|
||||
func RunBindSetAd(args ...string) error {
|
||||
fs := flag.NewFlagSet("setad", flag.ExitOnError)
|
||||
cfgFile := paramConfig(fs)
|
||||
target := paramTarget(fs)
|
||||
account := paramAccount(fs)
|
||||
ads := paramAds(fs)
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// 切割字符串
|
||||
idsSlice := strings.Split(*ads, ",")
|
||||
if len(idsSlice) == 1 && idsSlice[0] == "" {
|
||||
idsSlice = []string{}
|
||||
}
|
||||
|
||||
if fs.NArg() > 0 || len(idsSlice) == 0 || (len(idsSlice) == 1 && idsSlice[0] == "") {
|
||||
fs.PrintDefaults()
|
||||
return nil
|
||||
}
|
||||
|
||||
numAds := []int64{}
|
||||
for _, v := range idsSlice {
|
||||
n, err := strconv.ParseInt(v, 10, 64)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "account parse error", "value", v, "err", err)
|
||||
return err
|
||||
}
|
||||
numAds = append(numAds, n)
|
||||
}
|
||||
|
||||
cfg, err := LoadConfigFile(*cfgFile)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
bindSetAdParams := bindSetAdParams{
|
||||
target: *target,
|
||||
account: *account,
|
||||
ads: numAds,
|
||||
saasHttp: &saashttp.SaasClient{
|
||||
Client: &http.Client{},
|
||||
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
|
||||
Auth: &cfg.Auth,
|
||||
},
|
||||
}
|
||||
|
||||
return doBindSetAd(bindSetAdParams)
|
||||
}
|
||||
|
||||
func doBindSetAd(p bindSetAdParams) error {
|
||||
bindSet := &saasapi.BindSet{}
|
||||
|
||||
saasReq := &saasapi.SaasReq{
|
||||
Cmd: &saasapi.SaasReq_BindSet{
|
||||
BindSet: bindSet,
|
||||
},
|
||||
}
|
||||
|
||||
for _, v := range p.ads {
|
||||
bindSet.Binds = append(bindSet.Binds, &saasapi.Bind{
|
||||
BindId: v,
|
||||
BindType: saasapi.BindType_AdgroupId,
|
||||
TargetId: p.target,
|
||||
AccountId: p.account,
|
||||
})
|
||||
}
|
||||
|
||||
res, err := p.saasHttp.BindSet(saasReq)
|
||||
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "submit Bind Set error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if res.Code != saasapi.ErrorCode_SUCC {
|
||||
fmt.Fprintln(os.Stderr, "Bind Set failed", "code", res.Code, "status", res.Status)
|
||||
return nil
|
||||
}
|
||||
|
||||
bindRes := res.GetBindSetRes()
|
||||
|
||||
fmt.Printf("bind res: %v\n", protojson.Format(bindRes))
|
||||
return nil
|
||||
}
|
||||
@@ -15,13 +15,16 @@ const usage = `
|
||||
Usage: saastool COMMAND [OPTIONS]
|
||||
|
||||
Commands:
|
||||
info Saas Info
|
||||
write Write user's 'bytes / uint32s / flags'
|
||||
read Read user's 'bytes / uint32s / flags'
|
||||
columnwrite Write columns for 'deviceid / openid' users
|
||||
|
||||
convert Convert data to write format
|
||||
|
||||
task Task commands
|
||||
task Task commands
|
||||
target Target commands
|
||||
bind Bind commands
|
||||
|
||||
"help" is the default command.
|
||||
|
||||
|
||||
71
cmd/saastool/info.go
Normal file
71
cmd/saastool/info.go
Normal file
@@ -0,0 +1,71 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"e.coding.net/rta/public/saasapi"
|
||||
"e.coding.net/rta/public/saasapi/pkg/saashttp"
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
)
|
||||
|
||||
type infoParams struct {
|
||||
saasHttp *saashttp.SaasClient
|
||||
}
|
||||
|
||||
func RunInfo(args ...string) error {
|
||||
fs := flag.NewFlagSet("info", flag.ExitOnError)
|
||||
cfgFile := paramConfig(fs)
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if fs.NArg() > 0 {
|
||||
fs.PrintDefaults()
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg, err := LoadConfigFile(*cfgFile)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
infoParams := infoParams{
|
||||
saasHttp: &saashttp.SaasClient{
|
||||
Client: &http.Client{},
|
||||
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
|
||||
Auth: &cfg.Auth,
|
||||
},
|
||||
}
|
||||
return doInfo(infoParams)
|
||||
}
|
||||
|
||||
func doInfo(infoParams infoParams) error {
|
||||
saasReq := &saasapi.SaasReq{
|
||||
Cmd: &saasapi.SaasReq_Info{
|
||||
Info: &saasapi.Info{},
|
||||
},
|
||||
}
|
||||
|
||||
res, err := infoParams.saasHttp.Info(saasReq)
|
||||
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "submit Info error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if res.Code != saasapi.ErrorCode_SUCC {
|
||||
fmt.Fprintln(os.Stderr, "info failed", "code", res.Code, "status", res.Status)
|
||||
return nil
|
||||
}
|
||||
|
||||
infoRes := res.GetInfoRes()
|
||||
|
||||
fmt.Printf("info res: %v\n", protojson.Format(infoRes))
|
||||
return nil
|
||||
}
|
||||
@@ -18,6 +18,8 @@ func Run(args ...string) error {
|
||||
switch name {
|
||||
case "", "help":
|
||||
return RunHelp(args...)
|
||||
case "info":
|
||||
return RunInfo(args...)
|
||||
case "write":
|
||||
return RunWrite(args...)
|
||||
case "read":
|
||||
@@ -30,6 +32,10 @@ func Run(args ...string) error {
|
||||
return RunVerify(args...)
|
||||
case "task":
|
||||
return RunTask(args...)
|
||||
case "target":
|
||||
return RunTarget(args...)
|
||||
case "bind":
|
||||
return RunBind(args...)
|
||||
default:
|
||||
err := fmt.Errorf(`unknown command "%v"`+"\n"+`Run 'saastool help' for usage`, name)
|
||||
fmt.Fprintln(os.Stderr, err.Error())
|
||||
|
||||
@@ -62,7 +62,7 @@ func paramBatchSize(fs *flag.FlagSet) *uint {
|
||||
}
|
||||
|
||||
func paramBlockSize(fs *flag.FlagSet) *string {
|
||||
return fs.String("blocksize", "200M", "Block size to make hash. using size mode K, M, G, T")
|
||||
return fs.String("blocksize", "50M", "Block size to make hash. using size mode K, M, G, T")
|
||||
}
|
||||
|
||||
func paramClear(fs *flag.FlagSet) *bool {
|
||||
@@ -73,6 +73,38 @@ func paramDataSpaceId(fs *flag.FlagSet) *string {
|
||||
return fs.String("ds", "", "Data space ID")
|
||||
}
|
||||
|
||||
func paramTargets(fs *flag.FlagSet) *string {
|
||||
return fs.String("targets", "", "Target IDs. Use commas to separate multiple IDs")
|
||||
}
|
||||
|
||||
func paramListBinds(fs *flag.FlagSet) *bool {
|
||||
return fs.Bool("b", false, "List Binds")
|
||||
}
|
||||
|
||||
func paramTarget(fs *flag.FlagSet) *string {
|
||||
return fs.String("target", "", "Target ID")
|
||||
}
|
||||
|
||||
func paramAccount(fs *flag.FlagSet) *int64 {
|
||||
return fs.Int64("account", 0, "Advertiser ID")
|
||||
}
|
||||
|
||||
func paramAccounts(fs *flag.FlagSet) *string {
|
||||
return fs.String("accounts", "", "Advertiser IDs. Use commas to separate multiple IDs")
|
||||
}
|
||||
|
||||
func paramAds(fs *flag.FlagSet) *string {
|
||||
return fs.String("ads", "", "AdGroup IDs. Use commas to separate multiple IDs")
|
||||
}
|
||||
|
||||
func paramIdType(fs *flag.FlagSet) *int {
|
||||
return fs.Int("idtype", 0, "ID Type. empty is Automatic matching, 1=AdGroup, 3=Account")
|
||||
}
|
||||
|
||||
func paramIDs(fs *flag.FlagSet) *string {
|
||||
return fs.String("ids", "", "IDs for delete. Use commas to separate multiple IDs")
|
||||
}
|
||||
|
||||
// ParseByteSize 解析字节大小字符串为字节数
|
||||
func ParseByteSize(sizeStr string) (uint64, error) {
|
||||
sizeStr = strings.TrimSpace(sizeStr)
|
||||
|
||||
@@ -62,7 +62,7 @@ func RunRead(args ...string) error {
|
||||
ds: *ds,
|
||||
saasHttp: &saashttp.SaasClient{
|
||||
Client: &http.Client{},
|
||||
ApiUrls: &cfg.ApiUrls,
|
||||
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
|
||||
Auth: &cfg.Auth,
|
||||
},
|
||||
}
|
||||
|
||||
40
cmd/saastool/target.go
Normal file
40
cmd/saastool/target.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func RunTarget(args ...string) error {
|
||||
name, args := ParseCommandName(args)
|
||||
|
||||
// 从参数中解析出命令
|
||||
switch name {
|
||||
case "", "help":
|
||||
return RunTargetHelp(args...)
|
||||
case "list":
|
||||
return RunTargetList(args...)
|
||||
default:
|
||||
err := fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'saastool target help' for usage`, name)
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func RunTargetHelp(args ...string) error {
|
||||
fmt.Println(strings.TrimSpace(targetUsage))
|
||||
return nil
|
||||
}
|
||||
|
||||
const targetUsage = `
|
||||
Usage: saastoola target COMMAND [OPTIONS]
|
||||
|
||||
Commands:
|
||||
list List targets
|
||||
|
||||
"help" is the default command.
|
||||
|
||||
Use "saastool target COMMAND -help" for more information about a command.
|
||||
`
|
||||
88
cmd/saastool/target_list.go
Normal file
88
cmd/saastool/target_list.go
Normal file
@@ -0,0 +1,88 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"e.coding.net/rta/public/saasapi"
|
||||
"e.coding.net/rta/public/saasapi/pkg/saashttp"
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
)
|
||||
|
||||
type listTargetParams struct {
|
||||
targets []string
|
||||
listBinds bool
|
||||
saasHttp *saashttp.SaasClient
|
||||
}
|
||||
|
||||
func RunTargetList(args ...string) error {
|
||||
fs := flag.NewFlagSet("list", flag.ExitOnError)
|
||||
cfgFile := paramConfig(fs)
|
||||
targets := paramTargets(fs)
|
||||
listBinds := paramListBinds(fs)
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if fs.NArg() > 0 {
|
||||
fs.PrintDefaults()
|
||||
return nil
|
||||
}
|
||||
|
||||
// 切割字符串
|
||||
idsSlice := strings.Split(*targets, ",")
|
||||
if len(idsSlice) == 1 && idsSlice[0] == "" {
|
||||
idsSlice = []string{}
|
||||
}
|
||||
|
||||
cfg, err := LoadConfigFile(*cfgFile)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
listTargetParams := listTargetParams{
|
||||
listBinds: *listBinds,
|
||||
targets: idsSlice,
|
||||
saasHttp: &saashttp.SaasClient{
|
||||
Client: &http.Client{},
|
||||
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
|
||||
Auth: &cfg.Auth,
|
||||
},
|
||||
}
|
||||
|
||||
return doTargetList(listTargetParams)
|
||||
}
|
||||
|
||||
func doTargetList(listTargetParams listTargetParams) error {
|
||||
saasReq := &saasapi.SaasReq{
|
||||
Cmd: &saasapi.SaasReq_TargetList{
|
||||
TargetList: &saasapi.TargetList{
|
||||
Targets: listTargetParams.targets,
|
||||
ListBind: listTargetParams.listBinds,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
res, err := listTargetParams.saasHttp.TargetList(saasReq)
|
||||
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "submit List Target error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if res.Code != saasapi.ErrorCode_SUCC {
|
||||
fmt.Fprintln(os.Stderr, "Target list failed", "code", res.Code, "status", res.Status)
|
||||
return nil
|
||||
}
|
||||
|
||||
targetRes := res.GetTargetListRes()
|
||||
|
||||
fmt.Printf("target res: %v\n", protojson.Format(targetRes))
|
||||
return nil
|
||||
}
|
||||
@@ -25,6 +25,8 @@ func RunTask(args ...string) error {
|
||||
return RunTaskInfo(args...)
|
||||
case "upload":
|
||||
return RunTaskUpload(args...)
|
||||
case "download":
|
||||
return RunTaskDownload(args...)
|
||||
case "run":
|
||||
return RunTaskRun(args...)
|
||||
default:
|
||||
@@ -51,6 +53,7 @@ Commands:
|
||||
delete Delete a task on server
|
||||
info Get a task info on server
|
||||
upload Upload task's file block to server
|
||||
download Download task's file block to local
|
||||
|
||||
"help" is the default command.
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ func RunTaskCreate(args ...string) error {
|
||||
hashFile: *hashFile,
|
||||
saasHttp: &saashttp.SaasClient{
|
||||
Client: &http.Client{},
|
||||
ApiUrls: &cfg.ApiUrls,
|
||||
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
|
||||
Auth: &cfg.Auth,
|
||||
},
|
||||
task: &saasapi.Task{},
|
||||
|
||||
@@ -41,7 +41,7 @@ func RunTaskDelete(args ...string) error {
|
||||
taskSha256: *sha256,
|
||||
saasHttp: &saashttp.SaasClient{
|
||||
Client: &http.Client{},
|
||||
ApiUrls: &cfg.ApiUrls,
|
||||
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
|
||||
Auth: &cfg.Auth,
|
||||
},
|
||||
}
|
||||
|
||||
134
cmd/saastool/task_download.go
Normal file
134
cmd/saastool/task_download.go
Normal file
@@ -0,0 +1,134 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"e.coding.net/rta/public/saasapi"
|
||||
"e.coding.net/rta/public/saasapi/pkg/saashttp"
|
||||
)
|
||||
|
||||
type downloadTaskParams struct {
|
||||
taskSha256 string
|
||||
destPath string
|
||||
saasHttp *saashttp.SaasClient
|
||||
}
|
||||
|
||||
func RunTaskDownload(args ...string) error {
|
||||
fs := flag.NewFlagSet("download", flag.ExitOnError)
|
||||
cfgFile := paramConfig(fs)
|
||||
sha256 := paramSha256(fs)
|
||||
destPath := paramDestPath(fs)
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "command line parse error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if fs.NArg() > 0 || len(*sha256) == 0 || len(*destPath) == 0 {
|
||||
fs.PrintDefaults()
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg, err := LoadConfigFile(*cfgFile)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "LoadConfigFile error", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
downloadTaskParams := downloadTaskParams{
|
||||
taskSha256: *sha256,
|
||||
destPath: *destPath,
|
||||
saasHttp: &saashttp.SaasClient{
|
||||
Client: &http.Client{},
|
||||
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
|
||||
Auth: &cfg.Auth,
|
||||
},
|
||||
}
|
||||
|
||||
return doTaskDownload(downloadTaskParams)
|
||||
|
||||
}
|
||||
func doTaskDownload(downloadTaskParams downloadTaskParams) error {
|
||||
infoTaskParams := infoTaskParams{
|
||||
taskSha256: downloadTaskParams.taskSha256,
|
||||
saasHttp: downloadTaskParams.saasHttp,
|
||||
}
|
||||
taskInfo, err := doTaskInfo(infoTaskParams)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(taskInfo.GetSourcePath()) == 0 {
|
||||
err = fmt.Errorf("task download failed. task info source path is empty")
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return err
|
||||
}
|
||||
|
||||
totalFiles := len(taskInfo.GetTaskFileInfos())
|
||||
fi := 0
|
||||
for _, finfo := range taskInfo.GetTaskFileInfos() {
|
||||
fi++
|
||||
var f *os.File
|
||||
offset := int64(0)
|
||||
totalBlocks := len(finfo.GetFileBlocks())
|
||||
bi := 0
|
||||
for _, binfo := range finfo.GetFileBlocks() {
|
||||
bi++
|
||||
if binfo.GetUploaded() {
|
||||
if f == nil {
|
||||
fname := finfo.GetFileName()
|
||||
if len(downloadTaskParams.destPath) > 0 {
|
||||
fname = filepath.Join(downloadTaskParams.destPath, fname)
|
||||
}
|
||||
|
||||
os.MkdirAll(filepath.Dir(fname), os.ModePerm)
|
||||
|
||||
f, err = os.Create(fname)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
blockRes, err := downloadTaskParams.saasHttp.TaskDownload(
|
||||
binfo.GetBlockSha256(),
|
||||
f,
|
||||
offset,
|
||||
int(binfo.GetBlockLength()),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if blockRes.GetCode() != saasapi.ErrorCode_SUCC {
|
||||
err = fmt.Errorf("download block error, code %d, msg %s", blockRes.GetCode(), blockRes.GetStatus())
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return err
|
||||
} else {
|
||||
fmt.Printf("download block success. file: %v, sha256 %v. block %v/%v, file %v/%v\n",
|
||||
finfo.GetFileName(), binfo.GetBlockSha256(),
|
||||
bi, totalBlocks, fi, totalFiles,
|
||||
)
|
||||
}
|
||||
|
||||
} else {
|
||||
fmt.Printf("download block. file: %v, sha256 %v. block %v/%v, file %v/%v\n",
|
||||
finfo.GetFileName(), binfo.GetBlockSha256(),
|
||||
bi, totalBlocks, fi, totalFiles,
|
||||
)
|
||||
}
|
||||
offset += int64(binfo.GetBlockLength())
|
||||
|
||||
}
|
||||
|
||||
if f != nil {
|
||||
f.Close()
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -41,7 +41,7 @@ func RunTaskInfo(args ...string) error {
|
||||
taskSha256: *sha256,
|
||||
saasHttp: &saashttp.SaasClient{
|
||||
Client: &http.Client{},
|
||||
ApiUrls: &cfg.ApiUrls,
|
||||
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
|
||||
Auth: &cfg.Auth,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -40,7 +40,7 @@ func RunTaskList(args ...string) error {
|
||||
listTaskParams := listTaskParams{
|
||||
saasHttp: &saashttp.SaasClient{
|
||||
Client: &http.Client{},
|
||||
ApiUrls: &cfg.ApiUrls,
|
||||
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
|
||||
Auth: &cfg.Auth,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -81,7 +82,7 @@ func RunTaskMake(args ...string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
makeTaskParams := makeTaskParams{
|
||||
makeTaskParams := &makeTaskParams{
|
||||
sourcePath: *sourcePath,
|
||||
hashFile: *hashFile,
|
||||
task: &saasapi.Task{
|
||||
@@ -89,12 +90,13 @@ func RunTaskMake(args ...string) error {
|
||||
TaskDescription: *desc,
|
||||
Appid: *appid,
|
||||
DataspaceId: *ds,
|
||||
SourcePath: *sourcePath,
|
||||
},
|
||||
}
|
||||
return doMakeHash(makeTaskParams)
|
||||
return doMakeHash(makeTaskParams, true)
|
||||
}
|
||||
|
||||
func doMakeHash(makeTaskParams makeTaskParams) error {
|
||||
func doMakeHash(makeTaskParams *makeTaskParams, firstLevel bool) error {
|
||||
fsInfo, err := os.Stat(makeTaskParams.sourcePath)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -102,29 +104,39 @@ func doMakeHash(makeTaskParams makeTaskParams) error {
|
||||
|
||||
if !fsInfo.IsDir() {
|
||||
// 如果是文件,直接计算
|
||||
return doTaskMake(makeTaskParams)
|
||||
}
|
||||
|
||||
// 读取目录下信息
|
||||
dirEntry, err := os.ReadDir(makeTaskParams.sourcePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 遍历目录
|
||||
for _, dir := range dirEntry {
|
||||
newParam := makeTaskParams
|
||||
newParam.sourcePath = path.Join(makeTaskParams.sourcePath, dir.Name())
|
||||
|
||||
if err = doMakeHash(newParam); err != nil {
|
||||
if firstLevel {
|
||||
makeTaskParams.task.SourcePath = filepath.Dir(makeTaskParams.sourcePath)
|
||||
}
|
||||
err = doTaskMake(makeTaskParams)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// 读取目录下信息
|
||||
dirEntry, err := os.ReadDir(makeTaskParams.sourcePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 遍历目录
|
||||
for _, dir := range dirEntry {
|
||||
oldSourcePath := makeTaskParams.sourcePath
|
||||
makeTaskParams.sourcePath = path.Join(makeTaskParams.sourcePath, dir.Name())
|
||||
|
||||
if err = doMakeHash(makeTaskParams, false); err != nil {
|
||||
return err
|
||||
}
|
||||
makeTaskParams.sourcePath = oldSourcePath
|
||||
}
|
||||
}
|
||||
|
||||
return saveTaskFile(makeTaskParams)
|
||||
if firstLevel {
|
||||
return saveTaskFile(makeTaskParams)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func doTaskMake(makeTaskParams makeTaskParams) error {
|
||||
func doTaskMake(makeTaskParams *makeTaskParams) error {
|
||||
sourceFile, err := os.Open(makeTaskParams.sourcePath)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -200,10 +212,12 @@ func doTaskMake(makeTaskParams makeTaskParams) error {
|
||||
})
|
||||
|
||||
// 输出结果
|
||||
relPath, _ := filepath.Rel(makeTaskParams.task.GetSourcePath(), makeTaskParams.sourcePath)
|
||||
fileInfo := &saasapi.FileInfo{
|
||||
FileName: makeTaskParams.sourcePath,
|
||||
FileName: relPath,
|
||||
FileSize: uint64(fi.Size()),
|
||||
}
|
||||
|
||||
for _, r := range allResults {
|
||||
fileInfo.FileBlocks = append(fileInfo.FileBlocks, &saasapi.FileBlock{
|
||||
BlockSha256: r.hash,
|
||||
@@ -212,6 +226,8 @@ func doTaskMake(makeTaskParams makeTaskParams) error {
|
||||
}
|
||||
makeTaskParams.task.TaskFileInfos = append(makeTaskParams.task.TaskFileInfos, fileInfo)
|
||||
|
||||
makeTaskParams.task.TaskSize += uint64(fi.Size())
|
||||
|
||||
fmt.Println("")
|
||||
return nil
|
||||
}
|
||||
@@ -227,7 +243,7 @@ func hashWorker(tasks <-chan *hashTask, results chan<- *hashTask) {
|
||||
}
|
||||
}
|
||||
|
||||
func saveTaskFile(makeTaskParams makeTaskParams) error {
|
||||
func saveTaskFile(makeTaskParams *makeTaskParams) error {
|
||||
taskFile, err := os.Create(makeTaskParams.hashFile)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -41,7 +41,7 @@ func RunTaskRun(args ...string) error {
|
||||
taskSha256: *sha256,
|
||||
saasHttp: &saashttp.SaasClient{
|
||||
Client: &http.Client{},
|
||||
ApiUrls: &cfg.ApiUrls,
|
||||
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
|
||||
Auth: &cfg.Auth,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"e.coding.net/rta/public/saasapi"
|
||||
"e.coding.net/rta/public/saasapi/pkg/saashttp"
|
||||
@@ -40,7 +41,7 @@ func RunTaskUpload(args ...string) error {
|
||||
taskSha256: *sha256,
|
||||
saasHttp: &saashttp.SaasClient{
|
||||
Client: &http.Client{},
|
||||
ApiUrls: &cfg.ApiUrls,
|
||||
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
|
||||
Auth: &cfg.Auth,
|
||||
},
|
||||
}
|
||||
@@ -58,10 +59,10 @@ func doTaskUpload(uploadTaskParams uploadTaskParams) error {
|
||||
return err
|
||||
}
|
||||
|
||||
sourcePath := taskInfo.GetSourcePath()
|
||||
totalFiles := len(taskInfo.GetTaskFileInfos())
|
||||
fi := 0
|
||||
for _, finfo := range taskInfo.GetTaskFileInfos() {
|
||||
|
||||
fi++
|
||||
var f *os.File
|
||||
offset := int64(0)
|
||||
@@ -71,7 +72,11 @@ func doTaskUpload(uploadTaskParams uploadTaskParams) error {
|
||||
bi++
|
||||
if !binfo.GetUploaded() {
|
||||
if f == nil {
|
||||
f, err = os.Open(finfo.GetFileName())
|
||||
fname := finfo.GetFileName()
|
||||
if len(sourcePath) > 0 {
|
||||
fname = filepath.Join(sourcePath, fname)
|
||||
}
|
||||
f, err = os.Open(fname)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -89,7 +94,9 @@ func doTaskUpload(uploadTaskParams uploadTaskParams) error {
|
||||
}
|
||||
|
||||
if blockRes.GetCode() != saasapi.ErrorCode_SUCC {
|
||||
return fmt.Errorf("upload block error, code %d, msg %s", blockRes.GetCode(), blockRes.GetStatus())
|
||||
err = fmt.Errorf("upload block error, code %d, msg %s", blockRes.GetCode(), blockRes.GetStatus())
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return err
|
||||
} else {
|
||||
fmt.Printf("upload block success. file: %v, sha256 %v. block %v/%v, file %v/%v\n",
|
||||
finfo.GetFileName(), binfo.GetBlockSha256(),
|
||||
|
||||
@@ -63,7 +63,7 @@ func RunWrite(args ...string) error {
|
||||
clear: *clear,
|
||||
saasHttp: &saashttp.SaasClient{
|
||||
Client: &http.Client{},
|
||||
ApiUrls: &cfg.ApiUrls,
|
||||
ApiUrls: saashttp.InitAPIUrl(&cfg.ApiUrls),
|
||||
Auth: &cfg.Auth,
|
||||
},
|
||||
}
|
||||
|
||||
12
docker/saastool.Dockerfile
Normal file
12
docker/saastool.Dockerfile
Normal file
@@ -0,0 +1,12 @@
|
||||
FROM rta-docker.pkg.coding.net/public/docker/entre_dev:latest AS builder
|
||||
|
||||
COPY . /tmp/saasapi/
|
||||
WORKDIR /tmp/saasapi/
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
--mount=type=cache,target=/go/pkg \
|
||||
./make.sh
|
||||
|
||||
FROM rta-docker.pkg.coding.net/public/docker/alpine:latest AS product
|
||||
|
||||
COPY --from=builder /tmp/saasapi/build/saastool /bin/saastool
|
||||
3
make.sh
Executable file
3
make.sh
Executable file
@@ -0,0 +1,3 @@
|
||||
cd cmd/saastool
|
||||
go build -o ../../build/saastool
|
||||
cd ../../
|
||||
@@ -1,16 +1,135 @@
|
||||
package saashttp
|
||||
|
||||
const (
|
||||
baseUrl = "https://api.rta.qq.com"
|
||||
infoPath = "/saas/info"
|
||||
writePath = "/saas/write"
|
||||
readPath = "/saas/read"
|
||||
columnWritePath = "/saas/column_write"
|
||||
taskCreatePath = "/saas/task/create"
|
||||
taskListPath = "/saas/task/list"
|
||||
taskCancelPath = "/saas/task/delete"
|
||||
taskInfoPath = "/saas/task/info"
|
||||
taskUploadPath = "/saas/task/upload"
|
||||
taskDownloadPath = "/saas/task/download"
|
||||
taskDeletePath = "/saas/task/delete"
|
||||
taskRunPath = "/saas/task/run"
|
||||
targetListPath = "/saas/target/list"
|
||||
bindSetPath = "/saas/bind/set"
|
||||
bindDeletePath = "/saas/bind/delete"
|
||||
)
|
||||
|
||||
type ApiUrls struct {
|
||||
BaseUrl string
|
||||
WritePath string
|
||||
ReadPath string
|
||||
ColumnWritePath string
|
||||
TaskCreatePath string
|
||||
TaskListPath string
|
||||
TaskInfoPath string
|
||||
TaskDeletePath string
|
||||
TaskRunPath string
|
||||
TaskUploadPath string
|
||||
BaseUrl string
|
||||
InfoPath string
|
||||
WritePath string
|
||||
ReadPath string
|
||||
ColumnWritePath string
|
||||
TaskCreatePath string
|
||||
TaskListPath string
|
||||
TaskInfoPath string
|
||||
TaskDeletePath string
|
||||
TaskRunPath string
|
||||
TaskUploadPath string
|
||||
TaskDownloadPath string
|
||||
TargetListPath string
|
||||
BindSetPath string
|
||||
BindDeletePath string
|
||||
}
|
||||
|
||||
func InitAPIUrl(c *ApiUrls) *ApiUrls {
|
||||
r := &ApiUrls{}
|
||||
if c.BaseUrl != "" {
|
||||
r.BaseUrl = c.BaseUrl
|
||||
} else {
|
||||
r.BaseUrl = baseUrl
|
||||
}
|
||||
|
||||
if c.InfoPath != "" {
|
||||
r.InfoPath = c.InfoPath
|
||||
} else {
|
||||
r.InfoPath = infoPath
|
||||
}
|
||||
|
||||
if c.WritePath != "" {
|
||||
r.WritePath = c.WritePath
|
||||
} else {
|
||||
r.WritePath = writePath
|
||||
}
|
||||
|
||||
if c.ReadPath != "" {
|
||||
r.ReadPath = c.ReadPath
|
||||
} else {
|
||||
r.ReadPath = readPath
|
||||
}
|
||||
|
||||
if c.ColumnWritePath != "" {
|
||||
r.ColumnWritePath = c.ColumnWritePath
|
||||
} else {
|
||||
r.ColumnWritePath = columnWritePath
|
||||
}
|
||||
|
||||
if c.TaskCreatePath != "" {
|
||||
r.TaskCreatePath = c.TaskCreatePath
|
||||
} else {
|
||||
r.TaskCreatePath = taskCreatePath
|
||||
}
|
||||
|
||||
if c.TaskListPath != "" {
|
||||
r.TaskListPath = c.TaskListPath
|
||||
} else {
|
||||
r.TaskListPath = taskListPath
|
||||
}
|
||||
|
||||
if c.TaskInfoPath != "" {
|
||||
r.TaskInfoPath = c.TaskInfoPath
|
||||
} else {
|
||||
r.TaskInfoPath = taskInfoPath
|
||||
}
|
||||
|
||||
if c.TaskDeletePath != "" {
|
||||
r.TaskDeletePath = c.TaskDeletePath
|
||||
} else {
|
||||
r.TaskDeletePath = taskDeletePath
|
||||
}
|
||||
|
||||
if c.TaskRunPath != "" {
|
||||
r.TaskRunPath = c.TaskRunPath
|
||||
} else {
|
||||
r.TaskRunPath = taskRunPath
|
||||
}
|
||||
|
||||
if c.TaskUploadPath != "" {
|
||||
r.TaskUploadPath = c.TaskUploadPath
|
||||
} else {
|
||||
r.TaskUploadPath = taskUploadPath
|
||||
}
|
||||
|
||||
if c.TaskDownloadPath != "" {
|
||||
r.TaskDownloadPath = c.TaskDownloadPath
|
||||
} else {
|
||||
r.TaskDownloadPath = taskDownloadPath
|
||||
}
|
||||
|
||||
if c.TargetListPath != "" {
|
||||
r.TargetListPath = c.TargetListPath
|
||||
} else {
|
||||
r.TargetListPath = targetListPath
|
||||
}
|
||||
|
||||
if c.BindSetPath != "" {
|
||||
r.BindSetPath = c.BindSetPath
|
||||
} else {
|
||||
r.BindSetPath = bindSetPath
|
||||
}
|
||||
|
||||
if c.BindDeletePath != "" {
|
||||
r.BindDeletePath = c.BindDeletePath
|
||||
} else {
|
||||
r.BindDeletePath = bindDeletePath
|
||||
}
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
type Auth struct {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"e.coding.net/rta/public/saasapi"
|
||||
@@ -32,6 +33,11 @@ type SaasClient struct {
|
||||
ResponseEncoder ResponseEncoder
|
||||
}
|
||||
|
||||
func (c *SaasClient) Info(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
|
||||
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.InfoPath)
|
||||
return c.post(postUrl, saasReq)
|
||||
}
|
||||
|
||||
func (c *SaasClient) Write(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
|
||||
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.WritePath)
|
||||
return c.post(postUrl, saasReq)
|
||||
@@ -77,6 +83,26 @@ func (c *SaasClient) TaskUpload(sha256 string, file *os.File, offset int64, size
|
||||
return c.upload(postUrl, file, offset, size)
|
||||
}
|
||||
|
||||
func (c *SaasClient) TaskDownload(sha256 string, file *os.File, offset int64, size int) (saasRes *saasapi.SaasRes, err error) {
|
||||
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TaskDownloadPath, "sha256", sha256)
|
||||
return c.download(postUrl, file, offset, size)
|
||||
}
|
||||
|
||||
func (c *SaasClient) TargetList(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
|
||||
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.TargetListPath)
|
||||
return c.post(postUrl, saasReq)
|
||||
}
|
||||
|
||||
func (c *SaasClient) BindSet(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
|
||||
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.BindSetPath)
|
||||
return c.post(postUrl, saasReq)
|
||||
}
|
||||
|
||||
func (c *SaasClient) BindDelete(saasReq *saasapi.SaasReq) (saasRes *saasapi.SaasRes, err error) {
|
||||
postUrl := c.makeUrl(c.ApiUrls.BaseUrl, c.ApiUrls.BindDeletePath)
|
||||
return c.post(postUrl, saasReq)
|
||||
}
|
||||
|
||||
func (c *SaasClient) makeUrl(baseUrl, path string, params ...string) string {
|
||||
url, err := url.Parse(baseUrl)
|
||||
if err != nil {
|
||||
@@ -232,3 +258,86 @@ func (c *SaasClient) upload(url string, file *os.File, offset int64, size int) (
|
||||
|
||||
return saasRes, nil
|
||||
}
|
||||
|
||||
func (c *SaasClient) download(url string, file *os.File, offset int64, size int) (saasRes *saasapi.SaasRes, err error) {
|
||||
if file == nil {
|
||||
return nil, fmt.Errorf("file is nil")
|
||||
}
|
||||
|
||||
if size <= 0 {
|
||||
return nil, fmt.Errorf("size is invalid")
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", url, bytes.NewBuffer([]byte{}))
|
||||
if err != nil {
|
||||
fmt.Println("http new request error", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
timeStamp := strconv.FormatInt(time.Now().Unix(), 10)
|
||||
md5byte := md5.Sum([]byte(c.Auth.Account + c.Auth.Token + timeStamp))
|
||||
authorization := hex.EncodeToString(md5byte[:])
|
||||
|
||||
req.Header.Add("Account", c.Auth.Account)
|
||||
req.Header.Add("Time", timeStamp)
|
||||
req.Header.Add("Authorization", authorization)
|
||||
req.Header.Add("Accept-Encoding", "gzip")
|
||||
|
||||
res, err := c.Client.Do(req)
|
||||
if err != nil {
|
||||
fmt.Println("http send error", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode != 200 {
|
||||
err = fmt.Errorf("NOT 200. %v", res.StatusCode)
|
||||
fmt.Println("http state error", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bodyReader := res.Body
|
||||
if strings.Contains(res.Header.Get("Content-Encoding"), "gzip") {
|
||||
gz, err := gzip.NewReader(res.Body)
|
||||
if err != nil {
|
||||
fmt.Println("gzip newreader error", err)
|
||||
return nil, err
|
||||
}
|
||||
defer gz.Close()
|
||||
bodyReader = gz
|
||||
}
|
||||
|
||||
resBody, err := io.ReadAll(bodyReader)
|
||||
if err != nil {
|
||||
fmt.Println("read body error", err)
|
||||
return nil, err
|
||||
}
|
||||
saasRes = &saasapi.SaasRes{}
|
||||
|
||||
if strings.Contains(res.Header.Get("Content-Type"), "application/octet-stream") {
|
||||
if len(resBody) == size {
|
||||
file.WriteAt(resBody, offset)
|
||||
} else {
|
||||
err = fmt.Errorf("body size error. body:%v, want:%v", len(resBody), size)
|
||||
fmt.Println("http read body error", err)
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
if c.ResponseEncoder == RESPONSE_ENCODER_PROTOBUF {
|
||||
err = proto.Unmarshal(resBody, saasRes)
|
||||
if err != nil {
|
||||
fmt.Println("unmarshal response body to protobuf error", err)
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
err = json.Unmarshal(resBody, saasRes)
|
||||
if err != nil {
|
||||
fmt.Println("unmarshal response body to json error", err)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return saasRes, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user