使用 Go SDK 检查 AWS Data Pipeline 的状态
Checking status of AWS Data Pipeline using Go SDK
情况:我有 2 个数据管道 运行 按需。在管道 A 完成之前,管道 B 不能 运行。我正在尝试 运行 在单个 script/program 中自动化这两个管道,但我不确定如何在 Go 中完成所有这些操作。
我有一些激活数据管道的 Go 代码:
func awsActivatePipeline(pipelineID, region string) (*datapipeline.ActivatePipelineOutput, error) {
svc := datapipeline.New(session.New(&aws.Config{Region: aws.String(region)}))
input := &datapipeline.ActivatePipelineInput{
PipelineId: aws.String(pipelineID),
}
result, err := svc.ActivatePipeline(input)
if err != nil {
fmt.Println("error activating pipeline: ", err)
}
fmt.Println(result)
return result, nil
}
激活后,我希望能够监控该管道并确定它何时完成,以便我可以 运行 第二个管道。类似于 list-runs
CLI 命令,但我不确定相应的 Go 函数是什么。
$ aws datapipeline list-runs --region us-west-2 --pipeline-id df-EXAMPLE
Name Scheduled Start Status
ID Started Ended
---------------------------------------------------------------------------------------------------
1. EC2ResourceObj 2017-09-12T17:49:55 FINISHED
@EC2ResourceObj_2017-09-12T17:49:55 2017-09-12T17:49:58 2017-09-12T17:56:52
2. Installation 2017-09-12T17:49:55 FINISHED
@Installation_@ShellCommandActivityObj_2017-09-12T 2017-09-12T17:49:57 2017-09-12T17:54:09
3. S3OutputLocation 2017-09-12T17:49:55 FINISHED
@S3OutputLocation_2017-09-12T17:49:55 2017-09-12T17:49:58 2017-09-12T17:54:50
4. ShellCommandActivityObj 2017-09-12T17:49:55 FINISHED
@ShellCommandActivityObj_2017-09-12T17:49:55 2017-09-12T17:49:57 2017-09-12T17:54:49
所以一旦所有操作都被标记 'FINISHED',我想激活我的第二个管道。实现此目标的最佳方法是什么?
仅供参考,以防其他人遇到此问题,这就是我解决此问题的方法:
Golang AWS API 调用来描述 objects/actions 数据管道,returns 如果所有对象都完成则为真
func awsDescribeObjects(pipelineID, region string, objects []string) bool {
var r Object
var s []string
var f bool
svc := datapipeline.New(session.New(&aws.Config{Region: aws.String(region)}))
input := &datapipeline.DescribeObjectsInput{
PipelineId: aws.String(pipelineID),
ObjectIds: aws.StringSlice(objects),
}
result, err := svc.DescribeObjects(input)
if err != nil {
fmt.Println("error describing pipeline objects: ", err)
f = false
return f
}
//fmt.Println("original result: ", result)
result2 := re.ReplaceAllString(result.String(), `""`) //add "" around keys
result3 := re1.ReplaceAllString(result2, ``) //remove key and string/ref value from fields struct
result4 := strings.Replace(result3, "@", "", -1) //remove @ from keys and values
result5 := re2.ReplaceAllString(result4, ``) //remove "" from timestamps
result6 := re3.ReplaceAllString(result5, `,`) // remove {} from fields struct
json.Unmarshal([]byte(result6), &r)
// fmt.Printf("R: %+v\n", r)
p := r.PipelineObjects
// fmt.Printf("P: %+v\n", p)
for i := range p {
for m := range p[i].Fields {
fmt.Printf("%v STATUS: %v\n", p[i].Name, p[i].Fields[m].Status)
s = append(s, p[i].Fields[m].Status)
if p[i].Fields[m].Status != "FINISHED" {
f = false
} else {
f = true
}
}
// fmt.Println("bool: ", f)
}
return f
}
我的主要功能
func main() {
if *action == "describe" {
obj := strings.Split(*object, ",")
for i := 0; i <= 20; i++ {
f := awsDescribeObjects(*pipeline, *region, obj)
fmt.Printf("%v - Status Check %v - Finished?: %v\n", time.Now(), i, f)
if f == true {
fmt.Println("FINISHED describing pipeline complete")
break
}
time.Sleep(5 * time.Minute)
if i == 20 {
fmt.Println("TIME OUT - describe pipeline timed out, max time reached")
os.Exit(1)
}
}
}
}
Shell go 可执行脚本:
#PIPELINE 1
echo "Starting Pipeline 1..."
echo ./runpipeline.linux -region $REGION1 -pipeline-id $PIPELINEID1 -action activate
echo sleep 1m
echo ./runpipeline.linux -region $REGION1 -pipeline-id $PIPELINEID1 -action describe -object ShellCommandActivityObj
echo "Pipeline 1 complete"
#PIPELINE 2
echo "Starting Pipeline 2..."
echo ./runpipeline.linux -region $REGION2 -pipeline-id $PIPELINEID2 -action activate
echo sleep 1m
echo ./runpipeline.linux -region $REGION2 -pipeline-id $PIPELINEID2 -action describe -object ShellCommandActivityObj,CliActivity
echo "Pipeline 2 complete"
echo "FINISHED"
情况:我有 2 个数据管道 运行 按需。在管道 A 完成之前,管道 B 不能 运行。我正在尝试 运行 在单个 script/program 中自动化这两个管道,但我不确定如何在 Go 中完成所有这些操作。
我有一些激活数据管道的 Go 代码:
func awsActivatePipeline(pipelineID, region string) (*datapipeline.ActivatePipelineOutput, error) {
svc := datapipeline.New(session.New(&aws.Config{Region: aws.String(region)}))
input := &datapipeline.ActivatePipelineInput{
PipelineId: aws.String(pipelineID),
}
result, err := svc.ActivatePipeline(input)
if err != nil {
fmt.Println("error activating pipeline: ", err)
}
fmt.Println(result)
return result, nil
}
激活后,我希望能够监控该管道并确定它何时完成,以便我可以 运行 第二个管道。类似于 list-runs
CLI 命令,但我不确定相应的 Go 函数是什么。
$ aws datapipeline list-runs --region us-west-2 --pipeline-id df-EXAMPLE
Name Scheduled Start Status
ID Started Ended
---------------------------------------------------------------------------------------------------
1. EC2ResourceObj 2017-09-12T17:49:55 FINISHED
@EC2ResourceObj_2017-09-12T17:49:55 2017-09-12T17:49:58 2017-09-12T17:56:52
2. Installation 2017-09-12T17:49:55 FINISHED
@Installation_@ShellCommandActivityObj_2017-09-12T 2017-09-12T17:49:57 2017-09-12T17:54:09
3. S3OutputLocation 2017-09-12T17:49:55 FINISHED
@S3OutputLocation_2017-09-12T17:49:55 2017-09-12T17:49:58 2017-09-12T17:54:50
4. ShellCommandActivityObj 2017-09-12T17:49:55 FINISHED
@ShellCommandActivityObj_2017-09-12T17:49:55 2017-09-12T17:49:57 2017-09-12T17:54:49
所以一旦所有操作都被标记 'FINISHED',我想激活我的第二个管道。实现此目标的最佳方法是什么?
仅供参考,以防其他人遇到此问题,这就是我解决此问题的方法:
Golang AWS API 调用来描述 objects/actions 数据管道,returns 如果所有对象都完成则为真
func awsDescribeObjects(pipelineID, region string, objects []string) bool {
var r Object
var s []string
var f bool
svc := datapipeline.New(session.New(&aws.Config{Region: aws.String(region)}))
input := &datapipeline.DescribeObjectsInput{
PipelineId: aws.String(pipelineID),
ObjectIds: aws.StringSlice(objects),
}
result, err := svc.DescribeObjects(input)
if err != nil {
fmt.Println("error describing pipeline objects: ", err)
f = false
return f
}
//fmt.Println("original result: ", result)
result2 := re.ReplaceAllString(result.String(), `""`) //add "" around keys
result3 := re1.ReplaceAllString(result2, ``) //remove key and string/ref value from fields struct
result4 := strings.Replace(result3, "@", "", -1) //remove @ from keys and values
result5 := re2.ReplaceAllString(result4, ``) //remove "" from timestamps
result6 := re3.ReplaceAllString(result5, `,`) // remove {} from fields struct
json.Unmarshal([]byte(result6), &r)
// fmt.Printf("R: %+v\n", r)
p := r.PipelineObjects
// fmt.Printf("P: %+v\n", p)
for i := range p {
for m := range p[i].Fields {
fmt.Printf("%v STATUS: %v\n", p[i].Name, p[i].Fields[m].Status)
s = append(s, p[i].Fields[m].Status)
if p[i].Fields[m].Status != "FINISHED" {
f = false
} else {
f = true
}
}
// fmt.Println("bool: ", f)
}
return f
}
我的主要功能
func main() {
if *action == "describe" {
obj := strings.Split(*object, ",")
for i := 0; i <= 20; i++ {
f := awsDescribeObjects(*pipeline, *region, obj)
fmt.Printf("%v - Status Check %v - Finished?: %v\n", time.Now(), i, f)
if f == true {
fmt.Println("FINISHED describing pipeline complete")
break
}
time.Sleep(5 * time.Minute)
if i == 20 {
fmt.Println("TIME OUT - describe pipeline timed out, max time reached")
os.Exit(1)
}
}
}
}
Shell go 可执行脚本:
#PIPELINE 1
echo "Starting Pipeline 1..."
echo ./runpipeline.linux -region $REGION1 -pipeline-id $PIPELINEID1 -action activate
echo sleep 1m
echo ./runpipeline.linux -region $REGION1 -pipeline-id $PIPELINEID1 -action describe -object ShellCommandActivityObj
echo "Pipeline 1 complete"
#PIPELINE 2
echo "Starting Pipeline 2..."
echo ./runpipeline.linux -region $REGION2 -pipeline-id $PIPELINEID2 -action activate
echo sleep 1m
echo ./runpipeline.linux -region $REGION2 -pipeline-id $PIPELINEID2 -action describe -object ShellCommandActivityObj,CliActivity
echo "Pipeline 2 complete"
echo "FINISHED"