如何使用 Terraform 创建连接到 S3 的 AWS Kinesis Firehose？

Question

我正在尝试使用 kinesis_firehose_delivery_stream 资源创建具有 Direct PUT 源、无数据转换和 extended_s3 目标的 Kinesis Firehose。

我修改了 this example 中的代码（删除了 lambda 函数），现在它看起来像这样：

terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 3.0"
    }
  }
}

provider "aws" {
  region = "us-east-1"
}

resource "aws_s3_bucket" "bucket" {
  bucket = "test-kinesis-destination-bucket"
  acl    = "private"
}

resource "aws_kinesis_firehose_delivery_stream" "kinesis_event_stream" {
  name        = "kinesis-test-stream"
  destination = "extended_s3"

  extended_s3_configuration {
    role_arn   = aws_iam_role.firehose_role.arn
    bucket_arn = aws_s3_bucket.bucket.arn
    buffer_size = 1
    buffer_interval = 60
  }
}

resource "aws_iam_role" "firehose_role" {
  name = "firehose_test_role"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "firehose.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

Terraform 能够成功应用所有内容，但 Firehose 似乎无法写入 S3。

我的 IAM 角色是否遗漏了什么？如果是这样，我该如何解决？

编辑

根据@Marcin 的回答，我更新了我的 terraform 文件以更新 IAM 策略，以授予 Firehose 写入 S3 的权限。

terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 3.0"
    }
  }
}

provider "aws" {
  region = "us-west-2"
}

resource "aws_s3_bucket" "bucket" {
  bucket = "test-kinesis-destination-bucket"
  acl    = "private"
}

resource "aws_kinesis_firehose_delivery_stream" "kinesis_event_stream" {
  name        = "kinesis-test-stream"
  destination = "extended_s3"

  extended_s3_configuration {
    role_arn   = aws_iam_role.firehose_role.arn
    bucket_arn = aws_s3_bucket.bucket.arn
    buffer_size = 1
    buffer_interval = 60
  }
}

resource "aws_iam_role" "firehose_role" {
  name = "firehose_test_role"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",  
  "Statement":
  [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "firehose.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF

  inline_policy {
    name = "kinesis-s3-inline-policy"
    policy = jsonencode({
      Version = "2012-10-17"
      Statement = [
        {      
          Effect = "Allow",      
          Action = [
            "s3:AbortMultipartUpload",
            "s3:GetBucketLocation",
            "s3:GetObject",
            "s3:ListBucket",
            "s3:ListBucketMultipartUploads",
            "s3:PutObject"
          ]      
          Resource = [        
            "arn:aws:s3:::test-kinesis-destination-bucket",
            "arn:aws:s3:::test-kinesis-destination-bucket/*"            
          ]    
        },
        {
          Effect = "Allow"
          Action = [
            "kinesis:DescribeStream",
            "kinesis:GetShardIterator",
            "kinesis:GetRecords",
            "kinesis:ListShards"
          ]
          Resource = aws_kinesis_firehose_delivery_stream.kinesis_event_stream.arn
        }
      ]
    })
  }
}

但是当我运行 terraform plan 我得到以下错误：

$ terraform plan
╷
│ Error: Cycle: aws_kinesis_firehose_delivery_stream.kinesis_event_stream, aws_iam_role.firehose_role
│ 
│ 
|

如何在其 IAM 策略中引用 Firehose 的 ARN？

Answer 1

您创建的角色 firehose_role 只有信任关系，但 没有实际的 S3 权限。您的角色应具有以下权限，如 docs 中所述（如果您不将 lambda 与 kinesis、kms 或 firehose 可以使用的其他服务一起使用，则可以 trim 降低权限）：

{
    "Version": "2012-10-17",  
    "Statement":
    [    
        {      
            "Effect": "Allow",      
            "Action": [
                "s3:AbortMultipartUpload",
                "s3:GetBucketLocation",
                "s3:GetObject",
                "s3:ListBucket",
                "s3:ListBucketMultipartUploads",
                "s3:PutObject"
            ],      
            "Resource": [        
                "arn:aws:s3:::bucket-name",
                "arn:aws:s3:::bucket-name/*"            
            ]    
        },        
        {
            "Effect": "Allow",
            "Action": [
                "kinesis:DescribeStream",
                "kinesis:GetShardIterator",
                "kinesis:GetRecords",
                "kinesis:ListShards"
            ],
            "Resource": "arn:aws:kinesis:region:account-id:stream/stream-name"
        },
        {
           "Effect": "Allow",
           "Action": [
               "kms:Decrypt",
               "kms:GenerateDataKey"
           ],
           "Resource": [
               "arn:aws:kms:region:account-id:key/key-id"           
           ],
           "Condition": {
               "StringEquals": {
                   "kms:ViaService": "s3.region.amazonaws.com"
               },
               "StringLike": {
                   "kms:EncryptionContext:aws:s3:arn": "arn:aws:s3:::bucket-name/prefix*"
               }
           }
        },
        {
           "Effect": "Allow",
           "Action": [
               "logs:PutLogEvents"
           ],
           "Resource": [
               "arn:aws:logs:region:account-id:log-group:log-group-name:log-stream:log-stream-name"
           ]
        },
        {
           "Effect": "Allow", 
           "Action": [
               "lambda:InvokeFunction", 
               "lambda:GetFunctionConfiguration" 
           ],
           "Resource": [
               "arn:aws:lambda:region:account-id:function:function-name:function-version"
           ]
        }
    ]
}

Answer 2

@阿穷： But when I run terraform plan I get the following error:

你在循环中：

aws_kinesis_firehose_delivery_stream 正在等待 aws_iam_role 得到 aws_iam_role.firehose_role.arn
aws_iam_role 正在等待：aws_kinesis_firehose_delivery_stream 获取 aws_kinesis_firehose_delivery_stream.kinesis_event_stream.arn

所以我使用："Resource": "arn:aws:kinesis:ap-southeast-1:${var.account_id}:stream/${var.kinesis_firehose_delivery_stream_name}"

如何使用 Terraform 创建连接到 S3 的 AWS Kinesis Firehose？

How can I create an AWS Kinesis Firehose connected to S3 using Terraform?

amazon-s3

amazon-web-services

amazon-iam

amazon-kinesis

terraform

编辑