来自连接到 Azure 容器实例的 Azure 应用程序网关的 502 Bad Gateway

Question

我正在学习 Terraform 和 Azure Web 服务。在学习了一系列教程之后，我一直在努力获取与虚拟网络中的 CosmosDB 实例对话的 Azure 容器实例设置，我想要一个允许 HTTP 连接到 Azure 容器实例的应用程序网关设置。

目前，当我调用分配给应用程序网关的 IP 地址时，收到 502 Bad Gateway。我已验证我在 Azure 容器实例中运行的图像在本地工作。我感觉我面临的问题与我配置的后端地址池有关，并且可能与我在网络安全组 (nsg-myapp) 中设置的规则有关。

我想知道是否有人可以查看我的 Terraform 并确定我未正确配置的内容？我在 Whosebug 上发现的最接近的问题类似于我的场景 this unresolved question from last year.

network.tf

resource "azurerm_virtual_network" "myappdb" {
  name                = "myappdb-vnet"
  address_space       = ["10.7.0.0/16"]
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name
}

resource "azurerm_subnet" "internal" {
  name                 = "internal"
  resource_group_name  = azurerm_resource_group.rg.name
  virtual_network_name = azurerm_virtual_network.myappdb.name
  address_prefixes     = ["10.7.2.0/24"]
  service_endpoints    = ["Microsoft.AzureCosmosDB"]

  delegation {
    name = "acidelegationservice"

    service_delegation {
      name    = "Microsoft.ContainerInstance/containerGroups"
      actions = ["Microsoft.Network/virtualNetworks/subnets/join/action", "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action"]
    }
  }

  enforce_private_link_endpoint_network_policies = true
}

resource "azurerm_subnet" "frontend" {
  name                 = "myapp-frontend"
  resource_group_name  = azurerm_resource_group.rg.name
  virtual_network_name = azurerm_virtual_network.myappdb.name
  address_prefixes     = ["10.7.0.0/24"]
}

resource "azurerm_network_security_group" "nsg-myapp" {
  name                = "nsg-aci"
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name

  security_rule {
    name              = "from-gateway-subnet"
    priority          = 100
    direction         = "Inbound"
    access            = "Allow"
    protocol          = "Tcp"
    source_port_range = "*"

    destination_port_ranges    = [22, 80, 443, 445, 8000]
    source_address_prefixes    = azurerm_subnet.internal.address_prefixes
    destination_address_prefix = azurerm_subnet.internal.address_prefixes[0]
  }

  security_rule {
    name                       = "DenyAllInBound-Override"
    priority                   = 900
    direction                  = "Inbound"
    access                     = "Deny"
    protocol                   = "*"
    source_port_range          = "*"
    destination_port_range     = "*"
    source_address_prefix      = "*"
    destination_address_prefix = "*"
  }


  security_rule {
    name              = "to-internet"
    priority          = 100
    direction         = "Outbound"
    access            = "Allow"
    protocol          = "Tcp"
    source_port_range = "*"

    destination_port_ranges    = [80, 443, 445]
    source_address_prefix      = "*"
    destination_address_prefix = "*"
  }

  security_rule {
    name                       = "DenyAllOutBound-Override"
    priority                   = 900
    direction                  = "Outbound"
    access                     = "Deny"
    protocol                   = "*"
    source_port_range          = "*"
    destination_port_range     = "*"
    source_address_prefix      = "*"
    destination_address_prefix = "*"
  }
}

resource "azurerm_subnet_network_security_group_association" "sn-nsg-aci" {
  subnet_id                 = azurerm_subnet.internal.id
  network_security_group_id = azurerm_network_security_group.nsg-myapp.id
}

resource "azurerm_network_profile" "containergroup_profile" {
  name                = "acg-profile"
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name

  container_network_interface {
    name = "acg-nic"

    ip_configuration {
      name      = "aciipconfig"
      subnet_id = azurerm_subnet.internal.id
    }
  }
}

resource "azurerm_public_ip" "myappip" {
  name                = "myappip"
  resource_group_name = azurerm_resource_group.rg.name
  location            = azurerm_resource_group.rg.location
  allocation_method   = "Static"
  sku                 = "Standard"
}

locals {
  backend_address_pool_name      = "${azurerm_virtual_network.myappdb.name}-beap"
  frontend_port_name             = "${azurerm_virtual_network.myappdb.name}-feport"
  frontend_ip_configuration_name = "${azurerm_virtual_network.myappdb.name}-feip"
  http_setting_name              = "${azurerm_virtual_network.myappdb.name}-be-htst"
  listener_name                  = "${azurerm_virtual_network.myappdb.name}-httplstn"
  request_routing_rule_name      = "${azurerm_virtual_network.myappdb.name}-rqrt"
  redirect_configuration_name    = "${azurerm_virtual_network.myappdb.name}-rdrcfg"
}

resource "azurerm_application_gateway" "network" {
  name                = "myapp-appgateway"
  resource_group_name = azurerm_resource_group.rg.name
  location            = azurerm_resource_group.rg.location

  sku {
    name     = "Standard_v2"
    tier     = "Standard_v2"
    capacity = 2
  }

  gateway_ip_configuration {
    name      = "my-gateway-ip-configuration"
    subnet_id = azurerm_subnet.frontend.id
  }

  frontend_port {
    name = local.frontend_port_name
    port = 80
  }

  frontend_ip_configuration {
    name                 = local.frontend_ip_configuration_name
    public_ip_address_id = azurerm_public_ip.myappip.id
  }

  backend_address_pool {
    name  = local.backend_address_pool_name
    ip_addresses = [azurerm_container_group.tf_cg_sampleapi.ip_address]
  }

  backend_http_settings {
    name                  = local.http_setting_name
    cookie_based_affinity = "Disabled"
    path                  = "/path1/"
    port                  = 80
    protocol              = "Http"
    request_timeout       = 60
  }

  http_listener {
    name                           = local.listener_name
    frontend_ip_configuration_name = local.frontend_ip_configuration_name
    frontend_port_name             = local.frontend_port_name
    protocol                       = "Http"
  }

  request_routing_rule {
    name                       = local.request_routing_rule_name
    rule_type                  = "Basic"
    http_listener_name         = local.listener_name
    backend_address_pool_name  = local.backend_address_pool_name
    backend_http_settings_name = local.http_setting_name
  }
}

container.tf

resource "azurerm_container_group" "tf_cg_sampleapi" {
  depends_on = [azurerm_cosmosdb_account.db]

  name                = "cg_myapp"
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name

  network_profile_id = azurerm_network_profile.containergroup_profile.id

  ip_address_type = "Private"
  # dns_name_label  = "sampleapitf"
  os_type         = "Linux"

  identity {
    type = "SystemAssigned"
  }

  container {
    name   = "myapp"
    image  = "sample/myapp"
    cpu    = 1
    memory = 1

    ports {
      port     = 80
      protocol = "TCP"
    }

    ports {
      port     = 443
      protocol = "TCP"
    }

    secure_environment_variables = {
      "MYAPP_CONNECTION_STRING" = azurerm_cosmosdb_account.db.connection_strings[0]
    }
  }
}

Answer 1

我遇到了类似的问题，在我的情况下（Azure App Service 上的容器）我需要将 depends_on 块放在应用程序网关资源创建中，以便在第一个中创建应用程序服务地方。所以在你的情况下应该是：

resource "azurerm_application_gateway" "network" {
  name                = "myapp-appgateway"
  resource_group_name = azurerm_resource_group.rg.name
  location            = azurerm_resource_group.rg.location

  sku {
    name     = "Standard_v2"
    tier     = "Standard_v2"
    capacity = 2
  }

  gateway_ip_configuration {
    name      = "my-gateway-ip-configuration"
    subnet_id = azurerm_subnet.frontend.id
  }

  frontend_port {
    name = local.frontend_port_name
    port = 80
  }

  frontend_ip_configuration {
    name                 = local.frontend_ip_configuration_name
    public_ip_address_id = azurerm_public_ip.myappip.id
  }

  backend_address_pool {
    name  = local.backend_address_pool_name
    ip_addresses = [azurerm_container_group.tf_cg_sampleapi.ip_address]
  }

  backend_http_settings {
    name                  = local.http_setting_name
    cookie_based_affinity = "Disabled"
    path                  = "/path1/"
    port                  = 80
    protocol              = "Http"
    request_timeout       = 60
  }

  http_listener {
    name                           = local.listener_name
    frontend_ip_configuration_name = local.frontend_ip_configuration_name
    frontend_port_name             = local.frontend_port_name
    protocol                       = "Http"
  }

  request_routing_rule {
    name                       = local.request_routing_rule_name
    rule_type                  = "Basic"
    http_listener_name         = local.listener_name
    backend_address_pool_name  = local.backend_address_pool_name
    backend_http_settings_name = local.http_setting_name
  }
  depends_on = [ azurerm_container_group.tf_cg_sampleapi, ]
}

Answer 2

我发现我的 502 网关错误的根本原因是健康检查没有设置/不工作。因此，我设置了自定义探测器，该探测器将转到 API 端点以 return 200 OK 响应。当然，我将配置此端点以实际检查我是否可以连接到我的服务，但这只是一个验证问题所在的测试。

我还删除了 nsg-aci 安全组中的 DenyAllInBound-Override 和 DenyAllOutBound-Override 规则，因为这导致我的 ACI 连接到我的 Cosmos DB 时出现问题。

这是我生成的 network.tf 和 container.tf 文件：

network.tf

resource "azurerm_virtual_network" "myappdb" {
  name                = "myappdb-vnet"
  address_space       = ["10.7.0.0/16"]
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name
}

resource "azurerm_subnet" "internal" {
  name                 = "internal"
  resource_group_name  = azurerm_resource_group.rg.name
  virtual_network_name = azurerm_virtual_network.myappdb.name
  address_prefixes     = ["10.7.2.0/24"]
  service_endpoints    = ["Microsoft.AzureCosmosDB"]

  delegation {
    name = "acidelegationservice"

    service_delegation {
      name    = "Microsoft.ContainerInstance/containerGroups"
      actions = ["Microsoft.Network/virtualNetworks/subnets/join/action", "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action"]
    }
  }

  enforce_private_link_endpoint_network_policies = true
}

resource "azurerm_subnet" "frontend" {
  name                 = "myapp-frontend"
  resource_group_name  = azurerm_resource_group.rg.name
  virtual_network_name = azurerm_virtual_network.myappdb.name
  address_prefixes     = ["10.7.0.0/24"]
}

resource "azurerm_network_security_group" "nsg-myapp" {
  name                = "nsg-aci"
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name

  security_rule {
    name              = "from-gateway-subnet"
    priority          = 100
    direction         = "Inbound"
    access            = "Allow"
    protocol          = "Tcp"
    source_port_range = "*"

    destination_port_ranges      = [22, 80, 443, 445, 8000]
    source_address_prefixes      = azurerm_subnet.internal.address_prefixes
    destination_address_prefixes = azurerm_subnet.internal.address_prefixes
  }

  security_rule {
    name              = "to-internet"
    priority          = 100
    direction         = "Outbound"
    access            = "Allow"
    protocol          = "Tcp"
    source_port_range = "*"

    destination_port_ranges    = [80, 443, 445]
    source_address_prefix      = "*"
    destination_address_prefix = "*"
  }
}

resource "azurerm_subnet_network_security_group_association" "sn-nsg-aci" {
  subnet_id                 = azurerm_subnet.internal.id
  network_security_group_id = azurerm_network_security_group.nsg-myapp.id
}

resource "azurerm_network_profile" "containergroup_profile" {
  name                = "acg-profile"
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name

  container_network_interface {
    name = "acg-nic"

    ip_configuration {
      name      = "aciipconfig"
      subnet_id = azurerm_subnet.internal.id
    }
  }
}

resource "azurerm_public_ip" "myappip" {
  name                = "myappip"
  resource_group_name = azurerm_resource_group.rg.name
  location            = azurerm_resource_group.rg.location
  allocation_method   = "Static"
  sku                 = "Standard"
}

locals {
  backend_address_pool_name      = "${azurerm_virtual_network.myappdb.name}-beap"
  frontend_port_name             = "${azurerm_virtual_network.myappdb.name}-feport"
  frontend_ip_configuration_name = "${azurerm_virtual_network.myappdb.name}-feip"
  http_setting_name              = "${azurerm_virtual_network.myappdb.name}-be-htst"
  listener_name                  = "${azurerm_virtual_network.myappdb.name}-httplstn"
  request_routing_rule_name      = "${azurerm_virtual_network.myappdb.name}-rqrt"
  redirect_configuration_name    = "${azurerm_virtual_network.myappdb.name}-rdrcfg"
}

resource "azurerm_application_gateway" "network" {
  name                = "myapp-appgateway"
  resource_group_name = azurerm_resource_group.rg.name
  location            = azurerm_resource_group.rg.location

  sku {
    name     = "Standard_v2"
    tier     = "Standard_v2"
    capacity = 2
  }

  gateway_ip_configuration {
    name      = "my-gateway-ip-configuration"
    subnet_id = azurerm_subnet.frontend.id
  }

  frontend_port {
    name = local.frontend_port_name
    port = 80
  }

  frontend_ip_configuration {
    name                 = local.frontend_ip_configuration_name
    public_ip_address_id = azurerm_public_ip.myappip.id
  }

  backend_address_pool {
    name         = local.backend_address_pool_name
    ip_addresses = [azurerm_container_group.tf_cg_sampleapi.ip_address]
  }

  probe {
    interval            = 60
    timeout             = 60
    name                = "status"
    protocol            = "Http"
    path                = "/api/status/"
    unhealthy_threshold = 3
    host                = "127.0.0.1"
  }

  backend_http_settings {
    name                  = local.http_setting_name
    cookie_based_affinity = "Disabled"
    path                  = "/"
    port                  = 80
    protocol              = "Http"
    request_timeout       = 60
    probe_name            = "status"
  }

  http_listener {
    name                           = local.listener_name
    frontend_ip_configuration_name = local.frontend_ip_configuration_name
    frontend_port_name             = local.frontend_port_name
    protocol                       = "Http"
  }

  request_routing_rule {
    name                       = local.request_routing_rule_name
    rule_type                  = "Basic"
    http_listener_name         = local.listener_name
    backend_address_pool_name  = local.backend_address_pool_name
    backend_http_settings_name = local.http_setting_name
  }

  depends_on = [azurerm_container_group.tf_cg_sampleapi, ]
}

container.tf

resource "azurerm_container_group" "tf_cg_sampleapi" {
  depends_on = [azurerm_cosmosdb_account.db]

  name                = "cg_myapp"
  location            = azurerm_resource_group.rg.location
  resource_group_name = azurerm_resource_group.rg.name

  network_profile_id = azurerm_network_profile.containergroup_profile.id

  ip_address_type = "Private"
  # dns_name_label  = "sampleapitf"
  os_type = "Linux"

  container {
    name   = "myapp"
    image  = "sample/myapp"
    cpu    = 1
    memory = 1

    ports {
      port     = 80
      protocol = "TCP"
    }

    ports {
      port     = 443
      protocol = "TCP"
    }

    secure_environment_variables = {
      "MYAPP_CONNECTION_STRING" = azurerm_cosmosdb_account.db.connection_strings[0]
    }
  }
}

来自连接到 Azure 容器实例的 Azure 应用程序网关的 502 Bad Gateway

502 Bad Gateway from Azure Application Gateway Connecting to Azure Container Instance

azure

terraform

azure-application-gateway

azure-container-instances