领事模板意外响应代码：504

Question

我正在使用 Consul-Template 使用从 Consul 收集的数据呈现属性。文件正在成功呈现，但之后 Consul-Template 一遍又一遍地失败，直到崩溃：

2017/08/16 10:43:04.900637 [WARN] (view) kv.list(...): Unexpected response code: 504 (retry attempt 5 after "4s")

这种情况一直持续下去，暂停时间更长，直到达到限制/崩溃。

从日志中提取的配置：

2017/08/16 10:38:02.403969 [DEBUG] (runner) final config:
{
    "Consul": {
        "Address": "consul-client.development.discovery",
        "Auth": {
            "Enabled": false,
            "Username": "",
            "Password": ""
        },
        "Retry": {
            "Attempts": 12,
            "Backoff": 250000000,
            "MaxBackoff": 60000000000,
            "Enabled": true
        },
        "SSL": {
            "CaCert": "",
            "CaPath": "",
            "Cert": "",
            "Enabled": false,
            "Key": "",
            "ServerName": "",
            "Verify": true
        },
        "Token": "",
        "Transport": {
            "DialKeepAlive": 30000000000,
            "DialTimeout": 30000000000,
            "DisableKeepAlives": false,
            "IdleConnTimeout": 90000000000,
            "MaxIdleConns": 100,
            "MaxIdleConnsPerHost": 3,
            "TLSHandshakeTimeout": 10000000000
        }
    },
    "Dedup": {
        "Enabled": false,
        "MaxStale": 2000000000,
        "Prefix": "consul-template/dedup/",
        "TTL": 15000000000
    },
    "Exec": {
        "Command": "",
        "Enabled": false,
        "Env": {
            "Blacklist": [],
            "Custom": [],
            "Pristine": false,
            "Whitelist": []
        },
        "KillSignal": 2,
        "KillTimeout": 30000000000,
        "ReloadSignal": null,
        "Splay": 0,
        "Timeout": 0
    },
    "KillSignal": 2,
    "LogLevel": "DEBUG",
    "MaxStale": 2000000000,
    "PidFile": "",
    "ReloadSignal": 1,
    "Syslog": {
        "Enabled": false,
        "Facility": "LOCAL0"
    },
    "Templates": [
        {
            "Backup": false,
            "Command": "",
            "CommandTimeout": 30000000000,
            "Contents": "",
            "Destination": "/consul-template/output/service.properties",
            "Exec": {
                "Command": "",
                "Enabled": false,
                "Env": {
                    "Blacklist": [],
                    "Custom": [],
                    "Pristine": false,
                    "Whitelist": []
                },
                "KillSignal": 2,
                "KillTimeout": 30000000000,
                "ReloadSignal": null,
                "Splay": 0,
                "Timeout": 30000000000
            },
            "Perms": 420,
            "Source": "/consul-template/data/service.properties.tpl",
            "Wait": {
                "Enabled": false,
                "Min": 0,
                "Max": 0
            },
            "LeftDelim": "",
            "RightDelim": ""
        },
        {
            "Backup": false,
            "Command": "",
            "CommandTimeout": 30000000000,
            "Contents": "",
            "Destination": "/consul-template/output/application.properties",
            "Exec": {
                "Command": "",
                "Enabled": false,
                "Env": {
                    "Blacklist": [],
                    "Custom": [],
                    "Pristine": false,
                    "Whitelist": []
                },
                "KillSignal": 2,
                "KillTimeout": 30000000000,
                "ReloadSignal": null,
                "Splay": 0,
                "Timeout": 30000000000
            },
            "Perms": 420,
            "Source": "/consul-template/data/application.properties.tpl",
            "Wait": {
                "Enabled": false,
                "Min": 0,
                "Max": 0
            },
            "LeftDelim": "",
            "RightDelim": ""
        },
        {
            "Backup": false,
            "Command": "",
            "CommandTimeout": 30000000000,
            "Contents": "",
            "Destination": "/consul-template/output/anotherMicroConsul.properties",
            "Exec": {
                "Command": "",
                "Enabled": false,
                "Env": {
                    "Blacklist": [],
                    "Custom": [],
                    "Pristine": false,
                    "Whitelist": []
                },
                "KillSignal": 2,
                "KillTimeout": 30000000000,
                "ReloadSignal": null,
                "Splay": 0,
                "Timeout": 30000000000
            },
            "Perms": 420,
            "Source": "/consul-template/data/anotherMicroConsul.properties.tpl",
            "Wait": {
                "Enabled": false,
                "Min": 0,
                "Max": 0
            },
            "LeftDelim": "",
            "RightDelim": ""
        },
        {
            "Backup": false,
            "Command": "",
            "CommandTimeout": 30000000000,
            "Contents": "",
            "Destination": "/consul-template/output/master.properties",
            "Exec": {
                "Command": "",
                "Enabled": false,
                "Env": {
                    "Blacklist": [],
                    "Custom": [],
                    "Pristine": false,
                    "Whitelist": []
                },
                "KillSignal": 2,
                "KillTimeout": 30000000000,
                "ReloadSignal": null,
                "Splay": 0,
                "Timeout": 30000000000
            },
            "Perms": 420,
            "Source": "/consul-template/data/master.properties.tpl",
            "Wait": {
                "Enabled": false,
                "Min": 0,
                "Max": 0
            },
            "LeftDelim": "",
            "RightDelim": ""
        },
        {
            "Backup": false,
            "Command": "",
            "CommandTimeout": 30000000000,
            "Contents": "",
            "Destination": "/consul-template/output/microConsul.properties",
            "Exec": {
                "Command": "",
                "Enabled": false,
                "Env": {
                    "Blacklist": [],
                    "Custom": [],
                    "Pristine": false,
                    "Whitelist": []
                },
                "KillSignal": 2,
                "KillTimeout": 30000000000,
                "ReloadSignal": null,
                "Splay": 0,
                "Timeout": 30000000000
            },
            "Perms": 420,
            "Source": "/consul-template/data/microConsul.properties.tpl",
            "Wait": {
                "Enabled": false,
                "Min": 0,
                "Max": 0
            },
            "LeftDelim": "",
            "RightDelim": ""
        }
    ],
    "Vault": {
        "Address": "",
        "Enabled": false,
        "Grace": 15000000000,
        "RenewToken": true,
        "Retry": {
            "Attempts": 12,
            "Backoff": 250000000,
            "MaxBackoff": 60000000000,
            "Enabled": true
        },
        "SSL": {
            "CaCert": "",
            "CaPath": "",
            "Cert": "",
            "Enabled": true,
            "Key": "",
            "ServerName": "",
            "Verify": true
        },
        "Transport": {
            "DialKeepAlive": 30000000000,
            "DialTimeout": 30000000000,
            "DisableKeepAlives": false,
            "IdleConnTimeout": 90000000000,
            "MaxIdleConns": 100,
            "MaxIdleConnsPerHost": 3,
            "TLSHandshakeTimeout": 10000000000
        },
        "UnwrapToken": false
    },
    "Wait": {
        "Enabled": false,
        "Min": 0,
        "Max": 0
    }
}

通过代码筛选此错误消息似乎来自 session.go 中的 Renew。

我应该在 /session/list 下在 Consul 中看到一个会话吗？我有多个 Consul-Template 容器运行（每台主机一个）——我应该看到几个会话吗？现在我什么都没看到...

curl localhost:8500/v1/session/list
[]

Answer 1

原因：您的 consul-template 和 consul 之间有一个代理或负载平衡器。

解决方法：将空闲连接的负载平衡器或代理超时增加到 60 秒以上（例如：65 秒或 120 秒），因为在此之后保持活动将启动并保持连接还活着:)

详情

HTTP 504 是网关超时，这意味着连接已 "cut" 或服务变得无响应，但我假设您的 consul 集群仍在工作，因此这是由于中间有一个活动的网络设备（在 consul-template 和 consul 之间）
Consul-template 打开持久连接以监听变化。因此，如果连接被负载均衡器静默终止，您最终会遇到这些 504 错误。
这些消息是良性的，因为 consul-template 可以正常重新连接。但是它们很烦人，并且会一遍又一遍地用相同的警告填满您的日志和磁盘 space ;-)
如果在 AWS 中，将 ELB 空闲超时设置为 65 秒，这些消息就会消失

领事模板意外响应代码：504

Consul-Template Unexpected response code: 504

consul

consul-template

consul-kv