HAProxy 和 websockets 从不连接
HAProxy and websockets never connect
我认为 HAProxy 2.4 支持基于 h2 的 WebSockets，我最终能够通过 HAProxy 让 websockets 工作......但是没有......
当我的浏览器尝试连接时，它会立即放弃“connect?transport=webSockets”并尝试 SSE。这个“连接”也立即被放弃，但它“开始？传输= SSE”就好了。
登录 HAProxy 说它从未收到连接请求。
Nov 17 17:10:19 127.0.0.1 haproxy[26334]: 192.168.126.78:1228 [17/Nov/2021:17:10:18.520] www-httpsStrict~ test_backend/werdc2020 557/0/0/2/558 200 704 - - --VN 2/1/0/0/0 0/0 "GET https://myserver.com/signalr/negotiate?clientProtocol=2.1&connectionData=%5B%7B%22name%22%3A%22testservice%22%7D%5D&_=1637143818090 HTTP/2.0"
Nov 17 17:10:19 127.0.0.1 haproxy[26334]: 192.168.126.78:1228 [17/Nov/2021:17:10:19.075] www-httpsStrict~ test_backend/werdc2020 52/0/0/1/52 404 1392 - - --VN 2/1/0/0/0 0/0 "GET https://myserver.com/favicon.ico HTTP/2.0"
Nov 17 17:10:19 127.0.0.1 haproxy[26334]: 192.168.126.78:1228 [17/Nov/2021:17:10:19.279] www-httpsStrict~ test_backend/werdc2020 96/0/0/7/103 200 339 - - --VN 3/2/1/1/0 0/0 "GET https://myserver.com/signalr/start?transport=serverSentEvents&clientProtocol=2.1&connectionToken=%2FZ99uK3N88wIyXYxQlQc4w42M3jcf0Tz6tbLcm7CXhOcTgTu7qgbfsMxn9l6GNP%2FML39o%2BabOZ4GSQDgAxo7oYXIacee8Ku2Nd1QXvWRalisPY%2BTIgddfJsWdE828LaH&connectionData=%5B%7B%22name%22%3A%22testservice%22%7D%5D&_=1637143818091 HTTP/2.0"
Nov 17 17:10:19 127.0.0.1 haproxy[26334]: 192.168.126.78:1228 [17/Nov/2021:17:10:19.375] www-httpsStrict~ test_backend/werdc2020 65/0/0/20/84 200 348 - - --VN 3/2/1/1/0 0/0 "POST https://myserver.com/signalr/send?transport=serverSentEvents&clientProtocol=2.1&connectionToken=%2FZ99uK3N88wIyXYxQlQc4w42M3jcf0Tz6tbLcm7CXhOcTgTu7qgbfsMxn9l6GNP%2FML39o%2BabOZ4GSQDgAxo7oYXIacee8Ku2Nd1QXvWRalisPY%2BTIgddfJsWdE828LaH&connectionData=%5B%7B%22name%22%3A%22testservice%22%7D%5D HTTP/2.0"
如果我直接连接到服务器上，它工作得很好...
HAProxy 配置：
global
log 127.0.0.1 local0 debug
log-tag haproxy
maxconn 10000
user haproxy
group haproxy
daemon
nbproc 1
nbthread 8
#cpu-map auto:all 0-1
stats socket /var/run/haproxy.sock mode 600 level admin
stats timeout 30s
pidfile /var/run/haproxy.pid
tune.ssl.default-dh-param 2048
tune.ssl.cachesize 100000
tune.ssl.lifetime 600
tune.ssl.maxrecord 1460
# intermediate configuration
ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:!MD5:!aNULL:!DH:!RC4
ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
defaults
log global
mode http
option httplog
option dontlognull
timeout connect 5s
timeout client 1m
timeout client-fin 1m
timeout server 1h
timeout tunnel 1h
frontend stats
bind 0.0.0.0:8080
mode http
stats enable
stats hide-version
stats realm Haproxy\ Statistics
stats uri /
stats auth myuser:1111
stats refresh 60s
# Force front server MUST use redirect on HTTPS
frontend www-httpStrict
bind *:80
option splice-auto
# Test URI to see if its a letsencrypt request
acl letsencrypt-acl path_beg /.well-known
acl test hdr_sub(cookie) istest=true
redirect scheme https code 307 if !{ ssl_fc } !letsencrypt-acl
use_backend letsencrypt if letsencrypt-acl
use_backend test_backend if test
default_backend www-backendStrict
frontend www-httpsStrict
bind *:443 tfo ssl crt /etc/ssl/private alpn h2,http/1.1
option forwardfor
option splice-auto
acl test hdr_sub(cookie) istest=true
use_backend test_backend if test
default_backend www-backendStrict
backend letsencrypt
server nginx 127.0.0.1:8888
# this backend require haproxy open SSL tunel to port 443 on webservers
backend www-backendStrict
balance roundrobin
cookie MyWebFarm insert
option forwardfor
option splice-auto
option tcp-smart-connect
option httpchk
http-check connect ssl alpn h2 sni myserver.com
http-check send meth HEAD uri /login.aspx ver HTTP/2 hdr Host myserver.com
http-check expect status 200-399
http-request add-header X-Forwarded-Proto https if { ssl_fc }
retry-on all-retryable-errors
http-request disable-l7-retry if METH_POST
default-server ssl tfo verify none alpn h2,http/1.1 check allow-0rtt
server werdc01 192.168.1.5:443 cookie werc01 check
server werdc2020 192.168.1.6:443 cookie werdc2020 check
backend test_backend
balance roundrobin
cookie MyWebFarm insert
option forwardfor
option splice-auto
option tcp-smart-connect
option httpchk
http-check connect ssl alpn h2 sni myserver.com
http-check send meth HEAD uri /login.aspx ver HTTP/2 hdr Host myserver.com
http-check expect status 200-399
http-request add-header X-Forwarded-Proto https if { ssl_fc }
retry-on all-retryable-errors
http-request disable-l7-retry if METH_POST
default-server ssl verify none alpn h2,http/1.1 check allow-0rtt
server werdc2020 192.168.1.6:443 cookie werdc2020 check
我将 istest cookie 设置为始终点击 test_backend...
Headers 从浏览器发送：
GET /signalr/connect?transport=webSockets&clientProtocol=2.1&connectionToken=%2FZ99uK3N88wIyXYxQlQc4w42M3jcf0Tz6tbLcm7CXhOcTgTu7qgbfsMxn9l6GNP%2FML39o%2BabOZ4GSQDgAxo7oYXIacee8Ku2Nd1QXvWRalisPY%2BTIgddfJsWdE828LaH&connectionData=%5B%7B%22name%22%3A%22catiservice%22%7D%5D&tid=7 undefined
Host: myserver.com
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0
Accept: */*
Accept-Language: en-US,en;q=0.5
Accept-Encoding: gzip, deflate, br
Sec-WebSocket-Version: 13
Origin: https://myserver.com
Sec-WebSocket-Extensions: permessage-deflate
Sec-WebSocket-Key: P1GaqaLxIxBfuXDCtxuQwg==
DNT: 1
Connection: keep-alive, Upgrade
Cookie: _ga=GA1.2.95033637.1627965955; .MyAuth=D758016C45CF1B9BDC38D17DCAF7A3CEE7E528B97352267A1B0D019BFE20964BF2900C655950E126CD7FC08CB421C1A997D1A5FDB0266316B920634FB2C23B63080B271B6331096902B47BF73661240D; MyWebFarm=werdc2020; ASP.NET_SessionId=w1zibnnuht4ohicpntyhn55q; istest=true
Sec-Fetch-Dest: websocket
Sec-Fetch-Mode: websocket
Sec-Fetch-Site: same-origin
Pragma: no-cache
Cache-Control: no-cache
Upgrade: websocket
TE: trailers
为什么 HAProxy 看不到“连接”请求？
为什么它不与 WebSockets 连接？
这里有一个 github 问题：https://github.com/haproxy/haproxy/issues/162
虽然你的问题不是很有见地，但我建议将你的配置剥离到最低限度以测试 websocket。
这里有一个要考虑的片段
option http-server-close
option http-use-htx
acl hdr_connection_upgrade hdr(Connection) -i upgrade
acl hdr_upgrade_websocket hdr(Upgrade) -i websocket
Why doesn't HAProxy see the "connect" requests?
我想，只是因为CONNECT没有成功。如果您使用 tcpdump，您肯定会看到收到的请求。
使用 中的解释，此配置有效（至少在 2.5 修复问题之前）：
global
log 127.0.0.1 local0 debug
log-tag haproxy
maxconn 10000
user haproxy
group haproxy
daemon
nbproc 1
nbthread 8
#cpu-map auto:all 0-1
stats socket /var/run/haproxy.sock mode 600 level admin
stats timeout 30s
pidfile /var/run/haproxy.pid
tune.ssl.default-dh-param 2048
tune.ssl.cachesize 100000
tune.ssl.lifetime 600
tune.ssl.maxrecord 1460
# intermediate configuration
ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:!MD5:!aNULL:!DH:!RC4
ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
defaults all
log global
mode http
option httplog
option dontlognull
timeout connect 5s
timeout client 1m
timeout client-fin 1m
timeout server 1h
timeout tunnel 4h
option splice-auto
option tcp-smart-connect
balance roundrobin
option forwardfor
retry-on all-retryable-errors
frontend stats from all
bind *:8080
stats enable
stats hide-version
stats realm Haproxy\ Statistics
stats uri /
stats auth Myuser:11111
stats refresh 60s
listen www from all
bind *:80
bind *:443 tfo ssl crt /etc/ssl/private alpn h2,http/1.1 allow-0rtt
# Test URI to see if its a letsencrypt request
acs letsencrypt path_beg /.well-known
http-request redirect scheme https unless { ssl_fc } or letsencrypt
use_backend letsencrypt if letsencrypt
http-response set-header Strict-Transport-Security "max-age=10886400; includeSubDomains; preload;"
use_backend ws if { hdr(Connection) -i upgrade } or { hdr(Upgrade) -i websocket }
default_backend h2
backend letsencrypt
server letsencrypt 127.0.0.1:8888
defaults cg from all
cookie MyWebFarm insert
option httpchk
http-check connect ssl alpn h2 sni myserver.com
http-check send meth HEAD uri /login.aspx ver HTTP/2 hdr Host myserver.com
http-check expect status 200-399
backend h2 from cg
default-server ssl tfo verify none alpn h2 check allow-0rtt
http-request disable-l7-retry if METH_POST
use-server werdc2020 if { hdr_sub(cookie) istest=true }
server werdc01 192.168.1.5:443 cookie werc01
server werdc2020 192.168.1.6:443 cookie werdc2020
backend ws from cg
default-server ssl tfo verify none alpn http/1.1 check allow-0rtt
http-request disable-l7-retry if METH_POST
use-server werdc2020 if { hdr_sub(cookie) istest=true }
server werdc01 192.168.1.5:443 cookie werc01
server werdc2020 192.168.1.6:443 cookie werdc2020
我认为 HAProxy 2.4 支持基于 h2 的 WebSockets，我最终能够通过 HAProxy 让 websockets 工作......但是没有......
当我的浏览器尝试连接时，它会立即放弃“connect?transport=webSockets”并尝试 SSE。这个“连接”也立即被放弃，但它“开始？传输= SSE”就好了。
登录 HAProxy 说它从未收到连接请求。
Nov 17 17:10:19 127.0.0.1 haproxy[26334]: 192.168.126.78:1228 [17/Nov/2021:17:10:18.520] www-httpsStrict~ test_backend/werdc2020 557/0/0/2/558 200 704 - - --VN 2/1/0/0/0 0/0 "GET https://myserver.com/signalr/negotiate?clientProtocol=2.1&connectionData=%5B%7B%22name%22%3A%22testservice%22%7D%5D&_=1637143818090 HTTP/2.0"
Nov 17 17:10:19 127.0.0.1 haproxy[26334]: 192.168.126.78:1228 [17/Nov/2021:17:10:19.075] www-httpsStrict~ test_backend/werdc2020 52/0/0/1/52 404 1392 - - --VN 2/1/0/0/0 0/0 "GET https://myserver.com/favicon.ico HTTP/2.0"
Nov 17 17:10:19 127.0.0.1 haproxy[26334]: 192.168.126.78:1228 [17/Nov/2021:17:10:19.279] www-httpsStrict~ test_backend/werdc2020 96/0/0/7/103 200 339 - - --VN 3/2/1/1/0 0/0 "GET https://myserver.com/signalr/start?transport=serverSentEvents&clientProtocol=2.1&connectionToken=%2FZ99uK3N88wIyXYxQlQc4w42M3jcf0Tz6tbLcm7CXhOcTgTu7qgbfsMxn9l6GNP%2FML39o%2BabOZ4GSQDgAxo7oYXIacee8Ku2Nd1QXvWRalisPY%2BTIgddfJsWdE828LaH&connectionData=%5B%7B%22name%22%3A%22testservice%22%7D%5D&_=1637143818091 HTTP/2.0"
Nov 17 17:10:19 127.0.0.1 haproxy[26334]: 192.168.126.78:1228 [17/Nov/2021:17:10:19.375] www-httpsStrict~ test_backend/werdc2020 65/0/0/20/84 200 348 - - --VN 3/2/1/1/0 0/0 "POST https://myserver.com/signalr/send?transport=serverSentEvents&clientProtocol=2.1&connectionToken=%2FZ99uK3N88wIyXYxQlQc4w42M3jcf0Tz6tbLcm7CXhOcTgTu7qgbfsMxn9l6GNP%2FML39o%2BabOZ4GSQDgAxo7oYXIacee8Ku2Nd1QXvWRalisPY%2BTIgddfJsWdE828LaH&connectionData=%5B%7B%22name%22%3A%22testservice%22%7D%5D HTTP/2.0"
如果我直接连接到服务器上，它工作得很好...
HAProxy 配置：
global
log 127.0.0.1 local0 debug
log-tag haproxy
maxconn 10000
user haproxy
group haproxy
daemon
nbproc 1
nbthread 8
#cpu-map auto:all 0-1
stats socket /var/run/haproxy.sock mode 600 level admin
stats timeout 30s
pidfile /var/run/haproxy.pid
tune.ssl.default-dh-param 2048
tune.ssl.cachesize 100000
tune.ssl.lifetime 600
tune.ssl.maxrecord 1460
# intermediate configuration
ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:!MD5:!aNULL:!DH:!RC4
ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
defaults
log global
mode http
option httplog
option dontlognull
timeout connect 5s
timeout client 1m
timeout client-fin 1m
timeout server 1h
timeout tunnel 1h
frontend stats
bind 0.0.0.0:8080
mode http
stats enable
stats hide-version
stats realm Haproxy\ Statistics
stats uri /
stats auth myuser:1111
stats refresh 60s
# Force front server MUST use redirect on HTTPS
frontend www-httpStrict
bind *:80
option splice-auto
# Test URI to see if its a letsencrypt request
acl letsencrypt-acl path_beg /.well-known
acl test hdr_sub(cookie) istest=true
redirect scheme https code 307 if !{ ssl_fc } !letsencrypt-acl
use_backend letsencrypt if letsencrypt-acl
use_backend test_backend if test
default_backend www-backendStrict
frontend www-httpsStrict
bind *:443 tfo ssl crt /etc/ssl/private alpn h2,http/1.1
option forwardfor
option splice-auto
acl test hdr_sub(cookie) istest=true
use_backend test_backend if test
default_backend www-backendStrict
backend letsencrypt
server nginx 127.0.0.1:8888
# this backend require haproxy open SSL tunel to port 443 on webservers
backend www-backendStrict
balance roundrobin
cookie MyWebFarm insert
option forwardfor
option splice-auto
option tcp-smart-connect
option httpchk
http-check connect ssl alpn h2 sni myserver.com
http-check send meth HEAD uri /login.aspx ver HTTP/2 hdr Host myserver.com
http-check expect status 200-399
http-request add-header X-Forwarded-Proto https if { ssl_fc }
retry-on all-retryable-errors
http-request disable-l7-retry if METH_POST
default-server ssl tfo verify none alpn h2,http/1.1 check allow-0rtt
server werdc01 192.168.1.5:443 cookie werc01 check
server werdc2020 192.168.1.6:443 cookie werdc2020 check
backend test_backend
balance roundrobin
cookie MyWebFarm insert
option forwardfor
option splice-auto
option tcp-smart-connect
option httpchk
http-check connect ssl alpn h2 sni myserver.com
http-check send meth HEAD uri /login.aspx ver HTTP/2 hdr Host myserver.com
http-check expect status 200-399
http-request add-header X-Forwarded-Proto https if { ssl_fc }
retry-on all-retryable-errors
http-request disable-l7-retry if METH_POST
default-server ssl verify none alpn h2,http/1.1 check allow-0rtt
server werdc2020 192.168.1.6:443 cookie werdc2020 check
我将 istest cookie 设置为始终点击 test_backend...
Headers 从浏览器发送：
GET /signalr/connect?transport=webSockets&clientProtocol=2.1&connectionToken=%2FZ99uK3N88wIyXYxQlQc4w42M3jcf0Tz6tbLcm7CXhOcTgTu7qgbfsMxn9l6GNP%2FML39o%2BabOZ4GSQDgAxo7oYXIacee8Ku2Nd1QXvWRalisPY%2BTIgddfJsWdE828LaH&connectionData=%5B%7B%22name%22%3A%22catiservice%22%7D%5D&tid=7 undefined
Host: myserver.com
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0
Accept: */*
Accept-Language: en-US,en;q=0.5
Accept-Encoding: gzip, deflate, br
Sec-WebSocket-Version: 13
Origin: https://myserver.com
Sec-WebSocket-Extensions: permessage-deflate
Sec-WebSocket-Key: P1GaqaLxIxBfuXDCtxuQwg==
DNT: 1
Connection: keep-alive, Upgrade
Cookie: _ga=GA1.2.95033637.1627965955; .MyAuth=D758016C45CF1B9BDC38D17DCAF7A3CEE7E528B97352267A1B0D019BFE20964BF2900C655950E126CD7FC08CB421C1A997D1A5FDB0266316B920634FB2C23B63080B271B6331096902B47BF73661240D; MyWebFarm=werdc2020; ASP.NET_SessionId=w1zibnnuht4ohicpntyhn55q; istest=true
Sec-Fetch-Dest: websocket
Sec-Fetch-Mode: websocket
Sec-Fetch-Site: same-origin
Pragma: no-cache
Cache-Control: no-cache
Upgrade: websocket
TE: trailers
为什么 HAProxy 看不到“连接”请求？
为什么它不与 WebSockets 连接？
这里有一个 github 问题：https://github.com/haproxy/haproxy/issues/162
虽然你的问题不是很有见地，但我建议将你的配置剥离到最低限度以测试 websocket。
这里有一个要考虑的片段
option http-server-close
option http-use-htx
acl hdr_connection_upgrade hdr(Connection) -i upgrade
acl hdr_upgrade_websocket hdr(Upgrade) -i websocket
Why doesn't HAProxy see the "connect" requests?
我想，只是因为CONNECT没有成功。如果您使用 tcpdump，您肯定会看到收到的请求。
使用
global
log 127.0.0.1 local0 debug
log-tag haproxy
maxconn 10000
user haproxy
group haproxy
daemon
nbproc 1
nbthread 8
#cpu-map auto:all 0-1
stats socket /var/run/haproxy.sock mode 600 level admin
stats timeout 30s
pidfile /var/run/haproxy.pid
tune.ssl.default-dh-param 2048
tune.ssl.cachesize 100000
tune.ssl.lifetime 600
tune.ssl.maxrecord 1460
# intermediate configuration
ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:!MD5:!aNULL:!DH:!RC4
ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
defaults all
log global
mode http
option httplog
option dontlognull
timeout connect 5s
timeout client 1m
timeout client-fin 1m
timeout server 1h
timeout tunnel 4h
option splice-auto
option tcp-smart-connect
balance roundrobin
option forwardfor
retry-on all-retryable-errors
frontend stats from all
bind *:8080
stats enable
stats hide-version
stats realm Haproxy\ Statistics
stats uri /
stats auth Myuser:11111
stats refresh 60s
listen www from all
bind *:80
bind *:443 tfo ssl crt /etc/ssl/private alpn h2,http/1.1 allow-0rtt
# Test URI to see if its a letsencrypt request
acs letsencrypt path_beg /.well-known
http-request redirect scheme https unless { ssl_fc } or letsencrypt
use_backend letsencrypt if letsencrypt
http-response set-header Strict-Transport-Security "max-age=10886400; includeSubDomains; preload;"
use_backend ws if { hdr(Connection) -i upgrade } or { hdr(Upgrade) -i websocket }
default_backend h2
backend letsencrypt
server letsencrypt 127.0.0.1:8888
defaults cg from all
cookie MyWebFarm insert
option httpchk
http-check connect ssl alpn h2 sni myserver.com
http-check send meth HEAD uri /login.aspx ver HTTP/2 hdr Host myserver.com
http-check expect status 200-399
backend h2 from cg
default-server ssl tfo verify none alpn h2 check allow-0rtt
http-request disable-l7-retry if METH_POST
use-server werdc2020 if { hdr_sub(cookie) istest=true }
server werdc01 192.168.1.5:443 cookie werc01
server werdc2020 192.168.1.6:443 cookie werdc2020
backend ws from cg
default-server ssl tfo verify none alpn http/1.1 check allow-0rtt
http-request disable-l7-retry if METH_POST
use-server werdc2020 if { hdr_sub(cookie) istest=true }
server werdc01 192.168.1.5:443 cookie werc01
server werdc2020 192.168.1.6:443 cookie werdc2020