最大化从haproxy到jetty应用服务器的tcp连接速率
Maximizing tcp connection rate from haproxy to jetty application server
我正在使用带有 nginx 的 haproxy 前端在码头应用程序服务器之间进行负载平衡，所有 运行 ubuntu-14-04-x64。
Nginx 和 haproxy 共享一个 0.5G 1CPU VM。每个码头服务器都运行在 4G 2CPU VM 上。
Nginx前端haproxy，配置如下：
user myc;
worker_processes 1;
pid /run/nginx.pid;
events {
worker_connections 65536;
multi_accept on;
use epoll;
}
http {
include mime.types;
types_hash_max_size 2048;
default_type application/octet-stream;
proxy_cache_path /home/myc/cache levels=1:2 keys_zone=one:10m inactive=7d;
proxy_http_version 1.1;
proxy_set_header Connection "";
upstream loadbalancer {
server unix:/tmp/haproxy.sock;
keepalive 8192;
}
server {
listen 80 backlog=16384;
server_name example.org;
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
gzip on;
gzip_types text/plain application/javascript application/x-javascript text/javascript text/xml text/css application/json;
gzip_vary on;
gzip_min_length 10240;
open_file_cache max=2000 inactive=20s;
open_file_cache_valid 60s;
open_file_cache_min_uses 5;
open_file_cache_errors off;
sendfile on;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
error_page 502 /502.html;
location /502.html {
root /home/myc/html;
}
proxy_cache one;
location / {
proxy_pass http://loadbalancer/;
proxy_connect_timeout 60s;
proxy_read_timeout 60s;
add_header Cache-Control private; # Don't cache any of this publicly, because JSESSIONID will be cached, too
add_header X-Proxy-Cache $upstream_cache_status;
}
location /api {
proxy_pass http://loadbalancer/api;
proxy_connect_timeout 60s;
proxy_read_timeout 60s;
proxy_ignore_headers Expires;
proxy_hide_header Expires;
proxy_ignore_headers Set-Cookie; # Make sure JSESSIONID is not cached with static content
proxy_hide_header Set-Cookie;
proxy_cache_valid 200 10s;
add_header X-Proxy-Cache $upstream_cache_status;
}
location ~* ^.*\.(css|js|gif|jpe?g|png|ico)$ {
proxy_pass http://loadbalancer$uri;
proxy_connect_timeout 60s;
proxy_read_timeout 60s;
expires 1h;
access_log off;
log_not_found off;
proxy_ignore_headers Expires;
proxy_hide_header Expires;
proxy_ignore_headers Set-Cookie; # Make sure JSESSIONID is not cached with static content
proxy_hide_header Set-Cookie;
add_header Cache-Control public;
proxy_cache_valid 200 404 10s;
add_header X-Proxy-Cache $upstream_cache_status;
}
}
}
HAProxy 配置如下：
global
daemon
maxconn 262144
maxconnrate 128
chroot /home/jail/
defaults
mode http
timeout connect 10s
timeout client 120s
timeout server 120s
errorfile 502 /etc/haproxy/errors/502.http
errorfile 503 /etc/haproxy/errors/503.http
compression algo identity
option http-keep-alive
frontend http-in
bind /tmp/haproxy.sock user myc
default_backend makeyourcase
backend makeyourcase
option httpchk GET /
appsession JSESSIONID len 52 timeout 1h
server ny2-b-app01 <redacted>:8080 maxconn 1024 check inter 60000 fall 1 rise 1
server ny2-b-app02 <redacted>:8080 maxconn 1024 check inter 60000 fall 1 rise 1
TCP 堆栈在 nginx/haproxy 框和两个码头框上使用这些参数进行调整：
net.core.netdev_max_backlog = 65535
net.core.somaxconn = 65535
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rfc1337 = 1
net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.tcp_fin_timeout = 30
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_max_syn_backlog = 65535
net.ipv4.tcp_moderate_rcvbuf = 1
net.ipv4.tcp_syncookies = 0
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 16384 16777216
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_window_scaling = 1
我正在使用 The Grinder 来产生负荷。一旦负载增加到某个点，SYN_SENT 就会堆积在 haproxy 端。如果我将 maconnrate 减少到 96，连接率就会降低到 SYN_SENT 永远不会叠加。后端的负载永远不会超过 20% CPU，也不会超过 1 的平均负载。我在 nginx、haproxy 和 tcp 堆栈中尝试了各种调整选项。
似乎有一些 tcp 参数可以让更多 connections/second 进入码头服务器，这会让 haproxy 完全加载码头服务器。
OTOH，根据 http://cbonte.github.io/haproxy-dconv/configuration-1.5.html，“默认情况下，HAProxy 在持久性方面以保持活动模式运行
连接。”然而，haproxy 和应用程序服务器之间的连接并没有保持打开状态。是否有我指定的一些 haproxy 选项可以防止这种情况发生？或者是否有一些选项可以启用它？
虽然我花了一些时间调整设置，如下所示，但最终给出更合理结果（大约 14K 个同时连接）的更改是将托管 nginx 和 haproxy 的 VM 的大小从 0.5G 1CPU 增加到 2G 2CPU。
这是当前的 nginx 配置：
user myc;
worker_processes 1;
pid /run/nginx.pid;
worker_rlimit_nofile 131072;
events {
worker_connections 65536;
multi_accept on;
use epoll;
}
http {
include mime.types;
types_hash_max_size 2048;
default_type application/octet-stream;
proxy_cache_path /home/myc/cache levels=1:2 keys_zone=one:10m inactive=7d;
upstream loadbalancer {
server unix:/tmp/haproxy.sock;
keepalive 32768;
}
upstream haproxy_admin {
server 127.0.0.1:8081;
}
server {
listen 80 backlog=16384;
server_name makeyourcase.org;
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
gzip on;
gzip_types text/plain application/javascript application/x-javascript text/javascript text/xml text/css application/json;
gzip_vary on;
gzip_min_length 10240;
open_file_cache max=2000 inactive=20s;
open_file_cache_valid 60s;
open_file_cache_min_uses 5;
open_file_cache_errors off;
sendfile on;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
error_page 502 /502.html;
location /502.html {
root /home/myc/html;
}
proxy_cache one;
location ~ ^/api/auth/local/.*$ {
proxy_pass http://loadbalancer$uri;
}
location /admin {
proxy_pass http://loadbalancer/admin;
}
location /stats {
proxy_pass http://haproxy_admin/;
}
location / {
proxy_pass http://loadbalancer/;
proxy_connect_timeout 60s;
proxy_read_timeout 60s;
add_header Cache-Control private; # Don't cache any of this publicly, because JSESSIONID will be cached, too
add_header X-Proxy-Cache $upstream_cache_status;
}
location /api {
proxy_pass http://loadbalancer/api;
proxy_connect_timeout 60s;
proxy_read_timeout 60s;
proxy_ignore_headers Expires;
proxy_hide_header Expires;
proxy_ignore_headers Set-Cookie; # Make sure JSESSIONID is not cached with static content
proxy_hide_header Set-Cookie;
proxy_cache_valid 200 10s;
add_header X-Proxy-Cache $upstream_cache_status;
}
location ~* ^.*\.(css|js|gif|jpe?g|png|ico)$ {
proxy_pass http://loadbalancer$uri;
proxy_connect_timeout 60s;
proxy_read_timeout 60s;
expires 1h;
access_log off;
log_not_found off;
proxy_ignore_headers Expires;
proxy_hide_header Expires;
proxy_ignore_headers Set-Cookie; # Make sure JSESSIONID is not cached with static content
proxy_hide_header Set-Cookie;
add_header Cache-Control public;
proxy_cache_valid 200 404 10s;
add_header X-Proxy-Cache $upstream_cache_status;
}
}
}
和 HAProxy：
global
daemon
maxconn 32768
maxconnrate 256
chroot /home/jail/
defaults
mode http
timeout connect 10s
timeout client 120s
timeout server 120s
errorfile 502 /etc/haproxy/errors/502.http
errorfile 503 /etc/haproxy/errors/503.http
compression algo identity
frontend http-in
bind /tmp/haproxy.sock user myc
default_backend makeyourcase
backend makeyourcase
option httpchk GET /
option http-server-close
appsession JSESSIONID len 52 timeout 1h
server ny2-b-app01 <redacted>:8080 maxconn 1024 check inter 60000 fall 1 rise 1
server ny2-b-app02 <redacted>:8080 maxconn 1024 check inter 60000 fall 1 rise 1
listen stats localhost:8081
mode http
stats enable
stats hide-version
stats realm MYC
stats uri /
和 sysctl:
fs.file-max = 1048576
net.core.netdev_max_backlog = 250000
net.core.somaxconn = 262144
net.core.rmem_max = 134217728
net.core.wmem_max = 134217728
net.ipv4.tcp_rfc1337 = 1
net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.tcp_fin_timeout = 10
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_moderate_rcvbuf = 1
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_rmem = 4096 87380 67108864
net.ipv4.tcp_wmem = 4096 65536 67108864
net.ipv4.tcp_low_latency = 1
net.ipv4.tcp_tw_recycle = 0
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_window_scaling = 1
vm.swappiness = 0
我正在使用带有 nginx 的 haproxy 前端在码头应用程序服务器之间进行负载平衡，所有 运行 ubuntu-14-04-x64。
Nginx 和 haproxy 共享一个 0.5G 1CPU VM。每个码头服务器都运行在 4G 2CPU VM 上。
Nginx前端haproxy，配置如下：
user myc;
worker_processes 1;
pid /run/nginx.pid;
events {
worker_connections 65536;
multi_accept on;
use epoll;
}
http {
include mime.types;
types_hash_max_size 2048;
default_type application/octet-stream;
proxy_cache_path /home/myc/cache levels=1:2 keys_zone=one:10m inactive=7d;
proxy_http_version 1.1;
proxy_set_header Connection "";
upstream loadbalancer {
server unix:/tmp/haproxy.sock;
keepalive 8192;
}
server {
listen 80 backlog=16384;
server_name example.org;
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
gzip on;
gzip_types text/plain application/javascript application/x-javascript text/javascript text/xml text/css application/json;
gzip_vary on;
gzip_min_length 10240;
open_file_cache max=2000 inactive=20s;
open_file_cache_valid 60s;
open_file_cache_min_uses 5;
open_file_cache_errors off;
sendfile on;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
error_page 502 /502.html;
location /502.html {
root /home/myc/html;
}
proxy_cache one;
location / {
proxy_pass http://loadbalancer/;
proxy_connect_timeout 60s;
proxy_read_timeout 60s;
add_header Cache-Control private; # Don't cache any of this publicly, because JSESSIONID will be cached, too
add_header X-Proxy-Cache $upstream_cache_status;
}
location /api {
proxy_pass http://loadbalancer/api;
proxy_connect_timeout 60s;
proxy_read_timeout 60s;
proxy_ignore_headers Expires;
proxy_hide_header Expires;
proxy_ignore_headers Set-Cookie; # Make sure JSESSIONID is not cached with static content
proxy_hide_header Set-Cookie;
proxy_cache_valid 200 10s;
add_header X-Proxy-Cache $upstream_cache_status;
}
location ~* ^.*\.(css|js|gif|jpe?g|png|ico)$ {
proxy_pass http://loadbalancer$uri;
proxy_connect_timeout 60s;
proxy_read_timeout 60s;
expires 1h;
access_log off;
log_not_found off;
proxy_ignore_headers Expires;
proxy_hide_header Expires;
proxy_ignore_headers Set-Cookie; # Make sure JSESSIONID is not cached with static content
proxy_hide_header Set-Cookie;
add_header Cache-Control public;
proxy_cache_valid 200 404 10s;
add_header X-Proxy-Cache $upstream_cache_status;
}
}
}
HAProxy 配置如下：
global
daemon
maxconn 262144
maxconnrate 128
chroot /home/jail/
defaults
mode http
timeout connect 10s
timeout client 120s
timeout server 120s
errorfile 502 /etc/haproxy/errors/502.http
errorfile 503 /etc/haproxy/errors/503.http
compression algo identity
option http-keep-alive
frontend http-in
bind /tmp/haproxy.sock user myc
default_backend makeyourcase
backend makeyourcase
option httpchk GET /
appsession JSESSIONID len 52 timeout 1h
server ny2-b-app01 <redacted>:8080 maxconn 1024 check inter 60000 fall 1 rise 1
server ny2-b-app02 <redacted>:8080 maxconn 1024 check inter 60000 fall 1 rise 1
TCP 堆栈在 nginx/haproxy 框和两个码头框上使用这些参数进行调整：
net.core.netdev_max_backlog = 65535
net.core.somaxconn = 65535
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rfc1337 = 1
net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.tcp_fin_timeout = 30
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_max_syn_backlog = 65535
net.ipv4.tcp_moderate_rcvbuf = 1
net.ipv4.tcp_syncookies = 0
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 16384 16777216
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_window_scaling = 1
我正在使用 The Grinder 来产生负荷。一旦负载增加到某个点，SYN_SENT 就会堆积在 haproxy 端。如果我将 maconnrate 减少到 96，连接率就会降低到 SYN_SENT 永远不会叠加。后端的负载永远不会超过 20% CPU，也不会超过 1 的平均负载。我在 nginx、haproxy 和 tcp 堆栈中尝试了各种调整选项。
似乎有一些 tcp 参数可以让更多 connections/second 进入码头服务器，这会让 haproxy 完全加载码头服务器。
OTOH，根据 http://cbonte.github.io/haproxy-dconv/configuration-1.5.html，“默认情况下，HAProxy 在持久性方面以保持活动模式运行 连接。”然而，haproxy 和应用程序服务器之间的连接并没有保持打开状态。是否有我指定的一些 haproxy 选项可以防止这种情况发生？或者是否有一些选项可以启用它？
虽然我花了一些时间调整设置，如下所示，但最终给出更合理结果（大约 14K 个同时连接）的更改是将托管 nginx 和 haproxy 的 VM 的大小从 0.5G 1CPU 增加到 2G 2CPU。
这是当前的 nginx 配置：
user myc;
worker_processes 1;
pid /run/nginx.pid;
worker_rlimit_nofile 131072;
events {
worker_connections 65536;
multi_accept on;
use epoll;
}
http {
include mime.types;
types_hash_max_size 2048;
default_type application/octet-stream;
proxy_cache_path /home/myc/cache levels=1:2 keys_zone=one:10m inactive=7d;
upstream loadbalancer {
server unix:/tmp/haproxy.sock;
keepalive 32768;
}
upstream haproxy_admin {
server 127.0.0.1:8081;
}
server {
listen 80 backlog=16384;
server_name makeyourcase.org;
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
gzip on;
gzip_types text/plain application/javascript application/x-javascript text/javascript text/xml text/css application/json;
gzip_vary on;
gzip_min_length 10240;
open_file_cache max=2000 inactive=20s;
open_file_cache_valid 60s;
open_file_cache_min_uses 5;
open_file_cache_errors off;
sendfile on;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
error_page 502 /502.html;
location /502.html {
root /home/myc/html;
}
proxy_cache one;
location ~ ^/api/auth/local/.*$ {
proxy_pass http://loadbalancer$uri;
}
location /admin {
proxy_pass http://loadbalancer/admin;
}
location /stats {
proxy_pass http://haproxy_admin/;
}
location / {
proxy_pass http://loadbalancer/;
proxy_connect_timeout 60s;
proxy_read_timeout 60s;
add_header Cache-Control private; # Don't cache any of this publicly, because JSESSIONID will be cached, too
add_header X-Proxy-Cache $upstream_cache_status;
}
location /api {
proxy_pass http://loadbalancer/api;
proxy_connect_timeout 60s;
proxy_read_timeout 60s;
proxy_ignore_headers Expires;
proxy_hide_header Expires;
proxy_ignore_headers Set-Cookie; # Make sure JSESSIONID is not cached with static content
proxy_hide_header Set-Cookie;
proxy_cache_valid 200 10s;
add_header X-Proxy-Cache $upstream_cache_status;
}
location ~* ^.*\.(css|js|gif|jpe?g|png|ico)$ {
proxy_pass http://loadbalancer$uri;
proxy_connect_timeout 60s;
proxy_read_timeout 60s;
expires 1h;
access_log off;
log_not_found off;
proxy_ignore_headers Expires;
proxy_hide_header Expires;
proxy_ignore_headers Set-Cookie; # Make sure JSESSIONID is not cached with static content
proxy_hide_header Set-Cookie;
add_header Cache-Control public;
proxy_cache_valid 200 404 10s;
add_header X-Proxy-Cache $upstream_cache_status;
}
}
}
和 HAProxy：
global
daemon
maxconn 32768
maxconnrate 256
chroot /home/jail/
defaults
mode http
timeout connect 10s
timeout client 120s
timeout server 120s
errorfile 502 /etc/haproxy/errors/502.http
errorfile 503 /etc/haproxy/errors/503.http
compression algo identity
frontend http-in
bind /tmp/haproxy.sock user myc
default_backend makeyourcase
backend makeyourcase
option httpchk GET /
option http-server-close
appsession JSESSIONID len 52 timeout 1h
server ny2-b-app01 <redacted>:8080 maxconn 1024 check inter 60000 fall 1 rise 1
server ny2-b-app02 <redacted>:8080 maxconn 1024 check inter 60000 fall 1 rise 1
listen stats localhost:8081
mode http
stats enable
stats hide-version
stats realm MYC
stats uri /
和 sysctl:
fs.file-max = 1048576
net.core.netdev_max_backlog = 250000
net.core.somaxconn = 262144
net.core.rmem_max = 134217728
net.core.wmem_max = 134217728
net.ipv4.tcp_rfc1337 = 1
net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.tcp_fin_timeout = 10
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_moderate_rcvbuf = 1
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_rmem = 4096 87380 67108864
net.ipv4.tcp_wmem = 4096 65536 67108864
net.ipv4.tcp_low_latency = 1
net.ipv4.tcp_tw_recycle = 0
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_window_scaling = 1
vm.swappiness = 0