haproxy + nginx 实现四、七层IP透传

######环境：
10.0.0.150 ha1
10.0.0.170 web1
10.0.0.180 web2
10.0.0.190 client

############haproxy使用tcp模式4层代理，实现客户端ip透传。
######ha1配置haproxy服务
[root@ha1 ~]#yum install -y haproxy
[root@ha1 ~]#vim /etc/haproxy/haproxy.cfg
#其他配置默认，文件末尾加入以下配置
listen httpd_80
    bind 0.0.0.0:80
    mode tcp  #所处理的类别(7层代理http，4层代理tcp)
    server web1 10.0.0.170:80 send-proxy weight 1 check inter 2000 rise 2 fail 3
    #在HAProxy的TCP模式下，如果要向Nginx发送客户端的IP地址，它们之间要使用「PROXY协议」。需要在server中追加send-proxy配置
    server web2 10.0.0.180:80 send-proxy weight 2 check

listen stats
        mode http
        bind 10.0.0.150:9999
        stats enable
        log global
        stats uri     /haproxy-status
        stats auth   haadmin:123456


######web1配置nginx服务
[root@web1 ~]#yum install -y nginx
[root@web1 ~]#vim /etc/nginx/nginx.conf
    ...#...表示其他配置默认
http {
#    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
#                      '$status $body_bytes_sent "$http_referer" '
#                      '"$http_user_agent" "$http_x_forwarded_for"';
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" "$proxy_protocol_addr"';

    access_log  /var/log/nginx/access.log  main;

    server {
        listen       80  proxy_protocol default_server; #添加proxy_protocol参数配置Nginx服务接受PROXY协议
		...
	}
	...
[root@web1 ~]#echo "`hostname`" > /usr/share/nginx/html/index.html
[root@web1 ~]#systemctl restart nginx
[root@web1 ~]#curl localhost
web1.tan.com

######web2配置nginx服务
[root@web2 ~]#yum install -y nginx
[root@web2 ~]#vim /etc/nginx/nginx.conf
    ...#...表示其他配置默认
http {
#    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
#                      '$status $body_bytes_sent "$http_referer" '
#                      '"$http_user_agent" "$http_x_forwarded_for"';
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" "$proxy_protocol_addr"';

    access_log  /var/log/nginx/access.log  main;

    server {
        listen       80  proxy_protocol default_server; #添加proxy_protocol参数配置Nginx服务接受PROXY协议
		...
	}
	...
[root@web2 ~]#echo "`hostname`" > /usr/share/nginx/html/index.html
[root@web2 ~]#systemctl restart nginx
[root@web2 ~]#curl localhost
web2.tan.com


######client上访问ha1的地址
[root@client ~]#curl 10.0.0.150
web2.tan.com
[root@client ~]#curl 10.0.0.150
web1.tan.com
[root@client ~]#curl 10.0.0.150
web2.tan.com
[root@client ~]#curl 10.0.0.150
web2.tan.com
[root@client ~]#curl 10.0.0.150
web1.tan.com

######web1上查看访问日志
[root@web1 ~]#tail -f /var/log/nginx/access.log
::1 - - [27/Aug/2022:08:55:46 +0800] "GET / HTTP/1.1" 200 13 "-" "curl/7.61.1" "-"
10.0.0.150 - - [27/Aug/2022:09:06:31 +0800] "GET / HTTP/1.1" "10.0.0.190"
10.0.0.150 - - [27/Aug/2022:09:06:43 +0800] "GET / HTTP/1.1" "10.0.0.190"

######web2上查看访问日志
[root@web2 ~]#tail -f /var/log/nginx/access.log
::1 - - [27/Aug/2022:09:00:19 +0800] "GET / HTTP/1.1" 200 13 "-" "curl/7.61.1" "-"
10.0.0.150 - - [27/Aug/2022:09:06:29 +0800] "GET / HTTP/1.1" "10.0.0.190"
10.0.0.150 - - [27/Aug/2022:09:06:33 +0800] "GET / HTTP/1.1" "10.0.0.190"
10.0.0.150 - - [27/Aug/2022:09:06:34 +0800] "GET / HTTP/1.1" "10.0.0.190"

######因nginx开启了proxy_protocol所以不能只能访问nginx了，只能通过haproxy访问nginx服务。
[root@client ~]#curl 10.0.0.170
curl: (52) Empty reply from server
[root@client ~]#curl 10.0.0.180
curl: (52) Empty reply from server


############haproxy使用http模式7层代理，实现客户端ip透传。
######修改haproxy的配置，mode tcp改为mode http
[root@ha1 ~]#vim /etc/haproxy/haproxy.cfg
	...
	mode http
	...
[root@ha1 ~]#systemctl restart haproxy

######web1修改nginx配置,修改"$proxy_protocol_addr"为proxy_add_x_forwarded_for"
[root@web1 ~]#vim /etc/nginx/nginx.conf
	...
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request"  "$proxy_add_x_forwarded_for" ';
	...
[root@web1 ~]#systemctl restart nginx

######web2修改nginx配置,修改"$proxy_protocol_addr"为proxy_add_x_forwarded_for"
[root@web2 ~]#vim /etc/nginx/nginx.conf
	...
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request"  "$proxy_add_x_forwarded_for" ';
	...
[root@web2 ~]#systemctl restart nginx

######client访问ha1地址来访问nginx访问
[root@client ~]#curl 10.0.0.150
web1.tan.com
[root@client ~]#curl 10.0.0.150
web2.tan.com
[root@client ~]#curl 10.0.0.150
web2.tan.com
[root@client ~]#curl 10.0.0.150
web1.tan.com
[root@client ~]#curl 10.0.0.150
web2.tan.com
[root@client ~]#curl 10.0.0.150
web2.tan.com
[root@client ~]#curl 10.0.0.150
web1.tan.com
[root@client ~]#curl 10.0.0.150
web2.tan.com


######web1查看访问日志
[root@web1 ~]#tail -f /var/log/nginx/access.log
10.0.0.150 - - [27/Aug/2022:09:35:57 +0800] "GET / HTTP/1.1"  "10.0.0.190, 10.0.0.150"
10.0.0.150 - - [27/Aug/2022:09:35:59 +0800] "GET / HTTP/1.1"  "10.0.0.190, 10.0.0.150"
10.0.0.150 - - [27/Aug/2022:09:35:55 +0800] "GET / HTTP/1.1" "10.0.0.190, 10.0.0.150"

######web2查看访问日志
[root@web2 ~]#tail -f /var/log/nginx/access.log
10.0.0.150 - - [27/Aug/2022:09:35:57 +0800] "GET / HTTP/1.1" "10.0.0.190, 10.0.0.150"
10.0.0.150 - - [27/Aug/2022:09:35:58 +0800] "GET / HTTP/1.1" "10.0.0.190, 10.0.0.150"
10.0.0.150 - - [27/Aug/2022:09:35:58 +0800] "GET / HTTP/1.1" "10.0.0.190, 10.0.0.150"
10.0.0.150 - - [27/Aug/2022:09:35:59 +0800] "GET / HTTP/1.1" "10.0.0.190, 10.0.0.150"