nginx负载均衡，nginx负载均衡调度器高可用配置
nginx通常被用作后端服务器的反向代理，这样就可以很方便的实现动静分离以及负载均衡，从而大大提高服务器的处理能力。
nginx实现动静分离，其实就是在反向代理的时候，如果是静态资源，就直接从nginx发布的路径去读取，而不需要从后台服务器获取了。
但是要注意，这种情况下需要保证后端跟前端的程序保持一致，可以使用Rsync做服务端自动同步或者使用NFS、MFS分布式共享存储。
Http Proxy模块，功能很多，最常用的是proxy_pass和proxy_cache
如果要使用proxy_cache，需要集成第三方的ngx_cache_purge模块，用来清除指定的URL缓存。这个集成需要在安装nginx的时候去做，如：
./configure --add-module=../ngx_cache_purge-1.0 ......
nginx通过upstream模块来实现简单的负载均衡，upstream需要定义在http段内
在upstream段内，定义一个服务器列表，默认的方式是轮询，如果要确定同一个访问者发出的请求总是由同一个后端服务器来处理，可以设置ip_hash
| 主机名 | IP | 服务 |
|---|---|---|
| a | 192.168.29.129 | nginx keepalived |
| b | 192.168.29.130 | nginx keepalived |
| c | 192.168.29.131 | httpd |
| d | 192.168.29.132 | nginx |
[root@c ~]# dnf -y install httpd
[root@c ~]# systemctl enable --now httpd
Created symlink /etc/systemd/system/multi-user.target.wants/httpd.service → /usr/lib/systemd/system/httpd.service.
[root@c ~]# getenforce
Disabled
[root@c ~]# systemctl status firewalld
● firewalld.service - firewalld - dynamic firewall daemon
Loaded: loaded (/usr/lib/systemd/system/firewalld.service; enabled; vendor >
Active: inactive (dead) since Tue 2022-10-18 20:14:27 CST; 3min 38s ago
Docs: man:firewalld(1)
[root@c ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 *:80 *:*
LISTEN 0 128 [::]:22 [::]:*
[root@d ~]# dnf -y install nginx
[root@d ~]# systemctl status firewalld
● firewalld.service - firewalld - dynamic firewall daemon
Loaded: loaded (/usr/lib/systemd/system/firewalld.service; enabled; vendor >
Active: inactive (dead) since Tue 2022-10-18 20:26:16 CST; 2s ago
Docs: man:firewalld(1)
[root@d ~]# getenforce
Disabled
[root@d ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 [::]:80 [::]:*
LISTEN 0 128 [::]:22 [::]:*
[root@b ~]# systemctl status firewalld
● firewalld.service - firewalld - dynamic firewall daemon
Loaded: loaded (/usr/lib/systemd/system/firewalld.service; enabled; vendor >
Active: inactive (dead) since Tue 2022-10-18 20:26:16 CST; 2s ago
Docs: man:firewalld(1)
[root@b ~]# getenforce
Disabled
[root@b ~]# dnf -y install nginx
[root@b ~]# cd /etc/nginx/
[root@b nginx]# ls
conf.d koi-utf scgi_params
default.d koi-win scgi_params.default
fastcgi.conf mime.types uwsgi_params
fastcgi.conf.default mime.types.default uwsgi_params.default
fastcgi_params nginx.conf win-utf
fastcgi_params.default nginx.conf.default
[root@b nginx]# vim nginx.conf
......
upstream webservers {//添加这一栏
server 192.168.29.131;//添加这一栏
server 192.168.29.132;//添加这一栏
}//添加这一栏
server {
listen 80 ;//修改这一栏
listen [::]:80 default_server;//删除这一栏
......
location / {
proxy_pass http://webservers;//添加这一栏
}
[root@b nginx]# systemctl enable --now nginx
Created symlink /etc/systemd/system/multi-user.target.wants/nginx.service → /usr/lib/systemd/system/nginx.service.
[root@b nginx]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
[root@b nginx]#
刷新一下
刷新一下
[root@b nginx]# vim nginx.conf
upstream webservers {
server 192.168.29.131 weight=3;//修改这一栏
server 192.168.29.132;
}
[root@b nginx]# systemctl reload nginx
[root@c ~]# cd /var/www/html/
[root@c html]# ls
[root@c html]# echo 'apache' > index.html
[root@c html]# ls
index.html
[root@c html]#
[root@d ~]# cd /usr/share/nginx/html/
[root@d html]# ls
404.html 50x.html index.html nginx-logo.png poweredby.png
[root@d html]# echo 'nginx' > index.html
[root@d html]#
upstream webservers {
ip_hash;//添加这一栏
server 192.168.29.131 weight=3;
server 192.168.29.132;
}
[root@b nginx]# systemctl reload nginx
注意：这个方法本质还是轮询，而且由于客户端的ip可能是不断变化的，比如动态ip，代理，FQ等，因此ip_hash并不能完全保证同一个客户端总是由同一个服务器来处理。
[root@a ~]# dnf -y install nginx
[root@a ~]# getenforce
Disabled
[root@a ~]# systemctl status firewalld
● firewalld.service - firewalld - dynamic firewall daemon
Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor>
Active: inactive (dead)
Docs: man:firewalld(1)
[root@a ~]# scp 192.168.29.130:/etc/nginx/nginx.conf /etc/nginx/
The authenticity of host '192.168.29.130 (192.168.29.130)' can't be established.
ECDSA key fingerprint is SHA256:GKhyLI0ugf8uSsj22Zqei3oSFf76aLw8wpWYai88Fcc.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added '192.168.29.130' (ECDSA) to the list of known hosts.
root@192.168.29.130's password:
nginx.conf 100% 2555 3.1MB/s 00:00
[root@a ~]# systemctl enable --now nginx
Created symlink /etc/systemd/system/multi-user.target.wants/nginx.service → /usr/lib/systemd/system/nginx.service.
[root@a ~]# cd /etc/nginx/
[root@a nginx]# vim nginx.conf
...
upstream webservers {
ip_hash;//删除这一栏
server 192.168.29.131 weight=3;
server 192.168.29.132;
}
...
[root@a nginx]# systemctl restart nginx
[root@b nginx]# vim nginx.conf
...
upstream webservers {
ip_hash;//删除这一栏
server 192.168.29.131 weight=3;
server 192.168.29.132;
}
...
[root@b nginx]# systemctl restart nginx
[root@a ~]# dnf list all | grep keepalived
keepalived.x86_64 2.1.5-9.el8 appstream
[root@a ~]# dnf -y install keepalived
[root@a ~]# cd /etc/keepalived/
[root@a keepalived]# ls
keepalived.conf
[root@a keepalived]# mv keepalived.conf{,-bak}
[root@a keepalived]# ls
keepalived.conf-bak
[root@a keepalived]# vim keepalived.conf
! Configuration File for keepalived
global_defs {
router_id lb01
}
vrrp_instance VI_1 {
state MASTER
interface ens33
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass wangqing
}
virtual_ipaddress {
192.168.29.250
}
}
virtual_server 192.168.29.250 80 {
delay_loop 6
lb_algo rr
lb_kind DR
persistence_timeout 50
protocol TCP
real_server 192.168.29.129 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.29.130 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
[root@a keepalived]# systemctl start keepalived
[root@a keepalived]# systemctl enable keepalived
Created symlink /etc/systemd/system/multi-user.target.wants/keepalived.service → /usr/lib/systemd/system/keepalived.service.
[root@a keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:e8:e4:aa brd ff:ff:ff:ff:ff:ff
inet 192.168.29.129/24 brd 192.168.29.255 scope global dynamic noprefixroute ens33
valid_lft 910sec preferred_lft 910sec
inet 192.168.29.250/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::d966:4d98:b47c:e02b/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@b ~]# dnf list all | grep keepalived
keepalived.x86_64 2.1.5-9.el8 appstream
[root@b ~]# dnf -y install keepalived
[root@b ~]# cd /etc/keepalived/
[root@b keepalived]# ls
keepalived.conf
[root@b keepalived]# mv keepalived.conf{,-bak}
[root@b keepalived]# ls
keepalived.conf-bak
[root@a keepalived]# scp keepalived.conf 192.168.29.130:/etc/keepalived/
root@192.168.29.130's password:
keepalived.conf 100% 866 1.0MB/s 00:00
[root@b keepalived]# ls
keepalived.conf keepalived.conf-bak
[root@b keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:50:07:0e brd ff:ff:ff:ff:ff:ff
inet 192.168.29.130/24 brd 192.168.29.255 scope global dynamic noprefixroute ens33
valid_lft 1745sec preferred_lft 1745sec
inet6 fe80::d548:86d0:3cfd:1f06/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@b keepalived]# vim keepalived.conf
...
global_defs {
router_id lb02//修改这一栏
}
...
vrrp_instance VI_1 {
state BACKUP//修改这一栏
interface ens33
virtual_router_id 51
priority 90//修改这一栏
...
[root@b keepalived]# systemctl enable --now keepalived
Created symlink /etc/systemd/system/multi-user.target.wants/keepalived.service → /usr/lib/systemd/system/keepalived.service.
[root@b keepalived]# systemctl status keepalived
● keepalived.service - LVS and VRRP High Availability Monitor
Loaded: loaded (/usr/lib/systemd/system/keepalived.service; enabled; vendor>
Active: active (running) since Tue 2022-10-18 22:10:09 CST; 32s ago
Process: 109652 ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS (code=exi>
Main PID: 109653 (keepalived)
Tasks: 3 (limit: 11201)
Memory: 2.7M
CGroup: /system.slice/keepalived.service
├─109653 /usr/sbin/keepalived -D
├─109654 /usr/sbin/keepalived -D
└─109655 /usr/sbin/keepalived -D
10月 18 22:10:09 b Keepalived_vrrp[109655]: Assigned address fe80::d548:86d0:3>
10月 18 22:10:09 b Keepalived_vrrp[109655]: Registering gratuitous ARP shared >
10月 18 22:10:09 b Keepalived_vrrp[109655]: (VI_1) removing VIPs.
10月 18 22:10:09 b Keepalived_vrrp[109655]: (VI_1) Entering BACKUP STATE (init)
10月 18 22:10:09 b Keepalived_vrrp[109655]: VRRP sockpool: [ifindex( 2), fami>
10月 18 22:10:09 b Keepalived_healthcheckers[109654]: Gained quorum 1+0=1 <= 2>
10月 18 22:10:09 b Keepalived_healthcheckers[109654]: Activating healthchecker>
10月 18 22:10:09 b Keepalived_healthcheckers[109654]: Activating healthchecker>
10月 18 22:10:12 b Keepalived_healthcheckers[109654]: TCP connection to [192.1>
10月 18 22:10:14 b Keepalived_healthcheckers[109654]: TCP connection to [192.1>
[root@b keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:50:07:0e brd ff:ff:ff:ff:ff:ff
inet 192.168.29.130/24 brd 192.168.29.255 scope global dynamic noprefixroute ens33
valid_lft 1414sec preferred_lft 1414sec
inet6 fe80::d548:86d0:3cfd:1f06/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@a keepalived]# systemctl stop keepalived
[root@a keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:e8:e4:aa brd ff:ff:ff:ff:ff:ff
inet 192.168.29.129/24 brd 192.168.29.255 scope global dynamic noprefixroute ens33
valid_lft 1210sec preferred_lft 1210sec
inet6 fe80::d966:4d98:b47c:e02b/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@a keepalived]#
[root@b keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:50:07:0e brd ff:ff:ff:ff:ff:ff
inet 192.168.29.130/24 brd 192.168.29.255 scope global dynamic noprefixroute ens33
valid_lft 1310sec preferred_lft 1310sec
inet 192.168.29.250/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::d548:86d0:3cfd:1f06/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@b keepalived]#
[root@b keepalived]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
[root@a keepalived]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
[root@a keepalived]# systemctl stop nginx
[root@a keepalived]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
[root@a keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:e8:e4:aa brd ff:ff:ff:ff:ff:ff
inet 192.168.29.129/24 brd 192.168.29.255 scope global dynamic noprefixroute ens33
valid_lft 1210sec preferred_lft 1210sec
inet6 fe80::d966:4d98:b47c:e02b/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@b keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:50:07:0e brd ff:ff:ff:ff:ff:ff
inet 192.168.29.130/24 brd 192.168.29.255 scope global dynamic noprefixroute ens33
valid_lft 1310sec preferred_lft 1310sec
inet 192.168.29.250/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::d548:86d0:3cfd:1f06/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@a keepalived]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
[root@a keepalived]# systemctl start nginx keepalived
[root@a keepalived]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
[root@a keepalived]# systemctl status keepalived
● keepalived.service - LVS and VRRP High Availability Monitor
Loaded: loaded (/usr/lib/systemd/system/keepalived.service; enabled; vendor>
Active: active (running) since Tue 2022-10-18 22:17:28 CST; 35s ago
Process: 114058 ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS (code=exi>
Main PID: 114061 (keepalived)
Tasks: 3 (limit: 11201)
Memory: 2.1M
CGroup: /system.slice/keepalived.service
├─114061 /usr/sbin/keepalived -D
├─114062 /usr/sbin/keepalived -D
└─114063 /usr/sbin/keepalived -D
10月 18 22:17:31 a Keepalived_vrrp[114063]: Sending gratuitous ARP on ens33 fo>
10月 18 22:17:31 a Keepalived_vrrp[114063]: Sending gratuitous ARP on ens33 fo>
10月 18 22:17:31 a Keepalived_vrrp[114063]: Sending gratuitous ARP on ens33 fo>
10月 18 22:17:33 a Keepalived_healthcheckers[114062]: TCP connection to [192.1>
10月 18 22:17:36 a Keepalived_vrrp[114063]: (VI_1) Sending/queueing gratuitous>
10月 18 22:17:36 a Keepalived_vrrp[114063]: Sending gratuitous ARP on ens33 fo>
10月 18 22:17:36 a Keepalived_vrrp[114063]: Sending gratuitous ARP on ens33 fo>
10月 18 22:17:36 a Keepalived_vrrp[114063]: Sending gratuitous ARP on ens33 fo>
10月 18 22:17:36 a Keepalived_vrrp[114063]: Sending gratuitous ARP on ens33 fo>
10月 18 22:17:36 a Keepalived_vrrp[114063]: Sending gratuitous ARP on ens33 fo>
[root@a keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:e8:e4:aa brd ff:ff:ff:ff:ff:ff
inet 192.168.29.129/24 brd 192.168.29.255 scope global dynamic noprefixroute ens33
valid_lft 1736sec preferred_lft 1736sec
inet 192.168.29.250/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::d966:4d98:b47c:e02b/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@a keepalived]#
[root@b keepalived]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
[root@b keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:50:07:0e brd ff:ff:ff:ff:ff:ff
inet 192.168.29.130/24 brd 192.168.29.255 scope global dynamic noprefixroute ens33
valid_lft 945sec preferred_lft 945sec
inet6 fe80::d548:86d0:3cfd:1f06/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@b keepalived]# systemctl stop nginx
