Keepalive之nginx调度架构
author：JevonWei
版权声明：原创作品
单主模式Keepalive之Nginx调度
实验目的：实现Nginx调度的高可用，当一台Nginx调度器故障时，启用备用的Nginx调度，在架构中，启用了kepalive节点状态通知脚本机制，当keepalive的状态由MASTER变为BACKUP或由BACKUP变为MASTER时，以邮件通知用户，通过在通知脚本文件中定义BACKUP状态的节点启用nginx服务，是为了防止因Nginx服务故障导致节点权重降低从而变为BACKUP状态，为了避免此种现象，故在转换为BACKUP状态时需启动一次Nginx服务
实验环境
keepalive-A 172.16.253.108
keepalive-B 172.16.253.105
Nginx-A 172.16.253.108
Nginx-B 172.16.253.105
LVS-RS1 172.16.250.127
LVS-RS2 172.16.253.193
VIP 172.16.253.150
client 172.16.253.177
环境拓扑图
web服务集群
为了更好的观察实验结果，故在此将RS1和RS2的web页面内容设置不一致，以致可以更清晰的区分RS1服务端和RS2服务端
LVS-RS1
[root@LVS-RS1 ~]# systemctl restart chronyd \\多台服务器时间同步
[root@LVS-RS1 ~]# iptables -F
[root@LVS-RS1 ~]# setenforce 0
[root@LVS-RS1 ~]# yum -y install nginx
[root@LVS-RS1 ~]# vim /usr/share/nginx/html/index.html
<h1> Web RS1 </h1>
[root@LVS-RS1 ~]# systemctl start nginx
LVS-RS2
[root@LVS-RS2 ~]# systemctl restart chronyd \\多台服务器时间同步
[root@LVS-RS2 ~]# iptables -F
[root@LVS-RS2 ~]# setenforce 0
[root@LVS-RS2 ~]# yum -y install nginx
[root@LVS-RS2 ~]# vim /usr/share/nginx/html/index.html
<h1> Web RS2 </h1>
[root@LVS-RS2 ~]# systemctl start nginx
nginx调度集群
nginx-A
[root@nginx-A ~]# yum -y install nginx
[root@nginx-A ~]# vim /usr/share/nginx/html/index.html
</h1> sorry from keepalive-A </h1>
[root@nginx-A ~]# vim /etc/nginx/nginx.conf
http {
upstream websrvs {
server 172.16.250.127:80;
server 172.16.253.193:80;
}
}
server {
listen 80 default_server;
location / {
proxy_pass http://websrvs;
}
}
[root@nginx-A ~]# systemctl start nginx
nginx-B
[root@nginx-B ~]# yum -y install nginx
[root@nginx-B ~]# vim /usr/share/nginx/html/index.html
</h1> sorry from keepalive-B </h1>
[root@nginx-B ~]# vim /etc/nginx/nginx.conf
http {
upstream websrvs {
server 172.16.250.127:80;
server 172.16.253.193:80;
}
}
server {
listen 80 default_server;
location / {
proxy_pass http://websrvs;
}
}
[root@nginx-B ~]# systemctl start nginx
keepalive集群
keepalive-A
[root@keepalive-A ~]# yum -y install keepalived
[root@keepalive-A ~]# vim /etc/keepalived/notify.sh \\定义状态检测通知脚本，当节点变为BACKUP时，启动nginx服务
#!/bin/bash
#
contact='root@localhost' \\通知的联系者，即本地邮件服务器的通知邮件接受者
notify() {
local mailsubject="$(hostname) to be $1, vip floating" \\指定邮件标题
local mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1" \\指定邮件正文
echo "$mailbody" | mail -s "$mailsubject" $contact \\发送通知信息邮件到邮件接收者
}
case $1 in
master)
notify master
;;
backup)
notify backup
systemctl start nginx
;;
fault)
notify fault
;;
*)
echo "Usage: $(basename $0) {master|backup|fault}"
exit 1
;;
esac
[root@keepaliveA ~]# vim /etc/keepalived/keepalived.conf
global_defs {
notification_email {
jevon@danran.com
}
notification_email_from ka_admin@danran.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id keepaliveA
vrrp_mcast_group4 224.103.5.5
}
vrrp_script chk_mant_down { \\定义检测脚本
script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0" \\检测/etc/keepalived/down文件是否存在，若文件存在，则weight权重-10，若文件不存在，则每隔一秒检测
interval 1 \\每隔一秒检测
weight -10
fall 2 \\检测两次故障则表示节点故障
rise 1 \\检测一次正常则表示节点正常
}
vrrp_script chk_nginx { \\检测nginx服务是否运行脚本
script "killall -0 nginx" \\-0为探测信号，当nginx进程存在时，返回值$?=0,当nginx进程不存在时，返回值$?为非零，则权重减10
interval 2
weight -10
fall 2
rise 2
}
vrrp_instance VI_A {
state BACKUP
interface ens33
virtual_router_id 51
priority 95
advert_int 1
authentication {
auth_type PASS
auth_pass qr8hQHuL
}
virtual_ipaddress {
172.16.253.150/32
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
track_script { \\脚本调用
chk_mant_down
chk_nginx \\调用chk_nginx脚本
}
}
[root@keepalive-A ~]# systemctl start keepalived
[root@keepalive-A ~]# systemctl status keepalived
keepalive-B
[root@keepalive-B ~]# yum -y install keepalived
[root@keepalive-A ~]# vim /etc/keepalived/notify.sh \\定义状态检测通知脚本
#!/bin/bash
#
contact='root@localhost' \\通知的联系者，即本地邮件服务器的通知邮件接受者
notify() {
local mailsubject="$(hostname) to be $1, vip floating" \\指定邮件标题
local mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1" \\指定邮件正文
echo "$mailbody" | mail -s "$mailsubject" $contact \\发送通知信息邮件到邮件接收者
}
case $1 in
master)
notify master
;;
backup)
notify backup
systemctl start nginx
;;
fault)
notify fault
;;
*)
echo "Usage: $(basename $0) {master|backup|fault}"
exit 1
;;
esac
[root@keepalive-B ~]# vim /etc/keepalived/keepalived.conf
global_defs {
notification_email {
jevon@danran.com
}
notification_email_from ka_admin@danran.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id keepaliveA
vrrp_mcast_group4 224.103.5.5
}
vrrp_script chk_mant_down { \\定义检测脚本
script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0" \\检测/etc/keepalived/down文件是否存在，若文件存在，则weight权重-10，若文件不存在，则每隔一秒检测
interval 1 \\每隔一秒检测
weight -10
fall 2 \\检测两次故障则表示节点故障
rise 1 \\检测一次正常则表示节点正常
}
vrrp_script chk_nginx { \\检测nginx服务是否运行脚本
script "killall -0 nginx" \\-0为探测信号，当nginx进程存在时，返回值$?=0,当nginx进程不存在时，返回值$?为非零，则权重减10
interval 2
weight -10
fall 2
rise 2
}
vrrp_instance VI_A {
state BACKUP
interface ens33
virtual_router_id 51
priority 95
advert_int 1
authentication {
auth_type PASS
auth_pass qr8hQHuL
}
virtual_ipaddress {
172.16.253.150/32
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
track_script { \\脚本调用
chk_mant_down
chk_nginx \\调用chk_nginx脚本
}
}
[root@keepalive-B ~]# systemctl start keepalived
[root@keepalive-A ~]# systemctl start keepalived
测试
访问测试调度
[root@client ~]# for i in {1..10};do curl http://172.16.253.150;done
<h1> Web RS1 </h1>
<h1> Web RS2 </h1>
<h1> Web RS1 </h1>
<h1> Web RS2 </h1>
<h1> Web RS1 </h1>
测试检测监控Nginx服务资源的外部脚本
[root@keepalive-A ~]# ip a l
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:75:dc:3c brd ff:ff:ff:ff:ff:ff
inet 172.16.253.150/32 scope global ens33
valid_lft forever preferred_lft forever
手动停止Nginx服务，keepalive-A变为了BACKUP节点
[root@keepalive-A ~]# systemctl stop nginx
[root@keepalive-A ~]# ip a l
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:75:dc:3c brd ff:ff:ff:ff:ff:ff
几秒钟之后，Nginx服务再次自动启动，则Keepalive-A再次成为MASTER节点
[root@keepalive-A ~]# ip a l
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:75:dc:3c brd ff:ff:ff:ff:ff:ff
inet 172.16.253.150/32 scope global ens33
valid_lft forever preferred_lft forever
当nginx的80端口被占用时，Nginx服务停止之后不会再次启动，则keepalive-A变为BACKUP
[root@keepalive-A ~]# yum -y install httpd
[root@keepalive-A ~]# systemctl stop nginx && systemctl start httpd \\停止nginx的同时启动httpd
[root@keepalive-A ~]# ip a l
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:75:dc:3c brd ff:ff:ff:ff:ff:ff
[root@keepalive-A ~]# systemctl status keepalived.service
Sep 05 21:21:14 danran.com Keepalived_vrrp[24153]: VRRP_Script(chk_nginx) failed
在HTTPD进程停止之后，80端口被释放，则keepalive-A上的Nginx也不会再次重启
双主模式的Nginx调度(一般不做会话绑定)
双主模式的目的是为了访问web访问路径的单点故障，当一个web虚拟路由故障时，可使用另外一个web路径访问，最终是为了实现集群的高可用性
实验环境
keepalive-A 172.16.253.108
keepalive-B 172.16.253.105
Nginx-A 172.16.253.108
Nginx-B 172.16.253.105
LVS-RS1 172.16.250.127
LVS-RS2 172.16.253.193
VI-A 172.16.253.150
VI-B 172.16.253.140
client 172.16.253.177
环境拓扑图
keepalive-A
[root@keepalive-A ~]# vim /etc/keepalived/notify.sh
#!/bin/bash
#
contact='root@localhost'
notify() {
local mailsubject="$(hostname) to be $1, vip floating"
local mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"
echo "$mailbody" | mail -s "$mailsubject" $contact
}
case $1 in
master)
notify master
;;
backup)
notify backup
systemctl start nginx
;;
fault)
notify fault
;;
*)
echo "Usage: $(basename $0) {master|backup|fault}"
exit 1
;;
esac
[root@keepalive-A ~]# systemctl stop keepalived
[root@keepalive-A ~]# vim /etc/keepalived/keepalived.conf
global_defs {
notification_email {
jevon@danran.com
}
notification_email_from ka_admin@danran.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id keepaliveA
vrrp_mcast_group4 224.103.5.5
}
vrrp_script chk_mant_down {
script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"
interval 1
weight -10
fall 2
rise 1
}
vrrp_script chk_nginx {
script "killall -0 nginx"
interval 2
weight -10
fall 2
rise 2
}
vrrp_instance VI_A {
state MASTER
interface ens33
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass qr8hQHuL
}
virtual_ipaddress {
172.16.253.150/32
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
track_script {
chk_mant_down
chk_nginx
}
}
vrrp_instance VI_B {
state BACKUP
interface ens33
virtual_router_id 52
priority 95
advert_int 1
authentication {
auth_type PASS
auth_pass qr8hQH99
}
virtual_ipaddress {
172.16.253.140/32
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
track_script {
chk_mant_down
chk_nginx
}
}
[root@keepalive-A ~]# systemctl start keepalived
keepalive-B
[root@keepalive-A ~]# vim /etc/keepalived/notify.sh
#!/bin/bash
#
contact='root@localhost'
notify() {
local mailsubject="$(hostname) to be $1, vip floating"
local mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"
echo "$mailbody" | mail -s "$mailsubject" $contact
}
case $1 in
master)
notify master
;;
backup)
notify backup
systemctl start nginx
;;
fault)
notify fault
;;
*)
echo "Usage: $(basename $0) {master|backup|fault}"
exit 1
;;
esac
[root@keepalive-B ~]# systemctl stop keepalived
[root@keepalive-B ~]# vim /etc/keepalived/keepalived.conf
global_defs {
notification_email {
jevon@danran.com
}
notification_email_from ka_admin@danran.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id keepaliveA
vrrp_mcast_group4 224.103.5.5
}
vrrp_script chk_mant_down {
script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"
interval 1
weight -10
fall 2
rise 1
}
vrrp_script chk_nginx {
script "killall -0 nginx"
interval 2
weight -10
fall 2
rise 2
}
vrrp_instance VI_A {
state BACKUP
interface ens33
virtual_router_id 51
priority 95
advert_int 1
authentication {
auth_type PASS
auth_pass qr8hQHuL
}
virtual_ipaddress {
172.16.253.150/32
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
track_script {
chk_mant_down
chk_nginx
}
}
vrrp_instance VI_B {
state MASTER
interface ens33
virtual_router_id 52
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass qr8hQH99
}
virtual_ipaddress {
172.16.253.140/32
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
track_script {
chk_mant_down
chk_nginx
}
}
[root@keepalive-B ~]# systemctl start keepalived
查看keepalive-A和keepalive-B IP地址，确认虚拟路由配置正确
[root@keepalive-A ~]# ip a l
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:75:dc:3c brd ff:ff:ff:ff:ff:ff
inet 172.16.253.150/32 scope global ens33
valid_lft forever preferred_lft forever
[root@keepalive-B ~]# ip a l
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:0e:47:87 brd ff:ff:ff:ff:ff:ff
inet 172.16.253.140/32 scope global ens33
valid_lft forever preferred_lft forever
测试Nginx调度
[root@client ~]# for i in {1..10};do curl http://172.16.253.150;done
<h1> Web RS1 </h1>
<h1> Web RS2 </h1>
<h1> Web RS1 </h1>
<h1> Web RS2 </h1>
<h1> Web RS1 </h1>
[root@client ~]# for i in {1..10};do curl http://172.16.253.140;done
<h1> Web RS1 </h1>
<h1> Web RS2 </h1>
<h1> Web RS1 </h1>
<h1> Web RS2 </h1>
<h1> Web RS1 </h1>
<h1> Web RS2 </h1>
<h1> Web RS1 </h1>
danran
