Haproxy+Keepalived实现Haproxy的高可用

一、实验环境

　主机：四台 CentOS7系统的虚拟机，node1(192.168.27.7)，node2(192.168.27.17)，VIP(192.168.27.100)，web1(192.168.27.27)，web2(192.168.27.37)

　软件：haproxy-1.8.20.tar.gz，keepalived（光盘yum源），httpd（光盘yum源）

二、实验步骤

1、安装haproxy

　两台主机都源码安装haproxy-1.8.20.tar.gz，都两台主机操作一样，以下为node1的操作

[root@node1 ~]# ll haproxy-1.8.20.tar.gz 
-rw-r--r-- 1 root root 2083917 Jan 10 20:39 haproxy-1.8.20.tar.gz
#安装依赖包
[root@node1 ~]# yum install -y gcc gcc-c++ pcre pcre-devel openssl openssl-devel systemd-devel
#开始编译安装
[root@node1 ~]# tar xf haproxy-1.8.20.tar.gz 
[root@node1 ~]# cd haproxy-1.8.20/
[root@node1 haproxy-1.8.20]# make ARCH=x86_64 TARGET=linux2628 USE_PCRE=1 USE_OPENSSL=1 USE_ZLIB=1 USE_SYSTEMD=1 USE_CPU_AFFINITY=1 PREFIX=/usr/local/haproxy
[root@node1 haproxy-1.8.20]# make install PREFIX=/usr/local/haproxy
[root@node1 haproxy-1.8.20]# cp haproxy /usr/sbin/
#准备启动文件
[root@node1 haproxy-1.8.20]# vim /usr/lib/systemd/system/haproxy.service
[Unit]
Description=HAProxy Load Balancer
After=syslog.target network.target

[Service]
ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q
ExecStart=/usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /usr/local/haproxy/run/haproxy.pid
ExecReload=/bin/kill -USR2 $MAINPID

[Install]
WantedBy=multi-user.target
#创建用户与配置文件
[root@node1 haproxy-1.8.20]# useradd -r -s  /sbin/nologin haproxy
[root@node1 haproxy-1.8.20]# mkdir /etc/haproxy
[root@node1 haproxy-1.8.20]# mkdir /usr/local/haproxy/run
[root@node1 haproxy-1.8.20]# mkdir /var/lib/haproxy
[root@node1 haproxy-1.8.20]# chown -R haproxy:haproxy /var/lib/haproxy/
[root@node1 haproxy-1.8.20]# vim /etc/haproxy/haproxy.cfg

[root@node1 haproxy-1.8.20]# cat /etc/haproxy/haproxy.cfg 
global
maxconn 100000
chroot /usr/local/haproxy
#stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin
#uid 981
#gid 981
user haproxy
group haproxy
daemon
#nbproc 4
#cpu-map 1 0
#cpu-map 2 1
#cpu-map 3 2
#cpu-map 4 3
pidfile /usr/local/haproxy/run/haproxy.pid
log 127.0.0.1 local3 info

defaults
option http-keep-alive
option  forwardfor
maxconn 100000
mode http
timeout connect 300000ms
timeout client  300000ms
timeout server  300000ms

listen stats
 mode http
 bind 192.168.27.7:9999
 stats enable
 log global
 stats uri     /haproxy-status
 stats auth    haadmin:q1w2e3r4ys

listen  web_port
 bind 192.168.27.100:80    #此处为VIP
 mode http
 log global
 server web1  192.168.27.27:80  check inter 3000 fall 2 rise 5    #后端服务器
 server web1  192.168.27.37:80  check inter 3000 fall 2 rise 5    #后端服务器

2、安装keepalived

[root@node1 ~]# yum install -y keepalived

[root@node2 ~]# yum install -y keepalived

3、配置keepalived

[root@node1 ~]# vim /etc/keepalived/keepalived.conf 

! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from root@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node1    #node2节点此处改为node2
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_instance VI_1 {
    state MASTER    #node2节点此处改为BACKUP
    interface eth0
    virtual_router_id 51
    priority 100    #node2节点此处改为80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.27.100/24 dev eth0 label eth0:0
    }
}

4、修改内核参数

　因haproxy配置文件中绑定的IP是一个虚拟IP，会haproxy服务导致启动不了，此时需要修改内核参数

[root@node1 ~]# vim /etc/sysctl.conf
#添加以下两行
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
[root@node1 ~]# sysctl -p    #使配置生效
#node2节点上一样配置

5、配置后端服务器的web服务

[root@web1 ~]# yum install -y httpd
[root@web2 ~]# yum install -y httpd
#准备页面
[root@web1 ~]# echo 'web page 192.168.17.27' > /var/www/html/index.html
[root@web2 ~]# echo 'web page 192.168.17.37' > /var/www/html/index.html
#启动httpd服务
[root@web1 ~]# systemctl start httpd
[root@web2 ~]# systemctl start httpd

6、修改VIP防火墙策略

　keepalived会对VIP生成防火墙策略，导致访问不到后端服务器资源，此时可在keepalived配置文件中加一个配置vrrp_iptables 使其不生成防火墙策略，或手动删除防火墙规则 iptables -D INPUT -s 0.0.0.0/0 -d 192.168.27.100 -j DROP，本次就修改配置文件了

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from root@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node2
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_iptables    #添加此项，记得两台都要加
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}
...以下省略

7、启动服务，进行测试

[root@node1 ~]# systemctl start haproxy keepalived
[root@node2 ~]# systemctl start haproxy keepalived
#开始测试，当node1节点的keepalived挂了，VIP会转移到node2上，并保证业务不中断
[root@node1 ~]# ip a |grep 192.168.27.100    #vip在node1上
    inet 192.168.27.100/24 scope global secondary eth0:0
[root@node1 ~]# systemctl stop keepalived    #关掉node1的keepalived
[root@node2 ~]# ip a |grep 192.168.27.100    #vip转移到了node2上
    inet 192.168.27.100/24 scope global secondary eth0:0
[root@web1 ~]# while true;do curl http://192.168.27.100; sleep 1;done    #服务未中断
web page 192.168.17.27
web page 192.168.17.37
web page 192.168.17.27
web page 192.168.17.37
...
[root@node1 ~]# systemctl start keepalived    #恢复node1上的keepalived服务
[root@node1 ~]# ip a|grep 192.168.27.100    #vip又回到了node1上
    inet 192.168.27.100/24 scope global secondary eth0:0

8、实现haproxy高可用

　上述配置，只有前主节点故障时才会切换vip，当keepalived正常，但haproxy导常时并不会切换，这样也会导致业务访问出问题，此时，可以使用keepalived调用外部脚本进行资源监控，并根据监控的结果状态实现动态调整。

vrrp_script <SCRIPT_NAME> { #定义一个检测脚本，在global_defs 之外配置
    script <STRING>|<QUOTED-STRING>     #shell命令或脚本路径
    interval <INTEGER>     #间隔时间，单位为秒，默认1秒
    timeout <INTEGER>     #超时时间
    weight <INTEGER:-254..254>     #权重，脚本监测失败后会执行权重+/-操作
    fall <INTEGER>     #脚本几次失败转换为失败
    rise <INTEGER>     #脚本连续几次监测成功后，把服务器从失败标记为成功
    user USERNAME [GROUPNAME]     #执行监测的用户或组
    init_fail     #设置默认标记为失败状态，监测成功之后再转换为成功状态
}

vrrp_instance VI_1 {
    …
    track_script {    #触发脚本
        chk_down
    } 
}

#操作步骤
#创建脚本
[root@node1 ~]# vim /etc/keepalived/chk_haproxy.sh
#!/bin/bash
/usr/bin/killall -0 haproxy
#给脚本加执行权限
[root@node1 ~]# chmod +x /etc/keepalived/chk_haproxy.sh
[root@node1 ~]# vim /etc/keepalived/keepalived.conf 
[root@node1 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from root@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node1
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_iptables
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}
#添加以下段
vrrp_script chk_haproxy {
    script "/etc/keepalived/chk_haproxy.sh"
    interval 2
    weight -50
    fall 3
    rise 5 
    timeout 3
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.27.100/24 dev eth0 label eth0:0
    }
    #添加此段
    track_script {
    chk_haproxy
    }
}
#脚本与配置文件传到另一台主机
[root@node1 ~]# scp /etc/keepalived/chk_haproxy.sh 192.168.27.17:/etc/keepalived/
[root@node1 ~]# scp /etc/keepalived/keepalived.conf 192.168.27.17:/etc/keepalived/  
#重启服务
[root@node1 ~]# systemctl restart keepalived
[root@node2 ~]# systemctl restart keepalived
#测试
[root@node1 ~]# ip a|grep 192.168.27.100    #vip一开始在node1上
    inet 192.168.27.100/24 scope global secondary eth0:0
[root@node1 ~]# systemctl stop haproxy    #停掉node1上的haproxy服务
[root@node1 ~]# ip a|grep 192.168.27.100    #vip转移了

[root@node2 ~]# ip a|grep 192.168.27.100    #vip转移到了node2
    inet 192.168.27.100/24 scope global secondary eth0:0

9、实现keepalived的邮件通知功能

#定义通知脚本：
notify_master <STRING>|<QUOTED-STRING>：    当前节点成为主节点时触发的脚本
notify_backup <STRING>|<QUOTED-STRING>：    当前节点转为备节点时触发的脚本
notify_fault <STRING>|<QUOTED-STRING>：        当前节点转为“失败”状态时触发的脚本
notify <STRING>|<QUOTED-STRING>：            通用格式的通知触发机制，一个脚本可完成以上三种状态的转换时的通知

#安装邮件服务
[root@node1 ~]# yum install -y postfix

#发件人配置
[root@node1 ~]# vim /etc/mail.rc
set from=1954938301@qq.com
set smtp=smtp.qq.com
set smtp-auth-user=1954938301@qq.com    
set smtp-auth-password=mfcjxxjezahijgddj    #到QQ邮箱里配置生成
set smtp-auth=login
set ssl-verify=ignore

#准备通知脚本
[root@node1 ~]# cat /etc/keepalived/nodify.sh
#!/bin/bash

contact='1954938301@qq.com'

notify() {
    mailsubject="$(hostname) to be $1, vip 转移"
    mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"
    echo "$mailbody" | mail -s "$mailsubject" $contact
}
case $1 in

master)
    notify master
    ;;
backup)
    notify backup
    ;;
fault)
    notify fault
    ;;
*)
    echo "Usage: $(basename $0) {master|backup|fault}"
    exit 1
    ;;
esac

#脚本的调用方法，在vrrp_instance中配置即可
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"

#如下所示
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.27.100/24 dev eth0 label eth0:0
    }
    track_script {
        chk_haproxy
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}