Nginx+Keepalived主备模式配置文档-jesse于2011-0920

环境:
centos5.5x64
nginx-0.8.53.tar.gz
keepalived-1.1.17.tar.gz
pcre-8.11.tar.gz

定义如下:
nginx-master:10.0.0.15
nginx-backup:10.0.0.18
vip:10.0.0.200
#######################################
mkdir /tools
cd /tools
wget http://nginx.org/download/nginx-0.8.53.tar.gz
wget http://www.keepalived.org/software/keepalived-1.1.17.tar.gz
wget http://ftp.exim.llorien.org/pcre/pcre-8.11.tar.gz

一:分别安装Nginx负载均衡器及相关配置脚本

添加运行nginx的用户和组www

groupadd www
useradd -g www www

tar zxvf pcre-8.11.tar.gz
cd pcre-8.11/
./configure
make
make install
cd ..

tar zxvf nginx-0.8.53.tar.gz
cd nginx-0.8.53/
./configure --user=www \
--group=www \
--prefix=/usr/local/nginx-0.8.53 \
--with-http_stub_status_module \
--with-http_ssl_module
make
make install
cd ..

vi nginx.conf
#-----------------------------------------------------------------------------------
user www www;
worker_processes 8;
pid /usr/local/nginx-0.8.53/logs/nginx.pid;
worker_rlimit_nofile 65535;

events
{
use epoll;
worker_connections 65535;
}
http{
include mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 8m;
user www www;
worker_processes 8;
pid /usr/local/nginx-0.8.53/logs/nginx.pid;
worker_rlimit_nofile 65535;

events
{
use epoll;
worker_connections 65535;
}

http{
include mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 8m;
sendfile on;
tcp_nopush on;
keepalive_timeout 60;
tcp_nodelay on;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 64k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 128k;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary on;

upstream jesse
{
server 10.0.0.15:700;
server 10.0.0.15:800;
server 10.0.0.15:900;
}

server
{
listen 80;
server_name www.jesse.com;
location / {
proxy_pass http://jesse;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
log_format access '$remote_addr - $remote_user [$time_local] $request'
'"$status" $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /data/logs/jesse.log access;
}
}
#-----------------------------------------------------------------------------------

开启转发功能
sed -i 's#net.ipv4.ip_forward = 0#net.ipv4.ip_forward = 1#' /etc/sysctl.conf
sysctl -p

二:安装keepalived
tar zxvf keepalived-1.1.17.tar.gz
cd keepalived-1.1.17
./configure --prefix=/usr/local/keepalived-1.1.17
make
make install
cd ..
cp /usr/local/keepalived-1.1.17/sbin/keepalived /usr/sbin/
cp /usr/local/keepalived-1.1.17/etc/sysconfig/keepalived /etc/sysconfig/
cp /usr/local/keepalived-1.1.17/etc/rc.d/init.d/keepalived /etc/init.d/
mkdir /etc/keepalived
cd /etc/keepalived/

vi keepalived.conf
#################################[MASTER]
! Configuration File for keepalived
global_defs {
notification_email {
64125568@qq.com
}
notification_email_from keepalived@chtopnet.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}

vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
#define nginx-master-IP
mcast_src_ip 10.0.0.15 #这里是本机nginx-master-IP
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass jesse
}

#define vip
virtual_ipaddress {
10.0.0.200 #这个是VIP
}
}
####################################
/etc/init.d/keepalived start
echo "/etc/init.d/keepalived start" >>/etc/rc.local

我们来看一下日志:
[root@MASTER keepalived]# tail /var/log/messages
Sep 20 23:12:40 MASTER Keepalived_vrrp: Registering Kernel netlink reflector
Sep 20 23:12:40 MASTER Keepalived_vrrp: Registering Kernel netlink command channel
Sep 20 23:12:40 MASTER Keepalived_vrrp: Registering gratutious ARP shared channel
Sep 20 23:12:42 MASTER Keepalived_vrrp: Opening file '/etc/keepalived/keepalived.conf'.
Sep 20 23:12:42 MASTER Keepalived_vrrp: Configuration is using : 62588 Bytes
Sep 20 23:12:42 MASTER Keepalived_vrrp: VRRP sockpool: [ifindex(2), proto(112), fd(9,10)]
Sep 20 23:12:43 MASTER Keepalived_vrrp: VRRP_Instance(VI_1) Transition to MASTER STATE
Sep 20 23:12:44 MASTER Keepalived_vrrp: VRRP_Instance(VI_1) Entering MASTER STATE
Sep 20 23:12:44 MASTER Keepalived_vrrp: VRRP_Instance(VI_1) setting protocol VIPs.
Sep 20 23:12:44 MASTER Keepalived_vrrp: VRRP_Instance(VI_1) Sending gratuitous ARPs on eth0 for 10.0.0.200

命令检查vrrp是否启动
[root@MASTER keepalived]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
link/ether 00:0c:29:16:9f:16 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.15/24 brd 10.0.0.255 scope global eth0
inet 10.0.0.200/32 scope global eth0
inet6 fe80::20c:29ff:fe16:9f16/64 scope link
valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
link/ether 00:0c:29:16:9f:20 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.26/24 brd 10.0.0.255 scope global eth1
inet6 fe80::20c:29ff:fe16:9f20/64 scope link
valid_lft forever preferred_lft forever
4: sit0: <NOARP> mtu 1480 qdisc noop
link/sit 0.0.0.0 brd 0.0.0.0

说明vip已经启动

BACKUP机配置如下:

vi keepalived.conf
###################################[BACKUP]
! Configuration File for keepalived
global_defs {
notification_email {
64125568@qq.com
}
notification_email_from keepalived@chtopnet.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}

vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 51
#define nginx-backup-ip
mcast_src_ip 10.0.0.18
priority 99
advert_int 1
authentication {
auth_type PASS
auth_pass jesse
}
virtual_ipaddress {
10.0.0.200
}
}
###################################[BACKUP]###########################################
/etc/init.d/keepalived start
echo "/etc/init.d/keepalived start" >>/etc/rc.local

三:针对Keepalived的不足,用check_nginx.sh来监控nginx进程，实现真正意义上的负载均衡高可用.
此脚本思路其实也很简单，即放置在后台一直监控nginx进程；如进程消失，尝试重启nginx，
如是失败则立即停掉本机的keepalived服务，让另一台负载均衡器接手.

vi /shell/nginx_pid.sh
###################################################
#!/bin/bash
while :
do
nginxpid=`ps -C nginx --no-header | wc -l`

if [ $nginxpid -eq 0 ]
then
/usr/local/nginx-0.8.53/sbin/nginx
sleep 5
if [ $nginxpid -eq 0 ]
then
/etc/init.d/keepalived stop
fi
fi
sleep 5
done
###################################################
nohup /bin/bash /shell/nginx_pid.sh &

如果你正在运行一个进程，而且你觉得在退出帐户时该进程还不会结束，
那么可以使用nohup命令。该命令可以在你退出root帐户之后继续运行相应的进程.

 

###################################
WEB服务器部署:可以配置多端口的apache服务来测试,以为是测试，这里我就只安装apache了
10.0.0.15:700 显示内容:700
10.0.0.15:800 显示内容:800
10.0.0.15:900 显示内容:900

wget http://mirror.bjtu.edu.cn/apache//httpd/httpd-2.2.21.tar.gz

tar zvfx httpd-2.2.21.tar.gz
cd httpd-2.2.21
./configure --prefix=/usr/local/apache2.2.21 \
--enable-deflate \
--enable-headers \
--enable-modules=so \
--enable-so \
--enable-rewrite \
--enable-cgi \
--enable-file-cache \
--enable-cache \
--enable-disk-cache \
--enable-mem-cache
make
make install
cd ..
echo "/usr/local/apache2.2.21/bin/apachectl start" >> /etc/rc.local
注意，此处我采用的是perfork模式

useradd apache -M -s /sbin/nologin

vi /usr/local/apache2.2.21/conf/httpd.conf(做以下修改)
#-----------------------------------------------------------------------------------
1:默认用户和组daemon修改为apache

66 User apache
67 Group apache

2:注释掉80端口，因为nginx在使用这个
40 #Listen 80

3:将如下
105 DocumentRoot "/usr/local/apache2.2.21/htdocs"
132 <Directory "/usr/local/apache2.2.21/htdocs">

中的/usr/local/apache2.2.20/htdocs替换成/data

4:将371行打开并调优perfork模式
371 #Include conf/extra/httpd-mpm.conf

<IfModule mpm_prefork_module>
StartServers 10
MinSpareServers 10
MaxSpareServers 15
ServerLimit 2000
MaxClients 1000
MaxRequestsPerChild 5000
</IfModule>

五:打开虚拟主机
394 Include conf/extra/httpd-vhosts.conf

6:将次做如下修改:目的是禁止显示站点目录功
<Directory "/data">
Options -Indexes FollowSymLinks
AllowOverride None
Order allow,deny
Allow from all
</Directory>

/usr/local/apache2.2.21/bin/apachectl -t
/usr/local/apache2.2.21/bin/apachectl stop
/usr/local/apache2.2.21/bin/apachectl start
#-----------------------------------------------------------------------------------

vi httpd-vhosts.conf
###################################
NameVirtualHost *:700
Listen 700
Listen 800
Listen 900
<VirtualHost *:700>
ServerAdmin 64125568@qq.com
DocumentRoot "/data/700"
ServerName 700.com
ServerAlias www.700.com
ErrorLog "logs/700.com-error_log"
CustomLog "logs/700.com-access_log" common
</VirtualHost>

NameVirtualHost *:800
<VirtualHost *:800>
ServerAdmin 64125568@qq.com
DocumentRoot "/data/800"
ServerName 800.com
ServerAlias www.800.com
ErrorLog "logs/800.com-error_log"
CustomLog "logs/800.com-access_log" common
</VirtualHost>

NameVirtualHost *:900
<VirtualHost *:900>
ServerAdmin 64125568@qq.com
DocumentRoot "/data/900"
ServerName 900.com
ServerAlias www.900.com
ErrorLog "logs/900.com-error_log"
CustomLog "logs/900.com-access_log" common
</VirtualHost>
#######################################

[root@MASTER extra]# mkdir /data/{700,800,900}
[root@MASTER extra]# /usr/local/apache2.2.21/bin/apachectl -t
Syntax OK

echo "700" >/data/700/index.html
echo "800" >/data/800/index.html
echo "900" >/data/900/inhtmldex.

/usr/local/apache2.2.21/bin/apachectl start

访问测试：
down 掉nginx-master
ip add 发现vip 已经不存在了

备机执行: ip add
发现 vip 已经飘过来了

http://10.0.0.200 ctrl+f5强制刷新,发现每刷一次，出现的内容各不一样。

至此，nginx+keepalived的负载均衡主备模式配置成功！