LVS+NGINX+TOMCAT_集群实施操作记录.docx
LVS
IP：
Eth0:192.168.100.115
Eth1:192.168.100.215
Vi /etc/init.d./lvs
#!/bin/sh
#
# lvs Start lvs
#
# chkconfig: 2345 08 92
# description: Starts, stops and saves lvs
#
SNS_VIP=192.168.100.215
SNS_RIP1=192.168.100.114
SNS_RIP2=192.168.100.113
. /etc/rc.d/init.d/functions
#logger $0 called with $1
retval=0
start()
{
#set squid vip
/sbin/ipvsadm --set 30 5 60
#/sbin/ifconfig eth0:0 $SNS_VIP broadcast $SNS_VIP netmask 255.255.255.255 broadcast $SNS_VIP up
#/sbin/route add -host $SNS_VIP dev eth0:0
/sbin/ipvsadm -A -t $SNS_VIP:80 -s rr
/sbin/ipvsadm -a -t $SNS_VIP:80 -r $SNS_RIP1 -g
/sbin/ipvsadm -a -t $SNS_VIP:80 -r $SNS_RIP2 -g
touch /var/lock/subsys/ipvsadm > /dev/null 2 >&1
echo "ipvsadm started"
}
stop()
{
/sbin/ipvsadm -C
/sbin/ipvsadm -Z
#ifconfig eth0:0 down
#route del $SNS_VIP
rm -rf /var/lock/subsys/ipvsadm > /dev/null 2 >&1
echo "ipvsadm stoped"
}
status()
{
if [ ! -e /var/lock/subsys/ipvsadm ];then
echo "ipvsadm stoped"
exit 1
else
echo "ipvsadm OK"
fi
}
case "$1" in
start)
start
;;
stop)
stop
;;
status)
status
;;
restart)
stop
start
;;
*)
echo $"Usage: $0 {start|stop|status}"
retval=1
esac
exit $retval
keepalvived
tar xf keepalived-1.2.7.tar.gz
cd keepalived-1.2.7
./configure --prefix=/ --mandir=/usr/local/share/man/ --with-kernel-dir=/usr/src/kernels/2.6.32-279.el6.x86_64
Make
make install
cd /etc/keepalived/
mv keepalived.conf keepalived.conf.default
chkconfig --add keepalived
chkconfig keepalived on
vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from root@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id Director1
}
#VRRP（虚拟路由冗余协议）实例配置
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
priority 150
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.122.254/24 eth1
}
}
#LVS配置
virtual_server 192.168.100.215 80 {
delay_loop 3
lb_algo rr
lb_kind DR
nat_mask 255.255.255.0
# persistence_timeout 50
protocol TCP
real_server 192.168.100.113 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.100.114 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
}
}
}
}
LVS-realserver
Ip 192.168.100.113
Ip 192.168.100.114
Vi /etc/init.d/rsup
#!/bin/bash
VIP=192.168.100.215
ifconfig lo:0 $VIP broadcast $VIP netmask 255.255.255.255 up
#route add –host $VIP dev lo:0
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
#sysctl –p
Nginx
Ip 192.168.100.122（nfs）
Ip 192.168.100.113
Ip 192.168.100.114
#安装zlib
tar xzvf zlib-1.2.3.tar.gz
cd zlib-1.2.3
./configure
make && make install
#安装pcre
tar zxvf pcre-7.9.tar.gz
cd pcre-7.9
./configure --prefix=/usr/local/pcre
make && make install
wget http://h264.code-shop.com/download/nginx_mod_h264_streaming-2.2.7.tar.gz
tar -zxvf nginx_mod_h264_streaming-2.2.7.tar.gz
unzip nginx_upstream_check_module-master.zip
mv ./nginx_upstream_check_module-master /root/health
tar -xvf nginx-1.4.1.tar.gz -C /usr/src/
useradd nginx
cd /usr/src/nginx-1.4.1
patch -p1 < /root/health/check_1.2.6+.patch
./configure --user=nginx --group=nginx --with-http_stub_status_module --with-http_ssl_module --with-http_gzip_static_module --with-http_flv_module --add-module=../nginx_mod_h264_streaming-2.2.7 --with-pcre=/software/pcre-7.9 --with-zlib=/software/zlib-1.2.3 --prefix=/usr/local/nginx --add-module=/root/health
make && make install
/usr/local/nginx/sbin/nginx
开机自动启动
vi /etc/init.d/nginx
#!/bin/bash
#
# nginx - this script starts and stops the nginx daemin
#
# chkconfig: - 85 15
# description: Nginx is an HTTP(S) server, HTTP(S) reverse \
# proxy and IMAP/POP3 proxy server
# processname: nginx
# config: /usr/local/nginx/conf/nginx.conf
# pidfile: /usr/local/nginx/logs/nginx.pid
# Source function library.
. /etc/rc.d/init.d/functions
# Source networking configuration.
. /etc/sysconfig/network
# Check that networking is up.
[ "$NETWORKING" = "no" ] && exit 0
nginx="/usr/local/nginx/sbin/nginx"
prog=$(basename $nginx)
NGINX_CONF_FILE="/usr/local/nginx/conf/nginx.conf"
lockfile=/var/lock/subsys/nginx
start() {
[ -x $nginx ] || exit 5
[ -f $NGINX_CONF_FILE ] || exit 6
echo -n $"Starting $prog: "
daemon $nginx -c $NGINX_CONF_FILE
retval=$?
echo
[ $retval -eq 0 ] && touch $lockfile
return $retval
}
stop() {
echo -n $"Stopping $prog: "
killproc $prog -QUIT
retval=$?
echo
[ $retval -eq 0 ] && rm -f $lockfile
return $retval
}
restart() {
configtest || return $?
stop
start
}
reload() {
configtest || return $?
echo -n $"Reloading $prog: "
killproc $nginx -HUP
RETVAL=$?
echo
}
force_reload() {
restart
}
configtest() {
$nginx -t -c $NGINX_CONF_FILE
}
rh_status() {
status $prog
}
rh_status_q() {
rh_status >/dev/null 2>&1
}
case "$1" in
start)
rh_status_q && exit 0
$1
;;
stop)
rh_status_q || exit 0
$1
;;
restart|configtest)
$1
;;
reload)
rh_status_q || exit 7
$1
;;
force-reload)
force_reload
;;
status)
rh_status
;;
condrestart|try-restart)
rh_status_q || exit 0
;;
*)
echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload|configtest}"
exit 2
esac
##############################################################################
改权限
chmod 755 /etc/init.d/nginx
chkconfig nginx on
chkconfig --list nginx
nginx 0:off 1:off 2:on 3:on 4:on 5:on 6:off
service nginx start
service nginx stop
service nginx restart
service nginx reload
/etc/init.d/nginx start
/etc/init.d/nginx stop
/etc/init.d/nginx restart
/etc/init.d/nginx reload
Nginx 配置文件
vi /usr/local/nginx/conf/nginx.conf
#user nginx nginx;
worker_processes 12;
error_log /usr/local/nginx/logs/error.log crit;
pid /usr/local/nginx/logs/nginx.pid;
worker_rlimit_nofile 65535;
events
{
use epoll;
worker_connections 204800;
}
http
{
include mime.types;
default_type application/octet-stream;
charset utf-8;
server_names_hash_bucket_size 128;
client_header_buffer_size 2k;
large_client_header_buffers 4 4k;
client_max_body_size 8m;
sendfile on;
tcp_nopush on;
keepalive_timeout 60;
fastcgi_cache_path /usr/local/nginx/fastcgi_temp levels=1:2
keys_zone=TEST:10m
inactive=5m;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 1638;
fastcgi_buffers 16 16k;
fastcgi_busy_buffers_size 16k;
fastcgi_temp_file_write_size 16k;
fastcgi_cache TEST;
fastcgi_cache_valid 200 302 1h;
fastcgi_cache_valid 301 1d;
fastcgi_cache_valid any 1m;
fastcgi_cache_min_uses 1;
fastcgi_cache_use_stale error timeout invalid_header http_500;
open_file_cache max=204800 inactive=20s;
open_file_cache_min_uses 1;
open_file_cache_valid 30s;
tcp_nodelay on;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary on;
upstream nginx_server {
ip_hash;
server 192.168.100.122:80;
}
upstream web_server {
ip_hash;
server 192.168.100.131:9001;
server 192.168.100.132:9001;
server 192.168.100.133:9001;
server 192.168.100.134:9001;
check interval=3000 rise=2 fall=5 timeout=1000;
}
upstream napi_server {
ip_hash;
server 192.168.100.131:9002;
server 192.168.100.132:9002;
server 192.168.100.133:9002;
server 192.168.100.134:9002;
server 192.168.100.131:9003;
server 192.168.100.132:9003;
server 192.168.100.133:9003;
server 192.168.100.134:9003;
server 192.168.100.131:9004;
server 192.168.100.132:9004;
server 192.168.100.133:9004;
server 192.168.100.134:9004;
check interval=3000 rise=2 fall=5 timeout=1000;
}
upstream oapi_server {
ip_hash;
server 192.168.100.131:9005;
server 192.168.100.132:9005;
server 192.168.100.133:9005;
server 192.168.100.134:9005;
server 192.168.100.131:9006;
server 192.168.100.132:9006;
server 192.168.100.133:9006;
server 192.168.100.134:9006;
check interval=3000 rise=2 fall=5 timeout=1000;
}
server {
listen 80;
server_name localhost;
location ~ .*\.(gif|jpg|jpeg|png|bmp|swf|html|mp3|wma|wmv)$
{
root /usr/wodm/;
proxy_pass http://nginx_server;
}
location ~ .*\.(js|css)?$
{
expires 1h;
}
location /NginxStatus
{
stub_status on;
access_log on;
auth_basic "NginxStatus";
#auth_basic_user_file conf/htpasswd;
}
location /nstatus {
check_status;
access_log off;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
log_format access '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" $http_x_forwarded_for';
access_log /usr/local/nginx/logs/access.log access;
include /usr/local/nginx/conf/vhost/*.conf;
}
Nginx(nfs)
Nginx.conf
#user nginx nginx;
worker_processes 16;
#worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000;
error_log /usr/local/nginx/logs/error.log crit;
pid /usr/local/nginx/logs/nginx.pid;
worker_rlimit_nofile 204800;
events
{
use epoll;
worker_connections 204800;
}
http
{
include mime.types;
default_type application/octet-stream;
charset utf-8;
server_names_hash_bucket_size 128;
client_header_buffer_size 2k;
large_client_header_buffers 4 4k;
client_max_body_size 8m;
sendfile on;
tcp_nopush on;
keepalive_timeout 60;
fastcgi_cache_path /usr/local/nginx/fastcgi_cache levels=1:2
keys_zone=TEST:10m
inactive=5m;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 1638;
fastcgi_buffers 16 16k;
fastcgi_busy_buffers_size 16k;
fastcgi_temp_file_write_size 16k;
fastcgi_cache TEST;
fastcgi_cache_valid 200 302 1h;
fastcgi_cache_valid 301 1d;
fastcgi_cache_valid any 1m;
fastcgi_cache_min_uses 1;
fastcgi_cache_use_stale error timeout invalid_header http_500;
open_file_cache max=204800 inactive=20s;
open_file_cache_min_uses 1;
open_file_cache_valid 30s;
tcp_nodelay on;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary on;
server
{
listen 80;
server_name localhost;
index index.php index.htm;
root /usr/wodm/;
location /status
{
stub_status on;
}
location ~ .*\.(gif|jpg|jpeg|png|bmp|swf|html|mp3|wma|wmv)$
{
expires 30d;
}
}
log_format access '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" $http_x_forwarded_for';
access_log /usr/local/nginx/logs/access.log access;
标准字符集
vi /etc/sysconfig/i18n
LANG="zh_CN.UTF-8" SYSFONT="latarcyrheb-sun16" SUPPORTED="zh_CN.UTF-8:zh_CN:zh"
LANG="zh_CN.GB18030"
SUPPORTED="zh_CN.GB18030:zh_CN:zh:en_US.UTF-8:en_US:en"
打开文件数
vi /etc/security/limits.conf
* soft nofile 65535
* hard nofile 65535
vi /etc/pam.d/login
session required /lib/security/pam_limits.so
nginx 日志按天分割
vi nginx_log.sh
#!/bin/bash
logs_path="/usr/local/nginx/logs/"
pid_path="/usr/local/nginx/nginx.pid"
mv ${logs_path}access.log ${logs_path}access_$(date -d "yesterday" +"%Y%m%d").log
kill -USR1 `cat ${pid_path}`
chmod 755 nginx_log.sh
crontab –e
0 0 * * * bash /usr/local/nginx/nginx_log.sh
Tomcat memcache
tar xf libevent-1.4.11-stable.tar.gz
cd libevent-1.4.11-stable
./configure
make
make install
cd ../
tar xf memcached-1.4.5.tar.gz
cd memcached-1.4.5
./configure
make
make install
/usr/local/bin/memcached -d -m 10 -u root -l 192.168.1.113 -p 11211 -c 1024 -P /tmp/memcached.pid
Memcache+tomcat 共享session
后续计划尚未配置
