基于Tengine的反向代理详细配置
系统环境：
SUSE Linux Enterprise Server 10 SP1 (x86_64)
注：所有软件包都放置在/data/software目录下
nginx_tcp_proxy_module：
https://github.com/yaoweibin/nginx_tcp_proxy_module
nginx-hmux-module：
https://github.com/wangbin579/nginx-hmux-module
ngx_cache_purge：
http://labs.frickle.com/files/
#---------------------------------------------------------------------------------------------------------------------------------------------
1、内存管理库
# tar -zxvf libunwind-1.0.1.tar.gz
# cd libunwind-1.0.1
# LAGS=-fPIC ./configure --prefix=/usr/local
# make LAGS=-fPIC
# make LAGS=-fPIC install
# tar -zxvf gperftools-2.0.tar.gz
# cd gperftools-2.0
# ./configure --prefix=/usr/local
# make && make install
#---------------------------------------------------------------------------------------------------------------------------------------------
2、正则库
# tar -xvzf pcre-8.32.tar.gz
# cd pcre-8.32
# LAGS=-fPIC ./configure --prefix=/usr/local
# make LAGS=-fPIC
# make LAGS=-fPIC install
#---------------------------------------------------------------------------------------------------------------------------------------------
3、OpenSSL库
# tar xvzf openssl-1.0.1g.tar.gz
# cd openssl-1.0.1g
# ./config shared --prefix=/usr/local
# make && make install
#---------------------------------------------------------------------------------------------------------------------------------------------
4、IP地理位置定位组件
# tar xvzf GeoIP-latest.tar.gz
# cd GeoIP-1.5.0
# ./configure --prefix=/usr/local
# make && make install
#---------------------------------------------------------------------------------------------------------------------------------------------
5、相关目录创建
# mkdir -p /data/nginx_temp/{nginx_client,nginx_proxy,nginx_fastcgi,nginx_temp,nginx_cache}
# mkdir -p /data/logs/{nginx,web} /data/web/{data,conf}
#---------------------------------------------------------------------------------------------------------------------------------------------
6、Tengine编译安装
# tar xvzf nginx-hmux-module-1.3.tar.gz
# tar xvzf nginx_tcp_proxy_module-0.4.5.tar.gz
# tar xvzf tengine-1.5.2.tar.gz
# cd tengine-1.5.2
# patch -p1 < ../nginx_tcp_proxy_module-0.4.5/tcp.patch
# ./configure --prefix=/usr/local/nginx \
--lock-path=/var/lock/nginx.lock \
--pid-path=/var/run/nginx.pid \
--error-log-path=/data/logs/nginx/error.log \
--http-log-path=/data/logs/nginx/access.log \
--user=nobody \
--group=nogroup \
--with-pcre=../pcre-8.32 \
--with-pcre-opt=-fPIC \
--with-openssl=../openssl-1.0.1g \
--with-openssl-opt=-fPIC \
--with-backtrace_module \
--with-http_stub_status_module \
--with-http_gzip_static_module \
--with-http_realip_module \
--with-http_concat_module=shared \
--with-http_sysguard_module=shared \
--with-http_limit_conn_module=shared \
--with-http_limit_req_module=shared \
--with-http_split_clients_module=shared \
--with-http_footer_filter_module=shared \
--with-http_geoip_module=shared \
--with-http_sub_module=shared \
--with-http_access_module=shared \
--with-http_upstream_ip_hash_module=shared \
--with-http_upstream_least_conn_module=shared \
--with-http_referer_module=shared \
--with-http_rewrite_module=shared \
--with-http_memcached_module=shared \
--with-http_upstream_session_sticky_module=shared \
--with-http_addition_module=shared \
--with-http_xslt_module=shared \
--with-http_image_filter_module=shared \
--with-http_user_agent_module=shared \
--with-http_empty_gif_module=shared \
--with-http_browser_module=shared \
--with-google_perftools_module \
--with-http_map_module=shared \
--with-http_userid_filter_module=shared \
--with-http_charset_filter_module=shared \
--with-http_trim_filter_module=shared \
--with-http_lua_module=shared \
--without-http_fastcgi_module \
--without-http_uwsgi_module \
--without-http_scgi_module \
--without-select_module \
--without-poll_module \
--add-module=../nginx-hmux-module-1.3 \
--add-module=../nginx_tcp_proxy_module-0.4.5 \
--with-ld-opt='-ltcmalloc_minimal' \
--http-client-body-temp-path=/data/nginx_temp/nginx_client \
--http-proxy-temp-path=/data/nginx_temp/nginx_proxy \
--http-fastcgi-temp-path=/data/nginx_temp/nginx_fastcgi
# make && make install
#---------------------------------------------------------------------------------------------------------------------------------------------
7、Tengine缓存刷新模块
# cd /data/software
# tar xvzf ngx_cache_purge-2.0.tar.gz
# ./dso_tool --add-module=/data/software/ngx_cache_purge-2.0
#---------------------------------------------------------------------------------------------------------------------------------------------
8、Tengine配置
# rm -f /usr/local/nginx/html/*.html
# rm -f /usr/local/nginx/conf/*.default
# mkdir /usr/local/nginx/conf/SET
# vim /usr/local/nginx/conf/nginx.conf
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
|
user nobody nogroup; worker_processes auto;worker_cpu_affinity auto; error_log /data/logs/nginx/error.log crit;pid /var/run/nginx.pid;google_perftools_profiles /var/tmp/tcmalloc; worker_rlimit_nofile 65535; dso { load ngx_http_rewrite_module.so; load ngx_http_access_module.so; load ngx_http_concat_module.so; load ngx_http_limit_conn_module.so; load ngx_http_limit_req_module.so; load ngx_http_sysguard_module.so; load ngx_http_upstream_session_sticky_module.so; load ngx_http_cache_purge_module.so; load ngx_http_trim_filter_module.so;} events { use epoll; worker_connections 10240;} http { server_tokens off; server_tag off; autoindex off; access_log off; include mime.types; default_type application/octet-stream; server_names_hash_bucket_size 128; client_header_buffer_size 32k; large_client_header_buffers 4 32k; client_max_body_size 10m; client_body_buffer_size 256k; sendfile on; tcp_nopush on; keepalive_timeout 60; tcp_nodelay on; gzip on; gzip_min_length 1k; gzip_buffers 4 16k; gzip_http_version 1.0; gzip_comp_level 2; gzip_types text/plain application/x-javascript text/css application/xml; gzip_vary on; proxy_connect_timeout 600; proxy_read_timeout 600; proxy_send_timeout 600; proxy_buffer_size 128k; proxy_buffers 4 128k; proxy_busy_buffers_size 256k; proxy_temp_file_write_size 256k; proxy_headers_hash_max_size 1024; proxy_headers_hash_bucket_size 128; proxy_redirect off; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_temp_path /data/nginx_temp/nginx_temp; proxy_cache_path /data/nginx_temp/nginx_cache levels=1:2 keys_zone=cache_one:2048m inactive=30m max_size=60g; # backend web server address pool include SET/*.conf; log_format access '$remote_addr - $remote_user [$time_local] "$request"' '$status $body_bytes_sent "$http_referer"' '"$http_user_agent" $http_x_forwarded_for'; # system resource overload protect server { sysguard on; sysguard_load load=10.5 action=/loadlimit; sysguard_mem swapratio=20% action=/swaplimit; sysguard_mem free=100M action=/freelimit; location /loadlimit { return 503; } location /swaplimit { return 503; } location /freelimit { return 503; } } # refuse request server by ipaddr server { server_name _; return 404; } # web page cache and proxy setting include /data/web/conf/*.conf;} |
# vim /usr/local/nginx/conf/SET/NORTH1.conf
|
1
2
3
4
5
6
7
8
9
10
11
12
|
upstream NORTH1_SERVER_PROXY { consistent_hash $request_uri; server 192.168.1.101:80 weight=1; server 192.168.1.102:80 weight=1; server 192.168.1.103:80 weight=1; server 192.168.1.104:80 weight=1; session_sticky; check interval=3000 rise=2 fall=5 timeout=1000 type=http; check_http_send "GET / HTTP/1.0\r\n\r\n"; check_http_expect_alive http_2xx http_3xx;} |
# mkdir -p /data/logs/web/test.qq.com
# vim /data/web/conf/test.qq.com.conf
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
|
server { listen 80; server_name test.qq.com; index index.html index.htm index.php; root /data/nginx_temp/nginx_cache; access_log on; trim on; trim_jscss on; location / { proxy_next_upstream http_500 http_502 http_503 http_504 error timeout invalid_header; proxy_pass http://NORTH1_SERVER_PROXY; #存在静态首页时，才需添加此规则 if (-d $request_filename) { rewrite ^/(.*)$ http://$host/index.html break; } } location ~ .*\. (php)?$ { proxy_next_upstream http_500 http_502 http_503 http_504 error timeout invalid_header; proxy_pass http://NORTH1_SERVER_PROXY; } location ~ /purge(/.*) { allow 127.0.0.1; allow 192.168.1.0/24; deny all; proxy_cache_purge cache_one $host$1$is_args$args; } location ~ .*\.(htm|html|js|css|gif|jpg|jpeg|png|bmp|ico|swf|flv)$ { proxy_next_upstream http_500 http_502 http_503 http_504 error timeout invalid_header; proxy_cache cache_one; proxy_cache_valid 200 304 15m; proxy_cache_valid 301 302 10m; proxy_cache_valid any 1m; proxy_cache_key $host$uri$is_args$args; add_header Ten-webcache '$upstream_cache_status from $host'; proxy_pass http://NORTH1_SERVER_PROXY; expires 30m; } location ~ /\.ht { deny all; } access_log /data/logs/web/test.qq.com/access.log access;} |
#---------------------------------------------------------------------------------------------------------------------------------------------
9、Tengine启动脚本
# vim /etc/init.d/nginx
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
|
#!/bin/sh## nginx - this script start and stop the nginx daemon## chkconfig: 2345 55 25# description: Startup script for nginx# processname: nginx# config: /usr/local/nginx/conf/nginx.conf# pidfile: /var/run/nginx.pid#PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin DAEMON=/usr/local/nginx/sbin/nginxCONFIGFILE=/usr/local/nginx/conf/nginx.confPIDFILE=/var/run/nginx.pidSCRIPTNAME=/etc/init.d/nginxLOCKFILE=/var/lock/nginx.lock set -e[ -x "$DAEMON" ] || exit 0 start() { echo "Startting Nginx......" [ -x $DAEMON ] || exit 5 [ -f $CONFIGFILE ] || exit 6 $DAEMON -c $CONFIGFILE || echo -n "Nginx already running!" [ $? -eq 0 ] && touch $LOCKFILE} stop() { echo "Stopping Nginx......" MPID=`ps aux | grep nginx | awk '/master/{print $2}'` if [ "${MPID}X" != "X" ]; then kill -QUIT $MPID [ $? -eq 0 ] && rm -f $LOCKFILE else echo "Nginx server is not running!" fi} reload() { echo "Reloading Nginx......" MPID=`ps aux | grep nginx | awk '/master/{print $2}'` if [ "${MPID}X" != "X" ]; then kill -HUP $MPID else echo "Nginx can't reload!" fi} case "$1" instart) start ;; stop) stop ;; reload) reload ;; restart) stop sleep 1 start ;; *) echo "Usage: $SCRIPTNAME {start|stop|reload|restart}" exit 3 ;;esac exit 0 |
# chmod +x /etc/init.d/nginx
# chkconfig --add nginx
# service nginx start
#---------------------------------------------------------------------------------------------------------------------------------------------
10、Tengine健康检测
# mkdir -p /data/web/data/mycheckweb.act.qq.com
# echo "OK" > /data/web/data/mycheckweb.act.qq.com/index.html
# echo "你的内网IP mycheckweb.act.qq.com" >> /etc/hosts
# touch /var/lock/check_web.lock
#vim /data/web/conf/checkweb_for_nginx.conf
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
|
server { listen 80; server_name mycheckweb.act.qq.com; access_log off; location / { root /data/web/data/mycheckweb.act.qq.com; index index.html; } location ~ health_status { check_status; allow 127.0.0.1; allow 192.168.1.0/24; deny all; }} |
# vim /usr/local/nginx/sbin/check_web.sh
|
1
2
3
4
5
6
7
8
9
10
|
#!/bin/bashPATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin:/usr/local/sbin retval=`ping -c 3 mycheckweb.act.qq.com | awk '/received/ {print $4}'`[[ ${retval} -eq 0 ]] && exit 1 retval=`curl -I -s "http://mycheckweb.act.qq.com" | grep "200 OK"`if [[ "${retval}x" = "x" ]]; then [[ -e /usr/local/nginx ]] && /sbin/service nginx restart >/dev/null 2>&1fi |
#chmod +x /usr/local/nginx/sbin/check_web.sh
# crontab -e
*/5 * * * * (flock --timeout=0 /var/lock/check_web.lock /usr/local/nginx/sbin/check_web.sh >/dev/null 2>&1)
#---------------------------------------------------------------------------------------------------------------------------------------------
11、Tengine访问日志切割与清理
# vim /usr/local/nginx/sbin/cut_nginx_log.sh
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
|
#!/bin/bashPATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin:/usr/local/sbin ## the nginx access logs base pathWEBLOG_PATH="/data/logs/web" retval=`ps aux | grep ngin[x] | wc -l`if [ ${retval} -eq 0 ]; then echo "The daemon process for nginx has no found." exit 1fi ## avoid errors for USR1 signal, and modify 750 privilegechown -R nobody:nogroup /data/logs/{nginx,web}chmod -R 750 /data/logs/{nginx,web} ## cut nginx access logsfor LOGFILE in `find ${WEBLOG_PATH} -type f -name access.log`do LOGPATH=`dirname ${LOGFILE}` mv ${LOGPATH}/access.log ${LOGPATH}/access_$(date -d "yesterday" +"%Y-%m-%d").logdone kill -USR1 `ps aux | grep nginx | awk '/master/{print $2}'` ## and then modify original privilegeschown -R nobody:nogroup /data/logs/{nginx,web}chmod -R 640 /data/logs/{nginx,web} ## clear 10 days ago's nginx access logsLOGFILE=access_$(date -d "10 days ago" +"%Y-%m-%d").logfind ${WEBLOG_PATH} -type f -name ${LOGFILE} -exec rm -f {} \; |
# crontab -e
00 00 * * * /bin/bash /usr/local/nginx/sbin/cut_nginx_log.sh >/dev/null 2>&1
#---------------------------------------------------------------------------------------------------------------------------------------------
12、系统优化
##网络参数设置
# vim /etc/sysctl.conf
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
|
net.ipv4.tcp_syncookies = 1net.ipv4.tcp_tw_reuse = 1net.ipv4.tcp_tw_recycle = 1net.ipv4.tcp_fin_timeout = 30net.ipv4.tcp_keepalive_time = 1200 net.ipv4.ip_local_port_range = 1024 65000net.ipv4.tcp_max_syn_backlog = 8192net.ipv4.tcp_max_tw_buckets = 80000 net.core.somaxconn = 32768 net.ipv4.tcp_keepalive_probes = 5net.ipv4.tcp_keepalive_intvl = 20 net.core.wmem_default = 8388608net.core.rmem_default = 8388608net.core.rmem_max = 16777216net.core.wmem_max = 16777216 net.ipv4.tcp_rmem = 4096 87380 16777216net.ipv4.tcp_wmem = 4096 65536 16777216 net.core.netdev_max_backlog = 32768 net.ipv4.tcp_timestamps = 0net.ipv4.tcp_synack_retries = 2net.ipv4.tcp_syn_retries = 2net.ipv4.tcp_retries2 = 5 net.ipv4.tcp_mem = 41943040 73400320 94371840net.ipv4.tcp_max_orphans = 3276800fs.file-max = 1300000 |
# sysctl -p
## 文件描述符设置
# echo "ulimit -SHn 65535" >> /etc/profile
# source /etc/profile
#---------------------------------------------------------------------------------------------------------------------------------------------
13、测试
本地HOSTS绑定访问
