LVS-NAT搭建HTTP及HTTPS
author：JevonWei
版权声明：原创作品
搭建NAT模式的HTTP环境
网络拓扑图如下
网络环境
RS1 192.168.198.138
RS2 192.168.198.120
LVS：
DIP 192.168.198.128
vip 172.16.253.105
路由R1：
172.16.253.166
192.168.80.128
Client 192.168.80.129
RS1,RS2的网关指向192.168.198.128，client的网关指向R1
实现NAT模式的轮询rr模式
VS
添加路由转发选项
[root@VS ~]# vim /etc/sysctl.d/99-sysctl.conf
net.ipv4.ip_forward=1
[root@VS ~]# sysctl -p /etc/sysctl.d/99-sysctl.conf \\刷新生效
net.ipv4.ip_forward = 1
[root@VS ~]# route add -net 192.168.80.0/24 gw 172.16.253.166
[root@VS ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.198.1 0.0.0.0 UG 100 0 0 ens34
0.0.0.0 192.16.0.1 0.0.0.0 UG 101 0 0 ens33
172.16.0.0 0.0.0.0 255.255.0.0 U 100 0 0 ens33
192.16.0.1 0.0.0.0 255.255.255.255 UH 100 0 0 ens33
192.168.80.0 172.16.253.166 255.255.255.0 UG 0 0 0 ens33
192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0
192.168.198.0 0.0.0.0 255.255.255.0 U 100 0 0 ens34
配置LVS的调度算法为rr轮询
[root@VS ~]# yum -y install ipvsadm
[root@VS ~]# ipvsadm -A -t 172.16.253.105:80 -s rr \\-t指定TCP协议，-s指定调度算法为轮询
[root@VS ~]# ipvsadm -a -t 172.16.253.105:80 -r 192.168.198.138 -m \\添加192.168.198.138 RS1服务器到LVS调度，-m 为nat类型
[root@VS ~]# ipvsadm -a -t 172.16.253.105:80 -r 192.168.198.120 -m
[root@VS ~]# ipvsadm -Ln \\查看LVS调度信息
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 172.16.253.105:80 rr
-> 192.168.198.120:80 Masq 1 0 0
-> 192.168.198.138:80 Masq 1 0 0
[root@VS ~]# curl 192.168.198.120
welcome to RS2
[root@VS ~]# curl 192.168.198.138
welcome to RS1
修改LVS的调度模式为wrr
[root@VS ~]# ipvsadm -E -t 172.16.253.105:80 -s wrr
[root@VS ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 172.16.253.105:80 wrr
-> 192.168.198.120:80 Masq 1 0 0
-> 192.168.198.138:80 Masq 1 0 0
[root@VS ~]# ipvsadm -e -t 172.16.253.105:80 -r 192.168.198.138 -m -w 3 \\修改192.168.198.138 RS1主机的权重为3，-w 指定权重，-m为nat算法，192.168.198.120权重仍为1
[root@VS ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 172.16.253.105:80 wrr
-> 192.168.198.120:80 Masq 1 0 0
-> 192.168.198.138:80 Masq 3 0 0
脚本实现lvs-wrr的配置
[root@VS ~]# vim lvs_nat.sh
#! /bin/bash
vip=172.16.253.105:80
rip1=192.168.198.138
rip2=192.168.198.120:8080
sch=wrr
case $1 in
start)
ipvsadm -A -t $vip -s $sch
ipvsadm -a -t $vip -r $rip1 -m -w 3
ipvsadm -a -t $vip -r $rip2 -m -w 1
;;
stop)
ipvsadm -C
;;
*)
echo "Usage:$(basename $0) start|stop"
exit 1
;;
esac
RS1
[root@RS1 ~]# yum -y install httpd
[root@RS1 ~]# vim /var/www/html/index.html
welcome to RS1
[root@RS1 ~]# service httpd start
RS2
[root@RS2 ~]# yum -y install httpd
[root@RS2 ~]# vim /var/www/html/index.html
welcome to RS2
[root@RS2 ~]# service httpd start
路由器R1
[root@R1 ~]# vim /etc/sysctl.d/99-sysctl.conf
net.ipv4.ip_forward=1
[root@R1 ~]# sysctl -p /etc/sysctl.d/99-sysctl.conf
net.ipv4.ip_forward = 1
[root@R1 ~]# route add -net 172.16.0.0/16 gw 172.16.253.105
或
[root@R1 ~]# vim /etc/sysconfig/network-scripts/route-eth0
172.16.0.0/16 via 172.16.253.105
[root@R1 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
192.168.80.0 192.168.80.129 255.255.255.0 UG 0 0 0 eth1
192.168.80.0 0.0.0.0 255.255.255.0 U 1 0 0 eth1
172.16.0.0 172.16.253.105 255.255.0.0 UG 0 0 0 eth0
172.16.0.0 0.0.0.0 255.255.0.0 U 1 0 0 eth0
0.0.0.0 172.16.0.1 0.0.0.0 UG 0 0 0 eth0
client
[root@client ~]# route add -net 172.16.0.0/16 gw 192.168.80.128
访问rr轮询算法
[root@client ~]# for i in {1..10};do curl 172.16.253.105;sleep 1;done
welcome to RS2
welcome to RS1
welcome to RS2
welcome to RS1
welcome to RS2
访问wrr权重算法
[root@client ~]# for i in {1..10};do curl 172.16.253.105;sleep 1;done
welcome to RS2
welcome to RS1
welcome to RS1
welcome to RS1
welcome to RS2
welcome to RS1
welcome to RS1
welcome to RS1
welcome to RS2
welcome to RS1
查看LVS的信息
[root@VS ~]# ipvsadm -Ln --stats
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Conns InPkts OutPkts InBytes OutBytes
-> RemoteAddress:Port
TCP 172.16.253.105:80 29 158 139 10710 15609
-> 192.168.198.120:80 15 82 69 5554 7923
-> 192.168.198.138:80 14 76 70 5156 7686
[root@VS ~]# ipvsadm -Ln --connection \\查看网络连接数
[root@VS ~]# cut -d " " -f1 /var/log/httpd/access_log | sort -nr | uniq -c| sort -n \\查看网络连接
搭建NAT模式的HTTPS环境
在以上实验的基础上搭建HTTPS
- 拓扑图
VS搭建CA服务端
[root@VS ~]# cat /etc/pki/tls/openssl.cnf \\查看证书的相关路径
[root@VS ~]# (umask 077;openssl genrsa -out /etc/pki/CA/private/cakey.pem 1024) \\生成私钥文件
Generating RSA private key, 1024 bit long modulus
.............++++++
.........++++++
e is 65537 (0x10001)
[root@VS ~]# cd /etc/pki/CA
[root@VS CA]# openssl req -new -x509 -key private/cakey.pem -out cacert.pem -days 1024 \\生成自签名证书
You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [XX]:CN
State or Province Name (full name) []:henan
Locality Name (eg, city) [Default City]:zhengzhou
Organization Name (eg, company) [Default Company Ltd]:danran.com
Organizational Unit Name (eg, section) []:it
Common Name (eg, your name or your server's hostname) []:ca.danran.com
Email Address []:
[root@VS CA]# touch index.txt
[root@VS CA]# echo 00 > serial
RS1申请CA证书
[root@RS1 ~]# cd /etc/httpd/conf.d/
[root@RS1 conf.d]# (umask 077;openssl genrsa -out httpd.key 1024)
Generating RSA private key, 1024 bit long modulus
........++++++
...........++++++
e is 65537 (0x10001)
[[root@RS1 conf.d]# openssl req -new -key httpd.key -out httpd.csr -days 10
You are about to be asked to enter information that will be incorporated into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [XX]:CN
State or Province Name (full name) []:henan
Locality Name (eg, city) [Default City]:zhengzhou
Organization Name (eg, company) [Default Company Ltd]:danran.com
Organizational Unit Name (eg, section) []:it
Common Name (eg, your name or your server's hostname) []:ca.danran.com
Email Address []:
Please enter the following 'extra' attributes to be sent with your certificate request
A challenge password []:
An optional company name []:
[root@RS1 conf.d]# scp httpd.csr 192.168.198.128:/etc/pki/CA \\证书申请文件发送给CA服务端
CA服务端颁发证书
[root@VS CA]# openssl ca -in httpd.csr -out certs/httpd.crt -days 365
Using configuration from /etc/pki/tls/openssl.cnf
Check that the request matches the signature
Signature ok
Certificate Details:
Serial Number: 1 (0x1)
Validity
Not Before: Aug 19 13:00:12 2017 GMT
Not After : Aug 19 13:00:12 2018 GMT
Subject:
countryName = CN
stateOrProvinceName = henan
organizationName = danran.com
organizationalUnitName = it
commonName = ca.danran.com
X509v3 extensions:
X509v3 Basic Constraints:
CA:FALSE
Netscape Comment:
OpenSSL Generated Certificate
X509v3 Subject Key Identifier:
BB:DC:5C:85:69:2B:0A:41:98:3B:7F:3E:15:69:1D:2B:C3:81:3E:EF
X509v3 Authority Key Identifier:
keyid:91:15:B3:DB:2D:94:91:2E:12:87:26:ED:05:5E:08:78:E0:10:7C:F8
[root@VS CA]# scp certs/httpd.crt 192.168.198.138:/etc/httpd/conf.d \\将证书文件颁发给RS1申请者
[root@VS CA]# scp cacert.pem 192.168.198.138:/etc/httpd/conf.d \\将CA服务端证书发送给申请者
RS1
[root@RS1 conf.d]# scp cacert.pem httpd.crt httpd.key 192.168.198.120:/etc/httpd/conf.d \\将RS1的证书、私钥及CA证书文件发送给RS2
[root@RS1 ~]# yum -y install mod_ssl
[root@RS1 ~]# vim /etc/httpd/conf.d/ssl.conf \\修改如下证书私钥、证书文件及CA证书文件的存放路径
SSLCertificateFile /etc/httpd/conf.d/httpd.crt
SSLCertificateKeyFile /etc/httpd/conf.d/httpd.key
SSLCACertificateFile /etc/httpd/conf.d/cacert.pem
修改后如下所示
# Server Certificate:
# Point SSLCertificateFile at a PEM encoded certificate. If
# the certificate is encrypted, then you will be prompted for a
# pass phrase. Note that a kill -HUP will prompt again. A new
# certificate can be generated using the genkey(1) command.
SSLCertificateFile /etc/httpd/conf.d/httpd.crt
# Server Private Key:
# If the key is not combined with the certificate, use this
# directive to point at the key file. Keep in mind that if
# you've both a RSA and a DSA private key you can configure
# both in parallel (to also allow the use of DSA ciphers, etc.)
SSLCertificateKeyFile /etc/httpd/conf.d/httpd.key
# Server Certificate Chain:
# Point SSLCertificateChainFile at a file containing the
# concatenation of PEM encoded CA certificates which form the
# certificate chain for the server certificate. Alternatively
# the referenced file can be the same as SSLCertificateFile
# when the CA certificates are directly appended to the server
# certificate for convinience.
#SSLCertificateChainFile /etc/pki/tls/certs/server-chain.crt
# Certificate Authority (CA):
# Set the CA certificate verification path where to find CA
# certificates for client authentication or alternatively one
# huge file containing all of them (file must be PEM encoded)
SSLCACertificateFile /etc/httpd/conf.d/cacert.pem
[root@RS1 conf.d]# service httpd restart
RS2
[root@RS2 ~]# cd /etc/httpd/conf.d/
[root@RS2 conf.d]# ls
cacert.pem httpd.key php.conf welcome.conf
httpd.crt mod_dnssd.conf README
[root@RS2 conf.d]# yum -y install mod_ssl \\安装软件包
[root@RS2 conf.d]# vim ssl.conf \\修改如下证书私钥、证书文件及CA证书文件的存放路径
SSLCertificateFile /etc/httpd/conf.d/httpd.crt
SSLCertificateKeyFile /etc/httpd/conf.d/httpd.key
SSLCACertificateFile /etc/httpd/conf.d/cacert.pem
修改后如下所示
# Server Certificate:
# Point SSLCertificateFile at a PEM encoded certificate. If
# the certificate is encrypted, then you will be prompted for a
# pass phrase. Note that a kill -HUP will prompt again. A new
# certificate can be generated using the genkey(1) command.
SSLCertificateFile /etc/httpd/conf.d/httpd.crt
# Server Private Key:
# If the key is not combined with the certificate, use this
# directive to point at the key file. Keep in mind that if
# you've both a RSA and a DSA private key you can configure
# both in parallel (to also allow the use of DSA ciphers, etc.)
SSLCertificateKeyFile /etc/httpd/conf.d/httpd.key
# Server Certificate Chain:
# Point SSLCertificateChainFile at a file containing the
# concatenation of PEM encoded CA certificates which form the
# certificate chain for the server certificate. Alternatively
# the referenced file can be the same as SSLCertificateFile
# when the CA certificates are directly appended to the server
# certificate for convinience.
#SSLCertificateChainFile /etc/pki/tls/certs/server-chain.crt
# Certificate Authority (CA):
# Set the CA certificate verification path where to find CA
# certificates for client authentication or alternatively one
# huge file containing all of them (file must be PEM encoded)
SSLCACertificateFile /etc/httpd/conf.d/cacert.pem
[root@RS2 conf.d]# service httpd restart
VS
[root@VS ~]# vim lvs_nat.sh
#! /bin/bash
vip=172.16.253.105:443
rip1=192.168.198.138
rip2=192.168.198.120
sch=wrr
case $1 in
start)
ipvsadm -A -t $vip -s $sch
ipvsadm -a -t $vip -r $rip1 -m -w 3
ipvsadm -a -t $vip -r $rip2 -m -w 1
;;
stop)
ipvsadm -C
;;
*)
echo "Usage:$(basename $0) start|stop"
exit 1
;;
esac
[root@VS ~]# bash lvs_nat.sh stop
[root@VS ~]# bash lvs_nat.sh start
[root@VS ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 172.16.253.105:443 wrr
-> 192.168.198.120:443 Masq 1 0 0
-> 192.168.198.138:443 Masq 3 0 0
client客户端
[root@client ~]# for i in {1..10};do curl -k https://172.16.253.105;done \\-k跳过证书
welcome to RS1
welcome to RS1
welcome to RS1
welcome to RS2
welcome to RS1
welcome to RS1
welcome to RS1
welcome to RS2
welcome to RS1
welcome to RS1
实现一个LVS调用一组不同服务
VS
搭建https的LVS_nat
[root@VS ~]# vim lvs_nat.sh
#! /bin/bash
vip=172.16.253.105:443
rip1=192.168.198.138
rip2=192.168.198.120
sch=wrr
case $1 in
start)
ipvsadm -A -t $vip -s $sch
ipvsadm -a -t $vip -r $rip1 -m -w 3
ipvsadm -a -t $vip -r $rip2 -m -w 1
;;
stop)
ipvsadm -C
;;
*)
echo "Usage:$(basename $0) start|stop"
exit 1
;;
esac
[root@VS ~]# bash lvs_nat.sh stop
[root@VS ~]# bash lvs_nat.sh start
[root@VS ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 172.16.253.105:443 wrr
-> 192.168.198.120:443 Masq 1 0 0
-> 192.168.198.138:443 Masq 3 0 0
[root@VS ~]# bash lvs_nat.sh start
搭建http的LVS_nat
[root@VS ~]# vim lvs_nat2.sh
#! /bin/bash
vip=172.16.253.105:80
rip1=192.168.198.138
rip2=192.168.198.120:8080
sch=wrr
case $1 in
start)
ipvsadm -A -t $vip -s $sch
ipvsadm -a -t $vip -r $rip1 -m -w 3
ipvsadm -a -t $vip -r $rip2 -m -w 1
;;
stop)
ipvsadm -C
;;
*)
echo "Usage:$(basename $0) start|stop"
exit 1
;;
esac
[root@VS ~]# bash lvs_nat.sh stop
[root@VS ~]# bash lvs_nat.sh start
[root@VS ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 172.16.253.105:443 wrr
-> 192.168.198.120:443 Masq 1 0 0
-> 192.168.198.138:443 Masq 3 0 0
[root@VS ~]# bash lvs_nat2.sh start
[root@VS ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 172.16.253.105:80 wrr
-> 192.168.198.120:8080 Masq 1 0 0
-> 192.168.198.138:80 Masq 3 0 0
TCP 172.16.253.105:443 wrr
-> 192.168.198.120:443 Masq 1 0 0
-> 192.168.198.138:443 Masq 3 0 0
client
[root@client ~]# for i in {1..10};do curl -k https://172.16.253.105;done
welcome to RS2
welcome to RS1
welcome to RS1
welcome to RS2
welcome to RS1
welcome to RS1
welcome to RS2
welcome to RS1
welcome to RS1
welcome to RS2
[root@client ~]# for i in {1..10};do curl -k http://172.16.253.105;done
welcome to RS2
welcome to RS1
welcome to RS1
welcome to RS1
welcome to RS2
welcome to RS1
welcome to RS1
welcome to RS1
welcome to RS2
welcome to RS1
danran
