Linux路由机制打通网络
主机名：node2.smoke.com
操作系统：centos 7.5
内核版本：3.10.0-862.el7.x86_64
网卡1：vmnet0 172.16.100.68
网卡2：vmnet8 dhcp client
node2：
[root@node2 ~]# cd /etc/yum.repos.d/ [root@node2 yum.repos.d]# mkdir OldMirrorFile [root@node2 yum.repos.d]# mv CentOS-* OldMirrorFile/ [root@node2 yum.repos.d]# curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo [root@node2 yum.repos.d]# curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo [root@node2 yum.repos.d]# yum install -y yum-utils device-mapper-persistent-data lvm2
下载docker-ce的yum源报错如下：
[root@node2 yum.repos.d]# yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo Loaded plugins: fastestmirror adding repo from: https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo grabbing file https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo to /etc/yum.repos.d/docker-ce.repo Could not fetch/save url https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo to file /etc/yum.repos.d/docker-ce.repo: [Errno 14] curl#60 - "Peer's Certificate has expired."
由于系统时间问题导致：
[root@node2 yum.repos.d]# hwclock -s [root@node2 yum.repos.d]# date Wed Feb 26 21:57:39 EST 2020 [root@node2 yum.repos.d]# yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo [root@node2 yum.repos.d]# yum makecache fast [root@node2 yum.repos.d]# yum -y install docker-ce [root@node2 yum.repos.d]# systemctl enable docker
# 注意：
# 官方软件源默认启用了最新的软件，您可以通过编辑软件源的方式获取各个版本的软件包。例如官方并没有将测试版本的软件源置为可用，您可以通过以下方式开启。同理可以开启各种测试版本等。
# vim /etc/yum.repos.d/docker-ee.repo # 将[docker-ce-test]下方的enabled=0修改为enabled=1 #
# 安装指定版本的Docker-CE:
# Step 1: 查找Docker-CE的版本:
# yum list docker-ce.x86_64 --showduplicates | sort -r # Loading mirror speeds from cached hostfile # Loaded plugins: branch, fastestmirror, langpacks # docker-ce.x86_64 17.03.1.ce-1.el7.centos docker-ce-stable # docker-ce.x86_64 17.03.1.ce-1.el7.centos @docker-ce-stable # docker-ce.x86_64 17.03.0.ce-1.el7.centos docker-ce-stable # Available Packages
# Step2: 安装指定版本的Docker-CE: (VERSION例如上面的17.03.0.ce.1-1.el7.centos)
# sudo yum -y install docker-ce-[VERSION]
#安装校验
root@iZbp12adskpuoxodbkqzjfZ:$ docker version Client: Version: 17.03.0-ce API version: 1.26 Go version: go1.7.5 Git commit: 3a232c8 Built: Tue Feb 28 07:52:04 2017 OS/Arch: linux/amd64 Server: Version: 17.03.0-ce API version: 1.26 (minimum version 1.12) Go version: go1.7.5 Git commit: 3a232c8 Built: Tue Feb 28 07:52:04 2017 OS/Arch: linux/amd64 Experimental: false
注册阿里云账号密码https://cr.console.aliyun.com，登录后点击镜像加速器，复制加速器地址(https://bhzjdbej.mirror.aliyuncs.com)
[root@node2 ~]# mkdir -p /etc/docker
[root@node2 ~]# tee /etc/docker/daemon.json <<-'EOF'
> {
> "registry-mirrors": ["https://bhzjdbej.mirror.aliyuncs.com"]
> }
> EOF
{
"registry-mirrors": ["https://bhzjdbej.mirror.aliyuncs.com"]
}
[root@node2 ~]# systemctl daemon-reload
[root@node2 ~]# systemctl restart docker
[root@node2 ~]# systemctl | grep docker
[root@node2 ~]# docker info
关闭默认的frewalld防火墙，安装iptables-services
[root@node2 ~]# systemctl disable firewalld [root@node2 ~]# systemctl stop firewalld [root@node2 ~]# yum -y install iptables-services [root@node2 ~]# systemctl enable iptables [root@node2 ~]# systemctl start iptables
设置内核参数
[root@node2 ~]# vim /etc/sysctl.conf net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1 net.ipv4.ip_forward = 1 [root@node2 ~]# sysctl -p net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1 net.ipv4.ip_forward = 1
node2上修改Docker0的网络地址，与docker130不冲突
[root@node2 ~]# vim /usr/lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --bip=172.18.0.1/16
[root@node2 ~]# systemctl daemon-reload
[root@node2 ~]# systemctl restart docker
[root@node2 ~]# ip addr show docker0
4: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
link/ether 02:42:85:a4:9a:25 brd ff:ff:ff:ff:ff:ff
inet 172.18.0.1/16 brd 172.18.255.255 scope global docker0
valid_lft forever preferred_lft forever
node1：
[root@node1 ~]# vim /usr/lib/systemd/system/docker.service ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock [root@node1 ~]# systemctl daemon-reload [root@node1 ~]# systemctl restart docker [root@node1 ~]# ip route add 172.18.0.0/16 via 172.16.100.68 [root@node1 ~]# ip route show default via 192.168.204.2 dev ens37 proto dhcp metric 101 172.16.100.0/24 dev ens33 proto kernel scope link src 172.16.100.67 metric 100 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 172.18.0.0/16 via 172.16.100.68 dev ens33 192.168.204.0/24 dev ens37 proto kernel scope link src 192.168.204.135 metric 101
node2：
[root@node2 ~]# ip route add 172.17.0.0/16 via 172.16.100.67 [root@node2 ~]# ip route show default via 192.168.204.2 dev ens37 proto dhcp metric 101 172.16.100.0/24 dev ens33 proto kernel scope link src 172.16.100.68 metric 100 172.17.0.0/16 via 172.16.100.67 dev ens33 172.18.0.0/16 dev docker0 proto kernel scope link src 172.18.0.1 192.168.204.0/24 dev ens37 proto kernel scope link src 192.168.204.136 metric 101
node1：
[root@node1 ~]# docker run --rm=true -it java /bin/bash
root@c02064e68741:/# ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
211: eth0@if212: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 02:42:ac:11:00:02 brd ff:ff:ff:ff:ff:ff
inet 172.17.0.2/16 brd 172.17.255.255 scope global eth0
valid_lft forever preferred_lft forever
node2：
[root@node2 ~]# docker run --rm=true -it java /bin/bash
root@f8f3cd2ef86b:/# ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
5: eth0@if6: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 02:42:ac:12:00:02 brd ff:ff:ff:ff:ff:ff
inet 172.18.0.2/16 brd 172.18.255.255 scope global eth0
valid_lft forever preferred_lft forever
root@f8f3cd2ef86b:/# ping 172.17.0.1
PING 172.17.0.1 (172.17.0.1): 56 data bytes
64 bytes from 172.17.0.1: icmp_seq=0 ttl=63 time=1.402 ms
64 bytes from 172.17.0.1: icmp_seq=1 ttl=63 time=1.053 ms
^C--- 172.17.0.1 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max/stddev = 1.053/1.228/1.402/0.175 ms
root@f8f3cd2ef86b:/# ping 172.17.0.2 #ping对端node1上的容器不通是防火墙问题
PING 172.17.0.2 (172.17.0.2): 56 data bytes
92 bytes from 172.16.100.67: Dest Unreachable, Unknown Code: 10
92 bytes from 172.16.100.67: Dest Unreachable, Unknown Code: 10
92 bytes from 172.16.100.67: Dest Unreachable, Unknown Code: 10
^C--- 172.17.0.2 ping statistics ---
3 packets transmitted, 0 packets received, 100% packet loss
[root@node2 ~]# iptables -L FORWARD -n --line-number
Chain FORWARD (policy ACCEPT)
num target prot opt source destination
1 DOCKER-USER all -- 0.0.0.0/0 0.0.0.0/0
2 DOCKER-ISOLATION-STAGE-1 all -- 0.0.0.0/0 0.0.0.0/0
3 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED
4 DOCKER all -- 0.0.0.0/0 0.0.0.0/0
5 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0
6 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0
7 REJECT all -- 0.0.0.0/0 0.0.0.0/0 reject-with icmp-host-prohibited
[root@node2 ~]# iptables -D FORWARD 7
[root@node2 ~]# iptables -L FORWARD -n --line-number
Chain FORWARD (policy ACCEPT)
num target prot opt source destination
1 DOCKER-USER all -- 0.0.0.0/0 0.0.0.0/0
2 DOCKER-ISOLATION-STAGE-1 all -- 0.0.0.0/0 0.0.0.0/0
3 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED
4 DOCKER all -- 0.0.0.0/0 0.0.0.0/0
5 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0
6 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0
node1：
[root@node1 ~]# iptables -L FORWARD -n --line-number Chain FORWARD (policy ACCEPT) num target prot opt source destination 1 DOCKER-USER all -- 0.0.0.0/0 0.0.0.0/0 2 DOCKER-ISOLATION-STAGE-1 all -- 0.0.0.0/0 0.0.0.0/0 3 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED 4 DOCKER all -- 0.0.0.0/0 0.0.0.0/0 5 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 6 REJECT all -- 0.0.0.0/0 0.0.0.0/0 reject-with icmp-host-prohibited 7 DROP all -- 0.0.0.0/0 0.0.0.0/0 [root@node1 ~]# iptables -D FORWARD 6 [root@node1 ~]# iptables -L FORWARD -n --line-number Chain FORWARD (policy ACCEPT) num target prot opt source destination 1 DOCKER-USER all -- 0.0.0.0/0 0.0.0.0/0 2 DOCKER-ISOLATION-STAGE-1 all -- 0.0.0.0/0 0.0.0.0/0 3 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED 4 DOCKER all -- 0.0.0.0/0 0.0.0.0/0 5 ACCEPT all -- 0.0.0.0/0 0.0.0.0/0 6 DROP all -- 0.0.0.0/0 0.0.0.0/0
node2：
root@f8f3cd2ef86b:/# ping 172.17.0.2 PING 172.17.0.2 (172.17.0.2): 56 data bytes 64 bytes from 172.17.0.2: icmp_seq=0 ttl=62 time=2.085 ms 64 bytes from 172.17.0.2: icmp_seq=1 ttl=62 time=0.955 ms 64 bytes from 172.17.0.2: icmp_seq=2 ttl=62 time=1.060 ms ^C--- 172.17.0.2 ping statistics --- 3 packets transmitted, 3 packets received, 0% packet loss round-trip min/avg/max/stddev = 0.955/1.367/2.085/0.510 ms
node1：
root@c02064e68741:/# ping 172.18.0.2 PING 172.18.0.2 (172.18.0.2): 56 data bytes 64 bytes from 172.18.0.2: icmp_seq=0 ttl=62 time=3.560 ms 64 bytes from 172.18.0.2: icmp_seq=1 ttl=62 time=0.684 ms 64 bytes from 172.18.0.2: icmp_seq=2 ttl=62 time=1.008 ms ^C--- 172.18.0.2 ping statistics --- 3 packets transmitted, 3 packets received, 0% packet loss round-trip min/avg/max/stddev = 0.684/1.751/3.560/1.286 ms [root@node1 ~]# yum -y install wireshark [root@node1 ~]# tshark -i docker0 -f icmp Running as user "root" and group "root". This could be dangerous. Capturing on 'docker0' 1 0.000000000 172.17.0.2 -> 172.18.0.2 ICMP 98 Echo (ping) request id=0x0009, seq=16/4096, ttl=64 2 0.000674937 172.18.0.2 -> 172.17.0.2 ICMP 98 Echo (ping) reply id=0x0009, seq=16/4096, ttl=62 (request in 1) 3 1.002050708 172.17.0.2 -> 172.18.0.2 ICMP 98 Echo (ping) request id=0x0009, seq=17/4352, ttl=64 4 1.003267228 172.18.0.2 -> 172.17.0.2 ICMP 98 Echo (ping) reply id=0x0009, seq=17/4352, ttl=62 (request in 3) 4 packets captured [root@node1 ~]# tshark -i docker0 -f icmp Running as user "root" and group "root". This could be dangerous. Capturing on 'docker0' 1 0.000000000 172.17.0.2 -> 172.18.0.2 ICMP 98 Echo (ping) request id=0x0009, seq=139/35584, ttl=64 2 0.000629531 172.18.0.2 -> 172.17.0.2 ICMP 98 Echo (ping) reply id=0x0009, seq=139/35584, ttl=62 (request in 1) 3 1.002125513 172.17.0.2 -> 172.18.0.2 ICMP 98 Echo (ping) request id=0x0009, seq=140/35840, ttl=64 4 1.003037417 172.18.0.2 -> 172.17.0.2 ICMP 98 Echo (ping) reply id=0x0009, seq=140/35840, ttl=62 (request in 3) 4 packets captured
node2：
[root@node2 ~]# yum -y install wireshark [root@node2 ~]# tshark -i ens33 -f icmp Running as user "root" and group "root". This could be dangerous. Capturing on 'ens33' 1 0.000000000 172.16.100.67 -> 172.18.0.2 ICMP 98 Echo (ping) request id=0x0009, seq=334/19969, ttl=63 2 0.000112091 172.18.0.2 -> 172.16.100.67 ICMP 98 Echo (ping) reply id=0x0009, seq=334/19969, ttl=63 (request in 1) 3 0.918191754 172.16.100.67 -> 172.18.0.2 ICMP 98 Echo (ping) request id=0x0009, seq=335/20225, ttl=63 4 0.918338370 172.18.0.2 -> 172.16.100.67 ICMP 98 Echo (ping) reply id=0x0009, seq=335/20225, ttl=63 (request in 3) 4 packets captured [root@node2 ~]# tshark -i docker0 -f icmp Running as user "root" and group "root". This could be dangerous. Capturing on 'docker0' 1 0.000000000 172.16.100.67 -> 172.18.0.2 ICMP 98 Echo (ping) request id=0x0009, seq=352/24577, ttl=62 2 0.000067745 172.18.0.2 -> 172.16.100.67 ICMP 98 Echo (ping) reply id=0x0009, seq=352/24577, ttl=64 (request in 1) 3 0.919349700 172.16.100.67 -> 172.18.0.2 ICMP 98 Echo (ping) request id=0x0009, seq=353/24833, ttl=62 4 0.919480456 172.18.0.2 -> 172.16.100.67 ICMP 98 Echo (ping) reply id=0x0009, seq=353/24833, ttl=64 (request in 3) 4 packets captured
数据包流程：
ping request: c1 --> node1 docker0 --> node1 ens33 --> node2 ens33 --> node2 docker0 --> c2
ping reply: c2 --> node2 docker0 --> node2 ens33 --> node1 ens33 --> node1 docker0 --> c1
node1：
[root@node1 ~]# tshark -i docker0 -N m -e ip.src -e ip.dst -e eth.src -e eth.dst -T fields -E separator=, -f "net 172.18.0.0/16" -f icmp Running as user "root" and group "root". This could be dangerous. Capturing on 'docker0' 172.17.0.2,172.18.0.2,02:42:ac:11:00:02,02:42:1c:bd:fc:1e 172.18.0.2,172.17.0.2,02:42:1c:bd:fc:1e,02:42:ac:11:00:02 172.17.0.2,172.18.0.2,02:42:ac:11:00:02,02:42:1c:bd:fc:1e 172.18.0.2,172.17.0.2,02:42:1c:bd:fc:1e,02:42:ac:11:00:02 4 packets captured [root@node1 ~]# tshark -i ens33 -N m -e ip.src -e ip.dst -e eth.src -e eth.dst -T fields -E separator=, -f "net 172.18.0.0/16" -f icmp #mac地址发生改变 Running as user "root" and group "root". This could be dangerous. Capturing on 'ens33' 172.16.100.67,172.18.0.2,00:0c:29:67:e5:70,00:0c:29:70:86:6c 172.18.0.2,172.16.100.67,00:0c:29:70:86:6c,00:0c:29:67:e5:70 172.16.100.67,172.18.0.2,00:0c:29:67:e5:70,00:0c:29:70:86:6c 172.18.0.2,172.16.100.67,00:0c:29:70:86:6c,00:0c:29:67:e5:70 4 packets captured
node2：
[root@node2 ~]# tshark -i ens33 -N m -e ip.src -e ip.dst -e eth.src -e eth.dst -T fields -E separator=, -f "net 172.17.0.0/16" -f icmp Running as user "root" and group "root". This could be dangerous. Capturing on 'ens33' 172.16.100.67,172.18.0.2,00:0c:29:67:e5:70,00:0c:29:70:86:6c 172.18.0.2,172.16.100.67,00:0c:29:70:86:6c,00:0c:29:67:e5:70 172.16.100.67,172.18.0.2,00:0c:29:67:e5:70,00:0c:29:70:86:6c 172.18.0.2,172.16.100.67,00:0c:29:70:86:6c,00:0c:29:67:e5:70 4 packets captured [root@node2 ~]# tshark -i docker0 -N m -e ip.src -e ip.dst -e eth.src -e eth.dst -T fields -E separator=, -f "net 172.17.0.0/16" -f icmp #mac地址发生改变 Running as user "root" and group "root". This could be dangerous. Capturing on 'docker0' 172.16.100.67,172.18.0.2,02:42:85:a4:9a:25,02:42:ac:12:00:02 172.18.0.2,172.16.100.67,02:42:ac:12:00:02,02:42:85:a4:9a:25 172.16.100.67,172.18.0.2,02:42:85:a4:9a:25,02:42:ac:12:00:02 172.18.0.2,172.16.100.67,02:42:ac:12:00:02,02:42:85:a4:9a:25 4 packets captured
Docker网络方案
双网卡独立大二层交换（linux bridge)
Overlay网络
基于ovs的Overlay网络
neutron网络
官方的Libnetwork
Socketplane被docker公司收购，成为其官方网络的起源
开发者不想操作是否是 VLANs, VXLANs, Tunnels 或者是 TEPs. 对于架构人们最关心的是性能和可靠性。而SocketPlane 在 socket 层面提供了一个网络的抽象层，通过可管理的方式去解决各种网络问题。
主要特性：
•Open vSwitch 集成
•用于 Docker 的零配置多主机网络
•Docker/SocketPlane 集群的优雅增长
•支持多网络
•分布式 IP 地址管理 (IPAM)
1.将libnetwork集成到Docker Engine
2.在Docker CLI中使用新的network命令
3.撰写『-net』 参数的文档，以告知用户如何使用它来为容器指定网络
4.在network和endpoint中支持添加『label』
5.研发新的『bridge』 插件，以替换Docker目前的实现
6.研发『分布式bridge』 插件，以支持跨容器网络
目前看到有一些计划是打算将OVS项目关联到Docker上来，从Linux Kernel 3.3开始， OVS项目就是内核的一部分。当我听到这个的时候我觉得是不是脑袋让驴踢了。首先声明我并不是反对使用OVS，实际上，它是一个非常不错的网络工具套件。它的设置比较复杂，对于新手来说有一个陡峭的学习曲线，但是一旦学会， OVS就可以帮你事半功倍。关于这个话题我听到的一个讨论是：“如果OVS工作在Docker上，那么工作一切都变得很美好”。让我告诉你，亲们：如果让我花费大量时间学习它，最后的结果只能是：“还好，可以用”。我并不想说的那么愤世嫉俗，实际情况是在某些常用环境下OVS会崩溃。因此，使用OVS只是一种疯狂的想法罢了。
http://containertutorials.com/network/ovs_docker.html
https://github.com/openvswitch/ovs/blob/master/utilities/ovs-docker
Currently libnetwork is nothing more than an attempt to modularize the Docker platform's networking subsystem by moving it into libnetwork as a library.
浙公网安备 33010602011771号