Flannel网络部署

一、Flannel网络部署

为Flannel生成证书

[root@linux-node1 ssl]# vim flanneld-csr.json
{
  "CN": "flanneld",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

生成证书

[root@linux-node1 ssl]# cfssl gencert -ca=/opt/kubernetes/ssl/ca.pem \
    -ca-key=/opt/kubernetes/ssl/ca-key.pem \
    -config=/opt/kubernetes/ssl/ca-config.json \
    -profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld
2018/05/31 16:19:46 [INFO] generate received request
2018/05/31 16:19:46 [INFO] received CSR
2018/05/31 16:19:46 [INFO] generating key: rsa-2048
2018/05/31 16:19:46 [INFO] encoded CSR
2018/05/31 16:19:46 [INFO] signed certificate with serial number 655739903410343916627289078414999108781865584022
2018/05/31 16:19:46 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

[root@linux-node1 ssl]# ll flanneld*
-rw-r--r-- 1 root root  997 May 31 16:19 flanneld.csr
-rw-r--r-- 1 root root  221 May 31 16:18 flanneld-csr.json
-rw------- 1 root root 1679 May 31 16:19 flanneld-key.pem
-rw-r--r-- 1 root root 1391 May 31 16:19 flanneld.pem

分发证书

[root@linux-node1 ssl]# scp flanneld*.pem 192.168.56.12:/opt/kubernetes/ssl/
[root@linux-node1 ssl]# scp flanneld*.pem 192.168.56.13:/opt/kubernetes/ssl/

下载Flannel软件包

# wget https://github.com/coreos/flannel/releases/download/v0.10.0/flannel-v0.10.0-linux-amd64.tar.gz

[root@linux-node1 ssl]# cd /usr/local/src/
[root@linux-node1 src]# pwd
/usr/local/src
[root@linux-node1 src]# tar zxf flannel-v0.10.0-linux-amd64.tar.gz
[root@linux-node1 src]# cp flanneld mk-docker-opts.sh /opt/kubernetes/bin/
#复制到linux-node2、linux-node3节点上
[root@linux-node1 src]# scp flanneld mk-docker-opts.sh 192.168.56.12:/opt/kubernetes/bin/
[root@linux-node1 src]# scp flanneld mk-docker-opts.sh 192.168.56.13:/opt/kubernetes/bin/

复制对应脚本到/opt/kubernetes/bin目录下。
[root@linux-node1 src]# cd /usr/local/src/kubernetes/cluster/centos/node/bin/
[root@linux-node1 bin]# ll
total 8
-rwxr-xr-x 1 root root 2590 Apr 12 23:24 mk-docker-opts.sh
-rwxr-xr-x 1 root root  850 Apr 12 23:24 remove-docker0.sh
[root@linux-node1 bin]# cp remove-docker0.sh /opt/kubernetes/bin/
[root@linux-node1 bin]# scp remove-docker0.sh 192.168.56.12:/opt/kubernetes/bin/
[root@linux-node1 bin]# scp remove-docker0.sh 192.168.56.13:/opt/kubernetes/bin/
   

配置Flannel

[root@linux-node1 bin]# vim /opt/kubernetes/cfg/flannel
FLANNEL_ETCD="-etcd-endpoints=https://192.168.56.11:2379,https://192.168.56.12:2379,https://192.168.56.13:2379"
FLANNEL_ETCD_KEY="-etcd-prefix=/kubernetes/network"
FLANNEL_ETCD_CAFILE="--etcd-cafile=/opt/kubernetes/ssl/ca.pem"
FLANNEL_ETCD_CERTFILE="--etcd-certfile=/opt/kubernetes/ssl/flanneld.pem"
FLANNEL_ETCD_KEYFILE="--etcd-keyfile=/opt/kubernetes/ssl/flanneld-key.pem"

复制配置到其它节点上
[root@linux-node1 bin]# scp /opt/kubernetes/cfg/flannel 192.168.56.12:/opt/kubernetes/cfg/
[root@linux-node1 bin]# scp /opt/kubernetes/cfg/flannel 192.168.56.13:/opt/kubernetes/cfg/

设置Flannel系统服务

[root@linux-node1 bin]# vim /usr/lib/systemd/system/flannel.service
[root@linux-node1 bin]# cat /usr/lib/systemd/system/flannel.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
Before=docker.service

[Service]
EnvironmentFile=-/opt/kubernetes/cfg/flannel
ExecStartPre=/opt/kubernetes/bin/remove-docker0.sh
ExecStart=/opt/kubernetes/bin/flanneld ${FLANNEL_ETCD} ${FLANNEL_ETCD_KEY} ${FLANNEL_ETCD_CAFILE} ${FLANNEL_ETCD_CERTFILE} ${FLANNEL_ETCD_KEYFILE}
ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -d /run/flannel/docker

Type=notify

[Install]
WantedBy=multi-user.target
RequiredBy=docker.service

复制系统服务脚本到其它节点上
[root@linux-node1 bin]# scp /usr/lib/systemd/system/flannel.service 192.168.56.12:/usr/lib/systemd/system/
[root@linux-node1 bin]# scp /usr/lib/systemd/system/flannel.service 192.168.56.13:/usr/lib/systemd/system/

二、Flannel CNI集成

下载CNI插件

#下载CNI插件
https://github.com/containernetworking/plugins/releases
wget https://github.com/containernetworking/plugins/releases/download/v0.7.1/cni-plugins-amd64-v0.7.1.tgz
[root@linux-node1 bin]# mkdir /opt/kubernetes/bin/cni
[root@linux-node2 ssl]# mkdir /opt/kubernetes/bin/cni
[root@linux-node3 ssl]# mkdir /opt/kubernetes/bin/cni
[root@linux
-node1 bin]# cd /usr/local/src/ [root@linux-node1 src]# tar zxf cni-plugins-amd64-v0.7.1.tgz -C /opt/kubernetes/bin/cni [root@linux-node1 src]# scp -r /opt/kubernetes/bin/cni/* 192.168.56.12:/opt/kubernetes/bin/cni/ root@linux-node1 src]# scp -r /opt/kubernetes/bin/cni/* 192.168.56.13:/opt/kubernetes/bin/cni/

创建Etcd的key

root@linux-node1 src]# /opt/kubernetes/bin/etcdctl --ca-file /opt/kubernetes/ssl/ca.pem --cert-file /opt/kubernetes/ssl/flanneld.pem --key-file /opt/kubernetes/ssl/flanneld-key.pem \
       --no-sync -C https://192.168.56.11:2379,https://192.168.56.12:2379,https://192.168.56.13:2379 \
 mk /kubernetes/network/config '{ "Network": "10.2.0.0/16", "Backend": { "Type": "vxlan", "VNI": 1 }}' >/dev/null 2>&1

启动flannel

######################linux-node1启动#############################
[root@linux-node1 src]# systemctl daemon-reload
[root@linux-node1 src]# systemctl enable flannel
Created symlink from /etc/systemd/system/multi-user.target.wants/flannel.service to /usr/lib/systemd/system/flannel.service.
Created symlink from /etc/systemd/system/docker.service.requires/flannel.service to /usr/lib/systemd/system/flannel.service.
[root@linux-node1 src]# chmod +x /opt/kubernetes/bin/*
[root@linux-node1 src]# systemctl start flannel

######################linux-node2启动#############################
[root@linux-node2 src]# systemctl daemon-reload
[root@linux-node2 src]# systemctl enable flannel
Created symlink from /etc/systemd/system/multi-user.target.wants/flannel.service to /usr/lib/systemd/system/flannel.service.
Created symlink from /etc/systemd/system/docker.service.requires/flannel.service to /usr/lib/systemd/system/flannel.service.
[root@linux-node2 src]# chmod +x /opt/kubernetes/bin/*
[root@linux-node2 src]# systemctl start flannel

######################linux-node3启动#############################
[root@linux-node3 ssl]# systemctl daemon-reload
[root@linux-node3 ssl]# systemctl enable flannel
Created symlink from /etc/systemd/system/multi-user.target.wants/flannel.service to /usr/lib/systemd/system/flannel.service.
Created symlink from /etc/systemd/system/docker.service.requires/flannel.service to /usr/lib/systemd/system/flannel.service.
[root@linux-node3 ssl]# chmod +x /opt/kubernetes/bin/*
[root@linux-node3 ssl]# systemctl start flannel

查看服务状态

[root@linux-node1 src]# systemctl status flannel
[root@linux-node2 src]# systemctl status flannel
[root@linux-node3 ssl]# systemctl status flannel
[root@linux-node1 src]# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.56.11  netmask 255.255.255.0  broadcast 192.168.56.255
        inet6 fe80::20c:29ff:fea0:10ca  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:a0:10:ca  txqueuelen 1000  (Ethernet)
        RX packets 386026  bytes 62831393 (59.9 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 768980  bytes 1204891659 (1.1 GiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet 10.2.9.0  netmask 255.255.255.255  broadcast 0.0.0.0
        inet6 fe80::2404:bbff:fee0:41a6  prefixlen 64  scopeid 0x20<link>
        ether 26:04:bb:e0:41:a6  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 8 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1  (Local Loopback)
        RX packets 232562  bytes 59255512 (56.5 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 232562  bytes 59255512 (56.5 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
linux-node1上查看网卡ifconfig
[root@linux-node2 src]# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.56.12  netmask 255.255.255.0  broadcast 192.168.56.255
        inet6 fe80::20c:29ff:fef9:ca25  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:f9:ca:25  txqueuelen 1000  (Ethernet)
        RX packets 669708  bytes 372005370 (354.7 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 470811  bytes 63787801 (60.8 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet 10.2.13.0  netmask 255.255.255.255  broadcast 0.0.0.0
        inet6 fe80::4031:7ff:fe2d:9c2f  prefixlen 64  scopeid 0x20<link>
        ether 42:31:07:2d:9c:2f  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 8 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1  (Local Loopback)
        RX packets 960  bytes 72576 (70.8 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 960  bytes 72576 (70.8 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
linux-node2上查看网卡ifconfig
[root@linux-node3 ssl]# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.56.13  netmask 255.255.255.0  broadcast 192.168.56.255
        inet6 fe80::20c:29ff:feca:800d  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:ca:80:0d  txqueuelen 1000  (Ethernet)
        RX packets 548452  bytes 359854149 (343.1 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 353542  bytes 51778767 (49.3 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet 10.2.17.0  netmask 255.255.255.255  broadcast 0.0.0.0
        inet6 fe80::c86c:3ff:fe2e:d4ca  prefixlen 64  scopeid 0x20<link>
        ether ca:6c:03:2e:d4:ca  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 8 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1  (Local Loopback)
        RX packets 804  bytes 48598 (47.4 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 804  bytes 48598 (47.4 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
linux-node3上查看网卡ifconfig
[root@linux-node3 ssl]# cat /run/flannel/docker
DOCKER_OPT_BIP="--bip=10.2.17.1/24"
DOCKER_OPT_IPMASQ="--ip-masq=true"
DOCKER_OPT_MTU="--mtu=1450"
DOCKER_OPTS=" --bip=10.2.17.1/24 --ip-masq=true --mtu=1450"

三、配置Docker使用Flannel

root@linux-node1 ~]# vim /usr/lib/systemd/system/docker.service
[Unit] #在Unit下面修改After和增加Requires
After=network-online.target firewalld.service flannel.service
Wants=network-online.target
Requires=flannel.service

[Service] #增加EnvironmentFile=-/run/flannel/docker
Type=notify
EnvironmentFile=-/run/flannel/docker
ExecStart=/usr/bin/dockerd $DOCKER_OPTS

将配置复制到另外两个节点

[root@linux-node1 src]# scp /usr/lib/systemd/system/docker.service 192.168.56.12:/usr/lib/systemd/system/                                                                                         
[root@linux-node1 src]# scp /usr/lib/systemd/system/docker.service 192.168.56.13:/usr/lib/systemd/system/

重启Docker

#####################linux-node1重启Docker服务########################
[root@linux-node1 src]# systemctl daemon-reload
[root@linux-node1 src]# systemctl restart docker

#####################linux-node3重启Docker服务########################
[root@linux-node2 src]# systemctl daemon-reload
[root@linux-node2 src]# systemctl restart docker

#####################linux-node3重启Docker服务########################
[root@linux-node3 ssl]# systemctl daemon-reload
[root@linux-node3 ssl]# systemctl restart docker

 

[root@linux-node1 src]# ifconfig
docker0: flags=4099<UP,BROADCAST,MULTICAST>  mtu 1500
        inet 10.2.9.1  netmask 255.255.255.0  broadcast 10.2.9.255
        ether 02:42:18:d5:e0:d1  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.56.11  netmask 255.255.255.0  broadcast 192.168.56.255
        inet6 fe80::20c:29ff:fea0:10ca  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:a0:10:ca  txqueuelen 1000  (Ethernet)
        RX packets 426666  bytes 68740555 (65.5 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 809174  bytes 1211522272 (1.1 GiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet 10.2.9.0  netmask 255.255.255.255  broadcast 0.0.0.0
        inet6 fe80::2404:bbff:fee0:41a6  prefixlen 64  scopeid 0x20<link>
        ether 26:04:bb:e0:41:a6  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 8 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1  (Local Loopback)
        RX packets 254544  bytes 65389446 (62.3 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 254544  bytes 65389446 (62.3 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
linux-node1上查看网卡ifconfig
[root@linux-node2 src]# ifconfig
docker0: flags=4099<UP,BROADCAST,MULTICAST>  mtu 1500
        inet 10.2.13.1  netmask 255.255.255.0  broadcast 10.2.13.255
        ether 02:42:6a:cf:66:9f  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.56.12  netmask 255.255.255.0  broadcast 192.168.56.255
        inet6 fe80::20c:29ff:fef9:ca25  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:f9:ca:25  txqueuelen 1000  (Ethernet)
        RX packets 724138  bytes 379110250 (361.5 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 524129  bytes 71071850 (67.7 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet 10.2.13.0  netmask 255.255.255.255  broadcast 0.0.0.0
        inet6 fe80::4031:7ff:fe2d:9c2f  prefixlen 64  scopeid 0x20<link>
        ether 42:31:07:2d:9c:2f  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 8 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1  (Local Loopback)
        RX packets 1264  bytes 88416 (86.3 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 1264  bytes 88416 (86.3 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
linux-node2上查看网卡ifconfig
[root@linux-node3 ssl]# ifconfig
docker0: flags=4099<UP,BROADCAST,MULTICAST>  mtu 1500
        inet 10.2.17.1  netmask 255.255.255.0  broadcast 10.2.17.255
        ether 02:42:e3:78:53:2d  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.56.13  netmask 255.255.255.0  broadcast 192.168.56.255
        inet6 fe80::20c:29ff:feca:800d  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:ca:80:0d  txqueuelen 1000  (Ethernet)
        RX packets 584443  bytes 365047002 (348.1 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 390601  bytes 57109546 (54.4 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet 10.2.17.0  netmask 255.255.255.255  broadcast 0.0.0.0
        inet6 fe80::c86c:3ff:fe2e:d4ca  prefixlen 64  scopeid 0x20<link>
        ether ca:6c:03:2e:d4:ca  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 8 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1  (Local Loopback)
        RX packets 880  bytes 52582 (51.3 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 880  bytes 52582 (51.3 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
linux-node3上查看网卡ifconfig

 

posted @ 2018-05-31 21:48  hwlong  阅读(577)  评论(0编辑  收藏