部署kubernetes1.8.4+contiv高可用集群

原理和架构图参考上一篇,这里只记录操作步骤。由于东西较多,篇幅也会较长。

etcd version: 3.2.11
kube version: 1.8.4
contiv version: 1.1.7
docker version: 17.03.2-ce
OS version: debian stretch

三个ETCD节点(contiv插件也要使用etcd,这里每个节点复用跑2个etcd实例)

192.168.5.84    etcd0,contiv0
192.168.5.85    etcd1,contiv1
192.168.2.77    etcd2,contiv2

两个lvs节点,这里lvs代理了三个服务,分别是apiserver、contiv的netmaster、以及由于contiv不支持配置多个etcd所以代理三个etcd实例提供一个vip出来给contiv服务

192.168.2.56     master
192.168.2.57     backup

4个k8s节点(3个master,1个node)

192.168.5.62     master01
192.168.5.63     master02
192.168.5.107    master03
192.168.5.68     node

1、部署ETCD,由于这几个节点系统版本较低,所以没有使用systemd

a、部署k8s使用的etcd集群,直接使用etcd二进制文件启动即可,启动脚本如下:

# cat etcd-start.sh
#获取IP
localip=`ifconfig em2|grep -w inet| awk '{print $2}'|awk -F: '{print $2}'`
pubip=0.0.0.0
#启动服务
etcd --name etcd0 -data-dir /var/lib/etcd \
  --initial-advertise-peer-urls http://${localip}:2380 \
  --listen-peer-urls http://${localip}:2380 \
  --listen-client-urls http://${pubip}:2379 \
  --advertise-client-urls http://${pubip}:2379 \
  --initial-cluster-token my-etcd-token \
  --initial-cluster etcd0=http://192.168.5.84:2380,etcd1=http://192.168.5.85:2380,etcd2=http://192.168.2.77:2380 \
  --initial-cluster-state new >> /var/log/etcd.log 2>&1 &
# cat etcd-start.sh
#获取IP
localip=`ifconfig em2|grep -w inet| awk '{print $2}'|awk -F: '{print $2}'`
pubip=0.0.0.0
#启动服务
etcd --name etcd1 -data-dir /var/lib/etcd \
  --initial-advertise-peer-urls http://${localip}:2380 \
  --listen-peer-urls http://${localip}:2380 \
  --listen-client-urls http://${pubip}:2379 \
  --advertise-client-urls http://${pubip}:2379 \
  --initial-cluster-token my-etcd-token \
  --initial-cluster etcd0=http://192.168.5.84:2380,etcd1=http://192.168.5.85:2380,etcd2=http://192.168.2.77:2380 \
  --initial-cluster-state new >> /var/log/etcd.log 2>&1 &
# cat etcd-start.sh
#获取IP
localip=`ifconfig bond0|grep -w inet| awk '{print $2}'|awk -F: '{print $2}'`
pubip=0.0.0.0
#启动服务
etcd --name etcd2 -data-dir /var/lib/etcd \
  --initial-advertise-peer-urls http://${localip}:2380 \
  --listen-peer-urls http://${localip}:2380 \
  --listen-client-urls http://${pubip}:2379 \
  --advertise-client-urls http://${pubip}:2379 \
  --initial-cluster-token my-etcd-token \
  --initial-cluster etcd0=http://192.168.5.84:2380,etcd1=http://192.168.5.85:2380,etcd2=http://192.168.2.77:2380 \
  --initial-cluster-state new >> /var/log/etcd.log 2>&1 &

b、部署contiv使用的etcd:

# cat etcd-2-start.sh
#!/bin/bash
#获取IP
localip=`ifconfig em2|grep -w inet| awk '{print $2}'|awk -F: '{print $2}'`
pubip=0.0.0.0
#启动服务
etcd --name contiv0 -data-dir /var/etcd/contiv-data \
  --initial-advertise-peer-urls http://${localip}:6667 \
  --listen-peer-urls http://${localip}:6667 \
  --listen-client-urls http://${pubip}:6666 \
  --advertise-client-urls http://${pubip}:6666 \
  --initial-cluster-token contiv-etcd-token \
  --initial-cluster contiv0=http://192.168.5.84:6667,contiv1=http://192.168.5.85:6667,contiv2=http://192.168.2.77:6667 \
  --initial-cluster-state new >> /var/log/etcd-contiv.log 2>&1 &
# cat etcd-2-start.sh
#获取IP
localip=`ifconfig em2|grep -w inet| awk '{print $2}'|awk -F: '{print $2}'`
pubip='0.0.0.0'
#启动服务
etcd --name contiv1 -data-dir /var/etcd/contiv-data \
  --initial-advertise-peer-urls http://${localip}:6667 \
  --listen-peer-urls http://${localip}:6667 \
  --listen-client-urls http://${pubip}:6666 \
  --advertise-client-urls http://${pubip}:6666 \
  --initial-cluster-token contiv-etcd-token \
  --initial-cluster contiv0=http://192.168.5.84:6667,contiv1=http://192.168.5.85:6667,contiv2=http://192.168.2.77:6667 \
  --initial-cluster-state new >> /var/log/etcd-contiv.log 2>&1 &
# cat etcd-2-start.sh
#获取IP
localip=`ifconfig bond0|grep -w inet| awk '{print $2}'|awk -F: '{print $2}'`
pubip=0.0.0.0
#启动服务
etcd --name contiv2 -data-dir /var/etcd/contiv-data \
  --initial-advertise-peer-urls http://${localip}:6667 \
  --listen-peer-urls http://${localip}:6667 \
  --listen-client-urls http://${pubip}:6666 \
  --advertise-client-urls http://${pubip}:6666 \
  --initial-cluster-token contiv-etcd-token \
  --initial-cluster contiv0=http://192.168.5.84:6667,contiv1=http://192.168.5.85:6667,contiv2=http://192.168.2.77:6667 \
  --initial-cluster-state new >> /var/log/etcd-contiv.log 2>&1 &

c、启动服务,直接执行脚本即可。

# bash etcd-start.sh
# bash etcd-2-start.sh

d、验证集群状态

# etcdctl member list
4e2d8913b0f6d79d, started, etcd2, http://192.168.2.77:2380, http://0.0.0.0:2379
7b72fa2df0544e1b, started, etcd0, http://192.168.5.84:2380, http://0.0.0.0:2379
930f118a7f33cf1c, started, etcd1, http://192.168.5.85:2380, http://0.0.0.0:2379
# etcdctl --endpoints=http://192.168.6.17:6666 member list
21868a2f15be0a01, started, contiv0, http://192.168.5.84:6667, http://0.0.0.0:6666
63df25ae8bd96b52, started, contiv1, http://192.168.5.85:6667, http://0.0.0.0:6666
cf59e48c1866f41d, started, contiv2, http://192.168.2.77:6667, http://0.0.0.0:6666

e、配置lvs代理contiv的etcd,vip为192.168.6.17。这里顺便把其他两个服务的代理配置全部贴上来,实际上仅仅是多了两段配置而已,apiserver的vip为192.168.6.16

# vim vi_bgp_VI1_yizhuang.inc
vrrp_instance VII_1 {
    virtual_router_id 102
    interface eth0
    include /etc/keepalived/state_VI1.conf
    preempt_delay 120
    garp_master_delay 0
    garp_master_refresh 5
    lvs_sync_daemon_interface eth0
    authentication {
        auth_type PASS
        auth_pass opsdk
    }

    virtual_ipaddress {
        #k8s-apiserver
        192.168.6.16

        #etcd
        192.168.6.17
    }
}

这里单独使用了一个state.conf配置文件来区分主备角色,也就是master和backup节点的配置仅有这一部分不同,其他配置可以直接复制过去。

# vim /etc/keepalived/state_VI1.conf
#uy-s-07
     state MASTER
     priority 150
#uy-s-45
#    state BACKUP
#    priority 100
# vim /etc/keepalived/k8s.conf
virtual_server 192.168.6.16 6443 {
    lb_algo rr
    lb_kind DR
    persistence_timeout 0
    delay_loop 20
    protocol TCP

    real_server 192.168.5.62 6443 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }

    real_server 192.168.5.63 6443 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }

    real_server 192.168.5.107 6443 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }
}

virtual_server 192.168.6.17 6666 {
    lb_algo rr
    lb_kind DR
    persistence_timeout 0
    delay_loop 20
    protocol TCP

    real_server 192.168.5.84 6666 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }

    real_server 192.168.5.85 6666 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }

    real_server 192.168.2.77 6666 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }
}

virtual_server 192.168.6.16 9999 {
    lb_algo rr
    lb_kind DR
    persistence_timeout 0
    delay_loop 20
    protocol TCP

    real_server 192.168.5.62 9999 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }

    real_server 192.168.5.63 9999 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }

    real_server 192.168.5.107 9999 {
        weight 10
        TCP_CHECK {
            connect_timeout 10
        }
    }
}

为etcd的各real-server设置vip:

# vim /etc/network/interfaces
auto lo:17
iface lo:17 inet static
address 192.168.6.17
netmask 255.255.255.255

# ifconfig lo:17 192.168.6.17 netmask 255.255.255.255 up

为apiserver的各real-server设置vip:

# vim /etc/network/interfaces
auto lo:16
iface lo:16 inet static
address 192.168.6.16
netmask 255.255.255.255

# ifconfig lo:16 192.168.6.16 netmask 255.255.255.255 up

为所有real-server设置内核参数:

# vim /etc/sysctl.conf
net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
net.ipv4.ip_forward = 1
net.netfilter.nf_conntrack_max = 2048000

启动服务,查看服务状态:

# /etc/init.d/keepalived start
# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=1048576)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.6.16:6443 rr
  -> 192.168.5.62:6443            Route   10     1          0
  -> 192.168.5.63:6443            Route   10     0          0
  -> 192.168.5.107:6443           Route   10     4          0
TCP  192.168.6.16:9999 rr
  -> 192.168.5.62:9999            Route   10     0          0
  -> 192.168.5.63:9999            Route   10     0          0
  -> 192.168.5.107:9999           Route   10     0          0
TCP  192.168.6.17:6666 rr
  -> 192.168.2.77:6666            Route   10     24         14
  -> 192.168.5.84:6666            Route   10     22         13
  -> 192.168.5.85:6666            Route   10     18         14

2、部署k8s,由于上篇已经说了详细步骤,这里会略过一些内容

a、安装kubeadm,kubectl,kubelet,由于目前仓库已经更新到最新版本1.9了,所以这里如果要安装低版本需要手动指定版本号

# aptitude install -y kubeadm=1.8.4-00 kubectl=1.8.4-00 kubelet=1.8.4-00

b、使用kubeadm初始化第一个master节点。由于使用的是contiv插件,所以这里可以不设置网络参数podSubnet。因为contiv没有使用controller-manager的subnet-allocating特性,另外,weave也没有使用这个特性。

# cat kubeadm-config.yml
apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
  advertiseAddress: "192.168.5.62"
etcd:
  endpoints:
  - "http://192.168.5.84:2379"
  - "http://192.168.5.85:2379"
  - "http://192.168.2.77:2379"
kubernetesVersion: "v1.8.4"
apiServerCertSANs:
- uy06-04
- uy06-05
- uy08-10
- uy08-11
- 192.168.6.16
- 192.168.6.17
- 127.0.0.1
- 192.168.5.62
- 192.168.5.63
- 192.168.5.107
- 192.168.5.108
- 30.0.0.1
- 10.244.0.1
- 10.96.0.1
- kubernetes
- kubernetes.default
- kubernetes.default.svc
- kubernetes.default.svc.cluster
- kubernetes.default.svc.cluster.local
tokenTTL: 0s
networking:
  podSubnet: 30.0.0.0/10

执行初始化:

# kubeadm init --config=kubeadm-config.yml
[kubeadm] WARNING: kubeadm is in beta, please do not use it for production clusters.
[init] Using Kubernetes version: v1.8.4
[init] Using Authorization modes: [Node RBAC]
[preflight] Running pre-flight checks
[preflight] WARNING: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
[preflight] Starting the kubelet service
[kubeadm] WARNING: starting in 1.8, tokens expire after 24 hours by default (if you require a non-expiring token use --token-ttl 0)
[certificates] Generated ca certificate and key.
[certificates] Generated apiserver certificate and key.
[certificates] apiserver serving cert is signed for DNS names [uy06-04 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local uy06-04 uy06-05 uy08-10 uy08-11 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 192.168.5.62 192.168.6.16 192.168.6.17 127.0.0.1 192.168.5.62 192.168.5.63 192.168.5.107 192.168.5.108 30.0.0.1 10.244.0.1 10.96.0.1]
[certificates] Generated apiserver-kubelet-client certificate and key.
[certificates] Generated sa key and public key.
[certificates] Generated front-proxy-ca certificate and key.
[certificates] Generated front-proxy-client certificate and key.
[certificates] Valid certificates and keys now exist in "/etc/kubernetes/pki"
[kubeconfig] Wrote KubeConfig file to disk: "admin.conf"
[kubeconfig] Wrote KubeConfig file to disk: "kubelet.conf"
[kubeconfig] Wrote KubeConfig file to disk: "controller-manager.conf"
[kubeconfig] Wrote KubeConfig file to disk: "scheduler.conf"
[controlplane] Wrote Static Pod manifest for component kube-apiserver to "/etc/kubernetes/manifests/kube-apiserver.yaml"
[controlplane] Wrote Static Pod manifest for component kube-controller-manager to "/etc/kubernetes/manifests/kube-controller-manager.yaml"
[controlplane] Wrote Static Pod manifest for component kube-scheduler to "/etc/kubernetes/manifests/kube-scheduler.yaml"
[init] Waiting for the kubelet to boot up the control plane as Static Pods from directory "/etc/kubernetes/manifests"
[init] This often takes around a minute; or longer if the control plane images have to be pulled.
[apiclient] All control plane components are healthy after 28.502953 seconds
[uploadconfig] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[markmaster] Will mark node uy06-04 as master by adding a label and a taint
[markmaster] Master uy06-04 tainted and labelled with key/value: node-role.kubernetes.io/master=""
[bootstraptoken] Using token: 0c8921.578cf94fe0721e01
[bootstraptoken] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstraptoken] Configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstraptoken] Configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstraptoken] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[addons] Applied essential addon: kube-dns
[addons] Applied essential addon: kube-proxy

Your Kubernetes master has initialized successfully!

To start using your cluster, you need to run (as a regular user):

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  http://kubernetes.io/docs/admin/addons/

You can now join any number of machines by running the following on each node
as root:

  kubeadm join --token 0c8921.578cf94fe0721e01 192.168.5.62:6443 --discovery-token-ca-cert-hash sha256:58cf1826d49e44fb6ff1590ddb077dd4e530fe58e13c1502ec07ce41ba6cc39e

c、验证通过证书是否能访问到API(这里每个节点都务必验证一下,证书问题会导致各种其它问题)

# cd /etc/kubernetes/pki/
# curl --cacert ca.crt --cert apiserver-kubelet-client.crt --key apiserver-kubelet-client.key https://192.168.5.62:6443

d、让master节点参与调度

# kubectl taint nodes --all node-role.kubernetes.io/master-

e、安装contiv

下载安装包并解压

# curl -L -O https://github.com/contiv/install/releases/download/1.1.7/contiv-1.1.7.tgz
# tar xvf contiv-1.1.7.tgz

修改yaml文件

# cd contiv-1.1.7/
# vim install/k8s/k8s1.6/contiv.yaml

1、修改ca路径,并将k8s的ca复制到该路径下
    "K8S_CA": "/var/contiv/ca.crt"
2、修改netmaster的部署类型,把ReplicaSet改为DaemonSet(实现netmaster的高可用),这里使用了nodeSeletor,需要把三个master都打上master标签
    nodeSelector:
        node-role.kubernetes.io/master: ""
3、注释掉replicas指令

另外需要注意的是:

  • 将/var/contiv/目录下证书文件复制到三个master节点,netmaster pod需要挂载使用这些证书文件
  • 除了第一个节点外,需要为其他每个节点创建/var/run/contiv/目录,netplugin会生成两个socket文件,如果不手动创建目录,则无法生成socket

Contiv提供了一个安装脚本,执行脚本安装:

# ./install/k8s/install.sh -n 192.168.6.16 -w routing -s etcd://192.168.6.17:6666
Installing Contiv for Kubernetes
secret "aci.key" created
Generating local certs for Contiv Proxy
Setting installation parameters
Applying contiv installation
To customize the installation press Ctrl+C and edit ./.contiv.yaml.
Extracting netctl from netplugin container
dafec6d9f0036d4743bf4b8a51797ddd19f4402eb6c966c417acf08922ad59bb
clusterrolebinding "contiv-netplugin" created
clusterrole "contiv-netplugin" created
serviceaccount "contiv-netplugin" created
clusterrolebinding "contiv-netmaster" created
clusterrole "contiv-netmaster" created
serviceaccount "contiv-netmaster" created
configmap "contiv-config" created
daemonset "contiv-netplugin" created
daemonset "contiv-netmaster" created
Creating network default:contivh1
daemonset "contiv-netplugin" deleted
clusterrolebinding "contiv-netplugin" configured
clusterrole "contiv-netplugin" configured
serviceaccount "contiv-netplugin" unchanged
clusterrolebinding "contiv-netmaster" configured
clusterrole "contiv-netmaster" configured
serviceaccount "contiv-netmaster" unchanged
configmap "contiv-config" unchanged
daemonset "contiv-netplugin" created
daemonset "contiv-netmaster" configured
Installation is complete
=========================================================

Contiv UI is available at https://192.168.6.16:10000
Please use the first run wizard or configure the setup as follows:
 Configure forwarding mode (optional, default is routing).
 netctl global set --fwd-mode routing
 Configure ACI mode (optional)
 netctl global set --fabric-mode aci --vlan-range <start>-<end>
 Create a default network
 netctl net create -t default --subnet=<CIDR> default-net
 For example, netctl net create -t default --subnet=20.1.1.0/24 -g 20.1.1.1 default-net

=========================================================

这里使用了三个参数:

-n 表示netmaster的地址。为了实现高可用,这里我起了三个netmaster,然后用lvs代理三个节点提供vip
-w 表示转发模式
-s 表示外部etcd地址,如果指定了外部etcd则不会创建etcd容器,而且无需手动处理。

另外,contiv是自带UI的,监听10000端口,上面安装完成后有提示,可以通过UI来管理网络。默认账号和密码是admin/admin。

不过,如果你知道要做什么的话,用命令会更方便快捷。

创建一个subnet:

# netctl net create -t default --subnet=30.0.0.0/10 -g 30.0.0.1 default-net
# netctl network ls
Tenant   Network      Nw Type  Encap type  Packet tag  Subnet        Gateway    IPv6Subnet  IPv6Gateway  Cfgd Tag
------   -------      -------  ----------  ----------  -------       ------     ----------  -----------  ---------
default  contivh1     infra    vxlan       0           132.1.1.0/24  132.1.1.1
default  default-net  data     vxlan       0           30.0.0.0/10   30.0.0.1

创建好网络之后,这时kube-dns pod就能拿到IP地址并运行起来了。

f、部署另外两个master节点

将第一个节点的配置文件和证书全部复制过来:

# scp -r 192.168.5.62:/etc/kubernetes/* /etc/kubernetes/

为新的master节点生成新的证书:

# cat uy06-05.sh
#!/bin/bash

#apiserver-kubelet-client
openssl genrsa -out apiserver-kubelet-client.key 2048
openssl req -new -key apiserver-kubelet-client.key -out apiserver-kubelet-client.csr -subj "/O=system:masters/CN=kube-apiserver-kubelet-client"
openssl x509 -req -set_serial $(date +%s%N) -in apiserver-kubelet-client.csr -CA ca.crt -CAkey ca.key -out apiserver-kubelet-client.crt -days 365 -extensions v3_req -extfile apiserver-kubelet-client-openssl.cnf

#controller-manager
openssl genrsa -out controller-manager.key 2048
openssl req -new -key controller-manager.key -out controller-manager.csr -subj "/CN=system:kube-controller-manager"
openssl x509 -req -set_serial $(date +%s%N) -in controller-manager.csr -CA ca.crt -CAkey ca.key -out controller-manager.crt -days 365 -extensions v3_req -extfile controller-manager-openssl.cnf

#scheduler
openssl genrsa -out scheduler.key 2048
openssl req -new -key scheduler.key -out scheduler.csr -subj "/CN=system:kube-scheduler"
openssl x509 -req -set_serial $(date +%s%N) -in scheduler.csr -CA ca.crt -CAkey ca.key -out scheduler.crt -days 365 -extensions v3_req -extfile scheduler-openssl.cnf

#admin
openssl genrsa -out admin.key 2048
openssl req -new -key admin.key -out admin.csr -subj "/O=system:masters/CN=kubernetes-admin"
openssl x509 -req -set_serial $(date +%s%N) -in admin.csr -CA ca.crt -CAkey ca.key -out admin.crt -days 365 -extensions v3_req -extfile admin-openssl.cnf

#node
openssl genrsa -out $(hostname).key 2048
openssl req -new -key $(hostname).key -out $(hostname).csr -subj "/O=system:nodes/CN=system:node:$(hostname)" -config kubelet-openssl.cnf
openssl x509 -req -set_serial $(date +%s%N) -in $(hostname).csr -CA ca.crt -CAkey ca.key -out $(hostname).crt -days 365 -extensions v3_req -extfile kubelet-openssl.cnf

这里生成了四套证书,使用的openssl配置文件其实是相同的:

[ v3_req ]
# Extensions to add to a certificate request
keyUsage = critical, digitalSignature, keyEncipherment
extendedKeyUsage = clientAuth

用新的证书替换旧证书,这几套证书只有apiserver-kubelet-client的证书是路径引用的,其他的都是直接引用的证书加密过的内容:

#!/bin/bash

VIP=192.168.5.62
APISERVER_PORT=6443
HOSTNAME=$(hostname)
CA_CRT=$(cat ca.crt |base64 -w0)
CA_KEY=$(cat ca.key |base64 -w0)
ADMIN_CRT=$(cat admin.crt |base64 -w0)
ADMIN_KEY=$(cat admin.key |base64 -w0)
CONTROLLER_CRT=$(cat controller-manager.crt |base64 -w0)
CONTROLLER_KEY=$(cat controller-manager.key |base64 -w0)
KUBELET_CRT=$(cat $(hostname).crt |base64 -w0)
KUBELET_KEY=$(cat $(hostname).key |base64 -w0)
SCHEDULER_CRT=$(cat scheduler.crt |base64 -w0)
SCHEDULER_KEY=$(cat scheduler.key |base64 -w0)

#admin
sed -e "s/VIP/$VIP/g" -e "s/APISERVER_PORT/$APISERVER_PORT/g" -e "s/CA_CRT/$CA_CRT/g" -e "s/ADMIN_CRT/$ADMIN_CRT/g" -e "s/ADMIN_KEY/$ADMIN_KEY/g" admin.temp > admin.conf
cp -a admin.conf /etc/kubernetes/admin.conf

#kubelet
sed -e "s/VIP/$VIP/g" -e "s/APISERVER_PORT/$APISERVER_PORT/g" -e "s/HOSTNAME/$HOSTNAME/g" -e "s/CA_CRT/$CA_CRT/g" -e "s/CA_KEY/$CA_KEY/g" -e "s/KUBELET_CRT/$KUBELET_CRT/g" -e "s/KUBELET_KEY/$KUBELET_KEY/g" kubelet.temp > kubelet.conf
cp -a kubelet.conf /etc/kubernetes/kubelet.conf

#controller-manager
sed -e "s/VIP/$VIP/g" -e "s/APISERVER_PORT/$APISERVER_PORT/g" -e "s/CA_CRT/$CA_CRT/g" -e "s/CONTROLLER_CRT/$CONTROLLER_CRT/g" -e "s/CONTROLLER_KEY/$CONTROLLER_KEY/g" controller-manager.temp > controller-manager.conf
cp -a controller-manager.conf /etc/kubernetes/controller-manager.conf

#scheduler
sed -e "s/VIP/$VIP/g" -e "s/APISERVER_PORT/$APISERVER_PORT/g" -e "s/CA_CRT/$CA_CRT/g" -e "s/SCHEDULER_CRT/$SCHEDULER_CRT/g" -e "s/SCHEDULER_KEY/$SCHEDULER_KEY/g" scheduler.temp > scheduler.conf
cp -a scheduler.conf /etc/kubernetes/scheduler.conf

#manifest kube-apiserver-client
cp -a apiserver-kubelet-client.key /etc/kubernetes/pki/
cp -a apiserver-kubelet-client.crt /etc/kubernetes/pki/

另外,由于contiv的netmaster使用了nodeSelector,这里记得要把这两个新部署master节点也打上master角色标签。默认情况下,新加入集群的节点是没有角色标签的。

# kubectl label node uy06-05 node-role.kubernetes.io/master=
# kubectl label node uy08-10 node-role.kubernetes.io/master=

替换证书之后,还要将集群中所有需要访问apiserver的地方修改为vip,以及修改advertise-address为本机地址,修改本地配置之后记得重启kubelet服务。

# sed -i "s@192.168.5.62@192.168.6.16@g" admin.conf
# sed -i "s@192.168.5.62@192.168.6.16@g" controller-manager.conf
# sed -i "s@192.168.5.62@192.168.6.16@g" kubelet.conf
# sed -i "s@192.168.5.62@192.168.6.16@g" scheduler.conf
# kubectl edit cm cluster-info -n kube-public
# kubectl edit cm kube-proxy -n kube-system
# vim manifests/kube-apiserver.yaml
--advertise-address=192.168.5.63
# systemctl restart kubelet

g、验证,尝试通过vip请求apiserver将node节点加入到集群中。

# kubeadm join --token 0c8921.578cf94fe0721e01 192.168.6.16:6443 --discovery-token-ca-cert-hash sha256:58cf1826d49e44fb6ff1590ddb077dd4e530fe58e13c1502ec07ce41ba6cc39e
[kubeadm] WARNING: kubeadm is in beta, please do not use it for production clusters.
[preflight] Running pre-flight checks
[preflight] WARNING: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
[discovery] Trying to connect to API Server "192.168.6.16:6443"
[discovery] Created cluster-info discovery client, requesting info from "https://192.168.6.16:6443"
[discovery] Requesting info from "https://192.168.6.16:6443" again to validate TLS against the pinned public key
[discovery] Cluster info signature and contents are valid and TLS certificate validates against pinned roots, will use API Server "192.168.6.16:6443"
[discovery] Successfully established connection with API Server "192.168.6.16:6443"
[bootstrap] Detected server version: v1.8.4
[bootstrap] The server supports the Certificates API (certificates.k8s.io/v1beta1)

Node join complete:
* Certificate signing request sent to master and response
  received.
* Kubelet informed of new secure connection details.

Run 'kubectl get nodes' on the master to see this machine join.

h、至此,整个kubernetes集群搭建完成。

# kubectl get no
NAME      STATUS    ROLES     AGE       VERSION
uy06-04   Ready     master    1d        v1.8.4
uy06-05   Ready     master    1d        v1.8.4
uy08-10   Ready     master    1d        v1.8.4
uy08-11   Ready     <none>    1d        v1.8.4
# kubectl get po --all-namespaces
NAMESPACE     NAME                                    READY     STATUS    RESTARTS   AGE
development   snowflake-f88456558-55jk8               1/1       Running   0          3h
development   snowflake-f88456558-5lkjr               1/1       Running   0          3h
development   snowflake-f88456558-mm7hc               1/1       Running   0          3h
development   snowflake-f88456558-tpbhw               1/1       Running   0          3h
kube-system   contiv-netmaster-6ctqj                  3/3       Running   0          6h
kube-system   contiv-netmaster-w4tx9                  3/3       Running   0          3h
kube-system   contiv-netmaster-wrlgc                  3/3       Running   0          3h
kube-system   contiv-netplugin-nbhkm                  2/2       Running   0          6h
kube-system   contiv-netplugin-rf569                  2/2       Running   0          3h
kube-system   contiv-netplugin-sczzk                  2/2       Running   0          3h
kube-system   contiv-netplugin-tlf77                  2/2       Running   0          5h
kube-system   heapster-59ff54b574-jq52w               1/1       Running   0          3h
kube-system   heapster-59ff54b574-nhl56               1/1       Running   0          3h
kube-system   heapster-59ff54b574-wchcr               1/1       Running   0          3h
kube-system   kube-apiserver-uy06-04                  1/1       Running   0          7h
kube-system   kube-apiserver-uy06-05                  1/1       Running   0          5h
kube-system   kube-apiserver-uy08-10                  1/1       Running   0          3h
kube-system   kube-controller-manager-uy06-04         1/1       Running   0          7h
kube-system   kube-controller-manager-uy06-05         1/1       Running   0          5h
kube-system   kube-controller-manager-uy08-10         1/1       Running   0          3h
kube-system   kube-dns-545bc4bfd4-fcr9q               3/3       Running   0          7h
kube-system   kube-dns-545bc4bfd4-ml52t               3/3       Running   0          3h
kube-system   kube-dns-545bc4bfd4-p6d7r               3/3       Running   0          3h
kube-system   kube-dns-545bc4bfd4-t8ttx               3/3       Running   0          3h
kube-system   kube-proxy-bpdr9                        1/1       Running   0          3h
kube-system   kube-proxy-cjnt5                        1/1       Running   0          5h
kube-system   kube-proxy-l4w49                        1/1       Running   0          7h
kube-system   kube-proxy-wmqgg                        1/1       Running   0          3h
kube-system   kube-scheduler-uy06-04                  1/1       Running   0          7h
kube-system   kube-scheduler-uy06-05                  1/1       Running   0          5h
kube-system   kube-scheduler-uy08-10                  1/1       Running   0          3h
kube-system   kubernetes-dashboard-5c54687f9c-ssklk   1/1       Running   0          3h
production    frontend-987698689-7pc56                1/1       Running   0          3h
production    redis-master-5f68fbf97c-jft59           1/1       Running   0          3h
production    redis-slave-74855dfc5-2bfwj             1/1       Running   0          3h
production    redis-slave-74855dfc5-rcrkm             1/1       Running   0          3h
staging       cattle-5f67c7948b-2j8jf                 1/1       Running   0          2h
staging       cattle-5f67c7948b-4zcft                 1/1       Running   0          2h
staging       cattle-5f67c7948b-gk87r                 1/1       Running   0          2h
staging       cattle-5f67c7948b-gzhc5                 1/1       Running   0          2h
# kubectl get cs
NAME                 STATUS    MESSAGE              ERROR
scheduler            Healthy   ok
controller-manager   Healthy   ok
etcd-2               Healthy   {"health": "true"}
etcd-0               Healthy   {"health": "true"}
etcd-1               Healthy   {"health": "true"}
# kubectl cluster-info
Kubernetes master is running at https://192.168.6.16:6443
Heapster is running at https://192.168.6.16:6443/api/v1/namespaces/kube-system/services/heapster/proxy
KubeDNS is running at https://192.168.6.16:6443/api/v1/namespaces/kube-system/services/kube-dns/proxy


补充:

默认情况下,kubectl没有权限查看pod的日志,授权方法:

# vim kubelet.rbac.yaml
# This role allows full access to the kubelet API
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: kubelet-api-admin
  labels:
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
rules:
- apiGroups:
  - ""
  resources:
  - nodes/proxy
  - nodes/log
  - nodes/stats
  - nodes/metrics
  - nodes/spec
  verbs:
  - "*"

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: my-apiserver-kubelet-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kubelet-api-admin
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: User
  name: kube-apiserver-kubelet-client
# kubectl apply -f kubelet.rbac.yaml
posted @ 2017-12-28 16:18 KeithTt 阅读(...) 评论(...) 编辑 收藏