Kubernetes 安装升级

# 系统更新
yum install epel-release -y
yum update -y
yum install net-tools vim -y

##升级内核

##载入公钥
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org

##升级安装ELRepo
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm

##查看可用的rpm包
yum --disablerepo=* --enablerepo=elrepo-kernel repolist
yum --disablerepo=* --enablerepo=elrepo-kernel list kernel*

##安装最新版本的kernel
yum --disablerepo=* --enablerepo=elrepo-kernel install kernel-lt.x86_64 -y

##安装新版本开发工具包
yum --disablerepo=* --enablerepo=elrepo-kernel install kernel-lt-devel.x86_64 -y

##移除旧版本工具包
yum remove kernel-tools-libs.x86_64 kernel-tools.x86_64 -y
##安装新版本工具包
yum --disablerepo=* --enablerepo=elrepo-kernel install kernel-lt-tools.x86_64 -y

##查看内核插入顺序
awk -F \' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg
##grep "^menuentry" /boot/grub2/grub.cfg | cut -d "'" -f2

##查看当前实际启动顺序
grub2-editenv list

##设置默认启动
grub2-set-default 0

##重新创建内核配置
grub2-mkconfig -o /boot/grub2/grub.cfg

##移除旧的内核
rpm -qa | grep kernel
##yum remove
##清除iptables所有规则
ipvsadm --clear
iptables -t nat -F
iptables -t nat -X
iptables -t nat -P PREROUTING ACCEPT
iptables -t nat -P POSTROUTING ACCEPT
iptables -t nat -P OUTPUT ACCEPT
iptables -t mangle -F
iptables -t mangle -X
iptables -t mangle -P PREROUTING ACCEPT
iptables -t mangle -P INPUT ACCEPT
iptables -t mangle -P FORWARD ACCEPT
iptables -t mangle -P OUTPUT ACCEPT
iptables -t mangle -P POSTROUTING ACCEPT
iptables -F
iptables -X
iptables -P FORWARD ACCEPT
iptables -P INPUT ACCEPT
iptables -P OUTPUT ACCEPT
iptables -t raw -F
iptables -t raw -X
iptables -t raw -P PREROUTING ACCEPT
iptables -t raw -P OUTPUT ACCEPT
##重启
reboot

 

######################################################################################
# 设置内核参数
cat << EOF > /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-arptables = 1
net.ipv4.ip_forward = 1
net.ipv4.tcp_tw_recycle = 0 ##用来快速回收TIME_WAIT连接，不过如果在NAT环境下会引发问题 tcp_tw_recycle 和 Kubernetes的NAT冲突，必须关闭，否则会导致服务不通或丢包，在4.12之后的内核已移除tcp_tw_recycle内核参数:
vm.swappiness = 0 #最大限度使用物理内存，然后才是 swap空间
vm.overcommit_memory = 1
vm.panic_on_oom = 0
fs.inotify.max_user_watches = 89100
fs.file-max = 52706963
fs.nr_open = 52706963
net.ipv6.conf.default.disable_ipv6 = 1 ##关闭不使用的 IPV6 协议栈，防止触发 docker BUG。
net.netfilter.nf_conntrack_max = 2310720
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
fs.inotify.max_user_watches = 89100
EOF
sysctl --system
sysctl -p /etc/sysctl.d/k8s.conf

# 关闭 Swap，自 1.8 开始，k8s 要求关闭系统 Swap，如果不关闭，kubelet 无法启动。
swapoff -a
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
sed -i '/swap/d' /etc/fstab

# 关闭防火墙和 SELinux
systemctl disable firewalld && systemctl stop firewalld && setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

# kube-proxy 开启 ipvs 的前置条件
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
modprobe -- br_netfilter ##高版本内核已经编译进内核功能而不是模块CONFIG_BRIDGE_NETFILTER=y)cat /boot/config-$(uname -r) |grep -C5 BRIDGE
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

禁用postfix
systemctl stop postfix
systemctl disable postfix

# 添加docker源和安装docker
yum install -y yum-utils
yum-config-manager --add-repo https://download.daocloud.io/docker/linux/centos/docker-ce.repo
yum list docker-ce --show-duplicates
yum install -y --setopt=obsoletes=0 docker-ce-18.06.1.ce*

# 修改 docker cgoup driver 为 systemd
cat >> /etc/docker/daemon.json << EOF
{
"exec-opts": ["native.cgroupdriver=systemd"]
}
EOF
systemctl start docker
systemctl enable docker
docker info | grep Cgroup

# 添加kubernetes源和kubeadm,kubelet,kubectl,ipvsadm
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
yum install -y kubelet-1.14* kubeadm-1.14* kubectl-1.14* ipvsadm
systemctl daemon-reload
systemctl enable kubelet
systemctl start kubelet

 

##kubeadm初始化
kubeadm config print init-defaults > kubeadm.conf
kubeadm init --config kubeadm.conf

apiVersion: kubeadm.k8s.io/v1beta1
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.1.201
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: node1
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta1
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "192.168.1.200:6443"
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.14.0
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/12
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs

##拷贝证书到其它控制面节点

cd /etc/kubernetes && tar cvzf k8s-key.tgz pki/ca.* pki/sa.* pki/front-proxy-ca.* pki/etcd/ca.*
scp /etc/kubernetes/k8s-key.tgz $CP1_IP:/etc/kubernetes
ssh $CP1_IP 'tar xf /etc/kubernetes/k8s-key.tgz -C /etc/kubernetes/'
scp /etc/kubernetes/k8s-key.tgz $CP2_IP:/etc/kubernetes
ssh $CP2_IP 'tar xf /etc/kubernetes/k8s-key.tgz -C /etc/kubernetes/'

##网络插件
kubectl apply -f https://docs.projectcalico.org/v3.14/manifests/calico.yaml

#创建密钥
kubectl create secret docker-registry registry-secret --docker-server=registry.huoyancredit.com --docker-username=admin --docker-password=cqsd^A75 -n default

===================================================================
升级现有集群

先升级kubeadm kubelet kubectl到指定版本,例如从1.14版本升级到1.15版本
yum install kubeadm-1.15.0 kubelet-1.15.0 kubectl-1.15.0 -y

在每个master节点上升级版本
kubeadm upgrade apply v1.15.0

之后重启kubelet
systemctl daemon-reload && systemctl restart kubelet

问题点一：
较低版本etcd访问及查看状态问题
kubectl exec -it etcd-node1 -n kube-system -- etcdctl member list 返回：
client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 127.0.0.1:4001: connect: connection refused
; error #1: net/http: HTTP/1.x transport connection broken: malformed HTTP response "\x15\x03\x01\x00\x02\x02"

etcdctl --endpoints=https://192.168.1.201:2379 member list 返回“
client: etcd cluster is unavailable or misconfigured; error #0: x509: failed to load system roots and no roots provided

正常需要https和ca,证书,证书key
kubectl exec -it etcd-node1 -n kube-system -- etcdctl --endpoints=https://127.0.0.1:2379 --ca-file=/etc/kubernetes/pki/etcd/ca.crt --cert-file=/etc/kubernetes/pki/etcd/server.crt --key-file=/etc/kubernetes/pki/etcd/server.key member list

 

token获取

有关 token 的过期时间是24小时，certificate-key 过期时间是2小时

如果没有--discovery-token-ca-cert-hash值，也可以通过以下命令获取 

openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'

执行kubeadm token create --print-join-command，重新生成，重新生成基础的 join 命令（对于添加 master 节点还需要重新生成certificate-key，见下一步）

 

使用 kubeadm init phase upload-certs --experimental-upload-certs 重新生成certificate-key

参考

https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-join/#token-based-discovery-with-ca-pinning

https://kubernetes.io/docs/setup/independent/high-availability/#steps-for-the-first-control-plane-node (其中的note)