kubeadm HA安装
一、环境介绍
#阿里云服务器
#本来是要LSB作为api-server的负载均衡是最好的，但是阿里云的SLB对TCP方式的监听，如果是本服务器访问SLB最后又通过SLB访问到本机的话是走不通的，只有http和https的方式能通。
#node节点最好是使用阿里云的弹性伸缩服务创建，这样后面扩容和伸缩方便。
172.16.208.161 master1
172.16.208.159 master2
172.16.208.160 master3
172.16.208.163 haproxy
172.16.208.164 node1
二、master服务器操作(所有master节点)
#修改内核参数
echo net.bridge.bridge-nf-call-iptables = 1 >>/etc/sysctl.conf
echo net.ipv4.ip_forward=1 >>/etc/sysctl.conf
echo net.bridge.bridge-nf-call-iptables=1 >>/etc/sysctl.conf
echo net.bridge.bridge-nf-call-ip6tables=1 >>/etc/sysctl.conf
echo vm.swappiness=0 >>/etc/sysctl.conf
sysctl -p
#关闭swap
#swapoff -a
#sed -i '/swap/s/^/#/' /etc/fstab
#关闭firewalld
systemctl stop firewalld
systemctl disable firewalld
sed -i 's/=enforcing/=disabled/g' /etc/selinux/config
#配置IPVS模块
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules
lsmod | grep -e ip_vs -e nf_conntrack_ipv4
#配置源
cd /etc/yum.repos.d/
wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
cat>>/etc/yum.repos.d/kubrenetes.repo<<EOF
[kubernetes]
name=Kubernetes Repo
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
EOF
yum makecache
#安装docker
yum -y install docker-ce
#配置加速地址
mkdir -p /etc/docker
cat>/etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors": [
"https://1nj0zren.mirror.aliyuncs.com",
"https://docker.mirrors.ustc.edu.cn",
"http://f1361db2.m.daocloud.io",
"https://registry.docker-cn.com"
]
}
EOF
systemctl daemon-reload
systemctl restart docker
systemctl enable docker
#安装kubeadm等
yum install kubelet kubeadm kubectl -y
#安装ipvs
yum -y install ipvsadm ipset
#启动kubelet
systemctl enable kubelet && systemctl start kubelet
三、haproxy服务器操作
#haproxy服务器操作
yum install -y haproxy
#修改haproxy配置文件
[root@nginx-proxy ~]# egrep -v "^$|^#|#" /etc/haproxy/haproxy.cfg
global
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 4000
user haproxy
group haproxy
daemon
stats socket /var/lib/haproxy/stats
defaults
mode http
log global
option httplog
option dontlognull
option http-server-close
option forwardfor except 127.0.0.0/8
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 3000
frontend k8s-master
bind 0.0.0.0:6443
bind 127.0.0.1:6443
mode tcp
option tcplog
tcp-request inspect-delay 5s
default_backend k8s-master
backend k8s-master
mode tcp
option tcplog
option tcp-check
balance roundrobin
default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
server master1 172.16.208.161:6443 check
server master2 172.16.208.159:6443 check
server master3 172.16.208.160:6443 check
backend static
balance roundrobin
server static 127.0.0.1:4331 check
#启动haproxy
[root@haproxy ~]# systemctl start haproxy
#查看
[root@haproxy ~]# ss -lntp|grep 6443
LISTEN 0 128 127.0.0.1:6443 *:* users:(("haproxy",pid=11943,fd=6))
LISTEN 0 128 *:6443 *:* users:(("haproxy",pid=11943,fd=5))
四、一台master服务器kubeadm init操作
1、创建kubeadm配置的yaml文件
[root@master1 ~]# kubeadm config print init-defaults > kubeadm-init.yaml
2、修改配置
[root@master1 ~]# cat kubeadm-init.yaml
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
#ApiServer 程序绑定的 ip, 填写网卡实际ip
advertiseAddress: 172.16.208.161
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: master1
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "172.16.208.163:6443" #访问api-server的地址,填写haporyx的地址
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers #国内镜像仓库
kind: ClusterConfiguration
kubernetesVersion: v1.20.0
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/12
scheduler: {}
3、初始化集群(-upload-certs会在加入master节点的时候自动拷贝证书)
[root@master1 ~]# kubeadm init --config kubeadm-init.yaml --upload-certs
4、初始化结束的输出
Your Kubernetes control-plane has initialized successfully! To start using your cluster, you need to run the following as a regular user: mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config Alternatively, if you are the root user, you can run: export KUBECONFIG=/etc/kubernetes/admin.conf You should now deploy a pod network to the cluster. Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at: https://kubernetes.io/docs/concepts/cluster-administration/addons/ You can now join any number of the control-plane node running the following command on each as root: kubeadm join 172.16.208.163:6443 --token abcdef.0123456789abcdef \ --discovery-token-ca-cert-hash sha256:e8fb32223f9ddae02aced75e50cda25474fd803ac6ce0e5db2d73bff3272109c \ --control-plane --certificate-key 3d151f00234812596732feb72f6a52a7a190bb14325341fa65d5b288453d0827 Please note that the certificate-key gives access to cluster sensitive data, keep it secret! As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use "kubeadm init phase upload-certs --upload-certs" to reload certs afterward. Then you can join any number of worker nodes by running the following on each as root: kubeadm join 172.16.208.163:6443 --token abcdef.0123456789abcdef \ --discovery-token-ca-cert-hash sha256:e8fb32223f9ddae02aced75e50cda25474fd803ac6ce0e5db2d73bff3272109c
5、拷贝权限文件
mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config
五、其它master服务器接入集群
1、master2和master3服务器join
[root@master2 ~]# kubeadm join 172.16.208.163:6443 --token abcdef.0123456789abcdef \ > --discovery-token-ca-cert-hash sha256:e8fb32223f9ddae02aced75e50cda25474fd803ac6ce0e5db2d73bff3272109c \ > --control-plane --certificate-key 3d151f00234812596732feb72f6a52a7a190bb14325341fa65d5b288453d0827 [root@master3 ~]# kubeadm join 172.16.208.163:6443 --token abcdef.0123456789abcdef \ > --discovery-token-ca-cert-hash sha256:e8fb32223f9ddae02aced75e50cda25474fd803ac6ce0e5db2d73bff3272109c \ > --control-plane --certificate-key 3d151f00234812596732feb72f6a52a7a190bb14325341fa65d5b288453d0827
2、拷贝权限文件
mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config
六、安装calico
wget https://docs.projectcalico.org/manifests/calico.yaml kubectl apply -f calico.yaml
七、查看集群状态
[root@master1 ~]# kubectl get node NAME STATUS ROLES AGE VERSION master1 Ready control-plane,master 4m44s v1.20.0 master2 Ready control-plane,master 2m51s v1.20.0 master3 Ready control-plane,master 2m47s v1.20.0
八、处理cs组件的错误与修改master端kubelet访问api-server的ip地址
1、刚安装完cs组件中的scheduler和controller-manager会有connect: connection refused的错误
[root@master1 ~]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
scheduler Unhealthy Get "http://127.0.0.1:10251/healthz": dial tcp 127.0.0.1:10251: connect: connection refused
controller-manager Unhealthy Get "http://127.0.0.1:10252/healthz": dial tcp 127.0.0.1:10252: connect: connection refused
etcd-0 Healthy {"health":"true"}
2、注释3台master服务器上kube-controller-manager.yaml和kube-scheduler.yaml默认端口--port=0这行配置
cd /etc/kubernetes/manifests
[root@master3 manifests]# grep 'port=0' kube-scheduler.yaml
#- --port=0
[root@master3 manifests]# grep 'port=0' kube-controller-manager.yaml
#- --port=0
3、修改master服务器上的 /etc/kubernetes/kubelet.conf配置，修改成本机的内网ip
[root@master1 manifests]# grep 6443 /etc/kubernetes/kubelet.conf
server: https://172.16.208.161:6443
[root@master2 manifests]# grep 6443 /etc/kubernetes/kubelet.conf
server: https://172.16.208.159:6443
[root@master3 manifests]# grep 6443 /etc/kubernetes/kubelet.conf
server: https://172.16.208.160:6443
4、重启kubelet
systemctl restart kubelet
5、查看cs状态
[root@master1 manifests]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-0 Healthy {"health":"true"}
九、node节点接入集群操作
1、node节点配置
#修改内核参数 echo net.bridge.bridge-nf-call-iptables = 1 >>/etc/sysctl.conf echo net.ipv4.ip_forward=1 >>/etc/sysctl.conf echo net.bridge.bridge-nf-call-iptables=1 >>/etc/sysctl.conf echo net.bridge.bridge-nf-call-ip6tables=1 >>/etc/sysctl.conf echo vm.swappiness=0 >>/etc/sysctl.conf sysctl -p #关闭swap #swapoff -a #sed -i '/swap/s/^/#/' /etc/fstab #关闭firewalld systemctl stop firewalld systemctl disable firewalld sed -i 's/=enforcing/=disabled/g' /etc/selinux/config #配置IPVS模块 cat > /etc/sysconfig/modules/ipvs.modules <<EOF #!/bin/bash modprobe -- ip_vs modprobe -- ip_vs_rr modprobe -- ip_vs_wrr modprobe -- ip_vs_sh modprobe -- nf_conntrack_ipv4 EOF chmod 755 /etc/sysconfig/modules/ipvs.modules bash /etc/sysconfig/modules/ipvs.modules lsmod | grep -e ip_vs -e nf_conntrack_ipv4 #配置源 cd /etc/yum.repos.d/ wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo cat>>/etc/yum.repos.d/kubrenetes.repo<<EOF [kubernetes] name=Kubernetes Repo baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/ gpgcheck=0 gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg EOF yum makecache #安装docker yum -y install docker-ce #配置加速地址 mkdir -p /etc/docker cat>/etc/docker/daemon.json <<-'EOF' { "registry-mirrors": [ "https://1nj0zren.mirror.aliyuncs.com", "https://docker.mirrors.ustc.edu.cn", "http://f1361db2.m.daocloud.io", "https://registry.docker-cn.com" ] } EOF systemctl daemon-reload systemctl restart docker systemctl enable docker #安装kubeadm等 yum install kubelet kubeadm -y #安装ipvs yum -y install ipvsadm ipset
2、master节点创建新的token
[root@master1 manifests]# kubeadm token create --print-join-command kubeadm join 172.16.208.163:6443 --token c8i365.o0k3q1q8hhlowcx1 --discovery-token-ca-cert-hash sha256:e8fb32223f9ddae02aced75e50cda25474fd803ac6ce0e5db2d73bff3272109c
3、node节点join
[root@node1 ~]# kubeadm join 172.16.208.163:6443 --token c8i365.o0k3q1q8hhlowcx1 --discovery-token-ca-cert-hash sha256:e8fb32223f9ddae02aced75e50cda25474fd803ac6ce0e5db2d73bff3272109c
4、查看集群状态
[root@master1 manifests]# kubectl get node NAME STATUS ROLES AGE VERSION master1 Ready control-plane,master 46m v1.20.0 master2 Ready control-plane,master 44m v1.20.0 master3 Ready control-plane,master 44m v1.20.0 node1 Ready <none> 66s v1.20.0
