K8s1.25多master高可用集群

K8s1.25多master高可用集群

实验规划：
OS：centos7.9

pod网段：10.244.0.0/16

service网段：10.96.0.0/16

本篇虽然有三个master，但是只做master1和master2的主备

角色	IP	主机名	组件
master1	192.168.96.65	master1	apiserver、controller-manager、schedule、kubelet、etcd、kube-proxy、容器运行时、calico、keepalived、nginx
master2	192.168.96.66	master2	apiserver、controller-manager、schedule、kubelet、etcd、kube-proxy、容器运行时、calico、keepalived、nginx
master3	192.168.96.67	master3	apiserver、controller-manager、schedule、kubelet、etcd、kube-proxy、容器运行时、calico
node	192.168.96.68	node	Kube-proxy、calico、coredns、容器运行时、kubelet

默认配置好ip和主机名、本地解析

配置免密，便于传输

master1

ssh-keygen
ssh-copy-id root@master2
ssh-copy-id root@master3
ssh-copy-id root@node

master2

ssh-keygen
ssh-copy-id root@master1
ssh-copy-id root@master3
ssh-copy-id root@node

master3

ssh-keygen
ssh-copy-id root@master1
ssh-copy-id root@master2
ssh-copy-id root@node

node

ssh-keygen
ssh-copy-id root@master1
ssh-copy-id root@master2
ssh-copy-id root@master3

关闭swap分区【四节点执行】

swapoff -a
vim /etc/fstab

关闭防火墙和selinux【四节点】

systemctl disable --now firewalld
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

修改机器内核【四节点】

modprobe br_netfilter

cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-iptables = 1 
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF

sysctl -p /etc/sysctl.d/k8s.conf 
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1

配置阿里云仓库

rm -rf /etc/yum.repos.d/*

OS

curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo

docker

yum install -y yum-utils
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum install docker-ce -y 

docker加速器

mkdir -p /etc/docker
tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://ux1eh0cf.mirror.aliyuncs.com"]
}
EOF
systemctl daemon-reload
systemctl restart docker

K8s

cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

安装基础软件包【四节点】

yum install vim net-tools bash-completion wget chrony -y

配置NTP【四节点】

vim /etc/chrony.conf
server ntp4.aliyun.com iburst
systemctl restart chronyd
chronyc sources

配置containerd

containerd config default > /etc/containerd/config.toml 
vim /etc/containerd/config.toml
sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.6"  # 修改
SystemdCgroup = true  # 修改

systemctl enable --now containerd

[root@master1 ~]# crictl img
WARN[0000] image connect using default endpoints: [unix:///var/run/dockershim.sock unix:///run/containerd/containerd.sock unix:///run/crio/crio.sock unix:///var/run/cri-dockerd.sock]. As the default settings are now deprecated, you should set the endpoint instead. 
E0102 10:24:23.234047   11264 remote_image.go:119] "ListImages with filter from image service failed" err="rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial unix /var/run/dockershim.sock: connect: no such file or directory\"" filter="&ImageFilter{Image:&ImageSpec{Image:,Annotations:map[string]string{},},}"
FATA[0000] listing images: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing dial unix /var/run/dockershim.sock: connect: no such file or directory" 

查看镜像报错怎么办？

创建/etc/crictl.yaml

runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false

配置containerd镜像加速器

vim /etc/containerd/config.toml
# 过滤config_path
config_path = "/etc/containerd/certs.d" # 添加

mkdir certs.d
mkdir certs.d/docker.io

vim certs.d/docker.io/host.toml
[host."https://ux1eh0cf.mirror.aliyuncs.com"]
  capabilities = ["pull"]

systemctl restart containerd

安装K8s基础软件包

yum install kubelet-1.25.0 kubeadm-1.25.0 kubectl-1.25.0 -y
systemctl enable kubelet

• Kubeadm: kubeadm是一个工具，用来初始化k8s集群的
• kubelet: 安装在集群所有节点上，用于启动Pod的，kubeadm安装k8s，k8s控制节点和工作节点的组件，都是基于pod运行的，只要pod启动，就需要kubelet
• kubectl: 通过kubectl可以部署和管理应用，查看各种资源，创建、删除和更新各种组件

通过keepalived+nginx实现k8s apiserver高可用

1. 安装nginx
   在master1和master2上做主备

   yum install epel-release -y 
   yum install nginx keepalived nginx-mod-stream -y
   

2. 修改nginx配置文件

   rm -rf /etc/nginx.conf
   vim /etc/nginx/nginx.conf
   

   上传配置好的nginx

   user nginx;
   worker_processes auto;
   error_log /var/log/nginx/error.log;
   pid /run/nginx.pid;
   
   include /usr/share/nginx/modules/*.conf;
   
   events {
       worker_connections 1024;
   }
   
   # 四层负载均衡，为两台Master apiserver组件提供负载均衡
   stream {
   
       log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
   
       access_log  /var/log/nginx/k8s-access.log  main;
   
       upstream k8s-apiserver {
               server 192.168.96.65:6443 weight=5 max_fails=3 fail_timeout=30s;   # 修改ip
               server 192.168.96.66:6443 weight=5 max_fails=3 fail_timeout=30s;
               server 192.168.96.67:6443 weight=5 max_fails=3 fail_timeout=30s;
   
       }
   
       server {
          listen 16443; # 由于nginx与master节点复用，这个监听端口不能是6443，否则会冲突
          proxy_pass k8s-apiserver;
       }
   }
   
   http {
       log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                         '$status $body_bytes_sent "$http_referer" '
                         '"$http_user_agent" "$http_x_forwarded_for"';
   
       access_log  /var/log/nginx/access.log  main;
   
       sendfile            on;
       tcp_nopush          on;
       tcp_nodelay         on;
       keepalive_timeout   65;
       types_hash_max_size 2048;
   
       include             /etc/nginx/mime.types;
       default_type        application/octet-stream;
   
       server {
           listen       80 default_server;
           server_name  _;
   
           location / {
           }
       }
   }
   
   

3. 主keepalived配置

   rm -rf  /etc/keepalived/keepalived.conf
   

   上传文件

   global_defs {
      notification_email {
        acassen@firewall.loc
        failover@firewall.loc
        sysadmin@firewall.loc
      }
      notification_email_from Alexandre.Cassen@firewall.loc
      smtp_server 127.0.0.1
      smtp_connect_timeout 30
      router_id NGINX_MASTER
   }
   
   vrrp_script check_nginx {
       script "/etc/keepalived/check_nginx.sh"
   }
   
   vrrp_instance VI_1 {
       state MASTER
       interface ens32  # 修改为实际网卡名
       virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的
       priority 100    # 优先级，备服务器设置 90
       advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒
       authentication {
           auth_type PASS
           auth_pass 1111
       }
       # 虚拟IP
       virtual_ipaddress {
           192.168.96.69/24   # 修改本网段未占用的ip
       }
       track_script {
           check_nginx
       }
   }
   
   

4. 上传check_nginx.sh脚本到/etc/keepalived/

   #!/bin/bash
   #1、判断Nginx是否存活
   counter=$(ps -ef |grep nginx | grep sbin | egrep -cv "grep|$$" )
   if [ $counter -eq 0 ]; then
       #2、如果不存活则尝试启动Nginx
       service nginx start
       sleep 2
       #3、等待2秒后再次获取一次Nginx状态
       counter=$(ps -ef |grep nginx | grep sbin | egrep -cv "grep|$$" )
       #4、再次进行判断，如Nginx还不存活则停止Keepalived，让地址进行漂移
       if [ $counter -eq 0 ]; then
           service  keepalived stop
       fi
   fi
   

   chmod +x /etc/keepalived/check_nginx.sh
   

5. 传输文件到master2

   scp /etc/keepalived/check_nginx.sh root@master2:/etc/keepalived/check_nginx.sh
   scp /etc/keepalived/keepalived.conf root@master2:/etc/keepalived/keepalived.conf 
   

6. 修改keepalived.conf

   global_defs {
      notification_email {
        acassen@firewall.loc
        failover@firewall.loc
        sysadmin@firewall.loc
      }
      notification_email_from Alexandre.Cassen@firewall.loc
      smtp_server 127.0.0.1
      smtp_connect_timeout 30
      router_id NGINX_MASTER
   }
   
   vrrp_script check_nginx {
       script "/etc/keepalived/check_nginx.sh"
   }
   
   vrrp_instance VI_1 {
       state BACKUP      # 修改为备
       interface ens32  # 修改为实际网卡名
       virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的
       priority 90    # 优先级，备服务器设置 90 
       advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒
       authentication {
           auth_type PASS
           auth_pass 1111
       }
       # 虚拟IP
       virtual_ipaddress {
           192.168.96.69/24
       }
       track_script {
           check_nginx
       }
   }
   
   

7. 启动服务

   systemctl daemon-reload
   systemctl enable --now nginx
   systemctl enable --now keepalived
   

   验证

   2: ens32: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
       link/ether 00:0c:29:10:e9:8f brd ff:ff:ff:ff:ff:ff
       inet 192.168.96.65/24 brd 192.168.96.255 scope global noprefixroute ens32
          valid_lft forever preferred_lft forever
       inet 192.168.96.69/24 scope global secondary ens32
          valid_lft forever preferred_lft forever
       inet6 fe80::20c:29ff:fe10:e98f/64 scope link 
          valid_lft forever preferred_lft forever
   在主master上出现了我们配置的虚拟ip
   

使用kubeadm初始化集群

指定容器运行时

crictl config runtime-endpoint /run/containerd/containerd.sock

创建kubeadm.yaml

kubeadm config print init-defaults > kubeadm.yaml

修改如下

apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
#localAPIEndpoint:
#  advertiseAddress: 1.2.3.4
#  bindPort: 6443
nodeRegistration:
  criSocket: unix:///run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
#  name: node
  taints: null
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: 1.25.0
controlPlaneEndpoint:  192.168.96.69:16443
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.244.0.0/16
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd

将打包好的镜像传输给其他节点

scp k8s_1.25.0.tar.gz root@master2:/root/
scp k8s_1.25.0.tar.gz root@master3:/root/
scp k8s_1.25.0.tar.gz root@node:/root/

导入镜像【全部节点】

ctr -n k8s.io images import k8s_1.25.0.tar.gz

初始化

[root@master1 ~]# kubeadm  init  --config kubeadm.yaml --ignore-preflight-errors=SystemVerification 
[init] Using Kubernetes version: v1.25.0
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local master1] and IPs [10.96.0.1 192.168.96.65 192.168.96.69]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [localhost master1] and IPs [192.168.96.65 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [localhost master1] and IPs [192.168.96.65 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
W0102 11:24:28.968716   22193 endpoint.go:57] [endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "admin.conf" kubeconfig file
W0102 11:24:29.054540   22193 endpoint.go:57] [endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "kubelet.conf" kubeconfig file
W0102 11:24:29.120311   22193 endpoint.go:57] [endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
W0102 11:24:29.232906   22193 endpoint.go:57] [endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 31.527542 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Skipping phase. Please see --upload-certs
[mark-control-plane] Marking the node master1 as control-plane by adding the labels: [node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers]
[mark-control-plane] Marking the node master1 as control-plane by adding the taints [node-role.kubernetes.io/control-plane:NoSchedule]
[bootstrap-token] Using token: abcdef.0123456789abcdef
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] Configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] Configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
W0102 11:25:05.603696   22193 endpoint.go:57] [endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of control-plane nodes by copying certificate authorities
and service account keys on each node and then running the following as root:

  kubeadm join 192.168.96.69:16443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:2942ef510e4f6d4a44fa63cf6677060329f67694888941aef6489e06843cb084 \
	--control-plane 

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.96.69:16443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:2942ef510e4f6d4a44fa63cf6677060329f67694888941aef6489e06843cb084 

使用集群准备工作

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

为什么创建了config后就可以使用kubectl了？

因为kubectl默认从/root/找配置文件，并运行

添加控制节点

mkdir -p /etc/kubernetes/pki/etcd &&mkdir -p ~/.kube/

拷贝证书

scp /etc/kubernetes/pki/ca.*  root@master2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/sa.*  root@master2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/front-proxy-ca*  root@master2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/etcd/ca.*  root@master2:/etc/kubernetes/pki/etcd/

获取添加命令【有效期只有24小时】

[root@master1 ~]# kubeadm token create --print-join-command
kubeadm join 192.168.96.69:16443 --token m66ig7.8de6j94gq9r7qrib --discovery-token-ca-cert-hash sha256:2942ef510e4f6d4a44fa63cf6677060329f67694888941aef6489e06843cb084 

如果你添加控制节点，你要加参数

kubeadm join 192.168.96.69:16443 --token m66ig7.8de6j94gq9r7qrib --discovery-token-ca-cert-hash sha256:2942ef510e4f6d4a44fa63cf6677060329f67694888941aef6489e06843cb084  --control-plane

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

添加工作节点

[root@master1 ~]# kubeadm token create --print-join-command
kubeadm join 192.168.96.69:16443 --token m66ig7.8de6j94gq9r7qrib --discovery-token-ca-cert-hash sha256:2942ef510e4f6d4a44fa63cf6677060329f67694888941aef6489e06843cb084 

kubeadm join 192.168.96.69:16443 --token m66ig7.8de6j94gq9r7qrib --discovery-token-ca-cert-hash sha256:2942ef510e4f6d4a44fa63cf6677060329f67694888941aef6489e06843cb084  

验证

[root@master1 ~]# kubectl get node
NAME      STATUS     ROLES           AGE   VERSION
master1   NotReady   control-plane   77m   v1.25.0
master2   NotReady   control-plane   64m   v1.25.0
master3   NotReady   control-plane   63m   v1.25.0
node      NotReady   <none>          62m   v1.25.0

给node添加一个标签

kubectl label  nodes node node-role.kubernetes.io/work=work

[root@master1 ~]# kubectl get node
NAME      STATUS     ROLES           AGE   VERSION
master1   NotReady   control-plane   78m   v1.25.0
master2   NotReady   control-plane   65m   v1.25.0
master3   NotReady   control-plane   64m   v1.25.0
node      NotReady   work            62m   v1.25.0

安装calico网络插件

上传calico镜像文件，并传输给其他几个节点

scp calico.tar.gz root@master2:/root/
scp calico.tar.gz root@master3:/root/
scp calico.tar.gz root@node:/root/

并在所有节点执行

ctr -n k8s.io images import calico.tar.gz

在控制节点1执行calico.yaml

kubectl apply -f calico.yaml

稍等一会，验证

[root@master1 ~]# kubectl get pod -n kube-system 
NAME                                       READY   STATUS    RESTARTS      AGE
calico-kube-controllers-6744f6b6d5-lk96z   1/1     Running   0             25s   【做网络策略的时候用这个】
calico-node-hg597                          1/1     Running   0             25s
calico-node-rlg9q                          1/1     Running   0             25s
calico-node-v7xq2                          1/1     Running   0             25s
calico-node-wm2s2                          1/1     Running   0             25s
coredns-c676cc86f-kkdtc                    1/1     Running   0             83m
coredns-c676cc86f-rx7qh                    1/1     Running   0             83m

[root@master1 ~]# kubectl get node
NAME      STATUS   ROLES           AGE   VERSION
master1   Ready    control-plane   84m   v1.25.0
master2   Ready    control-plane   71m   v1.25.0
master3   Ready    control-plane   69m   v1.25.0
node      Ready    work            68m   v1.25.0

注：在线下载配置文件地址是： https://docs.projectcalico.org/manifests/calico.yaml

Calico架构图

Calico网络模型主要工作组件：

1.Felix：运行在每一台 Host 的 agent 进程，主要负责网络接口管理和监听、路由、ARP 管理、ACL 管理和同步、状态上报等。保证跨主机容器网络互通。

2.etcd：分布式键值存储，相当于k8s集群中的数据库，存储着Calico网络模型中IP地址等相关信息。主要负责网络元数据一致性，确保 Calico 网络状态的准确性；

3.BGP Client（BIRD）：Calico 为每一台 Host 部署一个 BGP Client，即每台host上部署一个BIRD。 主要负责把 Felix 写入 Kernel 的路由信息分发到当前 Calico 网络，确保 Workload 间的通信的有效性；

4.BGP Route Reflector：在大型网络规模中，如果仅仅使用 BGP client 形成 mesh 全网互联的方案就会导致规模限制，因为所有节点之间俩俩互联，需要 N^2 个连接，为了解决这个规模问题，可以采用 BGP 的 Router Reflector 的方法，通过一个或者多个 BGP Route Reflector 来完成集中式的路由分发。

1.9.2 calico网络插件配置文件说明

1、Daemonset配置

……

containers:

​ # Runs calico-node container on each Kubernetes node. This

​ # container programs network policy and routes on each

​ # host.

​ - name: calico-node

​ image: docker.io/calico/node:v3.18.0

……

​ env:

​ # Use Kubernetes API as the backing datastore.

​ - name: DATASTORE_TYPE

​ value: "kubernetes"

​ # Cluster type to identify the deployment type

​ - name: CLUSTER_TYPE

​ value: "k8s,bgp"

​ # Auto-detect the BGP IP address.

​ - name: IP

​ value: "autodetect"

​ #pod网段

​ - name: CALICO_IPV4POOL_CIDR

value: "10.244.0.0/16"

​ # Enable IPIP

​ - name: CALICO_IPV4POOL_IPIP

​ value: "Always"

calico-node服务的主要参数如下:

CALICO_IPV4POOL_IPIP：是否启用IPIP模式。启用IPIP模式时，Calico将在Node上创建一个名为tunl0的虚拟隧道。IP Pool可以使用两种模式：BGP或IPIP。使用IPIP模式时，设置CALICO_IPV4POOL_IPIP="Always"，不使用IPIP模式时，设置CALICO_IPV4POOL_IPIP="Off"，此时将使用BGP模式。

IP_AUTODETECTION_METHOD：获取Node IP地址的方式，默认使用第1个网络接口的IP地址，对于安装了多块网卡的Node，可以使用正则表达式选择正确的网卡，例如"interface=eth.*"表示选择名称以eth开头的网卡的IP地址。

- name: IP_AUTODETECTION_METHOD

value: "interface=ens33"

扩展：calico的IPIP模式和BGP模式对比分析

1）IPIP

把一个IP数据包又套在一个IP包里，即把IP层封装到IP层的一个 tunnel，它的作用其实基本上就相当于一个基于IP层的网桥，****一般来说，普通的网桥是基于mac层的，根本不需要IP，而这个ipip则是通过两端的路由做一个tunnel，把两个本来不通的网络通过点对点连接起来；

calico以ipip模式部署完毕后，node上会有一个tunl0的网卡设备，这是ipip做隧道封装用的,也是一种overlay模式的网络。当我们把节点下线，calico容器都停止后，这个设备依然还在，执行 rmmodipip命令可以将它删除。

2）BGP

BGP模式直接使用物理机作为虚拟路由路（vRouter），不再创建额外的tunnel

边界网关协议（BorderGateway Protocol, BGP）是互联网上一个核心的去中心化的自治路由协议。它通过维护IP路由表或‘前缀’表来实现自治系统（AS）之间的可达性，属于矢量路由协议。BGP不使用传统的内部网关协议（IGP）的指标，而是基于路径、网络策略或规则集来决定路由。因此，它更适合被称为矢量性协议，而不是路由协议，通俗的说就是将接入到机房的多条线路（如电信、联通、移动等）融合为一体，实现多线单IP；

BGP 机房的优点：服务器只需要设置一个IP地址，最佳访问路由是由网络上的骨干路由器根据路由跳数与其它技术指标来确定的，不会占用服务器的任何系统；

官方提供的calico.yaml模板里，默认打开了ip-ip功能，该功能会在node上创建一个设备tunl0，容器的网络数据会经过该设备被封装一个ip头再转发。这里，calico.yaml中通过修改calico-node的环境变量：CALICO_IPV4POOL_IPIP来实现ipip功能的开关：默认是Always，表示开启；Off表示关闭ipip。

- name: CLUSTER_TYPE

​ value: "k8s,bgp"

​ # Auto-detect the BGP IP address.

​ - name: IP

​ value: "autodetect"

​ # Enable IPIP

​ - name: CALICO_IPV4POOL_IPIP

​ value: "Always"

总结：

calico BGP通信是基于TCP协议的，所以只要节点间三层互通即可完成，即三层互通的环境bird就能生成与邻居有关的路由。但是这些路由和flannel host-gateway模式一样，需要二层互通才能访问的通，因此如果在实际环境中配置了BGP模式生成了路由但是不同节点间pod访问不通，可能需要再确认下节点间是否二层互通。

为了解决节点间二层不通场景下的跨节点通信问题，calico也有自己的解决方案——IPIP模式

配置etcd高可用集群

打开vim /etc/kubernetes/manifests/etcd.yaml

修改配置

# 原配置
spec:
  containers:
  - command:
    - etcd
    - --advertise-client-urls=https://192.168.96.65:2379
    - --cert-file=/etc/kubernetes/pki/etcd/server.crt
    - --client-cert-auth=true
    - --data-dir=/var/lib/etcd
    - --experimental-initial-corrupt-check=true
    - --experimental-watch-progress-notify-interval=5s
    - --initial-advertise-peer-urls=https://192.168.96.65:2380
    - --initial-cluster=master1=https://192.168.96.65:2380   # 修改这一行
    - --key-file=/etc/kubernetes/pki/etcd/server.key
    - --listen-client-urls=https://127.0.0.1:2379,https://192.168.96.65:2379
    - --listen-metrics-urls=http://127.0.0.1:2381
    - --listen-peer-urls=https://192.168.96.65:2380
    - --name=master1
    - --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt
    - --peer-client-cert-auth=true
    - --peer-key-file=/etc/kubernetes/pki/etcd/peer.key
    - --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
    - --snapshot-count=10000
    - --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt

spec:
  containers:
  - command:
    - etcd
    - --advertise-client-urls=https://192.168.96.65:2379
    - --cert-file=/etc/kubernetes/pki/etcd/server.crt
    - --client-cert-auth=true
    - --data-dir=/var/lib/etcd
    - --experimental-initial-corrupt-check=true
    - --experimental-watch-progress-notify-interval=5s
    - --initial-advertise-peer-urls=https://192.168.96.65:2380
    - --initial-cluster=master1=https://192.168.96.65:2380,master3=https://192.168.96.67:2380,master2=https://192.168.96.66:2380
    - --key-file=/etc/kubernetes/pki/etcd/server.key
    - --listen-client-urls=https://127.0.0.1:2379,https://192.168.96.65:2379
    - --listen-metrics-urls=http://127.0.0.1:2381
    - --listen-peer-urls=https://192.168.96.65:2380
    - --name=master1
    - --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt
    - --peer-client-cert-auth=true
    - --peer-key-file=/etc/kubernetes/pki/etcd/peer.key
    - --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
    - --snapshot-count=10000
    - --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt

重启服务

systemctl restart kubelet.service

配置命令补全

echo "source <(kubectl completion bash)" >> /etc/profile && source  /etc/profile