kubeasz部署-高可用Kubernetes
| hostname | IP | 规格 | ||
| k8s-master01 | 192.168.40.101 | 5G | ||
| k8s-master02 | 192.168.40.102 | 5G | ||
| k8s-master03 | 192.168.40.103 | 后续可加入 | 5G | |
| k8s-harbor | 192.168.40.104 | 4G | ||
| k8s-etcd01 | 192.168.40.106 | 2G | ||
| k8s-etcd02 | 192.168.40.107 | 2G | ||
| k8s-etcd03 | 192.168.40.108 | 2G | ||
| k8s-haproxy01 | 192.168.40.109 | VIP:192.168.40.188 | 2G | |
| k8s-haproxy02 | 192.168.40.110 | 2G | ||
| k8s-node01 | 192.168.40.111 | 8G | ||
| k8s-node02 | 192.168.40.112 | 8G | ||
| k8s-node03 | 192.168.40.113 | 后续可加入 | 8G | |
| k8s-deploy | 192.168.40.147 | 4G |
1、基础环境准备
#本实验 Centos7.9.2009
#1.配置hosts文件
vim /etc/hosts
192.168.40.101 k8s-master01
192.168.40.102 k8s-master02
192.168.40.103 k8s-master03
192.168.40.104 k8s-harbor
192.168.40.105 k8s-harbor02
192.168.40.106 k8s-etcd01
192.168.40.107 k8s-etcd02
192.168.40.108 k8s-etcd03
192.168.40.109 k8s-haproxy01
192.168.40.110 k8s-haproxy02
192.168.40.111 k8s-node01
192.168.40.112 k8s-node02
192.168.40.113 k8s-node03
#2.下载epel-release
https://www.rpmfind.net/linux/epel/7/x86_64/Packages/e/epel-release-7-14.noarch.rpm
[root@k8s-etcd01 ~]# yum install epel-release-7-14.noarch.rpm -y
#3.下载CentOS-Base.repo
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
#4安装ansible、pyhton3
yum install -y ansible
yum install -y pyhton3
#5.确认是否有python3
[root@k8s-master01 ~]# ls /usr/bin/python3
/usr/bin/python32、配置高可用负载均衡
2.1 调度器配置Haproxy（主/备）
【192.168.40.109/110】两台都要配置
[root@k8s-haproxy01 ~]# yum -y install haproxy
[root@k8s-haproxy01 ~]# cp -rf /etc/haproxy/haproxy.cfg{,.bak} #备份
[root@k8s-haproxy01 ~]# sed -i -r '/^[ ]*#/d;/^$/d' /etc/haproxy/haproxy.cfg #修改配置文件去掉注释
[root@k8s-haproxy01 haproxy]# cat /etc/haproxy/haproxy.cfg
global
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 4000
user haproxy
group haproxy
daemon
stats socket /var/lib/haproxy/stats
defaults
mode http
log global
option httplog
option dontlognull
option http-server-close
option forwardfor except 127.0.0.0/8
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 3000
#添加
listen kubernetes_api_nodes_6443
mode tcp
bind 192.168.40.188:6443
server 192.168.40.101 192.168.40.101:6443 check inter 2000 rise 3 fall 5
server 192.168.40.102 192.168.40.102:6443 check inter 2000 rise 3 fall 5
server 192.168.40.103 192.168.40.103:6443 check inter 2000 rise 3 fall 5
listen stats
mode http
bind *:81
stats enable
stats uri /haproxy-status
stats auth haadmin:sheca
[root@k8s-haproxy01haproxy]# systemctl start haproxy
[root@k8s-haproxy01haproxy]# systemctl enable haproxy
[root@k8s-haproxy02 haproxy]# systemctl start haproxy
[root@k8s-haproxy02 haproxy]# systemctl enable haproxy2.2 配置Keepalived-VIP
【192.168.40.109】
[root@k8s-haproxy01 ~]# yum install -y keepalived
[root@k8s-haproxy01 ~]# cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak #备份
[root@k8s-haproxy01 ~]# vim /etc/keepalived/keepalived.conf
[root@k8s-haproxy01 haproxy]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id directory1 #只是名字而已，辅节点改为directory2（两个名字一定不能一样）
}
vrrp_instance VI_1 {
state MASTER #定义主还是备，备用的话写backup
interface ens33 #VIP绑定接口
virtual_router_id 80 #整个集群的调度器一致（在同一个集群）
priority 100 #（优先权）back改为50（50一间隔）
advert_int 1 #发包
authentication {
auth_type PASS #主备节点认证
auth_pass 1111
}
virtual_ipaddress {
192.168.40.188/24 dev ens33 label ens33:0 #VIP（自己网段的）
}
}
[root@haproxy1~]# systemctl start keepalived
[root@haproxy1~]# systemctl enable keepalived
【192.168.40.110】
[root@haproxy2 ~]# yum install -y keepalived
[root@haproxy2 ~]# mv /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
[root@haproxy2 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id directory2
}
vrrp_instance VI_1 {
state BACKUP
interface ens33
nopreempt #设置到back上面，不抢占资源
virtual_router_id 80
priority 50 #辅助改为50
advert_int 1 #检测间隔1s
authentication {
auth_type PASS
auth_pass 1111 #认证类型和密码主备一样，要不然无法互相认证
}
virtual_ipaddress {
192.168.40.188/24 dev ens33 label ens33:0 #抢占的VIP也一样
}
}
[root@haproxy2~]# systemctl start keepalived
[root@haproxy2~]# systemctl enable keepalived
#查看是否正常
[root@k8s-haproxy01 haproxy]# ifconfig -a
ens33:0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.40.188 netmask 255.255.255.0 broadcast 0.0.0.0
ether 00:0c:29:ee:de:eb txqueuelen 1000 (Ethernet)3.配置Harbor-san签发证书
k8s-harbor【192.168.40.104】
参考：https://www.cnblogs.com/birkhoffxia/articles/17153652.html
本地配置域名解析：
192.168.40.104 harbor.sheca.com
#浏览器访问
访问地址：https://harbor.sheca.com
账户密码：admin/Harbor12345
3.1 部署节点安装docker并同步harbor crt证书
[root@k8s-deploy ~]# cd /usr/local/src/
#1.上传docker压缩包
[root@k8s-deploy src]# tar zxvf docker-20.10.17-binary-install.tar.gz
#2.执行脚本
[root@k8s-deploy src]# ./docker-install.sh
[root@k8s-deploy src]# mkdir /etc/docker/certs.d/harbor.sheca.com -p
#3.将k8s-harbor 客户端证书复制过来
[root@k8s-harbor ~]# cd /apps/harbor/certs/
[root@k8s-harbor certs]# scp sheca.com.crt 192.168.40.147:/etc/docker/certs.d/harbor.sheca.com
#4.配置hosts
[root@k8s-deploy harbor.sheca.com]# vim /etc/hosts
192.168.40.104 harbor.sheca.com
[root@k8s-deploy harbor.sheca.com]# systemctl restart docker
#6.客户端测试登录 Harbor
[root@k8s-deploy harbor.sheca.com]# docker login harbor.sheca.com
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
#7. 测试推送镜像到仓库中
[root@k8s-deploy harbor.sheca.com]# docker pull alpine
[root@k8s-deploy harbor.sheca.com]# docker tag alpine harbor.sheca.com/baseimages/alpine:latest
[root@k8s-deploy harbor.sheca.com]# docker push harbor.sheca.com/baseimages/alpine:latest
The push refers to repository [harbor.sheca.com/baseimages/alpine]
8d3ac3489996: Pushed
latest: digest: sha256:e7d88de73db3d3fd9b2d63aa7f447a10fd0220b7cbf39803c803f2af9ba256b3 size: 528
4.Kubeasz部署高可用Kubernetes集群
4.1 准备k8s-deploy 管理节点进行配置 使用ansible进行部署
#1.下载epel-release
https://www.rpmfind.net/linux/epel/7/x86_64/Packages/e/epel-release-7-14.noarch.rpm
[root@k8s-deploy ~]# rpm -ivh epel-release-7-14.noarch.rpm
#2.下载CentOS-Base.repo
[root@k8s-deploy ~]# wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
#3.安装ansible、python3
yum install -y ansible
yum install -y python3
#4.确认是否有python3
[root@k8s-master01 ~]# ls /usr/bin/python3
/usr/bin/python3
#5.生成密钥对
ssh-keygen
#6.安装sshpass命令用于同步公钥到各k8s服务器
yum -y install sshpass
#7.配置脚本
[root@master1-admin k8s-cluster-kubeasz]# cat key.sh
#192.168.40.103、192.168.40.113 暂时没有的话 后续可横向扩展
#!/bin/bash
#目标主机列表
IP="
192.168.40.101
192.168.40.102
192.168.40.103
192.168.40.106
192.168.40.107
192.168.40.108
192.168.40.111
192.168.40.112
192.168.40.113"
for node in ${IP};do
sshpass -p 123456 ssh-copy-id ${node} -o StrictHostKeyChecking=no
echo ${node} "密钥copy完成"
ssh ${node} ln -sv /usr/bin/python3 /usr/bin/python
echo "${node} /usr/bin/python3 软连接创建完成"
done
[root@master1-admin k8s-cluster-kubeasz]# bash key.sh
#8.找台机器机器进行测试是否可以访问
ssh 192.168.40.1114.2 下载kubeasz项目源码、二进制及离线镜像
#Github地址：https://github.com/easzlab/kubeasz/blob/master/docs/setup/00-planning_and_overall_intro.md
#1.下载工具脚本ezdown，举例使用kubeasz版本3.5.0
#或者提前下载好 3.3.1
export release=3.5.0
wget https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown
chmod +x ./ezdown
或者下载脚本上传到服务器-此实验使用*
https://github.com/easzlab/kubeasz/releases/download/3.3.1/ezdown
# 2.国内环境，执行脚本进行下载所需要的镜像 大约10分钟根据网速
./ezdown -D
#3.会自动启动一个registry:2 镜像
[root@k8s-deploy k8s-easzlab]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
a3759b03ab7c registry:2 "/entrypoint.sh /etc…" 10 minutes ago Up 10 minutes registry
#4.下载路径在 /etc/kubeasz
[root@k8s-deploy kubeasz]# ll
total 100
-rw-rw-r--. 1 root root 20304 Jul 3 2022 ansible.cfg
drwxr-xr-x. 3 root root 4096 Nov 15 09:41 bin
drwxrwxr-x. 8 root root 94 Jul 3 2022 docs
drwxr-xr-x. 2 root root 284 Nov 15 09:47 down
drwxrwxr-x. 2 root root 70 Jul 3 2022 example
-rwxrwxr-x. 1 root root 25012 Jul 3 2022 ezctl
-rwxrwxr-x. 1 root root 25266 Jul 3 2022 ezdown
drwxrwxr-x. 10 root root 145 Jul 3 2022 manifests
drwxrwxr-x. 2 root root 4096 Jul 3 2022 pics
drwxrwxr-x. 2 root root 4096 Jul 3 2022 playbooks
-rw-rw-r--. 1 root root 5058 Jul 3 2022 README.md
drwxrwxr-x. 22 root root 4096 Jul 3 2022 roles
drwxrwxr-x. 2 root root 48 Jul 3 2022 tools4.3 创建集群配置实例
#1.根据提示配置'/etc/kubeasz/clusters/k8s-01/hosts' 和 '/etc/kubeasz/clusters/k8s-01/config.yml'
#根据前面节点规划修改hosts 文件和其他集群层面的主要配置选项；其他集群组件等配置项可以在config.yml 文件中修改
[root@k8s-deploy kubeasz]# ./ezctl new k8s-cluster-kubeasz
2023-11-15 09:53:36 DEBUG generate custom cluster files in /etc/kubeasz/clusters/k8s-cluster-kubeasz
2023-11-15 09:53:36 DEBUG set versions
2023-11-15 09:53:36 DEBUG cluster k8s-cluster-kubeasz: files successfully created.
2023-11-15 09:53:36 INFO next steps 1: to config '/etc/kubeasz/clusters/k8s-cluster-kubeasz/hosts'
2023-11-15 09:53:36 INFO next steps 2: to config '/etc/kubeasz/clusters/k8s-cluster-kubeasz/config.yml'
#2.创建的配置文件放在/etc/kubeasz/clusters/k8s-cluster-kubeasz下2个文件
[root@k8s-deploy kubeasz]# ll /etc/kubeasz/clusters/k8s-cluster-kubeasz/
total 12
-rw-r--r--. 1 root root 6311 Nov 15 09:53 config.yml
-rw-r--r--. 1 root root 1751 Nov 15 09:53 hosts
====================================================================================
#3.配置hosts文件 此注销192.168.40.103、#192.168.40.113 为了后续 手动增加节点使用
[root@master1-admin k8s-cluster-kubeasz]# cat hosts
# 'etcd' cluster should have odd member(s) (1,3,5,...)
[etcd]
192.168.40.106
192.168.40.107
192.168.40.108
# master node(s)
[kube_master]
192.168.40.101
192.168.40.102
#192.168.40.103
# work node(s)
[kube_node]
192.168.40.111
192.168.40.112
#192.168.40.113
# [optional] harbor server, a private docker registry
# 'NEW_INSTALL': 'true' to install a harbor server; 'false' to integrate with existed one
[harbor]
#192.168.40.8 NEW_INSTALL=false
# [optional] loadbalance for accessing k8s from outside
[ex_lb]
#192.168.40.6 LB_ROLE=backup EX_APISERVER_VIP=192.168.40.250 EX_APISERVER_PORT=8443
#192.168.40.7 LB_ROLE=master EX_APISERVER_VIP=192.168.40.250 EX_APISERVER_PORT=8443
# [optional] ntp server for the cluster
[chrony]
#192.168.40.1
[all:vars]
# --------- Main Variables ---------------
# Secure port for apiservers
SECURE_PORT="6443"
# Cluster container-runtime supported: docker, containerd
# if k8s version >= 1.24, docker is not supported
CONTAINER_RUNTIME="containerd"
# Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn
CLUSTER_NETWORK="calico"
# Service proxy mode of kube-proxy: 'iptables' or 'ipvs'
PROXY_MODE="ipvs"
# K8S Service CIDR, not overlap with node(host) networking
SERVICE_CIDR="10.100.0.0/16"
# Cluster CIDR (Pod CIDR), not overlap with node(host) networking
CLUSTER_CIDR="10.200.0.0/16"
# NodePort Range
NODE_PORT_RANGE="30000-32767"
# Cluster DNS Domain
CLUSTER_DNS_DOMAIN="cluster.local"
# -------- Additional Variables (don't change the default value right now) ---
# Binaries Directory
#bin_dir="/opt/kube/bin"
#bin_dir="/usr/local/bin"
# Deploy Directory (kubeasz workspace)
base_dir="/etc/kubeasz"
# Directory for a specific cluster
cluster_dir="{{ base_dir }}/clusters/k8s-cluster-kubeasz"
# CA and other components cert/key Directory
ca_dir="/etc/kubernetes/ssl"
====================================================================================
#4.config.yml 配置
# 42行 可以改为自己harbor地址也可以不用改 为了更快的启动 有些服务器无法访问外网需要放到本地harbor上
SANDBOX_IMAGE: "easzlab.io.local:5000/easzlab/pause:3.7" （此实验使用原来的）*
#SANDBOX_IMAGE: "harbor.sheca.com/easzlab/pause:3.7"
# MASTER_CERT_HOSTS 配置VIP 或者 公网上的域名 否则之后无法访问
MASTER_CERT_HOSTS:
- "192.168.40.188"
- "api.xks.net"
#- "www.test.com"
# MAX_PODS
MAX_PODS: 500
# coredns 自动安装
dns_install: "no"
# ENABLE_LOCAL_DNS_CACHE 缓存关闭
ENABLE_LOCAL_DNS_CACHE: false
# metric server 自动安装
metricsserver_install: "no"
# dashboard 自动安装
dashboard_install: "no"
====================================================================================
#5.如果没有ex_lb chrony 进行取消 否则会超时报错
[root@k8s-deploy k8s-cluster-kubeasz]# cat /etc/kubeasz/playbooks/01.prepare.yml
# [optional] to synchronize system time of nodes with 'chrony'
- hosts:
- kube_master
- kube_node
- etcd
# - ex_lb
# - chrony
roles:
- { role: os-harden, when: "OS_HARDEN|bool" }
- { role: chrony, when: "groups['chrony']|length > 0" }
# to create CA, kubeconfig, kube-proxy.kubeconfig etc.
- hosts: localhost
roles:
- deploy
# prepare tasks for all nodes
- hosts:
- kube_master
- kube_node
- etcd
roles:
- prepare
====================================================================================
#6.以下是playbook 进行ansible配置
[root@k8s-deploy tasks]# ll /etc/kubeasz/roles/prepare/tasks/
total 20
-rw-rw-r--. 1 root root 1796 Jul 3 2022 centos.yml
-rw-rw-r--. 1 root root 1729 Jul 3 2022 common.yml
-rw-rw-r--. 1 root root 2559 Jul 3 2022 main.yml
-rw-rw-r--. 1 root root 3469 Jul 3 2022 offline.yml
-rw-rw-r--. 1 root root 1951 Jul 3 2022 ubuntu.yml
#7.全部进行安装或者分步安装，此实验使用分步安装
./ezctl setup k8s-cluster-kubeasz all
= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
#01 或者分步安装，具体使用 dk ezctl help setup 查看分步安装帮助信息
./ezctl setup k8s-cluster-kubeasz 01
= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
#02 部署etcd集群 /etc/kubeasz/roles/etcd/tasks/main.yml 这个文件进行配置任务
./ezctl setup k8s-cluster-kubeasz 02
#验证是否正常可以使用etcd
[root@k8s-etcd01 ~]# find / -name etcdctl
/opt/kube/bin/etcdctl
[root@k8s-etcd01 bin]# export NODE_IPS="192.168.40.106 192.168.40.107 192.168.40.108"
[root@k8s-etcd01 ~]#
for ip in ${NODE_IPS}; do \
ETCDCTL_API=3 \
/opt/kube/bin/etcdctl --endpoints=https://${ip}:2379 \
--cacert=/etc/kubernetes/ssl/ca.pem \
--cert=/etc/kubernetes/ssl/etcd.pem \
--key=/etc/kubernetes/ssl/etcd-key.pem \
endpoint health;
done
https://192.168.40.106:2379 is healthy: successfully committed proposal: took = 5.559316ms
https://192.168.40.107:2379 is healthy: successfully committed proposal: took = 6.123282ms
https://192.168.40.108:2379 is healthy: successfully committed proposal: took = 6.278146ms
= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
#03 部署运行时
master与node节点都要同时安装运行时(containerd或docker)，可以自行使用部署工具匹配安装、yum安装、或
自行使用二进制安装，因此此步骤为可选步骤!
#基础镜像可以先下载到本地进行仓库 如果可以连网就不需要修改
[root@k8s-master01 ~]# vim /etc/containerd/config.toml
sandbox_image = "easzlab.io.local:5000/easzlab/pause:3.7"
[root@k8s-deploy kubeasz]# grep SANDBOX_IMAGE ./clusters/* -R
./clusters/k8s-cluster-kubeasz/config.yml:SANDBOX_IMAGE: "easzlab.io.local:5000/easzlab/pause:3.7"
#修改为自己harbor地址 harbor.sheca.com/easzlab/pause:3.7
#并且需要配置/etc/containerd/config.toml 可信仓库
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor.sheca.com"]
endpoint = ["https://harbor.sheca.com"]
[plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.sheca.com".tls]
insecure_skip_verify = true
[plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.sheca.com".auth]
username = "admin"
password = "Harbor12345"
./ezctl setup k8s-cluster-kubeasz 03
# 修改自己阿里云docker加速器否则无法进行 下载镜像，
#"https://docker.mirrors.ustc.edu.cn", "http://hub-mirror.c.163.com"删除
vim /etc/containerd/config.toml
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
endpoint = ["https://qryj5zfu.mirror.aliyuncs.com"]
[root@k8s-master01 ~]# /opt/kube/bin/containerd -v
containerd github.com/containerd/containerd v1.6.4 212e8b6fa2f44b9c21b2798135fc6fb7c53efc16
= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
#04 部署master
./ezctl setup k8s-cluster-kubeasz 04
= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
#05 部署node
./ezctl setup k8s-cluster-kubeasz 05
#验证是否K8S起来
[root@k8s-master01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
192.168.40.101 Ready,SchedulingDisabled master 7m3s v1.24.2
192.168.40.102 Ready,SchedulingDisabled master 7m4s v1.24.2
192.168.40.111 Ready node 3m33s v1.24.2
192.168.40.112 Ready node 3m33s v1.24.2
= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
#06 部署网络插件calico
#确保能够上网 如果不能无网络 提前下载好镜像 然后传到本地harbor进行加载镜像
#这里有网
[root@k8s-deploy kubeasz]# vim ./clusters/k8s-cluster-kubeasz/config.yml
# ------------------------------------------- calico
# [calico]设置 CALICO_IPV4POOL_IPIP=“off”,可以提高网络性能，条件限制详见 docs/setup/calico.md
CALICO_IPV4POOL_IPIP: "Always"
# [calico]设置 calico-node使用的host IP，bgp邻居通过该地址建立，可手工指定也可以自动发现
IP_AUTODETECTION_METHOD: "can-reach={{ groups['kube_master'][0] }}"
# [calico]设置calico 网络 backend: brid, vxlan, none
CALICO_NETWORKING_BACKEND: "brid"
# [calico]设置calico 是否使用route reflectors
# 如果集群规模超过50个节点，建议启用该特性
CALICO_RR_ENABLED: false
# CALICO_RR_NODES 配置route reflectors的节点，如果未设置默认使用集群master节点
# CALICO_RR_NODES: ["192.168.1.1", "192.168.1.2"]
CALICO_RR_NODES: []
# [calico]更新支持calico 版本: [v3.3.x] [v3.4.x] [v3.8.x] [v3.15.x]
calico_ver: "v3.19.4"
# [calico]calico 主版本
calico_ver_main: "{{ calico_ver.split('.')[0] }}.{{ calico_ver.split('.')[1] }}"
#查看calico镜像：
[root@k8s-deploy kubeasz]# grep image roles/calico/templates/calico-v3.19.yaml.j2
image: easzlab.io.local:5000/calico/cni:{{ calico_ver }}
image: easzlab.io.local:5000/calico/pod2daemon-flexvol:{{ calico_ver }}
image: easzlab.io.local:5000/calico/node:{{ calico_ver }}
image: easzlab.io.local:5000/calico/kube-controllers:{{ calico_ver }}
#下载镜像
[root@k8s-master01 ~]# crictl pull easzlab.io.local:5000/calico/cni:v3.19.4
[root@k8s-master01 ~]# crictl pull easzlab.io.local:5000/calico/pod2daemon-flexvol:v3.19.4
[root@k8s-master01 ~]# crictl pull easzlab.io.local:5000/calico/node:v3.19.4
[root@k8s-master01 ~]# crictl pull easzlab.io.local:5000/calico/kube-controllers:v3.19.4
[root@k8s-master01 ~]# crictl images
IMAGE TAG IMAGE ID SIZE
easzlab.io.local:5000/calico/cni v3.19.4 84358b137f838 48.3MB
easzlab.io.local:5000/calico/kube-controllers v3.19.4 0db60d880d2db 25MB
easzlab.io.local:5000/calico/node v3.19.4 172a034f72979 56.3MB
easzlab.io.local:5000/calico/pod2daemon-flexvol v3.19.4 054ddbbe59755 8.59MB
easzlab.io.local:5000/easzlab/pause 3.7 221177c6082a8 309kB
./ezctl setup k8s-cluster-kubeasz 06
#验证calico
[root@k8s-master01 ~]# calicoctl node status
Calico process is running.
IPv4 BGP status
+----------------+-------------------+-------+----------+-------------+
| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO |
+----------------+-------------------+-------+----------+-------------+
| 192.168.40.102 | node-to-node mesh | up | 03:47:26 | Established |
| 192.168.40.112 | node-to-node mesh | up | 03:47:31 | Established |
| 192.168.40.111 | node-to-node mesh | up | 03:47:32 | Established |
+----------------+-------------------+-------+----------+-------------+
IPv6 BGP status
No IPv6 peers found.
#验证可用性
[root@k8s-master01 ~]# kubectl run net-tesing --image=alpine sleep 360000
[root@k8s-master01 ~]# kubectl run net-tesing-2 --image=alpine sleep 360000
[root@k8s-master01 ~]# kubectl get pods
NAME READY STATUS RESTARTS AGE
net-tesing-2 1/1 Running 0 17s
net-testing 1/1 Running 0 126m
[root@k8s-master01 ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
net-tesing-2 1/1 Running 0 4m34s 10.200.58.194 192.168.40.112 <none> <none>
net-testing 1/1 Running 0 130m 10.200.58.193 192.168.40.112 <none> <none>
[root@k8s-master01 ~]# kubectl exec -it net-tesing-2 sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/ # ping 10.200.58.193
PING 10.200.58.193 (10.200.58.193): 56 data bytes
64 bytes from 10.200.58.193: seq=0 ttl=63 time=0.101 ms
64 bytes from 10.200.58.193: seq=1 ttl=63 time=0.090 ms
64 bytes from 10.200.58.193: seq=2 ttl=63 time=0.077 ms
^C
--- 10.200.58.193 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 0.077/0.089/0.101 ms
/ # ping 10.200.58.194
PING 10.200.58.194 (10.200.58.194): 56 data bytes
64 bytes from 10.200.58.194: seq=0 ttl=64 time=0.038 ms
64 bytes from 10.200.58.194: seq=1 ttl=64 time=0.062 ms
64 bytes from 10.200.58.194: seq=2 ttl=64 time=0.057 ms
^C
--- 10.200.58.194 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 0.038/0.052/0.062 ms
#为了高可用 使用VIP进行查询
[root@k8s-deploy ~]# vim ~/.kube/config
server: https://192.168.40.188:6443
[root@k8s-deploy ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
192.168.40.101 Ready,SchedulingDisabled master 10d v1.24.2
192.168.40.102 Ready,SchedulingDisabled master 10d v1.24.2
192.168.40.111 Ready node 10d v1.24.2
192.168.40.112 Ready node 10d v1.24.2如果使用本地harbor需要配置/etc/containerd/config.toml
ezctl setup 加载过程
ezdown -D 下载过程
[root@master1-admin ~]# ./ezdown -D
2023-11-14 14:13:14 INFO Action begin: download_all
2023-11-14 14:13:14 INFO downloading docker binaries, version 20.10.16
--2023-11-14 14:13:14-- https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/static/stable/x86_64/docker-20.10.16.tgz
Resolving mirrors.tuna.tsinghua.edu.cn (mirrors.tuna.tsinghua.edu.cn)... 101.6.15.130, 2402:f000:1:400::2
Connecting to mirrors.tuna.tsinghua.edu.cn (mirrors.tuna.tsinghua.edu.cn)|101.6.15.130|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 64969189 (62M) [application/octet-stream]
Saving to: ‘docker-20.10.16.tgz’
100%[=======================================================================================================>] 64,969,189 59.3MB/s in 1.0s
2023-11-14 14:13:15 (59.3 MB/s) - ‘docker-20.10.16.tgz’ saved [64969189/64969189]
2023-11-14 14:13:17 DEBUG generate docker service file
2023-11-14 14:13:17 DEBUG generate docker config: /etc/docker/daemon.json
2023-11-14 14:13:17 DEBUG prepare register mirror for CN
2023-11-14 14:13:17 DEBUG turn off selinux in CentOS/Redhat
Disabled
2023-11-14 14:13:17 DEBUG enable and start docker
Created symlink from /etc/systemd/system/multi-user.target.wants/docker.service to /etc/systemd/system/docker.service.
2023-11-14 14:13:21 INFO downloading kubeasz: 3.3.1
2023-11-14 14:13:21 DEBUG run a temporary container
Unable to find image 'easzlab/kubeasz:3.3.1' locally
3.3.1: Pulling from easzlab/kubeasz
540db60ca938: Pull complete
d037ddac5dde: Pull complete
05d0edf52df4: Pull complete
54d94e388fb8: Pull complete
b25964b87dc1: Pull complete
aedfadb13329: Pull complete
8f6f8140f32b: Pull complete
Digest: sha256:c0cfc314c4caea45a7582a5e03b090901177c4c48210c3df8b209f5b03045f70
Status: Downloaded newer image for easzlab/kubeasz:3.3.1
c8edb1235f84892442c75509e414c9cb24e062783c0e679281cd55e4038da47c
2023-11-14 14:13:36 DEBUG cp kubeasz code from the temporary container
2023-11-14 14:13:36 DEBUG stop&remove temporary container
temp_easz
2023-11-14 14:13:36 INFO downloading kubernetes: v1.24.2 binaries
v1.24.2: Pulling from easzlab/kubeasz-k8s-bin
1b7ca6aea1dd: Pull complete
d2339c028cfd: Pull complete
Digest: sha256:1a41943faa18d7a69e243f4cd9b7b6f1cd7268be7c6358587170c3d3e9e1a34c
Status: Downloaded newer image for easzlab/kubeasz-k8s-bin:v1.24.2
docker.io/easzlab/kubeasz-k8s-bin:v1.24.2
2023-11-14 14:13:56 DEBUG run a temporary container
fcda7dcb33ee8ad7d4cb58cd133d1438cb4e4c4317ae9efcbd502dafd5f0b006
2023-11-14 14:13:56 DEBUG cp k8s binaries
2023-11-14 14:13:58 DEBUG stop&remove temporary container
temp_k8s_bin
2023-11-14 14:13:58 INFO downloading extral binaries kubeasz-ext-bin:1.2.0
1.2.0: Pulling from easzlab/kubeasz-ext-bin
1b7ca6aea1dd: Already exists
4a494a9b7425: Pull complete
b11479c0b3c6: Pull complete
0351e344774e: Pull complete
1c1e5d29db2d: Pull complete
Digest: sha256:a40f30978cca518503811db70ec7734b98ab4378a5c06546bf22de37900f252d
Status: Downloaded newer image for easzlab/kubeasz-ext-bin:1.2.0
docker.io/easzlab/kubeasz-ext-bin:1.2.0
2023-11-14 14:14:33 DEBUG run a temporary container
aa0396117e156f6751b940a0240fe6dcfade0b22810f1bd461648344479be25c
2023-11-14 14:14:34 DEBUG cp extral binaries
2023-11-14 14:14:35 DEBUG stop&remove temporary container
temp_ext_bin
2: Pulling from library/registry
96526aa774ef: Pull complete
834bccaa730c: Pull complete
87a69098c0a9: Pull complete
afc17120a9f7: Pull complete
e5ac04f3acf5: Pull complete
Digest: sha256:8a60daaa55ab0df4607c4d8625b96b97b06fd2e6ca8528275472963c4ae8afa0
Status: Downloaded newer image for registry:2
docker.io/library/registry:2
2023-11-14 14:14:48 INFO start local registry ...
dbf38aa75809182d0f45de1343dbd7ea19cb68952c677c794bb6dd8fa5504c66
2023-11-14 14:14:49 INFO download default images, then upload to the local registry
v3.19.4: Pulling from calico/cni
f3894d312a4e: Pull complete
8244094b678e: Pull complete
45b915a54b66: Pull complete
Digest: sha256:a866562105d3c18486879d313830d8b4918e8ba25ccd23b7dd84d65093d03c62
Status: Downloaded newer image for calico/cni:v3.19.4
docker.io/calico/cni:v3.19.4
v3.19.4: Pulling from calico/pod2daemon-flexvol
99aa522a8a66: Pull complete
beb35b03ed9b: Pull complete
8c61f8de6c67: Pull complete
622403455de3: Pull complete
a26eec45c530: Pull complete
b02e2914a61e: Pull complete
91f16e6ede78: Pull complete
Digest: sha256:d698fbda7a2e895ad45b478ab0b5fdd572cd80629e558dbfcf6e401c6ee6275e
Status: Downloaded newer image for calico/pod2daemon-flexvol:v3.19.4
docker.io/calico/pod2daemon-flexvol:v3.19.4
v3.19.4: Pulling from calico/kube-controllers
0a1506fb14ea: Pull complete
6abc1e849f8f: Pull complete
0cfea6002588: Pull complete
91d785239eb0: Pull complete
Digest: sha256:b15521e60d8bb04a501fe0ef4bf791fc8c164a175dd49a2328fb3f2b89838a68
Status: Downloaded newer image for calico/kube-controllers:v3.19.4
docker.io/calico/kube-controllers:v3.19.4
v3.19.4: Pulling from calico/node
7563b432e373: Pull complete
f1ad2d4094a4: Pull complete
Digest: sha256:df027832d91944516046f6baf3f6e74c5130046d2c56f88dc96296681771bc6a
Status: Downloaded newer image for calico/node:v3.19.4
docker.io/calico/node:v3.19.4
The push refers to repository [easzlab.io.local:5000/calico/cni]
e190560973d0: Pushed
237eb7dff52b: Pushed
7bdb7ca6a5a4: Pushed
v3.19.4: digest: sha256:9e1da653e987232cf18df3eb6967c9555a1235d212189b3e4c26f6f9d1601297 size: 946
The push refers to repository [easzlab.io.local:5000/calico/pod2daemon-flexvol]
0312eef4fc3a: Pushed
aeeffe0f6b8b: Pushed
672e236e33e9: Pushed
e5816bd252f3: Pushed
e29ee4bf6f3f: Pushed
9dd9977906c2: Pushed
cdc78476cc38: Pushed
v3.19.4: digest: sha256:152415638f6cc10fcbc2095069c5286df262c591422fb2608a14c7eee554c259 size: 1788
The push refers to repository [easzlab.io.local:5000/calico/kube-controllers]
568d0e1941e4: Pushed
7094539af214: Pushed
44bbcee30afb: Pushed
e47767779496: Pushed
v3.19.4: digest: sha256:214b5384028bac797ff16531d71d28f7d658ef3a26837db6bf5466bc5f113bfd size: 1155
The push refers to repository [easzlab.io.local:5000/calico/node]
f03078b73155: Pushed
14ec913b26f5: Pushed
v3.19.4: digest: sha256:393ff601623e04e685add605920e6c984a1ac74e23cc4232cec7f5013ba8caad size: 737
1.9.3: Pulling from coredns/coredns
d92bdee79785: Pull complete
f2401d57212f: Pull complete
Digest: sha256:8e352a029d304ca7431c6507b56800636c321cb52289686a581ab70aaa8a2e2a
Status: Downloaded newer image for coredns/coredns:1.9.3
docker.io/coredns/coredns:1.9.3
The push refers to repository [easzlab.io.local:5000/coredns/coredns]
df1818f16337: Pushed
256bc5c338a6: Pushed
1.9.3: digest: sha256:bdb36ee882c13135669cfc2bb91c808a33926ad1a411fee07bd2dc344bb8f782 size: 739
1.21.1: Pulling from easzlab/k8s-dns-node-cache
20b09fbd3037: Pull complete
af833073aa95: Pull complete
Digest: sha256:04c4f6b1f2f2f72441dadcea1c8eec611af4d963315187ceb04b939d1956782f
Status: Downloaded newer image for easzlab/k8s-dns-node-cache:1.21.1
docker.io/easzlab/k8s-dns-node-cache:1.21.1
The push refers to repository [easzlab.io.local:5000/easzlab/k8s-dns-node-cache]
8391095a8344: Pushed
87b6a930c8d0: Pushed
1.21.1: digest: sha256:04c4f6b1f2f2f72441dadcea1c8eec611af4d963315187ceb04b939d1956782f size: 741
v2.5.1: Pulling from kubernetesui/dashboard
d1d01ae59b08: Pull complete
a25bff2a339f: Pull complete
Digest: sha256:cc746e7a0b1eec0db01cbabbb6386b23d7af97e79fa9e36bb883a95b7eb96fe2
Status: Downloaded newer image for kubernetesui/dashboard:v2.5.1
docker.io/kubernetesui/dashboard:v2.5.1
The push refers to repository [easzlab.io.local:5000/kubernetesui/dashboard]
e98b3744f758: Pushed
dab46c9f5775: Pushed
v2.5.1: digest: sha256:0c82e96241aa683fe2f8fbdf43530e22863ac8bfaddb0d7d30b4e3a639d4e8c5 size: 736
v1.0.8: Pulling from kubernetesui/metrics-scraper
978be80e3ee3: Pull complete
5866d2c04d96: Pull complete
Digest: sha256:76049887f07a0476dc93efc2d3569b9529bf982b22d29f356092ce206e98765c
Status: Downloaded newer image for kubernetesui/metrics-scraper:v1.0.8
docker.io/kubernetesui/metrics-scraper:v1.0.8
The push refers to repository [easzlab.io.local:5000/kubernetesui/metrics-scraper]
bcec7eb9e567: Pushed
d01384fea991: Pushed
v1.0.8: digest: sha256:43227e8286fd379ee0415a5e2156a9439c4056807e3caa38e1dd413b0644807a size: 736
v0.5.2: Pulling from easzlab/metrics-server
e8614d09b7be: Pull complete
334ef31a5c43: Pull complete
Digest: sha256:b25c885ad398f4a1d82fa188160bd8bb9c6caad86069eb433ae923dad7ae78e2
Status: Downloaded newer image for easzlab/metrics-server:v0.5.2
docker.io/easzlab/metrics-server:v0.5.2
The push refers to repository [easzlab.io.local:5000/easzlab/metrics-server]
b2839a50be1a: Pushed
6d75f23be3dd: Pushed
v0.5.2: digest: sha256:bf6d56415a5a86e6e02b1ef9798f94da698da78b0727a1f0d4395e600c611b70 size: 739
3.7: Pulling from easzlab/pause
7582c2cc65ef: Pull complete
Digest: sha256:445a99db22e9add9bfb15ddb1980861a329e5dff5c88d7eec9cbf08b6b2f4eb1
Status: Downloaded newer image for easzlab/pause:3.7
docker.io/easzlab/pause:3.7
The push refers to repository [easzlab.io.local:5000/easzlab/pause]
1cb555415fd3: Pushed
3.7: digest: sha256:445a99db22e9add9bfb15ddb1980861a329e5dff5c88d7eec9cbf08b6b2f4eb1 size: 526
3.3.1: Pulling from easzlab/kubeasz
Digest: sha256:c0cfc314c4caea45a7582a5e03b090901177c4c48210c3df8b209f5b03045f70
Status: Image is up to date for easzlab/kubeasz:3.3.1
docker.io/easzlab/kubeasz:3.3.1
2023-11-14 14:20:14 INFO Action successed: download_allezctl setup k8s-cluster-kubeasz 01加载过程
[root@k8s-deploy kubeasz]# ./ezctl setup k8s-cluster-kubeasz 01
ansible-playbook -i clusters/k8s-cluster-kubeasz/hosts -e @clusters/k8s-cluster-kubeasz/config.yml playbooks/01.prepare.yml
2023-11-15 10:09:45 INFO cluster:k8s-cluster-kubeasz setup step:01 begins in 5s, press any key to abort:
PLAY [kube_master,kube_node,etcd] ********************************************************************************************************************************
TASK [Gathering Facts] *******************************************************************************************************************************************
ok: [192.168.40.112]
ok: [192.168.40.111]
ok: [192.168.40.106]
ok: [192.168.40.101]
ok: [192.168.40.107]
ok: [192.168.40.108]
ok: [192.168.40.102]
PLAY [localhost] *************************************************************************************************************************************************
TASK [Gathering Facts] *******************************************************************************************************************************************
ok: [localhost]
TASK [deploy : prepare some dirs] ********************************************************************************************************************************
changed: [localhost] => (item=/etc/kubeasz/clusters/k8s-cluster-kubeasz/ssl)
changed: [localhost] => (item=/etc/kubeasz/clusters/k8s-cluster-kubeasz/backup)
changed: [localhost] => (item=/etc/kubeasz/clusters/k8s-cluster-kubeasz/yml)
changed: [localhost] => (item=~/.kube)
TASK [deploy : 本地设置 bin 目录权限] ************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 读取ca证书stat信息] *************************************************************************************************************************************
ok: [localhost]
TASK [deploy : 准备CA配置文件和签名请求] ************************************************************************************************************************************
changed: [localhost] => (item=ca-config.json)
changed: [localhost] => (item=ca-csr.json)
TASK [deploy : 生成 CA 证书和私钥] **************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 准备kubectl使用的admin证书签名请求] **************************************************************************************************************************
changed: [localhost]
TASK [deploy : 创建admin证书与私钥] *************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置集群参数] *******************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置客户端认证参数] ****************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置上下文参数] ******************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 选择默认上下文] ******************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 安装kubeconfig] *************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 准备kube-proxy 证书签名请求] ******************************************************************************************************************************
changed: [localhost]
TASK [deploy : 创建 kube-proxy证书与私钥] *******************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置集群参数] *******************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置客户端认证参数] ****************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置上下文参数] ******************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 选择默认上下文] ******************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 准备kube-controller-manager 证书签名请求] *****************************************************************************************************************
changed: [localhost]
TASK [deploy : 创建 kube-controller-manager证书与私钥] ******************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置集群参数] *******************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置认证参数] *******************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置上下文参数] ******************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 选择默认上下文] ******************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 准备kube-scheduler 证书签名请求] **************************************************************************************************************************
changed: [localhost]
TASK [deploy : 创建 kube-scheduler证书与私钥] ***************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置集群参数] *******************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置认证参数] *******************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置上下文参数] ******************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 选择默认上下文] ******************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 本地创建 ezdown/ezctl 工具的软连接] *************************************************************************************************************************
changed: [localhost] => (item=ezdown)
changed: [localhost] => (item=ezctl)
TASK [deploy : ansible 控制端创建 kubectl 软链接] ************************************************************************************************************************
changed: [localhost]
PLAY [kube_master,kube_node,etcd] ********************************************************************************************************************************
TASK [prepare : 删除centos/redhat默认安装] *****************************************************************************************************************************
changed: [192.168.40.102] => (item=firewalld)
changed: [192.168.40.101] => (item=firewalld)
changed: [192.168.40.111] => (item=firewalld)
changed: [192.168.40.112] => (item=firewalld)
changed: [192.168.40.106] => (item=firewalld)
changed: [192.168.40.101] => (item=python-firewall)
changed: [192.168.40.112] => (item=python-firewall)
changed: [192.168.40.111] => (item=python-firewall)
changed: [192.168.40.102] => (item=python-firewall)
changed: [192.168.40.106] => (item=python-firewall)
changed: [192.168.40.101] => (item=firewalld-filesystem)
changed: [192.168.40.112] => (item=firewalld-filesystem)
changed: [192.168.40.102] => (item=firewalld-filesystem)
changed: [192.168.40.111] => (item=firewalld-filesystem)
changed: [192.168.40.106] => (item=firewalld-filesystem)
changed: [192.168.40.107] => (item=firewalld)
changed: [192.168.40.108] => (item=firewalld)
changed: [192.168.40.107] => (item=python-firewall)
changed: [192.168.40.108] => (item=python-firewall)
changed: [192.168.40.107] => (item=firewalld-filesystem)
changed: [192.168.40.108] => (item=firewalld-filesystem)
TASK [prepare : 安装基础软件包] *****************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.106]
changed: [192.168.40.112]
changed: [192.168.40.111]
changed: [192.168.40.108]
changed: [192.168.40.107]
TASK [prepare : 临时关闭 selinux] ************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 永久关闭 selinux] ************************************************************************************************************************************
ok: [192.168.40.101]
ok: [192.168.40.106]
ok: [192.168.40.112]
ok: [192.168.40.111]
ok: [192.168.40.102]
ok: [192.168.40.107]
ok: [192.168.40.108]
TASK [prepare : 禁止rsyslog获取journald日志1] **************************************************************************************************************************
[WARNING]: Module remote_tmp /root/.ansible/tmp did not exist and was created with a mode of 0700, this may cause issues when running as another user. To avoid
this, create the remote_tmp dir with the correct permissions manually
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 禁止rsyslog获取journald日志2] **************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 重启rsyslog服务] *************************************************************************************************************************************
changed: [192.168.40.102]
changed: [192.168.40.101]
changed: [192.168.40.111]
changed: [192.168.40.112]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 禁用系统 swap] ***************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 删除fstab swap 相关配置] *******************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 转换内核版本为浮点数] **************************************************************************************************************************************
ok: [192.168.40.101]
ok: [192.168.40.102]
ok: [192.168.40.111]
ok: [192.168.40.112]
ok: [192.168.40.106]
ok: [192.168.40.107]
ok: [192.168.40.108]
TASK [prepare : 加载内核模块] ******************************************************************************************************************************************
changed: [192.168.40.112] => (item=br_netfilter)
changed: [192.168.40.101] => (item=br_netfilter)
changed: [192.168.40.111] => (item=br_netfilter)
changed: [192.168.40.102] => (item=br_netfilter)
changed: [192.168.40.106] => (item=br_netfilter)
changed: [192.168.40.112] => (item=ip_vs)
changed: [192.168.40.101] => (item=ip_vs)
changed: [192.168.40.111] => (item=ip_vs)
changed: [192.168.40.102] => (item=ip_vs)
changed: [192.168.40.106] => (item=ip_vs)
changed: [192.168.40.112] => (item=ip_vs_rr)
changed: [192.168.40.101] => (item=ip_vs_rr)
changed: [192.168.40.102] => (item=ip_vs_rr)
changed: [192.168.40.106] => (item=ip_vs_rr)
changed: [192.168.40.111] => (item=ip_vs_rr)
changed: [192.168.40.112] => (item=ip_vs_wrr)
changed: [192.168.40.101] => (item=ip_vs_wrr)
changed: [192.168.40.111] => (item=ip_vs_wrr)
changed: [192.168.40.102] => (item=ip_vs_wrr)
changed: [192.168.40.106] => (item=ip_vs_wrr)
changed: [192.168.40.101] => (item=ip_vs_sh)
changed: [192.168.40.112] => (item=ip_vs_sh)
changed: [192.168.40.111] => (item=ip_vs_sh)
changed: [192.168.40.102] => (item=ip_vs_sh)
changed: [192.168.40.106] => (item=ip_vs_sh)
ok: [192.168.40.101] => (item=nf_conntrack)
ok: [192.168.40.112] => (item=nf_conntrack)
ok: [192.168.40.111] => (item=nf_conntrack)
ok: [192.168.40.106] => (item=nf_conntrack)
ok: [192.168.40.102] => (item=nf_conntrack)
changed: [192.168.40.107] => (item=br_netfilter)
changed: [192.168.40.108] => (item=br_netfilter)
changed: [192.168.40.107] => (item=ip_vs)
changed: [192.168.40.108] => (item=ip_vs)
changed: [192.168.40.107] => (item=ip_vs_rr)
changed: [192.168.40.108] => (item=ip_vs_rr)
changed: [192.168.40.107] => (item=ip_vs_wrr)
changed: [192.168.40.108] => (item=ip_vs_wrr)
changed: [192.168.40.107] => (item=ip_vs_sh)
changed: [192.168.40.108] => (item=ip_vs_sh)
ok: [192.168.40.107] => (item=nf_conntrack)
ok: [192.168.40.108] => (item=nf_conntrack)
TASK [prepare : 尝试加载nf_conntrack_ipv4] ***************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 启用systemd自动加载模块服务] *******************************************************************************************************************************
ok: [192.168.40.101]
ok: [192.168.40.102]
ok: [192.168.40.111]
ok: [192.168.40.112]
ok: [192.168.40.106]
ok: [192.168.40.107]
ok: [192.168.40.108]
TASK [prepare : 增加内核模块开机加载配置] ************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 设置系统参数] ******************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.112]
changed: [192.168.40.111]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 生效系统参数] ******************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 创建 systemd 配置目录] *********************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 设置系统 ulimits] ************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.106]
changed: [192.168.40.112]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 把SCTP列入内核模块黑名单] **********************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare some dirs] *****************************************************************************************************************************************
changed: [192.168.40.101] => (item=/opt/kube/bin)
changed: [192.168.40.102] => (item=/opt/kube/bin)
changed: [192.168.40.111] => (item=/opt/kube/bin)
changed: [192.168.40.112] => (item=/opt/kube/bin)
changed: [192.168.40.106] => (item=/opt/kube/bin)
changed: [192.168.40.101] => (item=/etc/kubernetes/ssl)
changed: [192.168.40.102] => (item=/etc/kubernetes/ssl)
changed: [192.168.40.111] => (item=/etc/kubernetes/ssl)
changed: [192.168.40.112] => (item=/etc/kubernetes/ssl)
changed: [192.168.40.106] => (item=/etc/kubernetes/ssl)
changed: [192.168.40.101] => (item=/root/.kube)
changed: [192.168.40.102] => (item=/root/.kube)
changed: [192.168.40.111] => (item=/root/.kube)
changed: [192.168.40.112] => (item=/root/.kube)
changed: [192.168.40.106] => (item=/root/.kube)
changed: [192.168.40.101] => (item=/etc/cni/net.d)
changed: [192.168.40.102] => (item=/etc/cni/net.d)
changed: [192.168.40.111] => (item=/etc/cni/net.d)
changed: [192.168.40.112] => (item=/etc/cni/net.d)
changed: [192.168.40.106] => (item=/etc/cni/net.d)
changed: [192.168.40.107] => (item=/opt/kube/bin)
changed: [192.168.40.108] => (item=/opt/kube/bin)
changed: [192.168.40.107] => (item=/etc/kubernetes/ssl)
changed: [192.168.40.108] => (item=/etc/kubernetes/ssl)
changed: [192.168.40.107] => (item=/root/.kube)
changed: [192.168.40.108] => (item=/root/.kube)
changed: [192.168.40.107] => (item=/etc/cni/net.d)
changed: [192.168.40.108] => (item=/etc/cni/net.d)
TASK [prepare : symlink /usr/bin/python -> /usr/bin/python3] *****************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 写入环境变量$PATH] *************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 添加 kubectl 自动补全] *********************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 添加 local registry hosts 解析] **********************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [prepare : 分发 kubeconfig配置文件] *******************************************************************************************************************************
changed: [192.168.40.102]
changed: [192.168.40.101]
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [prepare : 分发 kube-proxy.kubeconfig配置文件] ********************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [prepare : 分发controller/scheduler kubeconfig配置文件] ***********************************************************************************************************
changed: [192.168.40.101] => (item=kube-controller-manager.kubeconfig)
changed: [192.168.40.102] => (item=kube-controller-manager.kubeconfig)
changed: [192.168.40.101] => (item=kube-scheduler.kubeconfig)
changed: [192.168.40.102] => (item=kube-scheduler.kubeconfig)
PLAY RECAP *******************************************************************************************************************************************************
192.168.40.101 : ok=28 changed=24 unreachable=0 failed=0 skipped=112 rescued=0 ignored=0
192.168.40.102 : ok=28 changed=24 unreachable=0 failed=0 skipped=112 rescued=0 ignored=0
192.168.40.106 : ok=25 changed=21 unreachable=0 failed=0 skipped=115 rescued=0 ignored=0
192.168.40.107 : ok=25 changed=21 unreachable=0 failed=0 skipped=115 rescued=0 ignored=0
192.168.40.108 : ok=25 changed=21 unreachable=0 failed=0 skipped=115 rescued=0 ignored=0
192.168.40.111 : ok=27 changed=23 unreachable=0 failed=0 skipped=113 rescued=0 ignored=0
192.168.40.112 : ok=27 changed=23 unreachable=0 failed=0 skipped=113 rescued=0 ignored=0
localhost : ok=33 changed=31 unreachable=0 failed=0 skipped=11 rescued=0 ignored=002
[root@k8s-deploy kubeasz]# ./ezctl help setup
Usage: ezctl setup <cluster> <step>
available steps:
01 prepare to prepare CA/certs & kubeconfig & other system settings
02 etcd to setup the etcd cluster
03 container-runtime to setup the container runtime(docker or containerd)
04 kube-master to setup the master nodes
05 kube-node to setup the worker nodes
06 network to setup the network plugin
07 cluster-addon to setup other useful plugins
90 all to run 01~07 all at once
10 ex-lb to install external loadbalance for accessing k8s from outside
11 harbor to install a new harbor server or to integrate with an existed one
examples: ./ezctl setup test-k8s 01 (or ./ezctl setup test-k8s prepare)
./ezctl setup test-k8s 02 (or ./ezctl setup test-k8s etcd)
./ezctl setup test-k8s all
./ezctl setup test-k8s 04 -t restart_master
[root@k8s-deploy kubeasz]# ./ezctl setup k8s-cluster-kubeasz 02
ansible-playbook -i clusters/k8s-cluster-kubeasz/hosts -e @clusters/k8s-cluster-kubeasz/config.yml playbooks/02.etcd.yml
2023-11-15 10:20:50 INFO cluster:k8s-cluster-kubeasz setup step:02 begins in 5s, press any key to abort:
PLAY [etcd] ******************************************************************************************************************************************************
TASK [Gathering Facts] *******************************************************************************************************************************************
ok: [192.168.40.107]
ok: [192.168.40.108]
ok: [192.168.40.106]
TASK [etcd : prepare some dirs] **********************************************************************************************************************************
changed: [192.168.40.106]
changed: [192.168.40.108]
changed: [192.168.40.107]
TASK [下载etcd二进制文件] ***********************************************************************************************************************************************
changed: [192.168.40.108] => (item=etcd)
changed: [192.168.40.107] => (item=etcd)
changed: [192.168.40.106] => (item=etcd)
changed: [192.168.40.107] => (item=etcdctl)
changed: [192.168.40.108] => (item=etcdctl)
changed: [192.168.40.106] => (item=etcdctl)
TASK [创建etcd证书请求] ************************************************************************************************************************************************
changed: [192.168.40.106]
ok: [192.168.40.107]
ok: [192.168.40.108]
TASK [创建 etcd证书和私钥] **********************************************************************************************************************************************
changed: [192.168.40.108]
changed: [192.168.40.107]
changed: [192.168.40.106]
TASK [分发etcd证书相关] ************************************************************************************************************************************************
changed: [192.168.40.106] => (item=ca.pem)
changed: [192.168.40.107] => (item=ca.pem)
changed: [192.168.40.108] => (item=ca.pem)
changed: [192.168.40.106] => (item=etcd.pem)
changed: [192.168.40.107] => (item=etcd.pem)
changed: [192.168.40.108] => (item=etcd.pem)
changed: [192.168.40.106] => (item=etcd-key.pem)
changed: [192.168.40.107] => (item=etcd-key.pem)
changed: [192.168.40.108] => (item=etcd-key.pem)
TASK [创建etcd的systemd unit文件] *************************************************************************************************************************************
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [开机启用etcd服务] ************************************************************************************************************************************************
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
TASK [开启etcd服务] **************************************************************************************************************************************************
changed: [192.168.40.106]
changed: [192.168.40.108]
changed: [192.168.40.107]
TASK [etcd : 以轮询的方式等待服务同步完成] *************************************************************************************************************************************
changed: [192.168.40.106]
changed: [192.168.40.107]
changed: [192.168.40.108]
PLAY RECAP *******************************************************************************************************************************************************
192.168.40.106 : ok=10 changed=9 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
192.168.40.107 : ok=10 changed=8 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
192.168.40.108 : ok=10 changed=8 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0ezctl setup k8s-cluster-kubeasz 03加载过程
[root@k8s-deploy kubeasz]# ./ezctl setup k8s-cluster-kubeasz 03
ansible-playbook -i clusters/k8s-cluster-kubeasz/hosts -e @clusters/k8s-cluster-kubeasz/config.yml playbooks/03.runtime.yml
2023-11-15 10:43:15 INFO cluster:k8s-cluster-kubeasz setup step:03 begins in 5s, press any key to abort:
PLAY [kube_master,kube_node] *************************************************************************************************************************************
TASK [Gathering Facts] *******************************************************************************************************************************************
ok: [192.168.40.102]
ok: [192.168.40.112]
ok: [192.168.40.101]
ok: [192.168.40.111]
TASK [获取是否已经安装containerd] ****************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [准备containerd相关目录] ******************************************************************************************************************************************
ok: [192.168.40.101] => (item=/opt/kube/bin)
ok: [192.168.40.102] => (item=/opt/kube/bin)
ok: [192.168.40.111] => (item=/opt/kube/bin)
ok: [192.168.40.112] => (item=/opt/kube/bin)
changed: [192.168.40.102] => (item=/etc/containerd)
changed: [192.168.40.101] => (item=/etc/containerd)
changed: [192.168.40.112] => (item=/etc/containerd)
changed: [192.168.40.111] => (item=/etc/containerd)
TASK [containerd : 加载内核模块 overlay] *******************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.112]
changed: [192.168.40.111]
TASK [下载 containerd 二进制文件] ***************************************************************************************************************************************
changed: [192.168.40.102] => (item=containerd)
changed: [192.168.40.111] => (item=containerd)
changed: [192.168.40.101] => (item=containerd)
changed: [192.168.40.112] => (item=containerd)
changed: [192.168.40.102] => (item=containerd-shim)
changed: [192.168.40.111] => (item=containerd-shim)
changed: [192.168.40.101] => (item=containerd-shim)
changed: [192.168.40.112] => (item=containerd-shim)
changed: [192.168.40.102] => (item=containerd-shim-runc-v1)
changed: [192.168.40.111] => (item=containerd-shim-runc-v1)
changed: [192.168.40.101] => (item=containerd-shim-runc-v1)
changed: [192.168.40.112] => (item=containerd-shim-runc-v1)
changed: [192.168.40.102] => (item=containerd-shim-runc-v2)
changed: [192.168.40.111] => (item=containerd-shim-runc-v2)
changed: [192.168.40.101] => (item=containerd-shim-runc-v2)
changed: [192.168.40.112] => (item=containerd-shim-runc-v2)
changed: [192.168.40.101] => (item=crictl)
changed: [192.168.40.111] => (item=crictl)
changed: [192.168.40.102] => (item=crictl)
changed: [192.168.40.112] => (item=crictl)
changed: [192.168.40.101] => (item=ctr)
changed: [192.168.40.111] => (item=ctr)
changed: [192.168.40.102] => (item=ctr)
changed: [192.168.40.112] => (item=ctr)
changed: [192.168.40.101] => (item=runc)
changed: [192.168.40.111] => (item=runc)
changed: [192.168.40.102] => (item=runc)
changed: [192.168.40.112] => (item=runc)
TASK [创建 containerd 配置文件] ****************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [containerd : 创建systemd unit文件] *****************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [containerd : 创建 crictl 配置] *********************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [开机启用 containerd 服务] ****************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [开启 containerd 服务] ******************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.111]
changed: [192.168.40.102]
changed: [192.168.40.112]
TASK [轮询等待containerd服务运行] ****************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.111]
changed: [192.168.40.102]
changed: [192.168.40.112]
PLAY RECAP *******************************************************************************************************************************************************
192.168.40.101 : ok=11 changed=10 unreachable=0 failed=0 skipped=18 rescued=0 ignored=0
192.168.40.102 : ok=11 changed=10 unreachable=0 failed=0 skipped=15 rescued=0 ignored=0
192.168.40.111 : ok=11 changed=10 unreachable=0 failed=0 skipped=15 rescued=0 ignored=0
192.168.40.112 : ok=11 changed=10 unreachable=0 failed=0 skipped=15 rescued=0 ignored=0ezctl setup k8s-cluster-kubeasz 04加载过程
[root@k8s-deploy kubeasz]# ./ezctl setup k8s-cluster-kubeasz 04
ansible-playbook -i clusters/k8s-cluster-kubeasz/hosts -e @clusters/k8s-cluster-kubeasz/config.yml playbooks/04.kube-master.yml
2023-11-15 11:26:55 INFO cluster:k8s-cluster-kubeasz setup step:04 begins in 5s, press any key to abort:
PLAY [kube_master] ***********************************************************************************************************************************************
TASK [Gathering Facts] *******************************************************************************************************************************************
ok: [192.168.40.101]
ok: [192.168.40.102]
TASK [kube-lb : prepare some dirs] *******************************************************************************************************************************
changed: [192.168.40.102] => (item=/etc/kube-lb/sbin)
changed: [192.168.40.101] => (item=/etc/kube-lb/sbin)
changed: [192.168.40.101] => (item=/etc/kube-lb/logs)
changed: [192.168.40.102] => (item=/etc/kube-lb/logs)
changed: [192.168.40.102] => (item=/etc/kube-lb/conf)
changed: [192.168.40.101] => (item=/etc/kube-lb/conf)
TASK [下载二进制文件kube-lb(nginx)] *************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [创建kube-lb的配置文件] ********************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [创建kube-lb的systemd unit文件] **********************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [开机启用kube-lb服务] *********************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [开启kube-lb服务] ***********************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [以轮询的方式等待kube-lb服务启动] ***************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-master : 下载 kube_master 二进制] **************************************************************************************************************************
changed: [192.168.40.101] => (item=kube-apiserver)
changed: [192.168.40.102] => (item=kube-apiserver)
changed: [192.168.40.102] => (item=kube-controller-manager)
changed: [192.168.40.101] => (item=kube-controller-manager)
changed: [192.168.40.102] => (item=kube-scheduler)
changed: [192.168.40.101] => (item=kube-scheduler)
changed: [192.168.40.101] => (item=kubectl)
changed: [192.168.40.102] => (item=kubectl)
TASK [kube-master : 注册变量 KUBERNETES_SVC_IP] **********************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-master : 设置变量 CLUSTER_KUBERNETES_SVC_IP] **************************************************************************************************************
ok: [192.168.40.101]
ok: [192.168.40.102]
TASK [kube-master : 创建 kubernetes 证书签名请求] ************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-master : 创建 kubernetes 证书和私钥] *************************************************************************************************************************
changed: [192.168.40.102]
changed: [192.168.40.101]
TASK [kube-master : 创建 aggregator proxy证书签名请求] *******************************************************************************************************************
changed: [192.168.40.101]
ok: [192.168.40.102]
TASK [kube-master : 创建 aggregator-proxy证书和私钥] ********************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-master : 分发 kubernetes证书] *****************************************************************************************************************************
changed: [192.168.40.101] => (item=ca.pem)
changed: [192.168.40.102] => (item=ca.pem)
changed: [192.168.40.102] => (item=ca-key.pem)
changed: [192.168.40.101] => (item=ca-key.pem)
changed: [192.168.40.102] => (item=kubernetes.pem)
changed: [192.168.40.101] => (item=kubernetes.pem)
changed: [192.168.40.101] => (item=kubernetes-key.pem)
changed: [192.168.40.102] => (item=kubernetes-key.pem)
changed: [192.168.40.101] => (item=aggregator-proxy.pem)
changed: [192.168.40.102] => (item=aggregator-proxy.pem)
changed: [192.168.40.102] => (item=aggregator-proxy-key.pem)
changed: [192.168.40.101] => (item=aggregator-proxy-key.pem)
TASK [kube-master : 替换 kubeconfig 的 apiserver 地址] ****************************************************************************************************************
changed: [192.168.40.102] => (item=/root/.kube/config)
changed: [192.168.40.101] => (item=/root/.kube/config)
changed: [192.168.40.101] => (item=/etc/kubernetes/kube-controller-manager.kubeconfig)
changed: [192.168.40.102] => (item=/etc/kubernetes/kube-controller-manager.kubeconfig)
changed: [192.168.40.102] => (item=/etc/kubernetes/kube-scheduler.kubeconfig)
changed: [192.168.40.101] => (item=/etc/kubernetes/kube-scheduler.kubeconfig)
TASK [kube-master : 创建 master 服务的 systemd unit 文件] ***************************************************************************************************************
changed: [192.168.40.101] => (item=kube-apiserver.service)
changed: [192.168.40.102] => (item=kube-apiserver.service)
changed: [192.168.40.101] => (item=kube-controller-manager.service)
changed: [192.168.40.102] => (item=kube-controller-manager.service)
changed: [192.168.40.101] => (item=kube-scheduler.service)
changed: [192.168.40.102] => (item=kube-scheduler.service)
TASK [kube-master : enable master 服务] ****************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-master : 启动 master 服务] ********************************************************************************************************************************
changed: [192.168.40.102]
changed: [192.168.40.101]
TASK [kube-master : 轮询等待kube-apiserver启动] ************************************************************************************************************************
changed: [192.168.40.102]
changed: [192.168.40.101]
TASK [kube-master : 轮询等待kube-controller-manager启动] ***************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-master : 轮询等待kube-scheduler启动] ************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-master : 以轮询的方式等待master服务启动完成] ************************************************************************************************************************
changed: [192.168.40.102]
changed: [192.168.40.101]
TASK [kube-master : 获取user:kubernetes是否已经绑定对应角色] *****************************************************************************************************************
changed: [192.168.40.101]
TASK [kube-master : 创建user:kubernetes角色绑定] ***********************************************************************************************************************
changed: [192.168.40.101]
TASK [kube-node : 创建kube_node 相关目录] ******************************************************************************************************************************
changed: [192.168.40.101] => (item=/var/lib/kubelet)
changed: [192.168.40.102] => (item=/var/lib/kubelet)
changed: [192.168.40.101] => (item=/var/lib/kube-proxy)
changed: [192.168.40.102] => (item=/var/lib/kube-proxy)
ok: [192.168.40.101] => (item=/etc/cni/net.d)
ok: [192.168.40.102] => (item=/etc/cni/net.d)
TASK [kube-node : 下载 kubelet,kube-proxy 二进制和基础 cni plugins] ******************************************************************************************************
ok: [192.168.40.101] => (item=kubectl)
ok: [192.168.40.102] => (item=kubectl)
changed: [192.168.40.102] => (item=kubelet)
changed: [192.168.40.101] => (item=kubelet)
changed: [192.168.40.101] => (item=kube-proxy)
changed: [192.168.40.102] => (item=kube-proxy)
changed: [192.168.40.101] => (item=bridge)
changed: [192.168.40.102] => (item=bridge)
changed: [192.168.40.101] => (item=host-local)
changed: [192.168.40.102] => (item=host-local)
changed: [192.168.40.101] => (item=loopback)
changed: [192.168.40.102] => (item=loopback)
TASK [kube-node : 替换 kubeconfig 的 apiserver 地址] ******************************************************************************************************************
ok: [192.168.40.101]
ok: [192.168.40.102]
TASK [kube-node : 准备kubelet 证书签名请求] ******************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 创建 kubelet 证书与私钥] ******************************************************************************************************************************
changed: [192.168.40.102]
changed: [192.168.40.101]
TASK [kube-node : 分发ca 证书] ***************************************************************************************************************************************
ok: [192.168.40.101]
ok: [192.168.40.102]
TASK [kube-node : 分发kubelet 证书] **********************************************************************************************************************************
changed: [192.168.40.101] => (item=kubelet.pem)
changed: [192.168.40.102] => (item=kubelet.pem)
changed: [192.168.40.101] => (item=kubelet-key.pem)
changed: [192.168.40.102] => (item=kubelet-key.pem)
TASK [kube-node : 设置集群参数] ****************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 设置客户端认证参数] *************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 设置上下文参数] ***************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 选择默认上下文] ***************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 准备 cni配置文件] ************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 注册变量 DNS_SVC_IP] *******************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 设置变量 CLUSTER_DNS_SVC_IP] ***********************************************************************************************************************
ok: [192.168.40.101]
ok: [192.168.40.102]
TASK [kube-node : 创建kubelet的配置文件] ********************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 创建kubelet的systemd unit文件] **********************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 开机启用kubelet 服务] ********************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 开启kubelet 服务] **********************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 替换 kube-proxy.kubeconfig 的 apiserver 地址] *******************************************************************************************************
changed: [192.168.40.102]
changed: [192.168.40.101]
TASK [kube-node : 创建kube-proxy 配置] *******************************************************************************************************************************
changed: [192.168.40.102]
changed: [192.168.40.101]
TASK [kube-node : 创建kube-proxy 服务文件] *****************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 开机启用kube-proxy 服务] *****************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 开启kube-proxy 服务] *******************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 轮询等待kube-proxy启动] ******************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 轮询等待kubelet启动] *********************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
FAILED - RETRYING: 轮询等待node达到Ready状态 (8 retries left).
FAILED - RETRYING: 轮询等待node达到Ready状态 (8 retries left).
TASK [kube-node : 轮询等待node达到Ready状态] *****************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [kube-node : 设置node节点role] **********************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [Making master nodes SchedulingDisabled] ********************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
TASK [Setting master role name] **********************************************************************************************************************************
changed: [192.168.40.102]
changed: [192.168.40.101]
PLAY RECAP *******************************************************************************************************************************************************
192.168.40.101 : ok=55 changed=50 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
192.168.40.102 : ok=53 changed=47 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0ezctl setup k8s-cluster-kubeasz 05 加载过程
[root@k8s-deploy kubeasz]# cat echo.txt
ansible-playbook -i clusters/k8s-cluster-kubeasz/hosts -e @clusters/k8s-cluster-kubeasz/config.yml playbooks/05.kube-node.yml
2023-11-15 11:30:46 INFO cluster:k8s-cluster-kubeasz setup step:05 begins in 5s, press any key to abort:
PLAY [kube_node] *****************************************************************************************************************************************************************************
TASK [Gathering Facts] ***********************************************************************************************************************************************************************
ok: [192.168.40.111]
ok: [192.168.40.112]
TASK [kube-lb : prepare some dirs] ***********************************************************************************************************************************************************
changed: [192.168.40.112] => (item=/etc/kube-lb/sbin)
changed: [192.168.40.111] => (item=/etc/kube-lb/sbin)
changed: [192.168.40.112] => (item=/etc/kube-lb/logs)
changed: [192.168.40.111] => (item=/etc/kube-lb/logs)
changed: [192.168.40.112] => (item=/etc/kube-lb/conf)
changed: [192.168.40.111] => (item=/etc/kube-lb/conf)
TASK [下载二进制文件kube-lb(nginx)] *****************************************************************************************************************************************************************
changed: [192.168.40.112]
changed: [192.168.40.111]
TASK [创建kube-lb的配置文件] ************************************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [创建kube-lb的systemd unit文件] **************************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [开机启用kube-lb服务] *************************************************************************************************************************************************************************
changed: [192.168.40.112]
changed: [192.168.40.111]
TASK [开启kube-lb服务] ***************************************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [以轮询的方式等待kube-lb服务启动] *******************************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 创建kube_node 相关目录] **********************************************************************************************************************************************************
changed: [192.168.40.111] => (item=/var/lib/kubelet)
changed: [192.168.40.112] => (item=/var/lib/kubelet)
changed: [192.168.40.111] => (item=/var/lib/kube-proxy)
changed: [192.168.40.112] => (item=/var/lib/kube-proxy)
ok: [192.168.40.111] => (item=/etc/cni/net.d)
ok: [192.168.40.112] => (item=/etc/cni/net.d)
TASK [kube-node : 下载 kubelet,kube-proxy 二进制和基础 cni plugins] **********************************************************************************************************************************
changed: [192.168.40.111] => (item=kubectl)
changed: [192.168.40.112] => (item=kubectl)
changed: [192.168.40.112] => (item=kubelet)
changed: [192.168.40.111] => (item=kubelet)
changed: [192.168.40.112] => (item=kube-proxy)
changed: [192.168.40.111] => (item=kube-proxy)
changed: [192.168.40.112] => (item=bridge)
changed: [192.168.40.111] => (item=bridge)
changed: [192.168.40.112] => (item=host-local)
changed: [192.168.40.111] => (item=host-local)
changed: [192.168.40.112] => (item=loopback)
changed: [192.168.40.111] => (item=loopback)
TASK [kube-node : 替换 kubeconfig 的 apiserver 地址] **********************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 准备kubelet 证书签名请求] **********************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 创建 kubelet 证书与私钥] **********************************************************************************************************************************************************
changed: [192.168.40.112]
changed: [192.168.40.111]
TASK [kube-node : 分发ca 证书] *******************************************************************************************************************************************************************
changed: [192.168.40.112]
changed: [192.168.40.111]
TASK [kube-node : 分发kubelet 证书] **************************************************************************************************************************************************************
changed: [192.168.40.111] => (item=kubelet.pem)
changed: [192.168.40.112] => (item=kubelet.pem)
changed: [192.168.40.111] => (item=kubelet-key.pem)
changed: [192.168.40.112] => (item=kubelet-key.pem)
TASK [kube-node : 设置集群参数] ********************************************************************************************************************************************************************
changed: [192.168.40.112]
changed: [192.168.40.111]
TASK [kube-node : 设置客户端认证参数] *****************************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 设置上下文参数] *******************************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 选择默认上下文] *******************************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 准备 cni配置文件] ****************************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 注册变量 DNS_SVC_IP] ***********************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 设置变量 CLUSTER_DNS_SVC_IP] ***************************************************************************************************************************************************
ok: [192.168.40.111]
ok: [192.168.40.112]
TASK [kube-node : 创建kubelet的配置文件] ************************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 创建kubelet的systemd unit文件] **************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 开机启用kubelet 服务] ************************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 开启kubelet 服务] **************************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 替换 kube-proxy.kubeconfig 的 apiserver 地址] ***********************************************************************************************************************************
changed: [192.168.40.112]
changed: [192.168.40.111]
TASK [kube-node : 创建kube-proxy 配置] ***********************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 创建kube-proxy 服务文件] *********************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 开机启用kube-proxy 服务] *********************************************************************************************************************************************************
changed: [192.168.40.112]
changed: [192.168.40.111]
TASK [kube-node : 开启kube-proxy 服务] ***********************************************************************************************************************************************************
changed: [192.168.40.112]
changed: [192.168.40.111]
TASK [kube-node : 轮询等待kube-proxy启动] **********************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 轮询等待kubelet启动] *************************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [kube-node : 轮询等待node达到Ready状态] *********************************************************************************************************************************************************
changed: [192.168.40.111]
FAILED - RETRYING: 轮询等待node达到Ready状态 (8 retries left).
FAILED - RETRYING: 轮询等待node达到Ready状态 (7 retries left).
changed: [192.168.40.112]
TASK [kube-node : 设置node节点role] **************************************************************************************************************************************************************
changed: [192.168.40.111]
changed: [192.168.40.112]
PLAY RECAP ***********************************************************************************************************************************************************************************
192.168.40.111 : ok=35 changed=33 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
192.168.40.112 : ok=35 changed=33 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0ezctl setup k8s-cluster-kubeasz 06过程加载
[root@k8s-deploy kubeasz]# ./ezctl setup k8s-cluster-kubeasz 06
ansible-playbook -i clusters/k8s-cluster-kubeasz/hosts -e @clusters/k8s-cluster-kubeasz/config.yml playbooks/06.network.yml
2023-11-15 11:47:06 INFO cluster:k8s-cluster-kubeasz setup step:06 begins in 5s, press any key to abort:
PLAY [kube_master,kube_node] *************************************************************************************************************************************
TASK [Gathering Facts] *******************************************************************************************************************************************
ok: [192.168.40.111]
ok: [192.168.40.112]
ok: [192.168.40.101]
ok: [192.168.40.102]
TASK [calico : 在节点创建相关目录] ****************************************************************************************************************************************
changed: [192.168.40.101] => (item=/etc/calico/ssl)
changed: [192.168.40.111] => (item=/etc/calico/ssl)
changed: [192.168.40.112] => (item=/etc/calico/ssl)
changed: [192.168.40.102] => (item=/etc/calico/ssl)
TASK [创建calico 证书请求] *********************************************************************************************************************************************
changed: [192.168.40.101]
ok: [192.168.40.111]
ok: [192.168.40.112]
ok: [192.168.40.102]
TASK [创建 calico证书和私钥] ********************************************************************************************************************************************
changed: [192.168.40.112]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.101]
TASK [分发calico证书相关] **********************************************************************************************************************************************
changed: [192.168.40.101] => (item=ca.pem)
changed: [192.168.40.102] => (item=ca.pem)
changed: [192.168.40.111] => (item=ca.pem)
changed: [192.168.40.112] => (item=ca.pem)
changed: [192.168.40.101] => (item=calico.pem)
changed: [192.168.40.112] => (item=calico.pem)
changed: [192.168.40.102] => (item=calico.pem)
changed: [192.168.40.111] => (item=calico.pem)
changed: [192.168.40.102] => (item=calico-key.pem)
changed: [192.168.40.112] => (item=calico-key.pem)
changed: [192.168.40.101] => (item=calico-key.pem)
changed: [192.168.40.111] => (item=calico-key.pem)
TASK [get calico-etcd-secrets info] ******************************************************************************************************************************
changed: [192.168.40.101]
TASK [创建 calico-etcd-secrets] ************************************************************************************************************************************
changed: [192.168.40.101]
TASK [配置 calico DaemonSet yaml文件] ********************************************************************************************************************************
changed: [192.168.40.101]
TASK [运行 calico网络] ***********************************************************************************************************************************************
changed: [192.168.40.101]
TASK [calico : 删除默认cni配置] ****************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
TASK [下载calicoctl 客户端] *******************************************************************************************************************************************
changed: [192.168.40.102] => (item=calicoctl)
changed: [192.168.40.101] => (item=calicoctl)
changed: [192.168.40.111] => (item=calicoctl)
changed: [192.168.40.112] => (item=calicoctl)
TASK [准备 calicoctl配置文件] ******************************************************************************************************************************************
changed: [192.168.40.101]
changed: [192.168.40.102]
changed: [192.168.40.111]
changed: [192.168.40.112]
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (15 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (15 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (15 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行，视下载镜像速度而定 (15 retries left).
TASK [轮询等待calico-node 运行，视下载镜像速度而定] ******************************************************************************************************************************
changed: [192.168.40.102]
changed: [192.168.40.101]
changed: [192.168.40.112]
changed: [192.168.40.111]
PLAY RECAP *******************************************************************************************************************************************************
192.168.40.101 : ok=13 changed=12 unreachable=0 failed=0 skipped=36 rescued=0 ignored=0
192.168.40.102 : ok=9 changed=7 unreachable=0 failed=0 skipped=22 rescued=0 ignored=0
192.168.40.111 : ok=9 changed=7 unreachable=0 failed=0 skipped=22 rescued=0 ignored=0
192.168.40.112 : ok=9 changed=7 unreachable=0 failed=0 skipped=22 rescued=0 ignored=05. 增加master和node
#kube-lb.conf这个文件是 nginx配置文件 127.0.0.1:6443 转发到两台master上 如果在加一台会自动加入一台 进行分发
[root@k8s-master01 kubernetes]# cat /etc/kube-lb/conf/kube-lb.conf
user root;
worker_processes 1;
error_log /etc/kube-lb/logs/error.log warn;
events {
worker_connections 3000;
}
stream {
upstream backend {
server 192.168.40.101:6443 max_fails=2 fail_timeout=3s;
server 192.168.40.102:6443 max_fails=2 fail_timeout=3s;
}
server {
listen 127.0.0.1:6443;
proxy_connect_timeout 1s;
proxy_pass backend;
}
}
#此实验没有做
#添加node节点 192.168.40.113
[root@k8s-deploy kubeasz]# ./ezctl add-node k8s-cluster-kubeasz 192.168.40.113
#添加master节点 192.168.40.103
[root@k8s-deploy kubeasz]# ./ezctl add-master k8s-cluster-kubeasz 192.168.40.1036.升级版本kubernetes containerd
下载文件:https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.24.md
https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.24.md#downloads-for-v1243
上传文件
解压文件
停用master所有服务kube-controller-manager、kubectl、kubelet 、kube-proxy 、kube-scheduler
部署节点复制所有二进制kube-controller-manager、kubectl、kubelet 、kube-proxy 、kube-scheduler到master上
重新拉起所有服务kube-controller-manager、kubectl、kubelet 、kube-proxy 、kube-scheduler
#
升级master：
#systemctl stop kube-apiserver kube-controller-manager kube-scheduler kube-proxy kubelet
# scp kube-apiserver kube-controller-manager kube-scheduler kube-proxy kubelet kubectl 172.31.7.101:/usr/local/bin/
# systemctl start kube-apiserver kube-controller-manager kube-scheduler kube-proxy kubelet
升级node：
# systemctl stop kubelet kube-proxy.service
# scp kubelet kube-proxy kubectl 172.31.7.111:/usr/local/bin
# systemctl start kubelet kube-proxy.service
# kubectl uncordon 172.31.7.111
# \cp kube-apiserver kube-controller-manager kube-scheduler kube-proxy kubelet kubectl /etc/kubeasz/bin/
6.1 containerd
升级containerd-正常情况下，先驱逐pod，然后将服务停止或重启服务器，然后替换二进制再启动服务：
https://github.com/containerd/containerd/releases/download/v1.6.6/containerd-1.6.6-linux-amd64.tar.gz
\cp ./* /etc/kubeasz/bin/containerd-bin/
systemctl disable kubelet kube-proxy containerd
reboot
# systemctl enable kubelet kube-proxy containerd
# systemctl start kubelet kube-proxy containerd7. DNS 服务
目前常用的dns组件有kube-dns和coredns两个，即到目前k8s版本 1.17.X都可以使用,kube-dns和coredns用于解析k8s集群中service name所对应得到IP地址。
https://console.cloud.google.com/gcr/images/google-containers/GLOBAL #google的镜像仓库地址
k8s 1.18版本以后将不再支持kube-dns。
https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#downloads-for-v1180
kubeadm: kube-dns is deprecated and will not be supported in a future version
skyDNS/kube-dns/coreDNS
kube-dns: 提供service name域名的解析
dns-dnsmasg: 提供DNS缓存，降低kubedns负载，提高性能
dns-sidecar: 定期检查kubedns和dnsmasq的健康状态7.1 部署coredns
https://github.com/coredns/coredns
https://coredns.io/
https://github.com/coredns/deployment/tree/master/kubernetes #部署清单文件
#1.9.4版本
https://github.com/coredns/deployment/blob/master/kubernetes/coredns.yaml.sed
浙公网安备 33010602011771号