Rocky系统二进制部署k8s1.22.4版本
一、基础环境
1、firewalld配置
# 所有master节点开放相关防火墙端口 firewall-cmd --zone=public --add-port=179/tcp --permanent firewall-cmd --zone=public --add-port=16443/tcp --permanent firewall-cmd --zone=public --add-port=6443/tcp --permanent firewall-cmd --zone=public --add-port=2379-2380/tcp --permanent firewall-cmd --zone=public --add-port=10250/tcp --permanent firewall-cmd --zone=public --add-port=10251/tcp --permanent firewall-cmd --zone=public --add-port=10252/tcp --permanent firewall-cmd --zone=public --add-port=30000-32767/tcp --permanent firewall-cmd --permanent --add-rich-rule="rule family="ipv4" source address="10.96.0.0/16" port protocol="tcp" port="1-65535" accept" firewall-cmd --permanent --add-rich-rule="rule family="ipv4" source address="10.97.0.0/16" port protocol="tcp" port="1-65535" accept" firewall-cmd --add-masquerade --permanent firewall-cmd --reload firewall-cmd --list-all --zone=public # 所有master节点必须开启firewalld该设置，否则dns无法解释 firewall-cmd --add-masquerade --permanent firewall-cmd --reload firewall-cmd --list-all --zone=public ######################## # worker节点防火墙设置 ######################## # 所有worker节点开放相关防火墙端口 firewall-cmd --zone=public --add-port=179/tcp --permanent firewall-cmd --zone=public --add-port=10250/tcp --permanent firewall-cmd --zone=public --add-port=30000-32767/tcp --permanent firewall-cmd --permanent --add-rich-rule="rule family="ipv4" source address="10.97.0.0/16" port protocol="tcp" port="1-65535" accept" firewall-cmd --permanent --add-rich-rule="rule family="ipv4" source address="10.96.0.0/16" port protocol="tcp" port="1-65535" accept" firewall-cmd --add-masquerade --permanent firewall-cmd --reload firewall-cmd --list-all --zone=public # 所有worker节点必须开启firewalld该设置，否则dns无法解释 firewall-cmd --add-masquerade --permanent firewall-cmd --reload firewall-cmd --list-all --zone=public # 有iptables时候执行下边指令 # 所有节点清除iptables规则，解决firewalld引起nodeport无法访问问题 iptables -D INPUT -j REJECT --reject-with icmp-host-prohibited # 所有节点设置root的crontab，每十分钟设置一次 echo '5,15,25,35,45,55 * * * * /usr/sbin/iptables -D INPUT -j REJECT --reject-with icmp-host-prohibited' >> /var/spool/cron/root && crontab -l
2、基础配置
# 设置时间同步 yum install chrony -y timedatectl set-timezone Asia/Shanghai # 启用NTP时间同步 timedatectl set-ntp yes # 硬件时间默认为UTC： timedatectl set-local-rtc 1 # 校准时间服务器： chronyc tracking #关闭selinux，swap sed -i 's/enforcing/disabled/' /etc/selinux/config setenforce 0 swapoff -a sed -ri 's/.*swap.*/#&/' /etc/fstab #系统优化 cat > /etc/sysctl.d/k8s_better.conf << EOF net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 net.ipv4.ip_forward=1 net.ipv4.tcp_tw_recycle=0 vm.swappiness=0 vm.overcommit_memory=1 vm.panic_on_oom=0 fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=1048576 fs.file-max=52706963 fs.nr_open=52706963 net.ipv6.conf.all.disable_ipv6=1 net.netfilter.nf_conntrack_max=2310720 EOF sysctl -p /etc/sysctl.d/k8s_better.conf #修改hosts文件 cat >> /etc/hosts << "EOF" 10.20.1.44 master-20 10.30.1.44 master-30 172.168.0.44 master-00 10.20.1.100 node0-20 EOF #更改主机名，根据实际情况修改，分别在每台操作 hostnamectl --static set-hostname master-20 hostnamectl --static set-hostname master-30 hostnamectl --static set-hostname master-00 hostnamectl --static set-hostname node0-20 #配置免密登录，在master01上操作 ssh-keygen -t rsa ssh-copy-id root@10.20.1.100 ssh-copy-id root@172.168.0.44 #重启 reboot
3、配置keepalived、haproxy
# 所有节点安装 yum -y install ipvsadm ipset sysstat conntrack libseccomp # 各节点开启ipvs模块 cat > /etc/sysconfig/modules/ipvs.modules <<EOF #!/bin/sh modprobe -- ip_vs modprobe -- ip_vs_rr modprobe -- ip_vs_wrr modprobe -- ip_vs_sh modprobe -- nf_conntrack EOF chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4 # 所有master节点安装haproxy和keepalived服务 yum -y install haproxy keepalived # 修改master1节点keepalived配置文件 [root@master1 ~]# cat /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { router_id LVS_DEVEL # 添加如下内容 script_user root enable_script_security } vrrp_script check_haproxy { script "/etc/keepalived/check_haproxy.sh" # 检测脚本路径 interval 3 weight -2 fall 10 rise 2 } vrrp_instance VI_1 { state MASTER # MASTER interface eno1 # 本机网卡名 virtual_router_id 51 priority 100 # 权重100 advert_int 1 authentication { auth_type PASS auth_pass 1111 } virtual_ipaddress { 10.20.1.99 # 虚拟IP } track_script { check_haproxy # 模块 } } # 修改master2节点keepalived配置文件 [root@master2 ~]# cat /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { router_id LVS_DEVEL # 添加如下内容 script_user root enable_script_security } vrrp_script check_haproxy { script "/etc/keepalived/check_haproxy.sh" # 检测脚本路径 interval 3 weight -2 fall 10 rise 2 } vrrp_instance VI_1 { state BACKUP # BACKUP interface eno1 # 本机网卡名 virtual_router_id 51 priority 99 # 权重99 advert_int 1 authentication { auth_type PASS auth_pass 1111 } virtual_ipaddress { 10.20.1.99 # 虚拟IP } track_script { check_haproxy # 模块 } } # 修改master3节点keepalived配置文件 [root@master3 ~]# cat /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { router_id LVS_DEVEL # 添加如下内容 script_user root enable_script_security } vrrp_script check_haproxy { script "/etc/keepalived/check_haproxy.sh" # 检测脚本路径 interval 3 weight -2 fall 10 rise 2 } vrrp_instance VI_1 { state BACKUP # BACKUP interface enp5s0 # 本机网卡名 virtual_router_id 51 priority 98 # 权重98 advert_int 1 authentication { auth_type PASS auth_pass 1111 } virtual_ipaddress { 10.20.1.99 # 虚拟IP } track_script { check_haproxy # 模块 } } # 三台master节点haproxy配置都一样 [root@master1 ~]# cat /etc/haproxy/haproxy.cfg #--------------------------------------------------------------------- # Example configuration for a possible web application. See the # full configuration options online. # # http://haproxy.1wt.eu/download/1.4/doc/configuration.txt # #--------------------------------------------------------------------- #--------------------------------------------------------------------- # Global settings #--------------------------------------------------------------------- global # to have these messages end up in /var/log/haproxy.log you will # need to: # # 1) configure syslog to accept network log events. This is done # by adding the '-r' option to the SYSLOGD_OPTIONS in # /etc/sysconfig/syslog # # 2) configure local2 events to go to the /var/log/haproxy.log # file. A line like the following can be added to # /etc/sysconfig/syslog # # local2.* /var/log/haproxy.log # log 127.0.0.1 local2 chroot /var/lib/haproxy pidfile /var/run/haproxy.pid maxconn 4000 user haproxy group haproxy daemon # turn on stats unix socket stats socket /var/lib/haproxy/stats #--------------------------------------------------------------------- # common defaults that all the 'listen' and 'backend' sections will # use if not designated in their block #--------------------------------------------------------------------- defaults mode http log global option httplog option dontlognull option http-server-close option forwardfor except 127.0.0.0/8 option redispatch retries 3 timeout http-request 10s timeout queue 1m timeout connect 10s timeout client 1m timeout server 1m timeout http-keep-alive 10s timeout check 10s maxconn 3000 #--------------------------------------------------------------------- # main frontend which proxys to the backends #--------------------------------------------------------------------- frontend kubernetes-apiserver mode tcp bind *:16443 option tcplog default_backend kubernetes-apiserver #--------------------------------------------------------------------- # static backend for serving up images, stylesheets and such #--------------------------------------------------------------------- listen stats bind *:1080 stats auth admin:awesomePassword stats refresh 5s stats realm HAProxy\ Statistics stats uri /admin?stats #--------------------------------------------------------------------- # round robin balancing between the various backends #--------------------------------------------------------------------- backend kubernetes-apiserver mode tcp balance roundrobin server master1 10.20.1.44:6443 check server master2 10.30.1.44:6443 check server master3 172.168.0.44:6443 check # 每台master节点编写健康监测脚本 [root@master1 ~]# cat > /etc/keepalived/check_haproxy.sh <<EOF #!/bin/sh # HAPROXY down A=`ps -C haproxy --no-header | wc -l` if [ $A -eq 0 ] then systmectl start haproxy if [ ps -C haproxy --no-header | wc -l -eq 0 ] then killall -9 haproxy echo "HAPROXY down" | mail -s "haproxy" sleep 3600 fi fi EOF systemctl start keepalived systemctl enable keepalived systemctl enable haproxy systemctl start haproxy # 查看虚拟ip是否生效 ip a
三、部署etcd
下载地址：https://github.com/etcd-io/etcd/releases/download/v3.5.1/etcd-v3.5.1-linux-amd64.tar.gz
我当前使用的是3.5.0版本
3.1颁发证书
除了master，在其他机器上操作
mkdir -p /opt/cluster
在master上操作
1.创建目录
mkdir -p /opt/cluster/ssl/{rootca,etcd,kubernetes} mkdir -p /opt/cluster/kubelet/ssl mkdir -p /opt/cluster/log/{kube-apiserver,kube-controller-manager,kube-scheduler,kube-proxy,kubelet} mkdir -p /opt/cluster/plugins/{calico,coredns} mkdir -p /opt/cluster/etcd/{data,wal} mkdir ~/tools
2.上传cfssl证书签证工具到~/tools
cd ~/tools mv cfssl_linux-amd64 cfssl mv cfssl-certinfo_linux-amd64 cfssl-certinfo mv cfssljson_linux-amd64 cfssljson chmod +x cfssl* cp cfssl* /usr/local/bin
3.生成证书
cd /opt/cluster/ssl cat > cfssl-conf.json << EOF { "signing": { "default": { "expiry": "87600h" }, "profiles": { "common": { "usages": [ "signing", "key encipherment", "server auth", "client auth" ], "expiry": "87600h" } } } } EOF cd /opt/cluster/ssl cat > rootca/rootca-csr.json << "EOF" { "CN": "rootca", "key": { "algo": "ecdsa", "size": 256 }, "names": [{ "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "ROOTCA", "OU": "tz" }] } EOF cd /opt/cluster/ssl cat > etcd/etcd-csr.json << "EOF" { "CN": "etcd-cluster", "hosts": [ "127.0.0.1", "10.20.1.44", "10.30.1.44", "172.168.0.44" ], "key": { "algo": "ecdsa", "size": 256 }, "names": [{ "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "KUBERNETES-ETCD", "OU": "tz" }] } EOF cd /opt/cluster/ssl cfssl gencert -initca rootca/rootca-csr.json | cfssljson -bare rootca/rootca cfssl gencert \ -ca=rootca/rootca.pem \ -ca-key=rootca/rootca-key.pem \ --config=cfssl-conf.json \ -profile=common etcd/etcd-csr.json | cfssljson -bare etcd/etcd
4.将证书分发给其他机器
scp -r /opt/cluster/ssl 10.30.1.44:/opt/cluster/ scp -r /opt/cluster/ssl 172.168.0.44:/opt/cluster/
3.2部署etcd
1.上传工具
cd ~/tools/ tar zxvf etcd-v3.5.0-linux-amd64.tar.gz cp etcd-v3.5.0-linux-amd64/{etcd,etcdctl} /usr/local/bin chmod +x /usr/local/bin/ scp -r etcd-v3.5.0-linux-amd64/{etcd,etcdctl} root@10.30.1.44:/usr/local/bin scp -r etcd-v3.5.0-linux-amd64/{etcd,etcdctl} root@172.168.0.44:/usr/local/bin
2.编写system配置文件
k8s-master01配置文件如下
cat > /usr/lib/systemd/system/etcd.service << "EOF" [Unit] Description=Kubernetes:Etcd After=network.target network-online.target Wants=network-online.target [Service] Restart=on-failure RestartSec=5 ExecStart=/usr/local/bin/etcd \ --name=etcd01 \ --data-dir=/opt/cluster/etcd/data \ --wal-dir=/opt/cluster/etcd/wal \ --listen-peer-urls=https://10.20.1.44:2380 \ --listen-client-urls=https://10.20.1.44:2379,http://127.0.0.1:2379 \ --initial-advertise-peer-urls=https://10.20.1.44:2380 \ --initial-cluster=etcd01=https://10.20.1.44:2380,etcd02=https://10.30.1.44:2380,etcd03=https://172.168.0.44:2380 \ --initial-cluster-state=new \ --initial-cluster-token=373b3543a301630c \ --advertise-client-urls=https://10.20.1.44:2379 \ --cert-file=/opt/cluster/ssl/etcd/etcd.pem \ --key-file=/opt/cluster/ssl/etcd/etcd-key.pem \ --peer-cert-file=/opt/cluster/ssl/etcd/etcd.pem \ --peer-key-file=/opt/cluster/ssl/etcd/etcd-key.pem \ --trusted-ca-file=/opt/cluster/ssl/rootca/rootca.pem \ --peer-trusted-ca-file=/opt/cluster/ssl/rootca/rootca.pem \ --client-cert-auth=true \ --peer-client-cert-auth=true \ --logger=zap \ --log-outputs=default \ --log-level=info \ --listen-metrics-urls=https://10.20.1.44:2381 \ --enable-pprof=false [Install] WantedBy=multi-user.target EOF
k8s-node01配置文件如下
cat > /usr/lib/systemd/system/etcd.service << "EOF" [Unit] Description=Kubernetes:Etcd After=network.target network-online.target Wants=network-online.target [Service] Restart=on-failure RestartSec=5 ExecStart=/usr/local/bin/etcd \ --name=etcd02 \ --data-dir=/opt/cluster/etcd/data \ --wal-dir=/opt/cluster/etcd/wal \ --listen-peer-urls=https://10.30.1.44:2380 \ --listen-client-urls=https://10.30.1.44:2379,http://127.0.0.1:2379 \ --initial-advertise-peer-urls=https://10.30.1.44:2380 \ --initial-cluster=etcd01=https://10.20.1.44:2380,etcd02=https://10.30.1.44:2380,etcd03=https://172.168.0.44:2380 \ --initial-cluster-state=new \ --initial-cluster-token=373b3543a301630c \ --advertise-client-urls=https://10.30.1.44:2379 \ --cert-file=/opt/cluster/ssl/etcd/etcd.pem \ --key-file=/opt/cluster/ssl/etcd/etcd-key.pem \ --peer-cert-file=/opt/cluster/ssl/etcd/etcd.pem \ --peer-key-file=/opt/cluster/ssl/etcd/etcd-key.pem \ --trusted-ca-file=/opt/cluster/ssl/rootca/rootca.pem \ --peer-trusted-ca-file=/opt/cluster/ssl/rootca/rootca.pem \ --client-cert-auth=true \ --peer-client-cert-auth=true \ --logger=zap \ --log-outputs=default \ --log-level=info \ --listen-metrics-urls=https://10.30.1.44:2381 \ --enable-pprof=false [Install] WantedBy=multi-user.target EOF
k8s-node02配置文件如下
cat > /usr/lib/systemd/system/etcd.service << "EOF" [Unit] Description=Kubernetes:Etcd After=network.target network-online.target Wants=network-online.target [Service] Restart=on-failure RestartSec=5 ExecStart=/usr/local/bin/etcd \ --name=etcd03 \ --data-dir=/opt/cluster/etcd/data \ --wal-dir=/opt/cluster/etcd/wal \ --listen-peer-urls=https://172.168.0.44:2380 \ --listen-client-urls=https://172.168.0.44:2379,http://127.0.0.1:2379 \ --initial-advertise-peer-urls=https://172.168.0.44:2380 \ --initial-cluster=etcd01=https://10.20.1.44:2380,etcd02=https://10.30.1.44:2380,etcd03=https://172.168.0.44:2380 \ --initial-cluster-state=new \ --initial-cluster-token=373b3543a301630c \ --advertise-client-urls=https://172.168.0.44:2379 \ --cert-file=/opt/cluster/ssl/etcd/etcd.pem \ --key-file=/opt/cluster/ssl/etcd/etcd-key.pem \ --peer-cert-file=/opt/cluster/ssl/etcd/etcd.pem \ --peer-key-file=/opt/cluster/ssl/etcd/etcd-key.pem \ --trusted-ca-file=/opt/cluster/ssl/rootca/rootca.pem \ --peer-trusted-ca-file=/opt/cluster/ssl/rootca/rootca.pem \ --client-cert-auth=true \ --peer-client-cert-auth=true \ --logger=zap \ --log-outputs=default \ --log-level=info \ --listen-metrics-urls=https://172.168.0.44:2381 \ --enable-pprof=false [Install] WantedBy=multi-user.target EOF
3.启动etcd
所有机器都需要启动
systemctl daemon-reload && \ systemctl enable etcd.service && \ systemctl start etcd.service && \ systemctl status etcd.service
4.验证etcd集群（任意一台机器执行）
ETCDCTL_API=3 /usr/local/bin/etcdctl \ --cacert=/opt/cluster/ssl/rootca/rootca.pem \ --cert=/opt/cluster/ssl/etcd/etcd.pem \ --key=/opt/cluster/ssl/etcd/etcd-key.pem \ --endpoints="https://10.20.1.44:2379,https://10.30.1.44:2379,https://172.168.0.44:2379" \ endpoint health --write-out=table
四、部署API
4.1下载API
下载地址：https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.22.md
注：打开链接你会发现里面有很多包，下载一个Server Binaries包就够了，里面包含了Master和Worker节点的二进制文件。
4.2颁发证书
1.上传工具
cd ~/tools/ tar zxvf kubernetes-server-linux-amd64.tar.gz cd kubernetes/server/bin cp kube-apiserver kube-scheduler kube-controller-manager /usr/local/bin cp kubectl /usr/local/bin
2.生成证书
cd /opt/cluster/ssl cat > kubernetes/kube-apiserver-csr.json << "EOF" { "CN": "kube-apiserver", "hosts": [ "127.0.0.1", "10.20.1.44", "10.20.1.100", "10.20.1.101", "10.20.1.102", "10.20.1.103", "10.20.1.104", "10.20.1.105", "10.20.1.106", "10.30.1.44", "10.30.1.100", "10.30.1.101", "10.30.1.102", "10.30.1.103", "10.30.1.104", "10.30.1.105", "10.30.1.106", "172.168.0.44", "172.168.0.24", "172.168.0.100", "172.168.0.101", "172.168.0.102", "172.168.0.103", "172.168.0.104", "10.20.1.99", "10.96.0.1", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local" ], "key": { "algo": "ecdsa", "size": 256 }, "names": [{ "C": "CN", "L": "BeiJing", "ST": "BeiJing", "O": "system:masters", "OU": "tz" }] } EOF cd /opt/cluster/ssl cfssl gencert \ -ca=rootca/rootca.pem \ -ca-key=rootca/rootca-key.pem \ --config=cfssl-conf.json \ -profile=common kubernetes/kube-apiserver-csr.json | cfssljson -bare kubernetes/kube-apiserver
Work节点的证书使用API授权，不自己签发，所以这里的IP地址除了Work节点不用写，其他都要写。
10.96.0.1是service-cluster-ip的首个IP
4.3部署API
1.生成token.csv
cd /opt/cluster/ssl echo $(head -c 16 /dev/urandom | od -An -t x | tr -d ' '),kubelet-bootstrap,10001,"system:kubelet-bootstrap" > kubernetes/kube-apiserver.token.csv
Work节点请求证书需要用到，这里是注册了一个低权限的用户kubelet-bootstrap，工作节点使用该用户向API请求证书
2.编写systemd配置文件
cat > /usr/lib/systemd/system/kube-apiserver.service << "EOF" [Unit] Description=Kubernetes:Apiserver After=network.target network-online.target Wants=network-online.target [Service] Restart=on-failure RestartSec=5 ExecStart=/usr/local/bin/kube-apiserver \ --runtime-config=api/all=true \ --anonymous-auth=false \ --bind-address=0.0.0.0 \ --advertise-address=10.20.1.44 \ --secure-port=6443 \ --tls-cert-file=/opt/cluster/ssl/kubernetes/kube-apiserver.pem \ --tls-private-key-file=/opt/cluster/ssl/kubernetes/kube-apiserver-key.pem \ --client-ca-file=/opt/cluster/ssl/rootca/rootca.pem \ --etcd-cafile=/opt/cluster/ssl/rootca/rootca.pem \ --etcd-certfile=/opt/cluster/ssl/etcd/etcd.pem \ --etcd-keyfile=/opt/cluster/ssl/etcd/etcd-key.pem \ --etcd-servers=https://10.20.1.44:2379,https://10.30.1.44:2379,https://172.168.0.44:2379 \ --kubelet-client-certificate=/opt/cluster/ssl/kubernetes/kube-apiserver.pem \ --kubelet-client-key=/opt/cluster/ssl/kubernetes/kube-apiserver-key.pem \ --service-account-key-file=/opt/cluster/ssl/rootca/rootca-key.pem \ --service-account-signing-key-file=/opt/cluster/ssl/rootca/rootca-key.pem \ --service-account-issuer=https://kubernetes.default.svc.cluster.local \ --enable-bootstrap-token-auth=true \ --token-auth-file=/opt/cluster/ssl/kubernetes/kube-apiserver.token.csv \ --allow-privileged=true \ --service-cluster-ip-range=10.96.0.0/16 \ --service-node-port-range=30000-50000 \ --authorization-mode=RBAC,Node \ --enable-aggregator-routing=true \ --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \ --apiserver-count=3 \ --audit-log-maxage=30 \ --audit-log-maxbackup=3 \ --audit-log-maxsize=100 \ --audit-log-path=/opt/cluster/log/kube-apiserver/audit.log \ --logtostderr=false \ --v=2 \ --log-dir=/opt/cluster/log/kube-apiserver [Install] WantedBy=multi-user.target EOF
3.启动api
systemctl daemon-reload && \ systemctl enable --now kube-apiserver.service && \ systemctl status kube-apiserver.service
五、部署kubectl
1.生成证书
cd /opt/cluster/ssl
cat > kubernetes/kubectl-csr.json << "EOF"
{
"CN": "clusteradmin",
"key": {
"algo": "ecdsa",
"size": 256
},
"names": [{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:masters",
"OU": "tz"
}]
}
EOF
cd /opt/cluster/ssl
cfssl gencert -ca=rootca/rootca.pem \
-ca-key=rootca/rootca-key.pem \
--config=cfssl-conf.json \
-profile=common kubernetes/kubectl-csr.json | cfssljson -bare kubernetes/kubectl
2.生成kubeconfig配置文件
我是先部署的单点Master集群，还没用负载均衡器，所以这里的--server填写的是k8s-master01的地址，如果部署了负载均衡器，则填写VIP地址。
cd /opt/cluster/ssl kubectl config set-cluster kubernetes \ --certificate-authority=/opt/cluster/ssl/rootca/rootca.pem \ --embed-certs=true \ --server=https://10.20.1.99:16443 \ --kubeconfig=kubernetes/kubectl.kubeconfig kubectl config set-credentials clusteradmin \ --client-certificate=/opt/cluster/ssl/kubernetes/kubectl.pem \ --client-key=/opt/cluster/ssl/kubernetes/kubectl-key.pem \ --embed-certs=true \ --kubeconfig=kubernetes/kubectl.kubeconfig kubectl config set-context default \ --cluster=kubernetes \ --user=clusteradmin \ --kubeconfig=kubernetes/kubectl.kubeconfig kubectl config use-context default \ --kubeconfig=kubernetes/kubectl.kubeconfig mkdir /root/.kube cp /opt/cluster/ssl/kubernetes/kubectl.kubeconfig /root/.kube/config
3.获取集群信息
kubectl cluster-info kubectl get cs kubectl get all --all-namespaces
六、部署controller-manager
6.1签发证书
给master节点签发证书
cd /opt/cluster/ssl cat > kubernetes/kube-controller-manager-csr.json << "EOF" { "CN": "system:kube-controller-manager", "hosts": [ "127.0.0.1", "10.20.1.44", "10.30.1.44", "172.168.0.44" ], "key": { "algo": "ecdsa", "size": 256 }, "names": [{ "C": "CN", "ST": "BeiJing", "L": "BeiJing", "O": "KUBERNETES", "OU": "tz" }] } EOF cd /opt/cluster/ssl cfssl gencert -ca=rootca/rootca.pem \ -ca-key=rootca/rootca-key.pem \ --config=cfssl-conf.json \ -profile=common kubernetes/kube-controller-manager-csr.json | cfssljson -bare kubernetes/kube-controller-manager
6.2部署controller-manager
1.编写kubeconfig配置文件
cd /opt/cluster/ssl kubectl config set-cluster kubernetes --certificate-authority=/opt/cluster/ssl/rootca/rootca.pem \ --embed-certs=true --server=https://10.20.1.99:16443 \ --kubeconfig=kubernetes/kube-controller-manager.kubeconfig kubectl config set-credentials kube-controller-manager --client-certificate=kubernetes/kube-controller-manager.pem \ --client-key=kubernetes/kube-controller-manager-key.pem --embed-certs=true \ --kubeconfig=kubernetes/kube-controller-manager.kubeconfig kubectl config set-context default --cluster=kubernetes --user=kube-controller-manager \ --kubeconfig=kubernetes/kube-controller-manager.kubeconfig kubectl config use-context default --kubeconfig=kubernetes/kube-controller-manager.kubeconfig
2.编写systemd配置文件
cat > /usr/lib/systemd/system/kube-controller-manager.service << "EOF" [Unit] Description=Kubernetes:Kube-Controller-Manager After=network.target network-online.target Wants=network-online.target [Service] Restart=on-failure RestartSec=5 ExecStart=/usr/local/bin/kube-controller-manager \ --cluster-name=kubernetes \ --secure-port=10257 \ --bind-address=127.0.0.1 \ --service-cluster-ip-range=10.96.0.0/16 \ --allocate-node-cidrs=true \ --cluster-cidr=10.97.0.0/16 \ --leader-elect=true \ --controllers=*,bootstrapsigner,tokencleaner \ --kubeconfig=/opt/cluster/ssl/kubernetes/kube-controller-manager.kubeconfig \ --tls-cert-file=/opt/cluster/ssl/kubernetes/kube-controller-manager.pem \ --tls-private-key-file=/opt/cluster/ssl/kubernetes/kube-controller-manager-key.pem \ --cluster-signing-cert-file=/opt/cluster/ssl/rootca/rootca.pem \ --cluster-signing-key-file=/opt/cluster/ssl/rootca/rootca-key.pem \ --cluster-signing-duration=87600h0m0s \ --use-service-account-credentials=true \ --root-ca-file=/opt/cluster/ssl/rootca/rootca.pem \ --service-account-private-key-file=/opt/cluster/ssl/rootca/rootca-key.pem \ --logtostderr=false \ --v=2 \ --log-dir=/opt/cluster/log/kube-controller-manager [Install] WantedBy=multi-user.target EOF
3.启动controller-manager
systemctl daemon-reload && \ systemctl enable --now kube-controller-manager.service && \ systemctl status kube-controller-manager.service
验证
kubectl get componentstatuses
七、部署scheduler
7.1签发证书
给master节点签发证书
cd /opt/cluster/ssl cat > kubernetes/kube-scheduler-csr.json << "EOF" { "CN": "system:kube-scheduler", "hosts": [ "127.0.0.1", "10.20.1.44", "10.30.1.44", "172.168.0.44" ], "key": { "algo": "ecdsa", "size": 256 }, "names": [{ "C": "CN", "ST": "BeiJing", "L": "BeiJing", "O": "KUBERNETES", "OU": "tz" }] } EOF cd /opt/cluster/ssl cfssl gencert \ -ca=rootca/rootca.pem \ -ca-key=rootca/rootca-key.pem \ --config=cfssl-conf.json \ -profile=common kubernetes/kube-scheduler-csr.json | cfssljson -bare kubernetes/kube-scheduler
7.2部署kube-scheduler
1.编辑kubeconfig配置文件
cd /opt/cluster/ssl kubectl config set-cluster kubernetes --certificate-authority=/opt/cluster/ssl/rootca/rootca.pem \ --embed-certs=true --server=https://10.20.1.99:16443 \ --kubeconfig=kubernetes/kube-scheduler.kubeconfig kubectl config set-credentials kube-scheduler --client-certificate=kubernetes/kube-scheduler.pem \ --client-key=kubernetes/kube-scheduler-key.pem --embed-certs=true \ --kubeconfig=kubernetes/kube-scheduler.kubeconfig kubectl config set-context default --cluster=kubernetes --user=kube-scheduler \ --kubeconfig=kubernetes/kube-scheduler.kubeconfig kubectl config use-context default --kubeconfig=kubernetes/kube-scheduler.kubeconfig
2.编写systemd配置文件
cat > /usr/lib/systemd/system/kube-scheduler.service << "EOF" [Unit] Description=Kubernetes:Kube-Scheduler After=network.target network-online.target Wants=network-online.target [Service] Restart=on-failure RestartSec=5 ExecStart=/usr/local/bin/kube-scheduler \ --kubeconfig=/opt/cluster/ssl/kubernetes/kube-scheduler.kubeconfig \ --address=127.0.0.1 \ --leader-elect=true \ --logtostderr=false \ --v=2 \ --log-dir=/opt/cluster/log/kube-scheduler [Install] WantedBy=multi-user.target EOF
3.启动scheduler
systemctl daemon-reload && \ systemctl enable --now kube-scheduler.service && \ systemctl status kube-scheduler.service
验证
kubectl get cs
将证书和配置文件分发给其他master并启动
scp /opt/cluster/ssl/kubernetes/* 10.30.1.44:/opt/cluster/ssl/kubernetes/ scp -r ~/.kube/ 10.30.1.44:/root/ scp /usr/lib/systemd/system/kube-* 10.30.1.44:/usr/lib/systemd/system/ cd ~/tools/kubernetes/server/bin/ scp kube-apiserver kube-controller-manager kube-scheduler kubectl kubelet kube-proxy 10.30.1.44:/usr/local/bin/ scp /opt/cluster/ssl/kubernetes/* 172.168.0.44:/opt/cluster/ssl/kubernetes/ scp -r ~/.kube/ 172.168.0.44:/root/ scp /usr/lib/systemd/system/kube-* 172.168.0.44:/usr/lib/systemd/system/ cd ~/tools/kubernetes/server/bin/ scp kube-apiserver kube-controller-manager kube-scheduler kubectl kubelet kube-proxy 172.168.0.44:/usr/local/bin/ vim /usr/lib/systemd/system/kube-apiserver.service
# 修改--advertise-address=10.20.1.44的地址为本机地址
systemctl daemon-reload && \ systemctl enable --now kube-apiserver.service && \ systemctl status kube-apiserver.service systemctl daemon-reload && \ systemctl enable --now kube-controller-manager.service && \ systemctl status kube-controller-manager.service systemctl daemon-reload && \ systemctl enable --now kube-scheduler.service && \ systemctl status kube-scheduler.service kubectl get cs
八、部署kubelet
1.上传工具
cd /root/tools/kubernetes/server/bin cp kubelet kube-proxy /usr/local/bin scp -r kubelet kube-proxy root@10.30.1.44:/usr/local/bin scp -r kubelet kube-proxy root@172.168.0.44:/usr/local/bin scp -r kubelet kube-proxy root@10.20.1.100:/usr/local/bin
2.编写kubeconfig配置文件
cd /opt/cluster/ssl
kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap
kubectl config set-cluster kubernetes --certificate-authority=/opt/cluster/ssl/rootca/rootca.pem \
--embed-certs=true --server=https://10.20.1.99:16443 \
--kubeconfig=kubernetes/kubelet-bootstrap.kubeconfig
kubectl config set-credentials kubelet-bootstrap --token=$(awk -F "," '{print $1}' /opt/cluster/ssl/kubernetes/kube-apiserver.token.csv) \
--kubeconfig=kubernetes/kubelet-bootstrap.kubeconfig
kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap \
--kubeconfig=kubernetes/kubelet-bootstrap.kubeconfig
kubectl config use-context default --kubeconfig=kubernetes/kubelet-bootstrap.kubeconfig
3.编写kubelet.conf配置文件
cd /opt/cluster/ssl
cat > kubernetes/kubelet.conf << "EOF"
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 0.0.0.0
port: 10250
readOnlyPort: 0
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 2m0s
enabled: true
x509:
clientCAFile: /opt/cluster/ssl/rootca/rootca.pem
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 5m0s
cacheUnauthorizedTTL: 30s
cgroupDriver: systemd
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
healthzBindAddress: 127.0.0.1
healthzPort: 10248
rotateCertificates: true
evictionHard:
imagefs.available: 15%
memory.available: 100Mi
nodefs.available: 10%
nodefs.inodesFree: 5%
maxOpenFiles: 1000000
maxPods: 110
EOF
4.编写systemd配置文件
cat > /usr/lib/systemd/system/kubelet.service << "EOF"
[Unit]
Description=Kubernetes:Kubelet
After=network.target network-online.target docker.service
Requires=docker.service
[Service]
Restart=on-failure
RestartSec=5
ExecStart=/usr/local/bin/kubelet \
--bootstrap-kubeconfig=/opt/cluster/ssl/kubernetes/kubelet-bootstrap.kubeconfig \
--config=/opt/cluster/ssl/kubernetes/kubelet.conf \
--kubeconfig=/opt/cluster/kubelet/kubelet.kubeconfig \
--cert-dir=/opt/cluster/kubelet/ssl \
--network-plugin=cni \
--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2 \
--logtostderr=false \
--v=2 \
--log-dir=/opt/cluster/log/kubelet
[Install]
WantedBy=multi-user.target
EOF
5.复制证书配置文件到其他节点
scp -r /usr/lib/systemd/system/kubelet.service root@172.168.0.44:/usr/lib/systemd/system/kubelet.service scp -r /usr/lib/systemd/system/kubelet.service root@10.30.1.44:/usr/lib/systemd/system/kubelet.service scp -r /usr/lib/systemd/system/kubelet.service root@10.20.1.100:/usr/lib/systemd/system/kubelet.service scp /opt/cluster/ssl/kubernetes/kubelet* root@10.30.1.44:/opt/cluster/ssl/kubernetes/ scp /opt/cluster/ssl/kubernetes/kubelet* root@172.168.0.44:/opt/cluster/ssl/kubernetes/ scp /opt/cluster/ssl/kubernetes/kubelet* root@10.20.1.100:/opt/cluster/ssl/kubernetes/ scp /opt/cluster/ssl/rootca/* 172.168.0.44:/opt/cluster/ssl/rootca/ scp /opt/cluster/ssl/rootca/* 10.30.1.44:/opt/cluster/ssl/rootca/ scp /opt/cluster/ssl/rootca/* 10.20.1.100:/opt/cluster/ssl/rootca/
6.启动kubelet
所有节点都执行
systemctl daemon-reload && \ systemctl enable --now kubelet.service && \ systemctl status kubelet.service
7.授权证书
#查看需要授权的证书
kubectl get csr
#授权证书 kubectl certificate approve <CSR_NAME> kubectl get node
九、部署kube-proxy
1.生成证书
cd /opt/cluster/ssl
cat > kubernetes/kube-proxy-csr.json << "EOF"
{
"CN": "system:kube-proxy",
"key": {
"algo": "ecdsa",
"size": 256
},
"names": [{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "KUBERNETES",
"OU": "tz"
}]
}
EOF
cd /opt/cluster/ssl
cfssl gencert \
-ca=rootca/rootca.pem \
-ca-key=rootca/rootca-key.pem \
--config=cfssl-conf.json \
-profile=common kubernetes/kube-proxy-csr.json | cfssljson -bare kubernetes/kube-proxy
2.编写kubeconfig文件
cd /opt/cluster/ssl kubectl config set-cluster kubernetes --certificate-authority=/opt/cluster/ssl/rootca/rootca.pem \ --embed-certs=true --server=https://10.20.1.99:16443 \ --kubeconfig=kubernetes/kube-proxy.kubeconfig kubectl config set-credentials kube-proxy --client-certificate=/opt/cluster/ssl/kubernetes/kube-proxy.pem \ --client-key=/opt/cluster/ssl/kubernetes/kube-proxy-key.pem --embed-certs=true \ --kubeconfig=kubernetes/kube-proxy.kubeconfig kubectl config set-context default --cluster=kubernetes --user=kube-proxy \ --kubeconfig=kubernetes/kube-proxy.kubeconfig kubectl config use-context default --kubeconfig=kubernetes/kube-proxy.kubeconfig
3.编写kube-proxy配置文件
cat > kubernetes/kube-proxy.conf << "EOF" kind: KubeProxyConfiguration apiVersion: kubeproxy.config.k8s.io/v1alpha1 clientConnection: kubeconfig: /opt/cluster/ssl/kubernetes/kube-proxy.kubeconfig bindAddress: 0.0.0.0 clusterCIDR: "10.97.0.0/16" healthzBindAddress: "0.0.0.0:10256" metricsBindAddress: "0.0.0.0:10249" mode: ipvs ipvs: scheduler: "rr" EOF
4.编写systemd配置文件
这里需要注意，我把Master01也部署为Work节点，一方面可以监控，还能跑Pod，如果不想在Master上跑Pod，可以添加污点。
这里是在Master01上，所以--hostname-override值为master-20
cat > /usr/lib/systemd/system/kube-proxy.service << "EOF"
[Unit]
Description=Kubernetes:Kube-Proxy
After=network.target network-online.target
Wants=network-online.target
[Service]
Restart=on-failure
RestartSec=5
ExecStart=/usr/local/bin/kube-proxy \
--config=/opt/cluster/ssl/kubernetes/kube-proxy.conf \
--logtostderr=false \
--v=2 \
--log-dir=/opt/cluster/log/kube-proxy \
--hostname-override=master-20
[Install]
WantedBy=multi-user.target
EOF
5.复制证书跟配置文件到其他节点
scp -r /opt/cluster/ssl root@10.30.1.44:/opt/cluster/ scp -r /opt/cluster/ssl root@10.20.1.100:/opt/cluster/ scp -r /opt/cluster/ssl root@172.168.0.44:/opt/cluster/ scp -r /usr/lib/systemd/system/kube-proxy.service root@10.30.1.44:/usr/lib/systemd/system/kube-proxy.service scp -r /usr/lib/systemd/system/kube-proxy.service root@10.20.1.100:/usr/lib/systemd/system/kube-proxy.service scp -r /usr/lib/systemd/system/kube-proxy.service root@172.168.0.44:/usr/lib/systemd/system/kube-proxy.service
在k8s-node1跟k8s-node2节点上修改
#在node01上修改 vim /usr/lib/systemd/system/kube-proxy.service ... --hostname-override=k8s-node01 ... #在node02上修改 vim /usr/lib/systemd/system/kube-proxy.service ... --hostname-override=k8s-node02 ...
6.启动kube-proxy
systemctl daemon-reload && \ systemctl enable --now kube-proxy.service && \ systemctl status kube-proxy.service
十、部署网络组件
10.1部署calico插件
yaml文件下载地址：https://docs.projectcalico.org/v3.20/manifests/calico.yaml
cd /opt/cluster/plugins/calico/ wget https://docs.projectcalico.org/v3.20/manifests/calico.yaml
1.修改calico.yaml文件
vim calico.yaml - name: CALICO_IPV4POOL_CIDR value: "10.97.0.0/16"
2.应用yaml文件
kubectl apply -f calico.yaml
当无法启动calico插件时，需要先使用docker pull拉取它们以排查是否是网络原因造成的无法启动
3.验证集群
kubectl get pods -n kube-system #-w可以实时查看 kubectl get pods -n kube-system -w kubectl get node
正常情况下，calico插件状态为Running，各节点状态为Ready
10.2 CoreDNS插件
下载地址：https://github.com/coredns/deployment/blob/master/kubernetes/coredns.yaml.sed
1.修改coredns.yaml文件
cd /opt/cluster/plugins/coredns
vim coredns.yaml
---
...
data:
Corefile: |
.:53 {
errors
health {
lameduck 5s
}
ready
kubernetes CLUSTER_DOMAIN REVERSE_CIDRS { # 修改此处的"CLUSTER_DOMAIN"为"cluster.local"，表示集群域名
fallthrough in-addr.arpa ip6.arpa # 修改此处的"REVERSE_CIDRS"为"in-addr.arpa ip6.arpa"；本处的配置涉及的是DNS的反向解释功能
}
prometheus :9153
forward . UPSTREAMNAMESERVER { # 修改此处的"UPSTREAMNAMESERVER"为"/etc/resolv.conf"；本处的配置涉及的是DNS的正向解释功能
max_concurrent 1000
}
cache 30
loop
reload
loadbalance
}STUBDOMAINS # 删除此处的"STUBDOMAINS"；
# 新版本的YAML文件中有这个字段[若不存在则不需要任何操作]
---
...
spec:
selector:
k8s-app: kube-dns
clusterIP: CLUSTER_DNS_IP # 修改此处的"CLUSTER_DNS_IP"为"10.96.0.10";本处为定义K8S集群内的DNS服务器的地址；
# 这个值应该与"kubelet.conf"中定义的"clusterDNS"配置项的值相同；
修改如下所示
apiVersion: v1
kind: ServiceAccount
metadata:
name: coredns
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
kubernetes.io/bootstrapping: rbac-defaults
name: system:coredns
rules:
- apiGroups:
- ""
resources:
- endpoints
- services
- pods
- namespaces
verbs:
- list
- watch
- apiGroups:
- discovery.k8s.io
resources:
- endpointslices
verbs:
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
labels:
kubernetes.io/bootstrapping: rbac-defaults
name: system:coredns
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:coredns
subjects:
- kind: ServiceAccount
name: coredns
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns
namespace: kube-system
data:
Corefile: |
.:53 {
errors
health {
lameduck 5s
}
ready
kubernetes cluster.local in-addr.arpa ip6.arpa {
fallthrough in-addr.arpa ip6.arpa
}
prometheus :9153
forward . /etc/resolv.conf {
max_concurrent 1000
}
cache 30
loop
reload
loadbalance
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: coredns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/name: "CoreDNS"
spec:
# replicas: not specified here:
# 1. Default is 1.
# 2. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
selector:
matchLabels:
k8s-app: kube-dns
template:
metadata:
labels:
k8s-app: kube-dns
spec:
priorityClassName: system-cluster-critical
serviceAccountName: coredns
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"
nodeSelector:
kubernetes.io/os: linux
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: k8s-app
operator: In
values: ["kube-dns"]
topologyKey: kubernetes.io/hostname
containers:
- name: coredns
image: coredns/coredns:1.8.6
imagePullPolicy: IfNotPresent
resources:
limits:
memory: 170Mi
requests:
cpu: 100m
memory: 70Mi
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
readOnly: true
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
- containerPort: 9153
name: metrics
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_BIND_SERVICE
drop:
- all
readOnlyRootFilesystem: true
livenessProbe:
httpGet:
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
httpGet:
path: /ready
port: 8181
scheme: HTTP
dnsPolicy: Default
volumes:
- name: config-volume
configMap:
name: coredns
items:
- key: Corefile
path: Corefile
---
apiVersion: v1
kind: Service
metadata:
name: kube-dns
namespace: kube-system
annotations:
prometheus.io/port: "9153"
prometheus.io/scrape: "true"
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "CoreDNS"
spec:
selector:
k8s-app: kube-dns
clusterIP: 10.96.0.10
ports:
- name: dns
port: 53
protocol: UDP
- name: dns-tcp
port: 53
protocol: TCP
- name: metrics
port: 9153
protocol: TCP
2.应用yaml文件
cd /opt/cluster/plugins/coredns kubectl apply -f coredns.yaml
3.验证
#-w可以实时查看 kubectl get pods -n kube-system -w kubectl get node
十一、部署nginx进行测试
1.生成yaml配置文件
kubectl create deployment web --image=nginx -o yaml --dry-run >> nginx.yaml
2.使用yaml文件进行部署
kubectl apply -f nginx.yaml
3.查看创建的pod
kubectl get pods
4.对外暴露访问端口，生成yaml配置文件
kubectl expose deployment web --port=80 --type=NodePort --target-port=80 --name=web1 -o yaml >> nginx-web1.yaml
5.查看pod和svc
kubectl get pod,svc
6.访问nginx服务
十二、k8s补全命令
yum -y install bash-completion source /usr/share/bash-completion/bash_completion source <(kubectl completion bash) echo "source <(kubectl completion bash)" >> ~/.bashrc
浙公网安备 33010602011771号