centos7脚本部署kubernetes

一、思路：

1、centos7.9默认内核3.10，部署k8s有bug，先将内核升级到4.4或更高版本再部署k8s。

2、master节点和node节点都需要安装配置docker、kubelet等工具，放在一个脚本。

3、只需master节点安装配置的部分，放在单独的脚本中。

4、master节点配置完成后，node节点加入集群。

5、检查k8s集群状态。

二、实践：

1、如果centos7.9内核低于4.4，需要升级至4.4或更高版本

#!/bin/bash

# CentOS 7.x 系统自带的3.10.x内核存在一些Bugs.导致运行的Docker.Kubernetes不稳定
# 更新yum源仓库
#yum -y update

# 启用 ELRepo 仓库
# 导入ELRepo仓库的公共密钥
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org

#安装ELRepo仓库的yum源
rpm -Uvh https://mirrors.aliyun.com/elrepo/kernel/el7/x86_64/RPMS/elrepo-release-7.0-4.el7.elrepo.noarch.rpm

cp /etc/yum.repos.d/elrepo.repo /etc/yum.repos.d/elrepo.repo.bak
sed -i -e 's/^mirrorlist=/#&/' \
       -e 's#http://www.elrepo.org/linux#https://mirrors.aliyun.com/elrepo#' \
       -e 's#http://mirrors.coreix.net/elrepo#https://mirrors.tuna.tsinghua.edu.cn/#' \
       /etc/yum.repos.d/elrepo.repo

yum makecache

#安装最新版本内核
yum --enablerepo=elrepo-kernel install kernel-ml -y

#设置新的内核为grub2的默认版本
grub2-set-default 0

#生成 grub 配置文件并重启
#内核安装好后，需要设置为默认启动选项并重启后才会生效
grub2-mkconfig -o /boot/grub2/grub.cfg

#重启服务器
reboot

2、master节点和node节点都要安装配置

#cat deploy_k8s_common.sh

#!/bin/bash

# 调用方式 #bash deploy_k8s_common.sh (master|node1|node2)
if [ $# -ne 1 ]; then
  echo "$0 <hostname>"
  exit
fi

hostnamectl set-hostname $1

cat >> /etc/hosts<<EOF
10.0.0.17 node1
10.0.0.27 node2
10.0.0.47 master
EOF

yum -y install conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat \
       libseccomp wget git vim net-tools

#设置防火墙为lptables 并设置空规则
systemctl stop firewalld && systemctl disable firewalld
yum -y install iptables-services && systemctl start iptables && \
    systemctl enable iptables && iptables -F && service iptables save

#关闭swap分区和SELINUX(如果不关闭，默认配置下kubelet将无法启动。使用free -m确认swap已经关闭。)
swapoff -a && sed -ri 's/.*swap.*/#&/' /etc/fstab
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

#修改时区并关闭系统不需要的服务
timedatectl set-timezone Asia/Shanghai
timedatectl set-local-rtc 0
systemctl restart rsyslog
systemctl restart crond
systemctl stop postfix && systemctl disable postfix

#Kubernetes调整内核参数（三个重要部分是必须条件,将桥接的IPv4流量传递到iptables的链）
#cat > kubernetes.conf<<EOF
cat > /etc/sysctl.d/kubernetes.conf<<EOF
#开启网桥模式【重要】
net.bridge.bridge-nf-call-iptables=1
#开启网桥模式【重要】
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
#禁止使用swap空间，只有当系统OOM时才允许使用它
vm.swappiness=0
#不检查物理内存是否够用
vm.overcommit_memory=1
#开启OOM
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
#关闭ipv6【重要】
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF

#将优化内核文件拷贝到/etc/sysctl.d/文件夹下，这样优化文件开机的时候能够被调用
cp kubernetes.conf /etc/sysctl.d/kubernetes.conf

#手动加载内核文件，立即生效
modprobe ip_conntrack
sysctl -p /etc/sysctl.d/kubernetes.conf

#在Centos7以后，因为引导方式改为了system.d，所以有两个日志系统同时在工作，默认的是rsyslogd，以及systemd journald，使用systemd journald更好一些，因此我们更改默认为systemd journald，只保留一个日志的保存方式。
#持久化保存日志的目录
mkdir /var/log/journal
mkdir /etc/systemd/journal.conf.d

cat >/etc/systemd/journal.conf.d/99-prophet.conf <<EOF
[Journal]
#持久化保存到磁盘
Storage=persistent
#压缩历史日志
Compress=yes
SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000
#最大占用空间10G
SystemMaxUse=10G
#单日志文件最大200M
SystemMaxFileSize=200M
#日志保存时间2周
MaxRetentionSec=2week
#不将日志转发到syslog
ForwardToSyslog=no
EOF

systemctl restart systemd-journald


#kube-proxy开启ipvs的前置条件
modprobe br_netfilter

cat >/etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules && lsmod |grep -e ip_vs -e nf_conntrack


#一定要安装18.09版docker,否则在初始化k8s master的时候会提示：[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 19.03.3. Latest validated version: 18.09
yum install -y yum-utils
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum makecache fast
yum -y install docker-ce-18.09.0 docker-ce-cli-18.09.0 containerd.io
systemctl enable --now docker

# systemd是系统自带的cgroup管理器, 系统初始化就存在的, 和cgroups联系紧密,为每一个进程分配cgroups, 用它管理就行了
# 创建或修改/etc/docker/daemon.json：

cat >/etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts":{
    "max-size": "100m"
  }
}
EOF

systemctl daemon-reload
systemctl restart docker


#配置k8s源

cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF


#安装k8s组件，kubeadm,kubelet和kubectl
#yum makecache fast
yum install -y kubelet-1.15.0 kubeadm-1.15.0 kubectl-1.15.0
systemctl enable kubelet

#初始化准备
cat > k8s_image.sh <<EOF
#!/bin/sh
### 版本信息
K8S_VERSION=v1.15.1
ETCD_VERSION=3.3.10
#DASHBOARD_VERSION=v1.8.3
FLANNEL_VERSION=v0.12.0-amd64
#DNS_VERSION=1.14.8
PAUSE_VERSION=3.1
coredns_version=1.3.1
## 基本组件
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver-amd64:\$K8S_VERSION
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager-amd64:\$K8S_VERSION
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler-amd64:\$K8S_VERSION
docker pull aiotceo/kube-proxy:\$K8S_VERSION
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/etcd-amd64:\$ETCD_VERSION
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:\$PAUSE_VERSION
### 网络
#docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/k8s-dns-sidecar-amd64:\$DNS_VERSION
#docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/k8s-dns-kube-dns-amd64:\$DNS_VERSION
#docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/k8s-dns-dnsmasq-nanny-amd64:\$DNS_VERSION
docker pull quay.io/coreos/flannel:\$FLANNEL_VERSION
docker pull registry.cn-hangzhou.aliyuncs.com/openthings/k8s-gcr-io-coredns:\$coredns_version
### 前端
#docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kubernetes-dashboard-amd64:\$DASHBOARD_VER
## 修改tag
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver-amd64:\$K8S_VERSION k8s.gcr.io/kube-apiserver-amd64:\$K8S_VERSION
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager-amd64:\$K8S_VERSION k8s.gcr.io/kube-controller-manager-amd64:\$K8S_VERSION
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler-amd64:\$K8S_VERSION k8s.gcr.io/kube-scheduler-amd64:\$K8S_VERSION
#docker tag registry.cn-hangzhou.aliyuncs.com/openthings/k8s-gcr-io-kube-proxy-amd64:\$K8S_VERSION k8s.gcr.io/kube-proxy-amd64:\$K8S_VERSION
docker tag aiotceo/kube-proxy:\$K8S_VERSION k8s.gcr.io/kube-proxy-amd64:\$K8S_VERSION
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/etcd-amd64:\$ETCD_VERSION k8s.gcr.io/etcd-amd64:\$ETCD_VERSION
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:\$PAUSE_VERSION k8s.gcr.io/pause:\$PAUSE_VERSION
#docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/k8s-dns-sidecar-amd64:\$DNS_VERSION k8s.gcr.io/k8s-dns-sidecar-amd64:\$DNS_VERSION
#docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/k8s-dns-kube-dns-amd64:\$DNS_VERSION k8s.gcr.io/k8s-dns-kube-dns-amd64:\$DNS_VERSION
#docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/k8s-dns-dnsmasq-nanny-amd64:\$DNS_VERSION k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:\$DNS_VERSION
#docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kubernetes-dashboard-amd64:\$DASHBOARD_VERSION k8s.gcr.io/kubernetes-dashboard-amd64:\$DASHBOARD_VERSION
docker tag registry.cn-hangzhou.aliyuncs.com/openthings/k8s-gcr-io-coredns:\$coredns_version  k8s.gcr.io/coredns:\$coredns_version
## 删除镜像
docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver-amd64:\$K8S_VERSION
docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager-amd64:\$K8S_VERSION
docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler-amd64:\$K8S_VERSION
#docker rmi registry.cn-hangzhou.aliyuncs.com/openthings/k8s-gcr-io-kube-proxy-amd64:\$K8S_VERSION
docker rmi aiotceo/kube-proxy:\$K8S_VERSION
docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/etcd-amd64:\$ETCD_VERSION
docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:\$PAUSE_VERSION
#docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/k8s-dns-sidecar-amd64:\$DNS_VERSION
#docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/k8s-dns-kube-dns-amd64:\$DNS_VERSION
#docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/k8s-dns-dnsmasq-nanny-amd64:\$DNS_VERSION
#docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/kubernetes-dashboard-amd64:\$DASHBOARD_VERSION
docker rmi registry.cn-hangzhou.aliyuncs.com/openthings/k8s-gcr-io-coredns:\$coredns_version
EOF
#三台机器分别执行上面脚本下载镜像：
bash k8s_image.sh

#脚本开始时修改了主机名，执行"exec bash"命令可以看到修改后的效果。该命令执行后，结束执行脚本
exec bash

3、只在master节点上安装配置，需要预先把kube-flannel.yml文件放在脚本同级目录下。node节点加入集群后，会从master节点下载相关插件

#!/bin/bash

#显示默认的init初始化文件并打印出来存放到指定位置
kubeadm config print init-defaults >kubeadm-config.yaml

#添加pod网段（kubernetesVersion上一行添加即可）
sed -i -e '/advertiseAddress:/c\  advertiseAddress: 10.0.0.17' \
       -e '/kubernetesVersion:/c\  podSubnet: "10.244.0.0/16"\n  kubernetesVersion: v1.15.1' kubeadm-config.yaml

#另外再添加一个字段，把默认的调度方式改为ipvs模式（最后scheduler: {}下面）
cat >>kubeadm-config.yaml<<EOF
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
  SupportIPVSProxyMode: true
mode: ipvs
EOF

# 初始化安装并自动颁发证书
kubeadm init --image-repository=registry.aliyuncs.com/google_containers \
  --pod-network-cidr=10.244.0.0/16 --kubernetes-version=v1.15.1

# 设置k8s配置文件
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

#部署flannel前需要把kube-flannel.yml文件放在脚本同级目录下
mkdir -p install-k8s/{core,plugin/flannel}
mv kubeadm-config.yaml install-k8s/core/
mv kube-flannel.yml install-k8s/plugin/flannel/
cd install-k8s/plugin/flannel/

#创建flannel网络
kubectl create -f kube-flannel.yml

4、在node节点上执行类似以下的命令。该命令是上个步骤中，master节点执行初始化安装并自动颁发证书（kubeadm init）时的执行结果，复制到node节点执行即可。

kubeadm join 10.0.0.17:6443 --token apc1pa.sojpqedyvn881dci --discovery-token-ca-cert-hash sha256:3608a3713ae56ee508c779550b5e1782d77f892b4d7ded9b69d5b8c728090683

5、在master节点上用kubectl get nodes -o wide、kubectl get pods -n kube-system -o wide、kubectl get cs等命令，刷新查看k8s节点和pod状态。正常情况下，各个节点应为ready状态、pod为running状态、cs(componentstatuses)为healthy状态。