k8s集群搭建
1 环境说明
1.1 k8s-master 节点
#系统环境： 内存：4G 处理器：4核 #主机环境(待安装)： docker、kubelet、kubeadm、kubectl、flannel
1.2 k8s-node1 节点
#系统环境： 内存：2G 处理器：2核 #主机环境(待安装)： docker、kubelet、kubeadm、kubectl、flannel
1.3 k8s-node2 节点
#系统环境： 内存：2G 处理器：2核 #主机环境(待安装)： docker、kubelet、kubeadm、kubectl、flannel
2 系统初始化（三个节点）
2.1 修改主机名，配置 hosts 解析
hostnamectl set-hostname {k8s-master | k8s-node1 | k8s-node2} exit #退出重新登录，使主机名生效 cat >> /etc/hosts <<EOF 192.168.81.131 k8s-master 192.168.81.132 k8s-node1 192.168.81.133 k8s-node2
EOF
2.2 关闭防火墙、禁用 selinux
#查看防火墙状态 firewall-cmd --state #停止firewall；禁止firewall开机启动 systemctl stop firewalld.service systemctl disable firewalld.service #防火墙设置iptables，并清空 yum install iptables-services -y systemctl start iptables systemctl enable iptables iptables -F service iptables save -y #关闭selinux getenforce setenforce 0 sed -i 's/^ *SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
2.3 安装依赖包
yum install -y conntrack ntpdate ntp ipvsadm ipset jp iptables curl sysstat libseccomp wget vim net-tools git
2.4 调整 k8s 内核参数
#加载模块 modprobe br_netfilter #调整参数 cat > kubernetes.conf <<EOF net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 net.ipv4.ip_forward=1 net.ipv4.tcp_tw_recycle=0 vm.swappiness=0 vm.overcommit_memory=1 vm.panic_on_oom=0 fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=2310720 EOF cp kubernetes.conf /etc/sysctl.d/kubernetes.conf sysctl -p /etc/sysctl.d/kubernetes.conf
2.5 调整系统时区
#设置时区为中国/上海 timedatectl set-timezone Asia/Shanghai #将当前的UTC时间写入硬件时间 timedatectl set-local-rtc 0 #重启依赖于时间的服务 systemctl restart rsyslog systemctl restart crond
2.6 关闭系统不需要的服务
systemctl stop postfix && systemctl disable postfix
2.7 设置 rsyslogd 和 systemd journald
mkdir /var/log/journal #持久化保存日志目录 mkdir /etc/systemd/journald.conf.d cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF [Journal] #持久化保存在磁盘 Storage=persistent #压缩历史日志 Compress=yes SyncIntervalSec=5m RateLimitInterval=30s RateLimitBurst=1000 #最大占用空间10G SystemMaxUse=10G #单日志文件最大200M SystemMaxFileSize=200M #日志保存时间2周 MaxRetentionSec=2week #不将日志转发到syslog ForwardToSyslog=no EOF systemctl restart systemd-journald
2.8 升级内核版本至4.4
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org yum install https://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
#添加 repository 后, 列出可以使用的kernel包版本 yum --disablerepo="*" --enablerepo="elrepo-kernel" list available
#安装需要的kernel版本，这里安装 kernel-lt(长期维护版) yum --enablerepo=elrepo-kernel install kernel-lt
#安装完成检查/boot/grub2/grub.cfg中对应的内核menuentry中是否包含initrd16配置，如果没有再安装一次 cat /boot/grub2/grub.cfg vi /etc/default/grub GRUB_DEFAULT=0
#重新创建内核配置 grub2-mkconfig -o /boot/grub2/grub.cfg
#重启 reboot
3 kubernetes 集群部署
3.1 kube-proxy 开启 ipvs 的前置条件（三个节点）
旧版本使用 iptables 做转发，新版本默认使用 lvs 做转发，所以进行如下设置：
modprobe br_netfilter cat > /etc/sysconfig/modules/ipvs.modules <<EOF #!/bin/bash modprobe -- ip_vs modprobe -- ip_vs_rr modprobe -- ip_vs_wrr modprobe -- ip_vs_sh modprobe -- nf_conntrack_ipv4 EOF chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod |grep -e ip_vs -e nf_conntrack_ipv4
3.2 安装 docker（三个节点）
#1、配置yum源 wget -O /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo yum clean all yum makecache #2、docker安装 yum install -y yum-utils device-mapper-persistent-data lvm2 yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo yum update -ycat /boot/grub2/grub.cfg | grep "CentOS Linux" grub2-set-default 'CentOS Linux (4.4.230-1.el7.elrepo.x86_64) 7 (Core)' grub2-mkconfig -o /boot/grub2/grub.cfg reboot
yum install -y docker-ce
#3、创建/etc/docker目录
mkdir /etc/docker
#配置daemon
cat > /etc/docker/daemon.json <<EOF
{
"registry-mirrors": ["https://162ncvra.mirror.aliyuncs.com"]
}
EOF
mkdir -p /etc/systemd/system/docker.service.d
#4、启动服务
systemctl daemon-reload && systemctl restart docker && systemctl enable docker
#5、命令补全：安装/加载bash-completion
yum install bash-completion -y
source /etc/profile.d/bash_completion.sh
#6、验证
docker --version
docker run hello-world
3.3 设置 kubernetes 源（三个节点）
#1、验证mac和uuid，保证各节点mac和uuid唯一 cat /sys/class/net/ens32/address cat /sys/class/dmi/id/product_uuid #2、禁用swap #临时生效 swapoff -a #永久生效 sed -i.bak '/swap/s/^/#/' /etc/fstab #3、内核参数修改 cat /proc/sys/net/bridge/bridge-nf-call-iptables cat /proc/sys/net/bridge/bridge-nf-call-ip6tables (1)临时修改 sysctl net.bridge.bridge-nf-call-iptables=1 sysctl net.bridge.bridge-nf-call-ip6tables=1 (2)永久修改 cat <<EOF > /etc/sysctl.d/k8s.conf net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 EOF sysctl -p /etc/sysctl.d/k8s.conf #4、修改cgroup driver #修改daemon.json,新增 "exec-opts": ["native.cgroupdriver=systemd"] cat /etc/docker/daemon.json { "registry-mirrors": ["https://162ncvra.mirror.aliyuncs.com"], "exec-opts": ["native.cgroupdriver=systemd"] } #重新加载docker systemctl daemon-reload systemctl restart docker #修改cgroup driver，是为了消除告警 #5、设置kubernetes源 cat <<EOF > /etc/yum.repos.d/kubernetes.repo [kubernetes] name=Kubernetes baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/ enabled=1 gpgcheck=1 repo_gpgcheck=1 gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg EOF yum clean all yum -y makecache
3.4 安装 kubernetes（三个节点）
yum list kubelet --showduplicates | sort -r yum install -y kubelet-1.15.1 kubeadm-1.15.1 kubectl-1.15.1 #建议指定版本安装，保持环境一致。不指定版本安装为最新版。 #kubelet：运行在集群所有节点上，用于启动pod和容器等对象的工具 #kubeadm：用于初始化集群，启动集群的命令工具 #kubectl：用于和集群通信的命令行，通过kubectl可以部署和管理应用，查看各种资源，创建删除和更新各种组件 #启动kubelet systemctl enable kubelet && systemctl start kubelet #kubelet命令补全 echo "source <(kubectl completion bash)" >> ~/.bash_profile
3.5 下载镜像（三个节点）
将下载好的包上传到服务器并解压
链接：https://pan.baidu.com/s/1H8TtCc1VfPTw_iCMgODsdw
提取码：u7sx
tar xvf kubeadm-basic.images.tar.gz #编写安装脚本 vim image.sh #!/bin/bash ls /root/kubeadm-basic.images > /tmp/image-list.txt cd /root/kubeadm-basic.images for i in $( cat /tmp/image-list.txt ) do docker load -i $i done
rm -rf /tmp/image-list.txt
#运行导入 sh image.sh
3.6 初始化配置 master 节点
初始化完成记录 node 节点加入集群的命令：
#初始化配置Master [root@k8s-master ~]# kubeadm config print init-defaults > kubeadm-config.yaml
#打开该配置文件，修改如下配置：
advertiseAddress：192.168.81.131
kubernetesVersion:v1.15.1
添加：podSubnet：10.244.0.0/16
并开通ipvs认证方式，文末添加如下： --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: kubeProxyConfiguration featureGates: SupportIPVSProxyMode: true mode: ipvs
[root@k8s-master ~]# vim kubeadm-config.yaml [root@k8s-master ~]# cat kubeadm-config.yaml apiVersion: kubeadm.k8s.io/v1beta2 bootstrapTokens: - groups: - system:bootstrappers:kubeadm:default-node-token token: abcdef.0123456789abcdef ttl: 24h0m0s usages: - signing - authentication kind: InitConfiguration localAPIEndpoint: advertiseAddress: 192.168.81.131 bindPort: 6443 nodeRegistration: criSocket: /var/run/dockershim.sock name: k8s-master taints: - effect: NoSchedule key: node-role.kubernetes.io/master --- apiServer: timeoutForControlPlane: 4m0s apiVersion: kubeadm.k8s.io/v1beta2 certificatesDir: /etc/kubernetes/pki clusterName: kubernetes controllerManager: {} dns: type: CoreDNS etcd: local: dataDir: /var/lib/etcd imageRepository: k8s.gcr.io kind: ClusterConfiguration kubernetesVersion: v1.15.1 networking: dnsDomain: cluster.local podSubnet: 10.244.0.0/16 serviceSubnet: 10.96.0.0/12 scheduler: {} --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: kubeProxyConfiguration featureGates: SupportIPVSProxyMode: true mode: ipvs #初始化Master [root@k8s-master ~]# kubeadm init --config=kubeadm-config.yaml --experimental-upload-certs|tee kubeadim-init.log
[root@k8s-master ~]# mkdir -p $HOME/.kube
[root@k8s-master ~]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@k8s-master ~]# sudo chown $(id -u):$(id -g) $HOME/.kube/config
3.7 master 节点，安装 pod 网络
[root@k8s-master ~]# kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
3.8 node 节点加入集群
node1 节点：
[root@k8s-node1 ~]# kubeadm join 192.168.81.131:6443 --token abcdef.0123456789abcdef \ --discovery-token-ca-cert-hash sha256:071999240d2fd37b38cffde6918ad00afc0adb6ff5385547f3cf0f633289ed85
node2 节点：
[root@k8s-node2 ~]# kubeadm join 192.168.81.131:6443 --token abcdef.0123456789abcdef \ --discovery-token-ca-cert-hash sha256:071999240d2fd37b38cffde6918ad00afc0adb6ff5385547f3cf0f633289ed85
4 kubernetes 集群验证
[root@k8s-master ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION k8s-master NotReady master 11m v1.15.1 k8s-node1 NotReady <none> 6m57s v1.15.1 k8s-node2 NotReady <none> 2m40s v1.15.1
