k8s
2019-11-13 16:55 寒霜冻土 阅读(236) 评论(0) 收藏 举报Kubernetes-V1.14.2 二进制编译安装
一、环境准备
1、操作系统环境初始化和准备工作：
1.1、节点规划
准备部署一个 一主两从 的 三节点 Kubernetes集群，整体节点规划如下表所示：
|
主机名 |
IP |
角色 |
|
k8s-master |
192.168.189.139 |
k8s主节点 |
|
k8s-node-1 |
192.168.189.131 |
k8s从节点 |
|
k8s-node-2 |
192.168.189.132 |
k8s从节点 |
下面介绍一下各个节点的软件版本：
操作系统：CentOS-7.6（1810）-64Bit
内核版本：4.18.9
Docker版本：18.09.6-3.el7
Kubernetes版本：1.14.2
2、安装系统并升级内核
1、安装CentOS 7.6-64bit最小化系统，注意：在安装过程中记得设置好网络参数，IPV4： 选择手动设置地址，DNS等，可以将IPV6忽略掉
2、系统安装好后，先安装一些基础工具
yum -y install bash-completion.noarch net-tools \
vim lrzsz wget tree screen lsof tcpdump unzip tree
3、上载先准备好的 kernel-ml-4.18.9.tar.gz
tar xf kernel-ml-4.18.9.tar.gz
cd /kernel-ml-4.18.9 && yum -y install *.rpm
cd /tmp && yum -y install *.rpm
sed -i 's#GRUB_DEFAULT#\#GRUB_DEFAULT#' /etc/default/grub
sed -i '3a GRUB_DEFAULT=0' /etc/default/grub
grub2-mkconfig -o /boot/grub2/grub.cfg
reboot
#重启系统查看内核版本
uname -r
# 版本号 4.18.9-1.el7.elrepo.x86_64
3、系统准备和优化
#关闭selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
#关闭防火墙
setenforce 0
systemctl stop firewalld && systemctl disable firewalld
#远程连接优化
sed -i 's#GSSAPIAuthentication yes#GSSAPIAuthentication no#g' /etc/ssh/sshd_config
sed -i 's#\#UseDNS yes#UseDNS no#g' /etc/ssh/sshd_config
##开启密码登陆
sed -i 's#PasswordAuthentication no#PasswordAuthentication yes#g' /etc/ssh/sshd_config
#开启root远程登陆
sed -i 's#PermitRootLogin without-password#PermitRootLogin yes#g' /etc/ssh/sshd_config
#重启sshd服务
systemctl restart sshd
#修改系统时区
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
#修改配置yum源
rm -rf /etc/yum.repos.d/*
curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
yum makecache
#安装基础命令
yum -y install bash-completion.noarch net-tools vim lrzsz wget tree screen lsof tcpdump
#关闭swapp
swapoff -a
#为了防止开机自动挂载 swap 分区，可以注释 /etc/fstab 中相应的条目
$ sudo sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
# centos7 下需要给网卡建立桥接
vim /etc/sysctl.conf
vm.swappiness = 0
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
sysctl -p
###PS:建立桥接时出现文件不存在错误，用 modprobe br_netfilter
4、安装docker
#卸载旧版本
yum remove docker docker-client docker-client-latest docker-common \
docker-latest docker-latest-logrotate docker-logrotate docker-engine
#安装docker yum源
yum install -y yum-utils
yum-config-manager --add-repo \
http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
#安装docker-ce
yum -y install docker-ce-18.06.1.ce-3.el7
systemctl restart docker
systemctl enable docker && systemctl start docker
5、主机名、DNS设置、免密认证
1）分别设置各节点主机名
hostnamectl --static set-hostname k8s-master
hostnamectl --static set-hostname k8s-node-1
hostnamectl --static set-hostname k8s-node-2
2）所有节点 主机名/IP加入 hosts解析
vim /etc/hosts
192.168.189.139 k8s-master
192.168.189.131 k8s-node-1
192.168.189.132 k8s-node-2
3）免密认证
ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa
cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
scp -rp /root/.ssh k8s-node-1:/root
scp -rp /root/.ssh k8s-node-2:/root
ssh k8s-master
ssh k8s-node-1
ssh k8s-node-2
6、在各主机分别创建kubernetes目录
mkdir -p /app/kubernetes/{cfg,bin,ssl,log}
至此，编译安装kubernetes 1.14.2 二进制版准备工作完成
二、安装etcd
1、安装配置CFSSL
1.1、下载配置CFSSL
# 使用wget下载cfssl包，也可rz直接导入，文档附带资源包
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
# 移动下载好的包
mv cfssl_linux-amd64 /app/kubernetes/bin/cfssl
mv cfssljson_linux-amd64 /app/kubernetes/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /app/kubernetes/bin/cfssl-certinfo
# 赋予执行权限
chmod +x /app/kubernetes/bin/*
# 设置系统环境变量（可二选一，推荐1）
# 方式1：
vim /etc/profile
export PATH=$PATH:/app/kubernetes/bin
source /etc/profile
#方式2：
mv cfssl_linux-amd64 /usr/local/bin/cfssl
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo
# cfssl同步到两个节点上
scp /app/kubernetes/bin/cfssl* k8s-node-1:/app/kubernetes/bin
scp /app/kubernetes/bin/cfssl* k8s-node-2:/app/kubernetes/bin
# 同步profile配置文件
scp /etc/profile k8s-node-1:/etc
scp /etc/profile k8s-node-2:/etc
# 然后分别在两个节点上 执行
source /etc/profile
1.2、初始化CFSSL
mkdir /usr/local/src/ssl && cd /usr/local/src/ssl
cfssl print-defaults config > ca-config.json
cfssl print-defaults csr > ca-csr.json
1.3、创建用来生成 CA 文件的 JSON 配置文件
vim ca-config.json
{
"signing": {
"default": {
"expiry": "8760h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "8760h"
}
}
}
}
1.4、创建用来生成 CA 证书签名请求（CSR）的 JSON 配置文件
vim ca-csr.json
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
1.5、生成CA证书（ca.pem）和密钥（ca-key.pem）
cfssl gencert -initca ca-csr.json | cfssljson -bare ca
[root@k8s-master-2 ssl]# ls -l ca*
-rw-r--r-- 1 root root 290 6月 11 11:52 ca-config.json
-rw-r--r-- 1 root root 1001 6月 11 11:55 ca.csr
-rw-r--r-- 1 root root 208 6月 11 11:53 ca-csr.json
-rw------- 1 root root 1679 6月 11 11:55 ca-key.pem
-rw-r--r-- 1 root root 1359 6月 11 11:55 ca.pem
1.6、分发证书
cp ca.csr ca.pem ca-key.pem ca-config.json /app/kubernetes/ssl
scp ca.csr ca.pem ca-key.pem ca-config.json k8s-node-1:/app/kubernetes/ssl
scp ca.csr ca.pem ca-key.pem ca-config.json k8s-node-2:/app/kubernetes/ssl
2、安装etcd
2.1、 准备etcd软件包
wget https://github.com/etcd-io/etcd/releases/download/v3.3.13/\
etcd-v3.3.13-linux-amd64.tar.gz
tar xf etcd-v3.3.13-linux-amd64.tar.gz && cd etcd-v3.3.13-linux-amd64/
cp etcd* /app/kubernetes/bin/
scp etcd* k8s-node-1:/app/kubernetes/bin/
scp etcd* k8s-node-2:/app/kubernetes/bin/
2.2、 创建 etcd 证书签名请求
vim etcd-csr.json
# 注意将hosts中的ip地址改写为自己机器的，加入集群中所有节点
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.189.139",
"192.168.189.131",
"192.168.189.132"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
2.3、 生成 etcd 证书和私钥
cfssl gencert -ca=/app/kubernetes/ssl/ca.pem \
-ca-key=/app/kubernetes/ssl/ca-key.pem \
-config=/app/kubernetes/ssl/ca-config.json \
-profile=kubernetes etcd-csr.json | cfssljson -bare etcd
2.4、 将证书移动到/opt/kubernetes/ssl目录下
scp etcd*.pem /app/kubernetes/ssl/
scp etcd*.pem k8s-node-1:/app/kubernetes/ssl/
scp etcd*.pem k8s-node-2:/app/kubernetes/ssl/
rm -rf etcd.csr etcd-csr.json
2.5、 设置ETCD配置文件
vim /app/kubernetes/cfg/etcd.conf
#[member]
ETCD_NAME="etcd-node1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
# ETCD_SNAPSHOT_COUNTER="10000"
# ETCD_HEARTBEAT_INTERVAL="100"
# ETCD_ELECTION_TIMEOUT="1000"
# 注意将ip地址改为自己机器的
ETCD_LISTEN_PEER_URLS="https://192.168.189.139:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.189.139:2379,https://127.0.0.1:2379"
# ETCD_MAX_SNAPSHOTS="5"
# ETCD_MAX_WALS="5"
# ETCD_CORS=""
#[cluster]
# 注意将ip地址改为自己机器的
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.189.139:2380"
# if you use different ETCD_NAME (e.g. test),
# set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
ETCD_INITIAL_CLUSTER="etcd-node1=https://192.168.189.139:2380,etcd-node2=https://192.168.189.131:2380,etcd-node3=https://192.168.189.132:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
# 注意将ip地址改为自己机器的
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.189.139:2379"
#[security]
CLIENT_CERT_AUTH="true"
ETCD_CA_FILE="/app/kubernetes/ssl/ca.pem"
ETCD_CERT_FILE="/app/kubernetes/ssl/etcd.pem"
ETCD_KEY_FILE="/app/kubernetes/ssl/etcd-key.pem"
PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_CA_FILE="/app/kubernetes/ssl/ca.pem"
ETCD_PEER_CERT_FILE="/app/kubernetes/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/app/kubernetes/ssl/etcd-key.pem"
2.6、 创建etcd启动服务
vim /etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
[Service]
Type=simple
WorkingDirectory=/var/lib/etcd
EnvironmentFile=-/app/kubernetes/cfg/etcd.conf
# set GOMAXPROCS to number of processors
ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /app/kubernetes/bin/etcd"
Type=notify
[Install]
WantedBy=multi-user.target
2.7、 复制相关文件到其它机器
scp /app/kubernetes/cfg/etcd.conf k8s-node-1:/app/kubernetes/cfg/
scp /app/kubernetes/cfg/etcd.conf k8s-node-2:/app/kubernetes/cfg/
scp /etc/systemd/system/etcd.service k8s-node-1:/etc/systemd/system/
scp /etc/systemd/system/etcd.service k8s-node-2:/etc/systemd/system/
修改另外两台机器的etcd.conf文件中的ETCD_NAME和监听IP，如下图所示，其中标蓝线的地方，ip都更换为当前机器的ip
然后在每个机器上面创建 /var/lib/etcd的文件夹 mkdir /var/lib/etcd 然后启动服务
systemctl daemon-reload && systemctl enable etcd
systemctl start etcd && systemctl status etcd
systemctl daemon-reload&& systemctl restart etcd && systemctl status etcd
2.8、 验证集群
复制etcdctl可执行文件到/usr/local/bin下 cp etcdctl /usr/local/bin 文件在解压包中
etcdctl --endpoints=https://192.168.189.139:2379 \
--ca-file=/app/kubernetes/ssl/ca.pem \
--cert-file=/app/kubernetes/ssl/etcd.pem \
--key-file=/app/kubernetes/ssl/etcd-key.pem cluster-health
如果成功，则输出
[root@k8s-master-2 etcd-v3.3.13-linux-amd64]# etcdctl \
--endpoints=https://192.168.189.139:2379 \
--ca-file=/app/kubernetes/ssl/ca.pem \
--cert-file=/app/kubernetes/ssl/etcd.pem \
--key-file=/app/kubernetes/ssl/etcd-key.pem cluster-health
member 503e8f4db410fddd is healthy: got healthy result from \
https://192.168.189.139:2379
member 9e40edb005055a20 is healthy: got healthy result from \
https://192.168.189.131:2379
member e2e15ce4d2c75a68 is healthy: got healthy result from \
https://192.168.189.132:2379
三、master节点部署
3.1 复制二进制文件
tar xf kubernetes-server-linux-amd64.tar.gz
cd /root/kubernetes/server/bin
cp kube-apiserver kube-controller-manager kube-scheduler /app/kubernetes/bin/
3.2 部署kube-apiserver
3.2.1、 创建生成CSR的 JSON 配置文件
vim kubernetes-csr.json
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
### 需要将IP改为自己master的IP，并将注释删除
"192.168.189.139",
"10.1.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
3.2.2、 生成 kubernetes 证书和私钥
cfssl gencert -ca=/app/kubernetes/ssl/ca.pem \
-ca-key=/app/kubernetes/ssl/ca-key.pem \
-config=/app/kubernetes/ssl/ca-config.json \
-profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
cp kubernetes*.pem /app/kubernetes/ssl/
scp kubernetes*.pem k8s-node-1:/app/kubernetes/ssl/
scp kubernetes*.pem k8s-node-2:/app/kubernetes/ssl/
3.2.3、 创建 kube-apiserver 使用的客户端 token 文件
### 下面这个命令会产生一串字符串，请保存到文本，下面需要多次使用
head -c 16 /dev/urandom | od -An -t x | tr -d ' '
#### 创建这个文件
vim /app/kubernetes/ssl/bootstrap-token.csv
### 用生成的字符串替换下面的 “8fa448.... ”这一串数字
8fa4488b7751e2ba9f58555a8cb0cef4,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
3.2.4、 创建基础用户名/密码认证配置
vim /app/kubernetes/ssl/basic-auth.csv
admin,admin,1
readonly,readonly,2
3.2.5、 配置kubernetes apiserver 启动服务
vim /usr/lib/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
ExecStart=/app/kubernetes/bin/kube-apiserver \
--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,NodeRestriction \
--bind-address=192.168.189.139 \
--insecure-bind-address=127.0.0.1 \
--authorization-mode=Node,RBAC \
--runtime-config=rbac.authorization.k8s.io/v1 \
--kubelet-https=true \
--anonymous-auth=false \
--basic-auth-file=/app/kubernetes/ssl/basic-auth.csv \
--enable-bootstrap-token-auth \
--token-auth-file=/app/kubernetes/ssl/bootstrap-token.csv \
--service-cluster-ip-range=10.1.0.0/16 \
--service-node-port-range=9000-40000 \
--tls-cert-file=/app/kubernetes/ssl/kubernetes.pem \
--tls-private-key-file=/app/kubernetes/ssl/kubernetes-key.pem \
--client-ca-file=/app/kubernetes/ssl/ca.pem \
--service-account-key-file=/app/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/app/kubernetes/ssl/ca.pem \
--etcd-certfile=/app/kubernetes/ssl/kubernetes.pem \
--etcd-keyfile=/app/kubernetes/ssl/kubernetes-key.pem \
--etcd-servers=https://192.168.189.139:2379,https://192.168.189.131:2379,https://192.168.189.132:2379 \
--enable-swagger-ui=true \
--allow-privileged=true \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/app/kubernetes/log/api-audit.log \
--event-ttl=1h \
--v=2 \
--logtostderr=false \
--log-dir=/app/kubernetes/log
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
注意：
--bind-address=192.168.189.139 将其换成自己的master节点IP
--etcd-servers=https： 后面的IP分别换成自己的master和node节点的IP
3.2.6、 启动服务
systemctl daemon-reload
systemctl enable kube-apiserver
systemctl start kube-apiserver
systemctl status kube-apiserver
3.3 部署controller manager服务
3.3.1、创建kube-controller-manager.service
vim /usr/lib/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
[Service]
ExecStart=/app/kubernetes/bin/kube-controller-manager \
--address=127.0.0.1 \
--master=http://127.0.0.1:8080 \
--allocate-node-cidrs=true \
--service-cluster-ip-range=10.1.0.0/16 \
--cluster-cidr=10.2.0.0/16 \
--cluster-name=kubernetes \
--cluster-signing-cert-file=/app/kubernetes/ssl/ca.pem \
--cluster-signing-key-file=/app/kubernetes/ssl/ca-key.pem \
--service-account-private-key-file=/app/kubernetes/ssl/ca-key.pem \
--root-ca-file=/app/kubernetes/ssl/ca.pem \
--leader-elect=true \
--v=2 \
--logtostderr=false \
--log-dir=/app/kubernetes/log
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
3.3.2、启动kube-controller-manager
systemctl daemon-reload
systemctl enable kube-controller-manager && systemctl start kube-controller-manager
systemctl status kube-controller-manager
3.4 部署kube-scheduler服务
3.4.1创建kube-scheduler.service
vim /usr/lib/systemd/system/kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
[Service]
ExecStart=/app/kubernetes/bin/kube-scheduler \
--address=127.0.0.1 \
--master=http://127.0.0.1:8080 \
--leader-elect=true \
--v=2 \
--logtostderr=false \
--log-dir=/app/kubernetes/log
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
3.4.2启动kube-scheduler.service
systemctl daemon-reload
systemctl enable kube-scheduler && systemctl start kube-scheduler
systemctl status kube-scheduler
3.5 部署kubectl 命令行工具
3.5.1 复制二进制包
cd kubernetes/server/bin/
cp kubectl /app/kubernetes/bin/
3.5.2 创建 admin 证书签名请求
vim admin-csr.json
{
"CN": "admin",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "system:masters",
"OU": "System"
}
]
}
3.5.3 生成 admin 证书和私钥
cfssl gencert -ca=/app/kubernetes/ssl/ca.pem \
-ca-key=/app/kubernetes/ssl/ca-key.pem \
-config=/app/kubernetes/ssl/ca-config.json \
-profile=kubernetes admin-csr.json | cfssljson -bare admin
# 把生成好的密钥放入到kubernetes下的ssl中
cp admin*.pem /app/kubernetes/ssl/
3.5.4 设置集群参数
将kubectl可执行文件复制到 /usr/local/bin下 文件在解压包中
kubectl config set-cluster kubernetes \
--certificate-authority=/app/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=https://192.168.189.139:6443
#### 注意将IP换为自己的master节点IP
3.5.5 设置客户端认证参数
kubectl config set-credentials admin \
--client-certificate=/app/kubernetes/ssl/admin.pem \
--embed-certs=true \
--client-key=/app/kubernetes/ssl/admin-key.pem
3.5.6 设置上下文参数
kubectl config set-context kubernetes \
--cluster=kubernetes \
--user=admin
3.5.7 设置默认上下文
kubectl config use-context kubernetes
3.5.8 查看服务是否正常
kubectl get cs
正常则为：
[root@k8s-master bin]# kubectl get cs
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-0 Healthy {"health":"true"}
etcd-2 Healthy {"health":"true"}
etcd-1 Healthy {"health":"true"}
3.6 进行node节点安装部署
3.6.1 复制二进制包
cd ../kubernetes/server/bin/
cp kubelet kube-proxy /app/kubernetes/bin/
scp kubelet kube-proxy k8s-node-1:/app/kubernetes/bin/
scp kubelet kube-proxy k8s-node-2:/app/kubernetes/bin/
3.6.2 创建角色绑定
kubectl create clusterrolebinding kubelet-bootstrap \
--clusterrole=system:node-bootstrapper --user=kubelet-bootstrap
3.6.3 创建 kubelet bootstrapping kubeconfig 文件
设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/app/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=https://192.168.189.139:6443 \
--kubeconfig=bootstrap.kubeconfig
设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \
--token=8fa4488b7751e2ba9f58555a8cb0cef4 \
--kubeconfig=bootstrap.kubeconfig
### 注意：：--token的参数为上篇中的kube-apiserver
### 使用的客户端 token 文件中的那个token
设置上下文参数
kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=bootstrap.kubeconfig
选择默认上下文
kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
scp bootstrap.kubeconfig /app/kubernetes/cfg/
scp bootstrap.kubeconfig k8s-node-1:/app/kubernetes/cfg/
scp bootstrap.kubeconfig k8s-node-2:/app/kubernetes/cfg/
3.7 部署kubelet
# 设置CNI支持
mkdir -p /etc/cni/net.d
vim /etc/cni/net.d/10-default.conf
{
"name": "flannel",
"type": "flannel",
"delegate": {
"bridge": "docker0",
"isDefaultGateway": true,
"mtu": 1400
}
}
# 创建kubelet目录
mkdir /var/lib/kubelet
创建kubelet启动服务配置
vim /usr/lib/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service
[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/app/kubernetes/bin/kubelet \
--address=192.168.189.139 \
--hostname-override=192.168.189.139 \
--pod-infra-container-image=mirrorgooglecontainers/pause-amd64:3.0 \
--experimental-bootstrap-kubeconfig=/app/kubernetes/cfg/bootstrap.kubeconfig \
--kubeconfig=/app/kubernetes/cfg/kubelet.kubeconfig \
--cert-dir=/app/kubernetes/ssl \
--network-plugin=cni \
--cni-conf-dir=/etc/cni/net.d \
--cni-bin-dir=/app/kubernetes/bin/cni \
--cluster-dns=10.1.0.2 \
--cluster-domain=cluster.local. \
--hairpin-mode hairpin-veth \
--allow-privileged=true \
--fail-swap-on=false \
--logtostderr=true \
--v=2 \
--logtostderr=false \
--log-dir=/app/kubernetes/log
Restart=on-failure
RestartSec=5
启动服务
systemctl daemon-reload
systemctl enable kubelet && systemctl start kubelet
systemctl status kubelet
在其他机器上创建下面2个文件夹
mkdir -p /etc/cni/net.d
mkdir /var/lib/kubelet
在主节点使用下面命令分发
scp /etc/cni/net.d/10-default.conf k8s-node-1:/etc/cni/net.d/10-default.conf
scp /etc/cni/net.d/10-default.conf k8s-node-2:/etc/cni/net.d/10-default.conf
下面2个文件要去对应机器修改IP
scp /usr/lib/systemd/system/kubelet.service \
k8s-node-1:/usr/lib/systemd/system/kubelet.service
scp /usr/lib/systemd/system/kubelet.service \
k8s-node-2:/usr/lib/systemd/system/kubelet.service
# 使用下面的命令，里面的两个IP都改为当前IP
vim /usr/lib/systemd/system/kubelet.service
在各个机器启动服务
systemctl daemon-reload
systemctl enable kubelet && systemctl start kubelet
systemctl status kubelet
查看csr请求
kubectl get csr
结果如下：
[root@k8s-master bin]# kubectl get csr
NAME AGE REQUESTOR CONDITION
node-csr-M7NH4jxhj... 8m4s kubelet-bootstrap Pending
node-csr-XjMnA02kn... 30s kubelet-bootstrap Pending
node-csr-uIqmVSeTF... 26s kubelet-bootstrap Pending
通过kubelet 的 TLS 证书请求
kubectl get csr|grep 'Pending' | awk 'NR>0{print $1}'| xargs \
kubectl certificate approve
kubectl get node
3.8 部署kube-proxy
配置kube-proxy使用LVS
yum install -y ipvsadm ipset conntrack
创建 kube-proxy 证书请求
vim kube-proxy-csr.json
{
"CN": "system:kube-proxy",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
生成证书
cfssl gencert -ca=/app/kubernetes/ssl/ca.pem \
-ca-key=/app/kubernetes/ssl/ca-key.pem \
-config=/app/kubernetes/ssl/ca-config.json \
-profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
复制证书到所有node节点
cp kube-proxy*.pem /app/kubernetes/ssl/
scp kube-proxy*.pem k8s-node-1:/app/kubernetes/ssl/
scp kube-proxy*.pem k8s-node-2:/app/kubernetes/ssl/
创建kube-proxy配置文件
kubectl config set-cluster kubernetes \
--certificate-authority=/app/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=https://192.168.189.139:6443 \
--kubeconfig=kube-proxy.kubeconfig
kubectl config set-credentials kube-proxy \
--client-certificate=/app/kubernetes/ssl/kube-proxy.pem \
--client-key=/app/kubernetes/ssl/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig
kubectl config set-context default \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig
kubectl config use-context default \
--kubeconfig=kube-proxy.kubeconfig
分发kubeconfig配置文件
scp kube-proxy.kubeconfig /app/kubernetes/cfg/
scp kube-proxy.kubeconfig k8s-node-1:/app/kubernetes/cfg/
scp kube-proxy.kubeconfig k8s-node-2:/app/kubernetes/cfg/
配置启动服务
#各节点都执行 并修改对应IP
mkdir /var/lib/kube-proxy
创建 kube-proxy.service
vim /usr/lib/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/app/kubernetes/bin/kube-proxy \
--bind-address=192.168.189.139 \
--hostname-override=192.168.189.139 \
--kubeconfig=/app/kubernetes/cfg/kube-proxy.kubeconfig \
--masquerade-all \
--feature-gates=SupportIPVSProxyMode=true \
--proxy-mode=ipvs \
--ipvs-min-sync-period=5s \
--ipvs-sync-period=5s \
--ipvs-scheduler=rr \
--logtostderr=true \
--v=2 \
--logtostderr=false \
--log-dir=/app/kubernetes/log
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
同步到其他节点
scp /usr/lib/systemd/system/kube-proxy.service \
k8s-node-1:/usr/lib/systemd/system/
scp /usr/lib/systemd/system/kube-proxy.service \
k8s-node-2:/usr/lib/systemd/system/
修改kube-proxy.service中的IP
vim /usr/lib/systemd/system/kube-proxy.service
各节点都启动服务
systemctl daemon-reload
systemctl enable kube-proxy && systemctl start kube-proxy
systemctl status kube-proxy
3.9 配置flannel网络
3.9.1 生成flannel证书
vim flanneld-csr.json
{
"CN": "flanneld",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
3.9.2 生成证书
cfssl gencert -ca=/app/kubernetes/ssl/ca.pem \
-ca-key=/app/kubernetes/ssl/ca-key.pem \
-config=/app/kubernetes/ssl/ca-config.json \
-profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld
3.9.3 分发证书
cp flanneld*.pem /app/kubernetes/ssl/
scp flanneld*.pem k8s-node-1:/app/kubernetes/ssl/
scp flanneld*.pem k8s-node-2:/app/kubernetes/ssl/
3.9.4 下载Flannel软件包相关软件包
wget https://github.com/coreos/flannel/releases/download/v0.11.0/\
flannel-v0.11.0-linux-amd64.tar.gz
tar xf flannel-v0.11.0-linux-amd64.tar.gz
cd flannel-v0.11.0-linux-amd64/
cp flanneld mk-docker-opts.sh /app/kubernetes/bin/
scp flanneld mk-docker-opts.sh k8s-node-1:/app/kubernetes/bin/
scp flanneld mk-docker-opts.sh k8s-node-2:/app/kubernetes/bin/
wget https://github.com/kubernetes/kubernetes/releases/download/v1.14.2/\
kubernetes.tar.gz
tar xf kubernetes.tar.gz && cd kubernetes/cluster/centos/node/bin
cp remove-docker0.sh /app/kubernetes/bin/
scp remove-docker0.sh k8s-node-1:/app/kubernetes/bin/
scp remove-docker0.sh k8s-node-2:/app/kubernetes/bin/
3.9.5 配置flannel
vim /app/kubernetes/cfg/flannel
FLANNEL_ETCD="--etcd-endpoints=https://192.168.189.139:2379,
https://192.168.189.131:2379,https://192.168.189.132:2379"
FLANNEL_ETCD_KEY="--etcd-prefix=/kubernetes/network"
FLANNEL_ETCD_CAFILE="--etcd-cafile=/app/kubernetes/ssl/ca.pem"
FLANNEL_ETCD_CERTFILE="--etcd-certfile=/app/kubernetes/ssl/flanneld.pem"
FLANNEL_ETCD_KEYFILE="--etcd-keyfile=/app/kubernetes/ssl/flanneld-key.pem"
同步到其他节点
scp /app/kubernetes/cfg/flannel k8s-node-1:/app/kubernetes/cfg
scp /app/kubernetes/cfg/flannel k8s-node-2:/app/kubernetes/cfg
3.9.6 设置Flannel系统服务
vim /usr/lib/systemd/system/flannel.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
Before=docker.service
[Service]
EnvironmentFile=-/app/kubernetes/cfg/flannel
ExecStartPre=/app/kubernetes/bin/remove-docker0.sh
ExecStart=/app/kubernetes/bin/flanneld ${FLANNEL_ETCD} ${FLANNEL_ETCD_KEY} ${FLANNEL_ETCD_CAFILE} ${FLANNEL_ETCD_CERTFILE} ${FLANNEL_ETCD_KEYFILE}
ExecStartPost=/app/kubernetes/bin/mk-docker-apps.sh -d /run/flannel/docker
Type=notify
[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
由于上面文件的配置ExecStartPost=/app/kubernetes/bin/mk-docker-apps.sh
需要将原来 /app/kubernetes/bin下的mk-docker-opts.sh重命名成mk-docker-apps.sh
mv mk-docker-opts.sh mk-docker-apps.sh
分发启动服务
scp /usr/lib/systemd/system/flannel.service \
k8s-node-1:/usr/lib/systemd/system/flannel.service
scp /usr/lib/systemd/system/flannel.service \
k8s-node-2:/usr/lib/systemd/system/flannel.service
3.9.7 flannel cni插件配置
wget https://github.com/containernetworking/plugins/releases/download/\
v0.8.0/cni-plugins-linux-amd64-v0.8.0.tgz
mkdir /app/kubernetes/bin/cni
tar -zxf cni-plugins-linux-amd64-v0.8.0.tgz -C /app/kubernetes/bin/cni/
scp -r /app/kubernetes/bin/cni/ k8s-node-1:/app/kubernetes/bin/
scp -r /app/kubernetes/bin/cni/ k8s-node-2:/app/kubernetes/bin/
3.9.8 创建etcd的key
/app/kubernetes/bin/etcdctl --ca-file /app/kubernetes/ssl/ca.pem \
--cert-file /app/kubernetes/ssl/flanneld.pem \
--key-file /app/kubernetes/ssl/flanneld-key.pem \
--no-sync -C https://192.168.189.139:2379,https://192.168.189.131:2379,\
https://192.168.189.132:2379 \
mk /kubernetes/network/config '{ "Network": "10.2.0.0/16", "Backend": { "Type": "vxlan", "VNI": 1 }}' >/dev/null 2>&1
3.9.9 启动flannel
systemctl daemon-reload && systemctl start flannel
systemctl status flannel
3.9.10 修改docker启动服务
vim /usr/lib/systemd/system/docker.service
[Unit] #在Unit下面修改After和增加Requires
After=network-online.target firewalld.service flannel.service
Wants=network-online.target
Requires=flannel.service
[Service] #增加EnvironmentFile=-/run/flannel/docker
Type=notify
EnvironmentFile=-/run/flannel/docker
ExecStart=/usr/bin/dockerd $DOCKER_OPTS
3.9.11 分发到其它服务器
scp /usr/lib/systemd/system/docker.service \
k8s-node-1:/usr/lib/systemd/system/docker.service
scp /usr/lib/systemd/system/docker.service \
k8s-node-2:/usr/lib/systemd/system/docker.service
3.9.12 重启docker
systemctl daemon-reload && systemctl restart docker
浙公网安备 33010602011771号