临时用的

随便写写

脚本路径

hostnamectl set-hostname k8s01.example.com
systemctl stop firewalld
systemctl disable firewalld
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
sed -ri 's/.*swap.*/#&/' /etc/fstab
swapoff -a
yum install wget lrzsz bash-completion net-tools ipvsadm -y
yum install -y conntrack-tools
echo "* soft nofile 65536" >> /etc/security/limits.conf
echo "* hard nofile 65536" >> /etc/security/limits.conf
echo "* soft nproc 65536"  >> /etc/security/limits.conf
echo "* hard nproc 65536"  >> /etc/security/limits.conf
echo "* soft  memlock  unlimited"  >> /etc/security/limits.conf
echo "* soft  memlock  unlimited"  >> /etc/security/limits.conf

cat >> /etc/sysctl.conf << EOF
# 禁止ipv6的监听会导致rpcbind不能启动,因为nfs会监听ipv6
#net.ipv6.conf.all.disable_ipv6 = 1
#net.ipv6.conf.default.disable_ipv6 = 1
#net.ipv6.conf.lo.disable_ipv6 = 1

vm.swappiness = 0
net.ipv4.neigh.default.gc_stale_time=120
net.ipv4.ip_forward = 1

# see details in https://help.aliyun.com/knowledge_detail/39428.html
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.default.rp_filter=0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce=2
net.ipv4.conf.all.arp_announce=2


# see details in https://help.aliyun.com/knowledge_detail/41334.html
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
kernel.sysrq = 1
# 允许绑定不存在的ip
net.ipv4.ip_nonlocal_bind = 1
net.ipv6.ip_nonlocal_bind = 1

# 允许转发
net.ipv4.ip_forward = 1
#
# iptables透明网桥的实现
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-arptables = 1
EOF
sysctl -p

cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
modprobe br_netfilter
EOF
#echo "br_netfilter" | tee /etc/modprobe.d/br_netfilter.conf

 find . -type f | xargs chmod +x
mv cfssl/cfssl* /usr/local/bin/
mv etcd/etcd* /usr/local/bin/
mv kubernetes/* /usr/local/bin/
mv containerd/* /usr/local/bin/

• 证书的O等于创建后的k8s的rbac里面的组的意思，所以要一一对应

node01ip=192.168.58.162
master01ip=192.168.58.162
mkdir /root/ssl
cd /root/ssl
cat >> ca-config.json  << EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}
EOF
# 创建 CA 证书签名请求
cat >> ca-csr.json << EOF
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "ShenZhen",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF
/usr/local/bin/cfssl gencert -initca ca-csr.json | cfssljson -bare ca
cat >> etcd-csr.json << EOF
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "${node01ip}",
    "k8s01.example.com",
    "k8s02.example.com",
    "k8s03.example.com"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "ShenZhen",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF
/usr/local/bin/cfssl gencert -ca=ca.pem \
-ca-key=ca-key.pem \
-config=ca-config.json \
-profile=kubernetes etcd-csr.json | cfssljson -bare etcd
cat >> kubernetes-csr.json << EOF
{
  "CN": "kubernetes",
  "hosts": [
    "127.0.0.1",
    "10.96.0.1",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local",
    "${master01ip}",
    "${node01ip}",
    "k8s01.example.com",
    "k8s02.example.com",
    "k8s03.example.com"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "ShenZhen",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes

cat >> admin-csr.json << EOF
{
  "CN": "admin",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "ShenZhen",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin

cat >> kube-proxy-csr.json << EOF
{
  "CN": "system:kube-proxy",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "ShenZhen",
      "O": "system:kube-proxy",
      "OU": "System"
    }
  ]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes  kube-proxy-csr.json | cfssljson -bare kube-proxy

cat >> manager-csr.json << EOF
{
    "CN": "system:kube-controller-manager",
    "key": {
      "algo": "rsa",
      "size": 2048
    },
    "names": [
      {
        "C": "CN",
        "ST": "Fujian",
        "L": "XieMen",
        "O": "system:kube-controller-manager",
        "OU": "Kubernetes-manual"
      }
    ]
  }
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes  manager-csr.json | cfssljson -bare  controller-manager

cat > scheduler-csr.json << EOF
{
  "CN": "system:kube-scheduler",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "system:kube-scheduler",
      "OU": "Kubernetes-manual"
    }
  ]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes  scheduler-csr.json | cfssljson -bare  scheduler

cat >> sa-csr.json << EOF
{
    "CN": "system:kube-controller-manager",
    "key": {
      "algo": "rsa",
      "size": 2048
    },
    "names": [
      {
        "C": "CN",
        "ST": "Fujian",
        "L": "XieMen",
        "O": "system:kube-controller-manager",
        "OU": "Kubernetes-manual"
      }
    ]
  }
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes  sa-csr.json | cfssljson -bare sa


mkdir -p /etc/kubernetes/ssl
cp *.pem /etc/kubernetes/ssl


master01ip=192.168.58.162
export K8S_API_URL=${master01ip}

/usr/local/bin/kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=https://${K8S_API_URL}:6443 \
--kubeconfig=controller-manager.conf


/usr/local/bin/kubectl  config set-credentials system:kube-controller-manager \
--client-certificate=/etc/kubernetes/ssl/controller-manager.pem \
--client-key=/etc/kubernetes/ssl/controller-manager-key.pem \
--embed-certs=true \
--kubeconfig=controller-manager.conf


/usr/local/bin/kubectl  config set-context system:kube-controller-manager@kubernetes \
--cluster=kubernetes \
--user=system:kube-controller-manager \
--kubeconfig=controller-manager.conf

# 设置默认上下文
/usr/local/bin/kubectl  config use-context system:kube-controller-manager@kubernetes --kubeconfig=controller-manager.conf
\mv controller-manager.conf /etc/kubernetes/

/usr/local/bin/kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=https://${K8S_API_URL}:6443 \
--kubeconfig=scheduler.conf


/usr/local/bin/kubectl  config set-credentials system:kube-scheduler  \
--client-certificate=/etc/kubernetes/ssl/scheduler.pem \
--client-key=/etc/kubernetes/ssl/scheduler-key.pem \
--embed-certs=true \
--kubeconfig=scheduler.conf


/usr/local/bin/kubectl  config set-context system:kube-scheduler@kubernetes \
--cluster=kubernetes \
--user=system:kube-scheduler \
--kubeconfig=scheduler.conf

# 设置默认上下文
/usr/local/bin/kubectl  config use-context system:kube-scheduler@kubernetes --kubeconfig=scheduler.conf
\mv scheduler.conf /etc/kubernetes/

export BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')
cat > bootstrap-token.csv <<EOF
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
mv bootstrap-token.csv /etc/kubernetes/ssl/

cat > /usr/lib/systemd/system/etcd.service << \EOF
[Unit]
Description=Etcd Server
After=network.target

[Service]
Type=notify
ExecStartPre=-/bin/mkdir -p /etc/etcd/
ExecStartPre=-/bin/mkdir -p /var/lib/etcd
WorkingDirectory=/var/lib/etcd
EnvironmentFile=-/etc/etcd/etcd.conf
# set GOMAXPROCS to number of processors
ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/local/bin/etcd"

[Install]
WantedBy=multi-user.target
EOF

mkdir -p /etc/etcd/ /var/lib/etcd

cat > /etc/etcd/etcd.conf << \EOF
ETCD_NAME="etcd-node01"
ETCD_DATA_DIR="/var/lib/etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.58.162:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.58.162:2379,https://127.0.0.1:2379,http://127.0.0.1:4001"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.58.162:2380"
ETCD_INITIAL_CLUSTER="etcd-node01=https://192.168.58.162:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.58.162:2379"
CLIENT_CERT_AUTH="true"
ETCD_CA_FILE="/etc/kubernetes/ssl/ca.pem"
ETCD_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_CA_FILE="/etc/kubernetes/ssl/ca.pem"
ETCD_PEER_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
EOF
systemctl daemon-reload
systemctl enable etcd
systemctl start etcd

cat > /usr/lib/systemd/system/kube-apiserver.service << \EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \
--anonymous-auth=false \
--allow-privileged=true \
--authorization-mode=Node,RBAC \
--bind-address=0.0.0.0 \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--enable-admission-plugins=NodeRestriction \
--enable-bootstrap-token-auth=true \
--etcd-cafile=/etc/kubernetes/ssl/ca.pem   \
--etcd-certfile=/etc/kubernetes/ssl/etcd.pem   \
--etcd-keyfile=/etc/kubernetes/ssl/etcd-key.pem   \
--etcd-servers=https://127.0.0.1:2379 \
--proxy-client-cert-file=/etc/kubernetes/ssl/kube-proxy.pem \
--proxy-client-key-file=/etc/kubernetes/ssl/kube-proxy-key.pem \
--kubelet-certificate-authority=/etc/kubernetes/ssl/ca.pem \
--kubelet-client-certificate=/etc/kubernetes/ssl/kubernetes.pem \
--kubelet-client-key=/etc/kubernetes/ssl/kubernetes-key.pem \
--requestheader-allowed-names= \
--requestheader-client-ca-file=/etc/kubernetes/ssl/ca.pem \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-group-headers=X-Remote-Group \
--requestheader-username-headers=X-Remote-User \
--secure-port=6443 \
--service-account-issuer=https://kubernetes.default.svc.cluster.local \
--service-account-key-file=/etc/kubernetes/ssl/sa.pem \
--service-account-signing-key-file=/etc/kubernetes/ssl/sa-key.pem \
--service-cluster-ip-range=10.96.0.0/16 \
--tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem   \
--tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem   \
--v=2
Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable kube-apiserver
systemctl start kube-apiserver

cat > /usr/lib/systemd/system/kube-controller-manager.service << \EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-controller-manager \
--allocate-node-cidrs=true \
--authentication-kubeconfig=/etc/kubernetes/controller-manager.conf \
--authorization-kubeconfig=/etc/kubernetes/controller-manager.conf \
--bind-address=0.0.0.0 \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--cluster-cidr=10.244.0.0/16 \
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem   \
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem   \
--kubeconfig=/etc/kubernetes/controller-manager.conf \
--leader-elect=true \
--controllers=*,tokencleaner \
--node-cidr-mask-size=24 \
--requestheader-client-ca-file=/etc/kubernetes/ssl/ca.pem \
--root-ca-file=/etc/kubernetes/ssl/ca.pem \
--service-account-private-key-file=/etc/kubernetes/ssl/sa-key.pem  \
--service-cluster-ip-range=10.96.0.0/16 \
--use-service-account-credentials=true \
--v=2
Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable kube-controller-manager
systemctl start kube-controller-manager


cat > /usr/lib/systemd/system/kube-scheduler.service << \EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/GoogleCloudPlatform/kubernetes

[Service]
ExecStart=/usr/local/bin/kube-scheduler \
--authentication-kubeconfig=/etc/kubernetes/scheduler.conf \
--authorization-kubeconfig=/etc/kubernetes/scheduler.conf \
--bind-address=0.0.0.0 \
--feature-gates=RotateKubeletServerCertificate=true \
--kubeconfig=/etc/kubernetes/scheduler.conf \
--leader-elect=true \
--v=2

Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable kube-scheduler
systemctl start kube-scheduler

export KUBE_APISERVER="https://${master01ip}:6443"
# 设置集群参数
kubectl config set-cluster kubernetes \
  --certificate-authority=/etc/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER}
# 设置客户端认证参数
kubectl config set-credentials admin \
  --client-certificate=/etc/kubernetes/ssl/admin.pem \
  --embed-certs=true \
  --client-key=/etc/kubernetes/ssl/admin-key.pem
# 设置上下文参数
# 使用admin用户来管理集群。 配置项在默认的 ~/.kube/config
kubectl config set-context kubernetes \
  --cluster=kubernetes \
  --user=admin
# 设置默认上下文
kubectl config use-context kubernetes
kubectl create clusterrolebinding admin --clusterrole=system:kubelet-api-admin --user=admin



kubectl config set-cluster kubernetes     \
  --certificate-authority=/etc/kubernetes/ssl/ca.pem     \
  --embed-certs=true     \
  --server=https://192.168.58.162:6443     \
  --kubeconfig=/etc/kubernetes/admin.kubeconfig
kubectl config set-credentials kubernetes-admin  \
  --client-certificate=/etc/kubernetes/ssl/admin.pem     \
  --client-key=/etc/kubernetes/ssl/admin-key.pem     \
  --embed-certs=true     \
  --kubeconfig=/etc/kubernetes/admin.kubeconfig
kubectl config set-context kubernetes-admin@kubernetes    \
  --cluster=kubernetes     \
  --user=kubernetes-admin     \
  --kubeconfig=/etc/kubernetes/admin.kubeconfig
kubectl config use-context kubernetes-admin@kubernetes  --kubeconfig=/etc/kubernetes/admin.kubeconfig

cat > /usr/lib/systemd/system/containerd.service << \EOF
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target

[Service]
ExecStartPre=-/usr/sbin/modprobe overlay
ExecStartPre=-/usr/sbin/modprobe br_netfilter
ExecStartPre=-/bin/mkdir -p /var/lib/containerd
ExecStartPre=-/bin/mkdir -p /etc/containerd/

ExecStart=/usr/local/bin/containerd

Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=infinity
# Comment TasksMax if your systemd version does not supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable containerd
systemctl start containerd

mkdir -p /etc/containerd/ /var/lib/containerd
echo runtime-endpoint: unix:///run/containerd/containerd.sock > /etc/crictl.yaml
containerd config default | sudo tee /etc/containerd/config.toml
# sandbox_image = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.2"
sed -i 's#registry.k8s.io/pause:3.8#ccr.ccs.tencentyun.com/hank997/pause:3.8#g' /etc/containerd/config.toml
systemctl daemon-reload
systemctl restart containerd

# 创建bootstrap.kubernetes.io/token 就可以直接加入集群，https://kubernetes.io/zh-cn/docs/reference/access-authn-authz/bootstrap-tokens/
sort=$(head -c 6 /dev/urandom | md5sum | head -c 6)
long=$(head -c 16 /dev/urandom | md5sum | head -c 16)
expiration_24h=$(date -u -d "+24 hours" +"%Y-%m-%dT%H:%M:%SZ")
cat > tmp.yaml << EOF
apiVersion: v1
kind: Secret
metadata:
  name: bootstrap-token-${sort}
  namespace: kube-system
type: bootstrap.kubernetes.io/token
stringData:
  description: "The default bootstrap token generated by 'kubeadm init'."
  token-id: ${sort}
  token-secret: ${long}
  expiration: ${expiration_24h}
  usage-bootstrap-authentication: "true"
  usage-bootstrap-signing: "true"
  auth-extra-groups:  system:bootstrappers:default-node-token,system:bootstrappers:worker,system:bootstrappers:ingress
EOF
kubectl apply -f tmp.yaml

# 这里还需要创建三个yaml https://kubernetes.io/zh-cn/docs/reference/access-authn-authz/kubelet-tls-bootstrapping/

cat > 1.yaml << EOF
# 允许启动引导节点创建 CSR
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: create-csrs-for-bootstrapping
subjects:
- kind: Group
  name: system:bootstrappers
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: system:node-bootstrapper
  apiGroup: rbac.authorization.k8s.io
EOF
cat > 2.yaml << EOF
# 批复 "system:bootstrappers" 组的所有 CSR
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: auto-approve-csrs-for-group
subjects:
- kind: Group
  name: system:bootstrappers
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
  apiGroup: rbac.authorization.k8s.io
EOF
cat > 3.yaml << EOF
# 批复 "system:nodes" 组的 CSR 续约请求
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: auto-approve-renewals-for-nodes
subjects:
- kind: Group
  name: system:nodes
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
  apiGroup: rbac.authorization.k8s.io
EOF

kubectl apply -f 1.yaml 2.yaml 3.yaml

/usr/local/bin/kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

export K8S_API_URL=${master01ip}
/usr/local/bin/kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=https://${K8S_API_URL}:6443 \
--kubeconfig=bootstrap.kubeconfig


mkdir -p /etc/kubernetes/manifests

#export TOKEN=$(awk -F "," '{print $1}' /etc/kubernetes/ssl/bootstrap-token.csv)
TOKEN=$sort.$long
/usr/local/bin/kubectl config set-credentials kubelet-bootstrap \
--token=${TOKEN} \
--kubeconfig=bootstrap.kubeconfig

/usr/local/bin/kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=bootstrap.kubeconfig

/usr/local/bin/kubectl config use-context default --kubeconfig=bootstrap.kubeconfig

\mv bootstrap.kubeconfig /etc/kubernetes/

cat > /etc/kubernetes/kubelet-conf.yml << \EOF
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 2m0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/ssl/ca.pem
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 5m0s
    cacheUnauthorizedTTL: 30s
cgroupDriver: systemd
cgroupsPerQOS: true
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
containerLogMaxFiles: 5
containerLogMaxSize: 10Mi
contentType: application/vnd.kubernetes.protobuf
cpuCFSQuota: true
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebuggingHandlers: true
enforceNodeAllocatable:
- pods
eventBurst: 10
eventRecordQPS: 5
evictionHard:
  imagefs.available: 15%
  memory.available: 100Mi
  nodefs.available: 10%
  nodefs.inodesFree: 5%
evictionPressureTransitionPeriod: 5m0s
failSwapOn: true
fileCheckFrequency: 20s
hairpinMode: promiscuous-bridge
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 20s
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMinimumGCAge: 2m0s
iptablesDropBit: 15
iptablesMasqueradeBit: 14
kubeAPIBurst: 10
kubeAPIQPS: 5
makeIPTablesUtilChains: true
maxOpenFiles: 1000000
maxPods: 110
nodeStatusUpdateFrequency: 10s
oomScoreAdj: -999
podPidsLimit: -1
registryBurst: 10
registryPullQPS: 5
resolvConf: /etc/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 2m0s
serializeImagePulls: true
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 4h0m0s
syncFrequency: 1m0s
volumeStatsAggPeriod: 1m0s
EOF
mkdir -p /var/lib/kubelet
cat > /usr/lib/systemd/system/kubelet.service << \EOF
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/GoogleCloudPlatform/kubernetes

[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/local/bin/kubelet \
    --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig  \
    --client-ca-file=/etc/kubernetes/ssl/ca.pem \
    --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
    --config=/etc/kubernetes/kubelet-conf.yml \
    --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
    --node-labels=node.kubernetes.io/node= \
    --v=2
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF


systemctl daemon-reload
systemctl enable kubelet
systemctl start kubelet

# 自动批准已经被删除了，全部都需要手动同意添加
kubectl get csr
kubectl get csr|grep 'Pending' | awk 'NR>0{print $1}'| xargs kubectl certificate approve

export K8S_API_URL=${master01ip}
/usr/local/bin/kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=https://${K8S_API_URL}:6443 \
--kubeconfig=kube-proxy.kubeconfig

/usr/local/bin/kubectl  config set-credentials kube-proxy \
--client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \
--client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig

/usr/local/bin/kubectl  config set-context default \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig

# 设置默认上下文
/usr/local/bin/kubectl  config use-context default --kubeconfig=kube-proxy.kubeconfig

mv kube-proxy.kubeconfig /etc/kubernetes/
mkdir -p /var/lib/kube-proxy

cat > /etc/kubernetes/kube-proxy.yaml << \EOF
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
clientConnection:
  acceptContentTypes: ""
  burst: 10
  contentType: application/vnd.kubernetes.protobuf
  kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
  qps: 5
clusterCIDR: 10.244.0.0/16
configSyncPeriod: 15m0s
conntrack:
  max: null
  maxPerCore: 32768
  min: 131072
  tcpCloseWaitTimeout: 1h0m0s
  tcpEstablishedTimeout: 24h0m0s
enableProfiling: false
healthzBindAddress: 0.0.0.0:10256
hostnameOverride: ""
iptables:
  masqueradeAll: false
  masqueradeBit: 14
  minSyncPeriod: 0s
  syncPeriod: 30s
ipvs:
  masqueradeAll: true
  minSyncPeriod: 5s
  scheduler: "rr"
  syncPeriod: 30s
kind: KubeProxyConfiguration
metricsBindAddress: 127.0.0.1:10249
mode: "ipvs"
nodePortAddresses: null
oomScoreAdj: -999
portRange: ""
udpIdleTimeout: 250ms
EOF

cat >  /usr/lib/systemd/system/kube-proxy.service << \EOF
[Unit]
Description=Kubernetes Kube Proxy
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-proxy \
  --config=/etc/kubernetes/kube-proxy.yaml \
  --cluster-cidr=10.244.0.0/16 \
  --v=2
Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target

EOF
systemctl daemon-reload
systemctl enable kube-proxy
systemctl start kube-proxy

# 第一次部署 kubectl执行会有问题，不知道为什么会超时


systemctl restart etcd
systemctl restart kube-apiserver
systemctl restart kube-controller-manager
systemctl restart kube-scheduler
systemctl restart kubelet
systemctl restart kube-proxy


modprobe br_netfilter
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
kubectl apply -f kube-flannel.yml