k8s-集群部署
一、环境说明
k8s-node1 192.168.2.230
k8s-node2 192.168.2.231
二、初始化环境
2.1 配置IP地址
[root@localhost ~]# vim /etc/sysconfig/network-scripts/ifcfg-ens192 TYPE=Ethernet PROXY_METHOD=none BROWSER_ONLY=no BOOTPROTO=static DEFROUTE=yes IPV4_FAILURE_FATAL=no IPV6INIT=yes IPV6_AUTOCONF=yes IPV6_DEFROUTE=yes IPV6_FAILURE_FATAL=no IPV6_ADDR_GEN_MODE=stable-privacy NAME=ens192 UUID=ae2d396c-47c5-4009-af71-a4eef91de6fc HWADDR=00:50:56:b2:cf:b8 DEVICE=ens192 ONBOOT=yes IPADDR=192.168.2.230 NETMASK=255.255.255.0 GATEWAY=192.168.2.1 DNS1=202.96.134.133 [root@localhost ~]# systemctl restart network
2.2 配置主机名+ 配置hosts通信
[root@localhost ~]# hostnamectl set-hostname k8s-master-230 [root@k8s-master-230 ~]# vim /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.2.230 k8s-master-230 192.168.2.231 k8s-master-231
三、创建证书
[root@k8s-master-230 ~]# mkdir -pv /opt/local/cfssl mkdir: created directory ‘/opt/local’ mkdir: created directory ‘/opt/local/cfssl’ [root@k8s-master-230 ~]# cd /opt/local/cfssl/ [root@k8s-master-230cfssl]# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 [root@k8s-master-230 cfssl]# mv cfssl_linux-amd64 cfssl [root@k8s-master-230 cfssl]# wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 [root@k8s-master-230 cfssl]# mv cfssljson_linux-amd64 cfssljson [root@k8s-master-230 cfssl]# wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 [root@k8s-master-230 cfssl]# mv cfssl-certinfo_linux-amd64 cfssl-certinfo [root@k8s-master-230 cfssl]# chmod +x *
四、创建CA证书
[root@k8s-master-230 cfssl]# mkdir /opt/ssl [root@k8s-master-230 cfssl]# cd /opt/ssl/ [root@k8s-master-230 ssl]# vim config.json { "signing": { "default": { "expiry": "87600h" }, "profiles": { "kubernetes": { "usages": [ "signing", "key encipherment", "server auth", "client auth" ], "expiry": "87600h" } } } } [root@k8s-master-230 ssl]# vim csr.json { "CN": "kubernetes", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "ShenZhen", "L": "ShenZhen", "O": "k8s", "OU": "System" } ] }
4.1 生成CA证书和私钥
[root@k8s-master-230 ssl]# /opt/local/cfssl/cfssl gencert -initca csr.json | /opt/local/cfssl/cfssljson -bare ca 2018/09/28 18:14:13 [INFO] generating a new CA key and certificate from CSR 2018/09/28 18:14:13 [INFO] generate received request 2018/09/28 18:14:13 [INFO] received CSR 2018/09/28 18:14:13 [INFO] generating key: rsa-2048 2018/09/28 18:14:14 [INFO] encoded CSR 2018/09/28 18:14:14 [INFO] signed certificate with serial number 210512755176393999852585629849119329492232192621
4.2 分支证书
#创建证书目录 [root@k8s-master-230 ssl]# mkdir -pv /etc/kubernetes/ssl #复制所有pem文件到此目录下（MASTER所有节点都需要有） [root@k8s-master-230 ssl]# cp *.pem /etc/kubernetes/ssl/
五、安装docker
[root@k8s-master-230 ~]# yum -y install yum-utils [root@k8s-master-230 ~]# yum-config-manager \ > --add-repo \ > https://download.docker.com/linux/centos/docker-ce.repo [root@k8s-master-230 ~]# yum makecache [root@k8s-master-230 ~]# yum install docker-ce -y #修改docker服务配置 [root@k8s-master-230 ~]# vim /usr/lib/systemd/system/docker.service [Unit] Description=Docker Application Container Engine Documentation=https://docs.docker.com After=network-online.target firewalld.service Wants=network-online.target [Service] Type=notify ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS $DOCKER_OPTS $DOCKER_DNS_OPTIONS ExecReload=/bin/kill -s HUP $MAINPID LimitNOFILE=infinity LimitNPROC=infinity LimitCORE=infinity TimeoutStartSec=0 Delegate=yes KillMode=process Restart=on-failure StartLimitBurst=3 StartLimitInterval=60s [Install] WantedBy=multi-user.target #创建/usr/lib/systemd/system/docker.service.d/ [root@k8s-master-230 ~]# mkdir -pv /usr/lib/systemd/system/docker.service.d mkdir: created directory ‘/usr/lib/systemd/system/docker.service.d’ #registry.com是本地仓库地址，需要自行创建本地仓库 [root@k8s-master-230 ~]# vim /usr/lib/systemd/system/docker.service.d/docker-options.conf [Service] Environment="DOCKER_OPTS=--insecure-registry 192.168.2.252:5000 --graph=/data/docker --registry-mirror=https://registry.com" #重新读取配置 [root@k8s-master-230 ~]# systemctl daemon-reload #启动docker [root@k8s-master-230 ~]# systemctl start docker #开机启动docker [root@k8s-master-230 ~]# systemctl enable docker Created symlink from /etc/systemd/system/multi-user.target.wants/docker.service to /usr/lib/systemd/system/docker.service.
六、安装ETCD集群服务
[root@k8s-master-230 ~]# yum install etcd #创建etcd证书 [root@k8s-master-230 ~]# cd /opt/ssl/ [root@k8s-master-230 ~]# vim etcd-csr.json { "CN": "etcd", "hosts": [ "127.0.0.1", "192.168.2.230", "192.168.2.231" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "ShenZhen", "L": "ShenZhen", "O": "k8s", "OU": "System" } ] } [root@k8s-master-230 ssl]# /opt/local/cfssl/cfssl gencert -ca=/opt/ssl/ca.pem \ > -ca-key=/opt/ssl/ca-key.pem \ > -config=/opt/ssl/config.json \ > -profile=kubernetes etcd-csr.json | /opt/local/cfssl/cfssljson -bare etcd #复制到/etc/kubernetes/ssl/目录下（MASTER节点机器都需要） [root@k8s-master-230 ssl]# cp etcd*.pem /etc/kubernetes/ssl/ #修改etcd启动配置文件 #ETCD1配置 [root@k8s-master-230 ssl]# vim /usr/lib/systemd/system/etcd.service #etcd.service [Unit] Description=Etcd Server After=network.target After=network-online.target Wants=network-online.target [Service] Type=notify WorkingDirectory=/var/lib/etcd/ #EnvironmentFile=-/etc/etcd/etcd.conf User=etcd # set GOMAXPROCS to number of processors #ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd --name=\"${ETCD_NAME}\" --data-dir=\"${ETCD_DATA_DIR}\" --listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\"" ExecStart=/usr/bin/etcd \ --name=etcd1 \ --cert-file=/etc/kubernetes/ssl/etcd.pem \ --key-file=/etc/kubernetes/ssl/etcd-key.pem \ --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \ --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \ --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \ --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \ --initial-advertise-peer-urls=https://192.168.2.230:2380 \ --listen-peer-urls=https://192.168.2.230:2380 \ --listen-client-urls=https://192.168.2.230:2379,http://127.0.0.1:2379 \ --advertise-client-urls=https://192.168.2.230:2379,http://127.0.0.1:2379 \ --initial-cluster-token=k8s-etcd-cluster \ --initial-cluster=etcd1=https://192.168.2.230:2380,etcd2=https://192.168.2.231:2380 \ --initial-cluster-state new \ --data-dir=/var/lib/etcd Restart=on-failure RestartSec=5 LimitNOFILE=65536 [Install] WantedBy=multi-user.target #ETCD2配置文件 [root@k8s-master-231 ssl]# vim /usr/lib/systemd/system/etcd.service [Unit] Description=Etcd Server After=network.target After=network-online.target Wants=network-online.target [Service] Type=notify WorkingDirectory=/var/lib/etcd/ #EnvironmentFile=-/etc/etcd/etcd.conf User=etcd # set GOMAXPROCS to number of processors #ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd --name=\"${ETCD_NAME}\" --data-dir=\"${ETCD_DATA_DIR}\" --listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\"" ExecStart=/usr/bin/etcd \ --name=etcd2 \ --cert-file=/etc/kubernetes/ssl/etcd.pem \ --key-file=/etc/kubernetes/ssl/etcd-key.pem \ --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \ --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \ --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \ --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \ --initial-advertise-peer-urls=https://192.168.2.231:2380 \ --listen-peer-urls=https://192.168.2.231:2380 \ --listen-client-urls=https://192.168.2.231:2379,http://127.0.0.1:2379 \ --advertise-client-urls=https://192.168.2.231:2379,http://127.0.0.1:2379 \ --initial-cluster-token=k8s-etcd-cluster \ --initial-cluster=etcd1=https://192.168.2.230:2380,etcd2=https://192.168.2.231:2380 \ --initial-cluster-state new \ --data-dir=/var/lib/etcd Restart=on-failure RestartSec=5 LimitNOFILE=65536 [Install] WantedBy=multi-user.target #关闭SELINUX [root@k8s-master-230 system]# setenforce 0 #关闭iptables [root@k8s-master-230 system]# systemctl stop firewalld.service #给证书文件可读权限 [root@k8s-master-230 ssl]# chmod g+r,o+r * #开机启动etcd [root@k8s-master-231 system]# systemctl enable etcd Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service. #启动etcd [root@k8s-master-231 system]# systemctl start etcd #开机启动etcd [root@k8s-master-230 system]# systemctl enable etcd Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service. #启动etcd [root@k8s-master-230 system]# systemctl start etcd #验证集群状态 [root@k8s-master-230 ~]# etcdctl --endpoints=https://192.168.2.230:2379 \ > --cert-file=/etc/kubernetes/ssl/etcd.pem \ > --ca-file=/etc/kubernetes/ssl/ca.pem \ > --key-file=/etc/kubernetes/ssl/etcd-key.pem \ > cluster-health member 93a6fe8d5e0c09f8 is healthy: got healthy result from http://127.0.0.1:2379 member f89a9ab4224d15ce is healthy: got healthy result from http://127.0.0.1:2379 cluster is healthy [root@k8s-master-231 ~]# etcdctl --endpoints=https://192.168.2.230:2379 --cert-file=/etc/kubernetes/ssl/etcd.pem --ca-file=/etc/kubernetes/ssl/ca.pem --key-file=/etc/kubernetes/ssl/etcd-key.pem cluster-health member 93a6fe8d5e0c09f8 is healthy: got healthy result from http://127.0.0.1:2379 member f89a9ab4224d15ce is healthy: got healthy result from http://127.0.0.1:2379 cluster is healthy #查看etcd集群成员 [root@k8s-master-230 ~]# etcdctl --endpoints=https://192.168.2.230:2379 --cert-file=/etc/kubernetes/ssl/etcd.pem --ca-file=/etc/kubernetes/ssl/ca.pem --key-file=/etc/kubernetes/ssl/etcd-key.pem member list 93a6fe8d5e0c09f8: name=etcd2 peerURLs=https://192.168.2.231:2380 clientURLs=http://127.0.0.1:2379,https://192.168.2.231:2379 isLeader=false f89a9ab4224d15ce: name=etcd1 peerURLs=https://192.168.2.230:2380 clientURLs=http://127.0.0.1:2379,https://192.168.2.230:2379 isLeader=true [root@k8s-master-231 ~]# etcdctl --endpoints=https://192.168.2.230:2379 --cert-file=/etc/kubernetes/ssl/etcd.pem --ca-file=/etc/kubernetes/ssl/ca.pem --key-file=/etc/kubernetes/ssl/etcd-key.pem member list 93a6fe8d5e0c09f8: name=etcd2 peerURLs=https://192.168.2.231:2380 clientURLs=http://127.0.0.1:2379,https://192.168.2.231:2379 isLeader=false f89a9ab4224d15ce: name=etcd1 peerURLs=https://192.168.2.230:2380 clientURLs=http://127.0.0.1:2379,https://192.168.2.230:2379 isLeader=true
七、安装kubectl工具
[root@k8s-master-230 ~]# tar xvf kubernetes-server-linux-amd64.tar.tar [root@k8s-master-230 bin]# cd /root/kubernetes/server/bin [root@k8s-master-230 bin]# cp kubectl /usr/local/bin/ [root@k8s-master-230 bin]# kubectl version Client Version: version.Info{Major:"1", Minor:"8", GitVersion:"v1.8.1", GitCommit:"f38e43b221d08850172a9a4ea785a86a3ffa3b3a", GitTreeState:"clean", BuildDate:"2017-10-11T23:27:35Z", GoVersion:"go1.8.3", Compiler:"gc", Platform:"linux/amd64"} The connection to the server localhost:8080 was refused - did you specify the right host or port? #创建admin证书 [root@k8s-master-230 ssl]# vim admin-csr.json { "CN": "admin", "hosts": [], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "ShenZhen", "L": "ShenZhen", "O": "system:masters", "OU": "System" } ] } [root@k8s-master-230 ssl]# /opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem -ca-key=/etc/kubernetes/ssl/ca-key.pem -config=/opt/ssl/config.json -profile=kubernetes admin-csr.json | /opt/local/cfssl/cfssljson -bare admin 2018/09/28 20:30:05 [INFO] generate received request 2018/09/28 20:30:05 [INFO] received CSR 2018/09/28 20:30:05 [INFO] generating key: rsa-2048 2018/09/28 20:30:06 [INFO] encoded CSR 2018/09/28 20:30:06 [INFO] signed certificate with serial number 83426200686073795576519750120190884752065888474 2018/09/28 20:30:06 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for websites. For more information see the Baseline Requirements for the Issuance and Management of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org); specifically, section 10.2.3 ("Information Requirements"). [root@k8s-master-230 ssl]# cp admin*.pem /etc/kubernetes/ssl/ [root@k8s-master-230 ssl]# scp admin*.pem k8s-master-231:/etc/kubernetes/ssl/ #配置kubectl kubeconfig文件 #配置kubernetes集群 [root@k8s-master-230 ssl]# kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/ssl/ca.pem --embed-certs=true --server=https://192.168.2.230:6443 Cluster "kubernetes" set. #配置客户端认证 [root@k8s-master-230 ssl]# kubectl config set-credentials admin --client-certificate=/etc/kubernetes/ssl/admin.pem --embed-certs=true --client-key=/etc/kubernetes/ssl/admin-key.pem User "admin" set. [root@k8s-master-230 ssl]# kubectl config set-context kubernetes --cluster=kubernetes --user=admin Context "kubernetes" created. [root@k8s-master-230 ssl]# kubectl config use-context kubernetes Switched to context "kubernetes". #kubeconfig文件在以下目录 [root@k8s-master-230 .kube]# pwd /root/.kube [root@k8s-master-230 .kube]# ls config
八、部署kubernetes master节点
Master 需要部署kube-apiservice, kube-scheduler.kube-controller-manager 这三个组件，
kube-scheduler作用是调度pods分配到那个node里面，简单来说就是资源调度，
kube-controller-manager作用是对deployment controller ， replication controller ， endpoints controller ， and service accounts controller 等等的循环控制，与kube-apiservice 交互
#安装组件 https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.8.md https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.8.md#downloads-for-v181 [root@k8s-master-230 kubernetes]# cd /root/kubernetes/server/bin/ [root@k8s-master-230 bin]# cp kube-apiserver kube-controller-manager kube-scheduler kubectl kube-proxy kubelet /usr/local/bin/ cp: overwrite ‘/usr/local/bin/kubectl’? y #创建kubernetes证书 [root@k8s-master-230 ssl]# vim kubernetes-csr.json { "CN": "kubernetes", "hosts": [ "127.0.0.1", "192.168.2.230", "192.168.2.231", "10.254.0.1", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "ShenZhen", "L": "ShenZhen", "O": "k8s", "OU": "System" } ] } ## 这里 hosts 字段中 三个 IP 分别为 127.0.0.1 本机， 192.168.2.230 和 192.168.2.231 为 Master 的IP，多个Master需要写多个 10.254.0.1 为 kubernetes SVC 的 IP， 一般是 部署网络的第一个IP , 如: 10.254.0.1 ， 在启动完成后，我们使用 kubectl get svc ， 就可以查看到 #生成kubernetes证书和私钥 [root@k8s-master-230 ssl]# /opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem -ca-key=/etc/kubernetes/ssl/ca-key.pem -config=/opt/ssl/config.json -profile=kubernetes kubernetes-csr.json | /opt/local/cfssl/cfssljson -bare kubernetes 2018/09/29 10:06:40 [INFO] generate received request 2018/09/29 10:06:40 [INFO] received CSR 2018/09/29 10:06:40 [INFO] generating key: rsa-2048 2018/09/29 10:06:40 [INFO] encoded CSR 2018/09/29 10:06:40 [INFO] signed certificate with serial number 15070220706619436953102917755226550123483926972 2018/09/29 10:06:40 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for websites. For more information see the Baseline Requirements for the Issuance and Management of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org); specifically, section 10.2.3 ("Information Requirements"). [root@k8s-master-230 ssl]# cp -r kubernetes*.pem /etc/kubernetes/ssl/ [root@k8s-master-230 ssl]# scp -r kubernetes*.pem k8s-master-231:/etc/kubernetes/ssl/ #配置kube-apiserver #生成token [root@k8s-master-230 ssl]# head -c 16 /dev/urandom | od -An -t x | tr -d ' ' ed156e2287cb1b1d2095a39a8c57c1ed #创建token.csv文件 [root@k8s-master-230 ssl]# vim token.csv ed156e2287cb1b1d2095a39a8c57c1ed,kubelet-bootstrap,10001,"system:kubelet-bootstrap" [root@k8s-master-230 ssl]# cp token.csv /etc/kubernetes/ [root@k8s-master-230 ssl]# scp token.csv k8s-master-231:/etc/kubernetes/ #创建kube-apiserver.service文件 # 1.8 新增 (Node) --authorization-mode=Node,RBAC # 自定义 系统 service 文件一般存于 /etc/systemd/system/ 下 # 配置为 各自的本地 IP #K8S-MASTER-230(配置参数) [root@k8s-master-230 ssl]# vim /etc/systemd/system/kube-apiserver.service [Unit] Description=Kubernetes API Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=network.target [Service] User=root ExecStart=/usr/local/bin/kube-apiserver \ --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \ --advertise-address=192.168.2.230 \ --allow-privileged=true \ --apiserver-count=3 \ --audit-log-maxage=30 \ --audit-log-maxbackup=3 \ --audit-log-maxsize=100 \ --audit-log-path=/var/lib/audit.log \ --authorization-mode=Node,RBAC \ --bind-address=192.168.2.230 \ --client-ca-file=/etc/kubernetes/ssl/ca.pem \ --enable-swagger-ui=true \ --etcd-cafile=/etc/kubernetes/ssl/ca.pem \ --etcd-certfile=/etc/kubernetes/ssl/etcd.pem \ --etcd-keyfile=/etc/kubernetes/ssl/etcd-key.pem \ --etcd-servers=https://192.168.2.230:2379,https://192.168.2.231:2379 \ --event-ttl=1h \ --kubelet-https=true \ --insecure-bind-address=192.168.2.230 \ --runtime-config=rbac.authorization.k8s.io/v1alpha1 \ --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \ --service-cluster-ip-range=10.254.0.0/16 \ --service-node-port-range=30000-32000 \ --tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \ --tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \ --enable-bootstrap-token-auth \ --token-auth-file=/etc/kubernetes/token.csv \ --v=2 Restart=on-failure RestartSec=5 Type=notify LimitNOFILE=65536 [Install] WantedBy=multi-user.target #K8S-MASTER-231(配置参数) [Unit] Description=Kubernetes API Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=network.target [Service] User=root ExecStart=/usr/local/bin/kube-apiserver \ --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \ --advertise-address=192.168.2.231 \ --allow-privileged=true \ --apiserver-count=3 \ --audit-log-maxage=30 \ --audit-log-maxbackup=3 \ --audit-log-maxsize=100 \ --audit-log-path=/var/lib/audit.log \ --authorization-mode=Node,RBAC \ --bind-address=192.168.2.231 \ --client-ca-file=/etc/kubernetes/ssl/ca.pem \ --enable-swagger-ui=true \ --etcd-cafile=/etc/kubernetes/ssl/ca.pem \ --etcd-certfile=/etc/kubernetes/ssl/etcd.pem \ --etcd-keyfile=/etc/kubernetes/ssl/etcd-key.pem \ --etcd-servers=https://192.168.2.230:2379,https://192.168.2.231:2379 \ --event-ttl=1h \ --kubelet-https=true \ --insecure-bind-address=192.168.2.231 \ --runtime-config=rbac.authorization.k8s.io/v1alpha1 \ --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \ --service-cluster-ip-range=10.254.0.0/16 \ --service-node-port-range=30000-32000 \ --tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \ --tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \ --enable-bootstrap-token-auth \ --token-auth-file=/etc/kubernetes/token.csv \ --v=2 Restart=on-failure RestartSec=5 Type=notify LimitNOFILE=65536 [Install] WantedBy=multi-user.target # 这里面要注意的是 --service-node-port-range=30000-32000 # 这个地方是 映射外部端口时 的端口范围，随机映射也在这个范围内映射，指定映射端口必须也在这个范围内。 #启动kube-apiserver [root@k8s-master-230 system]# systemctl start kube-apiserver #服务自启动 [root@k8s-master-230 system]# systemctl enable kube-apiserver Created symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /etc/systemd/system/kube-apiserver.service. [root@k8s-master-230 system]# systemctl daemon-reload #配置kube-controller-manager #master 配置为 各自 本地 IP [root@k8s-master-230 system]# vim kube-controller-manager.service [Unit] Description=Kubernetes Controller Manager Documentation=https://github.com/GoogleCloudPlatform/kubernetes [Service] ExecStart=/usr/local/bin/kube-controller-manager \ --address=127.0.0.1 \ --master=http://192.168.2.230:8080 \ --allocate-node-cidrs=true \ --service-cluster-ip-range=10.254.0.0/16 \ --cluster-cidr=10.233.0.0/16 \ --cluster-name=kubernetes \ --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \ --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \ --service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \ --root-ca-file=/etc/kubernetes/ssl/ca.pem \ --leader-elect=true \ --v=2 Restart=on-failure RestartSec=5 [Install] WantedBy=multi-user.target #启动kube-controller-manager [root@k8s-master-230 system]# systemctl daemon-reload [root@k8s-master-230 system]# systemctl start kube-controller-manager [root@k8s-master-230 system]# systemctl enable kube-controller-manager Created symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /etc/systemd/system/kube-controller-manager.service. #配置kube-scheduler #MASTER配置为各自本地IP #创建kube-cheduler.service文件 [root@k8s-master-230 system]# vim kube-scheduler.service [Unit] Description=Kubernetes Scheduler Documentation=https://github.com/GoogleCloudPlatform/kubernetes [Service] ExecStart=/usr/local/bin/kube-scheduler \ --address=127.0.0.1 \ --master=http://192.168.2.230:8080 \ --leader-elect=true \ --v=2 Restart=on-failure RestartSec=5 [Install] WantedBy=multi-user.target #启动kube-scheduler [root@k8s-master-230 system]# systemctl daemon-reload [root@k8s-master-230 system]# systemctl start kube-scheduler [root@k8s-master-230 system]# systemctl enable kube-scheduler Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /etc/systemd/system/kube-scheduler.service #验证MASTER节点 [root@k8s-master-230 system]# kubectl get componentstatuses NAME STATUS MESSAGE ERROR scheduler Healthy ok controller-manager Healthy ok etcd-0 Healthy {"health": "true"} etcd-1 Healthy {"health": "true"} [root@k8s-master-231 .kube]# kubectl get componentstatuses NAME STATUS MESSAGE ERROR controller-manager Healthy ok scheduler Healthy ok etcd-0 Healthy {"health": "true"} etcd-1 Healthy {"health": "true"}
部署MASTER NODE 部分
这个master主机即是k8s-master 也是client node 部分
node 部分需要部署的组件有 docker calico kubectl bubelet kube-proxy 这几个组件
配置kubelet
kubelet 启动时向kube-apiservice 发送TLS bootstrapping 请求，需要先将bootstrap token 文件中的kubelet-bootsrap 用户赋予，system：node-bootstrapper 角色，然后kubelet才有权限创建认证请求（certificatesigningrequests）
# 先创建认证请求 # user 为 master 中 token.csv 文件里配置的用户 # 只需在MASTER创建一次就可以 [root@k8s-master-230 .kube]# kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap clusterrolebinding "kubelet-bootstrap" created #创建 kubelet kubeconfig 文件 #server 配置为 master 本机 IP #配置集群 [root@k8s-master-230 ~]# kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/ssl/ca.pem --embed-certs=true --server=https://192.168.2.230:6443 --kubeconfig=bootstrap.kubeconfig Cluster "kubernetes" set. [root@k8s-master-231 .kube]# kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/ssl/ca.pem --embed-certs=true --server=https://192.168.2.231:6443 --kubeconfig=bootstrap.kubeconfig Cluster "kubernetes" set. #配置客户端认证 [root@k8s-master-230 ~]# kubectl config set-credentials kubelet-bootstrap --token=ed156e2287cb1b1d2095a39a8c57c1ed --kubeconfig=bootstrap.kubeconfig User "kubelet-bootstrap" set. [root@k8s-master-231 .kube]# kubectl config set-credentials kubelet-bootstrap --token=ed156e2287cb1b1d2095a39a8c57c1ed --kubeconfig=bootstrap.kubeconfig User "kubelet-bootstrap" set. # 配置关联 [root@k8s-master-230 ~]# kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap --kubeconfig=bootstrap.kubeconfig Context "default" created. [root@k8s-master-231 .kube]# kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap --kubeconfig=bootstrap.kubeconfig Context "default" created. # 配置默认关联 [root@k8s-master-230 ~]# kubectl config use-context default --kubeconfig=bootstrap.kubeconfig Switched to context "default". # 拷贝生成的 bootstrap.kubeconfig 文件 [root@k8s-master-230 ~]# mv bootstrap.kubeconfig /etc/kubernetes/ #创建 kubelet.service 文件 [root@k8s-master-230 ~]# mkdir /var/lib/kubelet #注意--hostname-override为各自节点IP地址 [root@k8s-master-230 ~]# vim /etc/systemd/system/kubelet.service [Unit] Description=Kubernetes Kubelet Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=docker.service Requires=docker.service [Service] WorkingDirectory=/var/lib/kubelet ExecStart=/usr/local/bin/kubelet \ --cgroup-driver=cgroupfs \ --hostname-override=192.168.2.230 \ --pod-infra-container-image=registry.com/pause-amd64:3.0 \ --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \ --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \ --cert-dir=/etc/kubernetes/ssl \ --cluster_dns=10.254.0.2 \ --cluster_domain=cluster.local. \ --hairpin-mode promiscuous-bridge \ --allow-privileged=true \ --fail-swap-on=false \ --serialize-image-pulls=false \ --logtostderr=true \ --max-pods=512 \ --network-plugin=cni \ --v=2 [Install] WantedBy=multi-user.target # 如上配置: #192.168.2.230 为本机的IP #10.254.0.2 预分配的 dns 地址 #cluster.local. 为 kubernetes 集群的 domain #registry.com/pause-amd64:3.0 这个是 pod 的基础镜像，既 gcr 的 gcr.io/google_containers/pause-amd64:3.0 镜像， 下载下来修改为自己的仓库中的比较快。 [root@k8s-master-230 system]# systemctl daemon-reload [root@k8s-master-230 system]# systemctl start kubelet [root@k8s-master-230 system]# systemctl enable kubelet [root@k8s-master-231 system]# systemctl daemon-reload [root@k8s-master-231 system]# systemctl start kubelet [root@k8s-master-231 system]# systemctl enable kubelet
配置TLS认证
#未认证状态下CONDITION为Pending [root@k8s-master-230 system]# kubectl get csr NAME AGE REQUESTOR CONDITION node-csr-Bt5XvBJdBptZd1Vpyjbl_VFAqtgbjlh_KBbdMNyH1I8 55s kubelet-bootstrap Pending node-csr-GXJJtWYX60uVY1KCh9pDOmc-HHePsgs2pY2SarbRAfE 4m kubelet-bootstrap Pending [root@k8s-master-230 system]# kubectl certificate approve node-csr-Bt5XvBJdBptZd1Vpyjbl_VFAqtgbjlh_KBbdMNyH1I8 certificatesigningrequest "node-csr-Bt5XvBJdBptZd1Vpyjbl_VFAqtgbjlh_KBbdMNyH1I8" approved #认证后CONDITION为Approved,Issued [root@k8s-master-230 system]# kubectl get csr NAME AGE REQUESTOR CONDITION node-csr-Bt5XvBJdBptZd1Vpyjbl_VFAqtgbjlh_KBbdMNyH1I8 2m kubelet-bootstrap Approved,Issued node-csr-GXJJtWYX60uVY1KCh9pDOmc-HHePsgs2pY2SarbRAfE 6m kubelet-bootstrap Pending
配置 kube-proxy
#创建 kube-proxy 证书 [root@k8s-master-230 ssl]# vim kube-proxy-csr.json { "CN": "system:kube-proxy", "hosts": [], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "ShenZhen", "L": "ShenZhen", "O": "k8s", "OU": "System" } ] } #生成 kube-proxy 证书和私钥 [root@k8s-master-230 ssl]# /opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem -ca-key=/etc/kubernetes/ssl/ca-key.pem -config=/opt/ssl/config.json -profile=kubernetes kube-proxy-csr.json | /opt/local/cfssl/cfssljson -bare kube-proxy [root@k8s-master-230 ssl]# cp kube-proxy*.pem /etc/kubernetes/ssl/ #创建 kube-proxy kubeconfig 文件 #server 配置为各自 本机IP [root@k8s-master-230 ssl]# kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/ssl/ca.pem --embed-certs=true --server=https://192.168.2.230:6443 --kubeconfig=kube-proxy.kubeconfig # 配置客户端认证 [root@k8s-master-230 ssl]# kubectl config set-credentials kube-proxy --client-certificate=/etc/kubernetes/ssl/kube-proxy.pem --client-key=/etc/kubernetes/ssl/kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig #配置关联 [root@k8s-master-230 ssl]# kubectl config set-context default --cluster=kubernetes --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig # 配置默认关联 [root@k8s-master-230 ssl]# kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig [root@k8s-master-230 ssl]# mv kube-proxy.kubeconfig /etc/kubernetes/ #创建 kube-proxy.service 文件 #配置为 各自的 IP [root@k8s-master-230 ssl]# mkdir -p /var/lib/kube-proxy [root@k8s-master-230 ssl]# vim /etc/systemd/system/kube-proxy.service [Unit] Description=Kubernetes Kube-Proxy Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=network.target [Service] WorkingDirectory=/var/lib/kube-proxy ExecStart=/usr/local/bin/kube-proxy \ --bind-address=192.168.2.230 \ --hostname-override=192.168.2.230 \ --cluster-cidr=10.254.0.0/16 \ --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \ --logtostderr=true \ --v=2 Restart=on-failure RestartSec=5 LimitNOFILE=65536 [Install] WantedBy=multi-user.target #启动 kube-proxy [root@k8s-master-230 ssl]# systemctl daemon-reload [root@k8s-master-230 ssl]# systemctl start kube-proxy [root@k8s-master-230 ssl]# systemctl enable kube-proxy
安装calico
# 下载 yaml 文件 wget http://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/calico.yaml wget http://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/rbac.yaml #配置calico [root@k8s-master-230 calico]# vim calico.yaml #注意修改如下选项 #第16行 #原参数 etcd_endpoints: "http://127.0.0.1:2379" #修改为 etcd_endpoints: "https://192.168.2.230:2379,https://192.168.2.231:2379" #第48，49，50行 #原参数 48 etcd_ca: "" # "/calico-secrets/etcd-ca" 49 etcd_cert: "" # "/calico-secrets/etcd-cert" 50 etcd_key: "" # "/calico-secrets/etcd-key" #修改为 48 etcd_ca: "/calico-secrets/etcd-ca" 49 etcd_cert: "/calico-secrets/etcd-cert" 50 etcd_key: "/calico-secrets/etcd-key" #第67，68，69行 #这里面要写入base64的信息 #获取etcd-key的base64的信息（将得出来的信息填入etcd-key信息内） cat /etc/kubernetes/ssl/etcd-key.pem | base64 | tr -d '\n' #获取etcd-cert的base64的信息（将得出来的信息填入etcd-cert信息内） cat /etc/kubernetes/ssl/etcd.pem | base64 | tr -d '\n' #获取etcd-ca的base64的信息（将得出来的信息填入etcd-ca信息内 cat /etc/kubernetes/ssl/ca.pem | base64 | tr -d '\n' #原参数 etcd-key: null etcd-cert: null etcd-ca: null #修改为 etcd-key: (填入上面获取的base64信息的值) etcd-cert: (填入上面获取的base64信息的值) etcd-ca: (填入上面获取的base64信息的值) #第131行 #原参数 - name: CALICO_IPV4POOL_CIDR value: "192.168.0.0/16" #修改为(此IP为pod容器IP段) - name: CALICO_IPV4POOL_CIDR value: "10.233.0.0/16" #第167，168，169 #原参数 # Auto-detect the BGP IP address. - name: IP value: "" - name: FELIX_HEALTHENABLED value: "true" #修改为 - name: IP value: "autodetect" - name: IP_AUTODETECTION_METHOD #（新增参数） value: "interface=ens192" #（新增物理网卡绑定，如果不加，POD将无法获取到IP地址） - name: FELIX_HEALTHENABLED value: "true" #第203行 #原参数 image: quay.io/calico/cni:v1.11.7 #修改为 image: registry.com/calico/cni:v1.11.7 #第279行 #原参数 image: quay.io/calico/kube-controllers:v1.0.4 #修改为 image: registry.com/calico/kube-controllers:v1.0.4 #第345行 #原参数 image: quay.io/calico/kube-controllers:v1.0.4 #修改为 image: registry.com/calico/kube-controllers:v1.0.4 #第107行 #原参数 image: quay.io/calico/node:v2.6.11 #修改为 image: registry.com/calico/node:v2.6.11 #导入yaml文件 [root@k8s-master-230 calico]# kubectl apply -f calico.yaml configmap "calico-config" created secret "calico-etcd-secrets" created daemonset "calico-node" created deployment "calico-kube-controllers" created deployment "calico-policy-controller" created serviceaccount "calico-kube-controllers" created serviceaccount "calico-node" created [root@k8s-master-230 calico]# kubectl apply -f rbac.yaml clusterrole "calico-kube-controllers" created clusterrolebinding "calico-kube-controllers" created clusterrole "calico-node" created clusterrolebinding "calico-node" created #安装 Calicoctl wget -c https://github.com/projectcalico/calicoctl/releases/download/v1.3.0/calicoctl chmod +x calicoctl # 创建 calicoctl 配置文件 # 配置文件， 在 安装了 calico 网络的 机器下 [root@k8s-master-230 calico]# mkdir /etc/calico [root@k8s-master-230 calico]# vim /etc/calico/calicoctl.cfg apiVersion: v1 kind: calicoApiConfig metadata: spec: datastoreType: "etcdv2" etcdEndpoints: "https://192.168.2.230:2379,https://192.168.2.231:2379" etcdKeyFile: "/etc/kubernetes/ssl/etcd-key.pem" etcdCertFile: "/etc/kubernetes/ssl/etcd.pem" etcdCACertFile: "/etc/kubernetes/ssl/ca.pem" [root@k8s-master-230 calico]# scp /etc/calico/calicoctl.cfg 192.168.2.231:/etc/calico/
测试集群
[root@k8s-master-230 pod]# vim nginx.yaml apiVersion: extensions/v1beta1 kind: Deployment metadata: name: nginx-dm spec: replicas: 2 template: metadata: labels: name: nginx spec: containers: - name: nginx image: nginx:alpine imagePullPolicy: IfNotPresent ports: - containerPort: 80 --- apiVersion: v1 kind: Service metadata: name: nginx-svc spec: ports: - port: 80 targetPort: 80 protocol: TCP selector: name: nginx [root@k8s-master-230 pod]# kubectl apply -f nginx.yaml [root@k8s-master-230 pod]# kubectl get pods -o wide NAME READY STATUS RESTARTS AGE IP NODE nginx-dm-55b58f68b6-p9zp6 1/1 Running 0 12s 10.233.187.65 192.168.2.230 nginx-dm-55b58f68b6-wg6d7 1/1 Running 0 12s 10.233.10.1 192.168.2.231
配置kubeDNS
#下载 yaml 文件 curl -O https://raw.githubusercontent.com/kubernetes/kubernetes/master/cluster/addons/dns/kube-dns.yaml.base（地址已失效） # 修改后缀 mv kube-dns.yaml.base kube-dns.yaml #系统预定义的 RoleBinding #预定义的 RoleBinding system:kube-dns 将 kube-system 命名空间的 kube-dns ServiceAccount 与 system:kube-dns Role 绑定， 该 Role 具有访问 kube-apiserver DNS 相关 API 的权限； [root@k8s-master-230 kube-dns]# kubectl get clusterrolebindings system:kube-dns -o yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: rbac.authorization.kubernetes.io/autoupdate: "true" creationTimestamp: 2018-09-29T02:19:50Z labels: kubernetes.io/bootstrapping: rbac-defaults name: system:kube-dns resourceVersion: "78" selfLink: /apis/rbac.authorization.k8s.io/v1/clusterrolebindings/system%3Akube-dns uid: 24f1f600-c38e-11e8-93ce-005056b2cfb8 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:kube-dns subjects: - kind: ServiceAccount name: kube-dns namespace: kube-system #修改 kube-dns.yaml 1. # clusterIP: __PILLAR__DNS__SERVER__ 修改为我们之前定义的 dns IP 10.254.0.2 2. # 修改 --domain=__PILLAR__DNS__DOMAIN__. 为 我们之前 预定的 domain 名称 --domain=cluster.local. 3. # 修改 --server=/__PILLAR__DNS__DOMAIN__/127.0.0.1#10053 中 domain 为我们之前预定的 --server=/cluster.local./127.0.0.1#10053 4. # 修改 --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.__PILLAR__DNS__DOMAIN__, 中的 domain 为我们之前预定的 --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local., 5. # 修改 --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.__PILLAR__DNS__DOMAIN__, 中的 domain 为我们之前预定的 --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local., #导入 yaml 文件 #替换所有的image地址，全部改为本地仓库地址（registry.com） [root@k8s-master-230 kube-dns]# kubectl create -f k8s-dns.yaml service "kube-dns" created serviceaccount "kube-dns" created configmap "kube-dns" created deployment "kube-dns" created
查看kubedns服务
[root@k8s-master-230 kube-dns]# kubectl get all --namespace=kube-system NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE ds/calico-node 2 2 2 2 2 <none> 57m NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE deploy/calico-kube-controllers 1 1 1 1 57m deploy/calico-policy-controller 0 0 0 0 57m deploy/kube-dns 1 1 1 1 14s NAME DESIRED CURRENT READY AGE rs/calico-kube-controllers-755b75ffc9 1 1 1 57m rs/calico-policy-controller-585cc4d447 0 0 0 57m rs/kube-dns-7d765df5cb 1 1 1 14s NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE deploy/calico-kube-controllers 1 1 1 1 57m deploy/calico-policy-controller 0 0 0 0 57m deploy/kube-dns 1 1 1 1 14s NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE ds/calico-node 2 2 2 2 2 <none> 57m NAME DESIRED CURRENT READY AGE rs/calico-kube-controllers-755b75ffc9 1 1 1 57m rs/calico-policy-controller-585cc4d447 0 0 0 57m rs/kube-dns-7d765df5cb 1 1 1 15s NAME READY STATUS RESTARTS AGE po/calico-kube-controllers-755b75ffc9-9qnvk 1/1 Running 0 57m po/calico-node-djkjq 2/2 Running 0 57m po/calico-node-kddlc 2/2 Running 0 57m po/kube-dns-7d765df5cb-5btqm 3/3 Running 0 15s NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE svc/kube-dns ClusterIP 10.254.0.2 <none> 53/UDP,53/TCP 15s #验证 dns 服务 #在验证 dns 之前，在 dns 未部署之前创建的 pod 与 deployment 等，都必须删除，重新部署，否则无法解析 # 创建一个 pods 来测试一下 nameserver [root@k8s-master-230 pod]# vim alpine.yaml apiVersion: v1 kind: Pod metadata: name: alpine spec: containers: - name: alpine image: alpine command: - sh - -c - while true; do sleep 1; done [root@k8s-master-230 pod]# kubectl create -f alpine.yaml [root@k8s-master-230 pod]# kubectl exec -it alpine nslookup nginx-svc Name: nginx-svc Address 1: 10.254.237.47 nginx-svc.default.svc.cluster.local #部署 Ingress 与 Dashboard #官方 dashboard 的github https://github.com/kubernetes/dashboard # 替换所有的 images(改为本地仓库registry.com) # dashboard-controller.yaml增加 rbac 授权 # 在第二个 spec 最下面 增加 spec: serviceAccountName: dashboar # 因为开启了 RBAC 所以这里需要导入一个 RBAC 认证 [root@k8s-master-230 dashboard]# kubectl create -f dashboard-rbac.yaml serviceaccount "dashboard" created clusterrolebinding "dashboard" created #改入dashboard-controller.yaml 文件 [root@k8s-master-230 dashboard]# kubectl create -f dashboard-controller.yaml secret "kubernetes-dashboard-certs" created serviceaccount "kubernetes-dashboard" created role "kubernetes-dashboard-minimal" created rolebinding "kubernetes-dashboard-minimal" created deployment "kubernetes-dashboard" created service "kubernetes-dashboard" created #部署 Nginx Ingress Kubernetes 暴露服务的方式目前只有三种：LoadBlancer Service、NodePort Service、Ingress； 什么是 Ingress ? Ingress 就是利用 Nginx Haproxy 等负载均衡工具来暴露 Kubernetes 服务。 官方 Nginx Ingress github https://github.com/kubernetes/ingress-nginx #配置 调度 node # ingress 有多种方式 1. deployment 自由调度 replicas 2. daemonset 全局调度 分配到所有node里 # deployment 自由调度过程中，由于我们需要 约束 controller 调度到指定的 node 中，所以需要对 node 进行 label 标签 [root@k8s-master-230 ingress]# kubectl label nodes 192.168.2.230 ingress=proxy [root@k8s-master-230 ingress]# kubectl label nodes 192.168.2.231 ingress=proxy [root@k8s-master-230 ingress]# vim with-rbac.yam #第二个spec下添加 hostNetwork: true #第二个spec最后一行添加 ingress: proxy #导入namespace的yaml文件（加入ingress-nginx） [root@k8s-master-230 ingress]# kubectl create -f namespace.yaml #导入ingress的yaml文件（用于前端暴露代理） [root@k8s-master-230 ingress]# kubectl create -f with-rbac.yaml #导入rbac的yaml文件（用于rbac验证） [root@k8s-master-230 ingress]# kubectl create -f rbac.yaml #导入http-backend的yaml文件（当后端POD故障无法代理后自动代理回http-backend跳转404页面） [root@k8s-master-230 ingress]# kubectl create -f http-backend.yaml #导入dashboard-ingress的yaml文件（dashboardr系统的暴露服务） [root@k8s-master-230 ingress]# kubectl create -f dashboard-ingress.yaml
浙公网安备 33010602011771号