K8S学习笔记之kubectl get cs显示unhealthy的解决办法
$ kubectl get cs
NAME STATUS MESSAGE ERROR
controller-manager Unhealthy Get http://127.0.0.1:10252/healthz: dial tcp 127.0.0.1:10252: connect: connection refused
scheduler Unhealthy Get http://127.0.0.1:10251/healthz: dial tcp 127.0.0.1:10251: connect: connection refused
etcd-0 Healthy {"health":"true"}
原因1:
这两个pod的非安全端口没有开启，健康检查时报错，但是由于本身服务是正常的，只是健康检查的端口没启，所以不影响正常使用。
解决办法：
开启scheduler， control-manager的10251，10252端口
修改以下配置文件：
静态pod的路径：/etc/kubernetes/manifests
- vi kube-scheduler.yaml，把port=0那行注释
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
labels:
component: kube-scheduler
tier: control-plane
name: kube-scheduler
namespace: kube-system
spec:
containers:
- command:
- kube-scheduler
- --authentication-kubeconfig=/etc/kubernetes/scheduler.conf
- --authorization-kubeconfig=/etc/kubernetes/scheduler.conf
- --bind-address=127.0.0.1
- --kubeconfig=/etc/kubernetes/scheduler.conf
- --leader-elect=true
#- --port=0
image: registry.aliyuncs.com/google_containers/kube-scheduler:v1.18.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 10259
scheme: HTTPS
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-scheduler
resources:
requests:
cpu: 100m
volumeMounts:
- mountPath: /etc/kubernetes/scheduler.conf
name: kubeconfig
readOnly: true
hostNetwork: true
priorityClassName: system-cluster-critical
volumes:
- hostPath:
path: /etc/kubernetes/scheduler.conf
type: FileOrCreate
name: kubeconfig
status: {}
- vi kube-controller-manager.yaml，把port=0那行注释
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
labels:
component: kube-controller-manager
tier: control-plane
name: kube-controller-manager
namespace: kube-system
spec:
containers:
- command:
- kube-controller-manager
- --allocate-node-cidrs=true
- --authentication-kubeconfig=/etc/kubernetes/controller-manager.conf
- --authorization-kubeconfig=/etc/kubernetes/controller-manager.conf
- --bind-address=127.0.0.1
- --client-ca-file=/etc/kubernetes/pki/ca.crt
- --cluster-cidr=10.244.0.0/16
- --cluster-name=kubernetes
- --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
- --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
- --controllers=*,bootstrapsigner,tokencleaner
- --kubeconfig=/etc/kubernetes/controller-manager.conf
- --leader-elect=true
- --node-cidr-mask-size=24
#- --port=0
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
- --root-ca-file=/etc/kubernetes/pki/ca.crt
- --service-account-private-key-file=/etc/kubernetes/pki/sa.key
- --service-cluster-ip-range=10.96.0.0/12
- --use-service-account-credentials=true
image: registry.aliyuncs.com/google_containers/kube-controller-manager:v1.18.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 10257
scheme: HTTPS
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-controller-manager
resources:
requests:
cpu: 200m
volumeMounts:
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/ca-certificates
name: etc-ca-certificates
readOnly: true
- mountPath: /etc/pki
name: etc-pki
readOnly: true
- mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
name: flexvolume-dir
- mountPath: /etc/kubernetes/pki
name: k8s-certs
readOnly: true
- mountPath: /etc/kubernetes/controller-manager.conf
name: kubeconfig
readOnly: true
- mountPath: /usr/local/share/ca-certificates
name: usr-local-share-ca-certificates
readOnly: true
- mountPath: /usr/share/ca-certificates
name: usr-share-ca-certificates
readOnly: true
这时10251，10252端口就开启了，健康检查状态也正常了。
root@master:/etc/kubernetes/manifests# netstat -tulpn
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 192.168.74.140:2380 0.0.0.0:* LISTEN 3428/etcd
tcp 0 0 127.0.0.1:2381 0.0.0.0:* LISTEN 3428/etcd
tcp 0 0 127.0.0.1:10257 0.0.0.0:* LISTEN 12833/kube-controll
tcp 0 0 127.0.0.1:10259 0.0.0.0:* LISTEN 12385/kube-schedule
tcp 0 0 127.0.0.1:37845 0.0.0.0:* LISTEN 12520/kubelet
tcp 0 0 192.168.122.1:53 0.0.0.0:* LISTEN 3755/dnsmasq
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 1256/sshd
tcp 0 0 0.0.0.0:31359 0.0.0.0:* LISTEN 4358/kube-proxy
tcp 0 0 127.0.0.1:10248 0.0.0.0:* LISTEN 12520/kubelet
tcp 0 0 127.0.0.1:10249 0.0.0.0:* LISTEN 4358/kube-proxy
tcp 0 0 192.168.74.140:2379 0.0.0.0:* LISTEN 3428/etcd
tcp 0 0 127.0.0.1:2379 0.0.0.0:* LISTEN 3428/etcd
tcp6 0 0 :::10252 :::* LISTEN 12833/kube-controll
tcp6 0 0 :::10256 :::* LISTEN 4358/kube-proxy
tcp6 0 0 :::22 :::* LISTEN 1256/sshd
tcp6 0 0 :::10250 :::* LISTEN 12520/kubelet
tcp6 0 0 :::10251 :::* LISTEN 12385/kube-schedule
tcp6 0 0 :::6443 :::* LISTEN 3347/kube-apiserver
udp 0 0 192.168.122.1:53 0.0.0.0:* 3755/dnsmasq
udp 0 0 0.0.0.0:67 0.0.0.0:* 3755/dnsmasq
udp 0 0 0.0.0.0:68 0.0.0.0:* 1016/dhclient
udp 0 0 0.0.0.0:8472 0.0.0.0:* -
root@master:/etc/kubernetes/manifests# kubectl get cs
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-0 Healthy {"health":"true"}
原因2: kubelet配置文件里没有指定静态pod的路径
解决办法：
（5.1）查找kubelet的配置文件位置 systemctl cat kubelet
[root@cka-node01 ~]# systemctl cat kubelet
# /usr/lib/systemd/system/kubelet.service
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=https://kubernetes.io/docs/
[Service]
ExecStart=/usr/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
# /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf 这是kubelet的配置文件
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/sysconfig/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
（5.2）vim /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf 打开配置文件，添加以下内容
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests" 要添加这个配置
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/sysconfig/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
（5.3）重新加载daemon, 重启kubelet
[root@cka-node01 ~]# systemctl daemon-reload
[root@cka-node01 ~]# systemctl restart kubelet
添加前
在node上 ps -ef |grep kubelet 查看加载的配置文件
[root@cka-node01 ~]# ps -ef |grep kubelet
root 689 1 2 Aug16 ? 00:51:35 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/config.yaml --cgroup-driver=systemd --network-plugin=cni --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2
root 126462 126161 0 11:57 pts/0 00:00:00 grep --color=auto kubelet
添加后
[root@cka-node01 ~]# ps -ef |grep kubelet
root 4457 1 6 12:17 ? 00:00:02 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --pod-manifest-path=/etc/kubernetes/manifests --config=/var/lib/kubelet/config.yaml --cgroup-driver=systemd --network-plugin=cni --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2
root 5089 126161 0 12:18 pts/0 00:00:00 grep --color=auto kubelet
查找kubectl的帮助，grep manifest 可以看到
[root@cka-node01 ~]# kubelet --help |grep mani
manifest can be provided to the Kubelet.
(underspec'd currently) to submit a new manifest.
--cgroup-driver string Driver that the kubelet uses to manipulate cgroups on the host. Possible values: 'cgroupfs', 'systemd' (default "cgroupfs") (DEPRECATED: This parameter should be set via the config file specified by the Kubelet's --config flag. See https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/ for more information.)
--manifest-url string URL for accessing additional Pod specifications to run (DEPRECATED: This parameter should be set via the config file specified by the Kubelet's --config flag. See https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/ for more information.)
--manifest-url-header --manifest-url-header 'a:hello,b:again,c:world' --manifest-url-header 'b:beautiful' Comma-separated list of HTTP headers to use when accessing the url provided to --manifest-url. Multiple headers with the same name will be added in the same order provided. This flag can be repeatedly invoked. For example: --manifest-url-header 'a:hello,b:again,c:world' --manifest-url-header 'b:beautiful' (DEPRECATED: This parameter should be set via the config file specified by the Kubelet's --config flag. See https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/ for more information.)
--pod-manifest-path string Path to the directory containing static pod files to run, or the path to a single static pod file. Files starting with dots will be ignored. (DEPRECATED: This parameter should be set via the config file specified by the Kubelet's --config flag. See https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/ for more information.)
