k8s1.14.1 集群更新证书过期时间

适用场景

部分组件证书检查只有1年时间（即不止kubelet证书），但当前未过期

部分老包可能会出现一些master组件证书，或者kubelet证书只有1年的时间，如果检查发现若干证书都是1年，需要按如下步骤update为10/100年证书。

重新编译Kubeadm

Linux安装Go环境

说明因为kubernetes-1.14.1依赖go-1.12+，本文档采用1.12.1

wget https://dl.google.com/go/go1.12.1.linux-amd64.tar.gz
tar -zxf go1.12.1.linux-amd64.tar.gz -C /usr/local
vim /etc/profile
# 添加如下信息
export PATH=$PATH:/usr/local/go/bin
source /etc/profile
# 检查go是否配置正确
[root@k8s-Master ~]# go version
go version go1.12.1 linux/amd64

编译kubeadm

# 下载对应的kubernetes源代码，这里采用 "v1.14.1" 版本
wget https://codeload.github.com/kubernetes/kubernetes/tar.gz/v1.14.1

　修改源代码-cert.go

文件：staging/src/k8s.io/client-go/util/cert/cert.go
说明：1.14.0版本开始，此文件不需要修改
NewSelfSignedCACert 方法，签发以下证书，且默认为10年有效期：
- front-proxy-ca.crt
- front-proxy-client.crt
- ca.crt
- etcd/ca.crt
- etcd/peer.crt

 38 const duration365d = time.Hour * 24 * 365
 39 
 40 // Config contains the basic fields required for creating a certificate
 41 type Config struct {
 42         CommonName   string
 43         Organization []string
 44         AltNames     AltNames
 45         Usages       []x509.ExtKeyUsage
 46 }
 47 
 48 // AltNames contains the domain names and IP addresses that will be added
 49 // to the API Server's x509 certificate SubAltNames field. The values will
 50 // be passed directly to the x509.Certificate object.
 51 type AltNames struct {
 52         DNSNames []string
 53         IPs      []net.IP
 54 }
 55 
 56 // NewSelfSignedCACert creates a CA certificate
 57 func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, error) {
 58         now := time.Now()
 59         tmpl := x509.Certificate{
 60                 SerialNumber: new(big.Int).SetInt64(0),
 61                 Subject: pkix.Name{
 62                         CommonName:   cfg.CommonName,
 63                         Organization: cfg.Organization,
 64                 },
 65                 NotBefore:             now.UTC(),
 66                 NotAfter:              now.Add(duration365d * 10).UTC(), //在1.14.0以后的版本这个地方官方默认设置成了10年，所以可以忽略
 67                 KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
 68                 BasicConstraintsValid: true,
 69                 IsCA:                  true,
 70         }
 71 
 72         certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &tmpl, &tmpl, key.Public(), key)
 73         if err != nil {
 74                 return nil, err
 75         }
 76         return x509.ParseCertificate(certDERBytes)
 77 }

　修改源代码-pki_helpers.go

文件：cmd/kubeadm/app/util/pkiutil/pki_helpers.go
以下证书由 NewSignedCert 方法签发，但签发的证书默认只有一年有效期：
- apiserver.crt
- apiserver-etcd-client.crt
- etcd/server.crt
- etcd/healthcheck-client.crt
- apiserver-kubelet-client.crt

557 func NewSignedCert(cfg *certutil.Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) {
558         serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64))
559         if err != nil {
560                 return nil, err
561         }
562         if len(cfg.CommonName) == 0 {
563                 return nil, errors.New("must specify a CommonName")
564         }
565         if len(cfg.Usages) == 0 {
566                 return nil, errors.New("must specify at least one ExtKeyUsage")
567         }
568 
569         certTmpl := x509.Certificate{
570                 Subject: pkix.Name{
571                         CommonName:   cfg.CommonName,
572                         Organization: cfg.Organization,
573                 },
574                 DNSNames:     cfg.AltNames.DNSNames,
575                 IPAddresses:  cfg.AltNames.IPs,
576                 SerialNumber: serial,
577                 NotBefore:    caCert.NotBefore,
578                 NotAfter:     time.Now().Add(duration365d * 10).UTC(),// 添加 "* 10"
579                 KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
580                 ExtKeyUsage:  cfg.Usages,
581         }
582         certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey)
583         if err != nil {
584                 return nil, err
585         }
586         return x509.ParseCertificate(certDERBytes)
587 }

　　编译

# kubeadm
make WHAT=cmd/kubeadm GOFLAGS=-v

# 补充：编译kubelet
# make all WHAT=cmd/kubelet GOFLAGS=-v

# 补充：编译kubectl
# make all WHAT=cmd/kubectl GOFLAGS=-v

# 编译生成的二进制文件在 _output/bin/ 目录下

　　编译报错

报错一：
[root@k8s-Master kubernetes-1.14.1]# make WHAT=cmd/kubeadm GOFLAGS=-v
+++ [1122 15:05:41] Building go targets for linux/amd64:
    ./vendor/k8s.io/code-generator/cmd/conversion-gen
k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/conversion-gen/args
k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/conversion-gen/generators
k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/conversion-gen
find: ‘rsync’: 没有那个文件或目录
find: ‘rsync’: 没有那个文件或目录
find: ‘rsync’: 没有那个文件或目录
find: ‘rsync’: 没有那个文件或目录
./hack/run-in-gopath.sh:行33: _output/bin/conversion-gen: 权限不够

# 解决rsync没有哪个文件或目录错误
yum -y install rsync
# 解决权限不够问题
[root@k8s-Master kubernetes-1.14.1]# chmod +x _output/bin/conversion-gen

报错二：
[root@k8s-Master kubernetes]# kubeadm -h
-bash: /usr/bin/kubeadm: 无法执行二进制文件
该问题是由于跨平台导致，最初kubeadm是再mca os编译的拿到Linux系统无法使用，解决该问题就是使用Linux系统重新编译即可

　　替换kubeadm

# 替换前先生成现有集群中的kubeadm-config.yaml文件
/usr/bin/kubeadm config print init-defaults --kubeconfig ClusterConfiguration > ./kubeadm-config.yml
# 将kubeadm 文件拷贝替换系统中原有kubeadm
cp /usr/bin/kubeadm /usr/bin/kubeadm.origin
cp _output/bin/kubeadm /usr/bin/kubeadm

重新生成证书

更新 kube-master (任一)节点证书

# 备份
cp -r /etc/kubernetes/pki /etc/kubernetes/pki.origin

# 更新证书；
# "--config" 指定 "kubeadm" 的配置文件，建议使用部署集群时使用的配置文件；
# 其他参数可参考官方文档
cd /etc/kubernetes/pki
kubeadm alpha certs renew all --config=/root/kubeadm/kubeadm-config.yaml

# 验证新的证书有效期，以 "apiserver.crt" 为例
openssl x509 -in apiserver.crt -text -noout | grep Not

# 备份 /etc/kubernetes/*.conf 文件；
# 必须备份(mv)，否则无法更新 "*.conf" 文件
ll /etc/kubernetes/*.conf | awk '{print $9}' | xargs -i mv {} {}.`date "+%Y%m%d"`

# 更新/etc/kubernetes/*.conf文件；
# 如果没有 "mv" , 输出为 "[kubeconfig] Using existing kubeconfig file"，
kubeadm init phase kubeconfig all --config=/root/kubeadm/kubeadm-config.yaml
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file

# 在(所有) kube-master 节点重启 "kube-apiserver", "kube-controller-manager", "kube-scheduler", "etcd" 4个容器
docker ps | egrep "etcd|kube-apiserver|kube-controller-manager|kube-scheduler" | grep -v pause | awk '{print $1}' | xargs -i docker restart {}

# 覆盖 "$HOME/.kube/config" 文件
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config

　HA集群其余mater节点证书更新

在已更新证书的master节点运行脚本，将更新的证书同步到其余master节点。
笔者使用的是单Master集群，该步骤未执行。

# 如果可以，请提前在被同步master节点做备份
cat certificate_scp.sh
#!/bin/bash

USER=root # customizable
CONTROL_PLANE_IPS="1.1.1.1 2.2.2.2"
for host in ${CONTROL_PLANE_IPS}; do
    scp /etc/kubernetes/pki/ca.crt "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/ca.key "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/sa.key "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/sa.pub "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/front-proxy-ca.crt "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/front-proxy-ca.key "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/etcd/ca.crt "${USER}"@$host:/etc/kubernetes/pki/etcd/ca.crt
    scp /etc/kubernetes/pki/etcd/ca.key "${USER}"@$host:/etc/kubernetes/pki/etcd/ca.key
    scp /etc/kubernetes/admin.conf "${USER}"@$host:/etc/kubernetes/
done

　验证所有节点kubelet有无开启证书自动轮换

　　　在所有节点验证如下配置

## 执行如下命令，看输出是否有：rotateCertificates: true
cat /var/lib/kubelet/config.yaml |grep rotateCertificates

　　　如果有，则不需要添加如下配置直接进行第2.4步即可。

　　　如果没有，需要编辑/etc/kubernetes/kubelet.env增加，

--feature-gates=RotateKubeletClientCertificate=true \
--rotate-certificates=true \

　选取一个master节点生成永久不失效token

首先生成一个新的token，记录这个token后面会用，生成之前需要配置token有效期为forever

使用客户端证书轮换必须保证配置中token为有效的。
图中1画圈的为添加部分最终效果如图：

vim  /etc/kubernetes/kubeadm-config.yaml
##  添加内容
apiVersion: kubeadm.k8s.io/v1beta1
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  ttl: "0"
  usages:
  - signing
  - authentication
 
##  生成token
kubeadm token create --config /root/kubeadm/kubeadm-config.yaml
##  查看TTL失效为 forever 见图2
kubeadm token list

　　图1

　　图2

　在所有node节点替换token

vim /etc/kubernetes/bootstrap-kubelet.conf 
## 修改token的值为新生成的

　　图1，修改token后面的部分

重启kubelet

systemctl daemon-reload
systemctl restart kubelet

　修改controller-manager自动轮换的证书签署时间

在所有master节点操作
编辑/etc/kubernetes/manifests/kube-controller-manager.yaml增加签署时间，默认为8760，这里把签署时间改为10年

# 没有就新加
- --experimental-cluster-signing-duration=87600h0m0s

重启kubelet和controller-manager

systemctl daemon-reload
systemctl restart kubelet
 
 
## 检查controller-manager有无重启，没有则手动delete或者docker操作下触发重启，操作如下：
kubectl get po -n kube-system|grep controller-manager
kubectl delete po {查询到的所有controller-manager pod name} -n kube-system

　备份并让controller-manager重新生成kubelet证书

在所有节点操作

cd /var/lib/kubelet/pki && mkdir bak-kubelet-client
mv kubelet-client-* bak-kubelet-client/
systemctl restart kubelet

验证  验证k8s集群状态均为Ready

[root@k8s-Master kubeadm]# kubectl get  node
NAME         STATUS   ROLES    AGE   VERSION
k8s-master   Ready    master   20d   v1.14.1
k8s-node1    Ready    <none>   20d   v1.14.1
k8s-node2    Ready    <none>   20d   v1.14.1

验证所有证书过期时间

[root@k8s-Master kubeadm]# for i in $(find /etc/kubernetes/pki -type f -name "*.crt");do  echo "crt: $i" &&  openssl x509 -in $i -noout -text|grep Not; done
crt: /etc/kubernetes/pki/ca.crt
            Not Before: Nov  1 14:03:05 2022 GMT
            Not After : Oct 29 14:03:05 2032 GMT
crt: /etc/kubernetes/pki/apiserver.crt
            Not Before: Nov  1 14:03:05 2022 GMT
            Not After : Nov 19 07:22:08 2032 GMT
crt: /etc/kubernetes/pki/apiserver-kubelet-client.crt
            Not Before: Nov  1 14:03:05 2022 GMT
            Not After : Nov 19 07:22:08 2032 GMT
crt: /etc/kubernetes/pki/front-proxy-ca.crt
            Not Before: Nov  1 14:03:06 2022 GMT
            Not After : Oct 29 14:03:06 2032 GMT
crt: /etc/kubernetes/pki/front-proxy-client.crt
            Not Before: Nov  1 14:03:06 2022 GMT
            Not After : Nov 19 07:22:08 2032 GMT
crt: /etc/kubernetes/pki/etcd/ca.crt
            Not Before: Nov  1 14:03:06 2022 GMT
            Not After : Oct 29 14:03:06 2032 GMT
crt: /etc/kubernetes/pki/etcd/server.crt
            Not Before: Nov  1 14:03:06 2022 GMT
            Not After : Nov 19 07:22:08 2032 GMT
crt: /etc/kubernetes/pki/etcd/peer.crt
            Not Before: Nov  1 14:03:06 2022 GMT
            Not After : Nov 19 07:22:08 2032 GMT
crt: /etc/kubernetes/pki/etcd/healthcheck-client.crt
            Not Before: Nov  1 14:03:06 2022 GMT
            Not After : Nov 19 07:22:08 2032 GMT
crt: /etc/kubernetes/pki/apiserver-etcd-client.crt
            Not Before: Nov  1 14:03:06 2022 GMT
            Not After : Nov 19 07:22:08 2032 GMT

posted @ 2022-11-22 16:30 屌丝逆袭记阅读(554) 评论(0) 收藏举报

刷新页面返回顶部

Loading

走向人生巅峰