cfssl生成证书与etcd集群搭建与升级

[root@k8s01 cert]# cat ca-csr.json 
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "HangZhou",
      "L": "XS",
      "O": "k8s",
      "OU": "System"
    }
  ],
  "ca": {
    "expiry": "131400h"   #CA证书过期时间
  }
}

第一步:生成基础ca证书 包含

ca.csr  证书签名请求，一般用于提供给证书颁发机构，自签就不需要了

ca-key.pem 私钥

ca.pem 公钥

cfssl gencert -initca ca-csr.json | cfssl-json -bare ca

cfssl-certinfo -cert ca.pem #可以反解析出ca-csr.json 与公钥

 第二步: 生成etcd 证书

[root@k8s01 cert]# cat ca-config.json 
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {    # -profile 参数必须kubernetes不然会报错no key usage available
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"   #以此CA生成的证书过期时间都以此为准
      }
    }
  }
}
[root@k8s01 cert]# cat etcd-csr.json 
{
  "CN": "etcd",
  "hosts": [             #必须包含你需要放置证书的机器
    "172.25.32.231",
    "172.25.32.232",
    "172.25.32.233",
    "127.0.0.1"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "HangZhou",
      "L": "XS",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

cfssl gencert -config=ca-config.json -ca=ca.pem -ca-key=ca-key.pem -profile=kubernetes etcd-csr.json |cfssl-json -bare etcd
# -bare 生成key的名字 -profile 与ca-config.json一致

 etcd集群管理

#节点1启动
/opt/kube/bin/etcd   --name=etcd1   --cert-file=/etc/etcd/ssl/etcd.pem   --key-file=/etc/etcd/ssl/etcd-key.pem   --peer-cert-file=/etc/etcd/ssl/etcd.pem   --peer-key-file=/etc/etcd/ssl/etcd-key.pem   --trusted-ca-file=/etc/kubernetes/ssl/ca.pem   --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem   --initial-advertise-peer-urls=https://172.25.32.231:2380   --listen-peer-urls=https://172.25.32.231:2380   --listen-client-urls=https://172.25.32.231:2379,http://127.0.0.1:2379   --advertise-client-urls=https://172.25.32.231:2379   --initial-cluster-token=etcd-cluster-0   --initial-cluster=etcd1=https://172.25.32.231:2380,etcd2=https://172.25.32.232:2380,etcd3=https://172.25.32.233:2380   --initial-cluster-state=new   --data-dir=/var/lib/etcd
#节点二启动
/opt/kube/bin/etcd   --name=etcd2   --cert-file=/etc/etcd/ssl/etcd.pem   --key-file=/etc/etcd/ssl/etcd-key.pem   --peer-cert-file=/etc/etcd/ssl/etcd.pem   --peer-key-file=/etc/etcd/ssl/etcd-key.pem   --trusted-ca-file=/etc/kubernetes/ssl/ca.pem   --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem   --initial-advertise-peer-urls=https://172.25.32.232:2380   --listen-peer-urls=https://172.25.32.232:2380   --listen-client-urls=https://172.25.32.232:2379,http://127.0.0.1:2379   --advertise-client-urls=https://172.25.32.232:2379   --initial-cluster-token=etcd-cluster-0   --initial-cluster=etcd1=https://172.25.32.231:2380,etcd2=https://172.25.32.232:2380,etcd3=https://172.25.32.233:2380   --initial-cluster-state=new   --data-dir=/var/lib/etcd
#节点三启动
/opt/kube/bin/etcd   --name=etcd3   --cert-file=/etc/etcd/ssl/etcd.pem   --key-file=/etc/etcd/ssl/etcd-key.pem   --peer-cert-file=/etc/etcd/ssl/etcd.pem   --peer-key-file=/etc/etcd/ssl/etcd-key.pem   --trusted-ca-file=/etc/kubernetes/ssl/ca.pem   --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem   --initial-advertise-peer-urls=https://172.25.32.233:2380   --listen-peer-urls=https://172.25.32.233:2380   --listen-client-urls=https://172.25.32.233:2379,http://127.0.0.1:2379   --advertise-client-urls=https://172.25.32.233:2379   --initial-cluster-token=etcd-cluster-0   --initial-cluster=etcd1=https://172.25.32.231:2380,etcd2=https://172.25.32.232:2380,etcd3=https://172.25.32.233:2380   --initial-cluster-state=new   --data-dir=/var/lib/etcd
# 集群状态检查
##设置别名 需要加证书地址不然会认证不到，接口改成3
cat /root/.bashrc |grep etcdctl
alias etcdctl='export ETCDCTL_API=3&&etcdctl --endpoints=172.25.32.231:2379,172.25.32.232:2379,172.25.32.233:2379 --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem   --key=/etc/etcd/ssl/etcd-key.pem'
#状态
etcdctl endpoint status -w table
# 健康检查
etcdctl endpoint health
#插入数据
etcdctl put lxs "test"
#查询
etcdct get lxs
#获取所有key
etcdctl  --prefix --keys-only=true get /

 将服务设置为systemctl启动

32.231
cat>/etc/systemd/system/etcd.service<<EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/opt/kube/bin/etcd \
  --name=etcd1 \
  --cert-file=/etc/etcd/ssl/etcd.pem \
  --key-file=/etc/etcd/ssl/etcd-key.pem \
  --peer-cert-file=/etc/etcd/ssl/etcd.pem \
  --peer-key-file=/etc/etcd/ssl/etcd-key.pem \
  --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --initial-advertise-peer-urls=https://172.25.32.231:2380 \
  --listen-peer-urls=https://172.25.32.231:2380 \
  --listen-client-urls=https://172.25.32.231:2379,http://127.0.0.1:2379 \
  --advertise-client-urls=https://172.25.32.231:2379 \
  --initial-cluster-token=etcd-cluster-0 \
  --initial-cluster=etcd1=https://172.25.32.231:2380,etcd2=https://172.25.32.232:2380,etcd3=https://172.25.32.233:2380 \
  --initial-cluster-state=new \
  --data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

--name    #节点名字
--listen-peer-urls        #监听其他节点所用的地址
--listen-client-urls    #监听etcd客户端的地址
--initial-advertise-peer-urls    #与其他节点交互信息的地址
--advertise-client-urls    #与etcd客户端交互信息的地址

32.232
cat>/etc/systemd/system/etcd.service<<EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/opt/kube/bin/etcd \
  --name=etcd2 \
  --cert-file=/etc/etcd/ssl/etcd.pem \
  --key-file=/etc/etcd/ssl/etcd-key.pem \
  --peer-cert-file=/etc/etcd/ssl/etcd.pem \
  --peer-key-file=/etc/etcd/ssl/etcd-key.pem \
  --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --initial-advertise-peer-urls=https://172.25.32.232:2380 \
  --listen-peer-urls=https://172.25.32.232:2380 \
  --listen-client-urls=https://172.25.32.232:2379,http://127.0.0.1:2379 \
  --advertise-client-urls=https://172.25.32.232:2379 \
  --initial-cluster-token=etcd-cluster-0 \
  --initial-cluster=etcd1=https://172.25.32.231:2380,etcd2=https://172.25.32.232:2380,etcd3=https://172.25.32.233:2380 \
  --initial-cluster-state=new \
  --data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

32.233
cat>/etc/systemd/system/etcd.service<<EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/opt/kube/bin/etcd \
  --name=etcd3\
  --cert-file=/etc/etcd/ssl/etcd.pem \
  --key-file=/etc/etcd/ssl/etcd-key.pem \
  --peer-cert-file=/etc/etcd/ssl/etcd.pem \
  --peer-key-file=/etc/etcd/ssl/etcd-key.pem \
  --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --initial-advertise-peer-urls=https://172.25.32.233:2380 \
  --listen-peer-urls=https://172.25.32.233:2380 \
  --listen-client-urls=https://172.25.32.233:2379,http://127.0.0.1:2379 \
  --advertise-client-urls=https://172.25.32.233:2379 \
  --initial-cluster-token=etcd-cluster-0 \
  --initial-cluster=etcd1=https://172.25.32.231:2380,etcd2=https://172.25.32.232:2380,etcd3=https://172.25.32.233:2380 \
  --initial-cluster-state=new \
  --data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

 etcd升级官方文档

# 备份快照
snapshot save backup.db
# 查看备份
etcdctl --write-out=table snapshot status backup.db
# 恢复快照
ETCDCTL_API=3 etcdctl snapshot restore backup.db --data-dir=/var/lib/etcd
#参考官网逐个节点升级 3.3升级到3.4新增参数
+ --initial-cluster-state new \
+ --logger zap \
+ --log-outputs stderr