etcd基于https的集群部署
一.etcd基于https的集群部署
1.准备etcd程序包
1.1 下载etcd的软件包
wget https://github.com/etcd-io/etcd/releases/download/v3.5.17/etcd-v3.5.17-linux-amd64.tar.gz
svip:
[root@node-exporter41 ~]# wget http://192.168.15.253/Resources/Prometheus/softwares/Etcd/etcd-v3.5.17-linux-amd64.tar.gz
1.2 解压etcd的二进制程序包到PATH环境变量路径
[root@node-exporter41 ~]# tar -xf etcd-v3.5.17-linux-amd64.tar.gz --strip-components=1 -C /usr/local/bin etcd-v3.5.17-linux-amd64/etcd{,ctl}
[root@node-exporter41 ~]#
[root@node-exporter41 ~]# ll /usr/local/bin/etcd*
-rwxr-xr-x 1 oldboyedu oldboyedu 23625880 Nov 13 00:32 /usr/local/bin/etcd*
-rwxr-xr-x 1 oldboyedu oldboyedu 17899672 Nov 13 00:32 /usr/local/bin/etcdctl*
[root@node-exporter41 ~]#
[root@node-exporter41 ~]# etcdctl version
etcdctl version: 3.5.17
API version: 3.5
[root@node-exporter41 ~]#
[root@node-exporter41 ~]# scp /usr/local/bin/etcd* 10.0.0.42:/usr/local/bin
[root@node-exporter41 ~]# scp /usr/local/bin/etcd* 10.0.0.43:/usr/local/bin
2.准备etcd的证书文件
2.1 安装cfssl证书管理工具
[root@node-exporter41 ~]# wget http://192.168.15.253/Resources/Prometheus/softwares/Etcd/oldboyedu-cfssl-v1.6.5.zip
[root@node-exporter41 ~]#
[root@node-exporter41 ~]# unzip oldboyedu-cfssl-v1.6.5.zip
[root@node-exporter41 ~]#
[root@node-exporter41 ~]# apt install rename
[root@node-exporter41 ~]#
[root@node-exporter41 ~]# rename -v "s/_1.6.5_linux_amd64//g" cfssl*
[root@node-exporter41 ~]#
[root@node-exporter41 ~]# mv cfssl* /usr/local/bin/
[root@node-exporter41 ~]#
[root@node-exporter41 ~]# chmod +x /usr/local/bin/cfssl*
[root@node-exporter41 ~]#
[root@node-exporter41 ~]# ll /usr/local/bin/cfssl*
-rwxr-xr-x 1 root root 11890840 Jun 15 11:56 /usr/local/bin/cfssl*
-rwxr-xr-x 1 root root 8413336 Jun 15 11:56 /usr/local/bin/cfssl-certinfo*
-rwxr-xr-x 1 root root 6205592 Jun 15 11:56 /usr/local/bin/cfssljson*
[root@node-exporter41 ~]#
2.2 创建证书存储目录
[root@node-exporter41 ~]# mkdir -pv /oldboyedu/certs/etcd && cd /oldboyedu/certs/etcd/
2.3 生成证书的CSR文件： 证书签发请求文件，配置了一些域名，公司，单位
[root@node-exporter41 etcd]# cat > etcd-ca-csr.json <<EOF
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "etcd",
"OU": "Etcd Security"
}
],
"ca": {
"expiry": "876000h"
}
}
EOF
2.4 生成etcd CA证书和CA证书的key
[root@node-exporter41 etcd]# cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare /oldboyedu/certs/etcd/etcd-ca
[root@node-exporter41 etcd]#
[root@node-exporter41 etcd]# ll /oldboyedu/certs/etcd/etcd-ca*
-rw-r--r-- 1 root root 1050 Nov 15 10:42 /oldboyedu/certs/etcd/etcd-ca.csr
-rw-r--r-- 1 root root 249 Nov 15 10:42 /oldboyedu/certs/etcd/etcd-ca-csr.json
-rw------- 1 root root 1675 Nov 15 10:42 /oldboyedu/certs/etcd/etcd-ca-key.pem
-rw-r--r-- 1 root root 1318 Nov 15 10:42 /oldboyedu/certs/etcd/etcd-ca.pem
[root@node-exporter41 etcd]#
2.5 生成etcd证书的有效期为100年
[root@node-exporter41 etcd]# cat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "876000h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "876000h"
}
}
}
}
EOF
2.6 生成证书的CSR文件： 证书签发请求文件，配置了一些域名，公司，单位
[root@node-exporter41 etcd]# cat > etcd-csr.json <<EOF
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "etcd",
"OU": "Etcd Security"
}
]
}
EOF
2.7 基于自建的ectd ca证书生成etcd的证书
[root@node-exporter41 etcd]# cfssl gencert \
-ca=/oldboyedu/certs/etcd/etcd-ca.pem \
-ca-key=/oldboyedu/certs/etcd/etcd-ca-key.pem \
-config=ca-config.json \
--hostname=127.0.0.1,node-exporter41,node-exporter42,node-exporter43,10.0.0.41,10.0.0.42,10.0.0.43 \
--profile=kubernetes \
etcd-csr.json | cfssljson -bare /oldboyedu/certs/etcd/etcd-server
[root@k8s-master01 pki]# ll /oldboyedu/certs/etcd/etcd-server*
-rw-r--r-- 1 root root 1131 Jun 24 15:18 /oldboyedu/certs/etcd/etcd-server.csr
-rw------- 1 root root 1679 Jun 24 15:18 /oldboyedu/certs/etcd/etcd-server-key.pem
-rw-r--r-- 1 root root 1464 Jun 24 15:18 /oldboyedu/certs/etcd/etcd-server.pem
[root@k8s-master01 pki]#
2.8 将etcd证书拷贝到其他两个master节点
[root@node-exporter41 etcd]# scp -r /oldboyedu/certs/ 10.0.0.42:/oldboyedu
[root@node-exporter41 etcd]# scp -r /oldboyedu/certs/ 10.0.0.43:/oldboyedu
[root@node-exporter42 ~]# ll /oldboyedu/certs/etcd/
total 44
drwxr-xr-x 2 root root 4096 Nov 15 10:49 ./
drwxr-xr-x 3 root root 4096 Nov 15 10:49 ../
-rw-r--r-- 1 root root 294 Nov 15 10:49 ca-config.json
-rw-r--r-- 1 root root 1050 Nov 15 10:49 etcd-ca.csr
-rw-r--r-- 1 root root 249 Nov 15 10:49 etcd-ca-csr.json
-rw------- 1 root root 1675 Nov 15 10:49 etcd-ca-key.pem
-rw-r--r-- 1 root root 1318 Nov 15 10:49 etcd-ca.pem
-rw-r--r-- 1 root root 210 Nov 15 10:49 etcd-csr.json
-rw-r--r-- 1 root root 1143 Nov 15 10:49 etcd-server.csr
-rw------- 1 root root 1679 Nov 15 10:49 etcd-server-key.pem
-rw-r--r-- 1 root root 1476 Nov 15 10:49 etcd-server.pem
[root@node-exporter42 ~]#
[root@node-exporter43 ~]# ll /oldboyedu/certs/etcd/
total 44
drwxr-xr-x 2 root root 4096 Nov 15 10:49 ./
drwxr-xr-x 3 root root 4096 Nov 15 10:49 ../
-rw-r--r-- 1 root root 294 Nov 15 10:49 ca-config.json
-rw-r--r-- 1 root root 1050 Nov 15 10:49 etcd-ca.csr
-rw-r--r-- 1 root root 249 Nov 15 10:49 etcd-ca-csr.json
-rw------- 1 root root 1675 Nov 15 10:49 etcd-ca-key.pem
-rw-r--r-- 1 root root 1318 Nov 15 10:49 etcd-ca.pem
-rw-r--r-- 1 root root 210 Nov 15 10:49 etcd-csr.json
-rw-r--r-- 1 root root 1143 Nov 15 10:49 etcd-server.csr
-rw------- 1 root root 1679 Nov 15 10:49 etcd-server-key.pem
-rw-r--r-- 1 root root 1476 Nov 15 10:49 etcd-server.pem
[root@node-exporter43 ~]#
[root@node-exporter43 ~]#
3.创建etcd集群各节点配置文件
3.1node-exporter41节点的配置文件
[root@node-exporter41 ~]# mkdir -pv /oldboyedu/softwares/etcd
[root@node-exporter41 ~]#
[root@node-exporter41 ~]# cat > /oldboyedu/softwares/etcd/etcd.config.yml <<'EOF'
name: 'node-exporter41'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://10.0.0.41:2380'
listen-client-urls: 'https://10.0.0.41:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://10.0.0.41:2380'
advertise-client-urls: 'https://10.0.0.41:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'node-exporter41=https://10.0.0.41:2380,node-exporter42=https://10.0.0.42:2380,node-exporter43=https://10.0.0.43:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
cert-file: '/oldboyedu/certs/etcd/etcd-server.pem'
key-file: '/oldboyedu/certs/etcd/etcd-server-key.pem'
client-cert-auth: true
trusted-ca-file: '/oldboyedu/certs/etcd/etcd-ca.pem'
auto-tls: true
peer-transport-security:
cert-file: '/oldboyedu/certs/etcd/etcd-server.pem'
key-file: '/oldboyedu/certs/etcd/etcd-server-key.pem'
peer-client-cert-auth: true
trusted-ca-file: '/oldboyedu/certs/etcd/etcd-ca.pem'
auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF
3.2 node-exporter42节点的配置文件
[root@node-exporter42 ~]# mkdir -pv /oldboyedu/softwares/etcd
[root@node-exporter42 ~]#
[root@node-exporter42 ~]# cat > /oldboyedu/softwares/etcd/etcd.config.yml <<'EOF'
name: 'node-exporter42'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://10.0.0.42:2380'
listen-client-urls: 'https://10.0.0.42:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://10.0.0.42:2380'
advertise-client-urls: 'https://10.0.0.42:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'node-exporter41=https://10.0.0.41:2380,node-exporter42=https://10.0.0.42:2380,node-exporter43=https://10.0.0.43:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
cert-file: '/oldboyedu/certs/etcd/etcd-server.pem'
key-file: '/oldboyedu/certs/etcd/etcd-server-key.pem'
client-cert-auth: true
trusted-ca-file: '/oldboyedu/certs/etcd/etcd-ca.pem'
auto-tls: true
peer-transport-security:
cert-file: '/oldboyedu/certs/etcd/etcd-server.pem'
key-file: '/oldboyedu/certs/etcd/etcd-server-key.pem'
peer-client-cert-auth: true
trusted-ca-file: '/oldboyedu/certs/etcd/etcd-ca.pem'
auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF
3.3 node-exporter43节点的配置文件
[root@node-exporter43 ~]# mkdir -pv /oldboyedu/softwares/etcd
[root@node-exporter43 ~]#
[root@node-exporter43 ~]# cat > /oldboyedu/softwares/etcd/etcd.config.yml <<'EOF'
name: 'node-exporter43'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://10.0.0.43:2380'
listen-client-urls: 'https://10.0.0.43:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://10.0.0.43:2380'
advertise-client-urls: 'https://10.0.0.43:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'node-exporter41=https://10.0.0.41:2380,node-exporter42=https://10.0.0.42:2380,node-exporter43=https://10.0.0.43:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
cert-file: '/oldboyedu/certs/etcd/etcd-server.pem'
key-file: '/oldboyedu/certs/etcd/etcd-server-key.pem'
client-cert-auth: true
trusted-ca-file: '/oldboyedu/certs/etcd/etcd-ca.pem'
auto-tls: true
peer-transport-security:
cert-file: '/oldboyedu/certs/etcd/etcd-server.pem'
key-file: '/oldboyedu/certs/etcd/etcd-server-key.pem'
peer-client-cert-auth: true
trusted-ca-file: '/oldboyedu/certs/etcd/etcd-ca.pem'
auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF
4.编写etcd启动脚本
cat > /usr/lib/systemd/system/etcd.service <<'EOF'
[Unit]
Description=Jason Yin's Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target
[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/oldboyedu/softwares/etcd/etcd.config.yml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
Alias=etcd3.service
EOF
5.启动etcd集群
systemctl daemon-reload && systemctl enable --now etcd
systemctl status etcd
6.查看etcd集群状态
[root@node-exporter41 ~]# etcdctl --endpoints="10.0.0.41:2379,10.0.0.42:2379,10.0.0.43:2379" --cacert=/oldboyedu/certs/etcd/etcd-ca.pem --cert=/oldboyedu/certs/etcd/etcd-server.pem --key=/oldboyedu/certs/etcd/etcd-server-key.pem endpoint status --write-out=table
+----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| 10.0.0.41:2379 | 9378902f41df91e9 | 3.5.17 | 20 kB | true | false | 2 | 9 | 9 | |
| 10.0.0.42:2379 | 18f972748ec1bd96 | 3.5.17 | 20 kB | false | false | 2 | 9 | 9 | |
| 10.0.0.43:2379 | a3dfd2d37c461ee9 | 3.5.17 | 20 kB | false | false | 2 | 9 | 9 | |
+----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[root@node-exporter41 ~]#
7.验证etcd高可用集群
7.1 停止leader节点
[root@node-exporter41 ~]# ss -ntl | egrep "2379|2380"
LISTEN 0 4096 127.0.0.1:2379 0.0.0.0:*
LISTEN 0 4096 10.0.0.41:2379 0.0.0.0:*
LISTEN 0 4096 10.0.0.41:2380 0.0.0.0:*
[root@node-exporter41 ~]#
[root@node-exporter41 ~]# systemctl stop etcd
[root@node-exporter41 ~]#
[root@node-exporter41 ~]# ss -ntl | egrep "2379|2380"
[root@node-exporter41 ~]#
7.2 查看现有集群环境，发现新leader诞生
[root@node-exporter42 ~]# etcdctl --endpoints="10.0.0.41:2379,10.0.0.42:2379,10.0.0.43:2379" --cacert=/oldboyedu/certs/etcd/etcd-ca.pem --cert=/oldboyedu/certs/etcd/etcd-server.pem --key=/oldboyedu/certs/etcd/etcd-server-key.pem endpoint status --write-out=table
{"level":"warn","ts":"2024-11-15T11:12:19.054744+0800","logger":"etcd-client","caller":"v3@v3.5.17/retry_interceptor.go:63","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000456000/10.0.0.41:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 10.0.0.41:2379: connect: connection refused\""}
Failed to get the status of endpoint 10.0.0.41:2379 (context deadline exceeded)
+----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| 10.0.0.42:2379 | 18f972748ec1bd96 | 3.5.17 | 20 kB | true | false | 3 | 10 | 10 | |
| 10.0.0.43:2379 | a3dfd2d37c461ee9 | 3.5.17 | 20 kB | false | false | 3 | 10 | 10 | |
+----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[root@node-exporter42 ~]#
[root@node-exporter42 ~]#
7.3 再将之前的leader起来
[root@node-exporter41 ~]# systemctl start etcd
[root@node-exporter41 ~]#
[root@node-exporter41 ~]# ss -ntl | egrep "2379|2380"
LISTEN 0 4096 127.0.0.1:2379 0.0.0.0:*
LISTEN 0 4096 10.0.0.41:2379 0.0.0.0:*
LISTEN 0 4096 10.0.0.41:2380 0.0.0.0:*
[root@node-exporter41 ~]#
[root@node-exporter41 ~]# etcdctl --endpoints="10.0.0.41:2379,10.0.0.42:2379,10.0.0.43:2379" --cacert=/oldboyedu/certs/etcd/etcd-ca.pem --cert=/oldboyedu/certs/etcd/etcd-server.pem --key=/oldboyedu/certs/etcd/etcd-server-key.pem endpoint status --write-out=table
+----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| 10.0.0.41:2379 | 9378902f41df91e9 | 3.5.17 | 20 kB | false | false | 3 | 11 | 11 | |
| 10.0.0.42:2379 | 18f972748ec1bd96 | 3.5.17 | 20 kB | true | false | 3 | 11 | 11 | |
| 10.0.0.43:2379 | a3dfd2d37c461ee9 | 3.5.17 | 20 kB | false | false | 3 | 11 | 11 | |
+----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[root@node-exporter41 ~]#
浙公网安备 33010602011771号