06.主控节点安装-etcd集群
作用：用来存储k8s中网络及其他调度信息，可以理解为一个mysql数据库。etcd 的leader选举机制，要求至少为3台或以上的奇数台，所以必须至少两台存活。本次安装涉及：server11 server12 server13
1.先签发etcd证书，因为k8s中各节点通信需要使用证书。
登陆证书签发服务器：14服务器
• 创建ca的json配置: /opt/certs/ca-config.json
• server 表示服务端连接客户端时携带的证书，用于客户端验证服务端身份
• client 表示客户端连接服务端时携带的证书，用于服务端验证客户端身份
• peer 表示相互之间连接时使用的证书，如etcd节点之间验证
[root@server13 ~]# /opt/certs/ca-config.json
{
"signing": {
"default": {
"expiry": "175200h"
},
"profiles": {
"server": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"server auth"
]
},
"client": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"client auth"
]
},
"peer": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
• 创建etcd证书配置：/opt/certs/etcd-peer-csr.json
重点在hosts上，将所有可能的etcd服务器添加到host列表，不能使用网段，新增etcd服务器需要重新签发证书（因此本次规划安装三台etc集群服务器11、12、13，但地址地址写了6个。）。
[root@server14 certs]# cat /opt/certs/etcd-peer-csr.json
{
"CN": "k8s-etcd",
"hosts": [
"20.0.0.11",
"20.0.0.12",
"20.0.0.13",
"20.0.0.21",
"20.0.0.22",
"20.0.0.23"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "zhengzhou",
"L": "zhengzhou",
"O": "leon",
"OU": "ops"
}
]
}
[root@server14 certs]#
2.签发证书
[root@server14 certs]# cd /opt/certs/
[root@server14 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer etcd-peer-csr.json |cfssl-json -bare etcd-peer
2020/11/24 18:56:36 [INFO] generate received request
2020/11/24 18:56:36 [INFO] received CSR
2020/11/24 18:56:36 [INFO] generating key: rsa-2048
2020/11/24 18:56:36 [INFO] encoded CSR
2020/11/24 18:56:36 [INFO] signed certificate with serial number 501041424349560828988821268935687692162323792405
2020/11/24 18:56:36 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@server14 certs]#
[root@server14 certs]# ll etcd-peer*
-rw-r--r--. 1 root root 1086 Nov 24 18:56 etcd-peer.csr
-rw-r--r--. 1 root root 413 Nov 24 18:56 etcd-peer-csr.json
-rw-------. 1 root root 1675 Nov 24 18:56 etcd-peer-key.pem
-rw-r--r--. 1 root root 1456 Nov 24 18:56 etcd-peer.pem
[root@server14 certs]#
3.安装etcd
etcd地址：https://github.com/etcd-io/etcd/
实验使用版本: etcd-v3.1.20-linux-amd64.tar.gz
本次安装涉及：11，12，13服务器
3.1 下载etcd
[root@server11 ]# useradd -s /sbin/nologin -M etcd
[root@server11 ]# mkdir -p /opt/src/ && cd /opt/src/
[root@server11 ]# wget https://github.com/etcd-io/etcd/releases/download/v3.1.20/etcd-v3.1.20-linux-amd64.tar.gz
[root@server11 ]# tar -xf etcd-v3.1.20-linux-amd64.tar.gz
[root@server11 ]# mkdir -p /opt/release/ && mv etcd-v3.1.20-linux-amd64 /opt/release/etcd-v3.1.20
[root@server11 ]# mkdir -p /opt/apps/ && ln -s /opt/release/etcd-v3.1.20 /opt/apps/etcd
[root@server11 ]# ll /opt/apps/etcd
lrwxrwxrwx 1 root root 25 Jan 5 17:56 /opt/apps/etcd -> /opt/release/etcd-v3.1.20
[root@server11 ]# mkdir -p /opt/apps/etcd/certs /data/etcd /data/logs/etcd-server
3.2 证书服务器下发证书到各个etcd服务器上（提前做好免秘钥）
[root@server14 ~]# cd /opt/certs/
[root@server14 certs]# ls
ca-config.json ca-key.pem etcd-peer-csr.json
ca.csr ca.pem etcd-peer-key.pem
ca-csr.json etcd-peer.csr etcd-peer.pem
[root@server14 certs]#
[root@server14 certs]# scp ca.pem etcd-peer.pem etcd-peer-key.pem root@20.0.0.11:/opt/apps/etcd/certs/
ca.pem 100% 1354 2.8MB/s 00:00
etcd-peer.pem 100% 1456 2.6MB/s 00:00
etcd-peer-key.pem 100% 1675 3.4MB/s 00:00
[root@server14 certs]# scp ca.pem etcd-peer.pem etcd-peer-key.pem root@20.0.0.12:/opt/apps/etcd/certs/
ca.pem 100% 1354 2.2MB/s 00:00
etcd-peer.pem 100% 1456 2.8MB/s 00:00
etcd-peer-key.pem 100% 1675 3.8MB/s 00:00
[root@server14 certs]# scp ca.pem etcd-peer.pem etcd-peer-key.pem root@20.0.0.13:/opt/apps/etcd/certs/
ca.pem 100% 1354 3.5MB/s 00:00
etcd-peer.pem 100% 1456 3.7MB/s 00:00
etcd-peer-key.pem 100% 1675 4.9MB/s 00:00
[root@server14 certs]#
3.3 md5 验证：
[root@server14 certs]# md5sum ca.pem etcd-peer.pem etcd-peer-key.pem
1202dbc001b819b69a2fe87fefe1139b ca.pem
375f277917a6d93c1561e000b2f62096 etcd-peer.pem
0ade645ef56c85f2074a2ce9ca3753e2 etcd-peer-key.pem
[root@server14 certs]#
[root@server11 src]# md5sum /opt/apps/etcd/certs/*
1202dbc001b819b69a2fe87fefe1139b /opt/apps/etcd/certs/ca.pem
0ade645ef56c85f2074a2ce9ca3753e2 /opt/apps/etcd/certs/etcd-peer-key.pem
375f277917a6d93c1561e000b2f62096 /opt/apps/etcd/certs/etcd-peer.pem
[root@server11 src]#
3.4 创建启动脚本(部分参数每台机器不同)
# listen-peer-urls etcd节点之间通信端口
# listen-client-urls 客户端与etcd通信端口
# quota-backend-bytes 配额大小
# 需要修改的参数：name,listen-peer-urls,listen-client-urls,initial-advertise-peer-urls
[root@server11 ~]# cat /opt/apps/etcd/etcd-server-startup.sh
#!/bin/sh
WORK_DIR=$(dirname $(readlink -f $0))
[ $? -eq 0 ] && cd $WORK_DIR || exit
/opt/apps/etcd/etcd --name etcd-server-0-11 \
--data-dir /data/etcd/etcd-server \
--listen-peer-urls https://20.0.0.11:2380 \
--listen-client-urls https://20.0.0.11:2379 ,http://127.0.0.1:2379 \
--quota-backend-bytes 8000000000 \
--initial-advertise-peer-urls https://20.0.0.11:2380 \
--advertise-client-urls https://20.0.0.11:2379 ,http://127.0.0.1:2379 \
--initial-cluster etcd-server-0-11= https://20.0.0.11:2380,etcd-server-0-12=https://20.0.0.12:2380 ,etcd-server-0-13=https://20.0.0.13:2380 \
--ca-file ./certs/ca.pem \
--cert-file ./certs/etcd-peer.pem \
--key-file ./certs/etcd-peer-key.pem \
--client-cert-auth \
--trusted-ca-file ./certs/ca.pem \
--peer-ca-file ./certs/ca.pem \
--peer-cert-file ./certs/etcd-peer.pem \
--peer-key-file ./certs/etcd-peer-key.pem \
--peer-client-cert-auth \
--peer-trusted-ca-file ./certs/ca.pem \
--log-output stdout
[root@server11 ~]#
[root@server12 ~]# cat /opt/apps/etcd/etcd-server-startup.sh
#!/bin/sh
WORK_DIR=$(dirname $(readlink -f $0))
[ $? -eq 0 ] && cd $WORK_DIR || exit
/opt/apps/etcd/etcd --name etcd-server-0-12 \
--data-dir /data/etcd/etcd-server \
--listen-peer-urls https://20.0.0.12:2380 \
--listen-client-urls https://20.0.0.12:2379 ,http://127.0.0.1:2379 \
--quota-backend-bytes 8000000000 \
--initial-advertise-peer-urls https://20.0.0.12:2380 \
--advertise-client-urls https://20.0.0.12:2379 ,http://127.0.0.1:2379 \
--initial-cluster etcd-server-0-11= https://20.0.0.11:2380,etcd-server-0-12=https://20.0.0.12:2380 ,etcd-server-0-13=https://20.0.0.13:2380 \
--ca-file ./certs/ca.pem \
--cert-file ./certs/etcd-peer.pem \
--key-file ./certs/etcd-peer-key.pem \
--client-cert-auth \
--trusted-ca-file ./certs/ca.pem \
--peer-ca-file ./certs/ca.pem \
--peer-cert-file ./certs/etcd-peer.pem \
--peer-key-file ./certs/etcd-peer-key.pem \
--peer-client-cert-auth \
--peer-trusted-ca-file ./certs/ca.pem \
--log-output stdout
[root@server12 ~]#
[root@server13 ~]# cat /opt/apps/etcd/etcd-server-startup.sh
#!/bin/sh
WORK_DIR=$(dirname $(readlink -f $0))
[ $? -eq 0 ] && cd $WORK_DIR || exit
/opt/apps/etcd/etcd --name etcd-server-0-13 \
--data-dir /data/etcd/etcd-server \
--listen-peer-urls https://20.0.0.13:2380 \
--listen-client-urls https://20.0.0.13:2379 ,http://127.0.0.1:2379 \
--quota-backend-bytes 8000000000 \
--initial-advertise-peer-urls https://20.0.0.13:2380 \
--advertise-client-urls https://20.0.0.13:2379 ,http://127.0.0.1:2379 \
--initial-cluster etcd-server-0-11= https://20.0.0.11:2380,etcd-server-0-12=https://20.0.0.12:2380 ,etcd-server-0-13=https://20.0.0.13:2380 \
--ca-file ./certs/ca.pem \
--cert-file ./certs/etcd-peer.pem \
--key-file ./certs/etcd-peer-key.pem \
--client-cert-auth \
--trusted-ca-file ./certs/ca.pem \
--peer-ca-file ./certs/ca.pem \
--peer-cert-file ./certs/etcd-peer.pem \
--peer-key-file ./certs/etcd-peer-key.pem \
--peer-client-cert-auth \
--peer-trusted-ca-file ./certs/ca.pem \
--log-output stdout
[root@server13 ~]#
3.5 赋权：
[root@server11 ~]# chmod u+x /opt/apps/etcd/etcd-server-startup.sh
[root@server11 ~]# touch /data/logs/etcd-server/etcd.stdout.log
[root@server11 ~]# chown -R etcd.etcd /opt/apps/etcd/ /data/etcd /data/logs/etcd-server
3.6 启动etcd
因为这些进程都是要启动为后台进程，要么手动启动，要么采用后台进程管理工具，推荐使用后台管理工具supervisor
[root@server11 ~]# yum install -y supervisor
[root@server11 ~]# systemctl start supervisord ; systemctl enable supervisord
3.7 supervisord配置文件如下
[root@server11 ~]# cat /etc/supervisord.d/etcd-server.ini
[program:etcd-server-0-11]
command=/opt/apps/etcd/etcd-server-startup.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/apps/etcd ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=30 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=etcd ; setuid to this UNIX account to run the program
redirect_stderr=true ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=5 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
[root@server11 ~]#
[root@server12 ~]# cat /etc/supervisord.d/etcd-server.ini
[program:etcd-server-0-12]
command=/opt/apps/etcd/etcd-server-startup.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/apps/etcd ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=30 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=etcd ; setuid to this UNIX account to run the program
redirect_stderr=true ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=5 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
[root@server12 ~]#
[root@server13 ~]# cat /etc/supervisord.d/etcd-server.ini
[program:etcd-server-0-13]
command=/opt/apps/etcd/etcd-server-startup.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/apps/etcd ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=30 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=etcd ; setuid to this UNIX account to run the program
redirect_stderr=true ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=5 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
[root@server13 ~]#
3.8 更新supervisord状态（启动模块）
[root@server11 ~]# supervisorctl update
etcd-server-0-11: added process group
[root@server11 ~]#
3.8 etcd 进程状态查看
[root@server11 ~]# supervisorctl status
etcd-server-0-11 RUNNING pid 8792, uptime 0:31:44
[root@server11 ~]#
[root@server11 ~]# netstat -lntp|grep etcd
tcp 0 0 20.0.0.11:2379 0.0.0.0:* LISTEN 8796/etcd
tcp 0 0 127.0.0.1:2379 0.0.0.0:* LISTEN 8796/etcd
tcp 0 0 20.0.0.11:2380 0.0.0.0:* LISTEN 8796/etcd
[root@server11 ~]#
3.9 查看集群：
[root@server11 ~]# /opt/apps/etcd/etcdctl member list
c1b99d661e10b568: name=etcd-server-0-11 peerURLs=https://20.0.0.11:2380 clientURLs= http://127.0.0.1:2379 ,https://20.0.0.11:2379 isLeader=true
f33c8e897853b7c4: name=etcd-server-0-12 peerURLs=https://20.0.0.12:2380 clientURLs= http://127.0.0.1:2379 ,https://20.0.0.12:2379 isLeader=false
f895995aa044cd94: name=etcd-server-0-13 peerURLs=https://20.0.0.13:2380 clientURLs= http://127.0.0.1:2379 ,https://20.0.0.13:2379 isLeader=false
[root@server11 ~]#
[root@server12 ~]# /opt/apps/etcd/etcdctl member list
c1b99d661e10b568: name=etcd-server-0-11 peerURLs=https://20.0.0.11:2380 clientURLs= http://127.0.0.1:2379 ,https://20.0.0.11:2379 isLeader=true
f33c8e897853b7c4: name=etcd-server-0-12 peerURLs=https://20.0.0.12:2380 clientURLs= http://127.0.0.1:2379 ,https://20.0.0.12:2379 isLeader=false
f895995aa044cd94: name=etcd-server-0-13 peerURLs=https://20.0.0.13:2380 clientURLs= http://127.0.0.1:2379 ,https://20.0.0.13:2379 isLeader=false
[root@server12 ~]#
[root@server13 ~]# /opt/apps/etcd/etcdctl member list
c1b99d661e10b568: name=etcd-server-0-11 peerURLs=https://20.0.0.11:2380 clientURLs= http://127.0.0.1:2379 ,https://20.0.0.11:2379 isLeader=true
f33c8e897853b7c4: name=etcd-server-0-12 peerURLs=https://20.0.0.12:2380 clientURLs= http://127.0.0.1:2379 ,https://20.0.0.12:2379 isLeader=false
f895995aa044cd94: name=etcd-server-0-13 peerURLs=https://20.0.0.13:2380 clientURLs= http://127.0.0.1:2379 ,https://20.0.0.13:2379 isLeader=false
[root@server13 ~]#
集群健康状态：
[root@server13 ~]# /opt/apps/etcd/etcdctl cluster-health
member c1b99d661e10b568 is healthy: got healthy result from http://127.0.0.1:2379
member f33c8e897853b7c4 is healthy: got healthy result from http://127.0.0.1:2379
member f895995aa044cd94 is healthy: got healthy result from http://127.0.0.1:2379
cluster is healthy
[root@server13 ~]#
随着etcd关闭，leader会变化 随着server11服务器关闭etcd，etcd的leader会变化：
关闭11服务器： （或者关闭etcd和super进程）
[root@server11 ~]# shutdown -h now
13服务器查看状态：13升级为leader
[root@server13 ~]# /opt/apps/etcd/etcdctl member list
c1b99d661e10b568: name=etcd-server-0-11 peerURLs=https://20.0.0.11:2380 clientURLs= http://127.0.0.1:2379 ,https://20.0.0.11:2379 isLeader=false
f33c8e897853b7c4: name=etcd-server-0-12 peerURLs=https://20.0.0.12:2380 clientURLs= http://127.0.0.1:2379 ,https://20.0.0.12:2379 isLeader=false
f895995aa044cd94: name=etcd-server-0-13 peerURLs=https://20.0.0.13:2380 clientURLs= http://127.0.0.1:2379 ,https://20.0.0.13:2379 isLeader=true
[root@server13 ~]#
浙公网安备 33010602011771号