k8s二进制安装03-部署etcd

下载etcd

## 创建保存配置的文件夹
mkdir -p /root/etcd/{bin,config,service,ssl,app}
cd /root/etcd

## 下载etcd二进制文件
## github二进制包下载地址：https://github.com/etcd-io/etcd/releases
wget https://github.com/etcd-io/etcd/releases/download/v3.5.2/etcd-v3.5.2-linux-amd64.tar.gz -O app/etcd-v3.5.2.tar.gz

tar -xf app/etcd-v3.5.2.tar.gz --strip-components=1 -C bin/ etcd-v3.5.2-linux-amd64/etcd{,ctl}

生成证书

生成脚本gen_etcd_cert.sh

cat <<'EOF' | sudo tee gen_etcd_cert.sh
#!/bin/bash
## example: ./etcd-cert.sh 127.0.0.1,master01,master02,master03,192.168.10.51,192.168.10.52,192.168.10.53

HOSTNAME=$1

## etcd ca的配置文件
cat > ca-config.json <<EOF1
{
  "signing": {
    "default": {
      "expiry": "876000h"
    },
    "profiles": {
      "peer": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "876000h"
      }
    }
  }
}
EOF1

## etcd的ca证书签名请求文件
cat > etcd-ca-csr.json <<EOF2
{
  "CN": "etcd",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "GuangZhou",
      "O": "etcd",
      "OU": "Etcd Security"
    }
  ],
  "ca": {
    "expiry": "876000h"
  }
}
EOF2

## 生成etcd集群使用的ca根证书
cfssl gencert \
	-initca etcd-ca-csr.json | cfssljson -bare ssl/etcd-ca

## 生成etcd集群使用的证书申请签名文件
cat > etcd-csr.json <<EOF3
{
  "CN": "etcd",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "GuangZhou",
      "O": "etcd",
      "OU": "Etcd Security"
    }
  ]
}
EOF3

## 给etcd集群颁发一个peer类型的证书，可以用于集群内节点双向认证
## 又可用于服务器或客户端的单向认证
cfssl gencert \
	-ca=ssl/etcd-ca.pem \
	-ca-key=ssl/etcd-ca-key.pem \
	-config=ca-config.json \
	-hostname=${HOSTNAME} \
	-profile=peer etcd-csr.json | cfssljson -bare ssl/etcd
EOF

执行

## example：bash gen_etcd_cert.sh <etcd相关主机ip及主机名>
bash -x gen_etcd_cert.sh 127.0.0.1,m01,m02,m03,192.168.1.51,192.168.1.52,192.168.1.53

## 在ssl目录下生成
├── etcd-ca.csr
├── etcd-ca-key.pem
├── etcd-ca.pem
├── etcd.csr
├── etcd-key.pem
├── etcd.pem

生成参数文件及启动service文件

生成脚本etcd_config.sh

cat <<'EOF' | sudo tee etcd_config.sh
#!/bin/bash
## example: ./etcd_config.sh master01 192.168.1.51 master02=https://192.168.1.52:2380,master03=https://192.168.1.53:2380

ETCD_NAME=$1
ETCD_IP=$2
ETCD_CLUSTER=$3

WORK_DIR=/opt/etcd
ETCD_CONF_DIR=/opt/etcd/config
ETCD_CA_CERT=etcd-ca.pem
ETCD_SERVER_CERT_PREFIX=etcd

cat > config/etcd.config.yaml.$1 <<EOF1
name: '${ETCD_NAME}'
data-dir: ${WORK_DIR}/data
wal-dir: ${WORK_DIR}/data/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://${ETCD_IP}:2380'
listen-client-urls: 'https://${ETCD_IP}:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://${ETCD_IP}:2380'
advertise-client-urls: 'https://${ETCD_IP}:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: '${ETCD_NAME}=https://${ETCD_IP}:2380,${ETCD_CLUSTER}'
initial-cluster-token: 'etcd-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '${WORK_DIR}/ssl/${ETCD_SERVER_CERT_PREFIX}.pem'
  key-file: '${WORK_DIR}/ssl/${ETCD_SERVER_CERT_PREFIX}-key.pem'
  client-cert-auth: true
  trusted-ca-file: '${WORK_DIR}/ssl/${ETCD_CA_CERT}'
  auto-tls: true
peer-transport-security:
  cert-file: '${WORK_DIR}/ssl/${ETCD_SERVER_CERT_PREFIX}.pem'
  key-file: '${WORK_DIR}/ssl/${ETCD_SERVER_CERT_PREFIX}-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '${WORK_DIR}/ssl/${ETCD_CA_CERT}'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF1

cat > service/etcd.service <<EOF2

[Unit]
Description=Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target

[Service]
Type=notify
ExecStart=/usr/local/bin/etcd \\
--config-file=${ETCD_CONF_DIR}/etcd.config.yaml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
Alias=etcd3.service
EOF2
EOF

执行

## 3台etcd服务器的文件配置不同，生成3个配置文件
## example：./etcd_config.sh <ETCD主机名> <ETCD_IP> <ETCD集群其他的信息>
bash etcd_config.sh m01 192.168.1.51 m02=https://192.168.1.52:2380,m03=https://192.168.1.53:2380

bash etcd_config.sh m02 192.168.1.52 m01=https://192.168.1.51:2380,m03=https://192.168.1.53:2380

bash etcd_config.sh m03 192.168.1.53 m01=https://192.168.1.51:2380,m02=https://192.168.1.52:2380

## 在config目录下生成
├── etcd.config.yaml.m01
├── etcd.config.yaml.m02
├── etcd.config.yaml.m03

## 在service目录下生成
├── etcd.service

分发etcd二进制文件、证书、配置及服务文件

for i in m01 m02 m03; do \
ssh $i "mkdir -p /opt/etcd/{config,data,ssl}"; \
scp bin/etcd* $i:/usr/local/bin; \
scp ssl/etcd{,-key,-ca}.pem $i:/opt/etcd/ssl/; \
scp config/etcd.config.yaml.$i $i:/opt/etcd/config/etcd.config.yaml; \
scp service/etcd.service $i:/usr/lib/systemd/system/; \
done

启动etcd服务

for i in m01 m02 m03; do \
ssh $i "systemctl daemon-reload"; \
ssh $i "systemctl enable etcd"; \
ssh $i "systemctl restart etcd --no-block"; \
ssh $i "systemctl is-active etcd"; \
done

验证集群

## 查看集群
$ export ETCDCTL_API=3
etcdctl \
    --endpoints="192.168.1.51:2379,192.168.1.52:2379,192.168.1.53:2379" \
    --cacert=/opt/etcd/ssl/etcd-ca.pem \
    --cert=/opt/etcd/ssl/etcd.pem \
    --key=/opt/etcd/ssl/etcd-key.pem endpoint status \
    --write-out=table
+------------------+---------+------+---------------------------+---------------------------+------------+
|        ID        | STATUS  | NAME |        PEER ADDRS         |       CLIENT ADDRS        | IS LEARNER |
+------------------+---------+------+---------------------------+---------------------------+------------+
| 238b72cdd26e304f | started |  m02 | https://192.168.1.52:2380 | https://192.168.1.52:2379 |      false |
| 8034142cf01c5d1c | started |  m03 | https://192.168.1.53:2380 | https://192.168.1.53:2379 |      false |
| 8da171dbef9ded69 | started |  m01 | https://192.168.1.51:2380 | https://192.168.1.51:2379 |      false |
+------------------+---------+------+---------------------------+---------------------------+------------+

etcdctl \
    --endpoints="192.168.1.51:2379,192.168.1.52:2379,192.168.1.53:2379" \
    --cacert=/opt/etcd/ssl/etcd-ca.pem \
    --cert=/opt/etcd/ssl/etcd.pem \
    --key=/opt/etcd/ssl/etcd-key.pem member list \
    --write-out=table  
+-------------------+--------+--------------+---------------------------+
|     ENDPOINT      | HEALTH |     TOOK     |           ERROR           |
+-------------------+--------+--------------+---------------------------+
| 192.168.1.51:2379 |   true |  28.668399ms |                           |
| 192.168.1.53:2379 |   true |  29.078085ms |                           |
| 192.168.0.52:2379 |  false | 5.003967604s | context deadline exceeded |
+-------------------+--------+--------------+---------------------------+

etcdctl \
    --endpoints="192.168.1.51:2379,192.168.1.52:2379,192.168.1.53:2379" \
    --cacert=/opt/etcd/ssl/etcd-ca.pem \
    --cert=/opt/etcd/ssl/etcd.pem \
    --key=/opt/etcd/ssl/etcd-key.pem endpoint health \
    --write-out=table
+-------------------+--------+-------------+-------+
|     ENDPOINT      | HEALTH |    TOOK     | ERROR |
+-------------------+--------+-------------+-------+
| 192.168.1.51:2379 |   true | 30.342531ms |       |
| 192.168.1.52:2379 |   true | 31.598332ms |       |
| 192.168.1.53:2379 |   true | 40.204582ms |       |
+-------------------+--------+-------------+-------+