Dict.CN 在线词典, 英语学习, 在线翻译 ------------- MyGitee My腾云code My51cto

Happy_EveryDay

可以平凡 不可以平庸 无爱则无忧，无欲则无求，无怒而无敌，无怨才是佛。所有烦恼，都是放不下的执著 开源技群 328035181 MyGitee
博客园 首页 新随笔 管理
【k8s】 etcd服务端及客户端版本及快照【定时】备份 20260730

1、master节点-etcd服务端版本

####先获取 etcd 静态 Pod 名称

 

kubectl get pods -n kube-system | grep etcd

####查看集群 etcd 版本

kubectl exec -n kube-system etcd-k8s-master -- etcd --version

image

 服务端版本:etcd Version: 3.5.9

image

 

2、master节点-etcd客户端版本

客户端v3.5.31

 

wget https://github.com/etcd-io/etcd/releases/download/v3.5.33/etcd-v3.5.33-linux-amd64.tar.gz
tar -zxvf etcd-v3.5.33-linux-amd64.tar.gz
cd etcd-v3.5.33-linux-amd64
cp etcdctl etcdutl /usr/local/bin/

 

 

image

 

# 验证客户端版本

etcdctl version

image

 etcdutl version

 

image

 

 

3、master节点-etcd健康检测命令

ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key

endpoint health

 

image

https://127.0.0.1:2379 is healthy: successfully committed proposal: took = 26.893613ms 

####etcd 连接正常、集群健康探测成功！

 

 

 

4、etcd 快照备份（生产核心命令）

 

4.1 master 节点提前创建备份目录

mkdir -p /data/etcd-backup

 

 

 

4.2 备份完成立刻查看快照信息

ETCDCTL_API=3 etcdctl \
--endpoints=https://127.0.0.1:2379 \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key \
snapshot save /data/etcd-backup/etcd-$(date +%Y%m%d-%H%M).snap \
&& etcdutl snapshot status /data/etcd-backup/etcd-$(date +%Y%m%d-%H%M).snap

 

image

 

 

image

 

 

4.3

#集群成员列表

ETCDCTL_API=3 etcdctl \
--endpoints=https://127.0.0.1:2379 \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key \
member list

image

 

image

 

#集群版本

ETCDCTL_API=3 etcdctl \
--endpoints=https://127.0.0.1:2379 \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key \
version

image

 #集群领导者

ETCDCTL_API=3 etcdctl \
--endpoints=https://127.0.0.1:2379 \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key \
endpoint status

image

 

####可以把证书参数写入环境变量，后续命令不用重复携带证书

export ETCDCTL_API=3
export ETCDCTL_ENDPOINTS=https://127.0.0.1:2379
export ETCDCTL_CACERT=/etc/kubernetes/pki/etcd/ca.crt
export ETCDCTL_CERT=/etc/kubernetes/pki/etcd/server.crt
export ETCDCTL_KEY=/etc/kubernetes/pki/etcd/server.key

 

####简写

##健康检测命令

etcdctl endpoint health 

##备份快照命令
etcdctl snapshot save test.snap

image

####集群列表

etcdctl member list 

image

 

####单独查看快照命令（手动执行）

etcdutl snapshot status /data/etcd-backup/test.snap

image

 

image

 

# 查看集群领导者、版本、数据大小

etcdctl endpoint status

image

 

 

 

4.4 备份完成立刻查看快照信息脚本

 

#创建脚本
vi /data/etcd-backup/etcd-backup.sh
#赋予执行权限
chmod +x /data/etcd-backup/etcd-backup.sh
##手动立即执行测试

###手动临时清理环境变量再运行

unset ETCDCTL_API ETCDCTL_ENDPOINTS ETCDCTL_CACERT ETCDCTL_CERT ETCDCTL_KEY
./etcd-backup.sh

image

 

./etcd-backup.sh或/data/etcd-backup/etcd-backup.sh

image

 

 

4.5定时快照备份

###打开定时任务
crontab -e

image

 ###crontab 写法（最简）

###计划每 2 分钟

*/2 * * * * /bin/bash /data/etcd-backup/etcd-backup.sh

###凌晨 1 点执行一次

0 1 * * 0 /bin/bash /data/etcd-backup/etcd-backup.sh

image

 

 

 

 

# 查看cron系统日志（CentOS/RHEL）
tail -f /var/log/cron

image

 ###计划每 2 分钟 或凌晨 1 点执行一次生成备份记录

 

image

 

 

 

 

 

 

4.6 etcd-backup.sh脚本内容

 

#!/bin/bash

　#环境变量与日志输出
   export PATH=/usr/local/bin:/usr/bin:/bin:/sbin:/usr/sbin
   LOG_FILE="/data/etcd-backup/backup.log"
   exec >> ${LOG_FILE} 2>&1
   set -x


# 清除etcdctl环境变量冲突
unset ETCDCTL_API
unset ETCDCTL_ENDPOINTS
unset ETCDCTL_CACERT
unset ETCDCTL_CERT
unset ETCDCTL_KEY

SNAP=/data/etcd-backup/etcd-$(date +%Y%m%d-%H%M).snap

ETCDCTL_API=3 etcdctl \
--endpoints=https://127.0.0.1:2379 \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key \
snapshot save "${SNAP}"

echo "====快照信息===="
if [ -f "${SNAP}" ];then
etcdutl snapshot status "${SNAP}"
else
echo "❌快照文件生成失败！"
fi

echo "====删除7天前快照===="
# 算出7天前日期 YYYYMMDD
DEL_DATE=$(date -d "7 days ago" +%Y%m%d)
find /data/etcd-backup -name "etcd-*.snap" | while read file;do
# 提取文件名中的8位日期
FILE_DATE=$(echo "$file" | grep -oE "etcd-[0-9]{8}" | cut -d'-' -f2)
if [[ "$FILE_DATE" < "$DEL_DATE" ]];then
echo "删除旧备份：$file"
rm -f "$file"
fi
done

image

 

 

 

 

 

 

 5、etcd 快照恢复（生产核心命令）

 

5.1

# etcd 快照恢复脚本
# 适用：单 master + 多 worker 集群，etcd 以静态 Pod 运行
# 用法：./etcd_restore.sh [快照文件名]
# 示例：./etcd_restore.sh etcd-20260813-0100

image

 

5.2 验证

sleep 60

# 检查 etcd 容器是否已运行
crictl ps | grep etcd

# 再检查健康状态
ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key \
endpoint health

 

 

image

 

 

5.3 etcd-restore.sh脚本内容

 

 

#!/bin/bash
# ============================================================
# etcd 快照恢复脚本
# 适用：单 master + 多 worker 集群，etcd 以静态 Pod 运行
# 用法：./etcd_restore.sh [快照文件名]
# 示例：./etcd_restore.sh etcd-20260813-0100
# ============================================================

set -e

# 解析参数
SNAP_ARG="${1:-etcd-20260813-0100}"

# 自动补全 .snap 后缀
if [[ "$SNAP_ARG" == *.snap ]]; then
SNAP_FILE="/data/etcd-backup/$SNAP_ARG"
else
SNAP_FILE="/data/etcd-backup/${SNAP_ARG}.snap"
fi

ETCD_NAME="k8s-master"
ETCD_IP="192.168.91.166"
ETCD_DATA_DIR="/var/lib/etcd"
ETCD_MANIFEST="/etc/kubernetes/manifests/etcd.yaml"
ETCD_MANIFEST_BAK="/etc/kubernetes/manifests.bak"

# 颜色输出
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() {
echo -e "${GREEN}[INFO]${NC} $1"
}

log_warn() {
echo -e "${YELLOW}[WARN]${NC} $1"
}

log_error() {
echo -e "${RED}[ERROR]${NC} $1"
}

log_step() {
echo -e "${BLUE}[STEP]${NC} $1"
}

# 等待 etcd 容器启动的函数
wait_for_etcd_container() {
local max_wait=120
local waited=0
log_info "等待 etcd 容器启动（最多 ${max_wait} 秒）..."
while [ $waited -lt $max_wait ]; do
if crictl ps 2>/dev/null | grep -q etcd || docker ps 2>/dev/null | grep -q etcd; then
log_info "etcd 容器已启动"
return 0
fi
sleep 5
waited=$((waited + 5))
echo -n "."
done
echo ""
log_error "etcd 容器在 ${max_wait} 秒内未启动"
return 1
}

# 等待 etcd 健康的函数
wait_for_etcd_health() {
local max_wait=120
local waited=0
log_info "等待 etcd 健康（最多 ${max_wait} 秒）..."
while [ $waited -lt $max_wait ]; do
if ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key \
endpoint health >/dev/null 2>&1; then
log_info "etcd 健康检查通过"
return 0
fi
sleep 5
waited=$((waited + 5))
echo -n "."
done
echo ""
log_error "etcd 在 ${max_wait} 秒内未通过健康检查"
return 1
}

# 1. 前置检查
log_step "步骤 1/11：前置检查"
log_info "目标快照文件：$SNAP_FILE"

if [ ! -f "$SNAP_FILE" ]; then
log_error "快照文件不存在：$SNAP_FILE"
log_info "可用快照列表："
ls -lh /data/etcd-backup/*.snap 2>/dev/null || echo " (无)"
exit 1
fi

if [ ! -f "$ETCD_MANIFEST" ]; then
log_error "etcd manifest 不存在：$ETCD_MANIFEST"
exit 1
fi

log_info "快照文件确认存在"
log_info "etcd 节点名：$ETCD_NAME"
log_info "etcd IP：$ETCD_IP"

# 2. 停止 kubelet
log_step "步骤 2/11：停止 kubelet"
systemctl stop kubelet
if systemctl is-active --quiet kubelet; then
log_error "kubelet 停止失败"
exit 1
fi
log_info "kubelet 已停止"

# 3. 移走 etcd manifest
log_step "步骤 3/11：移走 etcd manifest"
mkdir -p "$ETCD_MANIFEST_BAK"
mv "$ETCD_MANIFEST" "$ETCD_MANIFEST_BAK/"

# 4. 强制停止 etcd 容器
log_step "步骤 4/11：强制停止 etcd 容器"
ETCD_CID=$(crictl ps -q --name etcd 2>/dev/null || true)
if [ -n "$ETCD_CID" ]; then
crictl stop $ETCD_CID 2>/dev/null || true
crictl rm $ETCD_CID 2>/dev/null || true
log_info "已停止并删除 crictl etcd 容器"
else
ETCD_CID=$(docker ps -q --filter name=etcd 2>/dev/null || true)
if [ -n "$ETCD_CID" ]; then
docker stop $ETCD_CID 2>/dev/null || true
docker rm $ETCD_CID 2>/dev/null || true
log_info "已停止并删除 docker etcd 容器"
fi
fi

# 5. 等待 etcd 完全停止
log_step "步骤 5/11：等待 etcd 完全停止"
sleep 15

# 6. 确认 etcd 已停止
log_step "步骤 6/11：确认 etcd 已停止"
if crictl ps 2>/dev/null | grep -q etcd; then
log_error "etcd 容器仍在运行（crictl），请手动检查"
exit 1
fi

if docker ps 2>/dev/null | grep -q etcd; then
log_error "etcd 容器仍在运行（docker），请手动检查"
exit 1
fi

log_info "etcd 已完全停止"

# 7. 备份当前 etcd 数据
log_step "步骤 7/11：备份当前 etcd 数据"
BACKUP_DIR="${ETCD_DATA_DIR}.bak.$(date +%Y%m%d%H%M%S)"
if [ -d "$ETCD_DATA_DIR" ]; then
mv "$ETCD_DATA_DIR" "$BACKUP_DIR"
log_info "旧数据已备份到：$BACKUP_DIR"
else
log_warn "原数据目录不存在，跳过备份"
fi

# 8. 创建新的空数据目录
log_step "步骤 8/11：创建新的空数据目录"
mkdir -p "$ETCD_DATA_DIR"

# 9. 恢复快照
log_step "步骤 9/11：恢复快照"
cd "$(dirname "$SNAP_FILE")"

ETCDCTL_API=3 etcdctl snapshot restore "$SNAP_FILE" \
--data-dir="$ETCD_DATA_DIR" \
--name="$ETCD_NAME" \
--initial-cluster="${ETCD_NAME}=https://${ETCD_IP}:2380" \
--initial-cluster-token=etcd-cluster-0 \
--initial-advertise-peer-urls="https://${ETCD_IP}:2380"

if [ $? -ne 0 ]; then
log_error "快照恢复失败"
exit 1
fi
log_info "快照恢复成功"

# 10. 恢复 etcd manifest 并启动 kubelet
log_step "步骤 10/11：恢复 etcd manifest 并启动 kubelet"
mv "$ETCD_MANIFEST_BAK/etcd.yaml" "$ETCD_MANIFEST"
systemctl start kubelet

# 11. 验证
log_step "步骤 11/11：验证 etcd 和 K8s 状态"

# 11.1 等待 etcd 容器启动
if ! wait_for_etcd_container; then
log_error "etcd 容器启动失败"
log_info "排查建议："
log_info " 1. 查看 kubelet 状态：systemctl status kubelet"
log_info " 2. 查看 kubelet 日志：journalctl -u kubelet -n 50"
exit 1
fi

# 11.2 等待 etcd 健康
if ! wait_for_etcd_health; then
log_error "etcd 健康检查未通过"
log_info "排查建议："
log_info " 1. 查看 etcd 日志："
log_info " crictl logs \$(crictl ps -q --name etcd) --tail=50"
log_info " 2. 检查 etcd 数据目录权限：ls -la /var/lib/etcd"
log_info " 3. 检查 etcd 容器挂载：crictl inspect \$(crictl ps -q --name etcd)"
exit 1
fi

# 自定义等待核验部分
log_warn "额外固定等待90秒，进行二次静态核验"
sleep 120

log_info "【二次核验】查看etcd容器运行列表："
crictl ps | grep etcd

echo ""
log_info "【二次核验】执行etcd端点健康检测："
ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key \
endpoint health

# 11.3 检查 K8s 节点
log_info "检查 K8s 节点状态..."
kubectl get nodes

# 11.4 检查 kube-system Pod
log_info "检查 kube-system Pod..."
kubectl get pods -n kube-system

# 11.5 检查 KubeSphere Pod
log_info "检查 KubeSphere Pod..."
kubectl get pods -n kubesphere-system

log_info "========================================"
log_info "etcd 恢复流程已完成"
log_info "如有问题，旧数据备份在：$BACKUP_DIR"
log_info "========================================"

 

posted on 2026-07-30 21:03  2026cn  阅读(12)  评论(0)    收藏  举报