velero结合minio实现kubernetes业务数据备份与恢复

部署一个 minio

docker run --name minio \
  -p 9000:9000 \
  -p 9999:9999 \
  -d --restart=always \
  -e "MINIO_ROOT_USER=admin" \
  -e "MINIO_ROOT_PASSWORD=12345678" \
  -v /data/minio/data:/data \
  minio/minio:RELEASE.2025-04-08T15-41-24Z server /data \
  --console-address '0.0.0.0:9999'

创建一个 buckets

下载 velero

wget https://github.com/vmware-tanzu/velero/releases/download/v1.16.0/velero-v1.16.0-linux-amd64.tar.gz

root@10:~# mkdir /data/velero -p
root@10:~# cd /data/velero/
root@10:/data/velero# 

root@10:/data/velero# vim velero-auth.txt 

[default]
aws_access_key_id = admin
aws_secret_access_key = 12345678

root@10:/data/velero# vim cutui-csr.json
{
  "CN": "cutui",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

root@10:/data/velero# apt install golang-cfssl -y

准备证书签发环境

wget https://github.com/cloudflare/cfssl/releases/download/v1.6.5/cfssl_1.6.5_linux_amd64
wget https://github.com/cloudflare/cfssl/releases/download/v1.6.5/cfssljson_1.6.5_linux_amd64
wget https://github.com/cloudflare/cfssl/releases/download/v1.6.5/cfssl-certinfo_1.6.5_linux_amd64

root@10:/data/velero# mv cfssl_1.6.5_darwin_amd64 cfssl
root@10:/data/velero# mv cfssljson_1.6.5_linux_amd64 cfssljson
root@10:/data/velero# mv cfssl-certinfo_1.6.5_linux_amd64 cfssl-certinfo
root@10:/data/velero# mv cfssl* /usr/local/bin/
root@10:/data/velero# chmod a+x /usr/local/bin/cfssl*

执行证书签发

root@10:/data/velero# cp /etc/kubeasz/clusters/k8s-cluster-yzy/ssl/ca-config.json /data/velero/

root@10:/data/velero# /usr/local/bin/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem -ca-key=/etc/kubernetes/ssl/ca-key.pem -config=/etc/kubeasz/clusters/k8s-cluster-yzy/ssl/ca-config.json -profile=kubernetes ./cutui-csr.json | cfssljson -bare cutui
2025/04/18 07:43:51 [INFO] generate received request
2025/04/18 07:43:51 [INFO] received CSR
2025/04/18 07:43:51 [INFO] generating key: rsa-2048
2025/04/18 07:43:51 [INFO] encoded CSR
2025/04/18 07:43:51 [INFO] signed certificate with serial number 693071236866306299359403150075131468643342340230
2025/04/18 07:43:51 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
root@10:/data/velero# ll
total 32
drwxr-xr-x 2 root root 4096 Apr 18 07:43 ./
drwxr-xr-x 3 root root 4096 Apr 18 06:42 ../
-rw-r--r-- 1 root root  459 Apr 18 07:29 ca-config.json
-rw-r--r-- 1 root root  993 Apr 18 07:43 cutui.csr
-rw-r--r-- 1 root root  218 Apr 18 07:43 cutui-csr.json
-rw------- 1 root root 1679 Apr 18 07:43 cutui-key.pem
-rw-r--r-- 1 root root 1387 Apr 18 07:43 cutui.pem
-rw-r--r-- 1 root root   69 Apr 18 06:44 velero-auth.txt

 分发证书到api-server证书路径

root@10:/data/velero# cp cutui-key.pem /etc/kubernetes/ssl/
root@10:/data/velero# cp cutui.pem /etc/kubernetes/ssl/

生成集群认证config文件

export KUBE_APISERVER="https://10.211.55.85:6443"
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=./cutui.kubeconfig

设置客户端证书认证

kubectl config set-credentials cutui \
--client-certificate=/etc/kubernetes/ssl/cutui.pem \
--client-key=/etc/kubernetes/ssl/cutui-key.pem \
--embed-certs=true \
--kubeconfig=./cutui.kubeconfig

 设置上下文参数

kubectl config set-context kubernetes \
--cluster=kubernetes \
--user=cutui \
--namespace=velero-system \
--kubeconfig=./cutui.kubeconfig

设置默认上下文

kubectl config use-context kubernetes --kubeconfig=cutui.kubeconfig

k8s集群中创建cutui账户

kubectl create clusterrolebinding cutui --clusterrole=cluster-admin --user=cutui

创建namespace

kubectl create ns velero-system

执行安装

velero --kubeconfig  ./cutui.kubeconfig \
	install \
    --provider aws \
    --plugins velero/velero-plugin-for-aws:v1.3.1 \
    --bucket velerodata  \
    --secret-file ./velero-auth.txt \
    --use-volume-snapshots=false \
	--namespace velero-system \
--backup-location-config region=minio,s3ForcePathStyle="true",s3Url=http://10.211.55.90:9000

查看安装状态

kubectl logs deployment/velero -n velero-system

root@10:/data/velero# kubectl get pods -n velero-system
NAME                      READY   STATUS    RESTARTS   AGE
velero-86ddd644f7-jwwv6   1/1     Running   0          23m

备份数据

DATE=`date +%Y%m%d%H%M%S`
velero backup create myserver-ns-backup-${DATE} \
--include-cluster-resources=true \
--include-namespaces myserver \
--kubeconfig=./cutui.kubeconfig \
--namespace velero-system

在 minio 上查看

可以查看备份的内容

root@10:/data/velero# velero backup describe  myserver-ns-backup-20250418082615 \
--kubeconfig=./cutui.kubeconfig \
--namespace velero-system

恢复备份

velero restore create --from-backup myserver-ns-backup-20250418082615 --wait \
--kubeconfig=./cutui.kubeconfig \
--namespace velero-system

备份脚本

#!/bin/bash
NS_NAME=$(kubectl get ns | awk '{if (NR>2) print}' | awk '{print $1}')
DATE=$(date +%Y%m%d%H%M%S)
cd /data/velero/

for i in $NS_NAME; do
  velero backup create ${i}-ns-backup-${DATE} \
    --include-cluster-resources=true \
    --include-namespaces ${i} \
    --kubeconfig=/root/.kube/config \
    --namespace velero-system
done