Velero结合minio实现kubernetes业务数据备份与恢复【一】

 

 

 

 

 

 部署环境

192.168.40.101(k8s-master01)	velero server
192.168.40.109:9999/9000(k8s-harbor)	minio

一、配置对象存储

minio

#tar xvf docker-20.10.17-binary-install.tar.gz
#bash docker-install.sh
#docker pull minio/minio:RELEASE.2022-04-12T06-55-35Z
#mkdir -p /data/minio

#启动命令 运行容器并挂在
docker run --name minio \
-p 9000:9000 \
-p 9999:9999 \
-d --restart=always \
-e "MINIO_ROOT_USER=admin" \
-e "MINIO_ROOT_PASSWORD=12345678" \
-v /data/minio/data:/data \
minio/minio:RELEASE.2022-04-12T06-55-35Z server /data \
--console-address '0.0.0.0:9999'

#浏览器运行  登陆账户密码：admin\12345678
#http://xxx.xxx.xxx.xxx:9999/login
http://192.168.40.109:9999/login

#Create a Bucket 创建 velerodataxks bucket 后续需要配置此参数：velerodataxks

二、部署Velero

#k8s-master01 之前kubeasz部署的 master01上操作
# 1.下载软件包 velero-v1.8.1-linux-amd64.tar.gz
[root@k8s-master01 src]# cd /usr/local/src/
[root@k8s-master01 src]# wget https://github.com/vmware-tanzu/velero/releases/download/v1.8.1/velero-v1.8.1-linux-amd64.tar.gz
[root@k8s-master01 src]# ll
-rw-r--r-- 1 root root 29570064 Nov 26 22:40 velero-v1.8.1-linux-amd64.tar.gz
[root@k8s-master01 src]# tar xvf velero-v1.8.1-linux-amd64.tar.gz
velero-v1.8.1-linux-amd64/LICENSE
velero-v1.8.1-linux-amd64/examples/README.md
velero-v1.8.1-linux-amd64/examples/minio
velero-v1.8.1-linux-amd64/examples/minio/00-minio-deployment.yaml
velero-v1.8.1-linux-amd64/examples/nginx-app
velero-v1.8.1-linux-amd64/examples/nginx-app/README.md
velero-v1.8.1-linux-amd64/examples/nginx-app/base.yaml
velero-v1.8.1-linux-amd64/examples/nginx-app/with-pv.yaml
velero-v1.8.1-linux-amd64/velero
[root@k8s-master01 src]# cp velero-v1.8.1-linux-amd64/velero  /usr/local/bin/
[root@k8s-master01 src]# velero --help

# 2.配置velero认证环境和授权文件
# 工作目录：
[root@k8s-master01 src]# mkdir  /data/velero -p
[root@k8s-master01 src]# cd /data/velero/
[root@k8s-master01 velero]# vim velero-auth.txt
[default]
aws_access_key_id = admin
aws_secret_access_key = 12345678

# 3.配置认证文件
[root@k8s-master01 velero]# vim awsuser-csr.json
{
  "CN": "awsuser",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

# 4.准备证书签发环境
# 下载软件
https://github.com/cloudflare/cfssl/releases/download/v1.6.1/cfssl_1.6.1_linux_amd64 
https://github.com/cloudflare/cfssl/releases/download/v1.6.1/cfssljson_1.6.1_linux_amd64 
https://github.com/cloudflare/cfssl/releases/download/v1.6.1/cfssl-certinfo_1.6.1_linux_amd64

cd /usr/local/src
mv cfssl-certinfo_1.6.1_linux_amd64 cfssl-certinfo
mv cfssl_1.6.1_linux_amd64 cfssl
mv cfssljson_1.6.1_linux_amd64 cfssljson

[root@k8s-master01 src]# cp cfssl-certinfo cfssl cfssljson /usr/local/bin/
[root@k8s-master01 src]# chmod  a+x /usr/local/bin/cfssl*
    
# 5.执行证书签发：
[root@k8s-master01 velero]# cd /data/velero/
# 确认是否有4个文件
[root@k8s-master01 src]# ll /etc/kubernetes/ssl/ca.pem
-rw-r--r-- 1 root root 1302 Nov 15 11:27 /etc/kubernetes/ssl/ca.pem
[root@k8s-master01 src]# ll /etc/kubernetes/ssl/ca-key.pem
-rw-r--r-- 1 root root 1679 Nov 15 11:27 /etc/kubernetes/ssl/ca-key.pem
[root@k8s-master01 velero]# ll ./awsuser-csr.json
-rw-r--r-- 1 root root 220 Nov 26 22:42 ./awsuser-csr.json
#在kubeasz 部署的 depoy节点 复制文件到服务器
[root@k8s-deploy ssl]# pwd
/etc/kubeasz/clusters/k8s-cluster-kubeasz/ssl
[root@k8s-deploy ssl]# scp /etc/kubeasz/clusters/k8s-cluster-kubeasz/ssl/ca-config.json 192.168.40.101:/data/velero

[root@k8s-master01 velero]# cd /data/velero
[root@k8s-master01 velero]# /usr/local/bin/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem -ca-key=/etc/kubernetes/ssl/ca-key.pem -config=./ca-config.json -profile=kubernetes ./awsuser-csr.json | cfssljson -bare awsuser
config=./ca-config.json -profile=kubernetes ./awsuser-csr.json | cfssljson -bare awsuser
2023/11/26 23:04:45 [INFO] generate received request
2023/11/26 23:04:45 [INFO] received CSR
2023/11/26 23:04:45 [INFO] generating key: rsa-2048
2023/11/26 23:04:45 [INFO] encoded CSR
2023/11/26 23:04:45 [INFO] signed certificate with serial number 335609343458255531945847074963848268577076484427
2023/11/26 23:04:45 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

# 6.验证证书
[root@k8s-master01 velero]# ll awsuser*
-rw-r--r-- 1 root root  997 Nov 26 23:04 awsuser.csr
-rw-r--r-- 1 root root  220 Nov 26 22:42 awsuser-csr.json
-rw------- 1 root root 1679 Nov 26 23:04 awsuser-key.pem
-rw-r--r-- 1 root root 1387 Nov 26 23:04 awsuser.pem

# 7.分发证书到api-server证书路径：
[root@k8s-master01 velero]# cp awsuser-key.pem /etc/kubernetes/ssl/
[root@k8s-master01 velero]# cp awsuser.pem /etc/kubernetes/ssl/

# 8.配置k8s所需要的各个配置参数
# 8.1生成集群认证config文件：
[root@k8s-master01 velero]# export KUBE_APISERVER="https://192.168.40.101:6443"
[root@k8s-master01 velero]# 
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=./awsuser.kubeconfig
Cluster "kubernetes" set.

# 8.2设置客户端证书认证:
kubectl config set-credentials awsuser \
--client-certificate=/etc/kubernetes/ssl/awsuser.pem \
--client-key=/etc/kubernetes/ssl/awsuser-key.pem \
--embed-certs=true \
--kubeconfig=./awsuser.kubeconfig

#8.3设置上下文参数:
kubectl config set-context kubernetes \
--cluster=kubernetes \
--user=awsuser \
--namespace=velero-system \
--kubeconfig=./awsuser.kubeconfig

# 8.4设置默认上下文:
kubectl config use-context kubernetes --kubeconfig=awsuser.kubeconfig

# 8.5k8s集群中创建awsuser账户：
kubectl create clusterrolebinding awsuser --clusterrole=cluster-admin --user=awsuser

# 8.6创建namespace：
kubectl create ns velero-system

# 9.执行安装：要稍等一会 因为要下载镜像 
#Velero is installed! ⛵ Use 'kubectl logs deployment/velero -n velero-system' to view the status.
velero --kubeconfig  ./awsuser.kubeconfig \
	install \
    --provider aws \
    --plugins velero/velero-plugin-for-aws:v1.3.1 \
    --bucket velerodataxks  \
    --secret-file ./velero-auth.txt \
    --use-volume-snapshots=false \
	--namespace velero-system \
--backup-location-config region=minio,s3ForcePathStyle="true",s3Url=http://192.168.40.109:9000

# 10.验证安装
[root@k8s-master01 velero]# kubectl get pods -n velero-system
NAME                      READY   STATUS    RESTARTS   AGE
velero-858b9459f9-ff5tt   1/1     Running   0          4m36s
# 查看日志 回显说明正常
[root@k8s-master01 velero]# kubectl logs deployment/velero -n velero-system
time="2023-11-26T15:21:48Z" level=info msg="Backup storage location valid, marking as available" backup-storage-location=default controller=backup-storage-location logSource="pkg/controller/backup_storage_location_controller.go:121"

# 11.执行备份
# 进行对default备份
[root@k8s-master01 velero]# kubectl get pod -n default
NAME           READY   STATUS    RESTARTS   AGE
net-tesing-2   1/1     Running   0          11d
net-testing    1/1     Running   0          11d

[root@k8s-master01 velero]# DATE=`date +%Y%m%d%H%M%S`
[root@k8s-master01 velero]# 
velero backup create default-ns-backup-${DATE} \
--include-namespaces default \
--kubeconfig=./awsuser.kubeconfig \
--namespace velero-system
Backup request "default-ns-backup-20231126232740" submitted successfully.
Run `velero backup describe default-ns-backup-20231126232740` or `velero backup logs default-ns-backup-20231126232740` for more details.

# 12.minio验证备份数据
[root@k8s-master01 velero]# velero backup describe default-ns-backup-20231126232740 -n velero-system
[root@k8s-master01 velero]# velero backup logs default-ns-backup-20231126232740 -n velero-system
#查看管理界面是否有备份数据
velerodataxks / backups / default-ns-backup-20231126232740

# 13.删除pod并验证数据恢复
[root@k8s-master01 velero]# kubectl get pods -n default
NAME           READY   STATUS    RESTARTS   AGE
net-tesing-2   1/1     Running   0          11d
net-testing    1/1     Running   0          11d
[root@k8s-master01 velero]# kubectl delete pod net-testing
pod "net-testing" deleted
[root@k8s-master01 velero]# kubectl get pods -n default
NAME           READY   STATUS    RESTARTS   AGE
net-tesing-2   1/1     Running   0          11d

[root@k8s-master01 velero]# velero restore create --from-backup default-ns-backup-20231126232740 --wait --kubeconfig=./awsuser.kubeconfig --namespace velero-system
Restore request "default-ns-backup-20231126232740-20231126233556" submitted successfully.
Waiting for restore to complete. You may safely press ctrl-c to stop waiting - your restore will continue in the background.

Restore completed with status: Completed. You may check for more information using the commands `velero restore describe default-ns-backup-20231126232740-20231126233556` and `velero restore logs default-ns-backup-20231126232740-20231126233556`.

[root@k8s-master01 velero]# kubectl get pod -n default
NAME           READY   STATUS    RESTARTS   AGE
net-tesing-2   1/1     Running   0          11d
net-testing    1/1     Running   0          35s

# 14.还可以对其他namespace进行备份


# 15.备份指定资源对象
# 备份指定namespace中的pod或者特定资源
velero backup create pod-backup-${DATE} \
--include-cluster-resources=true \
--ordered-resources 'pods=myserver/xxxxx,default/xxxxx'
--include-namespaces=myserver,default \
--kubeconfig=./awsuser.kubeconfig \
--namespace velero-system

# 16.批量备份所有namespace 最好每天低峰期进行备份
[root@xksmaster1 velero]# vim ns-backup.sh
#!/bin/bash
NS_NAME=`kubectl get ns | awk '{if(NR>1){print}}' | awk '{print $1}'`
DATE=`date +%Y%m%d%H%M%S`
cd /data/velero/

for i in $NS_NAME;do
velero backup create ${i}-ns-backup-${DATE} \
--include-cluster-resources=true \
--include-namespaces ${i} \
--kubeconfig=/root/.kube/config \
--namespace velero-system
done

[root@k8s-master01 velero]# bash ns-backup.sh
Backup request "default-ns-backup-20231126234338" submitted successfully.
Run `velero backup describe default-ns-backup-20231126234338` or `velero backup logs default-ns-backup-20231126234338` for more details.
Backup request "kube-node-lease-ns-backup-20231126234338" submitted successfully.
Run `velero backup describe kube-node-lease-ns-backup-20231126234338` or `velero backup logs kube-node-lease-ns-backup-20231126234338` for more details.
Backup request "kube-public-ns-backup-20231126234338" submitted successfully.
Run `velero backup describe kube-public-ns-backup-20231126234338` or `velero backup logs kube-public-ns-backup-20231126234338` for more details.
Backup request "kube-system-ns-backup-20231126234338" submitted successfully.
Run `velero backup describe kube-system-ns-backup-20231126234338` or `velero backup logs kube-system-ns-backup-20231126234338` for more details.
Backup request "kubernetes-dashboard-ns-backup-20231126234338" submitted successfully.
Run `velero backup describe kubernetes-dashboard-ns-backup-20231126234338` or `velero backup logs kubernetes-dashboard-ns-backup-20231126234338` for more details.
Backup request "velero-system-ns-backup-20231126234338" submitted successfully.
Run `velero backup describe velero-system-ns-backup-20231126234338` or `velero backup logs velero-system-ns-backup-20231126234338` for more details.