Ubuntu 22.04 LTS 快速部署nfs动态存储类
作者：尹正杰
版权声明：原创作品，谢绝转载！否则将追究法律责任。
一.Ubuntu 22.04 LTS 快速部署nfs服务器
1.nfs概述
nfs表示网络文件系统，存在客户端和服务端，需要单独部署服务端。
K8S在使用nfs时，集群应该安装nfs的相关模块。
nfs的应用场景：
- 1.实现跨节点不同Pod的数据共享;
- 2.实现跨节点存储数据;
2.K8S所有节点安装nfs程序
apt -y install nfs-kernel-server
3.服务端配置共享目录
[root@master231 ~]# mkdir -pv /yinzhengjie/data/nfs-server
4.配置共享目录
[root@master231 ~]# tail -1 /etc/exports
/yinzhengjie/data/nfs-server *(rw,no_root_squash)
[root@master231 ~]#
5.重启nfs服务使得配置生效
[root@master231 ~]# systemctl restart nfs-server
6.验证配置是否生效
[root@master231 ~]# exportfs
/yinzhengjie/data/nfs-server
<world>
[root@master231 ~]#
7.客户端测试验证
1.尝试挂载并写入数据
[root@worker233 ~]# mount -t nfs 10.0.0.231:/yinzhengjie/data/nfs-server /mnt
[root@worker233 ~]#
[root@worker233 ~]# df -h | grep mnt
10.0.0.231:/yinzhengjie/data/nfs-server 48G 16G 31G 34% /mnt
[root@worker233 ~]#
[root@worker233 ~]# cp /etc/os-release /mnt/
[root@worker233 ~]#
[root@worker233 ~]# ll /mnt/
total 12
drwxr-xr-x 2 root root 4096 Apr 15 16:37 ./
drwxr-xr-x 22 root root 4096 Apr 15 15:40 ../
-rw-r--r-- 1 root root 427 Apr 15 16:37 os-release
[root@worker233 ~]#
[root@worker233 ~]# umount /mnt/
[root@worker233 ~]#
[root@worker233 ~]# df -h | grep mnt
[root@worker233 ~]#
2.尝试挂载并读取数据
[root@worker232 ~]# mount -t nfs 10.0.0.231:/yinzhengjie/data/nfs-server /mnt
[root@worker232 ~]#
[root@worker232 ~]# ll /mnt
total 12
drwxr-xr-x 2 root root 4096 Apr 15 16:37 ./
drwxr-xr-x 22 root root 4096 Apr 15 15:40 ../
-rw-r--r-- 1 root root 427 Apr 15 16:37 os-release
[root@worker232 ~]#
[root@worker232 ~]# df -h | grep mnt
10.0.0.231:/yinzhengjie/data/nfs-server 48G 16G 31G 34% /mnt
[root@worker232 ~]#
[root@worker232 ~]# umount /mnt
[root@worker232 ~]#
[root@worker232 ~]# df -h | grep mnt
[root@worker232 ~]#
二.使用nfs存储卷实战案例
1. 编写资源清单
[root@master231 ~]# cat 01-deploy-nfs.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy-xiuxian-nfs
spec:
replicas: 3
selector:
matchLabels:
apps: xiuxian
template:
metadata:
labels:
apps: xiuxian
spec:
~:
- emptyDir: {}
name: data01
- name: data02
hostPath:
path: /etc/localtime
- name: data03
# 表示存储卷的类型是nfs
nfs:
# 指定nfs的服务器
server: 10.0.0.231
# 指定nfs共享的数据路径
path: /yinzhengjie/data/nfs-server
containers:
- name: c1
image: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1
ports:
- containerPort: 80
volumeMounts:
- name: data03
mountPath: /usr/share/nginx/html
- name: data02
mountPath: /etc/localtime
- name: data01
mountPath: /yinzhengjie-linux97
[root@master231 ~]#
2 测试验证
[root@master231 ~]# kubectl apply -f 04-deploy-nfs.yaml
deployment.apps/deploy-xiuxian-nfs created
[root@master231 ~]#
[root@master231 ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
deploy-xiuxian-nfs-785494468d-2ljcg 1/1 Running 0 3s 10.100.203.169 worker232 <none> <none>
deploy-xiuxian-nfs-785494468d-nndpb 1/1 Running 0 3s 10.100.140.106 worker233 <none> <none>
deploy-xiuxian-nfs-785494468d-tr2wx 1/1 Running 0 3s 10.100.140.105 worker233 <none> <none>
[root@master231 ~]#
[root@master231 ~]# kubectl exec -it deploy-xiuxian-nfs-785494468d-2ljcg -- sh
/ # echo www.yinzhengjie.com > /usr/share/nginx/html/index.html
/ #
/ #
[root@master231 ~]#
[root@master231 ~]# curl 10.100.203.169
www.yinzhengjie.com
[root@master231 ~]#
[root@master231 ~]# curl 10.100.140.106
www.yinzhengjie.com
[root@master231 ~]#
[root@master231 ~]# curl 10.100.140.105
www.yinzhengjie.com
[root@master231 ~]#
三.基于nfs4.9.0版本实现动态存储类实战案例
1.克隆代码
推荐阅读:
https://github.com/kubernetes-csi/csi-driver-nfs/blob/master/docs/install-csi-driver-v4.9.0.md
https://kubernetes.io/docs/concepts/storage/storage-classes/#nfs
[root@master231 nfs]# git clone https://github.com/kubernetes-csi/csi-driver-nfs.git
如果下载不了的SVIP:
wget http://192.168.16.253/Resources/Kubernetes/sc/nfs/code/csi-driver-nfs-4.9.0.tar.gz
tar xf csi-driver-nfs-4.9.0.tar.gz
2.安装nfs动态存储类
[root@master231 ~]# cd csi-driver-nfs-4.9.0/
[root@master231 csi-driver-nfs-4.9.0]#
[root@master231 csi-driver-nfs-4.9.0]# ./deploy/install-driver.sh v4.9.0 local
use local deploy
Installing NFS CSI driver, version: v4.9.0 ...
serviceaccount/csi-nfs-controller-sa created
serviceaccount/csi-nfs-node-sa created
clusterrole.rbac.authorization.k8s.io/nfs-external-provisioner-role created
clusterrolebinding.rbac.authorization.k8s.io/nfs-csi-provisioner-binding created
csidriver.storage.k8s.io/nfs.csi.k8s.io created
deployment.apps/csi-nfs-controller created
daemonset.apps/csi-nfs-node created
NFS CSI driver installed successfully.
[root@master231 csi-driver-nfs-4.9.0]#
3.验证是否安装成功
[root@master231 csi-driver-nfs]# kubectl -n kube-system get pod -o wide -l app
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
csi-nfs-controller-5c5c695fb-6psv8 4/4 Running 0 4s 10.0.0.232 worker232 <none> <none>
csi-nfs-node-bsmr7 3/3 Running 0 3s 10.0.0.232 worker232 <none> <none>
csi-nfs-node-ghtvt 3/3 Running 0 3s 10.0.0.231 master231 <none> <none>
csi-nfs-node-s4dm5 3/3 Running 0 3s 10.0.0.233 worker233 <none> <none>
[root@master231 csi-driver-nfs]#
温馨提示:
此步骤如果镜像下载不下来，则可以到我的仓库下载"http://192.168.16.253/Resources/Kubernetes/sc/nfs/images/"。
4.创建存储类
[root@master231 csi-driver-nfs]# mkdir /yinzhengjie/data/nfs-server/sc/
[root@master231 csi-driver-nfs]#
[root@master231 csi-driver-nfs]# cat deploy/v4.9.0/storageclass.yaml
...
parameters:
server: 10.0.0.231
share: /yinzhengjie/data/nfs-server/sc
...
[root@master231 csi-driver-nfs]#
[root@master231 csi-driver-nfs]# kubectl apply -f deploy/v4.9.0/storageclass.yaml
storageclass.storage.k8s.io/nfs-csi created
[root@master231 csi-driver-nfs]#
[root@master231 csi-driver-nfs]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-csi nfs.csi.k8s.io Delete Immediate false 3s
[root@master231 csi-driver-nfs]#
5.删除pv和pvc保证环境"干净"
[root@master231 volumes]# kubectl delete -f 21-deploy-pvc.yaml -f 20-pvc.yaml -f 19-pv.yaml
deployment.apps "deploy-pvc-demo" deleted
persistentvolumeclaim "yinzhengjie-linux-pvc" deleted
persistentvolume "yinzhengjie-linux-pv01" deleted
persistentvolume "yinzhengjie-linux-pv02" deleted
persistentvolume "yinzhengjie-linux-pv03" deleted
[root@master231 volumes]#
6.创建pvc测试
[root@master231 volumes]# cat 22-pvc-sc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: yinzhengjie-linux-pvc-sc
spec:
# 声明要是用的pv
# volumeName: yinzhengjie-linux-pv03
# 声明使用的存储类
storageClassName: nfs-csi
# 声明资源的访问模式
accessModes:
- ReadWriteMany
# 声明资源的使用量
resources:
limits:
storage: 2Mi
requests:
storage: 1Mi
[root@master231 volumes]#
[root@master231 volumes]# kubectl apply -f 22-pvc-sc.yaml
persistentvolumeclaim/yinzhengjie-linux-pvc-sc created
[root@master231 volumes]#
[root@master231 volumes]# kubectl get pvc,pv -o wide
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE VOLUMEMODE
persistentvolumeclaim/yinzhengjie-linux-pvc-sc Bound pvc-c23ad1b2-3f0b-4f53-bc51-9dbb20e6c037 1Mi RWX nfs-csi 6s Filesystem
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE VOLUMEMODE
persistentvolume/pvc-c23ad1b2-3f0b-4f53-bc51-9dbb20e6c037 1Mi RWX Delete Bound default/yinzhengjie-linux-pvc-sc nfs-csi 6s Filesystem
[root@master231 volumes]#
7.pod引用pvc
[root@master231 volumes]# cat 21-deploy-pvc.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy-pvc-demo
spec:
replicas: 1
selector:
matchLabels:
apps: v1
template:
metadata:
labels:
apps: v1
spec:
volumes:
- name: data
# 声明存储卷的类型是pvc
persistentVolumeClaim:
# 声明pvc的名称
# claimName: yinzhengjie-linux-pvc
claimName: yinzhengjie-linux-pvc-sc
- name: dt
hostPath:
path: /etc/localtime
initContainers:
- name: init01
image: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1
volumeMounts:
- name: data
mountPath: /yinzhengjie
- name: dt
mountPath: /etc/localtime
command:
- /bin/sh
- -c
- date -R > /yinzhengjie/index.html ; echo www.yinzhengjie.com >> /yinzhengjie/index.html
containers:
- name: c1
image: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1
volumeMounts:
- name: data
mountPath: /usr/share/nginx/html
- name: dt
mountPath: /etc/localtime
[root@master231 volumes]#
[root@master231 volumes]# kubectl apply -f 21-deploy-pvc.yaml
deployment.apps/deploy-pvc-demo created
[root@master231 volumes]#
[root@master231 volumes]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
deploy-pvc-demo-65d4b9bf97-st2ch 1/1 Running 0 3s 10.100.140.99 worker233 <none> <none>
[root@master231 volumes]#
[root@master231 volumes]# curl 10.100.140.99
Fri, 18 Apr 2025 10:31:32 +0800
www.yinzhengjie.com
[root@master231 volumes]#
8.验证pod的后端存储数据
[root@master231 volumes]# kubectl describe pod deploy-pvc-demo-688b57bdd-td2z7 | grep ClaimName
ClaimName: yinzhengjie-linux-pvc
[root@master231 volumes]#
[root@master231 volumes]# kubectl get pvc yinzhengjie-linux-pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
yinzhengjie-linux-pvc Bound pvc-b425481e-3b15-4854-bf34-801a29edfcc5 3Gi RWX nfs-csi 2m29s
[root@master231 volumes]#
[root@master231 volumes]# kubectl describe pv pvc-b425481e-3b15-4854-bf34-801a29edfcc5 | grep Source -A 5
Source:
Type: CSI (a Container Storage Interface (CSI) volume source)
Driver: nfs.csi.k8s.io
FSType:
VolumeHandle: 10.0.0.231#yinzhengjie/data/nfs-server/sc#pvc-c23ad1b2-3f0b-4f53-bc51-9dbb20e6c037##
ReadOnly: false
[root@master231 volumes]#
[root@master231 volumes]# ll /yinzhengjie/data/nfs-server/sc/pvc-c23ad1b2-3f0b-4f53-bc51-9dbb20e6c037/
total 12
drwxr-xr-x 2 root root 4096 Apr 18 10:31 ./
drwxr-xr-x 3 root root 4096 Apr 18 10:30 ../
-rw-r--r-- 1 root root 50 Apr 18 10:31 index.html
[root@master231 volumes]#
[root@master231 volumes]# cat /yinzhengjie/data/nfs-server/sc/pvc-c23ad1b2-3f0b-4f53-bc51-9dbb20e6c037/index.html
Fri, 18 Apr 2025 10:31:32 +0800
www.yinzhengjie.com
[root@master231 volumes]#
四.K8S配置默认的存储类及多个存储类定义
1.响应式配置默认存储类
[root@master231 nfs]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-csi nfs.csi.k8s.io Delete Immediate false 165m
[root@master231 nfs]#
[root@master231 nfs]# kubectl patch sc nfs-csi -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
storageclass.storage.k8s.io/nfs-csi patched
[root@master231 nfs]#
[root@master231 nfs]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-csi (default) nfs.csi.k8s.io Delete Immediate false 166m
[root@master231 nfs]#
2.响应式取消默认存储类
[root@master231 nfs]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-csi (default) nfs.csi.k8s.io Delete Immediate false 168m
[root@master231 nfs]#
[root@master231 nfs]# kubectl patch sc nfs-csi -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}'
storageclass.storage.k8s.io/nfs-csi patched
[root@master231 nfs]#
[root@master231 nfs]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-csi nfs.csi.k8s.io Delete Immediate false 168m
[root@master231 nfs]#
3.声明式配置多个存储类
[root@master231 storageclasses]# cat sc-multiple.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: yinzhengjie-sc-xixi
annotations:
storageclass.kubernetes.io/is-default-class: "false"
provisioner: nfs.csi.k8s.io
parameters:
server: 10.0.0.231
share: /yinzhengjie/data/nfs-server/sc-xixi
reclaimPolicy: Delete
volumeBindingMode: Immediate
mountOptions:
- nfsvers=4.1
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: yinzhengjie-sc-haha
annotations:
storageclass.kubernetes.io/is-default-class: "true"
provisioner: nfs.csi.k8s.io
parameters:
server: 10.0.0.231
share: /yinzhengjie/data/nfs-server/sc-haha
[root@master231 storageclasses]#
[root@master231 storageclasses]#
[root@master231 storageclasses]# kubectl apply -f sc-multiple.yaml
storageclass.storage.k8s.io/yinzhengjie-sc-xixi created
storageclass.storage.k8s.io/yinzhengjie-sc-haha created
[root@master231 storageclasses]#
[root@master231 storageclasses]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-csi nfs.csi.k8s.io Delete Immediate false 16m
yinzhengjie-sc-haha (default) nfs.csi.k8s.io Delete Immediate false 19s
yinzhengjie-sc-xixi nfs.csi.k8s.io Delete Immediate false 19s
[root@master231 storageclasses]#
4.准备目录
[root@master231 storageclasses]# mkdir -pv /yinzhengjie/data/nfs-server/sc-{xixi,haha}
5.测试验证
[root@master231 volumes]# cat 12-deploy-pvc-sc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: pvc-xiuxian
spec:
accessModes:
- ReadWriteMany
resources:
limits:
storage: 2Mi
requests:
storage: 1Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy-xiuxian
spec:
replicas: 1
selector:
matchLabels:
apps: v1
template:
metadata:
labels:
apps: v1
spec:
volumes:
- name: data
persistentVolumeClaim:
claimName: pvc-xiuxian
- name: dt
hostPath:
path: /etc/localtime
initContainers:
- name: init01
image: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1
volumeMounts:
- name: data
mountPath: /yinzhengjie
- name: dt
mountPath: /etc/localtime
command:
- /bin/sh
- -c
- date -R > /yinzhengjie/index.html ; echo www.yinzhengjie.com >> /yinzhengjie/index.html
containers:
- name: c1
image: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1
volumeMounts:
- name: data
mountPath: /usr/share/nginx/html
- name: dt
mountPath: /etc/localtime
[root@master231 volumes]#
[root@master231 volumes]#
[root@master231 volumes]# kubectl apply -f 12-deploy-pvc-sc.yaml
persistentvolumeclaim/pvc-xiuxian created
deployment.apps/deploy-xiuxian created
[root@master231 volumes]#
[root@master231 volumes]# kubectl get po,pvc,pv
NAME READY STATUS RESTARTS AGE
pod/deploy-xiuxian-587bcdb966-fk9s5 1/1 Running 0 5s
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
persistentvolumeclaim/pvc-xiuxian Bound pvc-691cf379-8889-4e5d-b346-b362d71ec1f0 1Mi RWX yinzhengjie-sc-haha 5s
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
persistentvolume/pvc-691cf379-8889-4e5d-b346-b362d71ec1f0 1Mi RWX Delete Bound default/pvc-xiuxian yinzhengjie-sc-haha 4s
[root@master231 volumes]#
[root@master231 volumes]# kubectl get po,pvc,pv -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
pod/deploy-xiuxian-587bcdb966-fk9s5 1/1 Running 0 10s 10.100.140.83 worker233 <none> <none>
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE VOLUMEMODE
persistentvolumeclaim/pvc-xiuxian Bound pvc-691cf379-8889-4e5d-b346-b362d71ec1f0 1Mi RWX yinzhengjie-sc-haha 10s Filesystem
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE VOLUMEMODE
persistentvolume/pvc-691cf379-8889-4e5d-b346-b362d71ec1f0 1Mi RWX Delete Bound default/pvc-xiuxian yinzhengjie-sc-haha 9s Filesystem
[root@master231 volumes]#
[root@master231 volumes]# curl 10.100.140.83
Fri, 06 Jun 2025 15:01:12 +0800
www.yinzhengjie.com
[root@master231 volumes]#
6.验证后端存储
[root@master231 volumes]# cat /usr/local/bin/get-pv.sh
#!/bin/bash
POD_NAME=$1
PVC_NAME=`kubectl describe pod $POD_NAME | grep ClaimName | awk '{print $2}'`
PV_NAME=`kubectl get pvc ${PVC_NAME} | awk 'NR==2{print $3}'`
kubectl describe pv $PV_NAME | grep Source -A 5
[root@master231 volumes]#
[root@master231 volumes]# chmod +x /usr/local/bin/get-pv.sh
[root@master231 volumes]#
[root@master231 volumes]# kubectl get pod
NAME READY STATUS RESTARTS AGE
deploy-xiuxian-587bcdb966-fk9s5 1/1 Running 0 2m36s
[root@master231 volumes]#
[root@master231 volumes]# get-pv.sh deploy-xiuxian-587bcdb966-fk9s5
Source:
Type: CSI (a Container Storage Interface (CSI) volume source)
Driver: nfs.csi.k8s.io
FSType:
VolumeHandle: 10.0.0.231#yinzhengjie/data/nfs-server/sc-haha#pvc-691cf379-8889-4e5d-b346-b362d71ec1f0##
ReadOnly: false
[root@master231 volumes]#
[root@master231 volumes]# ll /yinzhengjie/data/nfs-server/sc-haha/pvc-691cf379-8889-4e5d-b346-b362d71ec1f0/
total 12
drwxr-xr-x 2 root root 4096 Jun 6 15:01 ./
drwxr-xr-x 3 root root 4096 Jun 6 15:01 ../
-rw-r--r-- 1 root root 50 Jun 6 15:01 index.html
[root@master231 volumes]#
[root@master231 volumes]# cat /yinzhengjie/data/nfs-server/sc-haha/pvc-691cf379-8889-4e5d-b346-b362d71ec1f0/index.html
Fri, 06 Jun 2025 15:01:12 +0800
www.yinzhengjie.com
[root@master231 volumes]#
五.课后作业
部署WordPress到K8S集群，要求如下:
- 1.数据库一个副本;
- 2.wordpress为3个副本;
- 3.windows可以正常访问WordPress服务，正常发表博客;
- 4.删除所有wordpress相关Pod，实现秒级数据恢复;
本文来自博客园，作者：尹正杰，转载请注明原文链接：https://www.cnblogs.com/yinzhengjie/p/18968168，个人微信: "JasonYin2020"（添加时请备注来源及意图备注，有偿付费）
当你的才华还撑不起你的野心的时候，你就应该静下心来学习。当你的能力还驾驭不了你的目标的时候，你就应该沉下心来历练。问问自己，想要怎样的人生。
浙公网安备 33010602011771号