部署Ceph FS
root@ceph-mgr1:~# apt-cache madison ceph-mds
ceph-mds | 16.2.15-1focal | https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific focal/main amd64 Packages
ceph-mds | 15.2.17-0ubuntu0.20.04.6 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu focal-updates/main amd64 Packages
ceph-mds | 15.2.17-0ubuntu0.20.04.6 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu focal-security/main amd64 Packages
ceph-mds | 15.2.1-0ubuntu1 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu focal/main amd64 Packages
root@ceph-mgr1:~# apt install ceph-mds -y
cephadmin@ceph-deploy:~$ ceph-deploy mds create ceph-mgr1
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy mds create ceph-mgr1
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] subcommand : create
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f99c7a484c0>
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] func : <function mds at 0x7f99c7a73ee0>
[ceph_deploy.cli][INFO ] mds : [('ceph-mgr1', 'ceph-mgr1')]
[ceph_deploy.mds][DEBUG ] Deploying mds, cluster ceph hosts ceph-mgr1:ceph-mgr1
[ceph-mgr1][DEBUG ] connection detected need for sudo
[ceph-mgr1][DEBUG ] connected to host: ceph-mgr1
[ceph_deploy.mds][INFO ] Distro info: ubuntu 18.04 bionic
[ceph_deploy.mds][DEBUG ] remote host will use systemd
[ceph_deploy.mds][DEBUG ] deploying mds bootstrap to ceph-mgr1
[ceph-mgr1][WARNIN] mds keyring does not exist yet, creating one
[ceph-mgr1][INFO ] Running command: sudo ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.ceph-mgr1 osd allow rwx mds allow mon allow profile mds -o /var/lib/ceph/mds/ceph-ceph-mgr1/keyring
[ceph-mgr1][INFO ] Running command: sudo systemctl enable ceph-mds@ceph-mgr1
[ceph-mgr1][WARNIN] Created symlink /etc/systemd/system/ceph-mds.target.wants/ceph-mds@ceph-mgr1.service → /lib/systemd/system/ceph-mds@.service.
[ceph-mgr1][INFO ] Running command: sudo systemctl start ceph-mds@ceph-mgr1
[ceph-mgr1][INFO ] Running command: sudo systemctl enable ceph.target
#验证 MDS 服务:
#MDS 服务目前还无法正常使用，需要为 MDS 创建存储池用于保存 MDS 的数据
#standby 当前为备用状态，需要分配 pool 才可以使用
cephadmin@ceph-deploy:~$ ceph mds stat
1 up:standby
#:创建 CephFS metadata 和 data 存储池:
#使用 CephFS之前需要事先于集群中创建一个文件系统，并为其分别指定元数据和数据相关的存储池，
#如下命令将创建名为mycephfs 的文件系统，它使用 cephfs-metadata 作为元数据存储池，使用 cephfs-data 为数据存储池:
cephadmin@ceph-deploy:~$ ceph osd pool create cephfs-metadata 32 32 #保存metadata的pool
pool 'cephfs-metadata' created
cephadmin@ceph-deploy:~$ ceph osd pool create cephfs-data 64 64 #保存数据的pool
pool 'cephfs-data' created
cephadmin@ceph-deploy:~$ ceph -s
cluster:
id: 0d8fb726-ee6d-4aaf-aeca-54c68e2584af
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 5h)
mgr: ceph-mgr1(active, since 5h), standbys: ceph-mgr2
osd: 9 osds: 9 up (since 5h), 9 in (since 6h)
data:
pools: 4 pools, 129 pgs
objects: 0 objects, 0 B
usage: 2.6 GiB used, 267 GiB / 270 GiB avail
pgs: 129 active+clean
#创建cephFS并验证
cephadmin@ceph-deploy:~$ ceph fs new mycephfs cephfs-metadata cephfs-data
Pool 'cephfs-data' (id '4') has pg autoscale mode 'on' but is not marked as bulk.
Consider setting the flag by running
# ceph osd pool set cephfs-data bulk true
new fs with metadata pool 3 and data pool 4
cephadmin@ceph-deploy:~$ ceph fs ls
name: mycephfs, metadata pool: cephfs-metadata, data pools: [cephfs-data ]
#查看指定cephFS状态
cephadmin@ceph-deploy:~$ ceph fs status mycephfs
mycephfs - 0 clients
========
RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS
0 active ceph-mgr1 Reqs: 0 /s 10 13 12 0
POOL TYPE USED AVAIL
cephfs-metadata metadata 96.0k 84.6G
cephfs-data data 0 84.6G
MDS version: ceph version 16.2.15 (618f440892089921c3e944a991122ddc44e60516) pacific (stable)
#验证 cepfFS 服务状态 - 现在时正常的状态了
cephadmin@ceph-deploy:~$ ceph mds stat
mycephfs:1 {0=ceph-mgr1=up:active}
客户端挂载Ceph FS
cephadmin@ceph-deploy:~$ cat ceph.client.admin.keyring
[client.admin]
key = AQB/0+plFI5DJhAA+TEChHmsGs62RHhTaO/YUg==
caps mds = "allow *"
caps mgr = "allow *"
caps mon = "allow *"
caps osd = "allow *"
#使用Centos7
##ubuntu 及 centos client 挂载(内核版本 2.6.34 在 3.6.34 及以上):
[root@k8s-haproxy02 ~]# mkdir /cephfs
[root@k8s-haproxy02 ~]# mount -t ceph 192.168.40.151:6789:/ /cephfs -o name=admin,secret=AQB/0+plFI5DJhAA+TEChHmsGs62RHhTaO/YUg==
[root@k8s-haproxy02 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
devtmpfs 979M 0 979M 0% /dev
tmpfs 991M 0 991M 0% /dev/shm
tmpfs 991M 9.6M 981M 1% /run
tmpfs 991M 0 991M 0% /sys/fs/cgroup
/dev/mapper/centos-root 17G 2.2G 15G 13% /
/dev/sda1 1014M 138M 877M 14% /boot
tmpfs 199M 0 199M 0% /run/user/0
192.168.40.151:6789:/ 85G 0 85G 0% /cephfs
#
[root@k8s-haproxy02 ~]# cp /var/log/secure /cephfs/
[root@k8s-haproxy02 ~]# ll /cephfs/secure
-rw------- 1 root root 1131 Mar 12 10:12 /cephfs/secure
#测试写入数据
[root@k8s-haproxy02 ~]# dd if=/dev/zero of=/cephfs/ceph-fs-testfile bs=4M count=25
25+0 records in
25+0 records out
104857600 bytes (105 MB) copied, 0.398541 s, 263 MB/s
#ceph 集群验证存储
cephadmin@ceph-deploy:~$ ceph df
--- RAW STORAGE ---
CLASS SIZE AVAIL USED RAW USED %RAW USED
hdd 270 GiB 267 GiB 2.9 GiB 2.9 GiB 1.06
TOTAL 270 GiB 267 GiB 2.9 GiB 2.9 GiB 1.06
--- POOLS ---
POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL
device_health_metrics 1 1 0 B 0 0 B 0 85 GiB
mypool 2 32 0 B 0 0 B 0 85 GiB
cephfs-metadata 3 32 39 KiB 22 204 KiB 0 85 GiB
cephfs-data 4 64 100 MiB 26 300 MiB 0.12 85 GiB
#删除数据
[root@k8s-haproxy02 cephfs]# rm -rf *
#验证已经容量释放
cephadmin@ceph-deploy:~$ ceph df
--- RAW STORAGE ---
CLASS SIZE AVAIL USED RAW USED %RAW USED
hdd 270 GiB 267 GiB 2.6 GiB 2.6 GiB 0.95
TOTAL 270 GiB 267 GiB 2.6 GiB 2.6 GiB 0.95
--- POOLS ---
POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL
device_health_metrics 1 1 0 B 0 0 B 0 85 GiB
mypool 2 32 0 B 0 0 B 0 85 GiB
cephfs-metadata 3 32 50 KiB 23 252 KiB 0 85 GiB
cephfs-data 4 64 0 B 0 0 B 0 85 GiB
验证共享数据-Ubuntu&Centos
#Centos 挂载
cephadmin@ceph-deploy:~$ cat ceph.client.admin.keyring
[client.admin]
key = AQB/0+plFI5DJhAA+TEChHmsGs62RHhTaO/YUg==
caps mds = "allow *"
caps mgr = "allow *"
caps mon = "allow *"
caps osd = "allow *"
#使用Centos7
##ubuntu 及 centos client 挂载(内核版本 2.6.34 在 3.6.34 及以上):
[root@k8s-haproxy02 ~]# mkdir /cephfs
[root@k8s-haproxy02 ~]# mount -t ceph 192.168.40.151:6789:/ /cephfs -o name=admin,secret=AQB/0+plFI5DJhAA+TEChHmsGs62RHhTaO/YUg==
[root@k8s-haproxy02 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
devtmpfs 979M 0 979M 0% /dev
tmpfs 991M 0 991M 0% /dev/shm
tmpfs 991M 9.6M 981M 1% /run
tmpfs 991M 0 991M 0% /sys/fs/cgroup
/dev/mapper/centos-root 17G 2.2G 15G 13% /
/dev/sda1 1014M 138M 877M 14% /boot
tmpfs 199M 0 199M 0% /run/user/0
192.168.40.151:6789:/ 85G 0 85G 0% /cephfs
#
[root@k8s-haproxy02 ~]# cp /var/log/secure /cephfs/
[root@k8s-haproxy02 ~]# ll /cephfs/secure
-rw------- 1 root root 1131 Mar 12 10:12 /cephfs/secure
#测试写入数据
[root@k8s-haproxy02 ~]# dd if=/dev/zero of=/cephfs/ceph-fs-testfile bs=4M count=25
25+0 records in
25+0 records out
104857600 bytes (105 MB) copied, 0.398541 s, 263 MB/s
#ceph 集群验证存储
cephadmin@ceph-deploy:~$ ceph df
--- RAW STORAGE ---
CLASS SIZE AVAIL USED RAW USED %RAW USED
hdd 270 GiB 267 GiB 2.9 GiB 2.9 GiB 1.06
TOTAL 270 GiB 267 GiB 2.9 GiB 2.9 GiB 1.06
--- POOLS ---
POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL
device_health_metrics 1 1 0 B 0 0 B 0 85 GiB
mypool 2 32 0 B 0 0 B 0 85 GiB
cephfs-metadata 3 32 39 KiB 22 204 KiB 0 85 GiB
cephfs-data 4 64 100 MiB 26 300 MiB 0.12 85 GiB
#Ubuntu 挂载已经有的数据盘查看数据是否有
root@ceph-osd1:~# cat /etc/lsb-release
DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=22.04
DISTRIB_CODENAME=jammy
DISTRIB_DESCRIPTION="Ubuntu 22.04 LTS"
#挂载之前的数据盘 - 验证是共享的
root@ceph-osd1:~# mkdir /cephfsshare
root@ceph-osd1:~# mount -t ceph 192.168.40.151:6789:/ /cephfsshare -o name=admin,secret=AQB/0+plFI5DJhAA+TEChHmsGs62RHhTaO/YUg==
root@ceph-osd1:~# df -h
Filesystem Size Used Avail Use% Mounted on
tmpfs 589M 1.8M 587M 1% /run
/dev/mapper/ubuntu--vg-ubuntu--lv 9.8G 5.8G 3.5G 63% /
tmpfs 2.9G 0 2.9G 0% /dev/shm
tmpfs 5.0M 0 5.0M 0% /run/lock
/dev/sda2 1.8G 129M 1.5G 8% /boot
overlay 9.8G 5.8G 3.5G 63% /var/lib/docker/overlay2/06c0c2b4fcb68e23f75828bafe05d3898f4e0969caeb787dc4c41eb38bcfe122/merged
overlay 9.8G 5.8G 3.5G 63% /var/lib/docker/overlay2/dda5a3eec8971b3db4f54f6490bc5eaa93a5ab6138d11e43dca4dcbd87da2c1c/merged
overlay 9.8G 5.8G 3.5G 63% /var/lib/docker/overlay2/94eb4d5863029be88e4a7344cd5dfadd040a4432b5eaab23345d5220c5d717a7/merged
overlay 9.8G 5.8G 3.5G 63% /var/lib/docker/overlay2/0325ba60dcf4742f9db565c2b68768aa8ae6a901459e7aa77f6daae79036fd9a/merged
overlay 9.8G 5.8G 3.5G 63% /var/lib/docker/overlay2/e6a6b9c01fec410802eed9f9c967bc7434595f15b44c5d571c9aa445c264ce16/merged
tmpfs 589M 4.0K 589M 1% /run/user/0
192.168.40.151:6789:/ 85G 100M 85G 1% /cephfsshare
root@ceph-osd1:~# cd /cephfsshare/
root@ceph-osd1:/cephfsshare# ll
total 102404
drwxr-xr-x 2 root root 1 Mar 12 02:19 ./
drwxr-xr-x 20 root root 4096 Mar 12 02:20 ../
-rw-r--r-- 1 root root 104857600 Mar 12 02:19 ceph-fs-testfile
命令总结
#只显示存储池
cephadmin@ceph-deploy:~$ ceph osd pool ls
device_health_metrics
mypool
cephfs-metadata
cephfs-data
#列出存储池并显示id
cephadmin@ceph-deploy:~$ ceph osd lspools
1 device_health_metrics
2 mypool
3 cephfs-metadata
4 cephfs-data
#查看pg状态
cephadmin@ceph-deploy:~$ ceph pg stat
129 pgs: 129 active+clean; 2.3 KiB data, 2.6 GiB used, 267 GiB / 270 GiB avail
#查看指定pool或所有的pool的状态
cephadmin@ceph-deploy:~$ ceph osd pool stats mypool
pool mypool id 2
nothing is going on
#查看集群存储状态
cephadmin@ceph-deploy:~$ ceph df
--- RAW STORAGE ---
CLASS SIZE AVAIL USED RAW USED %RAW USED
hdd 270 GiB 267 GiB 2.6 GiB 2.6 GiB 0.95
TOTAL 270 GiB 267 GiB 2.6 GiB 2.6 GiB 0.95
--- POOLS ---
POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL
device_health_metrics 1 1 0 B 0 0 B 0 85 GiB
mypool 2 32 0 B 0 0 B 0 85 GiB
cephfs-metadata 3 32 2.3 KiB 22 96 KiB 0 85 GiB
cephfs-data 4 64 0 B 0 0 B 0 85 GiB
#查看集群存储状态的详情
cephadmin@ceph-deploy:~$ ceph df detail
--- RAW STORAGE ---
CLASS SIZE AVAIL USED RAW USED %RAW USED
hdd 270 GiB 267 GiB 2.6 GiB 2.6 GiB 0.95
TOTAL 270 GiB 267 GiB 2.6 GiB 2.6 GiB 0.95
--- POOLS ---
POOL ID PGS STORED (DATA) (OMAP) OBJECTS USED (DATA) (OMAP) %USED MAX AVAIL QUOTA OBJECTS QUOTA BYTES DIRTY USED COMPR UNDER COMPR
device_health_metrics 1 1 0 B 0 B 0 B 0 0 B 0 B 0 B 0 85 GiB N/A N/A N/A 0 B 0 B
mypool 2 32 0 B 0 B 0 B 0 0 B 0 B 0 B 0 85 GiB N/A N/A N/A 0 B 0 B
cephfs-metadata 3 32 2.3 KiB 2.3 KiB 0 B 22 96 KiB 96 KiB 0 B 0 85 GiB N/A N/A N/A 0 B 0 B
cephfs-data 4 64 0 B 0 B 0 B 0 0 B 0 B 0 B 0 85 GiB N/A N/A N/A 0 B 0 B
#查看osd状态
cephadmin@ceph-deploy:~$ ceph osd stat
9 osds: 9 up (since 5h), 9 in (since 6h); epoch: e75
#显示OSD的底层详细信息
ceph osd dump
#显示OSD和节点的对应关系
cephadmin@ceph-deploy:~$ ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 0.26367 root default
-3 0.08789 host ceph-node1
0 hdd 0.02930 osd.0 up 1.00000 1.00000
1 hdd 0.02930 osd.1 up 1.00000 1.00000
2 hdd 0.02930 osd.2 up 1.00000 1.00000
-5 0.08789 host ceph-node2
3 hdd 0.02930 osd.3 up 1.00000 1.00000
4 hdd 0.02930 osd.4 up 1.00000 1.00000
5 hdd 0.02930 osd.5 up 1.00000 1.00000
-7 0.08789 host ceph-node3
6 hdd 0.02930 osd.6 up 1.00000 1.00000
7 hdd 0.02930 osd.7 up 1.00000 1.00000
8 hdd 0.02930 osd.8 up 1.00000 1.00000
#查找OSD对应的硬盘
#先找到异常的OSD的ID，比如ID=4的OSD故障了找到对应的硬盘
ceph osd tree
#到 OSD 对用的 node 节点查看与 OSD 对用的硬盘:
root@ceph-node2:~# lsblk -f | grep -B1 ceph
sdb LVM2_member Iy1toC-Sxmo-dP4m-71tI-349Z-1ymy-gckh3B
└─ceph--f4d6b913--a9e2--408c--a691--dd0f73636781-osd--block--13ac2ce9--107f--4301--87d3--089b5cd8af89
sdc LVM2_member 6ktY5e-68X7-wGKH-HzjS-qolH-m5Va-jik0hO
└─ceph--e93a2451--d8a9--4b59--9b8b--de5d07580f37-osd--block--3b985c80--bdbd--4e90--950e--b42f720b2b1a
sdd LVM2_member bYOMyn-iywX-Mbgr-HMdl-tRlH-8k3Z-YWs2h9
└─ceph--11c5c9fd--8c07--4fcb--90f6--8e97e367eb11-osd--block--3f60d0f8--12ed--437a--b4e8--173bd89c5b66
#ceph-4 最后4位 2b1a 就是 sdb 有硬盘
root@ceph-node2:~# ll /var/lib/ceph/osd/ceph-4/block
lrwxrwxrwx 1 ceph ceph 93 Mar 11 11:07 /var/lib/ceph/osd/ceph-4/block -> /dev/ceph-e93a2451-d8a9-4b59-9b8b-de5d07580f37/osd-block-3b985c80-bdbd-4e90-950e-b42f720b2b1a
#查看mon 节点的状态
cephadmin@ceph-deploy:~$ ceph mon stat
e3: 3 mons at {ceph-mon1=[v2:192.168.40.151:3300/0,v1:192.168.40.151:6789/0],ceph-mon2=[v2:192.168.40.152:3300/0,v1:192.168.40.152:6789/0],ceph-mon3=[v2:192.168.40.153:3300/0,v1:192.168.40.153:6789/0]} removed_ranks: {}, election epoch 24, leader 0 ceph-mon1, quorum 0,1,2 ceph-mon1,ceph-mon2,ceph-mon3
#查看mon节点的dump信息
#epoch 3 #Ceph OSD 守护进程和 PG 的每个状态改变的历史(称之为"epoch"),osd.N 的失效会导致 osd map 变化，epoch 值也会自动调整
cephadmin@ceph-deploy:~$ ceph mon dump
epoch 3
fsid 0d8fb726-ee6d-4aaf-aeca-54c68e2584af
last_changed 2024-03-11T10:38:49.888985+0800
created 2024-03-08T16:59:43.135997+0800
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.40.151:3300/0,v1:192.168.40.151:6789/0] mon.ceph-mon1
1: [v2:192.168.40.152:3300/0,v1:192.168.40.152:6789/0] mon.ceph-mon2
2: [v2:192.168.40.153:3300/0,v1:192.168.40.153:6789/0] mon.ceph-mon3
dumped monmap epoch 3
#删除存储池
#如果把存储池删除会导致把存储池内的数据全部删除，因此 ceph 为了防止误删除存储池设置了两个机制来防止误删除操作。
#第一个机制是 NODELETE 标志,需要设置为 false 但是默认就是 false 了
cephadmin@ceph-deploy:~$ ceph osd pool create mypool2 32 32
pool 'mypool2' created
cephadmin@ceph-deploy:~$ ceph osd pool get mypool2 nodelete
nodelete: false
#如果设置true表示不能删除，可以使用set 设置false 表示可以进行删除
cephadmin@ceph-deploy:~$ ceph osd pool set mypool2 nodelete true
set pool 5 nodelete to true
cephadmin@ceph-deploy:~$ ceph osd pool get mypool2 nodelete
nodelete: true
cephadmin@ceph-deploy:~$ ceph osd pool set mypool2 nodelete false
set pool 5 nodelete to false
#删除会报错
cephadmin@ceph-deploy:~$ ceph osd pool rm mypool2
Error EPERM: WARNING: this will *PERMANENTLY DESTROY* all data stored in pool mypool2. If you are *ABSOLUTELY CERTAIN* that is what you want, pass the pool name *twice*, followed by --yes-i-really-really-mean-it.
#第二个机制是集群范围的配置参数 mon allow pool delete，默认值为 false，即监视器不,许删除存储池,
#可以在特定场合使用 tell 指令临时设置为(true)允许删除,在删除指定的 poool 之后再重新设置为 false.
cephadmin@ceph-deploy:~$ ceph tell mon.* injectargs --mon-allow-pool-delete=true
mon.ceph-mon1: {}
mon.ceph-mon1: mon_allow_pool_delete = 'true'
mon.ceph-mon2: {}
mon.ceph-mon2: mon_allow_pool_delete = 'true'
mon.ceph-mon3: {}
mon.ceph-mon3: mon_allow_pool_delete = 'true'
cephadmin@ceph-deploy:~$ ceph osd pool rm mypool2 mypool2 --yes-i-really-really-mean-it
pool 'mypool2' removed
#删除好之后把参数改为回来
cephadmin@ceph-deploy:~$ ceph tell mon.* injectargs --mon-allow-pool-delete=false
浙公网安备 33010602011771号