1、存储池管理常用命令
#列出存储池 - 不带 pool ID
cephadmin@ceph-deploy:~$ ceph osd pool ls
device_health_metrics
mypool
cephfs-metadata
cephfs-data
#带pool ID
cephadmin@ceph-deploy:~$ ceph osd lspools
1 device_health_metrics
2 mypool
3 cephfs-metadata
4 cephfs-data
#获取存储池的事件信息
cephadmin@ceph-deploy:~$ ceph osd pool stats mypool
pool mypool id 2
nothing is going on
#重命名存储池
cephadmin@ceph-deploy:~$ ceph osd pool rename old-name new-name
#显示存储池的用量信息
cephadmin@ceph-deploy:~$ rados df
POOL_NAME USED OBJECTS CLONES COPIES MISSING_ON_PRIMARY UNFOUND DEGRADED RD_OPS RD WR_OPS WR USED COMPR UNDER COMPR
cephfs-data 300 MiB 25 0 75 0 0 0 0 0 B 77 200 MiB 0 B 0 B
cephfs-metadata 300 KiB 23 0 69 0 0 0 50 52 KiB 100 93 KiB 0 B 0 B
device_health_metrics 0 B 0 0 0 0 0 0 0 0 B 0 0 B 0 B 0 B
mypool 0 B 0 0 0 0 0 0 2 374 KiB 4 189 KiB 0 B 0 B
total_objects 48
total_used 2.9 GiB
total_avail 267 GiB
total_space 270 GiB
2、存储池删除
#如果把存储池删除会导致把存储池内的数据全部删除，因此 ceph 为了防止误删除存储池设置了两个机制来防止误删除操作。
#第一个机制是 NODELETE 标志,需要设置为 false 但是默认就是 false 了
cephadmin@ceph-deploy:~$ ceph osd pool create mypool2 32 32
pool 'mypool2' created
cephadmin@ceph-deploy:~$ ceph osd pool get mypool2 nodelete
nodelete: false
#如果设置true表示不能删除，可以使用set 设置false 表示可以进行删除
cephadmin@ceph-deploy:~$ ceph osd pool set mypool2 nodelete true
set pool 5 nodelete to true
cephadmin@ceph-deploy:~$ ceph osd pool get mypool2 nodelete
nodelete: true
cephadmin@ceph-deploy:~$ ceph osd pool set mypool2 nodelete false
set pool 5 nodelete to false
#删除会报错
cephadmin@ceph-deploy:~$ ceph osd pool rm mypool2
Error EPERM: WARNING: this will *PERMANENTLY DESTROY* all data stored in pool mypool2. If you are *ABSOLUTELY CERTAIN* that is what you want, pass the pool name *twice*, followed by --yes-i-really-really-mean-it.
#第二个机制是集群范围的配置参数 mon allow pool delete，默认值为 false，即监视器不,许删除存储池,
#可以在特定场合使用 tell 指令临时设置为(true)允许删除,在删除指定的 poool 之后再重新设置为 false.
cephadmin@ceph-deploy:~$ ceph tell mon.* injectargs --mon-allow-pool-delete=true
mon.ceph-mon1: {}
mon.ceph-mon1: mon_allow_pool_delete = 'true'
mon.ceph-mon2: {}
mon.ceph-mon2: mon_allow_pool_delete = 'true'
mon.ceph-mon3: {}
mon.ceph-mon3: mon_allow_pool_delete = 'true'
cephadmin@ceph-deploy:~$ ceph osd pool rm mypool2 mypool2 --yes-i-really-really-mean-it
pool 'mypool2' removed
3、存储池配额
#存储池可以设置两个配对存储的对象进行限制，一个配额是对象最大数量(max obiects)、另外一个配额是最大空间(max_bytes)
cephadmin@ceph-deploy:~$ ceph osd pool get-quota mypool
quotas for pool 'mypool':
max objects: N/A #默认不限制对象数量
max bytes : N/A #默认不限制空间大小
#限制最大1000个对象
cephadmin@ceph-deploy:~$ ceph osd pool set-quota mypool max_objects 1000
set-quota max_objects = 1000 for pool mypool
限制最大10G 10*1024*1024*1024 字节
cephadmin@ceph-deploy:~$ ceph osd pool set-quota mypool max_bytes 10737418240
set-quota max_bytes = 10737418240 for pool mypool
#查看配置情况
cephadmin@ceph-deploy:~$ ceph osd pool get-quota mypool
quotas for pool 'mypool':
max objects: 1k objects (current num objects: 0 objects)
max bytes : 10 GiB (current num bytes: 0 bytes)
4、存储池可用参数
#size:存储池中的对象副本数,默认一主两个备3副本
cephadmin@ceph-deploy:~$ ceph osd pool get mypool size
size: 3
#min_size:提供服务所需要的最小副本数，如果定义 size为 3,min_size 也为 3.坏掉一个OSD,如果 pool池中有副本在此块 OSD 上面,那么此 pool将不提供服务
#如果将 min_size定义为 2，那么还可以提供服务，如果提供为1，表示只要有一块副本都提供服务,
cephadmin@ceph-deploy:~$ ceph osd pool get mypool min_size
min_size: 2
#pg_num:查看当前 PG 的数量
cephadmin@ceph-deploy:~$ ceph osd pool get mypool pg_num
pg_num: 32
#crush rule:设置 crush 算法规则 #默认为副本池
cephadmin@ceph-deploy:~$ ceph osd pool get mypool crush_rule
crush_rule: replicated_rule
#nodelete:控制是否可删除，默认可以
cephadmin@ceph-deploy:~$ ceph osd pool get mypool nodelete
nodelete: false
#nopgchange:控制是否可更改存储池的pg num 和 pgp num
cephadmin@ceph-deploy:~$ ceph osd pool get mypool nopgchange
nopgchange: false
#修改指定 pool 的 pg 数量
cephadmin@ceph-deploy:~$ ceph osd pool set mypool pg_num 64
set pool 2 pg_num to 64
cephadmin@ceph-deploy:~$ ceph osd pool set mypool pgp_num 64
set pool 2 pgp_num to 64
#nosizechange:控制是否可以更改存储池的大小 - #默认允许修改存储池大小
cephadmin@ceph-deploy:~$ ceph osd pool get mypool nosizechange
nosizechange: false
#noscrub 和 nodeep-scrub:控制是否不进行轻量扫描或是否深层扫描存储池，可临时解决高I/O 问题
##查看当前是否关闭轻量扫描数据，默认为不关闭，即开启
cephadmin@ceph-deploy:~$ ceph osd pool get mypool noscrub
noscrub: false
#可以修改某个指定的 pool 的轻量级扫描测量为 true，即不执行轻量级扫描
cephadmin@ceph-deploy:~$ ceph osd pool set mypool noscrub true
set pool 2 noscrub to true #再次查看就不进行轻量级扫描了
cephadmin@ceph-deploy:~$ ceph osd pool set mypool noscrub false
set pool 2 noscrub to false
#查看当前是否关闭深度扫描数据，默认为不关闭，即开启nodeep-scrub: false
cephadmin@ceph-deploy:~$ ceph osd pool get mypool nodeep-scrub
nodeep-scrub: false
#可以修改某个指定的 pool 的深度扫描测量为 true，即不执行深度扫描
cephadmin@ceph-deploy:~$ ceph osd pool set mypool nodeep-scrub true
set pool 2 nodeep-scrub to true #再次查看就不执行深度扫描了
cephadmin@ceph-deploy:~$ ceph osd pool set mypool nodeep-scrub false
set pool 2 nodeep-scrub to false
#scrub min interval:集群存储池的最小清理时间间隔，默认值没有设置，可以通过配置文件中的 osd_scrub_min_interval 参数指定间隔时间
cephadmin@ceph-deploy:~$ ceph osd pool get mypool scrub_min_interval
Error ENOENT: option 'scrub_min_interval' is not set on pool 'mypool'
#scrub_max_interva:整理存储池的最大清理时间间隔，默认值没有设置，可以通过配置文件中的 osd scrub_max_interval 参数指定.
cephadmin@ceph-deploy:~$ ceph osd pool get mypool scrub_max_interval
Error ENOENT: option 'scrub_max_interval' is not set on pool 'mypool'
#deep_scrub_interval:深层整理存储池的时间间隔，默认值没有设置，可以通过配置文件中的 osd deep scrub_interval 参数指定
cephadmin@ceph-deploy:~$ ceph osd pool get mypool deep_scrub_interval
Error ENOENT: option 'deep_scrub_interval' is not set on pool 'mypool'
#ceph node默认配置
root@ceph-node1:~# ll /var/run/ceph/
total 0
drwxrwx--- 2 ceph ceph 100 Mar 12 09:28 ./
drwxr-xr-x 25 root root 840 Mar 12 09:35 ../
srwxr-xr-x 1 ceph ceph 0 Mar 12 09:28 ceph-osd.0.asok=
srwxr-xr-x 1 ceph ceph 0 Mar 12 09:28 ceph-osd.1.asok=
srwxr-xr-x 1 ceph ceph 0 Mar 12 09:28 ceph-osd.2.asok=
root@ceph-node1:~# ceph daemon osd.2 config show | grep scrub
"mds_max_scrub_ops_in_progress": "5",
"mon_scrub_inject_crc_mismatch": "0.000000",
"mon_scrub_inject_missing_keys": "0.000000",
"mon_scrub_interval": "86400", #
"mon_scrub_max_keys": "100",
"mon_scrub_timeout": "300",
"mon_warn_pg_not_deep_scrubbed_ratio": "0.750000",
"mon_warn_pg_not_scrubbed_ratio": "0.500000",
"osd_debug_deep_scrub_sleep": "0.000000",
"osd_deep_scrub_interval": "604800.000000", #定义深度清晰间隔，7天
"osd_deep_scrub_keys": "1024",
"osd_deep_scrub_large_omap_object_key_threshold": "200000",
"osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824",
"osd_deep_scrub_randomize_ratio": "0.150000",
"osd_deep_scrub_stride": "524288",
"osd_deep_scrub_update_digest_min_age": "7200",
"osd_max_scrubs": "1", #定义一个ceph OSD daemon内能够同时进行scrubbing的操作数
"osd_requested_scrub_priority": "120",
"osd_scrub_auto_repair": "false",
"osd_scrub_auto_repair_num_errors": "5",
"osd_scrub_backoff_ratio": "0.660000",
"osd_scrub_begin_hour": "0",
"osd_scrub_begin_week_day": "0",
"osd_scrub_chunk_max": "25",
"osd_scrub_chunk_min": "5",
"osd_scrub_cost": "52428800",
"osd_scrub_during_recovery": "false",
"osd_scrub_end_hour": "0",
"osd_scrub_end_week_day": "0",
"osd_scrub_extended_sleep": "0.000000",
"osd_scrub_interval_randomize_ratio": "0.500000",
"osd_scrub_invalid_stats": "true", #定义scrub是否有效
"osd_scrub_load_threshold": "0.500000",
"osd_scrub_max_interval": "604800.000000", ##定义最大执行 scrub 间隔，7天
"osd_scrub_max_preemptions": "5",
"osd_scrub_min_interval": "86400.000000", #定义最小执行普通scrub间隔 1天
"osd_scrub_priority": "5",
"osd_scrub_sleep": "0.000000",
5、存储快照
#快照用于读存储池中的数据进行备份与还原,创建快照需要占用的磁盘空间会比较大，取决于存储池中的数据大小，使用以下命令创建快照
#创建快照
cephadmin@ceph-deploy:~$ ceph osd pool ls
device_health_metrics
mypool
cephfs-metadata
cephfs-data
cephadmin@ceph-deploy:~$ ceph osd pool mksnap mypool mypool-snap-202403121515
created pool mypool snap mypool-snap-202403121515
cephadmin@ceph-deploy:~$ rados -p mypool mksnap mypool-snap-202403121516
created pool mypool snap mypool-snap-202403121516
#验证快照
cephadmin@ceph-deploy:~$ rados lssnap -p mypool
1 mypool-snap-202403121515 2024.03.12 15:15:39
2 mypool-snap-202403121516 2024.03.12 15:16:06
2 snaps
#回滚快照-测试上传文件后创建快照，然后删除文件再还原文件,基于对象还原.
cephadmin@ceph-deploy:~$ rados -p mypool put testfile /etc/hosts #上传文件
cephadmin@ceph-deploy:~$ rados -p mypool ls #验证文件
testfile
cephadmin@ceph-deploy:~$ ceph osd pool mksnap mypool mypool-snapshot001 #创建快照
created pool mypool snap mypool-snapshot001
cephadmin@ceph-deploy:~$ rados lssnap -p mypool #验证快照
1 mypool-snap-202403121515 2024.03.12 15:15:39
2 mypool-snap-202403121516 2024.03.12 15:16:06
3 mypool-snapshot001 2024.03.12 15:18:03
3 snaps
cephadmin@ceph-deploy:~$ rados -p mypool rm testfile #删除文件
cephadmin@ceph-deploy:~$ rados -p mypool rm testfile #删除文件后 无法再次删除 文件不存在
error removing mypool>testfile: (2) No such file or directory
cephadmin@ceph-deploy:~$ rados rollback -p mypool testfile mypool-snapshot001 #通过快照还原某个文件
rolled back pool mypool to snapshot mypool-snapshot001
cephadmin@ceph-deploy:~$ rados -p mypool rm testfile #再次删除可以执行成功删除文件
#删除快照
cephadmin@ceph-deploy:~$ rados lssnap -p mypool
1 mypool-snap-202403121515 2024.03.12 15:15:39
2 mypool-snap-202403121516 2024.03.12 15:16:06
3 mypool-snapshot001 2024.03.12 15:18:03
3 snaps
cephadmin@ceph-deploy:~$ ceph osd pool rmsnap mypool mypool-snap-202403121515
removed pool mypool snap mypool-snap-202403121515
cephadmin@ceph-deploy:~$ ceph osd pool rmsnap mypool mypool-snap-202403121516
removed pool mypool snap mypool-snap-202403121516
cephadmin@ceph-deploy:~$ ceph osd pool rmsnap mypool mypool-snapshot001
removed pool mypool snap mypool-snapshot001
cephadmin@ceph-deploy:~$ rados lssnap -p mypool
0 snaps
6、存储压缩
#如果使用 bulestore 存储引擎，ceph 支持称为"实时数据压缩"即边压缩边保存数据的功能该功能有助于节省磁盘空间，
#可以在 BlueStore OSD 上创建的每个 Ceph 池上启用或禁用压缩，
#以节约磁盘空间，默认没有开启压缩，需要后期配置并开启。
#启用压缩并指定压缩算法
#snappy:该配置为指定压缩使用的算法默认为sanppy,还有 none、zlib、lz4、zstd 和 snappy等算法，zstd 压缩比好，
#但消耗 CPU，Iz4和 snappy 对 CPU 占用较低，不建议使用 zlib.
#指定压缩模式
root@ceph-node1:~# ceph daemon osd.2 config show | grep compression
"bluestore_compression_algorithm": "snappy", #默认算法是snappy
"bluestore_compression_max_blob_size": "0", #大于此的块在被压缩之前被分解成更小的 blob(块),此设置将覆盖 bluestore 压缩 max blob"的全局设置。
"bluestore_compression_max_blob_size_hdd": "65536", #默认64k
"bluestore_compression_max_blob_size_ssd": "65536", #默认64k
"bluestore_compression_min_blob_size": "0", #小于此的块不压缩，此设置将覆盖 bluestore 压缩 min blob *的全局配置
"bluestore_compression_min_blob_size_hdd": "8192", #默认8k
"bluestore_compression_min_blob_size_ssd": "8192", #默认8k
"bluestore_compression_mode": "none", #压缩模式
"bluestore_compression_required_ratio": "0.875000", #压缩后与压缩前的压缩比
"bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2,max_total_wal_size=1073741824",
"filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression",
"kstore_rocksdb_options": "compression=kNoCompression",
"leveldb_compression": "true",
"mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true",
"rbd_compression_hint": "none",
#修改算法
cephadmin@ceph-deploy:~$ ceph osd pool set mypool compression_algorithm snappy
set pool 2 compression_algorithm to snappy
cephadmin@ceph-deploy:~$ ceph osd pool get mypool compression_algorithm
compression_algorithm: snappy
#修改压缩模式
cephadmin@ceph-deploy:~$ ceph osd pool set mypool compression_mode passive
set pool 2 compression_mode to passive
cephadmin@ceph-deploy:~$ ceph osd pool get mypool compression_mode
compression_mode: passive
浙公网安备 33010602011771号