1. 官方文档
2. 基本命令
2.1 查看ceph使用状态
ceph df
2.2 指定mon查看状态
ceph -s --mon-host 192.168.88.10
2.3 指定配置文件查看状态
ceph -s -c /etc/ceph/ceph.conf
2.4 配置文件
2.4.1 cephadm集群文件目录
ll /var/lib/ceph/
# 注意修改配置了重启相关容器
2.4.2 集群日志文件
ll /var/log/ceph
2.4.3 开启debug
docker exec -it ceph-3104ac28-72bc-11ed-85c7-8d65cf0dc0c6-mon-ceph01 bash
# 自定义配置值
vim /etc/ceph/ceph.conf
'''
[global]
debug ms = 0
[osd]
debug ms = 1
[osd.1]
debug.ms = 10
[osd.2]
debug ms = 10
'''
3. 获取集群信息
3.1 显示整个集群的数据库信息
ceph config dump
3.2 显示某个守护进程的配置
- get值主要取决于 ceph config dump 输出中的 who 值
ceph config get mon
3.3 设置数据库
ceph config set mon.ceph01 debug_ms 5
3.4 显示ceph守护进程的详细信息
ceph config show osd.0
3.5 查看帮助
ceph config help mon_host
3.6 查看信息配置
ceph daemon osd.3 config help mon_host
ceph config help log_file -f json-pretty
3.7 临时修改选项值
ceph tell osd.1 config set debug_osd 30
ceph daemon mon.ceph01 config set debug_osd 30
3.8 查看当前节点所有选项值
ceph config show-with-defaults osd.0
4. ceph通用配置
4.1 mon
vim /etc/ceph/ceph.conf
[global]
fsid = 3104ac28-72bc-11ed-85c7-8d65cf0dc0c6
mon_host = [v2:192.168.88.10:3300/0,v1:192.168.88.10:6789/0] [v2:192.168.88.11:3300/0,v1:192.168.88.11:6789/0] [v2:192.168.88.12:3300/0,v1:192.168.88.12:6789/0]
4.2 osds
vim /etc/ceph/ceph.conf
# 定义osd日志大小
[global]
osd_journal_size = 10000
[osd.0]
host = [hostname] # manual deployments only
4.3 定义网络
[global]
public network = {public-network/netmask}
# 建议是内部网络
cluster network = {cluster-network/netmask}
5. 认证配置
5.1 安全协议
- 默认是配置好的,更改为None将没有认证功能,需要重启集群
[global]
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
5.2 证书导出
ceph auth get-or-create client.admin mon 'allow *' mds 'allow *' mgr 'allow *' osd 'allow *' -o ceph.client.admin.keyring
ceph auth get-or-create mgr.ceph01 mon 'allow profile mgr' mds 'allow *' osd 'allow *' -o mgr_keyring
ceph auth get-or-create osd.0 -o osd-4.keyring
ceph auth get-or-create mds.ceph01 mon 'allow rwx' osd 'allow *' mds 'allow *' mgr 'allow profile mds' -o mds.keyring
5.3 停用安全协议
[global]
auth_cluster_required = none
auth_service_required = none
auth_client_required = none
6. Bluestore
6.1 osd优化
- WAL设置适合SSD硬盘使用,主要预留日志,有一定程度提高ceph效率
- DB存储内部的元数据
ceph-volume lvm prepare --bluestore --data <device>
ceph-volume lvm prepare --bluestore --data <device> --block.wal <wal-device> --block.db <db-device>
vgcreate ceph-block-0 /dev/sda
vgcreate ceph-block-1 /dev/sdb
vgcreate ceph-block-2 /dev/sdc
vgcreate ceph-block-3 /dev/sdd
lvcreate -l 100%FREE -n block-0 ceph-block-0
lvcreate -l 100%FREE -n block-1 ceph-block-1
lvcreate -l 100%FREE -n block-2 ceph-block-2
lvcreate -l 100%FREE -n block-3 ceph-block-3
vgcreate ceph-db-0 /dev/sdx
lvcreate -L 50GB -n db-0 ceph-db-0
lvcreate -L 50GB -n db-0 ceph-db-0
lvcreate -L 50GB -n db-0 ceph-db-0
lvcreate -L 50GB -n db-0 ceph-db-0
ceph-volume lvm create --bluestore --data ceph-block-0/block-0 --block.db ceph-db-0/db-0
ceph-volume lvm create --bluestore --data ceph-block-0/block-1 --block.db ceph-db-0/db-1
ceph-volume lvm create --bluestore --data ceph-block-0/block-2 --block.db ceph-db-0/db-2
ceph-volume lvm create --bluestore --data ceph-block-0/block-3 --block.db ceph-db-0/db-3
7. ceph健康管理
7.1 关闭mon告警
ceph health detail
ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false
7.2 启动osd
cephadm deploy --name osd.2 --fsid 3104ac28-72bc-11ed-85c7-8d65cf0dc0c6 --osd-fsid 3104ac28-72bc-11ed-85c7-8d65cf0dc0c6
7.3 移除osd
ceph osd crush rm osd.<id>
# 停止osd
docker stop ceph-3104ac28-72bc-11ed-85c7-8d65cf0dc0c6-osd-2
# 踢出集群
ceph osd out 2
# 从CRUSH删除
ceph osd crush remove osd.2
# 删除osd集群密钥,从集群中移除该osd
ceph auth del osd.2
ceph osd rm 2
# 解除目录挂载(由于是容器化的,可以忽略),可以对磁盘进行格式化以及目录删除
cephadm ceph-volume lvm zap /dev/sdb --destroy
7.4 添加osd
ceph osd out osd.$N
while ! ceph osd safe-to-destroy osd.$N; do sleep 1m; done
ceph-volume lvm zap /path/to/device
ceph-volume lvm create --osd-id $N --data /path/to/device
7.5 设置osd阈值
ceph osd set-backfillfull-ratio <ratio>
ceph osd set-nearfull-ratio <ratio>
ceph osd set-full-ratio <ratio>
ceph osd dump | grep full_ratio
7.6 配置osd状态
full 群集被标记为已满,无法执行写入操作
pauserd,pausewr 暂停的读取或写入
noup 不允许启动OSD
nodown OSD故障报告被忽略,因此监视器不会将OSD标记为关闭
noin 以前标记的OSD在开始时不会被标记
noout 在配置的间隔之后,不会自动标记出关闭的OSD
nobackfill,norecover,norebalance 恢复或数据重新平衡已暂停
noscrub,nodeep_scrub 已禁用清理
notieragent 缓存分层活动已挂起
ceph osd set <flag>
ceph osd unset <flag>
7.7 详细osd信息
ceph df detail
7.8 存储池限额
# 最大对象数
ceph osd pool set-quota <poolname> max_objects <num-objects>
ceph osd pool set-quota <poolname> max_bytes <num-bytes>
7.9 修复osd
ceph-bluestore-tool repair --path /var/lib/ceph/3104ac28-72bc-11ed-85c7-8d65cf0dc0c6/osd.2/ceph_fsid
7.10 关闭bluestore告警
ceph config set global bluestore_warn_on_legacy_statfs false
7.11 pool缓存最大配置
# 目标最大缓存字节
ceph osd pool set <cache-pool-name> target_max_bytes <bytes>
# 目标最大缓存字节数
ceph osd pool set <cache-pool-name> target_max_objects <bytes>
7.12 pg计算公式
Total PGs = (osds * 100) / pool size
7.13 pool的pg设置
ceph osd pool get cephfs-data pg_num
ceph osd set <pool-name> pg_num <value>
ceph config set global mon_warn_on_pool_pg_num_not_power_of_two false
7.14 关闭pg数过多告警
ceph osd pool set <pool-name> pg_autoscale_mode off
7.15 允许集群自动调整pg数
ceph osd pool set <pool-name> pg_autoscale_mode on
7.16 初始化存储池
rbd pool init <poolname>
ceph osd pool application enable <poolname>
7.17 查看osd吞吐速率
docker exec -it ceph-3104ac28-72bc-11ed-85c7-8d65cf0dc0c6-mon-ceph01 bash
ceph daemon osd.5 ops
7.18 查看osd详细信息
ceph osd find osd.5
docker exec -it ceph-3104ac28-72bc-11ed-85c7-8d65cf0dc0c6-mon-ceph01 bash
ceph daemon osd.5 dump_historic_ops
7.19 关闭进程有异常的警告信息
ceph crash ls-new
# 设置开启
ceph config set mgr mgr/crash/warn_recent_interval 10
# 关闭
ceph config set mgr mgr/crash/warn_recent_interval 0
ceph config show mgr.ceph01.rrwcxs
8. 删除存储池
vim /var/lib/ceph/587b3aae-7ba6-11ed-91b7-4768cb2da4c8/mon.ceph01/config
mon_allow_pool_delete = true
# 重启mon进程
ceph osd pool delete rbd rbd --yes-i-really-really-mean-it
9.调整副本池
ceph osd pool get $pool size
ceph osd pool set $pool size 2
10.用户管理
10.1 创建用户
ceph auth add client.testuser mon 'allow r' osd 'allow rw pool=rbdpool'
ceph auth get-or-create client.testuser mon 'allow r' osd 'allow rw pool=rbdpool'
10.2 修改用户
ceph auth caps client.testuser mon 'allow rw' osd 'allow rw pool=rbdpool'
10.3 删除用户
ceph auth del client.testuser
11.dashboard操作
11.1 修改HTTPS端口号
ceph config set mgr mgr/dashboard/ssl_server_port 5050
11.2 关闭dashboard证书认证
ceph config set mgr mgr/dashboard/ssl false #关闭 SSL
11.3 指定 dashboard 监听地址
ceph config set mgr mgr/dashboard/server_addr 192.168.18.130
11.4 指定 dashboard 监听端口
ceph config set mgr mgr/dashboard/server_port 5050
11.5 重启dashboard模块生效
ceph mgr module disable dashboard
ceph mgr module enable dashboard
12. Grafana对接URL问题
vim /var/lib/ceph/1201424c-909d-11ed-8b70-cf1f91e7fda7/grafana.storage01/etc/grafana/grafana.ini
# This file is generated by cephadm.
[users]
default_theme = light
[auth.anonymous]
enabled = true
org_name = 'Main Org.'
org_role = 'Viewer'
[server]
domain = 'bootstrap.storage.lab'
protocol = http
http_port = 3000
http_addr =
[snapshots]
external_enabled = false
[security]
disable_initial_admin_creation = true
cookie_secure = true
cookie_samesite = none
allow_embedding = true
# 修改url
ceph dashboard set-grafana-api-url http://10.0.0.10:3000/
# 重启grafana容器
docker restart ceph-1201424c-909d-11ed-8b70-cf1f91e7fda7-grafana-storage01
13.调整ceph平衡
13.1 调整WEIGHT平衡
ceph osd reweight 5 0.7
ceph osd crush reweight osd.5 6.39999
14.rbd块设备锁处理
rbd lock ls volumes/volume-4cb6acc8-807e-42fb-b178-adfd1d7ce425
rbd lock rm volumes/volume-4cb6acc8-807e-42fb-b178-adfd1d7ce425 "auto 139945604373728" client.10499600
15. 单机版部署问题处理
15.1 ceph蓝图导出
# 安装工具
ceph-base
# 导出蓝图
ceph osd getcrushmap -o map.bin
# 转换蓝图为可读格式
crushtool -d map.bin -o map.txt
15.2 修改蓝图规则并导入
# 将规则host改为osd
vim map.txt
55 step chooseleaf firstn 0 type osd
# 转换为蓝图可识别的文件
crushtool -c map.txt -o map.bin.new
# 导入到集群
ceph osd setcrushmap -i map.bin.new