Ceph分布式存储系统

环境

节点	IP	主机名
node1	172.129.33.31	ceph-node1
node2	172.129.33.32	ceph-node2
node3	172.129.33.34	ceph-node3

 

Ceph分布式存储系统

配置映射

# cat /etc/hosts
172.129.33.31 ceph-node1
172.129.33.32 ceph-node2
172.129.33.34 ceph-node3

 

配置源

[ceph]
name=mariadb
baseurl=http://172.25.253.3/colony/ceph/
gpgcheck=0
enabled1=1

[centos]
name=centos
baseurl=http://172.25.253.3/centos/
gpgcheck=0
enabled=1

 

node1节点

# mkdir /etc/ceph  
# cd /etc/ceph/
# ceph-deploy new ceph-node1
# ceph-deploy install ceph-node1 ceph-node2 ceph-node3
# ceph-deploy --overwrite-conf mon create-initial
# ceph -s
     health HEALTH_ERR

 

创建osd

# lsblk 
NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sda      8:0    0  100G  0 disk 
└─sda1   8:1    0  100G  0 part /
sdb      8:16   0   50G  0 disk 

# parted /dev/sdb
(parted) mklabel                                                          
New disk label type? gpt       *gpt分区                                                                                             
(parted) mkpart         *分区所占比例                                              
Partition name?  []?                                                      
File system type?  [ext2]?                                                
Start? 0%                                                                 
End? 100%                                                                 
# mkfs.xfs /dev/sdb1
# mkdir /opt/osd1
# mount /dev/sdb1 /opt/osd1/
# chmod 777 /opt/osd1/

# cd /etc/ceph/
# ceph-deploy osd prepare ceph-node1:/opt/osd1 ceph-node2:/opt/osd2 ceph-node3:/opt/osd3
# chmod 777 /opt/osd1/*

# cd /etc/ceph/
# ceph-deploy osd activate ceph-node1:/opt/osd1/ ceph-node2:/opt/osd2 ceph-node3:/opt/osd3
# ceph -s
     health HEALTH_OK
# ceph-deploy admin ceph-node{1,2,3}

 

 

Ceph集群运维命令

① 检查 Ceph 的安装状态。
# ceph status
② 观察集群的健康状况。
# ceph w
③ 检查 Ceph monitor 仲裁状态。
# ceph quorum_status --format json-pretty
④ 导出 Ceph monitor 信息。
# ceph mon dump
⑤ 检查集群使用状态。
# ceph df
⑥ 检查 Ceph Monitor、OSD 和 PG（配置组）状态。
# ceph mon stat
# ceph osd stat
# ceph pg stat
⑦ 列表 PG。
# ceph pg dump
⑧ 列表 Ceph 存储池。
# ceph osd lspools
⑨ 检查 OSD 的 CRUSH。
# ceph osd tree
⑩ 列表集群的认证密钥。93
# ceph auth list

 

ceph客户端使用

添加一个客户端 172.129.33.37 ceph-client

node1

# ceph-deploy install ceph-client
# ceph-deploy admin ceph-client

 

ceph-client

# rbd create foo --size 4096 -m 172.129.33.31 -k /etc/ceph/ceph.client.admin.keyring 
# rbd map foo --name client.admin -m 172.129.33.31 -k /etc/ceph/ceph.client.admin.keyring 
/dev/rbd0
# lsblk 
NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sda      8:0    0  100G  0 disk 
└─sda1   8:1    0  100G  0 part /
rbd0   253:0    0    4G  0 disk 
# mkfs.ext4 /dev/rbd0
# mount /dev/rbd0 /mnt/
# df -h
/dev/rbd0       3.9G   16M  3.6G   1% /mnt

• 查看foo大小、调整foo的大小、使用resize2fs命令调整文件系统大小并查看

# rbd info foo
rbd image 'foo':
        size 4096 MB in 1024 objects
        order 22 (4096 kB objects)
        block_name_prefix: rb.0.1019.238e1f29
        format: 1
# rbd resize --size 8191 foo
Resizing image: 100% complete...done.
# rbd info foo              
rbd image 'foo':
        size 8191 MB in 2048 objects
        order 22 (4096 kB objects)
        block_name_prefix: rb.0.1019.238e1f29
        format: 1
# resize2fs /dev/rbd0
resize2fs 1.42.9 (28-Dec-2013)
Filesystem at /dev/rbd0 is mounted on /mnt; on-line resizing required
old_desc_blocks = 1, new_desc_blocks = 1
The filesystem on /dev/rbd0 is now 2096896 blocks long.

•  删除块设备

# umount /mnt/
# rbd rm foo
Removing image: 100% complete...done.

•  解决拉入黑名单

# rbd status foo
Watchers:
        watcher=172.129.33.37:0/267871751 client.4125 cookie=18446462598732840961
# ceph osd  blacklist add 172.129.33.37:0/267871751
blacklisting 172.129.33.37:0/267871751 until 2021-06-11 10:45:06.506103 (3600 sec)
# rbd rm foo
Removing image: 100% complete...done.