ubuntu安装ceph pacific

环境准备

# 准备6台机器，系统为Ubuntu 20.04
ceph1 mon ceph-deploy
ceph2 mon mgr
ceph3 mon mgr
ceph4 osd
ceph5 osd
ceph6 osd
# 配置hosts，在ceph1上操作
root@ceph1:~# cat /etc/hosts
192.168.1.20 ceph1
192.168.1.21 ceph2
192.168.1.22 ceph3
192.168.1.23 ceph4
192.168.1.24 ceph5
192.168.1.25 ceph6
# 免密钥登录，在ceph1上操作
root@ceph1:~# echo -e "\n"|ssh-keygen -t rsa -N ""
root@ceph1:~# ssh-copy-id ceph1
root@ceph1:~# ssh-copy-id ceph2
root@ceph1:~# ssh-copy-id ceph3
root@ceph1:~# ssh-copy-id ceph4
root@ceph1:~# ssh-copy-id ceph5
root@ceph1:~# ssh-copy-id ceph6
root@ceph1:~# scp /etc/hosts ceph2
root@ceph1:~# scp /etc/hosts ceph3
root@ceph1:~# scp /etc/hosts ceph4
root@ceph1:~# scp /etc/hosts ceph5
root@ceph1:~# scp /etc/hosts ceph6
# 配置源，在每台机器上操作
echo "deb http://mirrors.aliyun.com/ceph/debian-pacific/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/ceph.list
wget -q -O- 'https://download.ceph.com/keys/release.asc'| apt-key add -
apt update
# 安装ceph-deploy，在ceph1上操作
apt-cache madison ceph-deploy
apt -y install ceph-deploy
ceph-deploy --version
mkdir -p /data/ceph-cluster && cd /data/ceph-cluster

# 安装python3.6，在每台机器上操作
apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl gcc
mkdir -p /data/tools && cd /data/tools
wget https://www.python.org/ftp/python/3.6.9/Python-3.6.9.tar.xz
tar xf Python-3.6.9.tar.xz Python-3.6.9/
cd ./Python-3.6.9/
./configure && make all && make install
mv /usr/bin/python /usr/bin/python3.8.10
ln -s /usr/local/bin/python3.6 /usr/bin/python

# 初始化集群配置文件
ceph-deploy new --public-network 192.168.1.0/24 ceph1
# ceph-deploy new --public-network 192.168.1.0/24 --cluster-network 192.168.10.0/24 ceph1

# 安装monitor节点
# 远程安装，在ceph1上操作
ceph-deploy install --no-adjust-repos --nogpgcheck --mon ceph1 ceph2 ceph3
# 本地安装，在ceph1，ceph2，ceph3上操作
apt -y install ceph-mon
# 生成monitor配置文件
ceph-deploy --overwrite-conf mon create-initial

# 安装管理节点，在ceph1节点操作
apt -y install ceph-common
# 推送管理key到管理节点
ceph-deploy admin ceph1

# 查看ceph集群
root@ceph1:/data/ceph-cluster# ceph -s
  cluster:
    id:     07d59d84-b56b-4889-980d-56cffb09b471
    health: HEALTH_WARN
            mon is allowing insecure global_id reclaim
 
  services:
    mon: 1 daemons, quorum ceph1 (age 16m)
    mgr: no daemons active
    osd: 0 osds: 0 up, 0 in
 
  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   0 B used, 0 B / 0 B avail
    pgs:     

# 解决报错HEALTH_WARN
root@ceph1:/data/ceph-cluster# ceph config set mon auth_allow_insecure_global_id_reclaim false
root@ceph1:/data/ceph-cluster# ceph -s
  cluster:
    id:     07d59d84-b56b-4889-980d-56cffb09b471
    health: HEALTH_OK
 
  services:
    mon: 1 daemons, quorum ceph1 (age 17m)
    mgr: no daemons active
    osd: 0 osds: 0 up, 0 in
 
  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   0 B used, 0 B / 0 B avail
    pgs:     
 
# 安装mgr节点
# 可以远程安装，在ceph1上操作
ceph-deploy install --mgr ceph2 ceph3
# 本地安装，在ceph2，ceph3上操作
apt -y install ceph-mgr

# 安装osd节点
# 远程安装，在ceph1上操作
ceph-deploy install --release pacific --osd ceph4 ceph5 ceph6
# 本地安装，在ceph4，ceph5，ceph6上操作
apt -y install ceph-osd
# 查看包安装情况
dpkg -l|grep ceph

# 解决报错
root@ceph1:/data/ceph-cluster# sed -i 's#line.startswith(#line.startswith(b#g' /usr/lib/python3/dist-packages/ceph_deploy/osd.py
root@ceph1:/data/ceph-cluster# ceph-deploy disk list ceph4
# 添加osd，在ceph1上操作
ceph-deploy --overwrite-conf osd create ceph4 --data /dev/vdb
ceph-deploy --overwrite-conf osd create ceph4 --data /dev/vdc
ceph-deploy --overwrite-conf osd create ceph5 --data /dev/vdb
ceph-deploy --overwrite-conf osd create ceph5 --data /dev/vdc
ceph-deploy --overwrite-conf osd create ceph6 --data /dev/vdb
ceph-deploy --overwrite-conf osd delete ceph6 --data /dev/vdc

# 查看添加的osd状态，在ceph1节点操作
root@ceph1:/data/ceph-cluster# ceph osd tree
ID  CLASS  WEIGHT   TYPE NAME       STATUS  REWEIGHT  PRI-AFF
-1         0.58612  root default                             
-3         0.19537      host ceph4                           
 0    hdd  0.09769          osd.0       up   1.00000  1.00000
 1    hdd  0.09769          osd.1       up   1.00000  1.00000
-5         0.19537      host ceph5                           
 2    hdd  0.09769          osd.2       up   1.00000  1.00000
 3    hdd  0.09769          osd.3       up   1.00000  1.00000
-7         0.19537      host ceph6                           
 4    hdd  0.09769          osd.4       up   1.00000  1.00000
 5    hdd  0.09769          osd.5       up   1.00000  1.00000
root@ceph1:/data/ceph-cluster# ceph osd status
ID  HOST    USED  AVAIL  WR OPS  WR DATA  RD OPS  RD DATA  STATE      
 0  ceph4  6304k  99.9G      0        0       0        0   exists,up  
 1  ceph4  6240k  99.9G      0        0       0        0   exists,up  
 2  ceph5  6176k  99.9G      0        0       0        0   exists,up  
 3  ceph5  6172k  99.9G      0        0       0        0   exists,up  
 4  ceph6  5916k  99.9G      0        0       0        0   exists,up  
 5  ceph6  5852k  99.9G      0        0       0        0   exists,up

# Ubuntu删除snap卷
apt autoremove --purge snapd -y

# 添加mon节点，在ceph1上操作
ceph-deploy --overwrite-conf mon add ceph2 ceph3
# 添加ceph2，ceph3的ip，，在ceph1上操作
root@ceph1:/data/ceph-cluster# cat ceph.conf 
[global]
fsid = 07d59d84-b56b-4889-980d-56cffb09b471
mon_initial_members = ceph1
mon_host = 192.168.1.20,192.168.1.21,192.168.1.22
public_network = 192.168.1.0/24
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
#推送配置文件到其他节点，在ceph1上操作
root@ceph1:/data/ceph-cluster# ceph-deploy --overwrite-conf config push ceph{1,2,3,4,5,6}

# 添加mgr节点，在ceph1上操作
root@ceph1:/data/ceph-cluster# ceph-deploy mgr create ceph3