银河麒麟 V10 安装 ceph reef 集群

背景

操作系统： Kylin Linux Advanced Server V10 (Lance)

IP	主机名	OSD 设备
ceph-01	192.168.10.151	/dev/xvdb
ceph-02	192.168.10.152	/dev/xvdb
ceph-03	192.168.10.153	/dev/xvdb

前期准备(所有节点都要操作)

# 设置主机名，这里以 ceph-01 为例
hostnamectl set-hostname ceph-01

# 配置 hosts
cat << EOF >> /etc/hosts
192.168.10.151 ceph-01
192.168.10.152 ceph-02
192.168.10.153 ceph-03
EOF

cat /etc/hosts

# 安装 docker 
rpm -e podman-1:0.10.1-8.ky10.x86_64
rpm -e docker-runc-1.0.0.rc3-203.p01.ky10.x86_64

mkdir -p /data/software/
cd /data/software/

## 把 docker-23.0.4.tgz 上传到这里
tar -xvf docker-23.0.4.tgz
cp docker/* /usr/bin/
  
cat >> /lib/systemd/system/docker.service << EOF
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target
  
[Service]
Type=notify
ExecStart=/usr/bin/dockerd  
ExecReload=/bin/kill -s HUP
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s
  
[Install]
WantedBy=multi-user.target
EOF
  
chmod +x /lib/systemd/system/docker.service
  
systemctl daemon-reload
systemctl start docker
systemctl enable docker
systemctl status docker
docker -v
 
 
# 创建 docker 配置
mkdir -p /etc/docker
 
cat >/etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "registry-mirrors": [
    "https://docker.xuanyuan.me",
    "https://docker.1ms.run",  
    "https://dockerproxy.com",
    "https://hub.rat.dev" 
  ],
  "max-concurrent-downloads": 10,
  "log-driver": "json-file",
  "log-level": "warn",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
    },
  "data-root": "/var/lib/docker"
}
EOF
 
# 重启 docker
systemctl restart docker

# ceph-01 会使用到下面这些镜像，可提前下载
quay.io/ceph/ceph:v18
quay.io/ceph/ceph-grafana:9.4.7
quay.io/prometheus/prometheus:v2.43.0
quay.io/prometheus/alertmanager:v0.25.0
quay.io/prometheus/node-exporter:v1.5.0

# ceph-02，ceph-03 会使用到下面这些镜像，可提前下载
quay.io/ceph/ceph:v18
quay.io/prometheus/node-exporter:v1.5.0   

部署 ceph 集群(ceph-01 上操作)

# 下载 cephadm 命令
CEPH_RELEASE=18.2.0 
curl --silent --remote-name --location https://download.ceph.com/rpm-${CEPH_RELEASE}/el9/noarch/cephadm
chmod +x cephadm
mv cephadm /usr/local/bin/

# 初始化集群
cephadm bootstrap --mon-ip 192.168.10.151

# 初始化成功后，会出现下面类似信息，可登录 Dashboard 修改密码
Ceph Dashboard is now available at:

             URL: https://ceph-01:8443/
            User: admin
        Password: hx2035ge4n

# 把密钥传到其他两台服务器上
ssh-copy-id -f -i /etc/ceph/ceph.pub ceph-02
ssh-copy-id -f -i /etc/ceph/ceph.pub ceph-03

# 添加主机
cephadm shell

ceph orch host add ceph-02 192.168.10.152
ceph orch host add ceph-03 192.168.10.153

# 添加块设备
ceph orch daemon add osd ceph-01:/dev/xvdb
ceph orch daemon add osd ceph-02:/dev/xvdb
ceph orch daemon add osd ceph-03:/dev/xvdb

在 Dashboard 查看

常用命令

# 查看主机
ceph orch host ls

# 查看由 Ceph Orchestrator 管理的所有服务的状态
ceph orch ps

# 列出集群中所有可用的存储设备及其状态
ceph orch device ls

# 列出集群中所有的存储池（pools）
ceph osd lspools

# 查看配置
ceph config dump