DRBD + keepalived实现文件实时同步和双机热备
DRBD + keepalived实现文件实时同步和双机热备
安装DRBD
系统初始化设置
注意： 需要有数据盘或者多的分区
yum update -y
关闭防火墙
systemctl stop firewalld
systemctl disable firewalld
修改host文件
vim /etc/hosts
192.168.1.240 Primary kylin-01
192.168.1.241 Secondary kylin-02
关闭SELINUX
vim /etc/sysconfig/selinux
SELINUX=disabled
安装依赖
yum install gcc libxslt-devel libxslt perl keyutils-libs-devel net-tools -y
下载源码编译安装
drbd需要两个安装报：drbd drbd-utils
drbd 安装
wget https://pkg.linbit.com//downloads/drbd/9/drbd-9.2.8.tar.gz
tar -zxvf drbd-9.2.8.tar.gz
cd drbd-9.2.8
make && make install
drbd-utils 安装
wget https://pkg.linbit.com//downloads/drbd/utils/drbd-utils-9.27.0.tar.gz
tar -zxvf drbd-utils-9.27.0.tar.gz
cd drbd-utils-9.27.0
./configure --prefix=/usr/local/drbd --without-83support --with-udev --with-initscripttype=systemd --without-manual
make && make install
安装配置路径：/usr/local/drbd/etc/drbd.d
安装路径： /usr/sbin/drbdsetup /usr/sbin/drbdmeta /usr/sbin/drbdadm
配置drbd
磁盘分区 此处不要格式化磁盘
fdisk /dev/sdb
全局配置
global_common.conf内容如下：
# DRBD is the result of over a decade of development by LINBIT.
# In case you need professional services for DRBD or have
# feature requests visit http://www.linbit.com
global {
usage-count yes;
# Decide what kind of udev symlinks you want for "implicit" volumes
# (those without explicit volume <vnr> {} block, implied vnr=0):
# /dev/drbd/by-resource/<resource>/<vnr> (explicit volumes)
# /dev/drbd/by-resource/<resource> (default for implict)
udev-always-use-vnr; # treat implicit the same as explicit volumes
# minor-count dialog-refresh disable-ip-verification
# cmd-timeout-short 5; cmd-timeout-medium 121; cmd-timeout-long 600;
}
common {
handlers {
# These are EXAMPLE handlers only.
# They may have severe implications,
# like hard resetting the node under certain circumstances.
# Be careful when choosing your poison.
# IMPORTANT: most of the following scripts symlink to "notify.sh" which tries to send mail via "mail".
# If you intend to use this notify.sh script make sure that "mail" is installed.
#
pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";
# fence-peer "/usr/lib/drbd/crm-fence-peer.sh";
# split-brain "/usr/lib/drbd/notify-split-brain.sh root";
# out-of-sync "/usr/lib/drbd/notify-out-of-sync.sh root";
# before-resync-target "/usr/lib/drbd/snapshot-resync-target-lvm.sh -p 15 -- -c 16k";
# after-resync-target /usr/lib/drbd/unsnapshot-resync-target-lvm.sh;
# quorum-lost "/usr/lib/drbd/notify-quorum-lost.sh root";
# disconnected /bin/true;
}
startup {
# wfc-timeout degr-wfc-timeout outdated-wfc-timeout wait-after-sb
}
options {
# cpu-mask on-no-data-accessible
# RECOMMENDED for three or more storage nodes with DRBD 9:
# quorum majority;
# on-no-quorum suspend-io | io-error;
}
disk {
on-io-error detach; #配置I/O错误处理策略为分离
# size on-io-error fencing disk-barrier disk-flushes
# disk-drain md-flushes resync-rate resync-after al-extents
# c-plan-ahead c-delay-target c-fill-target c-max-rate
# c-min-rate disk-timeout
}
net {
# protocol timeout max-epoch-size max-buffers
# connect-int ping-int sndbuf-size rcvbuf-size ko-count
# allow-two-primaries cram-hmac-alg shared-secret after-sb-0pri
# after-sb-1pri after-sb-2pri always-asbp rr-conflict
# ping-timeout data-integrity-alg tcp-cork on-congestion
# congestion-fill congestion-extents csums-alg verify-alg
# use-rle
}
}
资源配置
在node1、node2 上分别建立drbd.res：
drbd.d目录下新建资源文件（drbd.res）
vim /usr/local/drbd/etc/drbd.d/drbd.res
resource r1 { #这个r1是定义资源的名字
protocol C;
on kylin-01 { #on开头，后面是主机名称
device /dev/drbd0; #drbd设备名称
disk /dev/sdb1; #drbd0使用的磁盘分区为sdb1
address 192.168.1.240:7789; #设置drbd监听地址与端口
meta-disk internal;
}
on kylin-02 { #on开头，后面是主机名称
device /dev/drbd0; #drbd设备名称
disk /dev/sdb1; #drbd0使用的磁盘分区为sdb1
address 192.168.1.241:7789; #设置drbd监听地址与端口
meta-disk internal;
}
}
在node1、node2 上初始化资源、启动drbd：
[root@kylin-01 drbd.d]# drbdadm create-md r1
initializing activity log
initializing bitmap (640 KB) to all zero
Writing meta data...
New drbd meta data block successfully created.
#启动 两个节点需要同时启动才生效
[root@kylin-01 drbd.d]# systemctl start drbd
# 开机启动
[root@kylin-01 drbd.d]# systemctl enable drbd
# 查看状态
[root@kylin-01 drbd.d]# systemctl status drbd
# 查看状态
netstat -anput|grep 7789
tcp 0 0 192.168.1.240:33015 192.168.1.241:7789 ESTABLISHED -
tcp 0 0 192.168.1.240:40897 192.168.1.241:7789 ESTABLISHED -
检查资源状态：
#查看节点角色状态：
drbdadm role r1
注：第一次启动drbd时，两个drbd节点默认都处于Secondary状态
主节点执行设置主节点：
#初始化资源
drbdadm primary --force r1
#查看资源状态
drbdadm status r1
r1 role:Primary
disk:UpToDate
kylin-02 role:Secondary
replication:SyncSource peer-disk:Inconsistent done:6.16
#查看同步状态
cat /proc/drbd
version: 9.2.8 (api:2/proto:86-122)
GIT-hash: e163b05a76254c0f51f999970e861d72bb16409a build by root@kylin-01, 2024-04-08 15:26:56
Transports (api:20): tcp (9.2.8)
会把主机上的数据传到备机，开始会显示同步进度，过一会显示状态都是”实时”,表示数据同步完成了。接下来就可以使用DRBD了。
DRBD使用（测试）
你现在可以把主机上的DRBD设备挂载到一个目录上进行使用，备机的DRBD设备无法被挂载,因为它是用来接收主机数据的,由DRBD负责操作.
格式化文件系统（文件格式根据自己的系统环境选择）
mkfs.ext4 /dev/drbd0
挂载此文件系统
mkdir /data
mount /dev/drbd0 /data/
在挂载data目录中创建一个测试文件，然后卸载挂载目录，然后切换主备节点，在备用节点上查看刚刚建立的测试文件还是否存在
node1:
[root@node1 ~]# mkdir /data/test
将node1变为备用节点
[root@node1 ~]# umount /data/
[root@node1 ~]# drbdadm secondary r1
[root@node1 ~]# drbdadm role r1
Secondary
node2:
将node2变为主节点
[root@node2 ~]# drbdadm primary r1
[root@node2 ~]# drbdadm role r1
Primary/Secondary
挂载设备，然后看文件是否存在
[root@node2 ~]# mount /dev/drbd0 /data
[root@node2 ~]# cd /data/
[root@node2 mnt]# ls
test
OK! 到这里已经算是完成了！
同样，在Node2上建立文件，然后
- 卸载/mnt/：umount /mnt/
- 将Node2降级成备用节点：drbdadm secondary r1
- 在Node1上升级为主机节点：drbdadm primary r1
- 在Node1上挂载：mount /dev/drbd0 /data
会发现Node2上的文件也同步到了Node1上。
问题
1、umount时，如果提示device is busy，使用下面方法解决：
fuser -m /data
显示：/data: 25023c
然后kill -9 25023 即可
2、"Split-Brain"（脑裂）的情况：
假设把Primary主机的的eth0设备宕掉，然后直接在Secondary主机上进行提权升级为DRBD的主节点，并且mount挂载DRBD，这时会发现之前在Primary主机上写入的数据文件确实同步过来了。
接着再把Primary主机的eth0设备恢复，看看有没有自动恢复 主从关系。经过查看，发现DRBD检测出了Split-Brain的状况，也就是两个节点都处于standalone状态，
故障描述如下：Split-Brain detected,dropping connection! 这就是传说中的“脑裂”。
DRBD官方推荐的手动恢复方案：
- Secondary主机上的操作
drbdadm secondary r0
drbdadm disconnect all
drbdadm --discard-my-data connect r0 //或者"drbdadm -- --discard-my-data connect r0"
- Primary主机上的操作
drbdadm disconnect all
drbdadm connect r0
drbdsetup /dev/drbd0 primary
- 检查drdb状态
[root@kylin-01 ~]# drbdadm status r1
r1 role:Primary
disk:UpToDate
kylin-02 role:Secondary
peer-disk:UpToDate
安装keepalived
直接采用yum安装
yum install -y keepalived
查看keepalived版本
[root@kylin-02 keepalived]# keepalived -v
Keepalived v2.0.20 (01/22,2020)
Copyright(C) 2001-2020 Alexandre Cassen, <acassen@gmail.com>
Built with kernel headers for Linux 4.19.90
Running on Linux 4.19.90-52.25.v2207.ky10.x86_64 #1 SMP Fri Jun 2 12:05:28 CST 2023
..........
更改keepalived配置
注：本处采用pgsql测试两个机器数据同步
pg docker-compose的文件(docker的配置安装请自行百度)
version: "3"
services:
postgresql:
image: postgres:11.8
container_name: postgres
hostname: postgres
ports:
- "5432:5432"
volumes:
- "/data/pgsql:/var/lib/postgresql/data"
- "/etc/localtime:/etc/localtime"
restart: on-failure
logging:
driver: "json-file"
options:
tag: postgres
cap_add:
- ALL
environment:
POSTGRES_USER: "root"
POSTGRES_PASSWORD: "123456"
ALLOW_IP_RANGE: "0.0.0.0/0"
cd /etc/keepalived
vim keepalived.conf
主节点keepalived.conf 文件内容如下：
注意：再执行stop脚本时不能直接执行，需要采用脚本调用脚本的方式执行，否则执行不完就会被kill掉（原因暂时没弄清楚）
! Configuration File for keepalived
global_defs {
#notification_email {
# acassen@firewall.loc
# failover@firewall.loc
# sysadmin@firewall.loc
#}
#notification_email_from Alexandre.Cassen@firewall.loc
#smtp_server 192.168.200.1
#smtp_connect_timeout 30
router_id kylin-02 # 节点标识，主机名
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state BACKUP
interface ens32 # 网卡
virtual_router_id 51
priority 100 # 节点权重，主节点100 备节点小于100，数字越大优先级越高
mcast_src_ip 192.168.1.240 # 本机IP
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.1.239 # 绑定的虚拟IP
}
notify_master "/etc/keepalived/notify.sh" # 节点为master时执行脚本
notify_backup "/etc/keepalived/notify_back.sh" # 切换为备节点时执行脚本
notify_stop "/etc/keepalived/notify_back.sh" # stop keepalived时执行的脚本
}
备节点keepalived.conf 文件内容如下：
! Configuration File for keepalived
global_defs {
#notification_email {
# acassen@firewall.loc
# failover@firewall.loc
# sysadmin@firewall.loc
#}
#notification_email_from Alexandre.Cassen@firewall.loc
#smtp_server 192.168.200.1
#smtp_connect_timeout 30
router_id kylin-02 # 节点标识，主机名
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state BACKUP
interface ens32
virtual_router_id 51
priority 99 # 节点权重，主节点100 备节点小于100，数字越大优先级越高
mcast_src_ip 192.168.1.241 # 本机IP
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.1.239
}
notify_master "/etc/keepalived/notify.sh"
notify_backup "/etc/keepalived/notify_back.sh"
notify_stop "/etc/keepalived/notify_back.sh"
}
notify.sh
#!/bin/bash
drbdadm primary r1
while true
do
drdbs=$(drbdadm role r1)
echo "drbd status is $drdbs"
if [[ "$drdbs" == "Primary" ]];then
break
else
drbdadm primary r1
sleep 3
fi
done
mount /dev/drbd0 /data
docker-compose -f /opt/pgsql/docker-compose.yml up -d
stop.sh
#!/bin/bash
docker stop postgres
umount /data/
drbdadm secondary r1
while true
do
drdbs=$(drbdadm role r1)
echo "drbd status is $drdbs"
if [[ "$drdbs"=="Secondary" ]];then
break
else
drbdadm secondary r1
sleep 3
fi
done
notify_back.sh
#!/bin/bash
/etc/keepalived/stop.sh
启动keepalived
systemctl start keepalived
开机启动
systemctl enable keepalived
验证
在数据库里面执行创建删除，切换keepalived节点后查看数据是否同步
