搭建redis集群高可用及宕机处理

1先获取所有主节点信息

[root@db03 ~]# redis-cli -c -h db01 -p 6381 cluster nodes  #先查看节点信息并保存
ddee11b9c4c6622bbf28497c53aa1eb84846084f 10.0.0.203:6381 slave 4c007a5e2d894aa4c6ce950467221f01fd50f645 0 1618327466372 1 connected
74bdf79d95556ffd38c47b01d0c76f393706bbda 10.0.0.202:6380 master - 0 1618327468387 5 connected 5462-10922
6139c63daaac449a5173a41ae3be3f9d86ed8fa8 10.0.0.201:6381 myself,slave 74bdf79d95556ffd38c47b01d0c76f393706bbda 0 0 2 connected
4c007a5e2d894aa4c6ce950467221f01fd50f645 10.0.0.201:6380 master - 0 1618327463345 1 connected 0-5461
070b91942844a34c1871642eef2d24bd3e816403 10.0.0.203:6380 master - 0 1618327467379 4 connected 10923-16383
2e616daa28ca6518b908148227edf562280a2d61 10.0.0.202:6381 slave 070b91942844a34c1871642eef2d24bd3e816403 0 1618327465362 4 connected
[root@db03 ~]# redis-cli -c -h db01 -p 6381 cluster nodes|grep -v "6381"  #删除6381
74bdf79d95556ffd38c47b01d0c76f393706bbda 10.0.0.202:6380 master - 0 1618327504683 5 connected 5462-10922
4c007a5e2d894aa4c6ce950467221f01fd50f645 10.0.0.201:6380 master - 0 1618327503675 1 connected 0-5461
070b91942844a34c1871642eef2d24bd3e816403 10.0.0.203:6380 master - 0 1618327504179 4 connected 10923-16383
[root@db03 ~]# redis-cli -c -h db01 -p 6381 cluster nodes|grep -v "6381"|awk '{print $1,$2}' #只取前2列数据
74bdf79d95556ffd38c47b01d0c76f393706bbda 10.0.0.202:6380
4c007a5e2d894aa4c6ce950467221f01fd50f645 10.0.0.201:6380
070b91942844a34c1871642eef2d24bd3e816403 10.0.0.203:6380
[root@db03 ~]# 

 

 2根据高可用图先在txt里把命令敲好

	db01里的6381复制的是db02里的6380
	db02里的6381复制的是db03里的6380
	db03里的6381复制的是db01里的6380
redis-cli -c -h db01 6381 cluster replicate 74bdf79d95556ffd38c47b01d0c76f393706bbda redis-cli -c -h db02 6381 cluster replicate 070b91942844a34c1871642eef2d24bd3e816403 redis-cli -c -h db03 6381 cluster replicate 4c007a5e2d894aa4c6ce950467221f01fd50f645

 3在从库中执行复制主库命令并检查状态（配置高可用）：

[root@db01 ~]# redis-cli -c -h db01 -p 6381 cluster replicate 74bdf79d95556ffd38c47b01d0c76f393706bbda
d50f645OK
[root@db01 ~]# redis-cli -c -h db02 -p 6381 cluster replicate 070b91942844a34c1871642eef2d24bd3e816403
OK
[root@db01 ~]# redis-cli -c -h db03 -p 6381 cluster replicate 4c007a5e2d894aa4c6ce950467221f01fd50f645
OK
[root@db01 ~]# sh redis_shell.sh login 6380
10.0.0.201:6380> cluster nodes
4c007a5e2d894aa4c6ce950467221f01fd50f645 10.0.0.201:6380 myself,master - 0 0 1 connected 0-5461
74bdf79d95556ffd38c47b01d0c76f393706bbda 10.0.0.202:6380 master - 0 1618328735307 5 connected 5462-10922
ddee11b9c4c6622bbf28497c53aa1eb84846084f 10.0.0.203:6381 slave 4c007a5e2d894aa4c6ce950467221f01fd50f645 0 1618328736315 1 connected
2e616daa28ca6518b908148227edf562280a2d61 10.0.0.202:6381 slave 070b91942844a34c1871642eef2d24bd3e816403 0 1618328733294 4 connected
070b91942844a34c1871642eef2d24bd3e816403 10.0.0.203:6380 master - 0 1618328734301 4 connected 10923-16383
6139c63daaac449a5173a41ae3be3f9d86ed8fa8 10.0.0.201:6381 slave 74bdf79d95556ffd38c47b01d0c76f393706bbda 0 1618328732286 5 connected
10.0.0.201:6380> cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:5
cluster_my_epoch:1
cluster_stats_messages_sent:24302
cluster_stats_messages_received:23760
10.0.0.201:6380> 

 4模拟db01的主库6380挂了，看看是否是高可用：

[root@db01 ~]# sh redis_shell.sh stop 6380
root        779      1  0 19:57 ?        00:00:32 /usr/local/bin/redis-server 127.0.0.1:6379
root       1009      1  0 20:33 ?        00:00:42 redis-server 10.0.0.201:6381 [cluster]
root       1177   1071  0 23:51 pts/0    00:00:00 sh redis_shell.sh stop 6380
root       1186   1177  0 23:51 pts/0    00:00:00 grep redis
[root@db01 ~]# redis-cli -c -h db01 -p 6381 cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:6
cluster_my_epoch:5
cluster_stats_messages_sent:26204
cluster_stats_messages_received:25411
[root@db01 ~]# sh redis_shell.sh tail 6381
1009:S 13 Apr 23:44:57.799 * Master replied to PING, replication can continue...
1009:S 13 Apr 23:44:57.801 * Partial resynchronization not possible (no cached master)
1009:S 13 Apr 23:44:57.804 * Full resync from master: 2deb732a092fb7fb43ba536f30f360dea85b8ea2:12559
1009:S 13 Apr 23:44:57.966 * MASTER <-> SLAVE sync: receiving 76 bytes from master
1009:S 13 Apr 23:44:57.966 * MASTER <-> SLAVE sync: Flushing old data
1009:S 13 Apr 23:44:57.966 * MASTER <-> SLAVE sync: Loading DB in memory
1009:S 13 Apr 23:44:57.967 * MASTER <-> SLAVE sync: Finished with success
1009:S 13 Apr 23:51:27.583 * FAIL message received from 070b91942844a34c1871642eef2d24bd3e816403 about 4c007a5e2d894aa4c6ce950467221f01fd50f645
1009:S 13 Apr 23:51:27.585 # Cluster state changed: fail
1009:S 13 Apr 23:51:28.455 # Cluster state changed: ok

[root@db01 ~]# redis-cli -c -h db01 -p 6381 cluster nodes
ddee11b9c4c6622bbf28497c53aa1eb84846084f 10.0.0.203:6381 master - 0 1618329321855 6 connected 0-5461  #从库变为了主库
74bdf79d95556ffd38c47b01d0c76f393706bbda 10.0.0.202:6380 master - 0 1618329322362 5 connected 5462-10922
6139c63daaac449a5173a41ae3be3f9d86ed8fa8 10.0.0.201:6381 myself,slave 74bdf79d95556ffd38c47b01d0c76f393706bbda 0 0 2 connected
4c007a5e2d894aa4c6ce950467221f01fd50f645 10.0.0.201:6380 master,fail - 1618329071574 1618329068021 1 disconnected #主库宕机
070b91942844a34c1871642eef2d24bd3e816403 10.0.0.203:6380 master - 0 1618329322865 4 connected 10923-16383
2e616daa28ca6518b908148227edf562280a2d61 10.0.0.202:6381 slave 070b91942844a34c1871642eef2d24bd3e816403 0 1618329320840 4 connected
[root@db01 ~]# redis-cli -c -h db01 6381 set k1 v1
Could not connect to Redis at db01:6379: Connection refused
[root@db01 ~]# redis-cli -c -h db01 -p 6381 set k1 v1
OK
[root@db01 ~]# redis-cli -c -h db01 -p 6381 set k2 v2
OK
[root@db01 ~]#

 5修复宕机主库后发现db01的6380变成了从库：

[root@db01 ~]# sh redis_shell.sh start 6380 #修复宕机
root        779      1  0 4月13 ?       00:00:33 /usr/local/bin/redis-server 127.0.0.1:6379
root       1009      1  0 4月13 ?       00:00:47 redis-server 10.0.0.201:6381 [cluster]
root       1219   1071  0 00:01 pts/0    00:00:00 sh redis_shell.sh start 6380
root       1227      1  0 00:01 ?        00:00:00 redis-server 10.0.0.201:6380 [cluster]
root       1229   1219  0 00:01 pts/0    00:00:00 grep redis
[root@db01 ~]# redis-cli -c -h db01 -p 6381 cluster info #查看状态
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:6
cluster_my_epoch:5
cluster_stats_messages_sent:33245
cluster_stats_messages_received:26719
[root@db01 ~]# redis-cli -c -h db01 -p 6381 cluster nodes
ddee11b9c4c6622bbf28497c53aa1eb84846084f 10.0.0.203:6381 master - 0 1618329888815 6 connected 0-5461
74bdf79d95556ffd38c47b01d0c76f393706bbda 10.0.0.202:6380 master - 0 1618329890836 5 connected 5462-10922
6139c63daaac449a5173a41ae3be3f9d86ed8fa8 10.0.0.201:6381 myself,slave 74bdf79d95556ffd38c47b01d0c76f393706bbda 0 0 2 connected
4c007a5e2d894aa4c6ce950467221f01fd50f645 10.0.0.201:6380 slave ddee11b9c4c6622bbf28497c53aa1eb84846084f 0 1618329889821 6 connected
070b91942844a34c1871642eef2d24bd3e816403 10.0.0.203:6380 master - 0 1618329885787 4 connected 10923-16383
2e616daa28ca6518b908148227edf562280a2d61 10.0.0.202:6381 slave 070b91942844a34c1871642eef2d24bd3e816403 0 1618329887807 4 connected
[root@db01 ~]# 

 6我想恢复到之前的状态，还是把db01的6380变成主库，把db03中的6381变成从库（db03中操作）：

[root@db03 ~]# sh redis_shell.sh login 6381
10.0.0.203:6381> cluster nodes
ddee11b9c4c6622bbf28497c53aa1eb84846084f 10.0.0.203:6381 myself,master - 0 0 6 connected 0-5461
070b91942844a34c1871642eef2d24bd3e816403 10.0.0.203:6380 master - 0 1618126051084 4 connected 10923-16383
4c007a5e2d894aa4c6ce950467221f01fd50f645 10.0.0.201:6380 slave ddee11b9c4c6622bbf28497c53aa1eb84846084f 0 1618126047056 6 connected
74bdf79d95556ffd38c47b01d0c76f393706bbda 10.0.0.202:6380 master - 0 1618126049074 5 connected 5462-10922
2e616daa28ca6518b908148227edf562280a2d61 10.0.0.202:6381 slave 070b91942844a34c1871642eef2d24bd3e816403 0 1618126045042 4 connected
6139c63daaac449a5173a41ae3be3f9d86ed8fa8 10.0.0.201:6381 slave 74bdf79d95556ffd38c47b01d0c76f393706bbda 0 1618126050078 5 connected
10.0.0.203:6381> cluster failover
(error) ERR You should send CLUSTER FAILOVER to a slave  #你应该在从库中执行这个命令
10.0.0.203:6381> 

[root@db01 ~]# redis-cli -c -h db01 -p 6380
db01:6380> cluster failover
OK
db01:6380> cluster nodes
74bdf79d95556ffd38c47b01d0c76f393706bbda 10.0.0.202:6380 master - 0 1618331006574 5 connected 5462-10922
2e616daa28ca6518b908148227edf562280a2d61 10.0.0.202:6381 slave 070b91942844a34c1871642eef2d24bd3e816403 0 1618331009593 4 connected
4c007a5e2d894aa4c6ce950467221f01fd50f645 10.0.0.201:6380 myself,master - 0 0 7 connected 0-5461 #变成主库了
6139c63daaac449a5173a41ae3be3f9d86ed8fa8 10.0.0.201:6381 slave 74bdf79d95556ffd38c47b01d0c76f393706bbda 0 1618331011607 5 connected
070b91942844a34c1871642eef2d24bd3e816403 10.0.0.203:6380 master - 0 1618331010601 4 connected 10923-16383
ddee11b9c4c6622bbf28497c53aa1eb84846084f 10.0.0.203:6381 slave 4c007a5e2d894aa4c6ce950467221f01fd50f645 0 1618331008587 7 connected
db01:6380>