mysql+es+fastdfs+MongoDB的备份及灾备

rsync 的配置

一、服务端：
1.安装、
cd  /home/rongda
mkdir rsync

[root@rongda rongda]# ls rsync/
log  rsync.passwd  run  tool 

yum  localinstall rsync  -y   【默认安装的是rsync-3.1.2-4.el7.x86_64版本的】

2.配置

[root@backup ~]# vim /etc/rsyncd.conf
uid = rsync
gid = rsync
port = 873
fake super = yes
use chroot = no
max connections = 200
timeout = 600
ignore errors
read only = false
list = false
hosts allow = 0.0.0.0/0
auth users = rsync_backup
secrets file = /home/rongda/rsync/rsync.passwd
log file = /home/rongda/rsync/log/rsyncd.log
pid file = /home/rongda/rsync/run/rsyncd.pid  
lock file = /home/rongda/rsync/run/rsyncd.lock

#####################################
[backup]
path = /backup


#3.初始化的操作
    3.1）创建Rsync进程运行的用户  （ rsync ）
        [root@backup ~]# useradd -M -s /sbin/nologin rsync
        [root@backup ~]# id rsync
        uid=1008(rsync) gid=1008(rsync) 组=1008(rsync)
    3.2) 定义虚拟用户名rsync_bakcup的密码.
        mkdir /home/rongda/rsync/
        echo "rsync_backup:123456" > /home/rongda/rsync/rsync.passwd  
        chmod 600  /home/rongda/rsync/rsync.passwd 

    3.3) 创建存储的数据的目录 /backup
        [root@backup ~]# mkdir /backup
        [root@backup ~]# chown -R rsync.rsync /backup/
    
    3.4)创建启动时的日志目录及存放进程号的目录
        mkdir /home/rongda/rsync/log/
        mkdir /home/rongda/rsync/run
    
#4.启动并检测
[root@backup ~]# systemctl start rsyncd
[root@backup ~]# systemctl enable rsyncd
[root@backup ~]# systemctl status rsyncd
[root@backup ~]#  netstat -lntp  |grep 873

#5.客户端可进行一波测试
无需安装此服务
mkdir  /home/rongda/rsync/
echo "123456" > /home/rongda/rsync/rsync.passwd  
chmod 600  /home/rongda/rsync/rsync.passwd

[root@nfs ~]# rsync  -avz --password-file=/home/rongda/rsync/rsync.passwd  $Path  rsync_backup@172.16.10.65::backup

二、客户端上传并推送数据


1.编写脚本:备份的脚本  vim scripts.sh  

#!/usr/bin/bash
#0.将目录需要用的IP，时间，定义为变量
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin
Addr=$(hostname  -I |awk  '{print $1}')
Time=$(date +%F)
Path=/backup
DirName="${Addr}_${Time}"
current_time=`date +%F-%H-%M`

#准备一个目录
mkdir -p "$Path/$DirName.full"

#将文件备份到该目录中-mongodb
/home/rongda/mongodb/bin/mongodump -h 192.168.254.101:27017  --authenticationDatabase admin  -o $Path/$DirName.full/mongodb  > /dev/null 2>&1
cp -rp  /home/rongda/mongodb/data/  $Path/$DirName.full/mongodb/ > /dev/null 2>&1

#备份mysql的数据
mkdir  $Path/$DirName.full/mysql/
#/usr/bin/mysqldump -uib_test -piB_test2019  ib_test >>$Path/$DirName.full/mysql/backupdb.sql > /dev/null 2>&1
/usr/bin/mysqldump -uroot -pGotoipo20\!9 ib_test >>$Path/$DirName.full/mysql/ib_test.sql


#备份fastdfs的数据
#cp -rp   /home/rongda/fastdfs/file $Path/$DirName.full/fastdfs/  > /dev/null 2>&1
mkdir $Path/$DirName.full/fastdfs/ -p
cp -rp /home/rongda/fastdfs/file  $Path/$DirName.full/fastdfs/

#备份es的数据
cd  /home/rongda/backup/es/
/bin/sh  esdump.sh && \
mkdir $Path/$DirName.full/es/  && \
cp  *.tar.gz   $Path/$DirName.full/es/ > /dev/null 2>&1 && \
rm -fr  /home/rongda/backup/es/esdump-$current_time*.tar.gz   && \
rm -fr /home/rongda/backup/es/esdump-$current_time  && \
cp -rp   /home/rongda/elasticsearch-2.4.5/data   $Path/$DirName.full/es/ > /dev/null 2>&1

#使用rsync命令进行远程传输
#rsync  -avz --password-file=/home/rongda/rsync/rsync.passwd  $Path/$DirName.full  rsync_backup@192.168.254.102::backup


2.推送的脚本
[root@rongda rongda]# cat scripts_push.sh 
#!/usr/bin/bash
#0.将目录需要用的IP，时间，定义为变量
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin
Addr=$(hostname  -I |awk  '{print $1}')
Time=$(date +%F)
Path=/backup
DirName="${Addr}_${Time}"
current_time=`date +%F-%H-%M`

#1.我们要准备一个目录
mkdir -p  $Path/check

#2.将文件目录备份到该目录下
tar   czf  $Path/$DirName.full.tar.gz    $Path/$DirName.full && \
md5sum $Path/$DirName.full.tar.gz  >   $Path/check/check_flag_$Time
rm -fr $Path/$DirName.full


#3.将文件进行传输使用rsync命令进行远程传输
rsync  -avz --password-file=/home/rongda/rsync/rsync.passwd  $Path/$DirName.full.tar.gz   rsync_backup@192.168.254.102::backup

rsync  -avz --password-file=/home/rongda/rsync/rsync.passwd  $Path/check/check_flag_$Time    rsync_backup@192.168.254.102::backup


3.编写定时任务：
[root@nfs backup]# crontab -l
#backup
* 01 * * * /usr/bin/sh /home/rongda/scripts.sh  &>/dev/null

#push
* 02 * * * /usr/bin/sh /home/rongda/scripts_push.sh   &>/dev/null



三、服务端检查客户端传过来的数据是否是完整的
[root@rongda rongda]# cat script_check_data.sh 
#!/usr/bin/bash
#检查客户端推送过来的数据是否是完整的
#将目录定义为时间变量
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin
Time=$(date +%F)
Path=/backup

#1.找到当天的check_flag文件,校验数据是否是完整的，并且将校验结果存储至某个文件中
mkdir $Path/check_result
find $Path/ -type f -name "check_flag_$Time" | xargs md5sum -c > $Path/check_result/result_$Time

#2.将推送过来的校验值删除掉

find $Path/ -type f -name "check_flag_$Time" | xargs rm -fr  

#2.服务端仅保留6个月的备份数据,其余的全部删除
#find $Path  -type d -mtime +180  | xargs rm -rf

    
    
编写定时任务：
[root@rongda rongda]# crontab  -l
#check
* 04 * * * /usr/bin/sh  /home/rongda/script_check_data.sh  &>/dev/null    




===========================================
es部分的备份
cd  /home/rongda/backup/es

vim  esdump.sh
#!/bin/bash

es_host=192.168.254.101
es_port=9200
es_index=wisdom_doc
#scp_url=192.168.6.230:/home/

current_time=`date +%F-%H-%M`
mkdir ./esdump-$current_time

/home/rongda/backup/es/esdump dump --host $es_host --port $es_port --index $es_index --directory /home/rongda/backup/es/esdump-$current_time --chunk 500 --format bulk_indexing
tar -zcf esdump-$current_time.tar.gz esdump-$current_time

#scp esdump-$current_time.tar.gz $scp_url