maxwell安装部署

环境:
Os:Centos 7
maxwell:v1.40.2
jdk:21
mysql:5.7

说明:v1.29.2之后的版本需要jdk11以上

1.下载安装介质
[root@host113 /]# cd /soft
[root@host113 soft]#wget https://github.com/zendesk/maxwell/releases/download/v1.40.2/maxwell-1.40.2.tar.gz

2.创建maxwell数据库
create database db_maxwell;

 

3.创建maxwell同步账号
grant select, replication slave, replication client on *.* to 'maxwell'@'%' identified by 'maxwell';
grant all on db_maxwell.* to 'maxwell'@'%';

 

4.解压安装
[root@host113 soft]# cd /soft
[root@host113 soft]# tar -xvf maxwell-1.40.2.tar.gz
[root@host113 soft]# mv maxwell-1.40.2 /opt/

 

5.修改配置文件
[root@host113 maxwell-1.29.2]#cd /opt/maxwell-1.40.2
[root@host113 maxwell-1.29.2]# cp config.properties.example config.properties

vi config.properties
修改的项如下:

# 日志级别
log_level=info
client_id=maxwell-mysql
replica_server_id=11
# Maxwell 元数据数据库信息
host=192.168.1.14
port=3306
user=maxwell
password=maxwell
##会自动过滤掉这个元数据库
schema_database=db_maxwell


#producer kafka的配置信息
producer=kafka
kafka.bootstrap.servers=192.168.1.134:9092,192.168.1.135:9092,192.168.1.136:9092
kafka_topic=maxwell-mysql
kafka_partition_hash=murmur3
kafka.max.request.size=10485760
producer_partition_by=primary_key

#被同步的数据库的信息,若maxwell元数据库所在实例与被同步的实例是同一个,replication段不需要写
replication_host=192.168.1.14
replication_user=maxwell
replication_password=maxwell
replication_port=3306
jdbc_options=useSSL=false&serverTimezone=Asia/Shanghai

#kafka高吞吐配置，只要leader数据写入磁盘表示成功
kafka.acks = 1
kafka.max.in.flight.requests.per.connection = 1
kafka.compression.type = snappy
kafka.retries = 10

# 配置过滤器，可以配置只同步指定的数据库，指定的表的数据变动。
#不能分2行写
##filter.exclude=*.*
##filter.include=^db_.+\\..*

#直接一行写,同步db_开头的库和test库
filter=exclude:*.*,include:/^db_.+\./.*,include:test.*


output_push_timestamp=true
output_binlog_position=true
output_old = true
http_port=8081

 

6.启动
/opt/maxwell-1.40.2
nohup /opt/maxwell-1.40.2/bin/maxwell --config config.properties > maxwell.log 2>&1 &

/opt/maxwell-1.40.2/bin/maxwell --config config.properties

7.查看运行情况
curl http://192.168.1.113:8081/status

 

8.查看kafka的消费
/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server 192.168.1.134:9092,192.168.1.135:9092,192.168.1.136:9092 --topic maxwell-mysql

 

9.数据库写入一条记录
/opt/mysql57/bin/mysql -h localhost -uroot -pyeemiao1117 -S /opt/mysql57/data/mysql.sock
use db_admin;
insert into user (name,email) values('hhh','11@qq.com');

这个时候在部署8的窗口会看得到如下的记录
{"database":"db_admin","table":"user","type":"insert","ts":1789110500,"xid":998530263,"commit":true,"position":"binlog.000299:55310031","push_ts":1789110501.043155,"data":{"id":4,"name":"hhh","email":"11@qq.com"}}

 

10.过滤规则
例子1:
只同步db_开头的库,test库,cat库,必须写在一行
filter=exclude:*.*,include:/^db_.+\./.*,include:test.*,include:cat.*

例子2:
只同步库db_linkman下表名以tb_link开头的表
filter=exclude:*.*,include:db_link./^tb_link.*/

 

例子3:
只同步库db_link下表名以tb_link开头的表,以及以db开头的库
filter=exclude:*.*,include:db_link./^tb_link.*/,include:/^db_.+\./.*

例子4:
同步db开头下的所有的库,但是排除表 db_admin.user
filter=exclude:*.*,include:/^db_.+\./.*,exclude:db_admin.user

 

11.全量同步

主进程需要运行
/opt/maxwell-1.40.2/bin/maxwell --config config.properties
启用全量同步
/opt/maxwell-1.40.2/bin/maxwell-bootstrap --config /opt/maxwell-1.40.2/config.properties --database db_admin --table user

 

配置文件中定义的filter规则对全量同步不起作用的.