使用flink将mysql数据同步到kafka

环境:
OS:ubuntu 26
flink:1.20
原理是读取mysql binlog同步,flink只能同步dml语句 不能同步ddl语句

 

1.下载依赖包
cd /opt/flink/lib
wget https://repo1.maven.org/maven2/org/apache/flink/flink-sql-connector-mysql-cdc/3.4.0/flink-sql-connector-mysql-cdc-3.4.0.jar
wget https://repo1.maven.org/maven2/org/apache/flink/flink-sql-connector-kafka/3.4.0-1.20/flink-sql-connector-kafka-3.4.0-1.20.jar

 

红色标注的版本号需要与flink版本号保持一致

 

2.重启集群
/opt/flink/bin/stop-cluster.sh
执行jps命令确保没有相关的flink进程

/opt/flink/bin/start-cluster.sh

 

3.mysql创建同步账号
CREATE USER 'flink'@'%' IDENTIFIED BY 'flink';
GRANT REPLICATION SLAVE,REPLICATION CLIENT,SELECT ON *.* TO flink@'%';
FLUSH PRIVILEGES;

 

4.准备同步文件
mkdir /opt/flink/flinksql
vi /opt/flink/flinksql/mysql2kafka.sql

SET 'pipeline.name' = '测试';
SET 'execution.checkpointing.interval' = '10s';
SET 'state.backend' = 'rocksdb';
SET 'state.backend.rocksdb.incremental' = 'true';
SET 'table.exec.sink.upsert-materialize' = 'NONE';

CREATE TABLE mysql_user_source (
            id BIGINT,
            name STRING,
            email STRING,
            PRIMARY KEY(id) NOT ENFORCED
) WITH (
    'connector' = 'mysql-cdc',
    'hostname' = '192.168.1.14',
    'port' = '3306',
    'username' = 'flink',
    'password' = 'flink',
    'database-name' = 'db_admin',
    'table-name' = 'user',
    'server-id' = '5401',
    'server-time-zone' = 'Asia/Shanghai',
    'scan.startup.mode' = 'initial',    -- initial = 全量快照 + 后续增量binlog,scan.startup.mode=latest-offset 为增量同步
    'scan.incremental.snapshot.enabled' = 'true', -- 开启增量快照（分片并行读，减少锁）
    'scan.incremental.snapshot.chunk.size' = '8096', -- 分片大小，可根据表数据量调
    'debezium.connect.keep.alive.interval.ms' = '40000'
);

CREATE TABLE kafka_user_sink (
            id BIGINT,
            name STRING,
            email STRING
) WITH (
    'connector' = 'kafka',
    'topic' = 'mysql_user_cdc',
    'properties.bootstrap.servers' = '192.168.1.134:9092,192.168.1.135:9092,192.168.1.136:9092',
    'format' = 'debezium-json',
    'key.fields' = 'id',
    'key.format' = 'json',
    'properties.batch.size' = '16384',
    'properties.linger.ms' = '5',
    'properties.acks' = '1'
);

INSERT INTO kafka_user_sink SELECT id, name, email FROM mysql_user_source;

 

6.kafka端查看数据
在kafka服务器上执行
/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server 192.168.1.134:9092,192.168.1.135:9092,192.168.1.136:9092 --topic mysql_user_cdc