使用flink-sql同步oracle数据到kafka

环境:
os:ubuntu 26.04
flink:1.20.5
说明:
a.使用flink-sql,每提交一个sql(一个同步任务),就会单独启动logminer,同步多个表的话,很耗性能;
b.可以使用debezium 同步多个表,只启用一个logminer进程

1.下载相应的jar包
cd /opt/flink-1.20.5/lib

##ojdbc8 Oracle驱动
wget https://maven.aliyun.com/repository/public/com/oracle/database/jdbc/ojdbc8/19.21.0.0/ojdbc8-19.21.0.0.jar
wget https://repo1.maven.org/maven2/org/apache/flink/flink-sql-connector-oracle-cdc/3.6.0-1.20/flink-sql-connector-oracle-cdc-3.6.0-1.20.jar

##kafka驱动
wget https://repo1.maven.org/maven2/org/apache/flink/flink-sql-connector-kafka/3.4.0-1.20/flink-sql-connector-kafka-3.4.0-1.20.jar

2.重启flink
/opt/flink-1.20.5/bin/stop-cluster.sh
/opt/flink-1.20.5/bin/start-cluster.sh

3.准备同步配置文件
vi /opt/flink-1.20.5/flinksql/oracle2kafka_tb_test.sql

 

SET 'pipeline.name' = 'oracle同步到kafka_tb_test';
-- Oracle CDC源表 TB_TEST
CREATE TABLE oracle_source_tb_test (
    ID INT,
    NAME STRING,
    PRIMARY KEY (ID) NOT ENFORCED
) WITH (
    'connector' = 'oracle-cdc',
    'hostname' = '192.168.1.134',
    'port' = '1521',
    'username' = 'hxl',
    'password' = 'oracle',
    'database-name' = 'slnngk',
    'schema-name' = 'HXL',
    'table-name' = 'TB_TEST',
    'scan.startup.mode' = 'initial',
    'scan.incremental.snapshot.chunk.size' = '4000'
);

CREATE TABLE kafka_sink_tb_test (
    ID          INT,
    NAME        STRING
) WITH (
    'connector' = 'kafka',
    'topic' = 'oracle_cdc_topic_tb_test',
    'properties.bootstrap.servers' = '192.168.1.134:9092',
    'format' = 'debezium-json',
    'debezium-json.ignore-parse-errors' = 'true',
    'sink.partitioner' = 'fixed',
    'properties.retries' = '3'
);

INSERT INTO kafka_sink_tb_test
SELECT ID, NAME FROM oracle_source_tb_test;

 

4.启动任务
/opt/flink-1.20.5/bin/sql-client.sh -f /opt/flink-1.20.5/flinksql/oracle2kafka_tb_test.sql

5.查看kafka数据
[root@host134 ~]# /opt/kafka/bin/kafka-console-consumer.sh \
--bootstrap-server 192.168.1.134:9092,192.168.1.135:9092,192.168.1.136:9092 \
--topic oracle_cdc_topic_tb_test \
--from-beginning