Flink-CDC-PostgreSQL 之 自定义同步实例
一、依赖引入
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<flink.version>1.16.0</flink.version>
<!-- <flink.version>1.17.0</flink.version>-->
<!-- <flink-cdc.version>3.1.0</flink-cdc.version>-->
<flink-cdc.version>3.0.1</flink-cdc.version>
<!-- <flink-cdc.version>2.3.0</flink-cdc.version>-->
</propert>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-api-java-bridge</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-planner-loader</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-runtime</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-base</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-jdbc</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<!-- <groupId>org.apache.flink</groupId>-->
<groupId>com.ververica</groupId>
<artifactId>flink-sql-connector-postgres-cdc</artifactId>
<version>${flink-cdc.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-statebackend-rocksdb</artifactId>
<version>${flink.version}</version>
<!-- <version>2.1.1</version>--
</dependency>
二、数据库的设置
1、数据库配置
wal_level = logical
max_replication_slots = 20
max_wal_senders = 20
wal_sender_timeout = 180s
shared_preload_libraries = 'pgoutput'
2、数据库权限设置
CREATE USER cdc_user WITH PASSWORD 'cdc@dip001';
ALTER ROLE cdc_user REPLICATION;
GRANT CONNECT ON DATABASE dip_basedata TO cdc_user;
GRANT USAGE ON SCHEMA public TO cdc_user;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO cdc_user;
-- 使用超级用户登录后执行
-- CREATE PUBLICATION flink_cdc_pub FOR ALL TABLES;
CREATE PUBLICATION tasktable_cdc_pub FOR TABLE public.t_common_sync_task;
ALTER TABLE public.t_common_sync_task REPLICA IDENTITY FULL;
三、代码编写
1、创建flink执行环境
// 创建Flink流执行环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
// 启用检查点
env.enableCheckpointing(30000); // 每30秒触发一次检查点
// env.setStateBackend(new RocksDBStateBackend("hdfs:///flink/checkpoints", true));// 第二个参数启用增量检查点
env.setStateBackend(new RocksDBStateBackend("file:///tmp/flink/checkpoints", true));// 第二个参数启用增量检查点
env.getCheckpointConfig().setTolerableCheckpointFailureNumber(3); // 允许3次检查点失败
env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE); // 精确一次语义
env.getCheckpointConfig().setMinPauseBetweenCheckpoints(10000); // 强制两次检查点间隔至少10000ms（防止频繁触发）,如果设置的检查点触发时间小于此值，则会强制使用此值作为检查点间隔
env.getCheckpointConfig().setCheckpointTimeout(600000); // 检查点超时时间（10分钟）
env.getCheckpointConfig().setTolerableCheckpointFailureNumber(3); // 允许连续失败次数
env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); // 作业取消后保留检查点
// env.getCheckpointConfig().setString("state.backend.rocksdb.compression", "zstd");
// 设置并行度
env.setParallelism(1);
2、debezium属性配置
Properties properties = new Properties();
// 好像不起作用使用slot.name
// properties.setProperty("debezium.slot.name", "bgpc_cdc_slot");
// properties.setProperty("slot.name", "bgpc_flink_slot");
properties.setProperty("slot.name", "dbz_customerdb_europe_slot");
properties.setProperty("publication.name", "tasktable_cdc_pub");
properties.setProperty("publication.create.enable", "true");
properties.setProperty("debezium.slot.drop.on.top", "false");
properties.setProperty("include.schema.changes", "false");//忽略ddl变更
// 指定连接器启动时执行快照的条件：****重要*****
//initial- 连接器仅在没有为逻辑服务器名称记录偏移量时才执行快照。
//always- 连接器每次启动时都会执行快照。
//never- 连接器从不执行快照。
//initial_only- 连接器执行初始快照然后停止，不处理任何后续更改。
//exported- 连接器根据创建复制槽的时间点执行快照。这是一种以无锁方式执行快照的绝佳方式。
//custom- 连接器根据snapshot.custom.class属性的设置执行快照
if (initReadIgnore) {
properties.setProperty("debezium.snapshot.mode", "never");
properties.setProperty("snapshot.mode", "never");
// deserialization = new PostgreSQLDeserialization(new PostgreSQLReadDataFilter());
} else {
properties.setProperty("debezium.snapshot.mode", "initial");
properties.setProperty("snapshot.mode", "initial");
// deserialization = new PostgreSQLDeserialization();
}
3、创建CDC源
// 创建PostgreSQL CDC源
// PostgreSQLSource<String> source = PostgreSQLSource.<String>builder()
DebeziumSourceFunction<String> source = PostgreSQLSource.<String>builder()
.hostname("10.17.16.204")
.port(5432)
.username("cdc_user")
.password("cdc@dip001")
.database("dip_basedata")
.schemaList("public")
.tableList("public.t_common_sync_task")
.debeziumProperties(properties)
.deserializer(new JsonDebeziumDeserializationSchema())
.decodingPluginName("pgoutput") // PostgreSQL 10+ 使用pgoutput
.build();
4、启动任务
DataStreamSource<String> sourceStream = env.addSource(source, "PG_SOURCE").setParallelism(1);
SinkFunction<String> sink=new SinkFunction<String>() {
@Override
public void invoke(String value, Context context) throws Exception {
log.info("监听数据: {}", JSONUtil.toJsonStr(value));
SinkFunction.super.invoke(value, context);
}
};
sourceStream.addSink(sink).name("PG_SINK");
env.execute("PG_JOB");
三、结果验证
-- 日志
Starting PostgresConnectorTask with configuration:
connector.class = io.debezium.connector.postgresql.PostgresConnector
slot.name = dbz_customerdb_europe_slot
publication.name = tasktable_cdc_pub
schema.include.list = public
include.schema.changes = false
debezium.slot.drop.on.top = true
tombstones.on.delete = false
offset.storage.file.filename =
publication.create.enable = true
value.converter = com.ververica.cdc.connectors.shaded.org.apache.kafka.connect.json.JsonConverter
database.history.instance.name = b95c8e68-db7b-4b1a-ad9f-7472bb4d04f9
key.converter = com.ververica.cdc.connectors.shaded.org.apache.kafka.connect.json.JsonConverter
database.user = cdc_user
database.dbname = dip_basedata
offset.storage = com.ververica.cdc.debezium.internal.FlinkOffsetBackingStore
database.server.name = postgres_cdc_source
offset.flush.timeout.ms = 5000
heartbeat.interval.ms = 300000
database.port = 5432
plugin.name = pgoutput
offset.flush.interval.ms = 9223372036854775807
internal.key.converter = com.ververica.cdc.connectors.shaded.org.apache.kafka.connect.json.JsonConverter
debezium.snapshot.mode = never
database.hostname = 108.19.16.205
database.password = ********
name = engine
internal.value.converter = com.ververica.cdc.connectors.shaded.org.apache.kafka.connect.json.JsonConverter
table.include.list = public.t_common_sync_task
snapshot.mode = never
database.history = com.ververica.cdc.debezium.internal.FlinkDatabaseSchemaHistory
本文来自博客园，作者：蓝迷梦，转载请注明原文链接：https://www.cnblogs.com/hewei-blogs/articles/19277596
浙公网安备 33010602011771号