flinktemp

CREATE TEMPORARY TABLE odps_sink (
  id  INT,
  len INT,
  content VARCHAR,
  ds VARCHAR --需要显式声明动态分区列。
) WITH (
  'connector' = 'odps',
  'endpoint' = '<yourEndpoint>',
  'project' = '<yourProjectName>',
  'tableName' = '<yourTableName>',
  'accessId' = '${secret_values.ak_id}',
  'accessKey' = '${secret_values.ak_secret}',
  'partition' = 'ds' --不写分区的值，表示根据ds字段的值写入不同分区。
);

INSERT INTO odps_sink
SELECT
   id,
   len,
   content,
   DATE_FORMAT(c, 'yyMMdd') as ds
FROM datagen_source;

Flink 1.13 实时写入 MaxCompute 项目

项目结构

flink-maxcompute-demo/
├── pom.xml
├── src/
│   ├── main/
│   │   ├── java/
│   │   │   └── com/
│   │   │       └── example/
│   │   │           ├── KafkaToMaxComputeJob.java
│   │   │           ├── KafkaSourceBuilder.java
│   │   │           ├── MaxComputeSinkBuilder.java
│   │   │           └── model/
│   │   │               └── KafkaMessage.java
│   │   └── resources/
│   │       └── log4j.properties
│   └── test/
│       └── java/
└── target/

主要代码实现

1. pom.xml 依赖配置

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>com.example</groupId>
    <artifactId>flink-maxcompute-demo</artifactId>
    <version>1.0-SNAPSHOT</version>
    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <flink.version>1.13.6</flink.version>
        <scala.binary.version>2.12</scala.binary.version>
    </properties>
    <dependencies>
        <!-- Flink dependencies -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <!-- Kafka connector -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <!-- MaxCompute connector -->
        <dependency>
            <groupId>com.alibaba.ververica</groupId>
            <artifactId>ververica-connector-odps</artifactId>
            <version>1.13-vvr-4.0.7</version>
        </dependency>
        <!-- JSON processing -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-json</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <!-- Logging -->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.30</version>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.2.4</version>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <artifactSet>
                                <excludes>
                                    <exclude>org.apache.flink:force-shading</exclude>
                                    <exclude>com.google.code.findbugs:jsr305</exclude>
                                    <exclude>org.slf4j:*</exclude>
                                    <exclude>log4j:*</exclude>
                                </excludes>
                            </artifactSet>
                            <filters>
                                <filter>
                                    <artifact>*:*</artifact>
                                    <excludes>
                                        <exclude>META-INF/*.SF</exclude>
                                        <exclude>META-INF/*.DSA</exclude>
                                        <exclude>META-INF/*.RSA</exclude>
                                    </excludes>
                                </filter>
                            </filters>
                            <transformers>
                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                    <mainClass>com.example.KafkaToMaxComputeJob</mainClass>
                                </transformer>
                            </transformers>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>

2. Kafka 消息模型 (KafkaMessage.java)

package com.example.model;

import java.util.Date;

public class KafkaMessage {
    private String id;
    private String content;
    private Date eventTime;
    private String partitionValue; // 用于MaxCompute分区字段的值

    // Getters and Setters
    public String getId() {
        return id;
    }

    public void setId(String id) {
        this.id = id;
    }

    public String getContent() {
        return content;
    }

    public void setContent(String content) {
        this.content = content;
    }

    public Date getEventTime() {
        return eventTime;
    }

    public void setEventTime(Date eventTime) {
        this.eventTime = eventTime;
    }

    public String getPartitionValue() {
        return partitionValue;
    }

    public void setPartitionValue(String partitionValue) {
        this.partitionValue = partitionValue;
    }

    @Override
    public String toString() {
        return "KafkaMessage{" +
                "id='" + id + '\'' +
                ", content='" + content + '\'' +
                ", eventTime=" + eventTime +
                ", partitionValue='" + partitionValue + '\'' +
                '}';
    }
}

3. Kafka 数据源构建器 (KafkaSourceBuilder.java)

package com.example;

import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.connector.kafka.source.KafkaSource;
import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer;
import org.apache.flink.connector.kafka.source.reader.deserializer.KafkaRecordDeserializationSchema;
import org.apache.kafka.clients.consumer.ConsumerConfig;

import java.util.Properties;

public class KafkaSourceBuilder {

    public static KafkaSource<String> buildKafkaSource() {
        // Kafka配置
        String bootstrapServers = "your-kafka-brokers:9092";
        String topic = "your-topic";
        String groupId = "flink-maxcompute-group";
        
        // SASL认证配置
        Properties props = new Properties();
        props.setProperty("security.protocol", "SASL_PLAINTEXT");
        props.setProperty("sasl.mechanism", "PLAIN");
        props.setProperty("sasl.jaas.config", 
            "org.apache.kafka.common.security.plain.PlainLoginModule required " +
            "username=\"your-username\" " +
            "password=\"your-password\";");

        return KafkaSource.<String>builder()
                .setBootstrapServers(bootstrapServers)
                .setTopics(topic)
                .setGroupId(groupId)
                .setStartingOffsets(OffsetsInitializer.earliest())
                .setDeserializer(KafkaRecordDeserializationSchema.valueOnly(new SimpleStringSchema()))
                .setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest")
                .setProperties(props)
                .build();
    }
}

4. MaxCompute Sink 构建器 (MaxComputeSinkBuilder.java)

package com.example;

import com.alibaba.ververica.connectors.odps.OdpsSinkFunction;
import com.alibaba.ververica.connectors.odps.sink.OdpsSinkOptions;
import com.alibaba.ververica.connectors.odps.sink.OdpsSinkOptionsBuilder;
import com.example.model.KafkaMessage;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.typeutils.RowTypeInfo;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.types.Row;

import java.sql.Timestamp;

public class MaxComputeSinkBuilder {

    public static SinkFunction<Row> buildMaxComputeSink() {
        // MaxCompute配置
        String projectName = "your_project";
        String tableName = "your_table";
        String partition = "pt"; // 分区字段名
        String accessId = "your_access_id";
        String accessKey = "your_access_key";
        String endpoint = "http://service.cn.maxcompute.aliyun.com/api";

        // 定义字段类型
        TypeInformation<?>[] fieldTypes = new TypeInformation[]{
                TypeInformation.of(String.class),  // id
                TypeInformation.of(String.class),  // content
                TypeInformation.of(Timestamp.class),  // event_time
                TypeInformation.of(String.class)   // 分区字段
        };

        // 定义字段名称
        String[] fieldNames = new String[]{"id", "content", "event_time", partition};

        // 创建RowTypeInfo
        RowTypeInfo rowTypeInfo = new RowTypeInfo(fieldTypes, fieldNames);

        // 构建OdpsSinkOptions
        OdpsSinkOptions odpsSinkOptions = new OdpsSinkOptionsBuilder()
                .withProjectName(projectName)
                .withTableName(tableName)
                .withPartition(partition)
                .withAccessId(accessId)
                .withAccessKey(accessKey)
                .withEndpoint(endpoint)
                .withOverwrite(false)
                .withCreatePartition(true)
                .withRowTypeInfo(rowTypeInfo)
                .build();

        return new OdpsSinkFunction(odpsSinkOptions);
    }
}

5. 主程序 (KafkaToMaxComputeJob.java)

package com.example;

import com.example.model.KafkaMessage;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.types.Row;
import org.apache.flink.util.Collector;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode;
import org.apache.flink.streaming.api.functions.ProcessFunction;

import java.sql.Timestamp;
import java.util.Date;

public class KafkaToMaxComputeJob {

    public static void main(String[] args) throws Exception {
        // 创建执行环境
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        // 设置并行度
        env.setParallelism(1);

        // 添加Kafka源
        DataStream<String> kafkaSource = env.fromSource(
                KafkaSourceBuilder.buildKafkaSource(),
                WatermarkStrategy.noWatermarks(),
                "Kafka Source"
        );

        // 解析JSON并转换为KafkaMessage对象
        DataStream<KafkaMessage> messageStream = kafkaSource
                .map(new MapFunction<String, KafkaMessage>() {
                    private transient ObjectMapper mapper;

                    @Override
                    public KafkaMessage map(String value) throws Exception {
                        if (mapper == null) {
                            mapper = new ObjectMapper();
                        }
                        
                        JsonNode jsonNode = mapper.readTree(value);
                        KafkaMessage message = new KafkaMessage();
                        message.setId(jsonNode.get("id").asText());
                        message.setContent(jsonNode.get("content").asText());
                        
                        // 假设eventTime是时间戳
                        long timestamp = jsonNode.get("eventTime").asLong();
                        message.setEventTime(new Date(timestamp));
                        
                        // 设置分区值，例如按天分区 "20230520"
                        String partitionValue = new SimpleDateFormat("yyyyMMdd").format(new Date(timestamp));
                        message.setPartitionValue(partitionValue);
                        
                        return message;
                    }
                });

        // 转换为Row类型，以便写入MaxCompute
        DataStream<Row> rowStream = messageStream
                .process(new ProcessFunction<KafkaMessage, Row>() {
                    @Override
                    public void processElement(
                            KafkaMessage value,
                            ProcessFunction<KafkaMessage, Row>.Context ctx,
                            Collector<Row> out) throws Exception {
                        Row row = new Row(4);
                        row.setField(0, value.getId());
                        row.setField(1, value.getContent());
                        row.setField(2, new Timestamp(value.getEventTime().getTime()));
                        row.setField(3, value.getPartitionValue());
                        out.collect(row);
                    }
                })
                .returns(TypeInformation.of(Row.class));

        // 添加MaxCompute Sink
        rowStream.addSink(MaxComputeSinkBuilder.buildMaxComputeSink())
                .name("MaxCompute Sink");

        // 执行任务
        env.execute("Kafka to MaxCompute Job");
    }
}

配置说明

Kafka认证配置：

在KafkaSourceBuilder中配置了SASL_PLAINTEXT认证
需要替换your-username和your-password为实际的Kafka认证信息

MaxCompute配置：

在MaxComputeSinkBuilder中配置了MaxCompute的连接信息
需要替换your_project, your_table, your_access_id, your_access_key等参数

数据处理流程：

从Kafka读取JSON格式数据
解析JSON并转换为KafkaMessage对象
转换为Flink Row类型
写入MaxCompute分区表

注意事项

确保MaxCompute表的结构与程序中定义的Row类型匹配
根据实际业务需求调整分区策略
生产环境中应考虑错误处理和重试机制
敏感信息(如accessKey)应通过安全方式配置，而不是硬编码在代码中
根据实际数据量调整Flink作业的并行度和资源配置

这个示例提供了从认证的Kafka读取数据并实时写入MaxCompute分区表的基本框架，您可以根据实际业务需求进行修改和扩展。

重构为单独的处理函数类

为了更好地组织代码和管理处理函数，我将重构项目结构，将处理函数提取为单独的类。以下是重构后的代码：

更新后的项目结构

flink-maxcompute-demo/
├── pom.xml
├── src/
│   ├── main/
│   │   ├── java/
│   │   │   └── com/
│   │   │       └── example/
│   │   │           ├── KafkaToMaxComputeJob.java
│   │   │           ├── KafkaSourceBuilder.java
│   │   │           ├── MaxComputeSinkBuilder.java
│   │   │           ├── functions/
│   │   │           │   ├── JsonToKafkaMessageFunction.java
│   │   │           │   └── KafkaMessageToRowFunction.java
│   │   │           └── model/
│   │   │               └── KafkaMessage.java
│   │   └── resources/
│   │       └── log4j.properties
│   └── test/
│       └── java/
└── target/

新增的处理函数类

1. JsonToKafkaMessageFunction.java

package com.example.functions;

import com.example.model.KafkaMessage;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.flink.api.common.functions.MapFunction;

import java.text.SimpleDateFormat;
import java.util.Date;

public class JsonToKafkaMessageFunction implements MapFunction<String, KafkaMessage> {
    
    private transient ObjectMapper mapper;
    private transient SimpleDateFormat dateFormat;

    @Override
    public KafkaMessage map(String value) throws Exception {
        if (mapper == null) {
            mapper = new ObjectMapper();
        }
        if (dateFormat == null) {
            dateFormat = new SimpleDateFormat("yyyyMMdd");
        }
        
        JsonNode jsonNode = mapper.readTree(value);
        KafkaMessage message = new KafkaMessage();
        message.setId(jsonNode.get("id").asText());
        message.setContent(jsonNode.get("content").asText());
        
        // 假设eventTime是时间戳
        long timestamp = jsonNode.get("eventTime").asLong();
        message.setEventTime(new Date(timestamp));
        
        // 设置分区值，例如按天分区 "20230520"
        message.setPartitionValue(dateFormat.format(new Date(timestamp)));
        
        return message;
    }
}

2. KafkaMessageToRowFunction.java

package com.example.functions;

import com.example.model.KafkaMessage;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.types.Row;
import org.apache.flink.util.Collector;

import java.sql.Timestamp;

public class KafkaMessageToRowFunction extends ProcessFunction<KafkaMessage, Row> {

    @Override
    public void processElement(
            KafkaMessage value,
            ProcessFunction<KafkaMessage, Row>.Context ctx,
            Collector<Row> out) throws Exception {
        Row row = new Row(4);
        row.setField(0, value.getId());
        row.setField(1, value.getContent());
        row.setField(2, new Timestamp(value.getEventTime().getTime()));
        row.setField(3, value.getPartitionValue());
        out.collect(row);
    }

    @Override
    public TypeInformation<Row> getProducedType() {
        return TypeInformation.of(Row.class);
    }
}

更新后的主程序 (KafkaToMaxComputeJob.java)

package com.example;

import com.example.functions.JsonToKafkaMessageFunction;
import com.example.functions.KafkaMessageToRowFunction;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;

public class KafkaToMaxComputeJob {

    public static void main(String[] args) throws Exception {
        // 创建执行环境
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        // 设置并行度
        env.setParallelism(1);

        // 添加Kafka源
        DataStream<String> kafkaSource = env.fromSource(
                KafkaSourceBuilder.buildKafkaSource(),
                WatermarkStrategy.noWatermarks(),
                "Kafka Source"
        );

        // 解析JSON并转换为KafkaMessage对象
        DataStream<KafkaMessage> messageStream = kafkaSource
                .map(new JsonToKafkaMessageFunction())
                .name("JSON to KafkaMessage Mapper");

        // 转换为Row类型，以便写入MaxCompute
        DataStream<Row> rowStream = messageStream
                .process(new KafkaMessageToRowFunction())
                .name("KafkaMessage to Row Converter");

        // 添加MaxCompute Sink
        rowStream.addSink(MaxComputeSinkBuilder.buildMaxComputeSink())
                .name("MaxCompute Sink");

        // 执行任务
        env.execute("Kafka to MaxCompute Job");
    }
}

重构后的优势

更好的代码组织：将处理逻辑分离到专门的类中，使主程序更简洁
可重用性：处理函数可以在多个作业中重复使用
可测试性：独立的函数类更容易编写单元测试
可维护性：修改处理逻辑时只需修改对应的函数类，不会影响其他部分
更好的命名：通过函数类名可以更清晰地表达处理逻辑的意图
状态管理：可以在函数类中更好地管理状态变量（如ObjectMapper）

其他建议

可以考虑为处理函数添加日志记录，便于调试和监控
可以添加参数校验和错误处理逻辑
对于更复杂的处理逻辑，可以考虑使用RichFunction系列类（如RichMapFunction）以获得更多功能
考虑将日期格式等配置参数提取为构造函数参数，使函数更灵活

这种结构化的方式更适合生产环境，特别是当处理逻辑变得复杂时，能够保持代码的清晰和可维护性。

posted @ 2025-05-20 22:11 ---江北阅读(42) 评论(0) 收藏举报

刷新页面返回顶部

---江北

冰冻三尺，非一日之寒