Flink和Kafka

来源于 https://blog.csdn.net/wangchao_cn/article/details/104707274/

 

使用DataStream来消费Kafka。

下面Schema类很重要，实现对消息的序列化和反序列化

KafkaSerializationSchema<Quotes>, KafkaDeserializationSchema<Quotes>

 

 

import com.redblue.entity.Quotes;
import com.redblue.kafka.QuotesSchema;
import lombok.extern.slf4j.Slf4j;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.java.io.jdbc.JDBCInputFormat;
import org.apache.flink.api.java.typeutils.RowTypeInfo;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.java.StreamTableEnvironment;
import org.apache.flink.types.Row;

import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.Properties;

@Slf4j
public class TestKafka {
    public static void main(String[] args) throws Exception {

        EnvironmentSettings envSettings = EnvironmentSettings.newInstance().useOldPlanner().inStreamingMode().build();
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env, envSettings);

        String mysqlUrl = "jdbc:mysql://xxxx.com:3306/test?useSSL=false";
        String driverName = "com.mysql.jdbc.Driver";
        String user = "root";
        String password = "xxx";
        Properties properties = new Properties();
        properties.setProperty("user", user);
        properties.setProperty("password", password);
        String tableName = "stock_quotes";

        RowTypeInfo rowTypeInfo = new RowTypeInfo(BasicTypeInfo.STRING_TYPE_INFO,
                BasicTypeInfo.STRING_TYPE_INFO,
                BasicTypeInfo.INT_TYPE_INFO,
                BasicTypeInfo.STRING_TYPE_INFO,
                BasicTypeInfo.STRING_TYPE_INFO,
                BasicTypeInfo.STRING_TYPE_INFO,
                BasicTypeInfo.BIG_DEC_TYPE_INFO,
                BasicTypeInfo.BIG_DEC_TYPE_INFO,
                BasicTypeInfo.BIG_DEC_TYPE_INFO,
                BasicTypeInfo.BIG_DEC_TYPE_INFO,
                BasicTypeInfo.BIG_DEC_TYPE_INFO,
                BasicTypeInfo.BIG_DEC_TYPE_INFO);
        StringBuilder sqlBuilder = new StringBuilder();
        sqlBuilder.append("select stock_full_code,stock_name,id,market_code,stock_code,deal_date");
        sqlBuilder.append(",open_price,close_price,lowest_price,highest_price");
        sqlBuilder.append(",cast(deal_volume as decimal) deal_volume,deal_amount");
        sqlBuilder.append(" from ").append(tableName);

        log.info(sqlBuilder.toString());

        JDBCInputFormat jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat()
                .setDrivername(driverName)
                .setDBUrl(mysqlUrl)
                .setUsername(user)
                .setPassword(password)
                .setQuery(sqlBuilder.toString())
                .setRowTypeInfo(rowTypeInfo)
                .finish();
        DataStreamSource<Row> sourceQuotes = env.createInput(jdbcInputFormat);
        tableEnv.registerDataStream("quotes", sourceQuotes);
        Table tableQuotes = tableEnv.sqlQuery("select * from quotes" );
        DataStream<Quotes> dsQuotes = tableEnv.toAppendStream(tableQuotes,rowTypeInfo).map(new MapFunction<Row, Quotes>() {
            @Override
            public Quotes map(Row row) throws Exception {
                return new Quotes(
                        row.getField(0).toString(),
                        row.getField(1).toString(),
                        new BigInteger(row.getField(2).toString()),
                        row.getField(3).toString(),
                        row.getField(4).toString(),
                        row.getField(5).toString(),
                        new BigDecimal(row.getField(6).toString()),
                        new BigDecimal(row.getField(7).toString()),
                        new BigDecimal(row.getField(8).toString()),
                        new BigDecimal(row.getField(9).toString()),
                        new BigInteger(row.getField(10).toString()),
                        new BigDecimal(row.getField(11).toString()));
            }
        });

        Properties propertiesKafka = new Properties();
        propertiesKafka.setProperty("bootstrap.servers", "xxx.com:31092");
        String topic = "flink-demo-simple";

        FlinkKafkaProducer kafkaProducer = new FlinkKafkaProducer<Quotes>(topic, new QuotesSchema(topic), propertiesKafka, FlinkKafkaProducer.Semantic.EXACTLY_ONCE);
        kafkaProducer.setWriteTimestampToKafka(true);
        dsQuotes.addSink(kafkaProducer);

        env.execute("TestKafka");
    }
}

import com.redblue.entity.Quotes;
import com.redblue.kafka.QuotesSchema;
import org.apache.flink.streaming.api.*;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;

import java.util.Properties;

public class TestKafkaStream {
    public static void main(String[] args) throws Exception {
        final  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.enableCheckpointing(5000);

        Properties properties = new Properties();
        properties.setProperty("bootstrap.servers", "xxx.com:31092");
        properties.setProperty("group.id", "flink");

        String topic = "flink-demo-simple";
        DataStream kafkaStream = env.addSource(new FlinkKafkaConsumer<Quotes>(topic,new QuotesSchema(topic), properties));
        kafkaStream.print();

        env.execute("TestKafkaStream");

    }
}
 

import com.alibaba.fastjson.JSONObject;
import com.redblue.entity.Quotes;
import lombok.extern.slf4j.Slf4j;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.streaming.connectors.kafka.KafkaDeserializationSchema;
import org.apache.flink.streaming.connectors.kafka.KafkaSerializationSchema;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.producer.ProducerRecord;

import javax.annotation.Nullable;

@Slf4j
public class QuotesSchema implements KafkaSerializationSchema<Quotes>, KafkaDeserializationSchema<Quotes> {
    private String topic;

    public QuotesSchema(String topic) {
        this.topic = topic;
    }

    @Override
    public ProducerRecord<byte[], byte[]> serialize(Quotes quotes, @Nullable Long timestamp) {
        log.info("===============================================================================");
        log.info(quotes.toJsonString());
        return new ProducerRecord<byte[], byte[]>(topic, quotes.toJsonString().getBytes());
    }

    @Override
    public boolean isEndOfStream(Quotes nextElement) {
        return false;
    }

    @Override
    public Quotes deserialize(ConsumerRecord<byte[], byte[]> record) throws Exception {
        Quotes quotes = null;
        String key = null;
        String jsonString = null;

        if (record != null) {

            if (record.key() != null) {
                key = new String(record.key());
            }

            if (record.value() != null) {
                jsonString = new String(record.value());
                log.info("-----------------------------------------------------------------------------------");
                log.info(jsonString);
            }

            quotes = JSONObject.parseObject(jsonString, Quotes.class);
        }

        return quotes;
    }

    @Override
    public TypeInformation<Quotes> getProducedType() {
        return TypeInformation.of(new TypeHint<Quotes>() {
        });
    }
}