Structured Streaming消费rocketMQ

一、背景

最近有一个需求是：要求有一个类对象为Order，它有string类型的字段orderNo和Long类型的字段cost，生产者写到rocketMQ的value是Order对象序列化后的字节数组、key值是orderNo字段，要求spark以自动提交的方式消费rocketMQ，并将数据依次写入到hive表中，并且spark中有一个5分钟滑动窗口，滑动步长为1分钟，统计5分钟内的cost总值并输出。

 

二、实战演练

1. Order 类

package com.example;

import java.io.Serializable;

public class Order implements Serializable {
    private static final long serialVersionUID = 1L;

    private String orderNo;
    private Long cost;

    public Order() {}

    public Order(String orderNo, Long cost) {
        this.orderNo = orderNo;
        this.cost = cost;
    }

    public String getOrderNo() {
        return orderNo;
    }

    public void setOrderNo(String orderNo) {
        this.orderNo = orderNo;
    }

    public Long getCost() {
        return cost;
    }

    public void setCost(Long cost) {
        this.cost = cost;
    }

    @Override
    public String toString() {
        return "Order{" +
                "orderNo='" + orderNo + '\'' +
                ", cost=" + cost +
                '}';
    }
}

2. RocketMQ Producer 示例

package com.example;

import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;

import java.io.ByteArrayOutputStream;
import java.io.ObjectOutputStream;
import java.util.UUID;

public class RocketMQOrderProducer {
    public static void main(String[] args) throws Exception {
        DefaultMQProducer producer = new DefaultMQProducer("order_producer_group");
        producer.setNamesrvAddr("localhost:9876"); // RocketMQ NameServer 地址
        producer.start();

        for (int i = 0; i < 20; i++) {
            String orderNo = "ORD-" + UUID.randomUUID().toString().substring(0, 8);
            long cost = (long) (Math.random() * 1000);
            Order order = new Order(orderNo, cost);

            byte[] body;
            try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
                 ObjectOutputStream oos = new ObjectOutputStream(bos)) {
                oos.writeObject(order);
                body = bos.toByteArray();
            }

            Message msg = new Message("OrderTopic", orderNo, body);
            SendResult result = producer.send(msg);
            System.out.printf("Sent: %s, result=%s%n", order, result);
        }

        producer.shutdown();
    }
}

3. Hive 建表 SQL

-- 原始订单表
DROP TABLE IF EXISTS demo_db.orders;
CREATE TABLE demo_db.orders (
  orderNo STRING,
  cost BIGINT
)
STORED AS PARQUET;

-- 聚合结果表
DROP TABLE IF EXISTS demo_db.order_cost_window_agg;
CREATE TABLE demo_db.order_cost_window_agg (
  window_start TIMESTAMP,
  window_end TIMESTAMP,
  total_cost BIGINT
)
STORED AS PARQUET;

4. Spark Structured Streaming 消费 RocketMQ & 写入 Hive

这里用 rocketmq-spark connector，Structured Streaming 模式下，offset 会自动 checkpoint。

package com.example;

import org.apache.spark.sql.*;
import org.apache.spark.sql.streaming.StreamingQuery;
import org.apache.spark.sql.streaming.Trigger;
import org.apache.spark.sql.types.DataTypes;

import java.io.ByteArrayInputStream;
import java.io.ObjectInputStream;

public class StructuredStreamingRocketMQToHive {
    public static void main(String[] args) throws Exception {
        SparkSession spark = SparkSession.builder()
                .appName("StructuredStreamingRocketMQToHive")
                .enableHiveSupport()
                .getOrCreate();

        // 注册反序列化 UDF
        spark.udf().register("deserializeOrderNo", (byte[] bytes) -> {
            try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes))) {
                Order o = (Order) ois.readObject();
                return o.getOrderNo();
            } catch (Exception e) {
                return null;
            }
        }, DataTypes.StringType);

        spark.udf().register("deserializeCost", (byte[] bytes) -> {
            try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes))) {
                Order o = (Order) ois.readObject();
                return o.getCost();
            } catch (Exception e) {
                return null;
            }
        }, DataTypes.LongType);

        // 读取 RocketMQ 流
        Dataset<Row> df = spark.readStream()
                .format("org.apache.rocketmq.spark")
                .option("namesrvAddr", "localhost:9876")
                .option("consumerGroup", "order_consumer_group")
                .option("topics", "OrderTopic")
                .load();

        // RocketMQ Connector 输出的 DataFrame 包含 key、body、topic、offset 等字段
        Dataset<Row> orderDF = df.withColumn("orderNo", functions.callUDF("deserializeOrderNo", df.col("body")))
                .withColumn("cost", functions.callUDF("deserializeCost", df.col("body")))
                .select("orderNo", "cost")
                .filter("orderNo IS NOT NULL AND cost IS NOT NULL");

        // 写入 Hive 原始订单表
        StreamingQuery orderQuery = orderDF.writeStream()
                .format("hive")
                .outputMode("append")
                .option("checkpointLocation", "/user/hive/warehouse/checkpoints/orders")
                .toTable("demo_db.orders");

        // 窗口统计（处理时间窗口）
        Dataset<Row> aggDF = orderDF
                .withColumn("event_time", functions.current_timestamp())
                .groupBy(functions.window(functions.col("event_time"), "5 minutes", "1 minute"))
                .agg(functions.sum("cost").alias("total_cost"))
                .selectExpr("window.start as window_start", "window.end as window_end", "total_cost");

        // 写入 Hive 聚合表
        StreamingQuery aggQuery = aggDF.writeStream()
                .format("hive")
                .outputMode("append")
                .option("checkpointLocation", "/user/hive/warehouse/checkpoints/orders_agg")
                .toTable("demo_db.order_cost_window_agg");

        spark.streams().awaitAnyTermination();
    }
}

5. 关键点

1. Structured Streaming 自动管理 offset，checkpoint 存在 HDFS/S3，恢复时会从 checkpoint 继续。

2. outputMode("append") + Hive sink，需要 Spark 开启 enableHiveSupport()。

3. 这里为了简单，用的是 处理时间窗口（current_timestamp()），如果你想改成 事件时间窗口，需要在 Order 类里加 ts 字段并解析出来，再配合 withWatermark() 使用。

 转发请注明出处：https://www.cnblogs.com/fnlingnzb-learner/p/19073626