十五、FLINK应用一：关联查询

一、需求：给定一个字符串，其中包含id字段，需从数据库解析相应的名称返回

二、核心代码：

public class QueryRelaDb {
    public static void main(String[] args) throws Exception {
        DataStreamSource<String> source = KafkaUtil.kafkaSource(args[0], args[1], args[2]);
        SingleOutputStreamOperator<Acitivity>  oper = source.map(new ActivityMapFunc());
        oper.print();
        KafkaUtil.getStreamExecutionEnvironment().execute("QueryRelaDb");
    }
}

ActivityMapFunc:

public class ActivityMapFunc extends RichMapFunction<String, Acitivity>{
    private static final long serialVersionUID = -5958085170167500655L;
    private Connection conn = null;
    
    @Override
    public void open(Configuration parameters) throws Exception {
        conn = DriverManager.getConnection("xxxx", "xdsu", "123456");
    }
    
    @Override
    public Acitivity map(String input) throws Exception {
        String[] splitStr = input.split(",");
        Long acitivityId = Long.parseLong(splitStr[1]);
        String name = null;
        PreparedStatement state = conn.prepareStatement("select activity_name from activity where activity_id = ?");
        state.setLong(1, acitivityId);
        ResultSet rs = state.executeQuery();
        if(rs.next()) {
            name = rs.getString(1);
        }
        rs.close();
        Acitivity acitivity = Acitivity.of(splitStr[0], acitivityId, name, splitStr[2]);
        return acitivity;
    }

    @Override
    public void close() throws Exception {
        if(conn != null) {
            conn.close();
        }
    }
}

KafkaUtil

public class KafkaUtil {
    private static StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    
    public static DataStreamSource<String> kafkaSource(String topic, String brokerList, String groupId) {
        Properties props = new Properties();
        //指定kafka的Broke地址
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList);
        //指定组ID
        props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
        //如果没有记录偏移量，第一次从最开始消费
        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
        //kafka消费者自动提交偏移量
        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");

        //KafkaSource
        FlinkKafkaConsumer<String> kafkaSource = new FlinkKafkaConsumer<String>(topic, new SimpleStringSchema(), props);
        return env.addSource(kafkaSource);
    }
    
    public static StreamExecutionEnvironment getStreamExecutionEnvironment() {
        return env;
    }
}

 三、异步实现方式

核心代码：

    public static void main(String[] args) throws Exception {
        DataStreamSource<String> source = KafkaUtil.kafkaSource(args[0], args[1], args[2]);
        SingleOutputStreamOperator<String>  oper = AsyncDataStream.unorderedWait(source, new AsyncActivityMapFunc(), 0, TimeUnit.MICROSECONDS);
        oper.print();
        KafkaUtil.getStreamExecutionEnvironment().execute("QueryRelaDb");
    }

AsyncActivityMapFunc:

public class AsyncActivityMapFunc extends RichAsyncFunction<String, String>{
    private static final long serialVersionUID = -5958085170167500655L;
    private transient ExecutorService executorService;
    private transient DruidDataSource dataSource;

    @Override
    public void open(Configuration parameters) throws Exception {
        executorService = Executors.newFixedThreadPool(10);
        dataSource = new DruidDataSource();
        dataSource.setDriverClassName("com.mysql.jdbc.Driver");
        dataSource.setUrl("xxxxxxxxxxxx");
        dataSource.setUsername("xxx");
        dataSource.setPassword("xxx");
        dataSource.setMaxActive(50);
        dataSource.setMinIdle(10);
        dataSource.setInitialSize(5);
    }
    
    @Override
    public void asyncInvoke(String id, ResultFuture<String> resultFuture)
            throws Exception {
        Future<String> future = executorService.submit(new Callable<String>() {
            @Override
            public String call() throws Exception {
                return queryFromMysql(id);
            }
        });
        CompletableFuture.supplyAsync(new Supplier<String>() {
            @Override
            public String get() {
                try {
                    return future.get();
                } catch (Exception e) {
                    return null;
                }
            }
        }).thenAccept(new Consumer<String>() {
            @Override
            public void accept(String activeName) {
                resultFuture.complete(Collections.singleton(activeName));
            }
        });
    }
    
    private String queryFromMysql(String id) throws Exception {
        String name = null;
        Connection conn = null;
        PreparedStatement state = null;
        ResultSet rs = null;
        try {
            conn = dataSource.getConnection();
            state = conn.prepareStatement("select activity_name from activity where activity_id = ?");
            state.setString(1, id);
            rs = state.executeQuery();
            if (rs.next()) {
                name = rs.getString(1);
            }
        } finally {
            rs.close();
            state.close();
            conn.close();
        }
        return name;
    }

    @Override
    public void close() throws Exception {
        if(executorService != null) {
            executorService.shutdown();
        }
        if(dataSource != null) {
            dataSource.close();
        }
    }
}