Springboot如何实现消费不同的kafka集群

1、问题

因业务需求，同一个springboot项目需要消费不同的kafka集群，那么该如何配置和开发呢？

2、主要技术点实现

• KafkaListenerContainerFactory<> 加入不同的kafka集群配置。其中bootstrap.servers ，group.id可以分别配置

@Configuration
public class KafkaConfig {

    @Bean
    public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
        ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory("group1", "bootstrap.servers=server1:9092", false));
        factory.setConcurrency(3);
        return factory;
    }

    @Bean
    public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory2() {
        ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory("group2", "bootstrap.servers=server2:9092", true));
        factory.setConcurrency(3);
        return factory;
    }

    private ConsumerFactory<String, String> consumerFactory(String groupId, String bootstrapServers, Boolean needAuth) {
        Map<String, Object> props = new HashMap<>();
        props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        if (needAuth) {
            properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, security_protocol);
            properties.put(SaslConfigs.SASL_MECHANISM, sasl_mechanism);
            properties.put(SaslConfigs.SASL_JAAS_CONFIG, sasl_jaas_config);
        }
        return new DefaultKafkaConsumerFactory<>(props);
    }
}

 然后，在创建 KafkaListener 时，使用 containerFactory 属性来选择要使用的 KafkaListenerContainerFactory，如下所示：

@KafkaListener(topics = "topic1", containerFactory = "kafkaListenerContainerFactory")
public void listenTopic1(String message) {
    // handle message from topic1
}

@KafkaListener(topics = "topic2", containerFactory = "kafkaListenerContainerFactory2")
public void listenTopic2(ConsumerRecord<?, ?> record, Acknowledgment ack) {
 // handle message from topic2 using a different KafkaListenerContainerFactory

　　Optional<?> kafkaMessage = Optional.ofNullable(record.value());

　　if (kafkaMessage.isPresent()) {
　　　　Object message = kafkaMessage.get();
   　　log.info("kafka listen apiGroup msg: {}", message);
   　　this.getReportSourceService().handleApiGroupKafkaMsg(message);
   　　ack.acknowledge();

　　}
}