Fork me on GitHub

通过spring-kafka完成kafka消费和生产

通过spring-kafka完成kafka消费和生产

1、引入maven依赖

<dependency>
			<groupId>org.springframework.kafka</groupId>
			<artifactId>spring-kafka</artifactId>
			<version>2.2.0.RELEASE</version>
</dependency>

2、application.properties文件配置

kafka.producer.server=127.0.0.1:9092
kafka.producer.retries=2
kafka.producer.batch-size=1
kafka.producer.linger=1
kafka.producer.topic=my_data

kafka.bootstrap-servers=127.0.0.1:11056
kafka.consumer.enable.auto.commit=false
kafka.consumer.session.timeout=60000
kafka.heartbeat.interval.ms=20000
kafka.consumer.group.id=mygroup
kafka.consumer.auto.offset.reset=latest
msg.consumer.max.poll.records=1
kafka.consumer.thread.min=1
kafka.consumer.thread.max=1
msg.topic=my_data_2

3、配置文件Configuration

import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.config.SaslConfigs;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.config.KafkaListenerContainerFactory;
import org.springframework.kafka.core.*;
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
import org.springframework.kafka.listener.ContainerProperties;
import java.util.HashMap;
import java.util.Map;
@Configuration
@EnableKafka
public class KafkaConfig {
    @Value("${kafka.bootstrap-servers}")
    private String servers;
    @Value("${kafka.producer.retries}")
    private int retries;
    @Value("${kafka.producer.batch-size}")
    private int batchSize;
    @Value("${kafka.producer.linger}")
    private int linger;
    @Value("${kafka.producer.server}")
    private String producerServer;
    @Value("${kafka.consumer.enable.auto.commit}")
    private boolean enableAutoCommit;
    @Value("${kafka.consumer.session.timeout}")
    private String sessionTimeout;
    @Value("${kafka.heartbeat.interval.ms}")
    private String heatbeatInterVal;
    @Value("${kafka.consumer.group.id}")
    private String groupId;
    @Value("${kafka.consumer.auto.offset.reset}")
    private String autoOffsetReset;
    @Value("${msg.consumer.max.poll.records}")
    private int maxPollRecords;

    public Map<String, Object> producerConfigs() {
        Map<String, Object> props = new HashMap<>();
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, producerServer);
        props.put(ProducerConfig.RETRIES_CONFIG, retries);
        props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize);
        props.put(ProducerConfig.LINGER_MS_CONFIG, linger);
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        return props;
    }

    public ProducerFactory producerFactory() {
        return new DefaultKafkaProducerFactory(producerConfigs());
    }

    @Bean
    public KafkaTemplate kafkaTemplate() {
        return new KafkaTemplate(producerFactory());
    }

    @Bean
    public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, Object>>
    kafkaListenerContainerFactory() {
        ConcurrentKafkaListenerContainerFactory<String, Object> factory =
                new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory());
        factory.setBatchListener(true);
        // 此处并发度设置的都是Consumer个数,可以设置1到partition总数,
        // 但是,所有机器实例上总的并发度之和必须小于等于partition总数
        // 如果,总的并发度小于partition总数,有一个Consumer实例会消费超过一个以上partition
        factory.setConcurrency(2);
        factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
        return factory;
    }

    public ConsumerFactory<String, Object> consumerFactory() {
        return new DefaultKafkaConsumerFactory<>(consumerConfigs());
    }

    public Map<String, Object> consumerConfigs() {
        Map<String, Object> propsMap = new HashMap<>();
        propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
        propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit);
        propsMap.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimeout);
        propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
        propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset);
        propsMap.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords);
        propsMap.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, heatbeatInterVal);
        propsMap.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
        propsMap.put(SaslConfigs.SASL_MECHANISM, "PLAIN");
        propsMap.put("sasl.jaas.config",
                "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"user001\" password=\"password@123\";");

        return propsMap;
    }
}

4、消费者(抽象类)

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationListener;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.event.ConsumerStoppedEvent;
import org.springframework.kafka.support.Acknowledgment;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.List;
import java.util.concurrent.*;

public abstract class BaseConsumer implements ApplicationListener<ConsumerStoppedEvent> {

    private static final Logger LOG = LoggerFactory.getLogger(BaseConsumer.class);

    @Value("${kafka.consumer.thread.min}")
    private int consumerThreadMin;

    @Value("${kafka.consumer.thread.max}")
    private int consumerThreadMax;

    private ThreadPoolExecutor consumeExecutor;

    private volatile boolean isClosePoolExecutor = false;

    @PostConstruct
    public void init() {

        this.consumeExecutor = new ThreadPoolExecutor(
                getConsumeThreadMin(),
                getConsumeThreadMax(),
                // 此处最大最小不一样没啥大的意义,因为消息队列需要达到 Integer.MAX_VALUE 才有点作用,
                // 矛盾来了,我每次批量拉下来不可能设置Integer.MAX_VALUE这么多,
                // 个人觉得每次批量下拉的原则 觉得消费可控就行,
                // 不然,如果出现异常情况下,整个服务示例突然挂了,拉下来太多,这些消息会被重复消费一次。
                1000 * 60,
                TimeUnit.MILLISECONDS,
                new LinkedBlockingQueue<>());
    }

    /**
     * 收到spring-kafka 关闭Consumer的通知
     * @param event 关闭Consumer 事件
     */
    @Override
    public void onApplicationEvent(ConsumerStoppedEvent event) {

        isClosePoolExecutor = true;
        closeConsumeExecutorService();

    }

    private void closeConsumeExecutorService() {

        if (!consumeExecutor.isShutdown()) {

            shutdownGracefully(consumeExecutor, 120, TimeUnit.SECONDS);
            LOG.info("consumeExecutor stopped");

        }

    }

    @PreDestroy
    public void doClose() {
        if (!isClosePoolExecutor) {
            closeConsumeExecutorService();
        }
    }

    @KafkaListener(topics = "${msg.topic}", containerFactory = "kafkaListenerContainerFactory")
    public void onPostMessage(List<String> msgList, Acknowledgment ack) {

        CountDownLatch countDownLatch = new CountDownLatch(msgList.size());

        for (String message : msgList) {
            submitConsumeTask(message, countDownLatch);
        }

        try {
            countDownLatch.await();
        } catch (InterruptedException e) {
            LOG.error("countDownLatch exception ", e);
        }

        // 本次批量消费完,手动提交
        ack.acknowledge();
        //LOG.info("finish commit offset");

    }

    /*@KafkaListener(topics = "${msg.comment.topic}", containerFactory = "kafkaListenerContainerFactory")
    public void onCommentMessage(List<String> msgList, Acknowledgment ack) {

        CountDownLatch countDownLatch = new CountDownLatch(msgList.size());

        for (String message : msgList) {
            submitConsumeTask(message, countDownLatch, "comment");
        }

        try {
            countDownLatch.await();
        } catch (InterruptedException e) {
            LOG.error("countDownLatch exception ", e);
        }

        // 本次批量消费完,手动提交
        ack.acknowledge();
        //LOG.info("finish commit offset");

    }*/

    private void submitConsumeTask(String message, CountDownLatch countDownLatch) {
        consumeExecutor.submit(() -> {
            try {
                onDealMessage(message);
            } catch (Exception ex) {
                LOG.error("on DealMessage exception:", ex);
            } finally {
                countDownLatch.countDown();
            }
        });
    }

    /**
     * 子类实现该抽象方法处理具体消息的业务逻辑
     * @param message kafka的消息
     */
    protected void onDealMessage(String message){
        LOG.info("");
        //System.out.println(message);
    }

    protected void onDealCommentMessage(String message){
        LOG.info("");
       // System.out.println(message);
    }


    private int getConsumeThreadMax() {
        return consumerThreadMax;
    }

    private int getConsumeThreadMin() {
        return consumerThreadMin;
    }

    public void setConsumerThreadMax(int consumerThreadMax) {
        this.consumerThreadMax = consumerThreadMax;
    }

    public void setConsumerThreadMin(int consumerThreadMin) {
        this.consumerThreadMin = consumerThreadMin;
    }

    public static void shutdownGracefully(ExecutorService executor, long timeout, TimeUnit timeUnit) {
        // Disable new tasks from being submitted.
        executor.shutdown();
        try {
            // Wait a while for existing tasks to terminate.
            if (!executor.awaitTermination(timeout, timeUnit)) {
                executor.shutdownNow();
                // Wait a while for tasks to respond to being cancelled.
                if (!executor.awaitTermination(timeout, timeUnit)) {
                    LOG.warn(String.format("%s didn't terminate!", executor));
                }
            }
        } catch (InterruptedException ie) {
            // (Re-)Cancel if current thread also interrupted.
            executor.shutdownNow();
            // Preserve interrupt status.
            Thread.currentThread().interrupt();
        }
    }
}

5、消费者(实现类)。消费者取到数据后再通过生产者扔到另一处kafka里

import org.apache.kafka.clients.producer.ProducerRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageHeaders;
import org.springframework.stereotype.Service;

@Service
public class ConsumerService extends BaseConsumer{
    @Value("${kafka.producer.topic}")
    private String producerTopic;
    @Autowired
    private KafkaTemplate kafkaTemplate;

    private static final Logger log = LoggerFactory.getLogger(ConsumerService.class);
    @Override
    protected void onDealMessage(String message) {
       JSONObject json = JSONObject.parseObject(message);
        //业务逻辑处理。。。。。。
       ProducerRecord record = new ProducerRecord(producerTopic, producerTopic,message);
       //生产数据 
       kafkaTemplate.send(record);
    }
}

posted @ 2021-03-30 10:21  ayueC  阅读(433)  评论(0)    收藏  举报