kafka消费者配置

最近的业务接触了一下消息插件kafka ，其他客户端负责发送，我方负责接收消费。功能很简单，但是！！！！我们要求把消息入库，而且数据量每天达到了千万级别，这就日了苟啊！

 

废话不多说，上代码！

 

由于消息需要入库，我使用的是手动提交消息，如果不入库，不要求准确定，使用自动提交就ok

 

消费者配置类

package com.asiainfo.hsop.server.kafka;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.config.KafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.listener.AbstractMessageListenerContainer;
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
import org.springframework.kafka.listener.config.ContainerProperties;

import java.util.HashMap;
import java.util.Map;
/**
 * kafka消息队列消费者 配置类
 *
 * @author cyc
 * @version 1.0.0
 * @date 2021-07-02
 *
 * */
@Configuration
@EnableKafka
public class KafkaConsumerConfig {
    /*
    *
    *启动项目之前 需要修改 etc/hosts 文件，把 kafka实例名称和主机ip对应关系填好，否则会报错 ，
    *
    *报错信息如下：
    *           Discovered group coordinator......
    *           (Re-)joining group......
    *           Marking the coordinator......
    *尝试升级kafka-clients ，但是没有啥卵用.......最好的解决方案 就是修改系统hosts文件
    *
    *   主机ip 和 实例名称 对应关系
    *   11.251.10.111 sss-nn-01
    *   
    *
    *  */
    public Map<String, Object> consumerConfig(String consumerGroupId) {
        Map<String, Object> props = new HashMap<>();
        //kafka服务地址
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "10.253.6.194:6667,10.253.6.195:6667,10.253.6.196:6667,10.253.6.197:6667,10.253.6.198:6667,10.253.6.199:6667,10.253.6.200:6667,10.253.6.201:6667");
        //消费后是否自动提交 当前为false
        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
        //获取消息后提交偏移量的最大时间
        props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "100");
        //超时时间，服务端没有收到心跳就会认为当前消费者失效
        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000");
        //默认消费组
        props.put(ConsumerConfig.GROUP_ID_CONFIG, consumerGroupId);
        //earliest  从头开始消费、latest获取最新消息 、none 没理解
        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
        //序列化
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        return props;
    }

    public ConsumerFactory<String, String> consumerFactory(String consumerGroupId) {
        return new DefaultKafkaConsumerFactory<>(consumerConfig(consumerGroupId));
    }

    /*有多个消费组监听，直接复制改方法，
    * 修改 @Bean(name = "kafkaListenerContainerFactory") name重新命名
    *
    * factory.setConsumerFactory(consumerFactory("etctu_tradeinfolist_test")); 改下消费组 名称
    *
    * 在调用的时候containerFactory="kafkaListenerContainerFactory" 改一下就ok
    *
    * */
    @Bean(name = "kafkaListenerContainerFactory")
    public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
        ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory("etctu_tradeinfolist_test"));
　　　　　//监听的线程数量，多个线程入库，数据库的id是自增的话，可能导致死锁，建议使用UUID
        factory.setConcurrency(1);
        /*消费者有两种消费模式，一种是kafka实例主动推送push模式，推送速度由kafka决定，很有可能导致消费端阻塞，
        *另一种就是 消费者主动拉取，poll模式
        * */
        factory.getContainerProperties().setPollTimeout(3000);
        //当使用手动提交时必须设置ackMode为MANUAL,否则会报错No Acknowledgment available as an argument, the listener container must have a MANUAL AckMode to populate the Acknowledgment.
        //
        factory.getContainerProperties().setAckMode(AbstractMessageListenerContainer.AckMode.MANUAL);
        // 下面两个条件哪个先满足，就会先使用那个
        factory.getContainerProperties().setAckCount(10);//达到10条数据的时候提交一次
        factory.getContainerProperties().setAckTime(10000);//10s提交一次
        return factory;
    }



    
    @Bean(name = "kafkaListenerContainerFactory2")
    public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory2() { ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory("exit_toll_host_test"));
        factory.setConcurrency(1);
        factory.getContainerProperties().setPollTimeout(3000);
        factory.getContainerProperties().setAckMode(AbstractMessageListenerContainer.AckMode.MANUAL);
        factory.getContainerProperties().setAckCount(10);
        factory.getContainerProperties().setAckTime(10000);
        return factory;
    }

  
    @Bean(name = "kafkaListenerContainerFactory3")
    public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory3() { ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory("entryinfo_test"));
        factory.setConcurrency(1);
        factory.getContainerProperties().setPollTimeout(3000);
        //当使用手动提交时必须设置ackMode为MANUAL,否则会报错No Acknowledgment available as an argument, the listener container must have a MANUAL AckMode to populate the Acknowledgment.
        factory.getContainerProperties().setAckMode(AbstractMessageListenerContainer.AckMode.MANUAL);
        factory.getContainerProperties().setAckCount(10);
        factory.getContainerProperties().setAckTime(10000);
        return factory;
    }


}

　

 

消费者类，里面包含了Hbase 引入的包，没啥用，删掉就行

package com.asiainfo.hsop.server.kafka;

import com.alibaba.fastjson.JSON;
import com.asiainfo.hsop.base.po.data.kafka.HospKafkaDetail;

import com.asiainfo.hsop.common.utils.HBaseUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.support.Acknowledgment;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.math.BigDecimal;
import java.text.ParseException;
import java.util.*;

/**
 *
 * @author cyc
 *
 * @date 2021-07-02
 *
 *
 *
 *
 * */
@Component
public class Consumer {

    private Logger logger= LoggerFactory.getLogger(Consumer.class);




        @KafkaListener(topics = "highwayetctu",containerFactory="kafkaListenerContainerFactory")
        public void consumer1(ConsumerRecord<?, ?> record, Acknowledgment ack) throws InterruptedException, ParseException, IOException {

          　　//具体业务

　　　　　　　　//手动提交
            ack.acknowledge();
        }






        




       
        @KafkaListener(topics = "exitList_0706",containerFactory="kafkaListenerContainerFactory2")
        public void consumer2(ConsumerRecord<?, ?> record,Acknowledgment ack ) throws IOException, ParseException {
　　　　　　　　//具体业务
　　　　　　　　//手动提交
            ack.acknowledge();
        }





        @KafkaListener(topics = "entryinfo",containerFactory="kafkaListenerContainerFactory3")
        public void consumer3(ConsumerRecord<?, ?> record,Acknowledgment ack ) throws IOException, ParseException {

           　　//具体业务
　　　　　　
　　　　　　　　//手动提交
            ack.acknowledge();
        }

  




}