通过spring-kafka完成kafka消费和生产
通过spring-kafka完成kafka消费和生产
1、引入maven依赖
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
<version>2.2.0.RELEASE</version>
</dependency>
2、application.properties文件配置
kafka.producer.server=127.0.0.1:9092
kafka.producer.retries=2
kafka.producer.batch-size=1
kafka.producer.linger=1
kafka.producer.topic=my_data
kafka.bootstrap-servers=127.0.0.1:11056
kafka.consumer.enable.auto.commit=false
kafka.consumer.session.timeout=60000
kafka.heartbeat.interval.ms=20000
kafka.consumer.group.id=mygroup
kafka.consumer.auto.offset.reset=latest
msg.consumer.max.poll.records=1
kafka.consumer.thread.min=1
kafka.consumer.thread.max=1
msg.topic=my_data_2
3、配置文件Configuration
import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.config.SaslConfigs;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.config.KafkaListenerContainerFactory;
import org.springframework.kafka.core.*;
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
import org.springframework.kafka.listener.ContainerProperties;
import java.util.HashMap;
import java.util.Map;
@Configuration
@EnableKafka
public class KafkaConfig {
@Value("${kafka.bootstrap-servers}")
private String servers;
@Value("${kafka.producer.retries}")
private int retries;
@Value("${kafka.producer.batch-size}")
private int batchSize;
@Value("${kafka.producer.linger}")
private int linger;
@Value("${kafka.producer.server}")
private String producerServer;
@Value("${kafka.consumer.enable.auto.commit}")
private boolean enableAutoCommit;
@Value("${kafka.consumer.session.timeout}")
private String sessionTimeout;
@Value("${kafka.heartbeat.interval.ms}")
private String heatbeatInterVal;
@Value("${kafka.consumer.group.id}")
private String groupId;
@Value("${kafka.consumer.auto.offset.reset}")
private String autoOffsetReset;
@Value("${msg.consumer.max.poll.records}")
private int maxPollRecords;
public Map<String, Object> producerConfigs() {
Map<String, Object> props = new HashMap<>();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, producerServer);
props.put(ProducerConfig.RETRIES_CONFIG, retries);
props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize);
props.put(ProducerConfig.LINGER_MS_CONFIG, linger);
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
return props;
}
public ProducerFactory producerFactory() {
return new DefaultKafkaProducerFactory(producerConfigs());
}
@Bean
public KafkaTemplate kafkaTemplate() {
return new KafkaTemplate(producerFactory());
}
@Bean
public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, Object>>
kafkaListenerContainerFactory() {
ConcurrentKafkaListenerContainerFactory<String, Object> factory =
new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(consumerFactory());
factory.setBatchListener(true);
// 此处并发度设置的都是Consumer个数,可以设置1到partition总数,
// 但是,所有机器实例上总的并发度之和必须小于等于partition总数
// 如果,总的并发度小于partition总数,有一个Consumer实例会消费超过一个以上partition
factory.setConcurrency(2);
factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
return factory;
}
public ConsumerFactory<String, Object> consumerFactory() {
return new DefaultKafkaConsumerFactory<>(consumerConfigs());
}
public Map<String, Object> consumerConfigs() {
Map<String, Object> propsMap = new HashMap<>();
propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit);
propsMap.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimeout);
propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset);
propsMap.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords);
propsMap.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, heatbeatInterVal);
propsMap.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
propsMap.put(SaslConfigs.SASL_MECHANISM, "PLAIN");
propsMap.put("sasl.jaas.config",
"org.apache.kafka.common.security.plain.PlainLoginModule required username=\"user001\" password=\"password@123\";");
return propsMap;
}
}
4、消费者(抽象类)
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationListener;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.event.ConsumerStoppedEvent;
import org.springframework.kafka.support.Acknowledgment;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.List;
import java.util.concurrent.*;
public abstract class BaseConsumer implements ApplicationListener<ConsumerStoppedEvent> {
private static final Logger LOG = LoggerFactory.getLogger(BaseConsumer.class);
@Value("${kafka.consumer.thread.min}")
private int consumerThreadMin;
@Value("${kafka.consumer.thread.max}")
private int consumerThreadMax;
private ThreadPoolExecutor consumeExecutor;
private volatile boolean isClosePoolExecutor = false;
@PostConstruct
public void init() {
this.consumeExecutor = new ThreadPoolExecutor(
getConsumeThreadMin(),
getConsumeThreadMax(),
// 此处最大最小不一样没啥大的意义,因为消息队列需要达到 Integer.MAX_VALUE 才有点作用,
// 矛盾来了,我每次批量拉下来不可能设置Integer.MAX_VALUE这么多,
// 个人觉得每次批量下拉的原则 觉得消费可控就行,
// 不然,如果出现异常情况下,整个服务示例突然挂了,拉下来太多,这些消息会被重复消费一次。
1000 * 60,
TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>());
}
/**
* 收到spring-kafka 关闭Consumer的通知
* @param event 关闭Consumer 事件
*/
@Override
public void onApplicationEvent(ConsumerStoppedEvent event) {
isClosePoolExecutor = true;
closeConsumeExecutorService();
}
private void closeConsumeExecutorService() {
if (!consumeExecutor.isShutdown()) {
shutdownGracefully(consumeExecutor, 120, TimeUnit.SECONDS);
LOG.info("consumeExecutor stopped");
}
}
@PreDestroy
public void doClose() {
if (!isClosePoolExecutor) {
closeConsumeExecutorService();
}
}
@KafkaListener(topics = "${msg.topic}", containerFactory = "kafkaListenerContainerFactory")
public void onPostMessage(List<String> msgList, Acknowledgment ack) {
CountDownLatch countDownLatch = new CountDownLatch(msgList.size());
for (String message : msgList) {
submitConsumeTask(message, countDownLatch);
}
try {
countDownLatch.await();
} catch (InterruptedException e) {
LOG.error("countDownLatch exception ", e);
}
// 本次批量消费完,手动提交
ack.acknowledge();
//LOG.info("finish commit offset");
}
/*@KafkaListener(topics = "${msg.comment.topic}", containerFactory = "kafkaListenerContainerFactory")
public void onCommentMessage(List<String> msgList, Acknowledgment ack) {
CountDownLatch countDownLatch = new CountDownLatch(msgList.size());
for (String message : msgList) {
submitConsumeTask(message, countDownLatch, "comment");
}
try {
countDownLatch.await();
} catch (InterruptedException e) {
LOG.error("countDownLatch exception ", e);
}
// 本次批量消费完,手动提交
ack.acknowledge();
//LOG.info("finish commit offset");
}*/
private void submitConsumeTask(String message, CountDownLatch countDownLatch) {
consumeExecutor.submit(() -> {
try {
onDealMessage(message);
} catch (Exception ex) {
LOG.error("on DealMessage exception:", ex);
} finally {
countDownLatch.countDown();
}
});
}
/**
* 子类实现该抽象方法处理具体消息的业务逻辑
* @param message kafka的消息
*/
protected void onDealMessage(String message){
LOG.info("");
//System.out.println(message);
}
protected void onDealCommentMessage(String message){
LOG.info("");
// System.out.println(message);
}
private int getConsumeThreadMax() {
return consumerThreadMax;
}
private int getConsumeThreadMin() {
return consumerThreadMin;
}
public void setConsumerThreadMax(int consumerThreadMax) {
this.consumerThreadMax = consumerThreadMax;
}
public void setConsumerThreadMin(int consumerThreadMin) {
this.consumerThreadMin = consumerThreadMin;
}
public static void shutdownGracefully(ExecutorService executor, long timeout, TimeUnit timeUnit) {
// Disable new tasks from being submitted.
executor.shutdown();
try {
// Wait a while for existing tasks to terminate.
if (!executor.awaitTermination(timeout, timeUnit)) {
executor.shutdownNow();
// Wait a while for tasks to respond to being cancelled.
if (!executor.awaitTermination(timeout, timeUnit)) {
LOG.warn(String.format("%s didn't terminate!", executor));
}
}
} catch (InterruptedException ie) {
// (Re-)Cancel if current thread also interrupted.
executor.shutdownNow();
// Preserve interrupt status.
Thread.currentThread().interrupt();
}
}
}
5、消费者(实现类)。消费者取到数据后再通过生产者扔到另一处kafka里
import org.apache.kafka.clients.producer.ProducerRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageHeaders;
import org.springframework.stereotype.Service;
@Service
public class ConsumerService extends BaseConsumer{
@Value("${kafka.producer.topic}")
private String producerTopic;
@Autowired
private KafkaTemplate kafkaTemplate;
private static final Logger log = LoggerFactory.getLogger(ConsumerService.class);
@Override
protected void onDealMessage(String message) {
JSONObject json = JSONObject.parseObject(message);
//业务逻辑处理。。。。。。
ProducerRecord record = new ProducerRecord(producerTopic, producerTopic,message);
//生产数据
kafkaTemplate.send(record);
}
}
既要仰望星空,又要脚踏实地