IO流从某路径下读取文件，并存储到es中（防重复消费，防消息丢失，防消息积压，消息顺序接收）

1.实体类

@Document(indexName = "index_log",type = "logread")
public class LogRead {
    /**
     * 日志id
     */
    private Integer id;
    /**
     * 日志内容
     */
    private String content;

    public LogRead(Integer id, String content) {
        this.id = id;
        this.content = content;
    }

    public Integer getId() {
        return id;
    }

    public void setId(Integer id) {
        this.id = id;
    }

    public String getContent() {
        return content;
    }

    public void setContent(String content) {
        this.content = content;
    }
}

 

2.es接口

package com.zcb.es;

import com.zcb.entity.LogRead;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;

public interface LogRepository extends ElasticsearchRepository<LogRead,Integer> {
}

 

3.es.xml配置文件

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xmlns:context="http://www.springframework.org/schema/context"
    xmlns:elasticsearch="http://www.springframework.org/schema/data/elasticsearch"
    xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
        http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-4.3.xsd
        http://www.springframework.org/schema/data/elasticsearch http://www.springframework.org/schema/data/elasticsearch/spring-elasticsearch-1.0.xsd">

    <!--项目路径扫描-->
    <!-- 扫描Dao包，自动创建实例(指定一个es仓库的包扫描位置) -->
    <!--我们 主要是用spring-data的方式来操作es的增删改查 -->
    <!-- 这个包下就是我们声明的es的仓库接口 :用来操作es的CRUD-->
     <elasticsearch:repositories base-package="com.zcb.es" />
    <!--ElasticsearchRepository<User,Integer>interfaceextends-->

<!-- 声明elasticsearchTemplate -->
    <bean name="elasticsearchTemplate"
        class="org.springframework.data.elasticsearch.core.ElasticsearchTemplate">
        <constructor-arg name="client" ref="client" />
    </bean>
    <!-- 声明elasticsearch客户端 -->
    <!-- 浏览器访问es的端口号是9200   Java代码连接es的端口号是9300 -->
    <elasticsearch:transport-client id="client" cluster-nodes="192.168.13.131:9300" />

</beans>

 

4.测试类

@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = "classpath:spring.xml")
public class test {

    @Autowired
    KafkaTemplate kafkaTemplate;


    @Test
    public void readFile() throws InterruptedException, IOException {
        String path = "d:/logs/";
        System.out.println("开始读取");
        int i = 0;
        this.getFile(new File(path));
        Thread.sleep(100000);
        System.out.println("结束读取");
    }

    private void getFile(File file) throws IOException {
        File[] files = file.listFiles();
        if(null!=files && files.length>0){
            for (File file1 : files) {
                if(file1.isFile()){
                    System.out.println("++++++>>>"+file1.getName());
                    BufferedReader reader = new BufferedReader(new FileReader(file1));
                    String str = "";
                    while((str=reader.readLine())!=null){
                        System.out.println("读取到的："+str);
　　　　　　　　　　　　　　　//UUID 是为了防重复消费
                        kafkaTemplate.sendDefault("import", UUID.randomUUID()+str);
                    }
                }else{
　　　　　　　　　　　　//如果是目录则调用自己
                    this.getFile(file1);
                }
            }
        }
    }
}

 

5.消费者

package com.zcb.kafka;

import com.alibaba.fastjson.JSON;
import com.zcb.entity.LogRead;
import com.zcb.entity.Menu;
import com.zcb.es.LogRepository;
import com.zcb.es.MenuElasticSearch;
import com.zcb.service.MenuService;
import com.zcb.utils.FileUtils;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.kafka.listener.AcknowledgingMessageListener;
import org.springframework.kafka.support.Acknowledgment;

import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;

/**
 * @author ：Lenny_z
 * @description：TODO
 * @date ：2021-12-16 14:55
 * spring_redis_demo2
 */
//消费者监听器
public class MyMessageListener implements AcknowledgingMessageListener<String,String> {

    @Autowired
    RedisTemplate redisTemplate;
    @Autowired
    LogRepository logRepository;


    //队列
    LinkedBlockingQueue<ConsumerRecord<String, String>> lbq = new LinkedBlockingQueue<>();
    //公平锁
    ReentrantLock lock = new ReentrantLock(true);

    /**
     * @author: zcb
     * @description: TODO
     * @date: 2021-12-16 16:34
     * @param data
     * @param acknowledgment
     * @return void
     */
    int i=0;
    @Override
    public void onMessage(ConsumerRecord<String, String> data, Acknowledgment acknowledgment) {
        //1.开启线程  用来吧消息放入队列
        new Thread(new Runnable() {
            @Override
            public void run() {
                try {
                    //放入队列
                    lbq.put(data);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        }).start();

        //2.开启线程 消费消息
        new Thread(new Runnable() {
            @Override
            public void run() {
                //判断是否为空 防止空指针异常
                ConsumerRecord<String, String> data = lbq.poll();
                if(null!=data){
                    myWork(data, acknowledgment);
                }
            }
        }).start();


    }

    /**
     * 具体业务
     */
    private void myWork(ConsumerRecord<String, String> data, Acknowledgment acknowledgment){
        //加锁
        lock.lock();
        String key = data.key();
        String value = data.value();
        if(key.equals("import")){
            /**
             *  读取日志文件
             *  1.截取日志文件的前36位
             *  2.防止重复消费 --默认该uuid存储七天
             */
            String uuid = value.substring(0, 35);
            if(redisTemplate.opsForValue().setIfAbsent(uuid,1,7,TimeUnit.DAYS)){
                logRepository.save(new LogRead(i++,value));
                acknowledgment.acknowledge();
            }
            System.out.println("log 已经存入es");
            //解锁
            lock.unlock();
        }
    }
}