java内存中快速数据处理方法

有一种场景,从上游消息队列中,实时获取大量数据,需要尽快处理完,入库,分析程序可以通过多线程往阻塞队列里存放,然后当数据达到一定条数(例如50)或者达到一定时间(例如3分钟),满足任一条件,则异步线程批量处理。

package com.nari.midplatform.local.email.task;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;



public class BatchQueue {
    private final int batchSize;
    private final Consumer<List<Object>> consumer;
    private final int timeoutInMs;

    private AtomicBoolean isLooping = new AtomicBoolean(false);
    private BlockingQueue<Object> queue = new LinkedBlockingQueue<2000>();
    private ExecutorService executorService = Executors.newCachedThreadPool();

    private AtomicLong start = new AtomicLong(System.currentTimeMillis());

    public BatchQueue(int batchSize, int timeoutInMs, Consumer<List<Object>> consumer) {
        this.batchSize = batchSize;
        this.timeoutInMs = timeoutInMs;
        this.consumer = consumer;
    }

    public BatchQueue(int batchSize, Consumer<List<Object>> consumer) {
        this(batchSize, 500, consumer);
    }

    public boolean add(Object t) {
        boolean result = queue.add(t);
        if(!isLooping.get() && result) {
            isLooping.set(true);
            startLoop();
        }
        return result;
    }

    public void completeAll() {
        while (!queue.isEmpty()) {
            drainToConsume();
        }
    }

    private void startLoop() {
        executorService.execute(new ExeThread());
    }

    private void drainToConsume() {
        List<Object> drained = new ArrayList<>();
        int num = queue.drainTo(drained, batchSize);
        if(num > 0) {
            consumer.accept(drained);
            start.set(System.currentTimeMillis());
        }
    }

    private class ExeThread implements Runnable {
        @Override
        public void run() {
            start = new AtomicLong(System.currentTimeMillis());
            while(true) {
                long last = System.currentTimeMillis() - start.get() ;
                if (queue.size() >= batchSize || (!queue.isEmpty() && last > timeoutInMs)) {
                    drainToConsume();
                } else if(queue.isEmpty()) {
                    isLooping.set(false);
                    break;
                }
            }
        }
    }
    
    public static void main(String[] args) {
        BatchQueue batchQueue = new BatchQueue(3, 5000, x -> exe(x));
        for (int i = 0; i < 36; i++) {
            batchQueue.add(i);
        }
    }
    
    private static void exe (List<Object> o) {
        System.out.println("处理数据:" + o);
    }

}

 

posted @ 2026-04-02 10:26  技术虫  阅读(10)  评论(0)    收藏  举报