java内存中快速数据处理方法
有一种场景,从上游消息队列中,实时获取大量数据,需要尽快处理完,入库,分析程序可以通过多线程往阻塞队列里存放,然后当数据达到一定条数(例如50)或者达到一定时间(例如3分钟),满足任一条件,则异步线程批量处理。
package com.nari.midplatform.local.email.task; import java.util.ArrayList; import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; public class BatchQueue { private final int batchSize; private final Consumer<List<Object>> consumer; private final int timeoutInMs; private AtomicBoolean isLooping = new AtomicBoolean(false); private BlockingQueue<Object> queue = new LinkedBlockingQueue<2000>(); private ExecutorService executorService = Executors.newCachedThreadPool(); private AtomicLong start = new AtomicLong(System.currentTimeMillis()); public BatchQueue(int batchSize, int timeoutInMs, Consumer<List<Object>> consumer) { this.batchSize = batchSize; this.timeoutInMs = timeoutInMs; this.consumer = consumer; } public BatchQueue(int batchSize, Consumer<List<Object>> consumer) { this(batchSize, 500, consumer); } public boolean add(Object t) { boolean result = queue.add(t); if(!isLooping.get() && result) { isLooping.set(true); startLoop(); } return result; } public void completeAll() { while (!queue.isEmpty()) { drainToConsume(); } } private void startLoop() { executorService.execute(new ExeThread()); } private void drainToConsume() { List<Object> drained = new ArrayList<>(); int num = queue.drainTo(drained, batchSize); if(num > 0) { consumer.accept(drained); start.set(System.currentTimeMillis()); } } private class ExeThread implements Runnable { @Override public void run() { start = new AtomicLong(System.currentTimeMillis()); while(true) { long last = System.currentTimeMillis() - start.get() ; if (queue.size() >= batchSize || (!queue.isEmpty() && last > timeoutInMs)) { drainToConsume(); } else if(queue.isEmpty()) { isLooping.set(false); break; } } } } public static void main(String[] args) { BatchQueue batchQueue = new BatchQueue(3, 5000, x -> exe(x)); for (int i = 0; i < 36; i++) { batchQueue.add(i); } } private static void exe (List<Object> o) { System.out.println("处理数据:" + o); } }

浙公网安备 33010602011771号