布隆过滤器实现部分场景下的高性能

过滤器

这个是概率型数据结构，主要用于判断元素是否可能存在于集合中，存在一定的误判率。在并发场景下，保证操作的线程安全，由于其概率特性，不同线程对元素存在性的判断可能会因为误判而出现不一致的情况。例如，一个线程判断某个元素存在，而实际上该元素可能并不存在于集合中。

 

maven依赖

<dependency>
    <groupId>com.google.guava</groupId>
    <artifactId>guava</artifactId>
    <version>32.1.3-jre</version>
</dependency>

实现布隆过滤器

import com.google.common.hash.BloomFilter;
import com.google.common.hash.Funnels;

import java.nio.charset.StandardCharsets;

public class RequestProcessor {
    // 预计插入的元素数量
    private static final int EXPECTED_INSERTIONS = 10000;
    // 期望的误判率
    private static final double FALSE_POSITIVE_PROBABILITY = 0.01;

    // 创建布隆过滤器
    private static final BloomFilter<CharSequence> processedRequestIds = BloomFilter.create(
            Funnels.stringFunnel(StandardCharsets.UTF_8),
            EXPECTED_INSERTIONS,
            FALSE_POSITIVE_PROBABILITY
    );

    public static boolean isRequestProcessed(String requestId) {
        // 判断请求 ID 是否已经存在于布隆过滤器中
        if (processedRequestIds.mightContain(requestId)) {
            // 这里存在一定的误判率，可能需要结合其他方式进一步确认
            return true;
        }
        // 如果不存在，则将请求 ID 添加到布隆过滤器中
        processedRequestIds.put(requestId);
        return false;
    }

    public static void main(String[] args) {
        String requestId1 = "12345";
        String requestId2 = "67890";

        // 处理第一个请求
        boolean isProcessed1 = isRequestProcessed(requestId1);
        System.out.println("Request ID 1 processed: " + isProcessed1);

        // 再次处理第一个请求
        boolean isProcessed2 = isRequestProcessed(requestId1);
        System.out.println("Request ID 1 processed again: " + isProcessed2);

        // 处理第二个请求
        boolean isProcessed3 = isRequestProcessed(requestId2);
        System.out.println("Request ID 2 processed: " + isProcessed3);
    }
}

测试并发安全

import com.google.common.hash.BloomFilter;
import com.google.common.hash.Funnels;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class BloomFilterConcurrentExample {
    private static final int EXPECTED_INSERTIONS = 10000;
    private static final double FALSE_POSITIVE_PROBABILITY = 0.01;
    private static final BloomFilter<CharSequence> processedRequestIds = BloomFilter.create(
            Funnels.stringFunnel(StandardCharsets.UTF_8),
            EXPECTED_INSERTIONS,
            FALSE_POSITIVE_PROBABILITY
    );

    public static void main(String[] args) {
        ExecutorService executor = Executors.newFixedThreadPool(10);
        for (int i = 0; i < 100; i++) {
            final int id = i;
            executor.submit(() -> {
                String requestId = String.valueOf(id);
                if (processedRequestIds.mightContain(requestId)) {
                    System.out.println("Request ID " + requestId + " might be processed.");
                } else {
                    processedRequestIds.put(requestId);
                    System.out.println("Request ID " + requestId + " is marked as processed.");
                }
            });
        }
        executor.shutdown();
    }
}