算法 - 布隆过滤器

要点：有n条数据，通过m个hash函数，将每条数据都转化为m个数，存到0-x的数集中。查询时，先将数据转化为m个数，如果m个数不能在数集中全都找到，说明该数据一定不存在。但是，全都找到，不能说明数据一定存在。

import redis.clients.jedis.Jedis;
import redis.clients.jedis.Pipeline;

import java.io.IOException;

public class BloomFilter {

    /**
     * m -> 【布隆过滤器长度】
     * n -> 插入元素个数
     * p -> 误报率
     * m = - nlnp / [(ln2)*(ln2)]
     * <p>
     * k -> 【哈希函数个数】
     * k = m * ln2 / n
     */

    private long filterLen;
    private int hashFuncNum;
    private static Jedis jedis;

    public BloomFilter(long elementNum, double errorRate) {
        this.filterLen = this.getFilterLen(elementNum, errorRate);
        this.hashFuncNum = this.getHashFuncNum(this.filterLen, elementNum);
    }

    static {
        jedis = new Jedis("127.0.0.1", 6379);
        jedis.auth("password");
    }

    /**
     * 计算哈希函数数量
     *
     * @param filterLen  过滤器长度
     * @param elementNum 元素个数
     * @return 哈希函数数量
     */
    private int getHashFuncNum(long filterLen, long elementNum) {
        return new Double(Math.floor(filterLen * Math.log(2) / elementNum)).intValue();
    }

    /**
     * 计算布隆过滤器长度
     *
     * @param elementNum 元素格式
     * @param errorRate  误报率
     * @return 布隆过滤器长度
     */
    private long getFilterLen(long elementNum, double errorRate) {
        double m = -1 * elementNum * Math.log(errorRate) / Math.pow(Math.log(2), 2);
        return new Double(m).longValue();
    }

    /**
     * 添加元素
     *
     * @param sign
     * @param key
     */
    public void add(String sign, String key) {
        long[] hashArr = hash(key);
        Pipeline pipeline = jedis.pipelined();
        try {
            for (long hash : hashArr) {
                pipeline.setbit(sign, hash, true);
            }
            pipeline.sync();
        } finally {
            try {
                pipeline.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    public boolean notExist(String sign, String key) {
        long[] hashArr = hash(key);
        boolean result;
        Pipeline pipeline = jedis.pipelined();
        try {
            for (long hash : hashArr) {
                pipeline.getbit(sign, hash);
            }
            result = pipeline.syncAndReturnAll().contains(false);
        } finally {
            try {
                pipeline.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        if (result) {
            add(sign, key);
        }
        return result;
    }

    private long[] hash(String key) {
        long hash1 = fnvHash(key);
        long hash2 = hash1 >>> 16;
        long[] result = new long[this.hashFuncNum];
        for (int i = 0; i < this.hashFuncNum; i++) {
            long hash = hash1 + i * hash2;
            if (hash < 0) {
                hash = ~hash;
            }
            result[i] = hash % this.filterLen;
        }
        return result;
    }

    public static int fnvHash(String key) {
        int hash = (int) 2166136261L;
        for (int i = 0; i < key.length(); i++) {
            hash += (hash * 16777619) ^ key.charAt(i);
        }
        return hash & Integer.MAX_VALUE;
    }

    public static void main(String[] args) {
        String sign = "Bloom-Filter";
        BloomFilter bf = new BloomFilter(100000, 0.01);
        bf.add(sign, "aaaaaa");
        bf.add(sign, "bbbbbb");
        System.out.println(bf.notExist(sign, "aaaaaa"));    // false
        System.out.println(bf.notExist(sign, "ffffff"));    // true 如果不存在会写入，再执行为false
    }

}