bloomFilter

struct bloomfilter_t
	{
		int entries;
		//输入的字符串的个数
		double error;
		//错误率
		int bits;
		//位数组的比特数
		int bytes;
		//位数组的字节数：bits/8
		int hashes;
		//hash函数的个数
		double bpe;
		//bpe=-log(f)/ln(2)^2
		unsigned char * bf;
		//位数组的首地址
		int ready;
		//记录bloomFilter是否正确初始化
	};

　

bloom->bf = (unsigned char *)calloc(bloom->bytes, sizeof(unsigned char));

　　

static unsigned int murmurhash2(const void * key, int len, const unsigned int seed)
{
	const unsigned int m = 0x5bd1e995;
	const int r = 24;
	unsigned int h = seed ^ len;
	const unsigned char * data = (const unsigned char *)key;
	while (len >= 4) {
		unsigned int k = *(unsigned int *)data;
		k *= m;
		k ^= k >> r;
		k *= m;
		h *= m;
		h ^= k;
		data += 4;
		len -= 4;
	}
	switch (len) {
	case 3:
		h ^= data[2] << 16;
	case 2:
		h ^= data[1] << 8;
	case 1:
		h ^= data[0];
		h *= m;
	};
	h ^= h >> 13;
	h *= m;
	h ^= h >> 15;
	return h;
}

　　

static int bloom_test_bit_set_bit(unsigned char * buf, unsigned int x, int set_bit)

　　

#include <assert.h>
#include <fcntl.h>
#include <math.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include<time.h>
#include<windows.h>

struct bloomfilter_t
{
	int entries;
	//输入的字符串的个数
	double error;
	//错误率
	int bits;
	//位数组的比特数
	int bytes;
	//位数组的字节数：bits/8
	int hashes;
	//hash函数的个数
	double bpe;
	//bpe=-log(f)/ln(2)^2
	unsigned char * bf;
	//位数组的首地址
	int ready;
	//记录bloomFilter是否正确初始化
};

static unsigned int murmurhash2(const void * key, int len, const unsigned int seed)
{
	const unsigned int m = 0x5bd1e995;
	const int r = 24;
	unsigned int h = seed ^ len;
	const unsigned char * data = (const unsigned char *)key;
	while (len >= 4) {
		unsigned int k = *(unsigned int *)data;
		k *= m;
		k ^= k >> r;
		k *= m;
		h *= m;
		h ^= k;
		data += 4;
		len -= 4;
	}
	switch (len) {
	case 3:
		h ^= data[2] << 16;
	case 2:
		h ^= data[1] << 8;
	case 1:
		h ^= data[0];
		h *= m;
	};
	h ^= h >> 13;
	h *= m;
	h ^= h >> 15;
	return h;
}
//检测这个比特位是否已经置为1，如果置为1，则返回1，否则根据是否set_bit,将对对应的比特位（c|mask）设置为1
static int bloom_test_bit_set_bit(unsigned char * buf, unsigned int x, int set_bit)
{
	unsigned int byte = x >> 3;
	// expensive memory access
	unsigned char c = buf[byte];
	unsigned int mask = 1 << (x % 8);
	if (c & mask) {
		return 1;
	}
	else {
		if (set_bit) {
			buf[byte] = c | mask;
		}
		return 0;
	}
}

static int bloom_check_add(struct bloomfilter_t * bloom, const void * buffer, int len, int add)
{
	if (bloom->ready == 0) {
		printf("bloom at %p not initialized!\n", (void *)bloom);
		return -1;
	}
	int hits = 0;
	register unsigned int a = murmurhash2(buffer, len, 0x9747b28c);
	register unsigned int b = murmurhash2(buffer, len, a);
	register unsigned int x;
	register unsigned int i;
	for (i = 0; i < (unsigned int)bloom->hashes; i++) {
		x = (a + i*b) % bloom->bits;
		if (bloom_test_bit_set_bit(bloom->bf, x, add)) {
			hits++;
		}
	}
	if (hits == bloom->hashes) {
		// 1 == element already in (or collision)
		return 1;
	}
	return 0;
}

static int bloomfilter_init(struct bloomfilter_t * bloom, int entries, double error)
{
	bloom->ready = 0;

	if (entries < 1 || error == 0) {
		return 1;
	}

	bloom->entries = entries;
	bloom->error = error;

	double num = log(bloom->error);
	double denom = 0.480453013918201; // ln(2)^2
	bloom->bpe = -(num / denom);

	double dentries = (double)entries;
	bloom->bits = (int)(dentries * bloom->bpe);

	if (bloom->bits % 8) {
		bloom->bytes = (bloom->bits / 8) + 1;
	}
	else {
		bloom->bytes = bloom->bits / 8;
	}

	bloom->hashes = (int)ceil(0.693147180559945 * bloom->bpe);  // ln(2)

	bloom->bf = (unsigned char *)calloc(bloom->bytes, sizeof(unsigned char));
	if (bloom->bf == NULL) {
		return 1;
	}
	bloom->ready = 1;
	return 0;
}

static int bloomfilter_check(struct bloomfilter_t * bloom, const void * buffer, int len)
{
	return bloom_check_add(bloom, buffer, len, 0);
}

static int bloomfilter_add(struct bloomfilter_t * bloom, const void * buffer, int len)
{
	return bloom_check_add(bloom, buffer, len, 1);
}

static void bloomfilter_print(struct bloomfilter_t * bloom)
{
	printf("bloom at %p\n", (void *)bloom);
	printf(" .entries = %d\n", bloom->entries);
	printf(" .error = %f\n", bloom->error);
	printf(" .bits = %d\n", bloom->bits);
	printf(" .bits-per-elem = %f\n", bloom->bpe);
	printf(" .bytes = %d\n", bloom->bytes);
	printf(" .hash-functions = %d\n", bloom->hashes);
}

static void bloomfilter_free(struct bloomfilter_t * bloom)
{
	if (bloom->ready) {
		free(bloom->bf);
	}
	bloom->ready = 0;
}

int main(){
	long long starttime, endtime;
	starttime = clock();

	struct bloomfilter_t bloom = { 0 };
	bloomfilter_init(&bloom, 1270574, 0.000001);

	FILE *fp; FILE *fp1, *fp2;
	char buff[1000];
	char buffer[1000];
	if ((fp = fopen("E:\\360MoveData\\Users\\Administrator\\Desktop\\practice\\dict.txt", "r")) == NULL){
		printf("file cannot be opened!\n");
		exit(1);
	}

	if ((fp1 = fopen("E:\\360MoveData\\Users\\Administrator\\Desktop\\practice\\string.txt", "r")) == NULL){
		printf("file cannot be opened!\n");
		exit(1);
	}

	if ((fp2 = fopen("E:\\360MoveData\\Users\\Administrator\\Desktop\\practice\\result.txt", "w")) == NULL){
		printf("file cannot be opened!\n");
		exit(1);
	}

	bloomfilter_print(&bloom);

	while (!feof(fp)){
		if (fgets(buff, 1000, fp) != NULL){
			int buflen = _snprintf(buffer, sizeof(buff), "%s", buff);
			//将字符串存储在buffer[16]中，将buffer[]和buflen传给bloomfilter_add()中的murmurhash进行hash
			bloomfilter_add(&bloom, buffer, buflen);
		}
	}
	int num = 0;
	while (!feof(fp1)){
		if (fgets(buff, 1000, fp1) != NULL){
			int buflen = _snprintf(buffer, sizeof(buff), "%s", buff);
			if (bloomfilter_check(&bloom, buffer, buflen)){
				//printf("%s", flag->key);
				//输出到文件result.txt
				num++;
				fputs(buffer, fp2);
				fputs("\n", fp2);
			}
		}
	}

	fclose(fp); fclose(fp1); fclose(fp2);
	bloomfilter_free(&bloom);
	printf("%d", num);
	Sleep(3);

	endtime = clock();
	printf("%ldms\n", endtime - starttime - 3000);//运行时间 

	system("pause");
	return 0;
}