学习hashtable，处理“海量”数据

直接上代码吧，哈希表的逻辑还是很简单的，目的是对比这几种方法的速度，重要的是参照代码，看输出结果：

#include <stdio.h>
#include <stdlib.h>
#include <sys/timeb.h>
#include <fstream>
#include <string>
#include <vector>
#include <list>
#include <algorithm>
#include <set>
#include <hash_set>
#include <hash_map>

using namespace std;

static const int hashtable_length    = 49157;

// 用于定位一个Bucket
unsigned int hash_function(const char* str)
{
    const char* end_of_str = str+strlen(str);
    unsigned int sum = 0;
    while (end_of_str - str > 3)
    {
        sum = (sum + (unsigned int)*((unsigned int*)str))%hashtable_length;
        str += 4;
    }
    char tmp[4] = {0};
    strcpy(tmp, str);
    sum = (sum + (unsigned int)*((unsigned int*)tmp))%hashtable_length;
    memset(tmp, 0, 4);

    return sum;
}

// 用于在一个Buchet中查找目标
bool find_in_bucket(list<string>& l, const char* str)
{
    list<string>::iterator iter;
    unsigned int hash_key = hash_function(str);
    bool exist = false;
    for (iter = l.begin(); iter != l.end(); iter++)
        if (strcmp(str, iter->c_str()) == 0)
                return true;
    return false;
}

// 用于把目标放到Bucket中
int insert_in_bucket(list<string>& l, const char* str)
{
    if (!find_in_bucket(l, str))
    {
        l.push_back(string(str));
        return l.size();
    }else
        return -1;
}

// 用于在整个hash表中查找目标
bool find_in_hashtable(vector<list<string>>& v, const char* str)
{
    return find_in_bucket(v[hash_function(str)], str);
}

// 用于在整个hash表中插入一个元素
int insert_in_hashtable(vector<list<string>>& v, const char* str)
{
    return insert_in_bucket(v[hash_function(str)], str);
}

// 过滤掉文本中的标点符号
void filter(char* str)
{
    while(*str++)
        if(*str == ',' || *str == '.' 
            || *str == '?' || *str == '-' 
            || *str == '\"' || *str == '\'' 
            || *str == ')' || *str == '('
            || *str == '!')
            *str = ' ';
}

// 读取一行中的一个单词
char* get_word_from_buff(char* &buff, char* word)
{
    while (*buff && *buff == ' ')
        buff++;
    if (!*buff)
        return NULL;
    int cnt = 0;
    while (*buff && *buff != ' ')
        word[cnt++] = *buff++;
    word[cnt] = 0;
    return buff;
}

int main()
{
    // 对比哈希表和朴素方法的差别
    // 任务是存储一个文件中的英文单词，要求不能重复

    timeb time_begin;
    timeb time_end;
    ifstream input_file;
    input_file.open("D:\\input.txt");
    char buff[10241] = {0};    // 10KB的缓冲区
    char word[100];
    vector<string> vector_of_words;
    ftime(&time_begin);
    // 下面代码速度奇慢无比
    while (input_file.getline(buff, 10240))
    {
        filter(buff);
        char* ptr_to_buff = buff;
        vector<string>::iterator iter = vector_of_words.begin();
        while (ptr_to_buff = get_word_from_buff(ptr_to_buff, word))
        {
            int i = 0;
            for (; i < vector_of_words.size(); i++)
                if(strcmp(vector_of_words[i].c_str(), word) == 0)
                    break;
            if (i >= vector_of_words.size())
                vector_of_words.push_back(string(word));
        }
    }

    ftime(&time_end);
    unsigned int seconds = time_end.time - time_begin.time;
    unsigned int miseconds = time_end.millitm - time_begin.millitm;
    miseconds = seconds * 1000 + miseconds;
    printf("朴素的方法：\t处理时间为：\t%u\t毫秒, 统计了%d个单词\n", miseconds, vector_of_words.size());

    input_file.close();
    input_file.open("D:\\input.txt");
    vector<list<string>> hashtable_of_words(hashtable_length, list<string>());
    ftime(&time_begin);
    int count = 0;
    while (input_file.getline(buff, 10240))
    {
        filter(buff);
        char* ptr_to_buff = buff;
        while (ptr_to_buff = get_word_from_buff(ptr_to_buff, word))
            if(insert_in_hashtable(hashtable_of_words, word) != -1)
                ++count;
    }
    ftime(&time_end);
    seconds = time_end.time - time_begin.time;
    miseconds = time_end.millitm - time_begin.millitm;
    miseconds = seconds * 1000 + miseconds;
    printf("hashtable：\t处理时间为：\t%u\t毫秒, 统计了%d个单词\n", miseconds, count);

    input_file.close();
    input_file.open("D:\\input.txt");
    set<string> set_of_words;
    ftime(&time_begin);
    while (input_file.getline(buff, 10240))
    {
        filter(buff);
        char* ptr_to_buff = buff;
        while (ptr_to_buff = get_word_from_buff(ptr_to_buff, word))
            set_of_words.insert(string(word));
    }
    ftime(&time_end);
    seconds = time_end.time - time_begin.time;
    miseconds = time_end.millitm - time_begin.millitm;
    miseconds = seconds * 1000 + miseconds;
    printf("rbtree-set：\t处理时间为：\t%u\t毫秒, 统计了%d个单词\n", miseconds, set_of_words.size());

    input_file.close();
    input_file.open("D:\\input.txt");
    hash_map<string, int> hashmap_of_words;
    ftime(&time_begin);
    while (input_file.getline(buff, 10240))
    {
        filter(buff);
        char* ptr_to_buff = buff;
        while (ptr_to_buff = get_word_from_buff(ptr_to_buff, word))
            hashmap_of_words[string(word)]++;
    }
    ftime(&time_end);
    seconds = time_end.time - time_begin.time;
    miseconds = time_end.millitm - time_begin.millitm;
    miseconds = seconds * 1000 + miseconds;
    printf("hash_map：\t处理时间为：\t%u\t毫秒, 统计了%d个单词\n", miseconds, hashmap_of_words.size());

    input_file.close();
    input_file.open("D:\\input.txt");
    hash_set<string> hashset_of_words;
    ftime(&time_begin);
#if 0    // 下面代码速度奇慢无比，所以注释掉了实际没有执行，我等了半天，没有计算完，不知道是不是逻辑有问题～
    while (input_file.getline(buff, 10240))
    {
        filter(buff);
        char* ptr_to_buff = buff;
        while (ptr_to_buff = get_word_from_buff(ptr_to_buff, word))
            hashset_of_words.insert(string(word));
    }
#endif
    ftime(&time_end);
    seconds = time_end.time - time_begin.time;
    miseconds = time_end.millitm - time_begin.millitm;
    miseconds = seconds * 1000 + miseconds;
    printf("hash_set：\t处理时间为：\t%u\t毫秒, 统计了%d个单词\n", miseconds, hashset_of_words.size());

    system("pause");
    return 0;
}

 

输出结果：

 

朴素的方法：     处理时间为：    238594  毫秒, 统计了28661个单词
hashtable：     处理时间为：    2312    毫秒, 统计了28661个单词
rbtree-set：    处理时间为：    13438   毫秒, 统计了28661个单词
hash_map：      处理时间为：    6953    毫秒, 统计了28661个单词
hash_set：      处理时间为：    0       毫秒, 统计了0个单词
请按任意键继续. . .

 后来又做了小幅的改动：

unsigned int hash_function_opt(const char* str)
{
    const char* end_of_str = str+strlen(str);
    unsigned int sum = 0;
    while (end_of_str - str > 3)
    {
        sum ^= *((unsigned int*)str);
        str += 4;
    }
    char tmp[4] = {0};
    strcpy(tmp, str);
    sum ^= (unsigned int)*((unsigned int*)tmp);
    sum %= hashtable_length;
    memset(tmp, 0, 4);

    return sum;
}

bool find_in_bucket_opt(list<string>& l, const char* str)
{
    list<string>::iterator iter;
    for (iter = l.begin(); iter != l.end(); iter++)
        if (strcmp(str, iter->c_str()) == 0)
            return true;
    return false;
}

void filter(char* str)
{
    while(*str)
    {
        if(!((*str >= 'a' && *str <= 'z') || (*str >= 'A' && *str <= 'Z')))
            *str = ' ';
        str++;
    }
}

    ofstream output_file;
    output_file.open("D:\\output.txt");
    vector<string> all_words;
    for(vector<list<string>>::iterator i_v = hashtable_of_words.begin(); i_v != hashtable_of_words.end(); i_v++)
        for(list<string>::iterator i_l = i_v->begin(); i_l != i_v->end(); i_l++)
            all_words.push_back(*i_l);
    sort(all_words.begin(), all_words.end());
    for(vector<string>::iterator i_v = all_words.begin(); i_v != all_words.end(); i_v++)
        output_file << *i_v <<endl;
    output_file.close();

改动之后发现，之前的版本其实是有些小错误的，新的输出结果为：

朴素的方法：    处理时间为：    0       毫秒, 统计了0个单词
hashtable：     处理时间为：    2079    毫秒, 统计了27735个单词
hashtable2：    处理时间为：    2047    毫秒, 统计了27735个单词
rbtree-set：    处理时间为：    0       毫秒, 统计了0个单词
hash_map：      处理时间为：    0       毫秒, 统计了0个单词
hash_set：      处理时间为：    0       毫秒, 统计了0个单词
请按任意键继续. . .