windows下简单的C++分词程序

这个程序实现的功能很简单，只是为了测试我的第一个篇随笔进行记录。

功能：

  • 读取分词表（通用分词表-p，所有分词表-others）、读取输入文件
  • 分词——分词规则是将句子按词表的词用空格分开，非词表的词每个字分开，英文和数字要连在一起
  • 输出文件就是输入文件每句的分词结果，要求两个文件格式一样
  1 #include <iostream>
  2 #include <vector>
  3 #include <string>
  4 #include<sstream>
  5 #include<fstream>
  6 #include <set>
  7 using namespace std;
  8 
  9 string word_files[] = {"wordlist_HNC_通用词表.txt"};
 10 set<string> dict;
 11 /************
 12 导入分词表
 13 ************/
 14 set<string> loader()
 15 {
 16     set<string> words;
 17     vector<string> files;
 18     cout << "输入“p”启用专业词表， 其他字符启用通用词表" << endl;
 19     string input;
 20     cin >> input;
 21     if (input == "p")
 22     {
 23         for (int i = 0; i < 1; i++)
 24         {
 25             files.push_back("C:\\wordlists\\" + word_files[i]);
 26         }
 27     }
 28     else
 29     {
 30         files.push_back("C:\\wordlists\\wordlist_HNC_通用词表.txt");
 31     }
 32     int file_num = files.size();
 33     for (int i = 0; i < file_num; i++)
 34     {
 35         string path = files[i];
 36         cout << "正在载入词表" << path << endl;
 37         ifstream wordsfile;
 38         wordsfile.open(path);
 39         string line;
 40         string word;
 41         while (getline(wordsfile, line))
 42         {
 43             if ('$' != line[0])
 44             {
 45                 istringstream inword(line);
 46                 inword >> word;
 47                 cout << word << endl;
 48                 words.insert(word);
 49             }
 50         }
 51     }
 52     cout << "词表加载完成" << endl;
 53     return words;
 54 }
 55 /************
 56 对输入的每行进行分词，输出为分词后的行
 57 ************/
 58 
 59 string split(string input_line)
 60 {
 61     string output = "";
 62     vector<int> pos_vec;
 63     int char_len = input_line.length();
 64     for (int i = 0; i < char_len; i++)
 65     {
 66         int pos = 0;
 67         if((unsigned)input_line[i]>0x80)
 68         {
 69             pos = i + 1;
 70             i++;
 71         }
 72         else
 73         {
 74             pos = i;
 75         }
 76         pos_vec.push_back(pos);
 77     }
 78     int start = 0;
 79     int end;
 80     int len = pos_vec.size();
 81     cout << "line***" << endl;
 82     cout << len << endl;
 83     while (start < len-1)
 84     {
 85         if (start < len - 16)
 86         {
 87             end = start + 15;
 88         }
 89         else
 90         {
 91             end = len-1;
 92         }
 93         while (dict.find(input_line.substr(pos_vec[start], pos_vec[end])) == dict.end())
 94         {
 95             if (1 == end - start)
 96             {
 97                 cout << "break" << endl;
 98                 break;
 99             }
100             end--;
101         }
102         output = output + input_line.substr(pos_vec[start], pos_vec[end]) + " ";
103         cout << output << endl;
104         start = end;
105     }
106     return output + '\n';
107 }
108 
109 
110 int main(int argc, char const *argv[])
111 {
112     dict = loader();
113     string input_file_path = "C:\\homework\\input.txt";
114     string output_file_path = "C:\\homework\\output.txt";
115     ifstream infile;
116     ofstream outfile;
117     infile.open(input_file_path);
118     outfile.open(output_file_path);
119     string input_line;
120     while (getline(infile, input_line))
121     {
122         string output_line = split(input_line);
123         outfile << input_line << endl;
124     }
125     cout << "结果文件已生成" << endl;
126     infile.close();
127     outfile.close();
128     return 0;
129 }

 

posted @ 2017-08-29 01:15  congshen  阅读(1630)  评论(0)    收藏  举报