CHzSeg

#ifndef _HZSEG_H_040415_
#define _HZSEG_H_040415_

#include <iostream>
#include <string>
#include <cstring>
#include <cstdlib>
#include <fstream>
#include "Dict.h"

using namespace std;

class CHzSeg
{
public:
    CHzSeg();
    ~CHzSeg();

    string SegmentSentenceMM (CDict&, string) const;//只保留下中文
    string SegmentHzStrMM (CDict&, string) const;//切词
    string SegmentURL(CDict&, string) const;

    // process a sentence before segmentation
    void Translate(char* SourceStr) const;
};
    
#endif /* _HZSEG_H_040415_ */

 

 

// HzSeg handling

#include "HzSeg.h"
#include "Dict.h"

const unsigned int MAX_WORD_LENGTH = 8;
const string SEPARATOR("/  "); // delimiter between words

CHzSeg::CHzSeg() {
}

CHzSeg::~CHzSeg() {
}

// Using Max Matching method to segment a character string.
string CHzSeg::SegmentHzStrMM(CDict &dict, string s1) const {
    string s2 = ""; //保存句子s1的分词结果
    while (!s1.empty()) {
        unsigned int len = s1.size();

        if (len > MAX_WORD_LENGTH)
            len = MAX_WORD_LENGTH;
        //如果待切分的句子大于最大切分单元
        //len=最大切分单元,否则len=句子的长度

        string w = s1.substr(0, len); //取s1句子最左边长度len为的子句子
        bool isw = dict.IsWord(w); //判断刚刚取出来的子句子是不是一个词

        while (len > 2 && isw == false) { //当w中至少有2个中文字&&不能构成字的时候,减去最右边的一个中文字符
            len -= 2; // cut a word
            w = w.substr(0, len);
            isw = dict.IsWord(w);
        }
        s2 += w + SEPARATOR;

        s1 = s1.substr(w.size());
    }

    return s2;
}

// process a sentence before segmentation
string CHzSeg::SegmentSentenceMM(CDict &dict, string s1) const {
    string s2 = "";
    unsigned int i, len;
    cout << endl << "I'm in SegmentSentenceMM" << endl;
    cout << s1 << endl;
    while (!s1.empty()) {
        unsigned char ch = (unsigned char) s1[0];
        if (ch < 128) { //吃掉一行中所有换行符以外的英文字符
            i = 1;
            len = s1.size();
            while (i < len && ((unsigned char) s1[i] < 128) && (s1[i] != 10)
                    && (s1[i] != 13)) { // LF, CR
                i++; //不是回车换行
            }

            if ((ch != 32) && (ch != 10) && (ch != 13)) { // SP, LF, CR
                s2 += s1.substr(0, i) + SEPARATOR;
            } else {
                if (ch == 10 || ch == 13) {
                    s2 += s1.substr(0, i);
                    cout << "当前s2:" << s2 << endl;
                }
            }

            if (i <= s1.size()) {
                s1 = s1.substr(i); //获得删去部分英文字符后的数据
            } else
                break; // 处理完英文字符

            continue;

        } else {
            if (ch < 176) { //中文标点等非汉字字符128<=ch<176
                i = 0;
                len = s1.length();

                while (i < len && ((unsigned char) s1[i] < 176)
                        && ((unsigned char) s1[i] >= 161)
                        && (!((unsigned char) s1[i] == 161
                                && ((unsigned char) s1[i + 1] >= 162
                                        && (unsigned char) s1[i + 1] <= 168)))
                        && (!((unsigned char) s1[i] == 161
                                && ((unsigned char) s1[i + 1] >= 171
                                        && (unsigned char) s1[i + 1] <= 191)))
                        && (!((unsigned char) s1[i] == 163
                                && ((unsigned char) s1[i + 1] == 172
                                        || (unsigned char) s1[i + 1] == 161)
                                || (unsigned char) s1[i + 1] == 168
                                || (unsigned char) s1[i + 1] == 169
                                || (unsigned char) s1[i + 1] == 186
                                || (unsigned char) s1[i + 1] == 187
                                || (unsigned char) s1[i + 1] == 191))) {
                    i = i + 2; //假定没有半个汉字
                }

                if (i == 0)
                    i = i + 2;

                if (!(ch == 161 && (unsigned char) s1[1] == 161)) { // 不处理中文空格
                    if (i <= s1.size()) // yhf
                        s2 += s1.substr(0, i) + SEPARATOR; // 其他的非汉字双字节字符可能连续输出
                    else
                        break; // yhf
                }

                if (i <= s1.size()) {
                    s1 = s1.substr(i); //取s1从下标i开始的子字符串

                } else
                    break; //yhf

                continue;
            }
        }

        i = 2;
        len = s1.length();
        while (i < len && (unsigned char) s1[i] >= 176)
//    while(i<len && (unsigned char)s1[i]>=128 && (unsigned char)s1[i]!=161)
            i += 2;

        s2 += SegmentHzStrMM(dict, s1.substr(0, i));

        if (i <= len) // yhf
            s1 = s1.substr(i);
        else
            break; // yhf
    }

    return s2;
}

// translate the encoded URL(%xx) to actual chars
void CHzSeg::Translate(char* SourceStr) const {
    int i = 0;
    int j = 0;
    char *tempstr, tempchar1, tempchar2;

    tempstr = (char*) malloc(strlen(SourceStr) + 1);
    if (tempstr == NULL) {
        return;
    }

    while (SourceStr[j]) {
        if ((tempstr[i] = SourceStr[j]) == '%') {
            if (SourceStr[j + 1] >= 'A')
                tempchar1 = ((SourceStr[j + 1] & 0xdf) - 'A') + 10;
            else
                tempchar1 = (SourceStr[j + 1] - '0');
            if (SourceStr[j + 2] >= 'A')
                tempchar2 = ((SourceStr[j + 2] & 0xdf) - 'A') + 10;
            else
                tempchar2 = (SourceStr[j + 2] - '0');
            tempstr[i] = tempchar1 * 16 + tempchar2;
            j = j + 2;
        }
        i++;
        j++;
    }
    tempstr[i] = '\0';
    strcpy(SourceStr, tempstr);

    if (tempstr)
        free(tempstr);
}

/*
 * segment the image URL by '/'
 * omit the domain name
 */
string CHzSeg::SegmentURL(CDict &dict, string url) const {
    string::size_type idx, nidx;
    char *curl = (char *) url.c_str();
    this->Translate(curl);
    url = curl;
    if ((idx = url.find("http://", 0)) != string::npos) {
        if ((nidx = url.find("/", 7)) != string::npos) {
            url = url.substr(nidx + 1); // cut the part of sitename
        }
    }
    idx = 0;
    while ((idx = url.find("/", idx)) != string::npos) {
        url.replace(idx, 1, SEPARATOR); // replace "/" with SEPARATOR "/  "
        idx += 3;
    }
    if ((idx = url.rfind(".")) != string::npos) {
        url = url.erase(idx); // erase the file extension
    }

    url += "/  ";

    // segment the string whose length is greater than 8 (4 HZ_chars)
    idx = 0;
    nidx = 0;
    bool isover = false;
    string stmp;
    while (!isover) {
        if ((nidx = url.find(SEPARATOR, idx)) == string::npos)
            isover = true;
        if (nidx - idx > 0) {
            stmp = url.substr(idx, nidx - idx);
            stmp = SegmentSentenceMM(dict, stmp);
            if (stmp.size() >= 3)
                stmp.erase(stmp.length() - 3); // erase the tail "/  "
            url = url.replace(idx, nidx - idx, stmp);
            idx += stmp.length() + 3;
        } else if (nidx == string::npos && idx < url.length()) {
            stmp = url.substr(idx);
            stmp = SegmentSentenceMM(dict, stmp);
            stmp.erase(stmp.length() - 3);
            url = url.substr(0, idx) + stmp;
        } else
            idx = nidx + 3;
    }

    return url;

}