1.读取

def read_dataset():

     file_path = r'SMSSpamCollection'

     sms = open(file_path, encoding='utf-8')

     sms_data = []

     sms_label = []

     csv_reader = csv.reader(sms, delimiter='\t')

     for line in csv_reader:

        sms_label.append(line[0])  # 提取出标签

        sms_data.append(preprocessing(line[1]))  # 提取出特征

     sms.close()

     return sms_data, sms_label

2.数据预处理

def preprocessing(text):

     tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]  # 分词

     stops = stopwords.words('english')  # 使用英文的停用词表

     tokens = [token for token in tokens if token not in stops]  # 去除停用词

     tokens = [token.lower() for token in tokens if len(token) >= 3]  # 大小写，短词

     lmtzr = WordNetLemmatizer()

     tag = nltk.pos_tag(tokens)  # 词性

     tokens = [lmtzr.lemmatize(token, pos=get_wordnet_pos(tag[i][1])) for i, token in enumerate(tokens)]  # 词性还原

     preprocessed_text = ' '.join(tokens)

     return preprocessed_text

3.数据划分—训练集和测试集数据划分

def split_dataset(data, label):

     x_train, x_test, y_train, y_test = train_test_split(data, label, test_size=0.2, random_state=0, stratify=label)

     return x_train, x_test, y_train, y_test

4.文本特征提取

sklearn.feature_extraction.text.CountVectorizer

https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html?highlight=sklearn%20feature_extraction%20text%20tfidfvectorizer

sklearn.feature_extraction.text.TfidfVectorizer

https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html?highlight=sklearn%20feature_extraction%20text%20tfidfvectorizer#sklearn.feature_extraction.text.TfidfVectorizer

from sklearn.feature_extraction.text import TfidfVectorizer

tfidf2 = TfidfVectorizer()

观察邮件与向量的关系

向量还原为邮件

def tfidf_dataset(x_train,x_test):

     tfidf = TfidfVectorizer()

     X_train = tfidf.fit_transform(x_train)  # X_train用fit_transform生成词汇表

     X_test = tfidf.transform(x_test)

     return X_train, X_test, tfidf

4.模型选择

from sklearn.naive_bayes import GaussianNB

from sklearn.naive_bayes import MultinomialNB

def mnb_model(x_train, x_test, y_train, y_test):

    mnb = MultinomialNB()

    mnb.fit(x_train, y_train)

    ypre_mnb = mnb.predict(x_test)

    print("总数：", len(y_test))

    print("预测正确数：", (ypre_mnb == y_test).sum())

return ypre_mnb

5.模型评价：混淆矩阵，分类报告

from sklearn.metrics import confusion_matrix

confusion_matrix = confusion_matrix(y_test, y_predict)

说明混淆矩阵的含义

from sklearn.metrics import classification_report

说明准确率、精确率、召回率、F值分别代表的意义 

 

 

6.比较与总结

如果用CountVectorizer进行文本特征生成，与TfidfVectorizer相比，效果如何？

CountVectorizer 与 TfidfVectorizer，这两个类都是特征数值计算的常见方法。对于每一个训练文本，CountVectorizer 只考虑每种词汇在该训练文本中出现的频率，而TfidfVectorizer 除了考量某一词汇在当前训练文本中出现的频率之外，同时关注包含这个词汇的其它训练文本数目的倒数。相比之下，训练文本的数量越多，TfidfVectorizer 这种特征量化方式就更有优势。