代码改变世界

深度学习 - Word2vec

2016-11-19 23:02  chenbinbin  阅读(603)  评论(0)    收藏  举报

词嵌入(word embedding)有很多方法，比较常用的是word2vec。以下利用gensim word2vec的例子

读取停用词

import gensim
import jieba
import os
import logging


## 停用词数据
def get_stopwords(file_name):
    with open(file_name) as f:
        stopwords = [line.strip('\r\t\n').decode('utf-8') for line in f]
    return stopwords

用结巴分词，对句子做切词和预处理

def sentence2word(sentence,stopwords=None):
    '''
    用结巴分词
    '''
    words = jieba.cut(sentence)
    if stopwords is not None:
        words = [w for w in words if w not in stopwords and w!=' ']
    else:
        words = list(words)
    return words

读取文件。为了便于逐行读取数据，利用一下的iterater.

class MySentences(object):
    def __init__(self, dirname,stopwords_fn=None):
        if stopwords_fn is not None:
            self.stopwords = get_stopwords(stopwords_fn)
        else:
            self.stopwords = None        
        self.dirname = dirname
 
    def __iter__(self):
        for fname in os.listdir(self.dirname):
            for line in open(os.path.join(self.dirname, fname)):
                line = sentence2word(line.strip(),stopwords=self.stopwords)
                if line:
                    yield line

以下用笑傲江湖跑的结果。

if __name__ == "__main__":
    # set up logging
    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',level=logging.INFO)
 
    dirname = '~/xajh'
    stopwords_fn = '~/all_stopword.txt'
    model_fn = '~/xajh_word2vec_model'
    sentences = MySentences(dirname,stopwords_fn)
    model = gensim.models.Word2Vec(sentences, size=128, max_vocab_size=40000)
    model.save(model_fn)
    
    ## 模型结果，令狐冲、盈盈、圣姑、华山、少林寺、风清扬、任我行
    for w in model.most_similar(u'令狐冲',topn=20):
        print w[0],w[1]
        
    ## 
    pos = [u'令狐冲',u'任盈盈']
    neg = [u'岳灵珊']
    for w in model.most_similar(positive=pos, negative=neg,topn=20):
        print w[0],w[1]

reference