veterbi

#!usr/bin/python
#coding=utf-8

import urllib2
import sys, time, re
import sys
import jieba
jieba.load_userdict("userdict.txt")
import jieba.analyse
import jieba.posseg as pseg
import os
jieba.initialize()
import operator
reload(sys);
sys.setdefaultencoding('utf8');


t1 = time.time()

url = "10.txt"
content = open(url, "rb").read()
#print type(content)
print '文章长度：', len(content)
strRe = re.sub('\s', '', content)   #用正则干掉所有的空白
print '用正则干掉所有的空白后，字符长度', len(strRe)

'''
fo = open("foo.txt", "wb")
fo.write(strRe);
# 关闭打开的文件
fo.close()
'''

#分词， 未登录词用veterbi分词
words = list(jieba.cut(strRe, cut_all=False))
print "分词的总数：", len(words)
wordset = sorted(set(words))
print "不重复的单词数：", len(wordset)

#TF-IDF
jieba.analyse.set_idf_path("extra_dict/idf.txt.big");
tf_idf_tags = jieba.analyse.extract_tags(strRe, topK = 10)
print "TF-IDF 未去除停用词, 获取10个关键词"
print(",".join(tf_idf_tags))

jieba.analyse.set_idf_path("extra_dict/idf.txt.big");
jieba.analyse.set_stop_words("extra_dict/cn_stop_words.txt")
tf_idf_stop_words_tags = jieba.analyse.extract_tags(strRe, topK = 10)
print "TF-IDF 去除停用词"
print(",".join(tf_idf_stop_words_tags))


#TextRank
#tagswords = jieba.analyse.textrank(content)
#print(",".join(tagswords))

print "TextRank, 获取10个关键词"
TextRank_words = jieba.analyse.textrank(strRe)
print(",".join(TextRank_words))

'''
list = words
fl = open('list.txt', 'wb')

for i in range(len(list)):
    fl.write(list[i].encode('utf-8')+'--')
    
fl.close()
'''


# 统计分词结果后，每个个分词的次数
wordsDict = {}
DictsMaxWordlen = 0
singal = ''
for w in words:
    if wordsDict.get(w) == None:
        wordsDict[w] = 1
    else:
        wordsDict[w] += 1
        
    if DictsMaxWordlen <= wordsDict[w]:
        DictsMaxWordlen = wordsDict[w]
        global singal 
        singal = w
        #print w

print "分词最多重复的次数：".decode('utf-8'), DictsMaxWordlen , "分词是：".decode('utf-8'),singal

#按字典值排序（默认为升序），返回值是字典{key, tuple}
sorted_wordsDict = sorted(wordsDict.iteritems(), key=operator.itemgetter(1))
#print type(sorted_wordsDict[1])    #tuple


classNumWord = {}

for w in sorted_wordsDict:
    if classNumWord.has_key(w[1]) == True:
        if w[0] not in classNumWord[w[1]]:  
            classNumWord[w[1]].append(w[0])
    else:
        classNumWord[w[1]] = []
        classNumWord[w[1]].append(w[0])
#将字典排序，按照升序, 通过键排序，
sort_classNumWord = sorted(classNumWord.iteritems(), key=lambda asd:asd[0], reverse = False)
#print sort_classNumWord[20][1][0].encode('gb2312') 

wordslength = 0     #分词的总数
worldsNum = 0       #分词有多少个不同的词或词组
wordsFequencelist = {}  #分词出现的频次等级，从1到N次,并存储所对应等级的词语个数
for w in sort_classNumWord:
    worldsNum += w[0]
    wordslength += len(w[1]) * w[0]
    
    wordsFequencelist[w[0]] = []
    wordsFequencelist[w[0]].append(len(w[1]))
        
    #print "============================" 
    #for i in range(len(w[1])):     #按照出现的频次，打印词组
     #   print w[1][i]
    #print "出现".decode('utf-8'),w[0], "次的有：".decode('utf-8') ,len(w[1])
    #print "============================"      

sort_wordsFequencelist = sorted(wordsFequencelist.iteritems(), key=lambda asd:asd[0], reverse = False)

print '\t\t频率是单词出现的次数, 次数是出现对应次数的所有不同单词的总和'
lenWords = 0
for wordsFequence in sort_wordsFequencelist:
	lenWords += 1
	print '频率:{0:<4} 词数:{1:>6}'.format(wordsFequence[0], wordsFequence[1]), " ",
	if lenWords % 4 == 0:
		print

print 
print "一共有".decode('utf-8'), worldsNum, '个不同的词或词组'.decode('utf-8')
print "一共有".decode('utf-8'), wordslength, '个词或词组'.decode('utf-8')


print 
print
t2 = time.time()
tm_cost = t2-t1
print '运行时间', tm_cost

　　

Building prefix dict from C:\Python27\lib\site-packages\jieba-0.36.2-py2.7.egg\jieba\dict.txt ...
Dumping model to file cache c:\users\og\appdata\local\temp\jieba.cache
Loading model cost 2.16899991035 seconds.
Prefix dict has been built succesfully.