综合练习：词频统计

world = f.read()
f.close()
xiaoqu = {'the','and','of','to', 'a', 'on','in','was','his','at','that','as','were','said','by','with', 'we','all','he',}
depart = ''',.？":!'''  
for c in depart:
    world = world.replace(c,' ')
wordlist = world.lower().split() 
wordDict = {}  
wordSet = set(wordlist) - xiaoqu
for a in wordSet:
    wordDict[a] = wordlist.count(a)
    dictlist = list(wordDict.items())
    dictlist.sort(key=lambda x:x[1], reverse=True)
for i in range(20):  
 print(dictlist[i])

　运行结果

　

 

2.中文词频统计

下载一长篇中文文章。

从文件读取待分析文本。

news = open('gzccnews.txt','r',encoding = 'utf-8')

安装与使用jieba进行中文分词。

pip install jieba

import jieba

list(jieba.lcut(news))

生成词频统计

排序

排除语法型词汇，代词、冠词、连词

输出词频最大TOP20（或把结果存放到文件里）

 

# -*- codding: UTF-8 -*-
# -*- author: WF -*-
import jieba

f = open('hongloumeng.txt','r',encoding='utf-8')   #将文本放在同等的路径下，以文件的方式读出来
hongloumeng = f.read()
f.close()

depart = '''？"：“。”．！'《》'''  #用循环列表，用空格来代替,.？":!标点符号
for c in depart:
    hongloumeng = hongloumeng.replace(c,' ')

hongloumenglist = list(jieba.cut(hongloumeng))

paichu = [' ','了','的','我','\n','他','道', '你','也','是','又','着','去','来', '在','都','不',
         '虽','为','却','那','亦','之','将','这','便','则','只','但','乃','再','因','得','此','与']

hongloumengDict = {}  #生成一个空字典
for a in hongloumenglist:
    hongloumengDict[a] = hongloumengDict.get(a,0)+1

for a in paichu:
    del (hongloumengDict[a]);

dictlist = list(hongloumengDict.items())
dictlist.sort(key=lambda x:x[1], reverse=True)

f = open('hongloumengcount.txt','a')
for i in range(20):  # 输出前20个
    f.write(dictlist[i][0]+" "+str(dictlist[i][1])+'\n')
f.close()