参考代码
ChineseClean_demo1.py：
# -*- coding:utf-8 -*-
import xlrd
import xlwt
'''
python3.4
'''
# file 表示源文件名字，修改此处即可
file="./data/answer_detail_5_15307860968687.xls"
dirs="./result"
def read_excel(rows_numb,cols_numb):
f = xlwt.Workbook() #创建工作簿
'''
创建第一个sheet:
sheet1
'''
sheet1 = f.add_sheet(u'sheet1_1',cell_overwrite_ok=True) #创建sheet
sheet2 = f.add_sheet(u'sheet1_2',cell_overwrite_ok=True) #创建sheet
row0 = [u'UserNo',u'Name',u'Question',u'Answer',u'Layer',u'Mark',u'Score',u'AbilityID']
# 打开文件
workbook = xlrd.open_workbook(file)
sheet0 = workbook.sheet_by_index(0) # sheet索引从0开始
cols = sheet0.col_values(cols_numb)
rows_list_1=[]
rows_list_2=[]
for i in range(1,len(cols)):
if cols[i] == '0':
rows_list_1.append(i)
else:
rows_list_2.append(i)
for i in range(0,len(row0)):
sheet1.write(0,i,row0[i])
sheet2.write(0,i,row0[i])
'''
sheet1_1保存0分数据
'''
for j in range(0,len(rows_list_1)):
rows = sheet0.row_values(rows_list_1[j]) # 获取行内容
for i in range(0,len(rows)):
sheet1.write(j+1,i,rows[i])
'''
sheet1_2保存非0分数据
'''
for j in range(0,len(rows_list_2)):
rows = sheet0.row_values(rows_list_2[j]) # 获取行内容
for i in range(0,len(rows)):
sheet2.write(j+1,i,rows[i])
f.save('./data/demo1.xls') #保存文件
if __name__ == '__main__':
# 读取文件的行和列
rows_numb=0
cols_numb=6
read_excel(rows_numb,cols_numb)
ChineseClean_demo2.py：
# -*- coding:utf-8 -*-
import xlrd
import xlwt
'''
python3.4
'''
# file 表示源文件名字，修改此处即可
file="./data/demo1.xls"
def read_excel(rows_numb,cols_numb):
f = xlwt.Workbook() #创建工作簿
'''
创建第sheet:
'''
sheet1 = f.add_sheet(u'sheet2_1',cell_overwrite_ok=True) #创建sheet
sheet2 = f.add_sheet(u'sheet2_2',cell_overwrite_ok=True) #创建sheet
sheet3 = f.add_sheet(u'sheet2_3',cell_overwrite_ok=True) #创建sheet
sheet4 = f.add_sheet(u'sheet2_4',cell_overwrite_ok=True) #创建sheet
row0 = [u'UserNo',u'Name',u'Question',u'Answer',u'Layer',u'Mark',u'Score',u'AbilityID']
for i in range(0,len(row0)):
sheet1.write(0,i,row0[i])
sheet2.write(0,i,row0[i])
sheet3.write(0,i,row0[i])
sheet4.write(0,i,row0[i])
# 打开文件
workbook = xlrd.open_workbook(file)
sheet0 = workbook.sheet_by_index(0) # sheet索引从0开始
cols = sheet0.col_values(cols_numb) # 获取列内容
rows_list_1=[]
rows_list_2=[]
rows_list_3=[]
rows_list_4=[]
for i in range(1,len(cols)):
if float(cols[i]) < 12.0:
rows_list_1.append(i)
if float(cols[i]) >= 12.0 and float(cols[i]) < 16.0:
rows_list_2.append(i)
if float(cols[i]) >= 16.0 and float(cols[i]) < 18.0:
rows_list_3.append(i)
if float(cols[i]) >= 18.0:
print(i)
print(type(cols[i]))
exit()
rows_list_4.append(i)
'''
sheet2_1保存差，小于12分
'''
for j in range(0,len(rows_list_1)):
rows = sheet0.row_values(rows_list_1[j]) # 获取行内容
for i in range(0,len(rows)):
sheet1.write(j+1,i,rows[i])
'''
sheet2_2保存中，大于等于12，且小于16分
'''
for j in range(0,len(rows_list_2)):
rows = sheet0.row_values(rows_list_2[j]) # 获取行内容
for i in range(0,len(rows)):
sheet2.write(j+1,i,rows[i])
'''
sheet2_3保存良，大于等于16，且小于18分
'''
for j in range(0,len(rows_list_3)):
rows = sheet0.row_values(rows_list_3[j]) # 获取行内容
for i in range(0,len(rows)):
sheet3.write(j+1,i,rows[i])
'''
sheet2_4保存优，大于等于18分
'''
for j in range(0,len(rows_list_4)):
rows = sheet0.row_values(rows_list_4[j]) # 获取行内容
for i in range(0,len(rows)):
sheet4.write(j+1,i,rows[i])
f.save('./data/demo2.xls')
if __name__ == '__main__':
# 读取文件的行和列
rows_numb=0
cols_numb=6
read_excel(rows_numb,cols_numb)
ChineseClean_demo3.py:
# -*- coding:utf-8 -*-
import xlrd
import xlwt
'''
python3.4
'''
file="./data/answer_detail_5_15307860968687.xls"
def read_excel(rows_numb,cols_numb):
f = xlwt.Workbook() #创建工作簿
'''
创建第一个sheet:
sheet1
'''
sheet1 = f.add_sheet(u'sheet1',cell_overwrite_ok=True) #创建sheet
sheet2 = f.add_sheet(u'sheet2',cell_overwrite_ok=True) #创建sheet
sheet3 = f.add_sheet(u'sheet3',cell_overwrite_ok=True) #创建sheet
sheet4 = f.add_sheet(u'sheet4',cell_overwrite_ok=True) #创建sheet
sheet5 = f.add_sheet(u'sheet5',cell_overwrite_ok=True)
row0 = [u'UserNo',u'Name',u'Question',u'Answer',u'Layer',u'Mark',u'Score',u'AbilityID']
for i in range(0,len(row0)):
sheet1.write(0,i,row0[i])
sheet2.write(0,i,row0[i])
sheet3.write(0,i,row0[i])
sheet4.write(0,i,row0[i])
sheet5.write(0,i,row0[i])
# 打开文件
workbook = xlrd.open_workbook(file)
sheet0 = workbook.sheet_by_index(0) # sheet索引从0开始
cols = sheet0.col_values(cols_numb) # 获取列内容
rows_list_1=[]
rows_list_2=[]
rows_list_3=[]
rows_list_4=[]
rows_list_5=[]
for i in range(1,len(cols)):
if cols[i] == '100012':
rows_list_1.append(i)
if cols[i] == '100014':
rows_list_2.append(i)
if cols[i] == '100007':
rows_list_3.append(i)
if cols[i] == '100016':
rows_list_4.append(i)
if cols[i] == '100017':
print(i)
print(type(cols[i]))
rows_list_5.append(i)
'''
sheet1保存
'''
for j in range(0,len(rows_list_1)):
rows = sheet0.row_values(rows_list_1[j]) # 获取第四行内容
for i in range(0,len(rows)):
sheet1.write(j+1,i,rows[i])
'''
sheet2保存
'''
for j in range(0,len(rows_list_2)):
rows = sheet0.row_values(rows_list_2[j]) # 获取第四行内容
for i in range(0,len(rows)):
sheet2.write(j+1,i,rows[i])
'''
sheet3保存
'''
for j in range(0,len(rows_list_3)):
rows = sheet0.row_values(rows_list_3[j]) # 获取第四行内容
for i in range(0,len(rows)):
sheet3.write(j+1,i,rows[i])
'''
sheet4保存
'''
for j in range(0,len(rows_list_4)):
rows = sheet0.row_values(rows_list_4[j]) # 获取第四行内容
for i in range(0,len(rows)):
sheet4.write(j+1,i,rows[i])
'''
sheet5保存
'''
for j in range(0,len(rows_list_5)):
rows = sheet0.row_values(rows_list_5[j]) # 获取第四行内容
for i in range(0,len(rows)):
sheet5.write(j+1,i,rows[i])
f.save('./data/demo3.xls') #保存文件
if __name__ == '__main__':
# 读取文件的行和列
rows_numb=0
cols_numb=7
read_excel(rows_numb,cols_numb)
ChineseClean_demo4or5.py：
同ChineseClean_demo3.py
ChineseClean_answer_QA.py：
# -*- coding:utf-8 -*-
import re
import xlrd
file="./data/demo5.xls"
dirs="./result"
def read_excel(rows_numb,cols1_numb):
number='1'
f2 = open(dirs+'./demo5_sheet1_%s.csv'%number, 'a', encoding='utf-8')
# 打开文件
workbook = xlrd.open_workbook(file)
sheet0 = workbook.sheet_by_index(int(number)-1) # sheet索引从0开始
cols1 = sheet0.col_values(cols1_numb[3]) [1:]# 获取列内容
p1 = r"(?:[\u2E80-\uFFFD]|[\u201c-\u201d]|[\u002d]|[\u003a])+"
pattern1 = re.compile(p1)
for i in range(len(cols1)):
matcher1 = re.findall(pattern1, cols1[i])
str1=str()
if matcher1:
str1 = ' '.join(matcher1)
f2.write(str1)
f2.write('\n')
f2.close()
if __name__ == '__main__':
# 读取文件的行和列
rows_numb=0
cols1_numb=[0,1,2,3,4,5,6,7]
read_excel(rows_numb,cols1_numb)
qa_test_clean_word.py：
# -*- coding: utf-8 -*-
import jieba
# 创建停用词list
def stopwordslist(filepath):
stopwords = [line.strip() for line in open(filepath, 'r', encoding='utf-8').readlines()]
return stopwords
# 对句子进行分词
def seg_sentence(sentence):
sentence_seged = jieba.lcut_for_search(sentence.strip(),HMM=True)
stopwords = stopwordslist('./test/stopwords.txt') # 这里加载停用词的路径
outstr = ''
for word in sentence_seged:
if word not in stopwords:
if word != '\t':
outstr += word
outstr += " "
return(outstr)
inputs = open('./data/demo5_answer_csv/demo5_sheet5_5.csv', 'r', encoding='utf-8')
outputs = open('./test/demo5_sheet5_5_5.csv', 'w')
for line in inputs:
line_seg = seg_sentence(line)
try:
if len(line_seg):
outputs.write(line_seg + '\n')
except:
pass
outputs.close()
inputs.close()
word_fre.py:
# -*- coding: utf-8 -*-
import matplotlib.pyplot as plt
from matplotlib.font_manager import *
import numpy as np
def drawStatBarh():
'''
画出词频统计条形图，用渐变颜色显示，选取前N个词频
'''
fig, ax = plt.subplots()
myfont = FontProperties(fname='./data/simfang.ttf')
N = 30
words = []
counts = []
for line in open('./data/word_fre.txt'):
if line == '\n':
continue
line.strip('\n')
words.append(line.split(' ')[0])
print(line.split(' ')[0])
# exit()
counts.append(int(line.split(' ')[1].strip('\n')))
y_pos = np.arange(N)
colors = ['#FA8072'] #这里是为了实现条状的渐变效果，以该色号为基本色实现渐变效果
for i in range(len(words[:N]) - 1):
colors.append('#FA' + str(int(colors[-1][3:]) - 1))
rects = ax.barh(y_pos, counts[:N], align='center', color=colors)
ax.set_yticks(np.arange(N))
ax.set_yticklabels(words[:N],fontproperties=myfont)
ax.invert_yaxis() # labels read top-to-bottom
ax.set_title('报告中的高频词汇',fontproperties=myfont, fontsize=17)
ax.set_xlabel(u"出现次数",fontproperties=myfont)
autolabel(rects, ax)
plt.show()
def autolabel(rects, ax):
"""
给条形图加上文字标签
"""
#fig, ax = plt.subplots()
for rect in rects:
width = rect.get_width()
ax.text(1.03 * width, rect.get_y() + rect.get_height()/2.,
'%d' % int(width),ha='center', va='center')
def wordCount(segment_list):
'''
该函数实现词频的统计，并将统计结果存储至本地。
在制作词云的过程中用不到，主要是在画词频统计图时用到。
'''
word_lst = []
word_dict = {}
with open('./data/word_fre.txt','w') as wf2:
word_lst.append(segment_list.split(' '))
for item in word_lst:
for item2 in item:
if item2 not in word_dict:
word_dict[item2] = 1
else:
word_dict[item2] += 1
# print(type(word_dict))
# print(word_dict)
word_dict_sorted =list(sorted(word_dict.items(),key = lambda jj:jj[1],reverse=True))#list是关键，按照词频从大到小排序
# word_dict_sorted = dict(sorted(word_dict.items(),key = lambda item:item[1], reverse=True))#按照词频从大到小排序
print(word_dict_sorted)
# exit()
for tup in word_dict_sorted:
# print(type(tup))
# print(tup)
# exit()
if tup[0] != '':
wf2.write(tup[0].strip('\n')+' '+str(tup[1])+'\n')
wf2.close()
if __name__ == "__main__":
segment_list_remove_stopwords=open('./data/demo5_sheet5_1_1.csv').read()
wordCount(segment_list_remove_stopwords)
drawStatBarh()
wordcloud_test2.py:
# - * - coding: utf - 8 -*-
from os import path
from scipy.misc import imread
import matplotlib.pyplot as plt
import jieba
# jieba.load_userdict("txt\userdict.txt")
# 添加用户词库为主词典,原词典变为非主词典
from wordcloud import WordCloud, ImageColorGenerator
# 获取当前文件路径
# __file__ 为当前文件, 在ide中运行此行会报错,可改为
# d = path.dirname('.')
d = path.dirname(__file__)
stopwords = {}
isCN = 1 #默认启用中文分词
back_coloring_path = "data/lz1.jpg" # 设置背景图片路径
text_path = 'data/demo5_sheet5_1_1.csv' #设置要分析的文本路径,讲原始文件转化为‘ANSI编码即可’
font_path = 'data/simfang.ttf' # 为matplotlib设置中文字体路径
stopwords_path = 'data/stopwords.txt' # 停用词词表
imgname1 = "data/WordCloudDefautColors.png" # 保存的图片名字1(只按照背景图片形状)
imgname2 = "data/WordCloudColorsByImg.png"# 保存的图片名字2(颜色按照背景图片颜色布局生成)
# my_words_list = ['CHENGLEI'] # 在结巴的词库中添加新词
back_coloring = imread(path.join(d, back_coloring_path))# 设置背景图片
# 设置词云属性
wc = WordCloud(font_path=font_path, # 设置字体
background_color="white", # 背景颜色
max_words=2000, # 词云显示的最大词数
mask=back_coloring, # 设置背景图片
max_font_size=100, # 字体最大值
random_state=42,
width=1000, height=860, margin=2,# 设置图片默认的大小,但是如果使用背景图片的话,那么保存的图片大小将会按照其大小保存,margin为词语边缘距离
)
# 添加自己的词库分词
# def add_word(list):
# for items in list:
# jieba.add_word(items)
# add_word(my_words_list)
text = open(path.join(d, text_path)).read()
# def jiebaclearText(text):
# mywordlist = []
# seg_list = jieba.cut(text, cut_all=False)
# liststr="/ ".join(seg_list)
# f_stop = open(stopwords_path)
# try:
# f_stop_text = f_stop.read( )
# f_stop_text=unicode(f_stop_text,'utf-8')
# finally:
# f_stop.close( )
# f_stop_seg_list=f_stop_text.split('\n')
# for myword in liststr.split('/'):
# if not(myword.strip() in f_stop_seg_list) and len(myword.strip())>1:
# mywordlist.append(myword)
# return ''.join(mywordlist)
#
# if isCN:
# text = jiebaclearText(text)
# 生成词云, 可以用generate输入全部文本(wordcloud对中文分词支持不好,建议启用中文分词),也可以我们计算好词频后使用generate_from_frequencies函数
wc.generate(text)
# wc.generate_from_frequencies(text)
# txt_freq例子为[('词a', 100),('词b', 90),('词c', 80)]
# 从背景图片生成颜色值
image_colors = ImageColorGenerator(back_coloring)
plt.figure()
# 以下代码显示图片
plt.imshow(wc)
plt.axis("off")
plt.show()
# 绘制词云
# 保存图片
wc.to_file(path.join(d, imgname1))
image_colors = ImageColorGenerator(back_coloring)
plt.imshow(wc.recolor(color_func=image_colors))
plt.axis("off")
# 绘制背景图片为颜色的图片
plt.figure()
plt.imshow(back_coloring, cmap=plt.cm.gray)
plt.axis("off")
plt.show()
# 保存图片
wc.to_file(path.join(d, imgname2))
lda_test_ok.py：
# coding=utf-8
import numpy as np
import matplotlib.pyplot as plt
from sklearn.feature_extraction.text import CountVectorizer
import lda
def doc_topic_word():
print(doc_topic[:, :3])#输出文档主题分布情况(前3列)
print(topic_word[:, :3])#输出主题词分布情况(前3列)，采用ifidf计算词频
#导出分布图
def plot_1():
# 计算各个主题中单词权重分布的情况
f, ax= plt.subplots(2, 1, figsize=(6, 6), sharex=True)
for i, k in enumerate([0, 9]): #任意选择两个主题
ax[i].stem(topic_word[k,:], linefmt='b-',
markerfmt='bo', basefmt='w-')
ax[i].set_xlim(-2,2000)
ax[i].set_ylim(0, 1)
ax[i].set_ylabel("Prob")
ax[i].set_title("topic {}".format(k))
ax[1].set_xlabel("word")
plt.tight_layout()
plt.show()
def plot_2():
# 计算文档具体分布在那个主题，代码如下所示：
f, ax= plt.subplots(2, 1, figsize=(8, 8), sharex=True)
for i, k in enumerate([0,9]): #任意选择两个主题
ax[i].stem(doc_topic[k,:], linefmt='r-',
markerfmt='ro', basefmt='w-')
ax[i].set_xlim(-1, 20) #x坐标下标,即主题的取值范围
ax[i].set_ylim(0, 1.2) #y坐标下标
ax[i].set_ylabel("Prob")
ax[i].set_title("Document {}".format(k))
ax[1].set_xlabel("Topic")
plt.tight_layout()
plt.show()
if __name__ == "__main__":
#存储读取语料 一行预料为一个文档
corpus = []
for line in open('./data/demo5_sheet5_1_1.csv', 'r').readlines():
corpus.append(line.strip())
#将文本中的词语转换为词频矩阵 矩阵元素a[i][j] 表示j词在i类文本下的词频
vectorizer = CountVectorizer()
print (vectorizer)
X = vectorizer.fit_transform(corpus)
analyze = vectorizer.build_analyzer()
weight = X.toarray()
print("type(X): {}".format(type(X)))
print("shape: {}\n".format(X.shape))
print (len(weight))
print (weight[:5, :5])
#LDA算法
print ('LDA:')
model = lda.LDA(n_topics=20, n_iter=50, random_state=1)
# model.fit_transform(X)
model.fit(np.asarray(weight)) # model.fit_transform(X) is also available?
topic_word = model.topic_word_ # model.components_ also works
#文档-主题（Document-Topic）分布
doc_topic = model.doc_topic_
print("type(doc_topic): {}".format(type(doc_topic)))
print("shape: {}".format(doc_topic.shape))
#输出前10篇文章最可能的Topic
label = []
for n in range(10):
topic_most_pr = doc_topic[n].argmax()
label.append(topic_most_pr)
print("doc: {} topic: {}".format(n, topic_most_pr))
#输出主题中的TopN关键词
word = vectorizer.get_feature_names()
n = 6
for i, topic_dist in enumerate(topic_word):
topic_words = np.array(word)[np.argsort(topic_dist)][:-(n+1):-1]
print(u'*Topic {}\n- {}'.format(i, ' '.join(topic_words)))
# doc_topic_word()
# plot_1()
plot_2()
gensimTopicTest0803.py:
# coding=utf-8
import re
import xlrd
import codecs
import jieba
from gensim import corpora, models, similarities
FILE="demo5"#选择要训练的文件
ID='1'#选择要训练的能力ID
# 读取停用词表
stopwords = [line.strip() for line in codecs.open('./data/stopwords.txt', 'r', encoding='utf-8').readlines()]
def cleanAnswer(cols_numb):
f1 = open('./result/%s_sheet%s.csv'%(FILE,ID), 'a', encoding='utf-8')
# 打开文件
workbook = xlrd.open_workbook('./data/%s.xls'%FILE)
#根据sheet索引或者名称获取sheet内容
sheet0 = workbook.sheet_by_index(int(ID)-1) # sheet索引从0开始
cols1 = sheet0.col_values(cols_numb[3])[1:]# 获取第三列内容,从第一行开始
p1 = r"(?:[\u2E80-\uFFFD]|[\u201c-\u201d]|[\u002d]|[\u003a])+"#(?:）不获取匹配，即不获取括号内的匹配，括号内使用UNICODE编码匹配
pattern1 = re.compile(p1)
for i in range(len(cols1)):
matcher1 = re.findall(pattern1, cols1[i])#以列表形式返回所有能匹配到的子串
str1=str()
if matcher1:
str1 =''.join(matcher1)
f1.write(str1.strip())
f1.write('\n')
f1.close()
def ldaAnaly():
print("构造分词库-----train-----")
#去停用词，构建分词库
train = []
fp = codecs.open('./result/%s_sheet%s.csv'%(FILE,ID),'r',encoding='utf8')
for line in fp.readlines():
line = line.strip()
if not len(line):#判断是否为空行
continue
outstr = ' '
seg_list =jieba.cut(line,cut_all=False)#采用精确模式分词，效果最好
for word in seg_list:
if word not in stopwords:
if word != '\t':
outstr += word
outstr += " "
train.append(outstr.strip().split(" "))#字符串转列表
fp.close()
print("构造分词库,并保存----“dict_v1.dict”----")
dic = corpora.Dictionary(train)
dic.save('./result/dict_v1.dict')
print("保存可读取的分词库----“dic.csv”----")
fd = codecs.open('./result/dic.csv', 'a',encoding = 'utf-8')
for word,index in dic.token2id.items():
fd.write(word +':'+ str(index)+'\n')
fd.close()
print("生成语料库，并保存-----“corpus.mm”-----")
corpus = [dic.doc2bow(text) for text in train]
corpora.MmCorpus.serialize('./result/corpus.mm', corpus)
print("保存tfidf模型-----“corpus.tfidf_model”-----")
tfidf = models.TfidfModel(corpus)
tfidf.save('./result/corpus.tfidf_model')
print("进行LDA主题分析，并保存-----“ldaModel.pkl”-----")
#使用tf-idf模型训练语料库
corpus_tfidf = tfidf[corpus]
#设置100个LDA主题，使用500次迭代
lda = models.LdaModel(corpus_tfidf, id2word=dic, num_topics=100, iterations=500 )
lda.save('./result/ldaModel.pkl')
print("评估文章属于不同主题的概率，一个词对文章的重要性-----“Demo：评估文章1”-----")
for index, score in sorted(lda[corpus_tfidf[0]], key=lambda tup: -1 * tup[1]):
print("Score: {}\t Topic: {}".format(score, lda.print_topic(index, 10)))
# 输出100个主题
# ldaOut = lda.print_topics(100)
# print("默认返回每个主题的前10的概率最大的词")
# print (ldaOut[0])
# print (ldaOut[1])
# print (ldaOut[2])
# corpus_lda = lda[corpus_tfidf]
# print("每篇文章属于不同主题的概率分布")
# k = 0
# for doc in corpus_lda:
# print(doc)
# k += 1
# if k == 3:
# break
def questionAnswer(cols_numb, questionNumber):
lda = models.LdaModel.load('./result/ldaModel.pkl')
dic = corpora.Dictionary.load('./result/dict_v1.dict')
corpus = corpora.MmCorpus('./result/corpus.mm')
tfidf = models.TfidfModel.load('./result/corpus.tfidf_model')
# print("输入一个问题------------------")
f1 = open('./result/%s_sheet%s.csv'%(FILE,ID), 'a', encoding='utf-8')
# 打开文件
workbook = xlrd.open_workbook('./data/%s.xls'%FILE)
sheet0 = workbook.sheet_by_index(int(ID)-1) # sheet索引从0开始
cols0 = sheet0.col_values(cols_numb[3])[questionNumber] # 获取第三列内容,从第一行开始
#对问题进行去乱码
p1 = r"(?:[\u2E80-\uFFFD]|[\u201c-\u201d]|[\u002d]|[\u003a])+"#(?:）不获取匹配，即不获取括号内的匹配，括号内使用UNICODE编码匹配
pattern1 = re.compile(p1)
matcher1 = re.findall(pattern1, cols0)#以列表形式返回所有能匹配到的子串
query=str()
if matcher1:
query =''.join(matcher1)
# print("待预测的问题(去乱码)：", query)
#对问题进行分词
seg_list = jieba.cut(query, cut_all=False)
outstr = ' '
for word in seg_list:
if word not in stopwords:
if word != '\t':
outstr += word
outstr += " "
inputTest=list(outstr.strip().split(" "))
# print("分词后的问题(去停用词)：", inputTest)
#将问题转成词袋
query_bow = dic.doc2bow(inputTest)
# print("生成的词袋：", query_bow)
#需要对查询语句进行tfidf转化
query_tfidf = tfidf[query_bow]
lda_vec_tfidf = lda[query_tfidf]
# print("问题对应的主题概率(tfidf)", lda_vec_tfidf)
# print("预测问题属于不同主题的概率--------------------")
#输出主题概率的代码
# for index, score in sorted(lda_vec_tfidf, key=lambda tup: -1 * tup[1]):
# print("Score: {}\t Topic: {}".format(score, lda.print_topic(index, 20)))
# print("预测问题与数据库中的哪些问题相似，并给出相似度排序(tfidf)--------------------")
#进行相似性检索
similarity = similarities.MatrixSimilarity(corpus)
#在TFIDF的基础上，进行相似性检测。query_lsi需要进行预先处理。先变化为dow2bow,然后tfidf.
lda_vec = lda[query_bow]
# sims = similarity[lda_vec] #相似度检测的词袋为no-tfidf
sims = similarity[lda_vec_tfidf] #相似度检测的词袋为tfidf
#先枚举出来，后进行排序输出
listSims = enumerate(sims)
sort_sims = sorted(listSims, key=lambda item: -item[1])
# print(sort_sims[0:6])#前n名效果最好
#进行分数预测--版本1---
sort_sims_list = sort_sims[0:6]
cols1 = sheet0.col_values(cols_numb[6])[1:]# 获取第三列内容,从第一行开始
f1.close()
#采用百分比形式的加权平均法，实质就是加权平均偏差法
sumCore1 = 0
sumPro = 0
for i in range(len(sort_sims_list)):
sumCore1 += float(cols1[sort_sims_list[i][0] - 1]) * sort_sims_list[i][1]
# print(cols1[sort_sims_list[i][0] - 1])
sumPro += sort_sims_list[i][1]
preCore1 = sumCore1 / sumPro
# print("采用加权平均偏差法，预测分数1为：%s,实际分数为%s"%(preCore1, cols1[questionNumber-1]))
print("保存预测结果----“pre.csv”----")
return preCore1, cols1[questionNumber-1], abs(preCore1 - float(cols1[questionNumber-1]))
if __name__ == '__main__':
cols_numb = [0,1,2,3,4,5,6,7] #读取文件的列标号
# questionNumber = 124 #待测试的问题号，最大不超过问题总数，主要用于测试
# cleanAnswer(cols_numb) #对数据库中的问题进行提取，并去乱码
# ldaAnaly() #对问题进行训练，生成主题模型
# questionAnswer(cols_numb, questionNumber) #对问题进行预测，给出预测分数
#循环预测的demo
fp = codecs.open('./result/pre_v1.csv', 'a', encoding='utf-8')
sum = 0
i = 1
count = 0
while( i < 8717 ):
questionNumber = i
a = questionAnswer(cols_numb, questionNumber)
sum += a[2]
# print(a, a[2])
# exit()
i += 8
count += 1
fp.write(str(i)+":"+str(a) + '\n')
fp.close()
ave = sum / count
print(ave)
