from pandas import read_csv
import numpy as np
from sklearn.datasets.base import Bunch
import pickle #导入cPickle包并且取一个别名pickle #持久化类
from sklearn.feature_extraction.text import TfidfVectorizer
import jieba
import xlwt
import operator#排序用
from sklearn import metrics
Straindata=[]
Strainlabel=[]
Sart_train=[]
Stestdata=[]
Stestlabel=[]
Sart_test=[]
Slast=[]
Snew=[]
class obj:
def __init__(self):
self.key=0
self.weight=0.0
def importSmallContentdata(file,data,art,label,f):
dataset=read_csv(file)
Sdata = dataset.values[:,:]
print(type(Sdata))
if f==1:
for line in Sdata:
ls=[]
ls.append(line[14])
ls.append(line[15])
ls.append(line[16])
ls.append(line[17])
Slast.append(ls)
#print(len(Slast))
#print("需要对照的小类数据准备完毕")
'''找到smalli不为0的装入Straindata,把数据分开'''
for smalli in range(14,18):
#print(smalli)
count=0
for line in Sdata:
count=count+1
if line[smalli]!='0' and line[smalli]!=0 :
k=1
ls=[]
for i in line:
if k==1:
art.append(i)
k=k+1
continue
if k==11:#k为14并不代表是line[14]，因为line是从0开始
break
ls.append(float(i))
k=k+1
data.append(ls)
label.append(line[smalli])
if f==1:
Snew.append(count)
#print("为什么都超限",len(Snew))
def getKvector(train_set,vec,n):
nonzero=train_set.tdm.nonzero()
k=0
lis=[]
gather=[]
p=-1
for i in nonzero[0]:
p=p+1
if k==i:
a=obj()
a.key=nonzero[1][p]
a.weight=train_set.tdm[i,nonzero[1][p]]
lis.append(a)
else:
lis.sort(key=lambda obj: obj.weight, reverse=True)#对链表内为类对象的排序
gather.append(lis)
while k < i:
k=k+1
lis=[]
a=obj()
a.key=nonzero[1][p]
a.weight=train_set.tdm[i,nonzero[1][p]]
lis.append(a)
gather.append(lis)#gather存储的是每条数据的事实描述的特征向量，已经从小到大排好了，只不过每个存既有key又有weight
#我们只要key，不再需要weight
sj=1
for i in gather:
ls=[]
for j in i:
sj=sj+1
ls.append(float(j.key))
while sj<=n:
sj=sj+1
ls.append(-1)
sj=1
vec.append(ls)
'''读取停用词'''
def _readfile(path):
with open(path, "rb") as fp:
content = fp.read()
return content
''' 读取bunch对象'''
def _readbunchobj(path):
with open(path, "rb") as file_obj:
bunch = pickle.load(file_obj)
return bunch
'''写入bunch对象'''
def _writebunchobj(path, bunchobj):
with open(path, "wb") as file_obj:
pickle.dump(bunchobj, file_obj)
def buildtrainbunch(bunch_path,art_train,trainlabel):
bunch = Bunch(label=[],contents=[])
for item1 in trainlabel:
bunch.label.append(item1)
#trainContentdatasave=[] #存储所有训练和测试数据的分词
for item2 in art_train:
item2=str(item2)
item2 = item2.replace("\r\n", "")
item2 = item2.replace(" ", "")
content_seg=jieba.cut(item2)
save2=''
for item3 in content_seg:
if len(item3) > 1 and item3!='\r\n':
#trainContentdatasave.append(item3)
save2=save2+","+item3
bunch.contents.append(save2)
with open(bunch_path, "wb") as file_obj:
pickle.dump(bunch, file_obj)
print("构建训练数据文本对象结束！！！")
def buildtestbunch(bunch_path,art_test,testlabel):
bunch = Bunch(label=[],contents=[])
for item1 in testlabel:
bunch.label.append(item1)
#testContentdatasave=[] #存储所有训练和测试数据的分词
for item2 in art_test:
item2=str(item2)
item2 = item2.replace("\r\n", "")
item2 = item2.replace(" ", "")
content_seg=jieba.cut(item2)
save2=''
for item3 in content_seg:
if len(item3) > 1 and item3!='\r\n':
#testContentdatasave.append(item3)
save2=save2+","+item3
bunch.contents.append(save2)
with open(bunch_path, "wb") as file_obj:
pickle.dump(bunch, file_obj)
print("构建测试数据文本对象结束！！！")
def vector_space(stopword_path,bunch_path,space_path):
stpwrdlst = _readfile(stopword_path).splitlines()#读取停用词
bunch = _readbunchobj(bunch_path)#导入分词后的词向量bunch对象
#构建tf-idf词向量空间对象
tfidfspace = Bunch(label=bunch.label,tdm=[], vocabulary={})
#权重矩阵tdm，其中，权重矩阵是一个二维矩阵，tdm[i][j]表示，第j个词（即词典中的序号）在第i个类别中的IF-IDF值
#使用TfidVectorizer初始化向量空间模型
vectorizer = TfidfVectorizer(stop_words=stpwrdlst, sublinear_tf=True, max_df=0.5, min_df=0.0001,use_idf=True,max_features=15000)
#print(vectorizer)
#文本转为词频矩阵，单独保存字典文件
tfidfspace.tdm = vectorizer.fit_transform(bunch.contents)
tfidfspace.vocabulary = vectorizer.vocabulary_
#创建词袋的持久化
_writebunchobj(space_path, tfidfspace)
print("if-idf词向量空间实例创建成功！！！")
def testvector_space(stopword_path,bunch_path,space_path,train_tfidf_path):
stpwrdlst = _readfile(stopword_path).splitlines()#把停用词变成列表
bunch = _readbunchobj(bunch_path)
tfidfspace = Bunch(label=bunch.label,tdm=[], vocabulary={})
#导入训练集的TF-IDF词向量空间 ★★
trainbunch = _readbunchobj(train_tfidf_path)
tfidfspace.vocabulary = trainbunch.vocabulary
vectorizer = TfidfVectorizer(stop_words=stpwrdlst, sublinear_tf=True, max_df=0.7, vocabulary=trainbunch.vocabulary, min_df=0.001)
tfidfspace.tdm = vectorizer.fit_transform(bunch.contents)
_writebunchobj(space_path, tfidfspace)
print("if-idf词向量空间实例创建成功！！！")
if __name__=="__main__":
'''============================先导入数据=================================='''
file_train = 'F:/goverment/exceloperating/all_tocai_train.csv'
file_test = 'F:/goverment/exceloperating/all_tocai_test.csv'
importSmallContentdata(file_train,Straindata,Sart_train,Strainlabel,0)
importSmallContentdata(file_test,Stestdata,Sart_test,Stestlabel,1)
#print("Stestlabel" ,len(Stestlabel))
#print("小类导入数据完毕")
#print("大类标签导入完毕")#共1329*4
'''==========================================================tf-idf对Bar进行文本特征提取============================================================================'''
#导入分词后的词向量bunch对象
train_bunch_path ="F:/goverment/exceloperating/trainbunch.bat"#Bunch保存路径
train_space_path = "F:/goverment/exceloperating/traintfdifspace.dat"
test_bunch_path ="F:/goverment/exceloperating/testbunch.bat"
test_space_path = "F:/goverment/exceloperating/testtfdifspace.dat"
stopword_path ="F:/goverment/exceloperating/hlt_stop_words.txt"
'''============================================================tf-idf对Sart进行文本特征提取=============================================================================='''
buildtrainbunch(train_bunch_path,Sart_train,Strainlabel)
buildtestbunch(test_bunch_path,Sart_test,Stestlabel)
vector_space(stopword_path,train_bunch_path,train_space_path)
testvector_space(stopword_path,test_bunch_path,test_space_path,train_space_path)
train_set=_readbunchobj(train_space_path)
test_set=_readbunchobj(test_space_path)
'''训练数据'''
S_vec_train=[]
getKvector(train_set,S_vec_train,76)
'''测试数据'''
S_vec_test=[]
getKvector(test_set,S_vec_test,76)
'''=================将得到的61个特征和之前的其它特征合并Btraindata=================='''
'''小类训练数据'''
S_vec_train=np.array(S_vec_train)
#print(type(S_vec_train))
#print(S_vec_train.shape)
Straindata=np.array(Straindata)
#print(type(Straindata))
#print(Straindata.shape)
Straindata=np.hstack((S_vec_train,Straindata))
#print(Straindata)
'''小类测试数据'''
S_vec_test=np.array(S_vec_test)
Stestdata=np.array(Stestdata)
Stestdata=np.hstack((S_vec_test,Stestdata))
print("分类算小类精度")
Strainlabel=np.array(Strainlabel)
Strainlabel=np.array(Strainlabel)
from xgboost import XGBClassifier
clf= XGBClassifier(learning_rate =0.1,
n_estimators=1150,
max_depth=2,
min_child_weight=1,
gamma=0,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,#没用
scale_pos_weight=1,#没用
seed=27)
clf.fit(Straindata, Strainlabel)
predict=clf.predict(Stestdata)
aa=metrics.accuracy_score(Stestlabel, predict)
print(aa)#40.09
''''============================输出技术问题及其可能性================'''
class attri:
def __init__(self):
self.key=0
self.weight=0.0
'''====================小类======================='''
attribute_proba=clf.predict_proba(Stestdata)
label=[]
for i in attribute_proba:
lis=[]
k=0
while k<4:
k=k+1
p=1
mm=0
sj=-1
for j in i:
sj=sj+1
if j>mm:
mm=j
p=sj
i[p]=0#难道是从1开始？
a=attri()
a.key=p
a.weight=mm
lis.append(a)
#lis.append(p)
label.append(lis)
#接下来将label和snew结合，再排序去重就可以和slast比较了
#print("为什么都超限",len(Snew))
print("label",len(label))
count=0
for lis in label:
lis.append(Snew[count])
count=count+1
print("结合完成，准备去重！")#此时label和Snew的长度都为1439
bol=np.zeros(len(label)+1)
Snew=[]
for lis in label:
if bol[lis[4]]==0:
Snew.append(lis)
bol[lis[4]]=1
#print(len(Snew))#去重后为1162
for i in range(len(Slast)+1):
if i==0:
continue
if bol[i]==0:
ls=[]
a=attri()
a.weight=1
a.key=0
ls.append(a)
ls.append(a)
ls.append(a)
ls.append(a)
ls.append(i)
Snew.append(ls)
#print("Snew",len(Snew)) #为1329
print("去重完毕，准备排序！")
Snew.sort(key=operator.itemgetter(4))
print("排序完毕，准备比较!")
myexcel = xlwt.Workbook()
sheet = myexcel.add_sheet('sheet')
si=-2
sj=-1
#cys=1
#print(Snew)
for i in Snew:
si=si+2
#print(si)
#print("对于记录 %d:" % cys)
#cys=cys+1
for j in range(len(i)):
if(j==len(i)-1):
continue
sj=sj+1
#sheet.write(si,sj,str(j))
sheet.write(si,sj,str(i[j].key))
sheet.write(si+1,sj,str(i[j].weight*100))
#print ("发生技术问题 %d 的可能性是:%.2f %%" % (j.key,j.weight*100))
sj=-1
myexcel.save("Snew.xls")
