期末大作业

#导入Boston房价数据集以及相关包
from sklearn import datasets
from sklearn.linear_model import LinearRegression
from sklearn.metrics import regression
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import PolynomialFeatures

data = datasets.load_boston()

#转化为DataFrame
import pandas
dataDF = pandas.DataFrame(data.data, columns=data.feature_names)
#划分数据集
x_train, x_test, y_train, y_test = train_test_split(data.data, data.target, test_size=0.3)

#建立线性回归模型
mlr = LinearRegression()
mlr.fit(x_train, y_train)
print('系数', mlr.coef_, "\n截距", mlr.intercept_)

#检测模型好坏
y_predict=mlr.predict(x_test)
#计算模型的预测指标
print("预测的均方误差：",regression.mean_squared_error(y_test,y_predict))
print("预测的平均绝对误差：",regression.mean_absolute_error(y_test,y_predict))

#模型分数
print("模型的准确率：",mlr.score(x_test,y_test))

#建立多项式回归模型
poly=PolynomialFeatures(degree=2)
x_poly_train=poly.fit_transform(x_train)
x_poly_test=poly.transform(x_test)
mlrpro=LinearRegression()
mlrpro.fit(x_poly_train,y_train)
print('系数',mlrpro.coef_,"\n截距",mlrpro.intercept_)

#检测模型好坏
y_predict2=mlrpro.predict(x_poly_test)
#计算模型的预测指标
print("预测的均方误差：",regression.mean_squared_error(y_test,y_predict2))
print("预测的平均绝对误差：",regression.mean_absolute_error(y_test,y_predict2))

#模型分数
print("模型的准确率：",mlrpro.score(x_poly_test,y_test))

 

 

 

文本分类

#导入数据包加载数据
import os
import codecs
import jieba
path=r'G:\147'

#导入停用词库
with open(r'G:\stopsCN.txt',encoding='utf-8')as s:
    stopwords = s.read().split('\n')

#存放文件(容器)
DataPaths=[]
#存放文件类型
DataClasses=[]
#存放读取的新闻内容
DataContents=[]

#遍历转码处理
for root,dirs,files in os.walk(path):#循环目录
    for n in files:
        DataPath = os.path.join(root,n)#把路径和文件串起来
        DataPaths.append(DataPath)#将上一步数据添加到外部容器
        DataClasses.append(DataPath.split('\\')[2])#提取新闻的类别(子文件夹名)
        s = codecs.open(DataPath,'r','utf-8')#转码
        DataContent = s.read()#读取
        DataContent = DataContent.replace('\n','')#去掉\n
        tokens = [token for token in jieba.cut(DataContent)]#结巴分词
        tokens = " ".join([token for token in tokens if token not in stopwords])#去掉停用词
        s.close()#关闭文件
        DataContents.append(tokens)#将关键字添加到外部容器

#矩阵排列
import pandas
all_datas = pandas.DataFrame({
    '新闻类别':DataClasses,
    '新闻内容':DataContents
})
print(all_datas)

#将所有数据合并为整条字符串
a=''
for i in range(len(DataContents)):
    a+=DataContents[i]

#用TF-IDF算法统计词频(前20最高频率)
import jieba.analyse
keywords = jieba.analyse.extract_tags(a,topK=20,withWeight=True,allowPOS=('n','nr','ns'))

#划分数据集
from sklearn.model_selection import train_test_split
x_train,x_test,y_train,y_test = train_test_split(DataContents,DataClasses,test_size=0.3,random_state=0,stratify=DataClasses)
#数据向量化
from sklearn.feature_extraction.text import TfidfVectorizer
vectors = TfidfVectorizer()
X_train = vectors.fit_transform(x_train)
X_test = vectors.transform(x_test)

#建立模型(贝叶斯)
from sklearn.naive_bayes import MultinomialNB
mx = MultinomialNB()
mod = mx.fit(X_train,y_train)
y_predict = mod.predict(X_test)#预测

#模型准确率
print("模型的准确率：",mod.score(X_test,y_test))

#模型评估报告
from sklearn.metrics import classification_report
print("模型评估报告：\n",classification_report(y_predict,y_test))

#新文本类别预测
import collections
testcount = collections.Counter(y_test)
predcount = collections.Counter(y_predict)
namelist = list(testcount.keys())
testlist = list(testcount.values())
predictlist = list(predcount.values())
x = list(range(len(namelist)))
print("类别：",namelist,'\n',"实际：",testlist,'\n',"预测：",predictlist)