分类文本代码

def loadDataset(filen):
    pattern = r'(\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})|(\d{2}:\d{2} \d{2}:\d{2}:\d{2}\.\d{3})'
    pattern_c = re.compile(pattern)
    dataset = []
    f = open(filen,'r')
    for line in f:
        line = re.sub(pattern_c,'',line)
        #line = textPrecessing(line)
        line = line.strip()
        dataset.append(line)
    f.close()
    return dataset

　　

#!--encoding=utf-8

from __future__ import print_function
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import HashingVectorizer
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans, MiniBatchKMeans
import re,os,sys

from sklearn.datasets import fetch_20newsgroups
import nltk
import string
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer

FILENAME = r'mengzhenhui.txt'
FILEPATH = r'D:\EntranceTest\nopassfile'

　　

def transform(dataset,n_features=5):
    vectorizer = TfidfVectorizer(max_df=0.5, max_features=n_features, min_df=2,use_idf=True)
    X = vectorizer.fit_transform(dataset)
    print('vectorizer = ',vectorizer)
    return X,vectorizer

def train(X,vectorizer,true_k=10,minibatch = False,showLable = False):
    #使用采样数据还是原始数据训练k-means，    
    if minibatch:
        km = MiniBatchKMeans(n_clusters=true_k, init='k-means++', n_init=1,
                             init_size=1000, batch_size=1000, verbose=False)
    else:
        km = KMeans(n_clusters=true_k, init='k-means++', max_iter=300, n_init=1,
                    verbose=False)
    km.fit(X)    
    if showLable:
        print("Top terms per cluster:")
        order_centroids = km.cluster_centers_.argsort()[:, ::-1]
        terms = vectorizer.get_feature_names()
        print (vectorizer.get_stop_words())
        for i in range(true_k):
            print("Cluster %d:" % i, end='')
            for ind in order_centroids[i, :20]:
                print(' %s' % terms[ind], end='')
            print()
    result = list(km.predict(X))
    print ('Cluster distribution:')
    print (dict([(i, result.count(i)) for i in result]))
    return -km.score(X)
    
def test():
    '''测试选择最优参数'''
    dataset = loadDataset(FILENAME)    
    print("%d documents" % len(dataset))
    X,vectorizer = transform(dataset,n_features=500)
    true_ks = []
    scores = []
    for i in range(3,80,1):        
        score = train(X,vectorizer,true_k=i)/len(dataset)
        print (i,score)
        true_ks.append(i)
        scores.append(score)
    plt.figure(figsize=(8,4))
    plt.plot(true_ks,scores,label="error",color="red",linewidth=1)
    plt.xlabel("n_features")
    plt.ylabel("error")
    plt.legend()
    plt.show()
    
def out():
    '''在最优参数下输出聚类结果'''
    #dataset = loadDataset(FILENAME)
    dataset = loadDataset_file(filelist)
    X,vectorizer = transform(dataset,n_features=100)
    #print(vectorizer[1])
    score = train(X,vectorizer,true_k=4,showLable=True)/len(dataset)
    print (score)
    return X,vectorizer

　　

X,V = out()

from scipy.spatial.distance import cdist
import numpy as np

K = range(1, 10)
meandistortions = []
for k in K:
    kmeans = KMeans(n_clusters=k)
    kmeans.fit(X)
    meandistortions.append(sum(np.min(cdist(X, kmeans.cluster_centers_, 'euclidean'), axis=1)) / X.shape[0])
plt.plot(K, meandistortions, 'bx-')
plt.xlabel('k')
plt.ylabel('平均畸变程度',fontproperties=font)
plt.title('用肘部法则来确定最佳的K值',fontproperties=font);

　　

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-15-6990a0ca6076> in <module>()
      4     kmeans = KMeans(n_clusters=k)
      5     kmeans.fit(X)
----> 6     meandistortions.append(sum(np.min(cdist(X, kmeans.cluster_centers_, 'euclidean'), axis=1)) / X.shape[0])
      7 plt.plot(K, meandistortions, 'bx-')
      8 plt.xlabel('k')

D:\anaconda\lib\site-packages\scipy\spatial\distance.py in cdist(XA, XB, metric, p, V, VI, w)
   2130 
   2131     if len(s) != 2:
-> 2132         raise ValueError('XA must be a 2-dimensional array.')
   2133     if len(sB) != 2:
   2134         raise ValueError('XB must be a 2-dimensional array.')

ValueError: XA must be a 2-dimensional array.