《机器学习实战》KNN算法实现

 

---

 

本系列都是参考《机器学习实战》这本书，只对学习过程一个记录，不做详细的描述！

注释：看了一段时间Ng的机器学习视频，感觉不能光看不练，现在一边练习再一边去学习理论！

KNN很早就之前就看过也记录过，在此不做更多说明，这是k-means之前的记录，感觉差不多：http://www.cnblogs.com/wjy-lulu/p/7002688.html

 

1.简单的分类

代码：

import numpy as np
import operator
import KNN

def classify0(inX,dataSet,labels,k):
     dataSetSize = dataSet.shape[0] #样本个数
     diffMat = np.tile(inX,(dataSetSize,1)) - dataSet#样本每个值和测试数据做差
     sqDiffMat = diffMat**2#平方
     sqDistances = sqDiffMat.sum(axis=1)#第二维度求和，也就是列
     distances = sqDistances**0.5#平方根
     sortedDistIndicies = distances.argsort()#下标排序
     classCount = {}

     for i in range(k):
          voteIlabel = labels[sortedDistIndicies[i]]#得到距离最近的几个数
          classCount[voteIlabel] = classCount.get(voteIlabel,0)+1#标签计数
     sortedClassCount = sorted(classCount.items(),key=operator.itemgetter(1),reverse=True)#按照数值排序operator.itemgetter(1)代表第二个域
     #上面排序之后就不是字典了，而是一个列表里面包含的元组[('c',2),('a',3)]
     return sortedClassCount[0][0]

if __name__ == '__main__':
     group,labels = KNN.createDataSet()
     result = classify0([0,0.5],group,labels,1)
     print (result)

KNN.Py文件

import numpy as np
import operator


def createDataSet():
    group = np.array([[1.0, 1.1], [1.0, 1.0], [0, 0], [0, 0.1]])
    labels = ['A', 'B', 'C', 'D']
    return group, labels

2.约会网站的预测

　　下面给出每个部分的代码和注释：

A.文本文件转换为可用数据

上面的文本中有空格和换行，而且样本和标签都在一起，必须的分开处理成矩阵才可以进行下一步操作。

def file2matrix(filename):#把文件转化为可操作数据
    fr = open(filename)#打开文件
    arrayOLines = fr.readlines()#读取每行文件
    numberOfLines = len(arrayOLines)#行数量
    returnMat = np.zeros([numberOfLines,3])#存储数据
    classLabelVector = []
    index = 0
    for line in arrayOLines:
        line = line.strip()#去除换行符
        listFromLine = line.split('\t')#按照空格去分割
        returnMat[index,:] = listFromLine[0:3]#样本
        classLabelVector.append(int(listFromLine[-1]))#labels
        index += 1
    return returnMat,classLabelVector#返回数据和标签

 B.归一化

数据大小差异太明显，比如有三个特征：a=[1,2,3],b=[1000,2000,3000],c=[0.1,0.2,0.3],我们发现c和a根本没啥作用，因为b的值太大了，或者说b的权重太大了，Ng中可以用惩罚系数去操作，或者正则化都可以处理这类数据，当然这是题外话。

def autoNorm(dataSet):#归一化函数
    #每列的最值
    minValue = dataSet.min(0)
    maxValue = dataSet.max(0)
    range = maxValue - minValue
    #创建最小值矩阵
    midData =  np.tile(minValue,[dataSet.shape[0],1])
    dataSet = dataSet - midData
    #创建range矩阵
    range = np.tile(range,[dataSet.shape[0],1])
    dataSet = dataSet / range #直接相除不是矩阵相除
    return dataSet,minValue,maxValue

C.预测

KNN的方法就是距离，计算K个距离，然后排序看哪个占得比重大就选哪个类。

def classify0(inX, dataSet, labels, k):#核心分类程序
    dataSetSize = dataSet.shape[0]  # 样本个数
    diffMat = np.tile(inX, (dataSetSize, 1)) - dataSet  # 样本每个值和测试数据做差
    sqDiffMat = diffMat ** 2  # 平方
    sqDistances = sqDiffMat.sum(axis=1)  # 第二维度求和，也就是列
    distances = sqDistances ** 0.5  # 平方根
    sortedDistIndicies = distances.argsort()  # 下标排序
    classCount = {}

    for i in range(k):
        voteIlabel = labels[sortedDistIndicies[i]]  # 得到距离最近的几个数
        classCount[voteIlabel] = classCount.get(voteIlabel, 0) + 1  # 标签计数
    sortedClassCount = sorted(classCount.items(), key=operator.itemgetter(1),
                              reverse=True)  # 按照数值排序operator.itemgetter(1)代表第二个域
    # 上面排序之后就不是字典了，而是一个列表里面包含的元组[('c',2),('a',3)]
    return sortedClassCount[0][0]

D.性能测试

比如1000个数据，900个用做样本，100用做测试，看看精确度是多少?

def datingClassTest():
    hoRatio = 0.2
    datingDataMat , datingLabels = file2matrix('datingTestSet2.txt')
    normMat = autoNorm(datingDataMat)
    n = normMat.shape[0]
    numTestVecs = int(n*hoRatio)#测试数据和样本数据的分割点
    erroCount = 0.0
    #numTestVecs:n样本，[i,numTestVecs]测试
    for i in range(numTestVecs):
        classfiResult = classify0(normMat[i,:],normMat[numTestVecs:n,:],
                                  datingLabels[numTestVecs:n],3)
        if (classfiResult!=datingLabels[i]): erroCount+=1.0
    print ("the totle error os: %f" %(erroCount/float(numTestVecs)))

 E.实战分类

注意输入的数据也得归一化

def classfiPerson():
    resultList = ['not at all','in small doses','in large doses']
    personTats = float(input('please input video game \n'))
    ffMiles = float(input('please input flier miles \n'))
    iceCream = float(input('please input ice cream \n'))
    datingData,datingLabels = file2matrix('datingTestSet2.txt')
    normData,minData,maxData = autoNorm(datingData)
    inputData = np.array([personTats,ffMiles,iceCream])#转化为矩阵
    inputData = (inputData - minData)/(maxData - minData)#输入归一化
    result = classify0(inputData,normData,datingLabels,3)
    print('等级是：',result)

F.可视化显示

 

     datingDatas, datingLabels = KNN.file2matrix('datingTestSet2.txt')
     #可视化样本数据显示
     fig = plt.figure('data_show')
     ax = fig.add_subplot(111)
     for i in range(datingDatas.shape[0]):
          if datingLabels[i]==1:
               ax.scatter(datingDatas[i, 0], datingDatas[i, 1], marker="*",c='r')  # 用后两个特征绘图

          if datingLabels[i]==2:
               ax.scatter(datingDatas[i, 0], datingDatas[i, 1], marker="s", c='g')  # 用后两个特征绘图

          if datingLabels[i]==3:
               ax.scatter(datingDatas[i, 0], datingDatas[i, 1], marker="^", c='b')  # 用后两个特征绘图
     plt.show()

 

 

G.完整代码

import numpy as np
import operator
#from numpy import *

def createDataSet():#创建简单测试的几个数
    group = np.array([[1.0, 1.1], [1.0, 1.0], [0, 0], [0, 0.1]])
    labels = ['A', 'B', 'C', 'D']
    return group, labels

def autoNorm(dataSet):#归一化函数
    #每列的最值
    minValue = dataSet.min(0)
    maxValue = dataSet.max(0)
    range = maxValue - minValue
    #创建最小值矩阵
    midData =  np.tile(minValue,[dataSet.shape[0],1])
    dataSet = dataSet - midData
    #创建range矩阵
    range = np.tile(range,[dataSet.shape[0],1])
    dataSet = dataSet / range #直接相除不是矩阵相除
    return dataSet,minValue,maxValue

def file2matrix(filename):#把文件转化为可操作数据
    fr = open(filename)#打开文件
    arrayOLines = fr.readlines()#读取每行文件
    numberOfLines = len(arrayOLines)#行数量
    returnMat = np.zeros([numberOfLines,3])#存储数据
    classLabelVector = []
    index = 0
    for line in arrayOLines:
        line = line.strip()#去除换行符
        listFromLine = line.split('\t')#按照空格去分割
        returnMat[index,:] = listFromLine[0:3]#样本
        classLabelVector.append(int(listFromLine[-1]))#labels
        index += 1
    return returnMat,classLabelVector#返回数据和标签

def classify0(inX, dataSet, labels, k):#核心分类程序
    dataSetSize = dataSet.shape[0]  # 样本个数
    diffMat = np.tile(inX, (dataSetSize, 1)) - dataSet  # 样本每个值和测试数据做差
    sqDiffMat = diffMat ** 2  # 平方
    sqDistances = sqDiffMat.sum(axis=1)  # 第二维度求和，也就是列
    distances = sqDistances ** 0.5  # 平方根
    sortedDistIndicies = distances.argsort()  # 下标排序
    classCount = {}

    for i in range(k):
        voteIlabel = labels[sortedDistIndicies[i]]  # 得到距离最近的几个数
        classCount[voteIlabel] = classCount.get(voteIlabel, 0) + 1  # 标签计数
    sortedClassCount = sorted(classCount.items(), key=operator.itemgetter(1),
                              reverse=True)  # 按照数值排序operator.itemgetter(1)代表第二个域
    # 上面排序之后就不是字典了，而是一个列表里面包含的元组[('c',2),('a',3)]
    return sortedClassCount[0][0]

def datingClassTest():
    hoRatio = 0.2
    datingDataMat , datingLabels = file2matrix('datingTestSet2.txt')
    normMat = autoNorm(datingDataMat)
    n = normMat.shape[0]
    numTestVecs = int(n*hoRatio)#测试数据和样本数据的分割点
    erroCount = 0.0
    #numTestVecs:n样本，[i,numTestVecs]测试
    for i in range(numTestVecs):
        classfiResult = classify0(normMat[i,:],normMat[numTestVecs:n,:],
                                  datingLabels[numTestVecs:n],3)
        if (classfiResult!=datingLabels[i]): erroCount+=1.0
    print ("the totle error os: %f" %(erroCount/float(numTestVecs)))

def classfiPerson():
    resultList = ['not at all','in small doses','in large doses']
    personTats = float(input('please input video game \n'))
    ffMiles = float(input('please input flier miles \n'))
    iceCream = float(input('please input ice cream \n'))
    datingData,datingLabels = file2matrix('datingTestSet2.txt')
    normData,minData,maxData = autoNorm(datingData)
    inputData = np.array([personTats,ffMiles,iceCream])#转化为矩阵
    inputData = (inputData - minData)/(maxData - minData)#输入归一化
    result = classify0(inputData,normData,datingLabels,3)
    print('等级是：',result)

 

3.手写数字识别

 A.转换文件

def img2vector(filename):
    returnVector = np.zeros([32,32])
    fr = open(filename)
    lineData = fr.readlines()
    count = 0
    for line in lineData:
        line = line.strip()#去除换行符
        for j in range(len(line)):
            returnVector[count,j] = line[j]
        count += 1
    returnVector = returnVector.reshape(1,1024).astype(int)#转化为1X1024
    return returnVector

B.识别分类

def handWriteringClassTest():
    #--------------------------读取数据---------------------------------
    hwLabels = []
    trainingFileList = os.listdir('trainingDigits')#获取文件目录
    m = len(trainingFileList)#获取目录个数
    trainingMat = np.zeros([m,1024])#全部样本
    for i in range(m):
        fileNameStr = trainingFileList[i]
        fileStr = fileNameStr.split('.')[0]#得到不带格式的文件名
        classNumStr = int(fileStr.split('_')[0])#得到最前面的数字类别0-9
        hwLabels.append(classNumStr)#存储
        dirList = 'trainingDigits/' + fileNameStr#绝对目录信息
        vectorUnderTest = img2vector(dirList)#读取第i个数据信息
        trainingMat[i,:] = vectorUnderTest #存储
    #--------------------------测试数据--------------------------------
    testFileList = os.listdir('testDigits')
    errorCount = 0.0
    m = len(testFileList)
    for i in range(m):
        fileNameStr = testFileList[i]
        fileInt = fileNameStr.split('.')[0].split('_')[0]
        dirList = 'testDigits/' + fileNameStr  # 绝对目录信息
        vectorUnderTest = img2vector(dirList)  # 读取第i个数据信息
        if int(fileInt) != int(classify0(vectorUnderTest,trainingMat,hwLabels,3)):
            errorCount += 1
    print('error count is : ',errorCount)
    print('error Rate is : ', (errorCount/m))

C.完整代码

import numpy as np
import operator
import os
#from numpy import *

def createDataSet():#创建简单测试的几个数
    group = np.array([[1.0, 1.1], [1.0, 1.0], [0, 0], [0, 0.1]])
    labels = ['A', 'B', 'C', 'D']
    return group, labels

def autoNorm(dataSet):#归一化函数
    #每列的最值
    minValue = dataSet.min(0)
    maxValue = dataSet.max(0)
    range = maxValue - minValue
    #创建最小值矩阵
    midData =  np.tile(minValue,[dataSet.shape[0],1])
    dataSet = dataSet - midData
    #创建range矩阵
    range = np.tile(range,[dataSet.shape[0],1])
    dataSet = dataSet / range #直接相除不是矩阵相除
    return dataSet,minValue,maxValue

def file2matrix(filename):#把文件转化为可操作数据
    fr = open(filename)#打开文件
    arrayOLines = fr.readlines()#读取每行文件
    numberOfLines = len(arrayOLines)#行数量
    returnMat = np.zeros([numberOfLines,3])#存储数据
    classLabelVector = []
    index = 0
    for line in arrayOLines:
        line = line.strip()#去除换行符
        listFromLine = line.split('\t')#按照空格去分割
        returnMat[index,:] = listFromLine[0:3]#样本
        classLabelVector.append(int(listFromLine[-1]))#labels
        index += 1
    return returnMat,classLabelVector#返回数据和标签

def classify0(inX, dataSet, labels, k):#核心分类程序
    dataSetSize = dataSet.shape[0]  # 样本个数
    diffMat = np.tile(inX, (dataSetSize, 1)) - dataSet  # 样本每个值和测试数据做差
    sqDiffMat = diffMat ** 2  # 平方
    sqDistances = sqDiffMat.sum(axis=1)  # 第二维度求和，也就是列
    distances = sqDistances ** 0.5  # 平方根
    sortedDistIndicies = distances.argsort()  # 下标排序
    classCount = {}

    for i in range(k):
        voteIlabel = labels[sortedDistIndicies[i]]  # 得到距离最近的几个数
        classCount[voteIlabel] = classCount.get(voteIlabel, 0) + 1  # 标签计数
    sortedClassCount = sorted(classCount.items(), key=operator.itemgetter(1),
                              reverse=True)  # 按照数值排序operator.itemgetter(1)代表第二个域
    # 上面排序之后就不是字典了，而是一个列表里面包含的元组[('c',2),('a',3)]
    return sortedClassCount[0][0]

def datingClassTest():
    hoRatio = 0.2
    datingDataMat , datingLabels = file2matrix('datingTestSet2.txt')
    normMat = autoNorm(datingDataMat)
    n = normMat.shape[0]
    numTestVecs = int(n*hoRatio)#测试数据和样本数据的分割点
    erroCount = 0.0
    #numTestVecs:n样本，[i,numTestVecs]测试
    for i in range(numTestVecs):
        classfiResult = classify0(normMat[i,:],normMat[numTestVecs:n,:],
                                  datingLabels[numTestVecs:n],3)
        if (classfiResult!=datingLabels[i]): erroCount+=1.0
    print ("the totle error os: %f" %(erroCount/float(numTestVecs)))

def classfiPerson():
    resultList = ['not at all','in small doses','in large doses']
    personTats = float(input('please input video game \n'))
    ffMiles = float(input('please input flier miles \n'))
    iceCream = float(input('please input ice cream \n'))
    datingData,datingLabels = file2matrix('datingTestSet2.txt')
    normData,minData,maxData = autoNorm(datingData)
    inputData = np.array([personTats,ffMiles,iceCream])#转化为矩阵
    inputData = (inputData - minData)/(maxData - minData)#输入归一化
    result = classify0(inputData,normData,datingLabels,3)
    print('等级是：',result)

def img2vector(filename):
    returnVector = np.zeros([32,32])
    fr = open(filename)
    lineData = fr.readlines()
    count = 0
    for line in lineData:
        line = line.strip()#去除换行符
        for j in range(len(line)):
            returnVector[count,j] = line[j]
        count += 1
    returnVector = returnVector.reshape(1,1024).astype(int)#转化为1X1024
    return returnVector

def img2vector2(filename):
    returnVect = np.zeros([1,1024])
    fr = open(filename)
    for i in range(32):
        lineStr = fr.readline()
        for j in range(32):
            returnVect[0,32*i+j] = int(lineStr[j])
    return returnVect

def handWriteringClassTest():
    #--------------------------读取数据---------------------------------
    hwLabels = []
    trainingFileList = os.listdir('trainingDigits')#获取文件目录
    m = len(trainingFileList)#获取目录个数
    trainingMat = np.zeros([m,1024])#全部样本
    for i in range(m):
        fileNameStr = trainingFileList[i]
        fileStr = fileNameStr.split('.')[0]#得到不带格式的文件名
        classNumStr = int(fileStr.split('_')[0])#得到最前面的数字类别0-9
        hwLabels.append(classNumStr)#存储
        dirList = 'trainingDigits/' + fileNameStr#绝对目录信息
        vectorUnderTest = img2vector(dirList)#读取第i个数据信息
        trainingMat[i,:] = vectorUnderTest #存储
    #--------------------------测试数据--------------------------------
    testFileList = os.listdir('testDigits')
    errorCount = 0.0
    m = len(testFileList)
    for i in range(m):
        fileNameStr = testFileList[i]
        fileInt = fileNameStr.split('.')[0].split('_')[0]
        dirList = 'testDigits/' + fileNameStr  # 绝对目录信息
        vectorUnderTest = img2vector(dirList)  # 读取第i个数据信息
        if int(fileInt) != int(classify0(vectorUnderTest,trainingMat,hwLabels,3)):
            errorCount += 1
    print('error count is : ',errorCount)
    print('error Rate is : ', (errorCount/m))