决策树（ID3、C4.5、CART算法numpy实现）

什么是决策树？

决策树（decision tree）是一个树结构（可以是二叉树或非二叉树）。

其每个非叶节点表示一个特征属性上的测试，每个分支代表这个特征属性在某个值域上的输出，而每个叶节点存放一个类别。

使用决策树进行决策的过程就是从根节点开始，测试待分类项中相应的特征属性，并按照其值选择输出分支，直到到达叶子节点，将叶子节点存放的类别作为决策结果。

 

 

 

 

 

 

 

 

 

（图片来源）

如何建树？

 建树的关键在于特征选择，我们应当选择怎么样的特征作为我们的判断条件。

大致有三种算法：

1、ID3算法：基于信息增益准则选择特征

2、C4.5算法：基于信息增益比来选择特征

3、CART算法：基于基尼指数来选择特征

1、ID3算法：

如何计算信息增益：

K是分类值y的种类，|  |表示样本数量，n表示特征A所有可能的取值（这句话的意思是枚举所有在训练集出现过的值，即使在某层中该取值的样本数为0）。

例如：（图片来源）

ID3算法流程如下：

 

代码实现：

使用信息增益进行特征选择的缺点：偏向于选择取值较多的特征，可以理解为倾向划分更多的子树。

 

import numpy as np
from math import log2 as log
class TreeNode(object):
    def __init__(self,X,Y,S):
        self.label = None
        self.divide_dim = None
        self.X = X
        self.Y = Y
        self.children = {}
        self.is_leaf = True
        self.source =S #为了display函数
        self.id = None #节点ID,确保唯一
        self.child_ids =[]

class DecisionTree(object):
    def __init__(self,X,Y,function='ID3'):
        self.X = X
        self.Y = Y
        self.F = function
        self.num_features = len(X[0]) #特征数
        self.feature_map = {} #特征映射为int值
        self.maps = {} #每个特征中对应的值的映射，例如第二个特征，有一个叫‘是’，那么maps[1]['是']=0/1
        self.label_map = {} #y 标签映射为数值
        self.root = None
        self.epsilon = 0
        self.inv_maps={}
        self.ID = 1

    def train(self):
        # train函数主要是处理数据，把所有特征转化为数值型，方便使用numpy处理
        X = self.X
        Y = self.Y
        # X row = Y row +1
        # 默认X的第一行为特征描述
        features = X[0]
        xx = np.zeros([len(X),self.num_features],dtype=int)
        yy = np.zeros(len(Y),dtype=int)
        for feature,j in zip(features,range(self.num_features)):
            self.feature_map[feature] = j
            self.maps[j] ={}
            self.inv_maps[j]={}
            for i in range(1,len(X)):
                k = X[i][j]
                if k in self.maps[j].keys():
                    xx[i][j] = self.maps[j][k]
                else:
                    self.maps[j][k] = len(self.maps[j])
                    xx[i][j] =self.maps[j][k]
                    self.inv_maps[j][xx[i][j]]=k

        for i in range(len(Y)):
            k= Y[i]
            if k in self.label_map.keys():
                yy[i] = self.label_map[k]
            else:
                self.label_map[k] = len(self.label_map)
                yy[i] = self.label_map[k]
        xx = xx[1:]
        X = X[1:]
        # xx , yy all int
        self.root = TreeNode(xx,yy,xx)
        self.root.id = self.ID
        self.ID = self.ID+1
        print(self.root.X)
        print(self.root.Y)
        self.build_tree(self.root)

    def get_information_gain(self,X,Y,dim,idx):
        # dim 表示特征索引
        nums = X.shape[0]
        cnt_class = len(self.label_map)
        cnt_x = len(self.maps[dim])
        xx = X[:,dim]
        yy = Y
        hd =0
        for v in range(cnt_class):
            nums_v = np.where(yy==v,1,0).sum()
            if nums_v >0 :
                hd = hd - nums_v*log(nums_v/nums)/nums
        hda = 0
        for i in range(cnt_x):
            di = np.where(xx==i,1,0).sum()
            if di>0:
                tmpy=yy[np.where(xx==i,True,False)]
                for k in range(cnt_class):
                    dik = np.where(tmpy== k,1,0).sum()
                    if dik >0:
                        hda = hda - (di/nums)*(dik/di)*log(dik/di)
        # 显示当前节点计算的指定维度的信息增益
        inv_feature_map = dict(zip(self.feature_map.values(), self.feature_map.keys()))
        print('当前节点序号: ', idx, ' 特征: ', inv_feature_map[dim])
        print('information gain', hd - hda)
        return hd-hda

    def build_tree(self,root):
        if self.check_all_y(root.Y): #所有实例属于同一类
            root.is_leaf=True
            root.label = root.Y[0]
        elif self.check_X_null(root.X): #特征为空，无法进一步选择划分,标记为-1的列即表示已经使用过
            root.is_leaf=True
            root.label = self.get_max_y(root.Y)
        else:
            epsilon = self.epsilon #epsilon
            mx_dim = -1 #选取信息增益最大的作为特征维度
            for dim in self.maps.keys():
                if root.X[0,dim] != -1:
                    gda = self.get_information_gain(root.X,root.Y,dim,root.id)
                    if gda > epsilon:
                        epsilon =gda
                        mx_dim=dim
            if mx_dim == -1:
                root.is_leaf = True
                root.label = self.get_max_y(root.Y)
            else:
                root.divide_dim = mx_dim
                root.is_leaf = False
                root.label = self.get_max_y(root.Y)
                xx = root.X
                yy = root.Y
                for i in range(len(self.maps[mx_dim])):
                    tmpx = xx[np.where(xx[:,mx_dim]==i,True,False)]
                    tmps = root.source[np.where(xx[:, mx_dim] == i, True, False)]
                    tmpx[:,mx_dim] =-1  #将使用过的特征标记为-1
                    tmpy = yy[np.where(xx[:,mx_dim]==i,True,False)]
                    child = TreeNode(tmpx,tmpy,tmps)
                    child.id = self.ID
                    self.ID = self.ID + 1
                    root.child_ids.append(child.id)
                    root.children[i] = child
                    if tmpx.shape[0] == 0: #如果为空集，子节点的类别和当前节点保持一致
                        child.is_leaf=True
                        child.label = root.label
                    else:
                        child.is_leaf=False
                        child.X = tmpx
                        child.Y = tmpy
                        self.build_tree(child)
        pass

    def check_all_y(self,Y):
        yy = Y - Y[0]
        if np.where(yy==0,0,1).sum()==0:
            return True
        else:
            return False
    def check_X_null(self,X):
        if np.where(X==-1,0,1).sum()==0:
            return True
        else:
            return False
    def get_max_y(self,Y): #选取最大类别
        mx = 0
        for k in range(len(self.label_map)):
            dk = np.where(Y == k, 1, 0).sum()
            if mx < dk:
                label = k
                mx = dk
        return label

    def display(self):
        mp = dict(zip(self.label_map.values(),self.label_map.keys()))
        inv_feature_map = dict(zip(self.feature_map.values(), self.feature_map.keys()))
        q= []
        q.append(self.root)
        while len(q)>0:
            root = q[0]
            c= q.pop(0)
            if root.is_leaf:
                print(root.id, mp[root.label],' is leaf')
                ss = []
                for idx in range(root.source.shape[0]):
                    s = root.source[idx]
                    sen = []
                    for i in range(s.shape[0]):
                       sen.append(self.inv_maps[i][s[i]])
                    ss.append(sen)
                print(ss)
            else :
                print(root.id, 'divide dim ', inv_feature_map[root.divide_dim], '*' * 20)
                print(root.child_ids)
                ss = []
                for idx in range(root.source.shape[0]):
                    s = root.source[idx]
                    sen = []
                    for i in range(s.shape[0]):
                        sen.append(self.inv_maps[i][s[i]])
                    ss.append(sen)
                print(ss)
                for i in range(len(self.maps[root.divide_dim])):
                    q.append(root.children[i])
        pass

    def _predict(self,root,x):

        dim = root.divide_dim
        if root.is_leaf:
            return root.label

        if x[dim] not in self.maps[dim].keys():
            return self.root.label

        return self._predict(root.children[self.maps[dim][x[dim]]], x)

    def predict(self,x):
        pred_label = self._predict(self.root,x)
        for k,v in self.label_map.items():
            if v==pred_label:
                return k



# X = [[1,2,3,4,5,6],['青年','否','一般'],['青年','否','好'],['青年','是','好'],['青年','是','一般'],
#      ['青年','否','一般'],['中年','否','一般'],['中年','否','一般'],
#      ['中年','是','好'],['中年','否','一般']]
# y = ['否','否','是','是','否','否','否','是','是']

X = [['色泽','根蒂','敲声','纹理','脐部','触感'],
     ['青绿','蜷缩','沉闷','清晰','凹陷','硬滑'],
     ['浅白','蜷缩','浊响','清晰','凹陷','硬滑'],
     ['乌黑','稍蜷','浊响','清晰','稍凹','硬滑'],
     ['乌黑','稍蜷','沉闷','稍糊','稍凹','硬滑'],
     ['浅白','硬挺','清脆','模糊','平坦','硬滑'],
     ['浅白','蜷缩','浊响','模糊','平坦','软粘'],
     ['青绿','稍蜷','浊响','稍糊','凹陷','硬滑']]
y = ['是','是','是','否','否','否','否']
dt = DecisionTree(X,y)
dt.train()
dt.display()
# 新增加了决策树的预测函数
print(dt.predict(['青绿','稍蜷','浊响','稍糊','非常平坦','硬滑']))

 

 

 

import numpy as np
from math import log2 as log
class TreeNode(object):
    def __init__(self,X,Y,S):
        self.label = None
        self.divide_dim = None
        self.X = X
        self.Y = Y
        self.children = {}
        self.is_leaf = True
        self.source =S #为了display函数
        self.id = None #节点ID,确保唯一
        self.child_ids =[]

class DecisionTree(object):
    def __init__(self,X,Y,function='ID3'):
        self.X = X
        self.Y = Y
        self.F = function
        self.num_features = len(X[0]) #特征数
        self.feature_map = {} #特征映射为int值
        self.maps = {} #每个特征中对应的值的映射，例如第二个特征，有一个叫‘是’，那么maps[1]['是']=0/1
        self.label_map = {} #y 标签映射为数值
        self.root = None
        self.epsilon = 0
        self.inv_maps={}
        self.ID = 1

    def train(self):
        # train函数主要是处理数据，把所有特征转化为数值型，方便使用numpy处理
        X = self.X
        Y = self.Y
        # X row = Y row +1
        # 默认X的第一行为特征描述
        features = X[0]
        xx = np.zeros([len(X),self.num_features],dtype=int)
        yy = np.zeros(len(Y),dtype=int)
        for feature,j in zip(features,range(self.num_features)):
            self.feature_map[feature] = j
            self.maps[j] ={}
            self.inv_maps[j]={}
            for i in range(1,len(X)):
                k = X[i][j]
                if k in self.maps[j].keys():
                    xx[i][j] = self.maps[j][k]
                else:
                    self.maps[j][k] = len(self.maps[j])
                    xx[i][j] =self.maps[j][k]
                    self.inv_maps[j][xx[i][j]]=k

        for i in range(len(Y)):
            k= Y[i]
            if k in self.label_map.keys():
                yy[i] = self.label_map[k]
            else:
                self.label_map[k] = len(self.label_map)
                yy[i] = self.label_map[k]
        xx = xx[1:]
        X = X[1:]
        # xx , yy all int
        self.root = TreeNode(xx,yy,xx)
        self.root.id = self.ID
        self.ID = self.ID+1
        print(self.root.X)
        print(self.root.Y)
        self.build_tree(self.root)

    def get_information_gain(self,X,Y,dim,idx):
        # dim 表示特征索引
        nums = X.shape[0]
        cnt_class = len(self.label_map)
        cnt_x = len(self.maps[dim])
        xx = X[:,dim]
        yy = Y
        hd =0
        for v in range(cnt_class):
            nums_v = np.where(yy==v,1,0).sum()
            if nums_v >0 :
                hd = hd - nums_v*log(nums_v/nums)/nums
        hda = 0
        for i in range(cnt_x):
            di = np.where(xx==i,1,0).sum()
            if di>0:
                tmpy=yy[np.where(xx==i,True,False)]
                for k in range(cnt_class):
                    dik = np.where(tmpy== k,1,0).sum()
                    if dik >0:
                        hda = hda - (di/nums)*(dik/di)*log(dik/di)
        # 显示当前节点计算的指定维度的信息增益
        inv_feature_map = dict(zip(self.feature_map.values(), self.feature_map.keys()))
        print('当前节点序号: ', idx, ' 特征: ', inv_feature_map[dim])
        print('information gain', hd - hda)
        return hd-hda

    def build_tree(self,root):
        if self.check_all_y(root.Y): #所有实例属于同一类
            root.is_leaf=True
            root.label = root.Y[0]
        elif self.check_X_null(root.X): #特征为空，无法进一步选择划分,标记为-1的列即表示已经使用过
            root.is_leaf=True
            root.label = self.get_max_y(root.Y)
        else:
            epsilon = self.epsilon #epsilon
            mx_dim = -1 #选取信息增益最大的作为特征维度
            for dim in self.maps.keys():
                if root.X[0,dim] != -1:
                    gda = self.get_information_gain(root.X,root.Y,dim,root.id)
                    if gda > epsilon:
                        epsilon =gda
                        mx_dim=dim
            if mx_dim == -1:
                root.is_leaf = True
                root.label = self.get_max_y(root.Y)
            else:
                root.divide_dim = mx_dim
                root.is_leaf = False
                root.label = self.get_max_y(root.Y)
                xx = root.X
                yy = root.Y
                for i in range(len(self.maps[mx_dim])):
                    tmpx = xx[np.where(xx[:,mx_dim]==i,True,False)]
                    tmps = root.source[np.where(xx[:, mx_dim] == i, True, False)]
                    tmpx[:,mx_dim] =-1  #将使用过的特征标记为-1
                    tmpy = yy[np.where(xx[:,mx_dim]==i,True,False)]
                    child = TreeNode(tmpx,tmpy,tmps)
                    child.id = self.ID
                    self.ID = self.ID + 1
                    root.child_ids.append(child.id)
                    root.children[i] = child
                    if tmpx.shape[0] == 0: #如果为空集，子节点的类别和当前节点保持一致
                        child.is_leaf=True
                        child.label = root.label
                    else:
                        child.is_leaf=False
                        child.X = tmpx
                        child.Y = tmpy
                        self.build_tree(child)
        pass
    def check_all_y(self,Y):
        yy = Y - Y[0]
        if np.where(yy==0,0,1).sum()==0:
            return True
        else:
            return False
    def check_X_null(self,X):
        if np.where(X==-1,0,1).sum()==0:
            return True
        else:
            return False
    def get_max_y(self,Y): #选取最大类别
        mx = 0
        for k in range(len(self.label_map)):
            dk = np.where(Y == k, 1, 0).sum()
            if mx < dk:
                label = k
                mx = dk
        return label

    def display(self):
        mp = dict(zip(self.label_map.values(),self.label_map.keys()))
        inv_feature_map = dict(zip(self.feature_map.values(), self.feature_map.keys()))
        q= []
        q.append(self.root)
        while len(q)>0:
            root = q[0]
            c= q.pop(0)
            if root.is_leaf:
                print(root.id, mp[root.label],' is leaf')
                ss = []
                for idx in range(root.source.shape[0]):
                    s = root.source[idx]
                    sen = []
                    for i in range(s.shape[0]):
                       sen.append(self.inv_maps[i][s[i]])
                    ss.append(sen)
                print(ss)
            else :
                print(root.id, 'divide dim ', inv_feature_map[root.divide_dim], '*' * 20)
                print(root.child_ids)
                ss = []
                for idx in range(root.source.shape[0]):
                    s = root.source[idx]
                    sen = []
                    for i in range(s.shape[0]):
                        sen.append(self.inv_maps[i][s[i]])
                    ss.append(sen)
                print(ss)
                for i in range(len(self.maps[root.divide_dim])):
                    q.append(root.children[i])
        pass



X = [['色泽','根蒂','敲声','纹理','脐部','触感'],
     ['青绿','蜷缩','沉闷','清晰','凹陷','硬滑'],
     ['浅白','蜷缩','浊响','清晰','凹陷','硬滑'],
     ['乌黑','稍蜷','浊响','清晰','稍凹','硬滑'],
     ['乌黑','稍蜷','沉闷','稍糊','稍凹','硬滑'],
     ['浅白','硬挺','清脆','模糊','平坦','硬滑'],
     ['浅白','蜷缩','浊响','模糊','平坦','软粘'],
     ['青绿','稍蜷','浊响','稍糊','凹陷','硬滑']]
y = ['是','是','是','否','否','否','否']
dt = DecisionTree(X,y)
dt.train()
dt.display()

 2、C4.5算法：

按信息增益比来选择特征：

缺点：偏向于选择取值数目较少的特征，因为HA(D)其实可以认为把特征A的取值看做一个随机变量。要让信息增益比尽可能大，会倾向选择特征取值的不确定度尽可能低的特征。比如特征A只有两个取值，特征B有三种取值，假设取值是均匀分布，那么经过计算HA(D)  < HB(D)，说明模型倾向选择不确定度小的特征，假设是取值是均匀分布，那么极有可能是选择取值数目较少的特征。

算法流程：

 C4.5 [Quinlan, 1993]使用了一个启发式方法：先从候选划分属性中找出信息增益高于平均水平的属性，再从中选取信息增益比最高的。
本人代码实现并没有用启发式的方法。C4.5的代码和ID3差不多。

import numpy as np
from math import log2 as log
class TreeNode(object):
    def __init__(self,X,Y,S):
        self.label = None
        self.divide_dim = None
        self.X = X
        self.Y = Y
        self.children = {}
        self.is_leaf = True
        self.source =S #为了display函数
        self.id = None #节点ID,确保唯一
        self.child_ids =[]

class DecisionTree(object):
    def __init__(self,X,Y,function='C4.5'):
        self.X = X
        self.Y = Y
        self.F = function
        self.num_features = len(X[0]) #特征数
        self.feature_map = {} #特征映射为int值
        self.maps = {} #每个特征中对应的值的映射，例如第二个特征，有一个叫‘是’，那么maps[1]['是']=0/1
        self.label_map = {} #y 标签映射为数值
        self.root = None
        self.epsilon = 0
        self.inv_maps={}
        self.ID = 1

    def train(self):
        # train函数主要是处理数据，把所有特征转化为数值型，方便使用numpy处理
        X = self.X
        Y = self.Y
        # X row = Y row +1
        # 默认X的第一行为特征描述
        features = X[0]
        xx = np.zeros([len(X),self.num_features],dtype=int)
        yy = np.zeros(len(Y),dtype=int)
        for feature,j in zip(features,range(self.num_features)):
            self.feature_map[feature] = j
            self.maps[j] ={}
            self.inv_maps[j]={}
            for i in range(1,len(X)):
                k = X[i][j]
                if k in self.maps[j].keys():
                    xx[i][j] = self.maps[j][k]
                else:
                    self.maps[j][k] = len(self.maps[j])
                    xx[i][j] =self.maps[j][k]
                    self.inv_maps[j][xx[i][j]]=k

        for i in range(len(Y)):
            k= Y[i]
            if k in self.label_map.keys():
                yy[i] = self.label_map[k]
            else:
                self.label_map[k] = len(self.label_map)
                yy[i] = self.label_map[k]
        xx = xx[1:]
        X = X[1:]
        # xx , yy all int
        self.root = TreeNode(xx,yy,xx)
        self.root.id = self.ID
        self.ID = self.ID+1
        print(self.root.X)
        print(self.root.Y)
        self.build_tree(self.root)

    def get_information_gain_rate(self,X,Y,dim,idx):
        # dim 表示特征索引
        nums = X.shape[0]
        cnt_class = len(self.label_map)
        cnt_x = len(self.maps[dim])
        xx = X[:,dim]
        yy = Y
        hd =0
        for v in range(cnt_class):
            nums_v = np.where(yy==v,1,0).sum()
            if nums_v >0 :
                hd = hd - nums_v*log(nums_v/nums)/nums
        hda = 0
        had = 0
        for i in range(cnt_x):
            di = np.where(xx==i,1,0).sum()
            if di>0:
                had = had - (di/nums)*log(di/nums)
                tmpy=yy[np.where(xx==i,True,False)]
                for k in range(cnt_class):
                    dik = np.where(tmpy== k,1,0).sum()
                    if dik >0:
                        hda = hda - (di/nums)*(dik/di)*log(dik/di)
        ## 显示计算的指定维度的信息增益和信息增益比
        inv_feature_map = dict(zip(self.feature_map.values(), self.feature_map.keys()))
        print('当前节点序号: ',idx,' 特征: ',inv_feature_map[dim])
        print('information gain',hd-hda)
        if had == 0:
            print('igr: ',(hd-hda)/1e-9)
            return (hd-hda)/1e-9
        else :
            print('igr: ', (hd - hda) / had)
            return (hd-hda)/had

    def build_tree(self,root):
        if self.check_all_y(root.Y): #所有实例属于同一类
            root.is_leaf=True
            root.label = root.Y[0]
        elif self.check_X_null(root.X): #特征为空，无法进一步选择划分,标记为-1的列即表示已经使用过
            root.is_leaf=True
            root.label = self.get_max_y(root.Y)
        else:
            epsilon = self.epsilon #epsilon
            mx_dim = -1 #选取信息增益比最大的作为特征维度
            for dim in self.maps.keys():
                if root.X[0,dim] != -1:
                    gda = self.get_information_gain_rate(root.X,root.Y,dim,root.id)
                    if gda > epsilon:
                        epsilon =gda
                        mx_dim=dim
            if mx_dim == -1:
                root.is_leaf = True
                root.label = self.get_max_y(root.Y)
            else:
                root.divide_dim = mx_dim
                root.is_leaf = False
                root.label = self.get_max_y(root.Y)
                xx = root.X
                yy = root.Y
                for i in range(len(self.maps[mx_dim])):
                    tmpx = xx[np.where(xx[:,mx_dim]==i,True,False)]
                    tmps = root.source[np.where(xx[:, mx_dim] == i, True, False)]
                    tmpx[:,mx_dim] =-1  #将使用过的特征标记为-1
                    tmpy = yy[np.where(xx[:,mx_dim]==i,True,False)]
                    child = TreeNode(tmpx,tmpy,tmps)
                    child.id = self.ID
                    self.ID = self.ID + 1
                    root.child_ids.append(child.id)
                    root.children[i] = child
                    if tmpx.shape[0] == 0: #如果为空集，子节点的类别和当前节点保持一致
                        child.is_leaf=True
                        child.label = root.label
                    else:
                        child.is_leaf=False
                        child.X = tmpx
                        child.Y = tmpy
                        self.build_tree(child)
        pass
    def check_all_y(self,Y):
        yy = Y - Y[0]
        if np.where(yy==0,0,1).sum()==0:
            return True
        else:
            return False
    def check_X_null(self,X):
        if np.where(X==-1,0,1).sum()==0:
            return True
        else:
            return False
    def get_max_y(self,Y): #选取最大类别
        mx = 0
        for k in range(len(self.label_map)):
            dk = np.where(Y == k, 1, 0).sum()
            if mx < dk:
                label = k
                mx = dk
        return label

    def display(self):
        mp = dict(zip(self.label_map.values(),self.label_map.keys()))
        inv_feature_map = dict(zip(self.feature_map.values(), self.feature_map.keys()))
        q= []
        q.append(self.root)
        while len(q)>0:
            root = q[0]
            c= q.pop(0)
            if root.is_leaf:
                print(root.id, mp[root.label],' is leaf')
                ss = []
                for idx in range(root.source.shape[0]):
                    s = root.source[idx]
                    sen = []
                    for i in range(s.shape[0]):
                       sen.append(self.inv_maps[i][s[i]])
                    ss.append(sen)
                print(ss)
            else :
                print(root.id,'divide dim ',inv_feature_map[root.divide_dim],'*'*20)
                print(root.child_ids)
                ss = []
                for idx in range(root.source.shape[0]):
                    s = root.source[idx]
                    sen = []
                    for i in range(s.shape[0]):
                        sen.append(self.inv_maps[i][s[i]])
                    ss.append(sen)
                print(ss)
                for i in range(len(self.maps[root.divide_dim])):
                    q.append(root.children[i])
        pass


X = [['色泽','根蒂','敲声','纹理','脐部','触感'],
     ['青绿','蜷缩','沉闷','清晰','凹陷','硬滑'],
     ['浅白','蜷缩','浊响','清晰','凹陷','硬滑'],
     ['乌黑','稍蜷','浊响','清晰','稍凹','硬滑'],
     ['乌黑','稍蜷','沉闷','稍糊','稍凹','硬滑'],
     ['浅白','硬挺','清脆','模糊','平坦','硬滑'],
     ['浅白','蜷缩','浊响','模糊','平坦','软粘'],
     ['青绿','稍蜷','浊响','稍糊','凹陷','硬滑']]
y = ['是','是','是','否','否','否','否']
dt = DecisionTree(X,y)
dt.train()
dt.display()

 3、CART(Classification And Regression Tree)算法：

本文仅做了分类树的实现。

CART算法得到的决策树是一棵二叉树。

特征选择使用的是基尼指数（Gini index）最小化原则。

      

 

 CART分类树的算法流程：

 

 算法的终止条件是结点中的样本数少于某个阈值，或者样本集的基尼指数小于预定值(样本基本属于同一类)，或者没有更多特征。

本文没有考虑第一个终止特征，使用了和ID3、C4.5一样的终止条件。

关于决策树的剪枝，笔者时间不充裕，有空再补一篇。

代码如下：

import numpy as np
from math import log2 as log
class TreeNode(object):
    def __init__(self,X,Y,S):
        self.label = None
        self.divide_dim = None
        self.X = X
        self.Y = Y
        self.children = {}
        self.is_leaf = True
        self.source =S #为了display函数，显示X设为-1的值
        self.id = None #节点ID,确保唯一
        self.child_ids =[]

class DecisionTree(object):
    def __init__(self,X,Y,function='CART'):
        self.X = X
        self.Y = Y
        self.F = function
        self.num_features = len(X[0]) #特征数
        self.feature_map = {} #特征映射为int值
        self.maps = {} #每个特征中对应的值的映射，例如第二个特征，有一个叫‘是’，那么maps[1]['是']=0/1
        self.label_map = {} #y 标签映射为数值
        self.root = None
        self.inv_maps={}
        self.ID = 1

    def train(self):
        # train函数主要是处理数据，把所有特征转化为数值型，方便使用numpy处理
        X = self.X
        Y = self.Y
        # X row = Y row +1
        # 默认X的第一行为特征描述
        features = X[0]
        xx = np.zeros([len(X),self.num_features],dtype=int)
        yy = np.zeros(len(Y),dtype=int)
        for feature,j in zip(features,range(self.num_features)):
            self.feature_map[feature] = j
            self.maps[j] ={}
            self.inv_maps[j]={}
            for i in range(1,len(X)):
                k = X[i][j]
                if k in self.maps[j].keys():
                    xx[i][j] = self.maps[j][k]
                else:
                    self.maps[j][k] = len(self.maps[j])
                    xx[i][j] =self.maps[j][k]
                    self.inv_maps[j][xx[i][j]]=k

        for i in range(len(Y)):
            k= Y[i]
            if k in self.label_map.keys():
                yy[i] = self.label_map[k]
            else:
                self.label_map[k] = len(self.label_map)
                yy[i] = self.label_map[k]
        xx = xx[1:]
        # xx , yy all int
        self.root = TreeNode(xx,yy,xx)
        self.root.id = self.ID
        self.ID = self.ID+1
        print(self.root.X)
        print(self.root.Y)
        self.build_tree(self.root)

    def get_mini_Gini_split(self,X,Y,dim,idx):
        #   获取当前维度最小基尼值的划分值
        # dim 表示特征索引
        nums = X.shape[0]
        cnt_class = len(self.label_map)
        cnt_x = len(self.maps[dim])
        xx = X[:,dim]
        yy = Y
        split_value = -1
        mini_gini = 1.1 #需要比最大值大一点
        for i in range(cnt_x):
            num_equal= np.where(xx==i,1,0).sum()
            num_diff = nums - num_equal
            if num_diff==0 or num_equal ==0:
                continue
            equal_y = yy[np.where(xx==i,True,False)]
            diff_y = yy[np.where(xx==i,False,True)]
            equal_gini = 1
            diff_gini = 1
            for k in range(cnt_class):
                equal_gini = equal_gini -((np.where(equal_y== k,1,0).sum())/num_equal)**2
                diff_gini = diff_gini - ((np.where(diff_y==k,1,0).sum())/num_diff)**2
            gini = num_equal*equal_gini/nums + num_diff*diff_gini/nums
            if gini < mini_gini:
                mini_gini=gini
                split_value = i
        #########
        inv_feature_map = dict(zip(self.feature_map.values(), self.feature_map.keys()))
        print('当前节点序号: ', idx, ' 特征: ', inv_feature_map[dim])
        if split_value==-1: #说明无法找到一个分割点将样本分为两份
            print('当前特征无法切分')
            return split_value,mini_gini
        print(mini_gini,split_value)
        print('基尼值: ',mini_gini,' 分割值: ',self.inv_maps[dim][split_value])

        return split_value,mini_gini

    def build_tree(self,root):
        if self.check_all_y(root.Y): #所有实例属于同一类
            root.is_leaf=True
            root.label = root.Y[0]
        elif self.check_X_null(root.X): #特征为空，无法进一步选择划分,标记为-1的列即表示已经使用过
            root.is_leaf=True
            root.label = self.get_max_y(root.Y)
        else:
            gini = 1.1 #需要比最大值大一点
            mx_dim = -1
            split_value = -1
            for dim in self.maps.keys():
                if root.X[0,dim] != -1:
                    split, mini_gini = self.get_mini_Gini_split(root.X,root.Y,dim,root.id)
                    #print(mx_dim,split_value,mini_gini)
                    if mini_gini < gini:
                        gini = mini_gini
                        mx_dim = dim
                        split_value = split
            if mx_dim == -1:
                root.is_leaf = True
                root.label = self.get_max_y(root.Y)
            else:
                root.divide_dim = mx_dim
                root.is_leaf = False
                root.label = self.get_max_y(root.Y)
                xx = root.X
                yy = root.Y
                flag = True #用来选择样本
                for i in range(2):
                    tmpx = xx[np.where(xx[:,mx_dim]==split_value,flag,not flag)]
                    tmps = root.source[np.where(xx[:, mx_dim] == split_value, flag,not flag)]
                    tmpx[:,mx_dim] =-1  #将使用过的特征标记为-1
                    tmpy = yy[np.where(xx[:,mx_dim]==split_value,flag,not flag)]
                    flag=False #转换一下
                    child = TreeNode(tmpx,tmpy,tmps)
                    child.id = self.ID
                    self.ID = self.ID + 1
                    root.child_ids.append(child.id)
                    root.children[i] = child
                    if tmpx.shape[0] == 0: #如果为空集，子节点的类别和当前节点保持一致
                        child.is_leaf=True
                        child.label = root.label
                    else:
                        child.is_leaf=False
                        child.X = tmpx
                        child.Y = tmpy
                        self.build_tree(child)
            pass
    def check_all_y(self,Y):
        yy = Y - Y[0]
        if np.where(yy==0,0,1).sum()==0:
            return True
        else:
            return False
    def check_X_null(self,X):
        if np.where(X==-1,0,1).sum()==0:
            return True
        else:
            return False
    def get_max_y(self,Y): #选取最大类别
        mx = 0
        for k in range(len(self.label_map)):
            dk = np.where(Y == k, 1, 0).sum()
            if mx < dk:
                label = k
                mx = dk
        return label

    def display(self):
        mp = dict(zip(self.label_map.values(),self.label_map.keys()))
        inv_feature_map = dict(zip(self.feature_map.values(), self.feature_map.keys()))
        q= []
        q.append(self.root)
        while len(q)>0:
            root = q[0]
            c= q.pop(0)
            if root.is_leaf:
                print(root.id, mp[root.label],' is leaf')
                ss = []
                for idx in range(root.source.shape[0]):
                    s = root.source[idx]
                    sen = []
                    for i in range(s.shape[0]):
                       sen.append(self.inv_maps[i][s[i]])
                    ss.append(sen)
                print(ss)
            else :
                print(root.id,'divide dim ',inv_feature_map[root.divide_dim],'*'*20)
                print(root.child_ids)
                ss = []
                for idx in range(root.source.shape[0]):
                    s = root.source[idx]
                    sen = []
                    for i in range(s.shape[0]):
                        sen.append(self.inv_maps[i][s[i]])
                    ss.append(sen)
                print(ss)
                for i in range(len(root.children)):
                    q.append(root.children[i])
        pass


X = [['色泽','根蒂','敲声','纹理','脐部','触感'],
     ['青绿','蜷缩','沉闷','清晰','凹陷','硬滑'],
     ['浅白','蜷缩','浊响','清晰','凹陷','硬滑'],
     ['乌黑','稍蜷','浊响','清晰','稍凹','硬滑'],
     ['乌黑','稍蜷','沉闷','稍糊','稍凹','硬滑'],
     ['浅白','硬挺','清脆','模糊','平坦','硬滑'],
     ['浅白','蜷缩','浊响','模糊','平坦','软粘'],
     ['青绿','稍蜷','浊响','稍糊','凹陷','硬滑']]
y = ['是','是','是','否','否','否','否']
dt = DecisionTree(X,y)
dt.train()
dt.display()

 参考文献：

1、周志华——《机器学习》

2、李航——《统计学习方法》(第二版)