【机器学习】决策树

Decision Tree

熵 - entropy

数学表达式

\[H(p_1) = -p_1 \text{log}_2(p_1) - (1- p_1) \text{log}_2(1- p_1) \]

代码

# UNQ_C1
# GRADED FUNCTION: compute_entropy

def compute_entropy(y):
    """
    Computes the entropy for 
    
    Args:
       y (ndarray): Numpy array indicating whether each example at a node is
           edible (`1`) or poisonous (`0`)
       
    Returns:
        entropy (float): Entropy at that node
        
    """
    # You need to return the following variables correctly
    entropy = 0.
    
    ### START CODE HERE ###
    if len(y) != 0:
        # len(y[y == 1]) 统计y中等于1元素的个数 等价于 np.sum(y == 1)
        p1 = len(y[y == 1]) / len(y)
        if p1 != 0 and p1 != 1:
            entropy = -p1 * np.log2(p1) - (1 - p1) * np.log2(1 - p1)
    ### END CODE HERE ###        
    
    return entropy

信息增益 - information gain

数学表达式

\[\text{Information Gain} = H(p_1^\text{node})- (w^{\text{left}}H(p_1^\text{left}) + w^{\text{right}}H(p_1^\text{right})) \]

代码

# UNQ_C3
# GRADED FUNCTION: compute_information_gain

def compute_information_gain(X, y, node_indices, feature):
    
    """
    Compute the information of splitting the node on a given feature
    
    Args:
        X (ndarray):            Data matrix of shape(n_samples, n_features)
        y (array like):         list or ndarray with n_samples containing the target variable
        node_indices (ndarray): List containing the active indices. I.e, the samples being considered in this step.
   
    Returns:
        cost (float):        Cost computed
    
    """    
    # Split dataset
    left_indices, right_indices = split_dataset(X, node_indices, feature)
    
    # Some useful variables
    X_node, y_node = X[node_indices], y[node_indices]
    X_left, y_left = X[left_indices], y[left_indices]
    X_right, y_right = X[right_indices], y[right_indices]
    
    # You need to return the following variables correctly
    information_gain = 0
    
    ### START CODE HERE ###
    
    # Weights 
    node_entropy = compute_entropy(y_node)
    left_entropy = compute_entropy(y_left)
    right_entropy = compute_entropy(y_right)
    w_left = len(X_left) / len(X_node)
    w_right = len(X_right) / len(y_node)
    # Weighted entropy
    weighted_entropy = w_left * left_entropy + w_right * right_entropy
    # Information gain                                                   
    information_gain = node_entropy - weighted_entropy
    ### END CODE HERE ###  
    
    return information_gain

# UNQ_C4
# GRADED FUNCTION: get_best_split

def get_best_split(X, y, node_indices):   
    """
    Returns the optimal feature and threshold value
    to split the node data 
    
    Args:
        X (ndarray):            Data matrix of shape(n_samples, n_features)
        y (array like):         list or ndarray with n_samples containing the target variable
        node_indices (ndarray): List containing the active indices. I.e, the samples being considered in this step.

    Returns:
        best_feature (int):     The index of the best feature to split
    """    
    
    # Some useful variables
    num_features = X.shape[1]
    
    # You need to return the following variables correctly
    best_feature = -1
    max_info_gain = 0
    ### START CODE HERE ###
    for feature in range(num_features):
        info_gain = compute_information_gain(X, y, node_indices, feature)
        if info_gain > max_info_gain:
            max_info_gain = info_gain
            best_feature = feature
    ### END CODE HERE ##    
   
    return best_feature

# Not graded
tree = []

def build_tree_recursive(X, y, node_indices, branch_name, max_depth, current_depth):
    """
    Build a tree using the recursive algorithm that split the dataset into 2 subgroups at each node.
    This function just prints the tree.
    
    Args:
        X (ndarray):            Data matrix of shape(n_samples, n_features)
        y (array like):         list or ndarray with n_samples containing the target variable
        node_indices (ndarray): List containing the active indices. I.e, the samples being considered in this step.
        branch_name (string):   Name of the branch. ['Root', 'Left', 'Right']
        max_depth (int):        Max depth of the resulting tree. 
        current_depth (int):    Current depth. Parameter used during recursive call.
   
    """ 

    # Maximum depth reached - stop splitting
    if current_depth == max_depth:
        formatting = " "*current_depth + "-"*current_depth
        print(formatting, "%s leaf node with indices" % branch_name, node_indices)
        return
   
    # Otherwise, get best split and split the data
    # Get the best feature and threshold at this node
    best_feature = get_best_split(X, y, node_indices) 
    tree.append((current_depth, branch_name, best_feature, node_indices))
    
    formatting = "-"*current_depth
    print("%s Depth %d, %s: Split on feature: %d" % (formatting, current_depth, branch_name, best_feature))
    
    # Split the dataset at the best feature
    left_indices, right_indices = split_dataset(X, node_indices, best_feature)
    
    # continue splitting the left and the right child. Increment current depth
    build_tree_recursive(X, y, left_indices, "Left", max_depth, current_depth+1)
    build_tree_recursive(X, y, right_indices, "Right", max_depth, current_depth+1)

import numpy as np
from sklearn.datasets import load_iris


class DecisionTree():
    def __init__(self, X_train, y_train, branch_name, max_depth) -> None:
        self.X_train = X_train
        self.y_train = y_train
        self.k = self.X_train.shape[1] # k类特征
        self.branch_name = branch_name
        self.max_depth = max_depth

    def split_dataset(self, data, featrue, value):
        '''分割数据集'''
        left_data, right_data = [], []    
        for i in range(len(data)):
            if data[i, featrue] <= value:
                left_data.append(data[i, featrue])
            else:
                right_data.append(data[i, featrue])
        return left_data, right_data

    def compute_entropy(self, data):
        '''计算熵'''
        entropy = 0.
        count = np.zeros(3) # 统计3类鸢尾花在数据集中对应的数量
        for i in range(len(data)):
            count[self.y_train[i]] += 1
        for i in range(3):
            term = count[i] / len(data)
            if term != 0:
                entropy += (term * np.log2(term))
        return -entropy
    
    def compute_information_gain(self, data, feature, value):
        '''计算信息增益'''
        # feature: 0 1 2 3 中的某个
        left_data, right_data = self.split_dataset(data, feature, value)
        left_entropy = self.compute_entropy(left_data)
        right_entropy = self.compute_entropy(right_data)
        w_left = len(left_data) / len(data)
        w_right = len(right_data) / len(data)
        weighted_entropy = w_left * left_entropy + w_right * right_entropy                                             
        information_gain = self.compute_entropy(data) - weighted_entropy
        return information_gain

    def get_best_split(self, data):
        '''决策本次划分'''   
        best_feature = -1
        max_info_gain = -1
        # 遍历四类特征，取其中信息增益最大的作为本次划分依据
        for feature in range(self.k):
            values = [row[feature] for row in data]
            for value in values:
                info_gain = self.compute_information_gain(data, feature, value)
                if info_gain > max_info_gain:
                    max_info_gain = info_gain

        return best_feature, max_info_gain

    def build_tree_recursive(self, data, branch_name, current_depth):
        if current_depth == self.max_depth:
            formatting = " "*current_depth + "-"*current_depth
            print(f"{formatting} {branch_name} leaf node with indices")
            return
        best_feature, max_info_gain, best_value = self.get_best_split(data) 
        tree.append((current_depth, branch_name, best_feature))
        
        formatting = "-"*current_depth
        print(f"{formatting} Depth {current_depth}, {branch_name}: Split on feature: {best_feature}")
        
        left_data, right_data = self.split_dataset(data, best_feature, best_value)
        
        self.build_tree_recursive(left_data, "Left", current_depth+1)
        self.build_tree_recursive(right_data, "Right", current_depth+1)

def load_data():
    iris = load_iris()
    iris_feature = iris.data
    iris_target = iris.target
    #数据进行分割（训练数据和测试数据）
    # X_train, X_test = iris_feature[:120, :], iris_feature[120:, :]
    # y_train, y_test = iris_target[:120], iris_target[120:]
    return iris_feature, iris_target

if __name__ == '__main__':
    tree = []
    # model = DecisionTree(X_train, y_train, root_indices, "Root", max_depth=2)
    # model.build_tree_recursive(root_indices, "Root", current_depth=0) 
    X_train, y_train = load_data()
    model = DecisionTree(X_train, y_train, "Root", max_depth=3)