ID3 decision tree algorithm implementation (Python version), id3python
1 #-*-coding: UTF-8-*-2 3 from numpy import * 4 import numpy as np 5 import pandas as pd 6 from math import log 7 import operator 8 9 # Calculate the Shannon entropy 10 def calcShannonEnt (dataSet): 11 numEntries = len (dataSet) 12 labelCounts = {} 13 # create a dictionary for all possible classes 14 for featVec in dataSet: 15 currentLabel = featVec [-1] 16 if currentLabel not in labelCounts. keys (): 17 labelCounts [currentLabel] = 0 18 labelCounts [currentLabel] + = 1 19 shannonEnt = 0.0 20 # Calculate Shannon entropy 21 for key in labelCounts: 22 prob = float (labelCounts [key]) based on 2 /numEntries 23 shannonEnt-= prob * log (prob, 2) 24 return shannonEnt 25 26 27 # divide data sets for discrete variables, retrieve all samples whose feature value is value 28 def splitDataSet (dataSet, axis, value): 29 retDataSet = [] 30 for featVec in dataSet: 31 if featVec [axis] = value: 32 reducedFeatVec = featVec [: axis] 33 reducedFeatVec. extend (featVec [axis + 1:]) 34 retDat ASet. append (reducedFeatVec) 35 return retDataSet 36 37 # classify a dataset with continuous variables, which is the direction specified by ction, 38 # determine whether to divide a data sample smaller than value or a data sample set greater than value 39 def splitContinuousDataSet (dataSet, axis, value, direction ): 40 retDataSet = [] 41 for featVec in dataSet: 42 if direction = 0: 43 if featVec [axis]> value: 44 reducedFeatVec = featVec [: axis] 45 reducedFeatVec. extend (featVec [axis + 1:]) 46 retDataSet. append (reducedFeatVec) 47 else: 48 If featVec [axis] <= value: 49 reducedFeatVec = featVec [: axis] 50 reducedFeatVec. extend (featVec [axis + 1:]) 51 retDataSet. append (reducedFeatVec) 52 return retDataSet 53 54 # select the best dataSet Partitioning Method 55 def chooseBestFeatureToSplit (dataSet, labels): 56 numFeatures = len (dataSet [0]) -1 57 baseEntropy = calcShannonEnt (dataSet) 58 bestInfoGain = 0.0 59 bestFeature =-1 60 bestSplitDict = {} 61 for I in range (numFeatures): 62 featLis T = [example [I] for example in dataSet] 63 # process continuous features 64 if type (featList [0]). _ name __= = 'float' or type (featList [0]). _ name __= = 'int': 65 # n-1 candidate division Points 66 sortfeatList = sorted (featList) 67 splitList = [] 68 for j in range (len (sortfeatList) -1): 69 splitList. append (sortfeatList [j] + sortfeatList [j + 1])/2.0) 70 71 bestSplitEntropy = 10000 72 slen = len (splitList) 73 # Calculate the information entropy obtained by dividing points by j candidates, and record the optimal division Point 74 for j in ran Ge (slen): 75 value = splitList [j] 76 newEntropy = 0.0 77 subDataSet0 = splitContinuousDataSet (dataSet, I, value, 0) 78 subDataSet1 = splitContinuousDataSet (dataSet, I, value, 1) 79 prob0 = len (subDataSet0)/float (len (dataSet) 80 newEntropy + = prob0 * calcShannonEnt (subDataSet0) 81 prob1 = len (subDataSet1) /float (len (dataSet) 82 newEntropy + = prob1 * calcShannonEnt (subDataSet1) 83 if newEntropy <bestSplitEntropy: 84 bestSpli TEntropy = newEntropy 85 bestSplit = j 86 # use a dictionary to record the optimal feature division 87 bestSplitDict [labels [I] = splitList [bestSplit] 88 infoGain = baseEntropy-bestSplitEntropy 89 # processing of discrete features 90 else: 91 uniqueVals = set (featList) 92 newEntropy = 0.0 93 # Calculate the information entropy of each classification of the feature 94 for value in uniqueVals: 95 subDataSet = splitDataSet (dataSet, I, value) 96 prob = len (subDataSet)/float (len (dataSet) 97 newEntropy + = prob * calcShannonEnt (subDataSet) 98 InfoGain = baseEntropy-newEntropy 99 if infoGain> bestInfoGain: 100 bestInfoGain = infoGain101 bestFeature = i102 # if the optimal feature of the current node is continuous, perform binarization based on the previously recorded partition points 103 # whether it is less than or equal to bestSplitValue104 if type (dataSet [0] [bestFeature]). _ name __= = 'float' or type (dataSet [0] [bestFeature]). _ name __= = 'int': 105 bestSplitValue = bestSplitDict [labels [bestFeature] 106 labels [bestFeature] = labels [bestFeature] + '<=' + str (bestSplitValue) 107 For I in range (shape (dataSet) [0]): 108 if dataSet [I] [bestFeature] <= bestSplitValue: 109 dataSet [I] [bestFeature] = 1110 else: 111 dataSet [I] [bestFeature] = 0112 return bestFeature113 114 # If the feature has been divided and the samples under the node have not yet taken the same value, you need to vote for 115 def majorityCnt (classList ): 116 classCount = {} 117 for vote in classList: 118 if vote not in classCount. keys (): 119 classCount [vote] = 0120 classCount [vote] + = 1121 return max (classCount) 122 123 # Main program, recursively generate decision tree 124 def createTree (dataSet, labels, data_full, labels_full): 125 classList = [example [-1] for example in dataSet] 126 if classList. count (classList [0]) = len (classList): 127 return classList [0] 128 if len (dataSet [0]) = 1:129 return majorityCnt (classList) 130 bestFeat = chooseBestFeatureToSplit (dataSet, labels) 131 bestFeatLabel = labels [bestFeat] 132 myTree = {bestFeatLabel :{}} 133 featValues = [example [bestFe At] for example in dataSet] 134 uniqueVals = set (featValues) 135 if type (dataSet [0] [bestFeat]). _ name __= = 'str': 136 currentlabel = labels_full.index (labels [bestFeat]) 137 featValuesFull = [example [currentlabel] for example in data_full] 138 uniqueValsFull = set (featValuesFull) 139 del (labels [bestFeat]) 140 # create a subtree for each value of bestFeat. 141 for value in uniqueVals: 142 subLabels = labels [:] 143 if type (dataSet [0] [bestFeat]). _ name __= = 'str': 144 uniqueValsFull. remove (value) 145 myTree [bestFeatLabel] [value] = createTree (splitDataSet \ 146 (dataSet, bestFeat, value), subLabels, data_full, labels_full) 147 if type (dataSet [0] [bestFeat]). _ name __= = 'str': 148 for value in uniqueValsFull: 149 myTree [bestFeatLabel] [value] = majorityCnt (classList) 150 return myTree151 152 import matplotlib. pyplot as plt153 decisionNode = dict (boxstyle = "sawtooth", fc = "0.8") 154 leafNode = dict (boxstyle = "round4", fc = "0.8 ") 155 arrow_args = dict (arrowstyle = "<-") 156 157 158 # Number of leaf nodes in the computing tree 159 def getNumLeafs (myTree): 160 numLeafs = 0161 firstSides = list (myTree. keys () 162 firstStr = firstSides [0] 163 secondDict = myTree [firstStr] 164 for key in secondDict. keys (): 165 if type (secondDict [key]). _ name __= = 'dict': 166 numLeafs + = getNumLeafs (secondDict [key]) 167 else: numLeafs + = 1168 return numLeafs169 170 # maximum depth of the computing tree 171 def getTreeDepth (myTree): 172 maxDepth = 0173 firstSides = list (myTree. keys () 174 firstStr = firstSides [0] 175 secondDict = myTree [firstStr] 176 for key in secondDict. keys (): 177 if type (secondDict [key]). _ name __= = 'dict ': 178 thisDepth = 1 + getTreeDepth (secondDict [key]) 179 else: thisDepth = 1180 if thisDepth> maxDepth: 181 maxDepth = thisDepth182 return maxDepth183 184 # Draw node 185 def plotNode (nodeTxt, centerPt, parentPt, nodeType): 186 createPlot. ax1.annotate (nodeTxt, xy = parentPt, xycoords = 'axes fraction', \ 187 xytext = centerPt, textcoords = 'axes fraction', va = "center", ha = "center ", \ 188 bbox = nodeType, arrowprops = arrow_args) 189 190 # Draw the text on the arrow 191 def plotMidText (cntrPt, parentPt, txtString): 192 lens = len (txtString) 193 xMid = (parentPt [0] + cntrPt [0])/2.0-lens * 0.002194 yMid = (parentPt [1] + cntrPt [1])/2.0195 createPlot. ax1.text (xMid, yMid, txtString) 196 197 def plotTree (myTree, parentPt, nodeTxt): 198 numLeafs = getNumLeafs (myTree) 199 depth = getTreeDepth (myTree) 200 firstSides = list (myTree. keys () 201 firstStr = firstSides [0] 202 cntrPt = (plotTree. x0ff + (1.0 + float (numLeafs)/2.0/plotTree. totalW, plotTree. y0ff) 203 plotMidText (cntrPt, parentPt, nodeTxt) 204 plotNode (firstStr, cntrPt, parentPt, decisionNode) 205 secondDict = myTree [firstStr] 206 plotTree. y0ff = plotTree. y0ff-1.0/plotTree. totalD207 for key in secondDict. keys (): 208 if type (secondDict [key]). _ name __= = 'dict': 209 plotTree (secondDict [key], cntrPt, str (key) 210 else: 211 plotTree. x0ff = plotTree. x0ff + 1.0/plotTree. totalW212 plotNode (secondDict [key], (plotTree. x0ff, plotTree. y0ff), cntrPt, leafNode) 213 plotMidText (plotTree. x0ff, plotTree. y0ff), cntrPt, str (key) 214 plotTree. y0ff = plotTree. y0ff + 1.0/plotTree. totalD215 216 def createPlot (inTree): 217 fig = plt. figure (1, facecolor = 'white') 218 fig. clf () 219 axprops = dict (xticks = [], yticks = []) 220 createPlot. ax1 = plt. subplot (111, frameon = False, ** axprops) 221 plotTree. totalW = float (getNumLeafs (inTree) 222 plotTree. totalD = float (getTreeDepth (inTree) 223 plotTree. x0ff =-0.5/plotTree. totalW224 plotTree. y0ff = 1.0225 plotTree (inTree, (0.5, 1.0), '') 226 plt. show () 227 228 df1_pd.read_csv('watermelon_4_3.csv ') 229 data = df. values [:, 1:]. tolist () 230 data_full = data [:] 231 labels = df. columns. values [1:-1]. tolist () 232 labels_full = labels [:] 233 myTree = createTree (data, labels, data_full, labels_full) 234 print (myTree) 235 createPlot (myTree)The final result is as follows: {'texture ': {'blur': 0, 'Little _ blur': {'touch': {'soft _ stick': 1, 'hard _ smooth ': 0 }}, 'distinct': {'density <= 0.38149999999999995 ': {0: 1, 1: 0 }}} the decision tree is as follows: reference: by Zhou Zhihua, machine learning practice