ID3 decision tree algorithm implementation (Python version), id3python

Source: Internet
Author: User

ID3 decision tree algorithm implementation (Python version), id3python

1 #-*-coding: UTF-8-*-2 3 from numpy import * 4 import numpy as np 5 import pandas as pd 6 from math import log 7 import operator 8 9 # Calculate the Shannon entropy 10 def calcShannonEnt (dataSet): 11 numEntries = len (dataSet) 12 labelCounts = {} 13 # create a dictionary for all possible classes 14 for featVec in dataSet: 15 currentLabel = featVec [-1] 16 if currentLabel not in labelCounts. keys (): 17 labelCounts [currentLabel] = 0 18 labelCounts [currentLabel] + = 1 19 shannonEnt = 0.0 20 # Calculate Shannon entropy 21 for key in labelCounts: 22 prob = float (labelCounts [key]) based on 2 /numEntries 23 shannonEnt-= prob * log (prob, 2) 24 return shannonEnt 25 26 27 # divide data sets for discrete variables, retrieve all samples whose feature value is value 28 def splitDataSet (dataSet, axis, value): 29 retDataSet = [] 30 for featVec in dataSet: 31 if featVec [axis] = value: 32 reducedFeatVec = featVec [: axis] 33 reducedFeatVec. extend (featVec [axis + 1:]) 34 retDat ASet. append (reducedFeatVec) 35 return retDataSet 36 37 # classify a dataset with continuous variables, which is the direction specified by ction, 38 # determine whether to divide a data sample smaller than value or a data sample set greater than value 39 def splitContinuousDataSet (dataSet, axis, value, direction ): 40 retDataSet = [] 41 for featVec in dataSet: 42 if direction = 0: 43 if featVec [axis]> value: 44 reducedFeatVec = featVec [: axis] 45 reducedFeatVec. extend (featVec [axis + 1:]) 46 retDataSet. append (reducedFeatVec) 47 else: 48 If featVec [axis] <= value: 49 reducedFeatVec = featVec [: axis] 50 reducedFeatVec. extend (featVec [axis + 1:]) 51 retDataSet. append (reducedFeatVec) 52 return retDataSet 53 54 # select the best dataSet Partitioning Method 55 def chooseBestFeatureToSplit (dataSet, labels): 56 numFeatures = len (dataSet [0]) -1 57 baseEntropy = calcShannonEnt (dataSet) 58 bestInfoGain = 0.0 59 bestFeature =-1 60 bestSplitDict = {} 61 for I in range (numFeatures): 62 featLis T = [example [I] for example in dataSet] 63 # process continuous features 64 if type (featList [0]). _ name __= = 'float' or type (featList [0]). _ name __= = 'int': 65 # n-1 candidate division Points 66 sortfeatList = sorted (featList) 67 splitList = [] 68 for j in range (len (sortfeatList) -1): 69 splitList. append (sortfeatList [j] + sortfeatList [j + 1])/2.0) 70 71 bestSplitEntropy = 10000 72 slen = len (splitList) 73 # Calculate the information entropy obtained by dividing points by j candidates, and record the optimal division Point 74 for j in ran Ge (slen): 75 value = splitList [j] 76 newEntropy = 0.0 77 subDataSet0 = splitContinuousDataSet (dataSet, I, value, 0) 78 subDataSet1 = splitContinuousDataSet (dataSet, I, value, 1) 79 prob0 = len (subDataSet0)/float (len (dataSet) 80 newEntropy + = prob0 * calcShannonEnt (subDataSet0) 81 prob1 = len (subDataSet1) /float (len (dataSet) 82 newEntropy + = prob1 * calcShannonEnt (subDataSet1) 83 if newEntropy <bestSplitEntropy: 84 bestSpli TEntropy = newEntropy 85 bestSplit = j 86 # use a dictionary to record the optimal feature division 87 bestSplitDict [labels [I] = splitList [bestSplit] 88 infoGain = baseEntropy-bestSplitEntropy 89 # processing of discrete features 90 else: 91 uniqueVals = set (featList) 92 newEntropy = 0.0 93 # Calculate the information entropy of each classification of the feature 94 for value in uniqueVals: 95 subDataSet = splitDataSet (dataSet, I, value) 96 prob = len (subDataSet)/float (len (dataSet) 97 newEntropy + = prob * calcShannonEnt (subDataSet) 98 InfoGain = baseEntropy-newEntropy 99 if infoGain> bestInfoGain: 100 bestInfoGain = infoGain101 bestFeature = i102 # if the optimal feature of the current node is continuous, perform binarization based on the previously recorded partition points 103 # whether it is less than or equal to bestSplitValue104 if type (dataSet [0] [bestFeature]). _ name __= = 'float' or type (dataSet [0] [bestFeature]). _ name __= = 'int': 105 bestSplitValue = bestSplitDict [labels [bestFeature] 106 labels [bestFeature] = labels [bestFeature] + '<=' + str (bestSplitValue) 107 For I in range (shape (dataSet) [0]): 108 if dataSet [I] [bestFeature] <= bestSplitValue: 109 dataSet [I] [bestFeature] = 1110 else: 111 dataSet [I] [bestFeature] = 0112 return bestFeature113 114 # If the feature has been divided and the samples under the node have not yet taken the same value, you need to vote for 115 def majorityCnt (classList ): 116 classCount = {} 117 for vote in classList: 118 if vote not in classCount. keys (): 119 classCount [vote] = 0120 classCount [vote] + = 1121 return max (classCount) 122 123 # Main program, recursively generate decision tree 124 def createTree (dataSet, labels, data_full, labels_full): 125 classList = [example [-1] for example in dataSet] 126 if classList. count (classList [0]) = len (classList): 127 return classList [0] 128 if len (dataSet [0]) = 1:129 return majorityCnt (classList) 130 bestFeat = chooseBestFeatureToSplit (dataSet, labels) 131 bestFeatLabel = labels [bestFeat] 132 myTree = {bestFeatLabel :{}} 133 featValues = [example [bestFe At] for example in dataSet] 134 uniqueVals = set (featValues) 135 if type (dataSet [0] [bestFeat]). _ name __= = 'str': 136 currentlabel = labels_full.index (labels [bestFeat]) 137 featValuesFull = [example [currentlabel] for example in data_full] 138 uniqueValsFull = set (featValuesFull) 139 del (labels [bestFeat]) 140 # create a subtree for each value of bestFeat. 141 for value in uniqueVals: 142 subLabels = labels [:] 143 if type (dataSet [0] [bestFeat]). _ name __= = 'str': 144 uniqueValsFull. remove (value) 145 myTree [bestFeatLabel] [value] = createTree (splitDataSet \ 146 (dataSet, bestFeat, value), subLabels, data_full, labels_full) 147 if type (dataSet [0] [bestFeat]). _ name __= = 'str': 148 for value in uniqueValsFull: 149 myTree [bestFeatLabel] [value] = majorityCnt (classList) 150 return myTree151 152 import matplotlib. pyplot as plt153 decisionNode = dict (boxstyle = "sawtooth", fc = "0.8") 154 leafNode = dict (boxstyle = "round4", fc = "0.8 ") 155 arrow_args = dict (arrowstyle = "<-") 156 157 158 # Number of leaf nodes in the computing tree 159 def getNumLeafs (myTree): 160 numLeafs = 0161 firstSides = list (myTree. keys () 162 firstStr = firstSides [0] 163 secondDict = myTree [firstStr] 164 for key in secondDict. keys (): 165 if type (secondDict [key]). _ name __= = 'dict': 166 numLeafs + = getNumLeafs (secondDict [key]) 167 else: numLeafs + = 1168 return numLeafs169 170 # maximum depth of the computing tree 171 def getTreeDepth (myTree): 172 maxDepth = 0173 firstSides = list (myTree. keys () 174 firstStr = firstSides [0] 175 secondDict = myTree [firstStr] 176 for key in secondDict. keys (): 177 if type (secondDict [key]). _ name __= = 'dict ': 178 thisDepth = 1 + getTreeDepth (secondDict [key]) 179 else: thisDepth = 1180 if thisDepth> maxDepth: 181 maxDepth = thisDepth182 return maxDepth183 184 # Draw node 185 def plotNode (nodeTxt, centerPt, parentPt, nodeType): 186 createPlot. ax1.annotate (nodeTxt, xy = parentPt, xycoords = 'axes fraction', \ 187 xytext = centerPt, textcoords = 'axes fraction', va = "center", ha = "center ", \ 188 bbox = nodeType, arrowprops = arrow_args) 189 190 # Draw the text on the arrow 191 def plotMidText (cntrPt, parentPt, txtString): 192 lens = len (txtString) 193 xMid = (parentPt [0] + cntrPt [0])/2.0-lens * 0.002194 yMid = (parentPt [1] + cntrPt [1])/2.0195 createPlot. ax1.text (xMid, yMid, txtString) 196 197 def plotTree (myTree, parentPt, nodeTxt): 198 numLeafs = getNumLeafs (myTree) 199 depth = getTreeDepth (myTree) 200 firstSides = list (myTree. keys () 201 firstStr = firstSides [0] 202 cntrPt = (plotTree. x0ff + (1.0 + float (numLeafs)/2.0/plotTree. totalW, plotTree. y0ff) 203 plotMidText (cntrPt, parentPt, nodeTxt) 204 plotNode (firstStr, cntrPt, parentPt, decisionNode) 205 secondDict = myTree [firstStr] 206 plotTree. y0ff = plotTree. y0ff-1.0/plotTree. totalD207 for key in secondDict. keys (): 208 if type (secondDict [key]). _ name __= = 'dict': 209 plotTree (secondDict [key], cntrPt, str (key) 210 else: 211 plotTree. x0ff = plotTree. x0ff + 1.0/plotTree. totalW212 plotNode (secondDict [key], (plotTree. x0ff, plotTree. y0ff), cntrPt, leafNode) 213 plotMidText (plotTree. x0ff, plotTree. y0ff), cntrPt, str (key) 214 plotTree. y0ff = plotTree. y0ff + 1.0/plotTree. totalD215 216 def createPlot (inTree): 217 fig = plt. figure (1, facecolor = 'white') 218 fig. clf () 219 axprops = dict (xticks = [], yticks = []) 220 createPlot. ax1 = plt. subplot (111, frameon = False, ** axprops) 221 plotTree. totalW = float (getNumLeafs (inTree) 222 plotTree. totalD = float (getTreeDepth (inTree) 223 plotTree. x0ff =-0.5/plotTree. totalW224 plotTree. y0ff = 1.0225 plotTree (inTree, (0.5, 1.0), '') 226 plt. show () 227 228 df1_pd.read_csv('watermelon_4_3.csv ') 229 data = df. values [:, 1:]. tolist () 230 data_full = data [:] 231 labels = df. columns. values [1:-1]. tolist () 232 labels_full = labels [:] 233 myTree = createTree (data, labels, data_full, labels_full) 234 print (myTree) 235 createPlot (myTree)
The final result is as follows: {'texture ': {'blur': 0, 'Little _ blur': {'touch': {'soft _ stick': 1, 'hard _ smooth ': 0 }}, 'distinct': {'density <= 0.38149999999999995 ': {0: 1, 1: 0 }}} the decision tree is as follows: reference: by Zhou Zhihua, machine learning practice

Contact Us

The content source of this page is from Internet, which doesn't represent Alibaba Cloud's opinion; products and services mentioned on that page don't have any relationship with Alibaba Cloud. If the content of the page makes you feel confusing, please write us an email, we will handle the problem within 5 days after receiving your email.

If you find any instances of plagiarism from the community, please send an email to: info-contact@alibabacloud.com and provide relevant evidence. A staff member will contact you within 5 working days.

A Free Trial That Lets You Build Big!

Start building with 50+ products and up to 12 months usage for Elastic Compute Service

  • Sales Support

    1 on 1 presale consultation

  • After-Sales Support

    24/7 Technical Support 6 Free Tickets per Quarter Faster Response

  • Alibaba Cloud offers highly flexible support services tailored to meet your exact needs.