Implementation Based on ID3 decision tree algorithm (Python version), id3python
Example:
#-*-Coding: UTF-8-*-from numpy import * import numpy as npimport pandas as pdfrom math import logimport operator # Calculate the Shannon entropy def calcShannonEnt (dataSet) of the dataSet ): numEntries = len (dataSet) labelCounts ={}# create a dictionary for all possible classes in featVec in dataSet: currentLabel = featVec [-1] if currentLabel not in labelCounts. keys (): labelCounts [currentLabel] = 0 labelCounts [currentLabel] + = 1 shannonEnt = 0.0 # Calculate Shannon entropy for key in labelCount based on 2 S: prob = float (labelCounts [key])/numEntries shannonEnt-= prob * log (prob, 2) return shannonEnt # divide data sets for discrete variables, retrieve all samples whose feature value is value def splitDataSet (dataSet, axis, value): retDataSet = [] for featVec in dataSet: if featVec [axis] = value: reducedFeatVec = featVec [: axis] reducedFeatVec. extend (featVec [axis + 1:]) retDataSet. append (reducedFeatVec) return retDataSet # define the direction of the dataset for continuous variables. # determine whether to divide data samples smaller than value or greater than value. Def splitContinuousDataSet (dataSet, axis, value, direction): retDataSet = [] for featVec in dataSet: if direction = 0: if featVec [axis]> value: reducedFeatVec = featVec [: axis] reducedFeatVec. extend (featVec [axis + 1:]) retDataSet. append (reducedFeatVec) else: if featVec [axis] <= value: reducedFeatVec = featVec [: axis] reducedFeatVec. extend (featVec [axis + 1:]) retDataSet. append (reducedFeatVec) return retDataSet # Select the best dataSet Partitioning Method def chooseBestFeatureToSplit (dataSet, labels): numFeatures = len (dataSet [0])-1 baseEntropy = calcShannonEnt (dataSet) bestInfoGain = 0.0 bestFeature =-1 bestSplitDict ={}for I in range (numFeatures ): featList = [example [I] for example in dataSet] # process continuous features if type (featList [0]). _ name __= = 'float' or type (featList [0]). _ name __= = 'int': # generate n-1 candidate division points sortfeatList = sorted (featList) splitList = [] for j in Range (len (sortfeatList)-1): splitList. append (sortfeatList [j] + sortfeatList [j + 1])/2.0) bestSplitEntropy = 10000 slen = len (splitList) # Calculate the Division time using the j-th candidate, obtain the information entropy and record the optimal division point for j in range (slen): value = splitList [j] newEntropy = 0.0 subDataSet0 = splitContinuousDataSet (dataSet, I, value, 0) subDataSet1 = splitContinuousDataSet (dataSet, I, value, 1) prob0 = len (subDataSet0)/float (len (dataSet) newEntropy + = prob0 * calcShannonEnt (SubDataSet0) prob1 = len (subDataSet1)/float (len (dataSet) newEntropy + = prob1 * calcShannonEnt (subDataSet1) if newEntropy <bestSplitEntropy: bytes = newEntropy bestSplit = j # use a dictionary to record the best feature division points bestSplitDict [labels [I] = splitList [bestSplit] infoGain = baseEntropy-bestSplitEntropy # else for processing discrete features: uniqueVals = set (featList) newEntropy = 0.0 # Calculate the information entropy of each classification under this feature for value in uniqueVals: subDataSet = splitDataSet (d AtaSet, I, value) prob = len (subDataSet)/float (len (dataSet) newEntropy + = prob * calcShannonEnt (subDataSet) infoGain = baseEntropy-newEntropy if infoGain> bestInfoGain: bestInfoGain = infoGain bestFeature = I # If the optimal feature of the current node is continuous, perform binarization based on the previously recorded points # whether the value is less than or equal to bestSplitValue if type (dataSet [0] [bestFeature]). _ name __= = 'float' or type (dataSet [0] [bestFeature]). _ name __= = 'int': bestSplitValue = bestSplitDict [labels [best Feature] labels [bestFeature] = labels [bestFeature] + '<=' + str (bestSplitValue) for I in range (shape (dataSet) [0]): if dataSet [I] [bestFeature] <= bestSplitValue: dataSet [I] [bestFeature] = 1 else: dataSet [I] [bestFeature] = 0 return bestFeature # If the feature has been divided and the samples under the node have not yet taken the same value, you need to vote for def majorityCnt (classList ): classCount ={} for vote in classList: if vote not in classCount. keys (): classCount [vote] = 0 classCount [vote] + = 1 r Eturn max (classCount) # The main program recursively generates the decision tree def createTree (dataSet, labels, data_full, labels_full): classList = [example [-1] for example in dataSet] if classList. count (classList [0]) = len (classList): return classList [0] if len (dataSet [0]) = 1: return majorityCnt (classList) bestFeat = chooseBestFeatureToSplit (dataSet, labels) bestFeatLabel = labels [bestFeat] myTree = {bestFeatLabel :{}} featValues = [example [bestFeat] for ex Ample in dataSet] uniqueVals = set (featValues) if type (dataSet [0] [bestFeat]). _ name __= = 'str': currentlabel = labels_full.index (labels [bestFeat]) featValuesFull = [example [currentlabel] for example in data_full] uniqueValsFull = set (featValuesFull) del (labels [bestFeat]) # divides a subtree for each value of bestFeat. For value in uniqueVals: subLabels = labels [:] if type (dataSet [0] [bestFeat]). _ name __= = 'str': uniqueValsFull. remove (value) myTree [bestFeatLabel] [value] = createTree (splitDataSet \ (dataSet, bestFeat, value), subLabels, data_full, labels_full) if type (dataSet [0] [bestFeat]). _ name __= = 'str': for value in uniqueValsFull: myTree [bestFeatLabel] [value] = majorityCnt (classList) return myTreeimport matplotlib. pyplot as pltdecisionNode = dict (boxstyle = "sawtooth", fc = "0.8") leafNode = dict (boxstyle = "round4", fc = "0.8 ") arrow_args = dict (arrowstyle = "<-") # calculate the number of leaf nodes in the tree def getNumLeafs (myTree): numLeafs = 0 firstSides = list (myTree. keys () firstStr = firstSides [0] secondDict = myTree [firstStr] for key in secondDict. keys (): if type (secondDict [key]). _ name __= = 'dict ': numLeafs + = getNumLeafs (secondDict [key]) else: numLeafs + = 1 return numLeafs # maximum computing tree depth def getTreeDepth (myTree ): maxDepth = 0 firstSides = list (myTree. keys () firstStr = firstSides [0] secondDict = myTree [firstStr] for key in secondDict. keys (): if type (secondDict [key]). _ name __= = 'dict ': thisDepth = 1 + getTreeDepth (secondDict [key]) else: thisDepth = 1 if thisDepth> maxDepth: maxDepth = thisDepth return maxDepth # Draw node def plotNode (nodeTxt, centerPt, parentPt, nodeType): createPlot. ax1.annotate (nodeTxt, xy = parentPt, xycoords = 'axes fraction', \ xytext = centerPt, textcoords = 'axes fraction', va = "center", ha = "center ", \ bbox = nodeType, arrowprops = arrow_args) # Draw the arrow on the text def plotMidText (cntrPt, parentPt, txtString): lens = len (txtString) xMid = (parentPt [0] + cntrPt [0])/2.0-lens * 0.002 yMid = (parentPt [1] + cntrPt [1])/2.0 createPlot. ax1.text (xMid, yMid, txtString) def plotTree (myTree, parentPt, nodeTxt): numLeafs = getNumLeafs (myTree) depth = getTreeDepth (myTree) firstSides = list (myTree. keys () firstStr = firstSides [0] cntrPt = (plotTree. x0ff + (1.0 + float (numLeafs)/2.0/plotTree. totalW, plotTree. y0ff) plotMidText (cntrPt, parentPt, nodeTxt) plotNode (firstStr, cntrPt, parentPt, decisionNode) secondDict = myTree [firstStr] plotTree. y0ff = plotTree. y0ff-1.0/plotTree. totalD for key in secondDict. keys (): if type (secondDict [key]). _ name __= = 'dict ': plotTree (secondDict [key], cntrPt, str (key) else: plotTree. x0ff = plotTree. x0ff + 1.0/plotTree. totalW plotNode (secondDict [key], (plotTree. x0ff, plotTree. y0ff), cntrPt, leafNode) plotMidText (plotTree. x0ff, plotTree. y0ff), cntrPt, str (key) plotTree. y0ff = plotTree. y0ff + 1.0/plotTree. totalDdef createPlot (inTree): fig = plt. figure (1, facecolor = 'white') fig. clf () axprops = dict (xticks = [], yticks = []) createPlot. ax1 = plt. subplot (111, frameon = False, ** axprops) plotTree. totalW = float (getNumLeafs (inTree) plotTree. totalD = float (getTreeDepth (inTree) plotTree. x0ff =-0.5/plotTree. totalW plotTree. y0ff = 1.0 plotTree (inTree, (0.5, 1.0), '') plt.show(1_df1_pd.read_csv('watermelon_4_3.csv ') data = df. values [:, 1:]. tolist () data_full = data [:] labels = df. columns. values [1:-1]. tolist () labels_full = labels [:] myTree = createTree (data, labels, data_full, labels_full) print (myTree) createPlot (myTree)
The final result is as follows:
{'Texture ': {'blur': 0, 'Little _ blur': {'touch': {'soft _ stick': 1, 'hard _ smooth ': 0 }}, 'distinct ': {'density <= 000000': {0: 1, 1: 0 }}}}
The resulting decision tree is as follows:
References:
Machine learning practices
Machine Learning by Zhou Zhihua
The above implementation based on the ID3 decision tree algorithm (Python version) is all the content shared by the editor. I hope to give you a reference, and I hope you can provide more support for the customer's house.