Implementation Based on ID3 decision tree algorithm (Python version), id3python

Source: Internet
Author: User

Implementation Based on ID3 decision tree algorithm (Python version), id3python

Example:

#-*-Coding: UTF-8-*-from numpy import * import numpy as npimport pandas as pdfrom math import logimport operator # Calculate the Shannon entropy def calcShannonEnt (dataSet) of the dataSet ): numEntries = len (dataSet) labelCounts ={}# create a dictionary for all possible classes in featVec in dataSet: currentLabel = featVec [-1] if currentLabel not in labelCounts. keys (): labelCounts [currentLabel] = 0 labelCounts [currentLabel] + = 1 shannonEnt = 0.0 # Calculate Shannon entropy for key in labelCount based on 2 S: prob = float (labelCounts [key])/numEntries shannonEnt-= prob * log (prob, 2) return shannonEnt # divide data sets for discrete variables, retrieve all samples whose feature value is value def splitDataSet (dataSet, axis, value): retDataSet = [] for featVec in dataSet: if featVec [axis] = value: reducedFeatVec = featVec [: axis] reducedFeatVec. extend (featVec [axis + 1:]) retDataSet. append (reducedFeatVec) return retDataSet # define the direction of the dataset for continuous variables. # determine whether to divide data samples smaller than value or greater than value. Def splitContinuousDataSet (dataSet, axis, value, direction): retDataSet = [] for featVec in dataSet: if direction = 0: if featVec [axis]> value: reducedFeatVec = featVec [: axis] reducedFeatVec. extend (featVec [axis + 1:]) retDataSet. append (reducedFeatVec) else: if featVec [axis] <= value: reducedFeatVec = featVec [: axis] reducedFeatVec. extend (featVec [axis + 1:]) retDataSet. append (reducedFeatVec) return retDataSet # Select the best dataSet Partitioning Method def chooseBestFeatureToSplit (dataSet, labels): numFeatures = len (dataSet [0])-1 baseEntropy = calcShannonEnt (dataSet) bestInfoGain = 0.0 bestFeature =-1 bestSplitDict ={}for I in range (numFeatures ): featList = [example [I] for example in dataSet] # process continuous features if type (featList [0]). _ name __= = 'float' or type (featList [0]). _ name __= = 'int': # generate n-1 candidate division points sortfeatList = sorted (featList) splitList = [] for j in Range (len (sortfeatList)-1): splitList. append (sortfeatList [j] + sortfeatList [j + 1])/2.0) bestSplitEntropy = 10000 slen = len (splitList) # Calculate the Division time using the j-th candidate, obtain the information entropy and record the optimal division point for j in range (slen): value = splitList [j] newEntropy = 0.0 subDataSet0 = splitContinuousDataSet (dataSet, I, value, 0) subDataSet1 = splitContinuousDataSet (dataSet, I, value, 1) prob0 = len (subDataSet0)/float (len (dataSet) newEntropy + = prob0 * calcShannonEnt (SubDataSet0) prob1 = len (subDataSet1)/float (len (dataSet) newEntropy + = prob1 * calcShannonEnt (subDataSet1) if newEntropy <bestSplitEntropy: bytes = newEntropy bestSplit = j # use a dictionary to record the best feature division points bestSplitDict [labels [I] = splitList [bestSplit] infoGain = baseEntropy-bestSplitEntropy # else for processing discrete features: uniqueVals = set (featList) newEntropy = 0.0 # Calculate the information entropy of each classification under this feature for value in uniqueVals: subDataSet = splitDataSet (d AtaSet, I, value) prob = len (subDataSet)/float (len (dataSet) newEntropy + = prob * calcShannonEnt (subDataSet) infoGain = baseEntropy-newEntropy if infoGain> bestInfoGain: bestInfoGain = infoGain bestFeature = I # If the optimal feature of the current node is continuous, perform binarization based on the previously recorded points # whether the value is less than or equal to bestSplitValue if type (dataSet [0] [bestFeature]). _ name __= = 'float' or type (dataSet [0] [bestFeature]). _ name __= = 'int': bestSplitValue = bestSplitDict [labels [best Feature] labels [bestFeature] = labels [bestFeature] + '<=' + str (bestSplitValue) for I in range (shape (dataSet) [0]): if dataSet [I] [bestFeature] <= bestSplitValue: dataSet [I] [bestFeature] = 1 else: dataSet [I] [bestFeature] = 0 return bestFeature # If the feature has been divided and the samples under the node have not yet taken the same value, you need to vote for def majorityCnt (classList ): classCount ={} for vote in classList: if vote not in classCount. keys (): classCount [vote] = 0 classCount [vote] + = 1 r Eturn max (classCount) # The main program recursively generates the decision tree def createTree (dataSet, labels, data_full, labels_full): classList = [example [-1] for example in dataSet] if classList. count (classList [0]) = len (classList): return classList [0] if len (dataSet [0]) = 1: return majorityCnt (classList) bestFeat = chooseBestFeatureToSplit (dataSet, labels) bestFeatLabel = labels [bestFeat] myTree = {bestFeatLabel :{}} featValues = [example [bestFeat] for ex Ample in dataSet] uniqueVals = set (featValues) if type (dataSet [0] [bestFeat]). _ name __= = 'str': currentlabel = labels_full.index (labels [bestFeat]) featValuesFull = [example [currentlabel] for example in data_full] uniqueValsFull = set (featValuesFull) del (labels [bestFeat]) # divides a subtree for each value of bestFeat. For value in uniqueVals: subLabels = labels [:] if type (dataSet [0] [bestFeat]). _ name __= = 'str': uniqueValsFull. remove (value) myTree [bestFeatLabel] [value] = createTree (splitDataSet \ (dataSet, bestFeat, value), subLabels, data_full, labels_full) if type (dataSet [0] [bestFeat]). _ name __= = 'str': for value in uniqueValsFull: myTree [bestFeatLabel] [value] = majorityCnt (classList) return myTreeimport matplotlib. pyplot as pltdecisionNode = dict (boxstyle = "sawtooth", fc = "0.8") leafNode = dict (boxstyle = "round4", fc = "0.8 ") arrow_args = dict (arrowstyle = "<-") # calculate the number of leaf nodes in the tree def getNumLeafs (myTree): numLeafs = 0 firstSides = list (myTree. keys () firstStr = firstSides [0] secondDict = myTree [firstStr] for key in secondDict. keys (): if type (secondDict [key]). _ name __= = 'dict ': numLeafs + = getNumLeafs (secondDict [key]) else: numLeafs + = 1 return numLeafs # maximum computing tree depth def getTreeDepth (myTree ): maxDepth = 0 firstSides = list (myTree. keys () firstStr = firstSides [0] secondDict = myTree [firstStr] for key in secondDict. keys (): if type (secondDict [key]). _ name __= = 'dict ': thisDepth = 1 + getTreeDepth (secondDict [key]) else: thisDepth = 1 if thisDepth> maxDepth: maxDepth = thisDepth return maxDepth # Draw node def plotNode (nodeTxt, centerPt, parentPt, nodeType): createPlot. ax1.annotate (nodeTxt, xy = parentPt, xycoords = 'axes fraction', \ xytext = centerPt, textcoords = 'axes fraction', va = "center", ha = "center ", \ bbox = nodeType, arrowprops = arrow_args) # Draw the arrow on the text def plotMidText (cntrPt, parentPt, txtString): lens = len (txtString) xMid = (parentPt [0] + cntrPt [0])/2.0-lens * 0.002 yMid = (parentPt [1] + cntrPt [1])/2.0 createPlot. ax1.text (xMid, yMid, txtString) def plotTree (myTree, parentPt, nodeTxt): numLeafs = getNumLeafs (myTree) depth = getTreeDepth (myTree) firstSides = list (myTree. keys () firstStr = firstSides [0] cntrPt = (plotTree. x0ff + (1.0 + float (numLeafs)/2.0/plotTree. totalW, plotTree. y0ff) plotMidText (cntrPt, parentPt, nodeTxt) plotNode (firstStr, cntrPt, parentPt, decisionNode) secondDict = myTree [firstStr] plotTree. y0ff = plotTree. y0ff-1.0/plotTree. totalD for key in secondDict. keys (): if type (secondDict [key]). _ name __= = 'dict ': plotTree (secondDict [key], cntrPt, str (key) else: plotTree. x0ff = plotTree. x0ff + 1.0/plotTree. totalW plotNode (secondDict [key], (plotTree. x0ff, plotTree. y0ff), cntrPt, leafNode) plotMidText (plotTree. x0ff, plotTree. y0ff), cntrPt, str (key) plotTree. y0ff = plotTree. y0ff + 1.0/plotTree. totalDdef createPlot (inTree): fig = plt. figure (1, facecolor = 'white') fig. clf () axprops = dict (xticks = [], yticks = []) createPlot. ax1 = plt. subplot (111, frameon = False, ** axprops) plotTree. totalW = float (getNumLeafs (inTree) plotTree. totalD = float (getTreeDepth (inTree) plotTree. x0ff =-0.5/plotTree. totalW plotTree. y0ff = 1.0 plotTree (inTree, (0.5, 1.0), '') plt.show(1_df1_pd.read_csv('watermelon_4_3.csv ') data = df. values [:, 1:]. tolist () data_full = data [:] labels = df. columns. values [1:-1]. tolist () labels_full = labels [:] myTree = createTree (data, labels, data_full, labels_full) print (myTree) createPlot (myTree)

The final result is as follows:

{'Texture ': {'blur': 0, 'Little _ blur': {'touch': {'soft _ stick': 1, 'hard _ smooth ': 0 }}, 'distinct ': {'density <= 000000': {0: 1, 1: 0 }}}}

The resulting decision tree is as follows:

References:

Machine learning practices

Machine Learning by Zhou Zhihua

The above implementation based on the ID3 decision tree algorithm (Python version) is all the content shared by the editor. I hope to give you a reference, and I hope you can provide more support for the customer's house.

Contact Us

The content source of this page is from Internet, which doesn't represent Alibaba Cloud's opinion; products and services mentioned on that page don't have any relationship with Alibaba Cloud. If the content of the page makes you feel confusing, please write us an email, we will handle the problem within 5 days after receiving your email.

If you find any instances of plagiarism from the community, please send an email to: info-contact@alibabacloud.com and provide relevant evidence. A staff member will contact you within 5 working days.

A Free Trial That Lets You Build Big!

Start building with 50+ products and up to 12 months usage for Elastic Compute Service

  • Sales Support

    1 on 1 presale consultation

  • After-Sales Support

    24/7 Technical Support 6 Free Tickets per Quarter Faster Response

  • Alibaba Cloud offers highly flexible support services tailored to meet your exact needs.