Python Data preprocessing exercises

#ecoding =utf-8 Import Math import re import csv def fileread (fileurl,access): "Incoming file path, return a two-dimensional list of stored file contents" LocalArray =
        [] # Create a list to store the contents of the Files CSVFile = file (FileURL, access) reader = Csv.reader (csvfile) for line in reader: Localarray.append (line) csvfile.close () return LocalArray def getline (inlist,line): "Get a row of data" return InL Ist[line] def getRow (inlist,row): "Get a column of data" Listreturn = [] for i in InList:listReturn.append (I[row) Return Listreturn def setline (inlist,childlist,line): "Set a row of the matrix data" inlist[line] = childlist def setrow (Inli St,chikdlist,row): "Set a column of a matrix" i = 0 for i in range (0,len (chikdlist)): Inlist[i][row] = Chikdlist[i] D
    EF AddLine (Inlist,childline): "Add a row to the data Matrix" Inlist.append (ChildLine) def addrow (Inlist,childrow): "Add a column to the data Matrix" j = 0 For I in Inlist:i.append (childrow[j]) j = j+1 def getavg (inList): "Calculate the mean value of a numeric property" Sumofl ist = 0 Lengoflist = 0 for I in Inlist:if Re.match (R ' [0-9]+ ', i): Sumoflist = sumoflist + float (i) lengofl ist = lengoflist + 1 else:continue if lengoflist!= 0:return sumoflist/lengoflist E Lse:return "current feature no mean" Def getave (inList): "Calculate variance of numeric Property" #先求平均数 sumoflist = 0 lengoflist = 0 su = 0 for I in Inlist:if Re.match (R ' [0-9]+ ', i): Sumoflist = sumoflist + float (i) Lengo
        Flist = lengoflist + 1 else:continue if lengoflist!= 0:avg = sumoflist/lengoflist
                For j in Inlist:if Re.match (R ' [0-9]+ ', j): Su + + (float (j)-AVG) * * 2 Else: Continue return Math.sqrt (su) else:return "Current feature variance" Def average (seq, total=0.0): n UM = 0 for item in Seq:total + + + + + 1 return total/num def getquantile (inlist,inlocaltion): "Please The number of decimal places for numeric Properties "if Inlocal"tion >1 or inlocaltion<0 or inlocaltion = = 1:return "Number of decimal points entered in numeric error" LOCALLST = [] Leng = 0 for I I N inlist:if re.match (R ' [0-9]+ ', I): Locallst.append (float (i)) Leng = Leng + 1 Else
         : Continue if Leng = = 0:return "Current feature not available median" locallst.sort () if inlocaltion = 0.5: If Len (locallst)%2 = = 1:return Locallst[len (locallst)//2] Else:return (locallst[le N (locallst)//2-1]+locallst[len (LOCALLST)//2])/2.0 elif inlocaltion<1 and Inlocaltion>=0:return [Int (len (locallst) *inlocaltion)] def fileread (fileurl,access): "Incoming file path, returns a two-dimensional list of stored file contents" LocalArray = [] # Create a list for Storage file Contents CSVFile = FileURL (access) reader = Csv.reader (csvfile) for line in Reader:localArray.app End (line) Csvfile.close () return LocalArray def removenoiseauto (inList): "Use IRQ to identify noise data and remove this data" Q3 = Getquan TILE (inlist,0.75) Q1 = Getquantile (inlist,0.25) IRQ = q3-q1 for i in Range (1,len (inList), 1): If float (inlist[i))-Q3 > 1 .5*irq or Q1-float (Inlist[i]) > 1.5*irq:inlist[i] = ' return inList def removenoisebythresholdmin ( Inlist,inthresholdmin): "Remove noise data based on minimum threshold to remove this data" for I in range (1, Len (inList), 1): If float (inlist[i)) < in Thresholdmin:inlist[i] = ' return inList def Removenoisebythresholdmax (Inlist,inthresholdmax): "According to the most Large threshold removal Noise data remove this data "for I in range (1, Len (inList), 1): If float (inlist[i]) > Inthresholdmax:inlist [I] = ' return inList def autopaddingbyavg (inList): "Use mean complement total missing value" avg = Getavg (inList) for I in range (1, l En (InList), 1): if inlist[i] = = ': inlist[i] = str (avg) return inList def Autopaddingbymedian (in List): "Complement all missing values with median" avg = Getquantile (inlist,0.5) for I in range (1, Len (inList), 1): if inlist[i] = = ' ': Inlist[i] = STR (avg) return inList def binningwidth (inlist,width): "Data discretization: Equal width compartment" dic = {} for I in range (1,len Inli ST): Dic[i] =float (inlist[i]) dict = sorted (Dic.iteritems (), Key=lambda d:d[1], reverse= False) # First press the list by Valu E sort dictlist = [] # assigns the sorted element to a list for storing k-v pair for Varlo in Dict:dictList.append (List (Varlo)) i = 0 # with To record the start position of each box J = 0 #用于记录每个箱结束位置 innerlist = [] for i in range (0, Len (dictlist)): If Dictlist[i][1]-di CTLIST[J][1] > Width:avg = average (innerlist) for K in range (J, I, 1): Dictlis T[K][1] = Avg Innerlist = [] J = i innerlist.append (dictlist[i][1]) if (i = = Len (d Ictlist)-1): avg = average (innerlist) for K in range (J, I, 1): dictlist[k][1] = av
        G innerlist = [] dictlist[i][1] = Avg Dic1 = {} for I in range (0, Len (dictlist)): Dic1[dictlist[i][0]] =DICTLIST[I][1] ad = sorted (Dic1.iteritems (), Key=lambda d:d[0], Reverse=false) # First sort list by key for I in range (0, le N (AD): Inlist[i + 1] = Ad[i][1] return inList def binningdeep (INLIST,DEEP1): "Data discretization: Equal frequency compartment" deep = Dee P1-1 dic = {} for I in range (1,len (inList)): Dic[i] =float (inlist[i)) Dict = sorted (Dic.iteritems (),
        Key=lambda d:d[1], reverse= False) # First sort the list by value dictlist = [] # Assign the sorted element to a list for storing k-v to for Varlo in Dict: Dictlist.append (List (Varlo)) innerlist = [] for I in range (0,deep): #为了排除0的干扰, first dispose of deep elements innerlist . Append (dictlist[i][1]) for I in range (deep, Len (dictlist)): If I% deep = 0:avg = Average (inner List) for J in Range (I-deep,i): dictlist[j][1] = avg Innerlist = [] Innerl Ist.append (dictlist[i][1]) if i = = Len (dictlist) -1:avg = average (innerlist) for J-in rang
E ((i+1)/deep*deep,i+1):                DICTLIST[J][1] = Avg Dic1 = {} for I in range (0, Len (dictlist)): dic1[dictlist[i][0] = d ICTLIST[I][1] ad = sorted (Dic1.iteritems (), Key=lambda d:d[0], reverse= False) # First sort the list by key for the I in range (0,len (AD): inlist[i+1] = ad[i][1] return inList def onehot (inlist,row): "Use Onehot encoding for a column of the input data Matrix" rowList0 = GetRow (inlist,row) rowhead = rowlist0[0] Rowlist = [] for I in range (1,len (rowList0)): Rowlist.appen D (rowlist0[i]) rowmsg = {} j = 0 for I in Rowlist:if Rowmsg.has_key (i): rowmsg[i] = rowms G[i] + 1 Else:rowmsg[i] = 1 for i in Rowmsg.keys (): Addlist = [] Addlist.append (i For J-in rowlist:if j = = I:addlist.append (' 1 ') else:a 
    Ddlist.append (' 0 ') addrow (inlist,addlist) for i in Inlist:print i def Minmax (inList): "Maximum minimum normalization"
    Innerlist = []For I in range (1,len (inList)): If Re.match (R ' [0-9]+ ', Inlist[i]): Innerlist.append (float (inlist[i)) MaxValue = max (innerlist) minvalue = min (innerlist) for I in Range (1,len (inList)): If Re.match (R ' [0-9] + ', Inlist[i]): a = (float (inlist[i))-minvalue)/(maxvalue-minvalue) b = "%.4f"%a in  List[i] = str (b) return inList def zscore (inList): "Zscore normalized" print inList u = getavg (inList) Ave =
            Getave (inList) stand = math.sqrt (ave) for I in Range (1,len (inList)): If Re.match (R ' [0-9]+ ', Inlist[i]):  A = (float (inlist[i])-u)/stand b = "%.4f"% a inlist[i] = str (b) return inList def Similaritydistance (inlist1,inlist2,n): "Distance similarity" sum = 0 for I in range (1,len (inList1)): sum = SUM + ABS
    (Float (inlist1[i])-float (inlist2[i]) * * N a = float (1)/2 return Pow (sum,a) def Similaritysim (INLIST1,INLIST2): "Cosine similarity calculation" sum = 0
    For I in range (1,len (inList1)): sum = sum + float (inlist1[i]) *float (inlist2[i]) sum1 = 0 sum2 = 0 For I in range (1,len (inList1)): sum1 = sum1 + float (inlist1[i]) **2 to I in range (1, Len (INLIST2)): s um2 = sum2 + float (inlist2[i]) * * * 2 return sum/(MATH.SQRT (sum1) *math.sqrt (sum2)) FileInput = Fileread ("D:\\pythonwor  Kspace\\externalfile\\train.csv ", R") # #获得某一行数据 # print Getline (fileinput,1) # #获得某一列数据 # print GetRow (fileinput,0) # #设置某一行数据 # Print before setting: "# Print Getline (fileinput,1) # Setline (Fileinput,getline (fileinput,2), 1) # print" After setting: "# Print G Etline (fileinput,1) # #设置某一列数据 # print before setting: "# Print GetRow (fileinput,1) # Setrow (Fileinput,getrow (fileinput,2), 1) # PRI NT "after setting:" # Print GetRow (fileinput,1) # #均值 # print Getavg (GetRow (fileinput,9)) # #方差 # print Getave (GetRow (fileinput,9) # #分位数 # Print Getquantile (GetRow (fileinput,9), 0.5) # #噪声数据过滤1 # print Removenoiseauto (getRow) # #噪声数据过 Filter 2 # Print RemovenoisebyThresholdmin (GetRow, fileinput,0) # #噪声数据过滤3 # print Removenoisebythresholdmax (GetRow (fileinput,0), ten) # #缺失值补全1 # Print Autopaddingbyavg (GetRow (fileinput,0)) # #缺失值补全2 # print Autopaddingbymedian (GetRow (fileinput,0)) # #等宽分箱 # Print Binningwidth (GetRow (fileinput,0), 3) # #等频分箱 # print Binningdeep (GetRow (fileinput,0), 3) # #ONE-hot code # onehot (fileInput , 1) # for I in FileInput: # print I # #最大最小归一化 # print Minmax (GetRow (fileinput,0)) # #zScore归一化 # print Zscore (getr ow (fileinput,0)) # #距离相似度 # print similaritydistance (GetRow (fileinput,0), GetRow (fileinput,0), 2) # cosine similarity calculation # print Simi Laritysim (GetRow (fileinput,0), GetRow (fileinput,1))