Random gradient descent algorithm and its annotation

#-*-Coding:utf8-*-

Import Math
Import Matplotlib.pyplot as Plt

def f (W, X):
N = Len (w)
i = 0
y = 0
While I < N-1:
Y + = w[i] * X[i]
i + = 1
Y + = w[n-1] # constant entry
Return y

def gradient (data, W, j):
M = Len (data) # Number of samples
N = Len (data[0])
i = 0
g = 0 # Gradient of the current dimension
While I < M:
y = f (w, Data[i])
if (j! = N-1):
G + = (data[i][n-1]-y) * Data[i][j]
Else
G + = data[i][n-1]-y
i + = 1
Return g/m

def gradientstochastic (Data, W, j):
N = Len (data) # Dimension
y = data[n-1]-F (w, data)
if (j! = N-1):
Return y * Data[j]
Return Y # Constant entry

Def issame (A, B):
n = Len (a)
i = 0
While I < n:
If ABS (A[i]-b[i]) > 0.01:
Return False
i + = 1
Return True

Def FW (W, data):
M = Len (data) # Number of samples
N = Len (data[0])
i = 0
s = 0
While I < M:
y = data[i][n-1]-F (w, Data[i])
s + = y * * 2
i + = 1
Return S/2

Def fwstochastic (W, data):
y = Data[len (data)-1]-F (w, data)
Y **= 2
Return Y/2

def numberproduct (N, VEC, W):
N = Len (VEC)
i = 0
While I < N:
W[i] + = vec[i] * n
i + = 1

def assign (a):
L = []
For X in a:
L.append (x)
Return L

# a = b
Def assign2 (A, B):
i = 0
While I < Len (a):
A[i] = B[i]
i + = 1

Def dotProduct (A, B):
N = Len (a)
i = 0
DP = 0
While I < N:
DP + = a[i] * B[i]
i + = 1
Return DP

# w current value; G current gradient direction; A current learning rate; data
Def Calcalpha (W, G, A, data):
C1 = 0.3
Now = FW (W, data)
Wnext = Assign (w) #复制一份w
Numberproduct (A, G, wnext) #利用w = w + alpha (y-f (x)) x Update W
Next = FW (Wnext, data) # Calculates the loss of the entire 1000 samples
# Look for a big enough to make H (a) >0
Count = 30
While next < now:
A *= 2
Wnext = Assign (W)
Numberproduct (A, G, Wnext)
Next = FW (Wnext, data)
Count-= 1
if Count = = 0:
Break

# Find the right learning rate a
Count = 50
While Next > Now-c1*a*dotproduct (G, g):
A/= 2
Wnext = Assign (W)
Numberproduct (A, G, Wnext)
Next = FW (Wnext, data)

Count-= 1
if Count = = 0:
Break
Return a

# w current value; G current gradient direction; A current learning rate; data
Def calcalphastochastic (W, G, A, data):
C1 = 0.01 # Because each sample is down, the parameters run a little bit more, namely: radical
now = fwstochastic (w, data) #计算当前的全体样本的LOSS
Wnext = Assign (w) #重新复制一份W
Numberproduct (A, G, wnext) #利用w = w + alpha* (y-f (x)) *x calculation Update W
Next = fwstochastic (Wnext, data) #利用更新后的W calculate the entire sample loss
# Look for a big enough to make H (a) >0
Count = 30
While next < now:
If a < 1e-10:
A = 0.01
Else
A *= 2
Wnext = Assign (W)
Numberproduct (A, G, Wnext)
Next = fwstochastic (Wnext, data)
Count-= 1
if Count = = 0:
Break

# Find the right learning rate a
Count = 50
While Next > Now-c1*a*dotproduct (G, g):
A/= 2
Wnext = Assign (W)
Numberproduct (A, G, Wnext)
Next = fwstochastic (Wnext, data)

Count-= 1
if Count = = 0:
Break
Return a

def normalize (g):
s = 0
For X in G:
s + = x * x
s = math.sqrt (s)
i = 0
N = Len (g)
While I < N:
G[i]/= S
i + = 1

def calccoefficient (data, ListA, LISTW, listlostfunction):
M = Len (data) # Number of samples
N = Len (data[0]) # dimension
W = [0 for I in range (N)]
WNEW = [0 for I in range (N)]
g = [0 for I in range (N)]

Times = 0
Alpha = 100.0 # learning rate Random Initialization
same = False
While times < 10000:
i = 0
While I < M:
j = 0
While J < N:
G[J] = gradientstochastic (Data[i], W, j)
J + = 1
Normalize (g) # regularization gradient
Alpha = calcalphastochastic (W, G, Alpha, data[i])
#alpha = 0.01
Numberproduct (Alpha, G, WNEW)

Print "Times,i, alpha,fw,w,g:\t", Times, I, Alpha, FW (W, data), W, G
If Issame (W, WNEW):
If times > 5: #防止训练次数过少
same = True
Break
Assign2 (W, wnew) # Update weights

Lista.append (Alpha)
Listw.append (Assign (W))
Listlostfunction.append (FW (W, data))

i + = 1
If same:
Break
Times + = 1
Return W

if __name__ = = "__main__":
FileData = open ("D8.txt")
data = []
For line in FileData:
D = map (float, line.split (', '))
Data.append (d)
Filedata.close ()

ListA = [] # Learning rate for each step
LISTW = [] # Weights for each step
Listlostfunction = [] # loss function value for each step
W = calccoefficient (data, ListA, LISTW, listlostfunction)

# Draw The Learning rate
Plt.plot (ListA, ' R ', linewidth=2)
Plt.plot (ListA, ' go ')
Plt.xlabel (' Times ')
Plt.ylabel (' Ratio/step ')
Plt.show ()

# Draw Loss
Listlostfunction = listlostfunction[0:100]
Listlostfunction[0]/= 2
Plt.plot (listlostfunction, ' R ', linewidth=2)
Plt.plot (listlostfunction, ' gv ', alpha = 0.75)
Plt.xlabel (' Times ')
Plt.ylabel (' Loss Value ')
Plt.grid (True)
Plt.show ()

# Draw Weights
X = []
Y = []
For D in data:
X.append (D[0])
Y.append (D[1])
Plt.plot (X, Y, ' CP ', Label=u ' Original Data ', alpha=0.75)
x = [min (x), Max (x)]
y = [w[0] * x[0] + w[1], w[0] * x[1] + w[1]]
Plt.plot (x, Y, ' R ', linewidth=3, label= ' Regression Curve ')
Plt.legend (loc= ' upper left ')
Plt.grid (True)
Plt.show ()

Random gradient descent algorithm and its annotation