卷積神經網路--代碼實現

import numpy as npclass ReluActivator(object):    def forward(self, weighted_input):        #return weighted_input        return max(0, weighted_input)    def backward(self, output):        return 1 if output > 0 else 0class IdentityActivator(object):    def forward(self, weighted_input):        return weighted_input    def backward(self, output):        return 1def get_patch(input_array, i, j, filter_width,              filter_height, stride):    start_i = i * stride    start_j = j * stride    if input_array.ndim == 2:        return input_array[             start_i : start_i + filter_height,             start_j : start_j + filter_width]    elif input_array.ndim == 3:        return input_array[:,             start_i : start_i + filter_height,             start_j : start_j + filter_width]def get_max_index(array):    max_i = 0    max_j = 0    max_value = array[0,0]    for i in range(array.shape[0]):        for j in range(array.shape[1]):            if array[i,j] > max_value:                max_value = array[i,j]                max_i, max_j = i, j    return max_i, max_jdef conv(input_array,          kernel_array,         output_array,          stride, bias):    channel_number = input_array.ndim    output_width = output_array.shape[1]    output_height = output_array.shape[0]    kernel_width = kernel_array.shape[-1]    kernel_height = kernel_array.shape[-2]    for i in range(output_height):        for j in range(output_width):            output_array[i][j] = (                    get_patch(input_array, i, j, kernel_width,                     kernel_height, stride) * kernel_array                ).sum() + biasdef padding(input_array, zp):    if zp == 0:        return input_array    else:        if input_array.ndim == 3:            input_width = input_array.shape[2]            input_height = input_array.shape[1]            input_depth = input_array.shape[0]            padded_array = np.zeros((                input_depth,                 input_height + 2 * zp,                input_width + 2 * zp))            padded_array[:,                zp : zp + input_height,                zp : zp + input_width] = input_array            return padded_array        elif input_array.ndim == 2:            input_width = input_array.shape[1]            input_height = input_array.shape[0]            padded_array = np.zeros((                input_height + 2 * zp,                input_width + 2 * zp))            padded_array[zp : zp + input_height,                zp : zp + input_width] = input_array            return padded_array# 對numpy數組進行element wise操作def element_wise_op(array, op):    for i in np.nditer(array,                       op_flags=['readwrite']):        i[...] = op(i)class Filter(object):    def __init__(self, width, height, depth):        self.weights = np.random.uniform(-1e-4, 1e-4,            (depth, height, width))        self.bias = 0        self.weights_grad = np.zeros(            self.weights.shape)        self.bias_grad = 0    def __repr__(self):        return 'filter weights:\n%s\nbias:\n%s' % (            repr(self.weights), repr(self.bias))    def get_weights(self):        return self.weights    def get_bias(self):        return self.bias    def update(self, learning_rate):        self.weights -= learning_rate * self.weights_grad        self.bias -= learning_rate * self.bias_gradclass ConvLayer(object):    def __init__(self, input_width, input_height,                  channel_number, filter_width,                  filter_height, filter_number,                  zero_padding, stride, activator,                 learning_rate):        self.input_width = input_width        self.input_height = input_height        self.channel_number = channel_number        self.filter_width = filter_width        self.filter_height = filter_height        self.filter_number = filter_number        self.zero_padding = zero_padding        self.stride = stride        self.output_width = \            ConvLayer.calculate_output_size(            self.input_width, filter_width, zero_padding,            stride)        self.output_height = \            ConvLayer.calculate_output_size(            self.input_height, filter_height, zero_padding,            stride)        self.output_array = np.zeros((self.filter_number,             int (self.output_height), int (self.output_width)))        self.filters = []        for i in range(filter_number):            self.filters.append(Filter(filter_width,                 filter_height, self.channel_number))        self.activator = activator        self.learning_rate = learning_rate    def forward(self, input_array):        self.input_array = input_array        self.padded_input_array = padding(input_array,            self.zero_padding)        for f in range(self.filter_number):            filter = self.filters[f]            conv(self.padded_input_array,                 filter.get_weights(), self.output_array[f],                self.stride, filter.get_bias())        element_wise_op(self.output_array,                         self.activator.forward)    def backward(self, input_array, sensitivity_array,                  activator):        self.forward(input_array)        self.bp_sensitivity_map(sensitivity_array,                                activator)        self.bp_gradient(sensitivity_array)    def update(self):        for filter in self.filters:            filter.update(self.learning_rate)    def bp_sensitivity_map(self, sensitivity_array,                           activator):        expanded_array = self.expand_sensitivity_map(            sensitivity_array)    nded_width = expanded_array.shape[2]        zp = (self.input_width +                self.filter_width - 1 - expanded_width) / 2        padded_array = padding(expanded_array, int (zp))        self.delta_array = self.create_delta_array()        for f in range(self.filter_number):            filter = self.filters[f]            flipped_weights = np.array([np.rot90(i, 2) for i in filter.get_weights()])            delta_array = self.create_delta_array()            for d in range(delta_array.shape[0]):                conv(padded_array[f], flipped_weights[d],                    delta_array[d], 1, 0)            self.delta_array += delta_array        derivative_array = np.array(self.input_array)        element_wise_op(derivative_array,                         activator.backward)        self.delta_array *= derivative_array    def bp_gradient(self, sensitivity_array):        expanded_array = self.expand_sensitivity_map(            sensitivity_array)        for f in range(self.filter_number):            filter = self.filters[f]            for d in range(filter.weights.shape[0]):                conv(self.padded_input_array[d],                      expanded_array[f],                     filter.weights_grad[d], 1, 0)            # 計算偏置項的梯度            filter.bias_grad = expanded_array[f].sum()    def expand_sensitivity_map(self, sensitivity_array):        depth = sensitivity_array.shape[0]        expanded_width = (self.input_width -             self.filter_width + 2 * self.zero_padding + 1)        expanded_height = (self.input_height -             self.filter_height + 2 * self.zero_padding + 1)        expand_array = np.zeros((depth, expanded_height,                                  expanded_width))        for i in range(int (self.output_height)):            for j in range(int (self.output_width)):                i_pos = i * self.stride                j_pos = j * self.stride                expand_array[:,i_pos,j_pos] = \                    sensitivity_array[:,i,j]        return expand_array    def create_delta_array(self):        return np.zeros((self.channel_number,            self.input_height, self.input_width))    @staticmethod    def calculate_output_size(input_size,            filter_size, zero_padding, stride):        return (input_size - filter_size +             2 * zero_padding) / stride + 1class MaxPoolingLayer(object):    def __init__(self, input_width, input_height,                  channel_number, filter_width,                  filter_height, stride):        self.input_width = input_width        self.input_height = input_height        self.channel_number = channel_number        self.filter_width = filter_width        self.filter_height = filter_height        self.stride = stride        self.output_width = (input_width -             filter_width) / self.stride + 1        self.output_height = (input_height -            filter_height) / self.stride + 1        self.output_array = np.zeros((self.channel_number,            int (self.output_height), int (self.output_width)))    def forward(self, input_array):        for d in range(self.channel_number):            for i in range(int (self.output_height)):                for j in range(int (self.output_width)):                    self.output_array[d,i,j] = (                            get_patch(input_array[d], i, j,                            self.filter_width,                             self.filter_height,                             self.stride).max())    def backward(self, input_array, sensitivity_array):        self.delta_array = np.zeros(input_array.shape)        for d in range(self.channel_number):            for i in range(int (self.output_height)):                for j in range(int (self.output_width)):                    patch_array = get_patch(                        input_array[d], i, j,                        self.filter_width,                         self.filter_height,                         self.stride)                    k, l = get_max_index(patch_array)                    self.delta_array[d,                         i * self.stride + k,                         j * self.stride + l] = \                        sensitivity_array[d,i,j]def init_test():    a = np.array(        [[[0,1,1,0,2],          [2,2,2,2,1],          [1,0,0,2,0],          [0,1,1,0,0],          [1,2,0,0,2]],         [[1,0,2,2,0],          [0,0,0,2,0],          [1,2,1,2,1],          [1,0,0,0,0],          [1,2,1,1,1]],         [[2,1,2,0,0],          [1,0,0,1,0],          [0,2,1,0,1],          [0,1,2,2,2],          [2,1,0,0,1]]])    b = np.array(        [[[0,1,1],          [2,2,2],          [1,0,0]],         [[1,0,2],          [0,0,0],          [1,2,1]]])    cl = ConvLayer(5,5,3,3,3,2,1,2,IdentityActivator(),0.001)    cl.filters[0].weights = np.array(        [[[-1,1,0],          [0,1,0],          [0,1,1]],         [[-1,-1,0],          [0,0,0],          [0,-1,0]],         [[0,0,-1],          [0,1,0],          [1,-1,-1]]], dtype=np.float64)    cl.filters[0].bias=1    cl.filters[1].weights = np.array(        [[[1,1,-1],          [-1,-1,1],          [0,-1,1]],         [[0,1,0],         [-1,0,-1],          [-1,1,0]],         [[-1,0,0],          [-1,0,1],          [-1,0,0]]], dtype=np.float64)    return a, b, cldef test():    a, b, cl = init_test()    cl.forward(a)    print (cl.output_array)def test_bp():    a, b, cl = init_test()    cl.backward(a, b, IdentityActivator())    cl.update()    print (cl.filters[0])    print (cl.filters[1])def gradient_check():    error_function = lambda o: o.sum()    a, b, cl = init_test()    cl.forward(a)    sensitivity_array = np.ones(cl.output_array.shape,                                dtype=np.float64)    cl.backward(a, sensitivity_array,                  IdentityActivator())    epsilon = 10e-4    for d in range(cl.filters[0].weights_grad.shape[0]):        for i in range(cl.filters[0].weights_grad.shape[1]):            for j in range(cl.filters[0].weights_grad.shape[2]):                cl.filters[0].weights[d,i,j] += epsilon                cl.forward(a)                err1 = error_function(cl.output_array)                cl.filters[0].weights[d,i,j] -= 2*epsilon                cl.forward(a)                err2 = error_function(cl.output_array)                expect_grad = (err1 - err2) / (2 * epsilon)                cl.filters[0].weights[d,i,j] += epsilon                print ('weights(%d,%d,%d): expected - actural %f - %f' % (                    d, i, j, expect_grad, cl.filters[0].weights_grad[d,i,j]))def init_pool_test():    a = np.array(        [[[1,1,2,4],          [5,6,7,8],          [3,2,1,0],          [1,2,3,4]],         [[0,1,2,3],          [4,5,6,7],          [8,9,0,1],          [3,4,5,6]]], dtype=np.float64)    b = np.array(        [[[1,2],          [2,4]],         [[3,5],          [8,2]]], dtype=np.float64)    mpl = MaxPoolingLayer(4,4,2,2,2,2)    return a, b, mpldef test_pool():    a, b, mpl = init_pool_test()    mpl.forward(a)    print ('input array:\n%s\noutput array:\n%s' % (a,        mpl.output_array))def test_pool_bp():    a, b, mpl = init_pool_test()    mpl.backward(a, b)    print ('input array:\n%s\nsensitivity array:\n%s\ndelta array:\n%s' % (        a, b, mpl.delta_array))if __name__ == '__main__':    test()    test_bp()    test_pool()    test_pool_bp()    gradient_check()