Lenet-5mnist handwritten numeral recognition model using L2 regularization and average sliding model

Lenet-5mnist handwritten numeral recognition model using L2 regularization and average sliding models feel useful, and welcome to discuss learning from each other ~follow Me

Reference TensorFlow Google Deep Learning framework
Experimental platform:
Tensorflow1.4.0
python3.5.0
mnist datasets are downloaded and placed into the Mnist_data folder under the current directory after downloading four files
Regularization of the L2
Dropout
Sliding Average method

Define model framework and forward propagation

Import TensorFlow as tf# Configure the parameters of the neural network Input_node = 784output_node = 10image_size = 28num_channels = 1num_labels = 10# The ruler of the first layer of the convolutional layer Inch and depth Conv1_deep = 32conv1_size = * The size and depth of the second layer of the convolutional layer conv2_deep = 64conv2_size = * The number of nodes of the full join layer fc_size = 512# Defines the forward propagation process of convolutional neural networks, which is added here A parameter train for distinguishing between the training process and the test process. # using the Dropout method, the dropout method can further enhance the reliability of the model and prevent overfitting, dropout used only during the training process. def inference (Input_tensor, Train, Regularizer): # By using a different namespace to isolate variables, you can make each layer variable named only if you want to consider the role of the current layer, without having to consider the problem with Tf.vari Able_scope (' Layer1-conv1 '): Conv1_weights = tf.get_variable ("Weight", [Conv1_size, Conv1_size, Num_chan Nels, Conv1_deep], Initializer=tf.truncated_normal_initializer (stddev=0.1)) conv1_biases = Tf.get_variab Le ("bias", [Conv1_deep], Initializer=tf.constant_initializer (0.0)) Conv1 = tf.nn.conv2d (Input_tensor, conv1_weights , Strides=[1, 1, 1, 1], padding= ' same ') RELU1 = Tf.nn.relu (Tf.nn.bias_add (CONV1, conv1_biases)) with Tf.name_sco PE ("Layer2-pool1"): Pool1 = Tf.nn.max_pool (RELU1, KSIZe=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding= "Same") with Tf.variable_scope ("Layer3-conv2"): Conv2_weights = t F.get_variable ("Weight", [Conv2_size, Conv2_size, Conv1_deep, Conv2_deep], initializer=tf.truncated _normal_initializer (stddev=0.1)) conv2_biases = Tf.get_variable ("bias", [conv2_deep], Initializer=tf.constant_initi Alizer (0.0)) Conv2 = tf.nn.conv2d (Pool1, Conv2_weights, strides=[1, 1, 1, 1], padding= ' same ') RELU2 = Tf.nn. Relu (Tf.nn.bias_add (Conv2, conv2_biases)) with Tf.name_scope ("Layer4-pool2"): Pool2 = Tf.nn.max_pool (RELU2, Ksiz        E=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding= ' same ') # Pool2.getshape function can get the dimension of the fourth output matrix without the need of manual calculation.        Note Because each layer of neural network input and output is a batch matrix, the resulting dimension also contains the number of data in a batch.        Pool_shape = Pool2.get_shape (). As_list () # Calculates the length of the matrix after it is straightened into a vector, which is the product of the length and depth of the matrix, and note that here the pool_shape[0] is the number of data in a batch  nodes = pool_shape[1]*pool_shape[2]*pool_shape[3] # The output of layer fourth is transformed into a vector of batch by the Tf.shape function      reshaped = Tf.reshape (Pool2, [pool_shape[0], nodes]) # dropout typically use with tf.variable_scope (' La ') only at the full connection layer instead of the convolution layer or the pooling layer Yer5-fc1 '): Fc1_weights = tf.get_variable ("Weight", [nodes, Fc_size], Initiali Zer=tf.truncated_normal_initializer (stddev=0.1) # Only the full join layer weights need to be added to the regularization if regularizer! = None:tf.add_to_collec tion (' Losses ', Regularizer (fc1_weights)) fc1_biases = Tf.get_variable ("bias", [fc_size], initializer=tf.constant_in Itializer (0.1)) FC1 = Tf.nn.relu (Tf.matmul (reshaped, fc1_weights) + fc1_biases) # If the train tag is true, the dropout function is introduced to make the output Half of the neurons out of the layer are inactivated if TRAIN:FC1 = Tf.nn.dropout (FC1, 0.5) with Tf.variable_scope (' LAYER6-FC2 '): fc2_weights = TF . get_variable ("Weight", [Fc_size, Num_labels], Initializer=tf.truncated_normal_initia Lizer (stddev=0.1)) if Regularizer! = None:tf.add_to_collection (' Losses ', Regularizer (fc2_weights)) Fc2_bias ES = tf.get_variable ("BiAs ", [Num_labels], Initializer=tf.constant_initializer (0.1)) Logit = Tf.matmul (FC1, fc2_weights) + fc2_biases re Turn logit

Training the Mnist model based on Lenet

Import TensorFlow as Tffrom tensorflow.examples.tutorials.mnist import Input_dataimport lenet5_inferneceimport Osimport NumPy as np# # 1. Define neural network-related parameters batch_size = 100 # Batch Quantity size learning_rate_base = 0.01 # Basic Learning Rate Learning_rate_decay = 0.99 # Learning Rate Decay rate regularizatio N_rate = 0.0001 # regularization parameter training_steps = 6000 # Number of training Cycles Moving_average_decay = 0.99 # Average Sliding Step # # # # # # # # # # # # # 2. Define the training Process Def train (mnist): # defines the output as a 4-dimensional matrix placeholder x = Tf.placeholder (Tf.float32, [Batch_size, lenet5_in Fernece. Image_size, Lenet5_infernece. Image_size, Lenet5_infernece. Num_channels], name= ' X-input ') # Y_ indicates the correct label y_ = Tf.placeholder (Tf.float32, [None, Lenet5_infer Nece. Output_node], name= ' y-input ') # defines L2 regularization Regularizer = Tf.contrib.layers.l2_regularizer (regularization_rate) y = L Enet5_infernece.inference (x, False, Regularizer) # indicates that dropout is not used, but regularization global_step = TF is used.    Variable (0, Trainable=false) # defines the loss function, the learning rate, the sliding average operation, and the training process. Variable_averages = TF.TRAIN.EXponentialmovingaverage (Moving_average_decay, Global_step) # using the average sliding model VARIABLES_AVERAGES_OP = VARIABLE_AVERAGES.APPL Y (Tf.trainable_variables ()) # Set with cross-entropy function cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits (logits=y, label S=tf.argmax (Y_, 1)) Cross_entropy_mean = Tf.reduce_mean (cross_entropy) # Add the L2 regularization portion of the weight to the loss function loss = Cross_entropy_ Mean + tf.add_n (tf.get_collection (' losses ')) # defines the descending learning rate learning_rate = Tf.train.exponential_decay (learning_    Rate_base, Global_step, Mnist.train.num_examples/batch_size, Learning_rate_decay, Staircase=true) Train_step = Tf.train.GradientDescentOptimizer (learning_rate). Minimize (loss, global_step=global_step) # with Tf.control_dependencies ([Train_step, Variables_averages_op]): # train_op = Tf.no_op (name= ' train ') # in reverse propagation gradient descent process    The sliding average of the updated variable train_op = Tf.group (Train_step, Variables_averages_op) # initializes the TensorFlow persistence class. Saver = Tf.train.Saver () with TF.     Session () as Sess:   Tf.global_variables_initializer (). Run () for I in Range (training_steps): xs, ys = Mnist.train.next_bat CH (batch_size) Reshaped_xs = Np.reshape (XS, (Batch_size, Lenet5_infernece. Image_size, Lenet5_infernece. Image_size, Lenet5_infernece. Num_channels) _, loss_value, step = Sess.run ([Train_op, Loss, Global_step], Feed_dict={x:reshaped_xs, Y_: ys} if i%1000 = = 0:print ("After%d training step (s), Loss on training batch is%g."% ( Step, Loss_value) # # # # # # # 3.  Main program Entry def main (argv=none): Mnist = Input_data.read_data_sets ("./mnist_data", One_hot=true) train (mnist) if __name__ = = ' __main__ ': Main ()

Lenet-5mnist handwritten numeral recognition model using L2 regularization and average sliding models