TensorFlow builds Neural Network best practices and tensorflow Best Practices

TensorFlow builds Neural Network best practices and tensorflow Best Practices

I. Complete TensorFLow example

On the MNIST dataset, a simple neural network structure and a two-layer Neural Network Containing ReLU unit nonlinear processing are built. When training a neural network, use the learning rate setting with exponential attenuation, use regularization to avoid overfitting, and use the moving average model to make the final model more robust.

The program defines an inference function for the Forward Propagation part of the computing neural network, a train function for the training part, and a main function for the training part.

Complete program:

#! /Usr/bin/env python3 #-*-coding: UTF-8-*-"Created on Thu May 25 08:56:30 2017 @ author: marsjhao "import tensorflow as tf from tensorflow. examples. tutorials. mnist import input_data INPUT_NODE = 784 # Number of input nodes OUTPUT_NODE = 10 # Number of output nodes LAYER1_NODE = 500 # Number of Hidden Layer Nodes BATCH_SIZE = 100 LEARNING_RETE_BASE = 0.8 # base learning rate attenuation = 0.99 # learning rate rate REGULARIZATION_RATE = 0.0001 # weight coefficient TRAININ of regularization items G_STEPS = 10000 # Number of iterative training times MOVING_AVERAGE_DECAY = 0.99 # Moving Average attenuation coefficient # weight and offset of the input neural network, which is used to calculate def inference (input_tensor, avg_class, weights1, biases1, weights2, biases2): # determine whether to pass in ExponentialMovingAverage class object if avg_class = None: layer1 = tf. nn. relu (tf. matmul (input_tensor, weights1) + biases1) return tf. matmul (layer1, weights2) + biases2 else: layer1 = tf. nn. relu (tf. matmul (input_tensor, avg_class. Average (weights1) + avg_class.average (biases1) return tf. matmul (layer1, avg_class.average (weights2) \ + avg_class.average (biases2) # def train (mnist): x = tf. placeholder (tf. float32, [None, INPUT_NODE], name = 'x-input') y _ = tf. placeholder (tf. float32, [None, OUTPUT_NODE], name = 'Y-input') # define the parameter weights1 = tf for the neural network structure. variable (tf. truncated_normal ([INPUT_NODE, LAYER1_NODE], stddev = 0.1) bia Ses1 = tf. variable (tf. constant (0.1, shape = [LAYER1_NODE]) weights2 = tf. variable (tf. truncated_normal ([LAYER1_NODE, OUTPUT_NODE], stddev = 0.1) biases2 = tf. variable (tf. constant (0.1, shape = [OUTPUT_NODE]) # Calculate the Forward Propagation result of parameters in the non-moving average model. y = inference (x, None, weights1, biases1, weights2, biases2) global_step = tf. variable (0, trainable = False) # defines the Variable that stores the number of training rounds of the current iteration # defines the ExponentialMovingAverage Class Object variable_average S = tf. train. exponentialMovingAverage (MOVING_AVERAGE_DECAY, global_step) # input the current iteration wheel Number Parameter # define the operation op variables_averages_op = variable_averages.apply (tf. trainable_variables () # Calculate the Forward Propagation result of parameters in the sliding model. average_y = inference (x, variable_averages, weights1, biases1, weights2, biases2) # define the cross entropy loss value cross_entropy = tf. nn. sparse_softmax_cross_entropy_with_logits (logits = y, label S = tf. argmax (y _, 1) cross_entropy_mean = tf. performance_mean (cross_entropy) # defines the L2 regularization device and regularizer = tf for weights1 and weights2. contrib. layers. l2_regularizer (REGULARIZATION_RATE) regularization = regularizer (weights1) + regularizer (weights2) loss = login + regularization # total loss value # define the exponential attenuation learning rate learning_rate = tf. train. exponential_decay (LEARNING_RETE_BASE, global_step, mnist. train. num_examples/ BATCH_SIZE, LEARNING_RETE_DECAY) # defines the gradient descent operation op. The global_step parameter can implement the auto-increment 1 operation train_step = tf. train. gradientDescentOptimizer (learning_rate )\. minimize (loss, global_step = global_step) # combine two operations op train_op = tf. group (train_step, variables_averages_op) ''' # Corresponds to tf. group () equivalent statement with tf. control_dependencies ([train_step, variables_averages_op]): train_op = tf. no_op (name = 'train') ''' # defines accuracy # In the final prediction, the output of the neural network is Calculation Result of Moving Average Forward Propagation: correct_prediction = tf. equal (tf. argmax (average_y, 1), tf. argmax (y _, 1) accuracy = tf. performance_mean (tf. cast (correct_prediction, tf. float32) # initialize the session sess and start iterative training with tf. session () as sess: sess. run (tf. global_variables_initializer () # validate_feed = {x: mnist. validation. images, y _: mnist. validation. labels} # Test Set Data to be fed test_feed = {x: mnist. test. images, y _: mnist. test. labels} fo R I in range (TRAINING_STEPS): if I % 1000 = 0: validate_acc = sess. run (accuracy, feed_dict = validate_feed) print ('after % d training steps, validation accuracy ''' using average model is % F' % (I, validate_acc) xs, ys = mnist. train. next_batch (BATCH_SIZE) sess. run (train_op, feed_dict = {x: xs, y _: ys}) test_acc = sess. run (accuracy, feed_dict = test_feed) print ('after % d training steps, test accuracy ''u Sing average model is % F' % (TRAINING_STEPS, test_acc) # main Function def main (argv = None): mnist = input_data.read_data_sets ("MNIST_data", one_hot = True) train (mnist) # The current python file is the entry file for shell File Execution, rather than the python module for import. If _ name _ = '_ main _': # run tf. app. run () in the module # Call the main function and input the required parameter list.

2. Analysis and Improvement Design

1. program analysis improvement

First, the inference function of the Forward propagation function needs to pass all the variables into the function as parameters. When the neural network structure becomes more complex and there are more parameters, program readability will become very poor.

Second, when the program exits, the trained model cannot be reused, and the training time of large-scale neural networks is long, during the training process, you need to save the intermediate results of model training at intervals. In this way, if the program crashes during the training process, the latest model parameters before the crash can still be retained, this eliminates the waste of time and resources.

Third, divide training and testing into two independent programs, and abstract the Forward propagation processes used in training and testing into separate library functions. This ensures that the Forward Propagation computing programs called during training and prediction are consistent.

2. Improved Program Design

Mnist_inference.py

This file defines the Forward Propagation Process of the neural network. The weights definition process used multiple times is also defined as a function separately.

Using tf. the get_variable function is used to obtain the variables. These variables are created during neural network training. During the test, the values of these variables are loaded through the saved model, and the sliding average value can be renamed during variable loading. Therefore, you can use the variable itself directly with the same name during training and use the sliding average value of the variable during testing.

Mnist_train.py

This program provides the complete training process of the neural network.

Mnist_eval.py

Perform a test on the moving average model.

You can use tf. train. get_checkpoint_state (mnist_train.MODEL_SAVE_PATH) to get the name of the latest model. Actually, you can get all the content of the checkpoint file.

Iii. TensorFlow best practices example

Mnist_inference.py

Import tensorflow as tf INPUT_NODE = 784 OUTPUT_NODE = 10 LAYER1_NODE = 500 def get_weight_variable (shape, regularizer): weights = tf. get_variable ("weights", shape, initializer = tf. truncated_normal_initializer (stddev = 0.1) if regularizer! = None: # Add the regularization item of the weight parameter to the loss set tf. add_to_collection ('losses ', regularizer (weights) return weights def inference (input_tensor, regularizer): with tf. variable_scope ('layer1'): weights = get_weight_variable ([INPUT_NODE, LAYER1_NODE], regularizer) biases = tf. get_variable ("biases", [LAYER1_NODE], initializer = tf. constant_initializer (0.0) layer1 = tf. nn. relu (tf. matmul (input_tensor, weights) + biases) with tf. variable_scope ('layer2'): weights = get_weight_variable ([LAYER1_NODE, OUTPUT_NODE], regularizer) biases = tf. get_variable ("biases", [OUTPUT_NODE], initializer = tf. constant_initializer (0.0) layer2 = tf. matmul (layer1, weights) + biases return layer2

Mnist_train.py

Import OS import tensorflow as tf from tensorflow. examples. tutorials. mnist import input_data import mnist_inference BATCH_SIZE = 100 LEARNING_RATE_BASE = 0.8 bytes = 0.99 REGULARIZATION_RATE = 0.0001 TRAINING_STEPS = 10000 bytes = 0.99 MODEL_SAVE_PATH = "Model_Folder/" MODEL_NAME = "model. ckpt "def train (mnist): # defines the input placeholder x = tf. placeholder (tf. float32, [None, mnist_inference.INPUT_NODE], name = 'x-input') y _ = tf. placeholder (tf. float32, [None, mnist_inference.OUTPUT_NODE], name = 'Y-input') # defines the regularization device and outputs regularizer = tf in the forward process of computing. contrib. layers. l2_regularizer (REGULARIZATION_RATE) y = mnist_inference.inference (x, regularizer) # defines the number of current training turns and the moving average model global_step = tf. variable (0, trainable = False) variable_averages = tf. train. exponentialMovingAverage (MOVING_AVERAGE_DECAY, global_step) variables_averages_op = variable_averages.apply (tf. trainable_variables () # define the loss function cross_entropy = tf. nn. sparse_softmax_cross_entropy_with_logits (logits = y, labels = tf. argmax (y _, 1) cross_entropy_mean = tf. performance_mean (cross_entropy) loss = cross_entropy_mean + tf. add_n (tf. get_collection ('losses ') # defines the exponential attenuation learning rate learning_rate = tf. train. exponential_decay (LEARNING_RATE_BASE, global_step, mnist. train. num_examples/BATCH_SIZE, LEARNING_RATE_DECAY) # define training operations, including model training and moving model operation train_step = tf. train. gradientDescentOptimizer (learning_rate )\. minimize (loss, global_step = global_step) train_op = tf. group (train_step, variables_averages_op) # defines Saver class objects, saves models, and TensorFlow persistence class saver = tf. train. saver () # defines the session and starts the training process with tf. session () as sess: tf. global_variables_initializer (). run () for I in range (TRAINING_STEPS): xs, ys = mnist. train. next_batch (BATCH_SIZE) _, loss_value, step = sess. run ([train_op, loss, global_step], feed_dict = {x: xs, y _: ys}) if I % 1000 = 0: print ("After % d training step (s), loss on training batch is % g. "\ % (step, loss_value) # The global_step parameter of the save method allows saver to be added at the end of the file name of each saved model. save (sess, OS. path. join (MODEL_SAVE_PATH, MODEL_NAME), global_step = global_step) def main (argv = None): mnist = input_data.read_data_sets ("MNIST_data", one_hot = True) train (mnist) if _ name _ = '_ main _': tf. app. run ()

Mnist_eval.py

Import time import tensorflow as tf from tensorflow. examples. tutorials. mnist import input_data import mnist_inference import mnist_train EVAL_INTERVAL_SECS = 10 def evaluate (mnist): with tf. graph (). as_default () as g: # defines the input placeholder x = tf. placeholder (tf. float32, [None, mnist_inference.INPUT_NODE], name = 'x-input') y _ = tf. placeholder (tf. float32, [None, mnist_inference.OUTPUT_NODE], name = 'Y-input') # define the feed dictionary validate_feed = {x: mnist. validation. images, y _: mnist. validation. labels} # loss due to normalization without parameters during testing y = mnist_inference.inference (x, None) # Calculation accuracy rate correct_prediction = tf. equal (tf. argmax (y, 1), tf. argmax (y _, 1) accuracy = tf. performance_mean (tf. cast (correct_prediction, tf. float32) # load the parameter value variable_averages = tf. train. exponentialMovingAverage (mnist_train.MOVING_AVERAGE_DECAY) saver = tf. train. saver (variable_averages.variables_to_restore () # Start a session every second in EVAL_INTERVAL_SECS while True: with tf. session () as sess: ckpt = tf. train. get_checkpoint_state (mnist_train.MODEL_SAVE_PATH) if ckpt and ckpt. model_checkpoint_path: saver. restore (sess, ckpt. model_checkpoint_path) # obtain the number of current iterations in the checkpoint file global_step = ckpt. model_checkpoint_path \. split ('/') [-1]. split ('-') [-1] accuracy_score = sess. run (accuracy, feed_dict = validate_feed) print ("After % s training step (s), validation accuracy = % g" \ % (global_step, accuracy_score) else: print ('no checkpoint file found ') return time. sleep (EVAL_INTERVAL_SECS) def main (argv = None): mnist = input_data.read_data_sets ("MNIST_data", one_hot = True) evaluate (mnist) if _ name _ = '_ main _': tf. app. run ()

The above is all the content of this article. I hope it will be helpful for your learning and support for helping customers.