How to generate and read TensorFlow tfrecords files

This article mainly introduces the TensorFlow Tfrecords file generation and reading methods, has a certain reference value, now share to everyone, have the need for friends can refer to

TensorFlow provides a tfrecords format to store data uniformly, in theory, tfrecords can store any form of data.

The data in the Tfrecords file is stored in the format tf.train.Example Protocol buffer. The following code gives the definition of tf.train.Example.

Message Example {   Features Features = 1;}; message Features {   map<string, feature> Feature = 1;}; message Feature {   oneof kind {   byteslist bytes_list = 1;   Floatlist float_list = 2;   Int64list int64_list = 3; } };

Here's how to build and read the Tfrecords file:

First introduce the Tfrecords file generation, directly on the code:

From random import shuffle import numpy as NP import glob import tensorflow as tf import cv2 import sys import OS # because I'm faking it. Is the CPU version, run up will have ' warning ', the solution into the next, the eye is not visible for net ~ os.environ[' tf_cpp_min_log_level '] = ' 2 ' shuffle_data = True image_path = '/path/ To/image/*.jpg ' # Gets the path of all pictures under that path, type (ADDRS) = List Addrs = Glob.glob (image_path) # Tag data for specific analysis, type (labels) = List Lab Els = ... # here is the order of the scrambled data if Shuffle_data:c = List (Zip (Addrs, labels)) shuffle (c) Addrs, labels = zip (*c) # split the dataset on demand Train_addrs = Addrs[0:int (0.7*len (Addrs))] Train_labels = Labels[0:int (0.7*len (labels))] Val_addrs = Addrs[int (0.7*len (Addrs)): Int (0.9*len (Addrs))] Val_labels = Labels[int (0.7*len (labels)): Int (0.9*len (labels))] Test_addrs = Addrs[int ( 0.9*len (Addrs)):] Test_labels = Labels[int (0.9*len (labels)):] # above is not get the address of the image, the following function is to obtain the image from the address def load_image (addr ): # A function to Load image img = cv2.imread (addr) img = Cv2.resize (IMG, (224, 224), Interpolation=cv2. Inter_cubic) img = Cv2.cvtcolor (IMG, Cv2. Color_bgr2RGB) # Here/255 is to normalized the pixel value to [0,1] img = img/255. img = Img.astype (NP.FLOAT32) return IMG # converts data to the corresponding attribute Def _int64_feature (value): Return Tf.train.Feature (int64_list= Tf.train.Int64List (Value=[value]) def _bytes_feature (value): Return Tf.train.Feature (bytes_list= Tf.train.BytesList (Value=[value]) def _float_feature (value): Return Tf.train.Feature (float_list= Tf.train.FloatList (Value=[value]) # The following paragraph begins to write data to the Tfrecods file Train_filename = '/path/to/train.tfrecords ' # output file address # Create a Writer to write Tfrecords file writer = tf.python_io. Tfrecordwriter (Train_filename) for I in Range (len (Train_addrs)): # This is a write operation visualized if not I% 1000:print (' train data : {}/{} '. Format (i, Len (Train_addrs))) Sys.stdout.flush () # loading picture img = Load_image (train_addrs[i]) label = Train_ Labels[i] # Create an attribute (feature) feature = {' Train/label ': _int64_feature (label), ' Train/image ': _bytes_feature (TF . Compat.as_bytes (Img.tostring ())} # Creates a example protocol buffer example = Tf.train.Example (Features=tf.train.features (feature=feature)) # writes the above Example protocol buffer to the file Writer.write (Example. Serializetostring ()) Writer.close () Sys.stdout.flush ()

The above only describes the generation of train.tfrecords files, the rest of the validation,test extrapolate bar.

The following describes the read of the Tfrecords file:

Import TensorFlow as TF import numpy as NP import matplotlib.pyplot as PLT import os os.environ[' tf_cpp_min_log_level '] = ' 2 ' data_path = ' train.tfrecords ' # tfrecords file address with TF. Session () as Sess: # defines feature first, which is consistent with the time it was created feature = {' Train/image ': TF. Fixedlenfeature ([], tf.string), ' Train/label ': TF. Fixedlenfeature ([], Tf.int64)} # Create a queue to maintain the input file list filename_queue = Tf.train.string_input_producer ([Data_path], Num_e Pochs=1) # defines a reader that reads the next record reader = TF. Tfrecordreader () _, Serialized_example = Reader.read (filename_queue) # Parse a record read in features = Tf.parse_single_exa Mple (Serialized_example, Features=feature) # Parses a string into an image corresponding to the pixel group image = Tf.decode_raw (features[' train/image '), TF.FLOAT3 2) # Convert the label to int32 label = Tf.cast (features[' Train/label '), Tf.int32) # Here the picture is restored to the original dimension image = Tf.reshape (image, [2 24, 224, 3]) # You can also do some other preprocessing .... # here is the Create order random batches (function does not understand self-Baidu) images, labels = Tf.train.shuffle_batch ([Image, LA BEL], Batch_size=10, capacity=30, min_after_dequeue=10) # Initialize INIT_OP = Tf.group (Tf.global_variables_initializer (), Tf.local_vari Ables_initializer ()) Sess.run (INIT_OP) # Start multithreading input data coord = Tf.train.Coordinator () threads = Tf.train.start_queu E_runners (Coord=coord) .... #关闭线程 coord.request_stop () coord.join (threads) sess.close ()