Mxnet trains its own dataset and tests

Using the Mxnet training picture class classifier
1, prepare the data:
(1) Create a root directory and then, for each category of pictures to create a subfolder, each type of picture into the corresponding subfolder.
--root:
----Class1
----Class2
......
----CLASSN
A list of the training sets and test sets is generated first, and the commands are as follows:

Python ~/mxnet/tools/im2rec.py--list True--recursive true--train-ratio 0.9 mydata/home/xxx/root/

–list: When you want to generate a list file, this parameter must be set to true to indicate the list file that is currently being generated; The default is to generate the rec file;

–recursive: Recursively iterate through all your datasets, to set to true;

–train-ratio: Used to split your entire dataset into two parts: the training set (train) and the cross validation set (Val), the specific number as a training set, as well as the number of validation set, is determined by this parameter;

–test-ratio: Ibid., divided into training set and test set two parts;

–exts: This is your data suffix (note, here we generally say the picture data), the current mxnet only supports two kinds of picture formats: JPG and JPEG.
After executing this command, you will find that two files are generated: Mydata_train.lst and Mydata_val.lst
(2) to generate the Rec file:

Python ~/mxnet/tools/im2rec.py–num-thread 4–pass-through 1 mydata/home/xxx/root/

–pass-through: Set to 1, that is, skip matrix transformation, or it will error: unknown array type;

MyData is the prefix for generating the. lst file in the first step, which is used to generate the rec;

After executing this command, you will see two files: Mydata_train.rec and Mydata_val.rec

2. Load training data and validate data set:

def get_iterators (Batch_size, data_shape= (3, 224, 224)):
    train = Mx.io.ImageRecordIter (
        path_imgrec         = './ Ld_train/my_images_train.rec ',
        data_name           = ' data ',
        label_name          = ' Softmax_label ',
        batch_size          = batch_size,
        data_shape          = Data_shape,
        Shuffle             = True,
        rand_crop           = True,
        Rand_mirror         = True)
    val = mx.io.ImageRecordIter (
        path_imgrec         = './ld_train/my_images_val.rec ',
        data_name           = ' data ',
        label_name          = ' Softmax_label ',
        batch_size          = batch_size,
        Data_shape          = data_shape,
        rand_crop           = False,
        rand_mirror         = False) return
    (Train, Val)
 train,val=get_iterators (128, (3,128,128)) # Specifies batch_size and picture size.

3. Define Network structure:
Here take ResNet as an example:

"' Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py Original author Wei Wu implemented 
The following paper:kaiming he, Xiangyu Zhang, shaoqing Ren, Jian Sun. "Identity mappings in Deep residual Networks" " Import mxnet as MX import numpy as NP def residual_unit (data, Num_filter, Stride, Dim_match, name, Bottle_neck=true, Bn_ mom=0.9, workspace=256, Memonger=false): "" "Return ResNet unit symbol for building ResNet Parameters--------- -DATA:STR Input Data num_filter:int number of output channels bnf:int bottle NE
        CK channels factor With regard to Num_filter stride:tupe stride used in convolution Dim_match:boolen True means channel number between input and output is the same, otherwise means differ name:str Base
        Name of the operators Workspace:int workspace used in convolution operator "" "If Bottle_neck: # the same as https://gIthub.com/facebook/fb.resnet.torch#notes, a bit difference with origin paper BN1 = Mx.sym.BatchNorm (data=data, fix _gamma=false, Eps=2e-5, momentum=bn_mom, Name=name + ' _bn1 ') Act1 = Mx.sym.Activation (data=bn1, act_type= ' Relu ', n Ame=name + ' _relu1 ') weight = mx.symbol.Variable (name=name + ' _conv1_weight ', dtype=np.float32) weight = m X.symbol.cast (Data=weight, dtype=np.float16) conv1 = Mx.sym.Convolution (Data=act1, Weight=weight, Num_filter=int (n um_filter*0.25), kernel= (1,1), stride= (1,1), pad= (0,0), No_bias=true, Workspace=workspa  CE, Name=name + ' _conv1 ') bn2 = Mx.sym.BatchNorm (Data=conv1, Fix_gamma=false, eps=2e-5, Momentum=bn_mom, Name=name + ' _bn2 ') Act2 = Mx.sym.Activation (data=bn2, act_type= ' Relu ', Name=name + ' _relu2 ') weight = Mx.symbol.Va
        Riable (name=name + ' _conv2_weight ', dtype=np.float32) weight = Mx.symbol.Cast (Data=weight, dtype=np.float16) Conv2 = Mx.sym.ConvolUtion (Data=act2, Weight=weight, Num_filter=int (num_filter*0.25), kernel= (3,3), Stride=stride, pad= (1,1), No_bias=true, Workspace=workspace, Name=name + ' _conv2 ') bn3 = Mx.sym.BatchNorm (Data=conv2, Fix_ Gamma=false, Eps=2e-5, momentum=bn_mom, Name=name + ' _bn3 ') Act3 = Mx.sym.Activation (data=bn3, act_type= ' Relu ', NA Me=name + ' _relu3 ') weight = mx.symbol.Variable (name=name + ' _conv3_weight ', dtype=np.float32) weight = mx . symbol. Cast (Data=weight, dtype=np.float16) conv3 = Mx.sym.Convolution (Data=act3, weight=weight, Num_filter=num_filter, ke Rnel= (1,1), stride= (1,1), pad= (0,0), No_bias=true, workspace=workspace, Name=name + ' _c  Onv3 ') If dim_match:shortcut = Data Else:weight = mx.symbol.Variable (Name=name +  ' _sc_weight ', dtype=np.float32) weight = Mx.symbol.Cast (Data=weight, dtype=np.float16) shortcut = Mx.sym.ConvolutIon (Data=act1, Weight=weight, Num_filter=num_filter, kernel= (1,1), Stride=stride, No_bias=true, Workspace=workspace, name=name+ ' _sc ') if memonger:shortcut._set_attr (mirror_stage= ' True ') return conv3 + shortcut else:bn1 = Mx.sym.BatchNorm (Data=data, Fix_gamma=false, Momentum=bn_mo
        M, eps=2e-5, Name=name + ' _bn1 ') Act1 = Mx.sym.Activation (data=bn1, act_type= ' Relu ', Name=name + ' _RELU1 ') Weight = mx.symbol.Variable (name=name + ' _conv1_weight ', dtype=np.float32) weight = Mx.symbol.Cast (Data=weight, D type=np.float16) Conv1 = Mx.sym.Convolution (Data=act1, Weight=weight, Num_filter=num_filter, kernel= (3,3), stride=
        Stride, pad= (1,1), No_bias=true, workspace=workspace, Name=name + ' _conv1 ') BN2 = Mx.sym.BatchNorm (Data=conv1, Fix_gamma=false, Momentum=bn_mom, eps=2e-5, Name=name + ' _bn2 ') Act2 = MX.SYM.A Ctivation (DATA=BN2, act_Type= ' Relu ', Name=name + ' _relu2 ') weight = mx.symbol.Variable (name=name + ' _conv2_weight ', Dtype=np.float32) Weight = Mx.symbol.Cast (Data=weight, dtype=np.float16) conv2 = Mx.sym.Convolution (Data=act2, Weight=weight, Nu M_filter=num_filter, Kernel= (3,3), stride= (1,1), pad= (1,1), No_bias=true, workspace= Workspace, Name=name + ' _conv2 ') if dim_match:shortcut = Data Else:weight = Mx.s Ymbol.
            Variable (name=name + ' _sc_weight ', dtype=np.float32) weight = Mx.symbol.Cast (Data=weight, dtype=np.float16) Shortcut = Mx.sym.Convolution (Data=act1, Weight=weight, Num_filter=num_filter, kernel= (1,1), Stride=stride, No_b
            Ias=true, Workspace=workspace, name=name+ ' _sc ') if Memonger: Shortcut._set_attr (mirror_stage= ' True ') return conv2 + shortcut def resnet (units, num_stages, filter_list, num _classes, Image_shApe, Bottle_neck=true, bn_mom=0.9, workspace=256, Memonger=false): "" "return resnet symbol of Parameters----- -----units:list number of units in stage num_stages:int number of stage filter_list : List Channel size of each stage num_classes:int ouput size of symbol DATASET:STR Dat
    Aset type, only Cifar10 and imagenet supports Workspace:int workspace used in convolution operator "" Num_unit = len (Units) assert (Num_unit = = num_stages) data = mx.sym.Variable (name= ' data ') data = MX.SYMBOL.C AST (Data=data, dtype=np.float16) data = Mx.sym.BatchNorm (Data=data, Fix_gamma=true, eps=2e-5, momentum=bn_mom, name= ' b N_data ') (Nchannel, height, width) = image_shape weight = mx.symbol.Variable (name= ' conv0_weight '), Dtype=np.float32
        ) Weight = Mx.symbol.Cast (Data=weight, dtype=np.float16) if height <=: # such as Cifar10 BODY = Mx.sym.ConvolutioN (data=data, Weight=weight, num_filter=filter_list[0], kernel= (3, 3), stride= (1,1), pad= (1, 1), No_bias=true, Name= "Conv0", Workspace=workspace) Else: # often expected to be 224 such a s imagenet BODY = mx.sym.Convolution (Data=data, Weight=weight, num_filter=filter_list[0], kernel= (7, 7), stride= (2 , 2), pad= (3, 3), No_bias=true, name= "Conv0", workspace=workspace) BODY = Mx.sym. Batchnorm (Data=body, Fix_gamma=false, eps=2e-5, Momentum=bn_mom, name= ' bn0 ') BODY = mx.sym.Activation (Data=body, a Ct_type= ' Relu ', name= ' relu0 ') BODY = mx.symbol.Pooling (Data=body, Kernel= (3, 3), stride= (2,2), pad= (1,1), Pool_typ E= ' Max ' for I in Range (num_stages): BODY = Residual_unit (body, filter_list[i+1], (1 if i==0 else 2, 1 if i== 0 Else 2), False, name= ' stage%d_unit%d '% (i + 1, 1), Bottle_neck=bottle_neck, Workspace=work
                 Space            Memonger=memonger) for J in Range (units[i]-1): BODY = Residual_unit (body, filter_list[i+1) , (1,1), True, name= ' stage%d_unit%d '% (i + 1, j + 2), Bottle_neck=bottle_neck, workspace =workspace, Memonger=memonger) bn1 = Mx.sym.BatchNorm (Data=body, Fix_gamma=false, eps=2e-5, Momentum=bn_mom, Name= ' BN1 ') RELU1 = Mx.sym.Activation (data=bn1, act_type= ' Relu ', name= ' RELU1 ') # Although kernel are not used Al_pool=true, we should put one pool1 = mx.symbol.Pooling (DATA=RELU1, Global_pool=true, kernel= (7, 7), pool_type= ' avg ' , name= ' pool1 ') flat = Mx.symbol.Flatten (data=pool1) weight = mx.symbol.Variable (name= ' fc1_weight ', dtype=np.float Bias = mx.symbol.Variable (name= ' Fc1_bias ', dtype=np.float32) weight = Mx.symbol.Cast (Data=weight, Dtype=np.flo at16) bias = Mx.symbol.Cast (Data=bias, dtype=np.float16) FC1 = mx.symbol.FullyConnected (Data=flat, Weight=weight, Bias=bias, Num_hidden=nuM_classes, name= ' fc1 ') FC1 = Mx.symbol.Cast (DATA=FC1, Dtype=np.float32) return Mx.symbol.SoftmaxOutput (DATA=FC1, n Ame= ' Softmax ') def get_symbol (num_classes, Num_layers, Image_shape, conv_workspace=256, **kwargs): "" "adapted fr Om https://github.com/tornadomeet/ResNet/blob/master/train_resnet.py Original author Wei Wu "" Image_shape =
    [Int (L) for L in Image_shape.split (', ')] (Nchannel, height, width) = image_shape if height <= 28:num_stages = 3 if (num_layers-2)% 9 = 0
            and num_layers >= 164:per_unit = [(num_layers-2)//9] Filter_list = [16, 64, 128, 256] Bottle_neck = True elif (num_layers-2)% 6 = 0 and Num_layers < 164:per_unit = [(num_layers-2 ) filter_list = [//6] Bottle_neck = False else:raise Valueerro R ("No experiments done in Num_layers {}, can do it youself". Format (num_layers)) units = PEr_unit * num_stages else:if num_layers >= 50:filter_list = [64, 256, 512, 1024, 2048]
        Bottle_neck = True Else:filter_list = [128, 256, bottle_neck] = False
            Num_stages = 4 if num_layers = = 18:units = [2, 2, 2, 2] elif num_layers = = 34:
            units = [3, 4, 6, 3] elif num_layers = = 50:units = [3, 4, 6, 3] elif num_layers = 101: units = [3, 4, 3] elif num_layers = = 152:units = [3, 8, 3] elif Num_layer s = = 200:units = [3, 3] elif num_layers = = 269:units = [3, M, 8] El Se:raise ValueError ("No experiments done in Num_layers {}, can do it youself". Format (num_layers)) re
                  Turn ResNet (units = units, Num_stages = num_stages, filter_list = Filter_list, Num_clAsses = num_classes, Image_shape = image_shape, Bottle_neck = Bottle_neck, Workspace = conv_workspace) resnet18=get_symbol (2,18, ' 3,128,128 ')

4, training and save the model:

# Construct a callback function to save checkpoints
Model_prefix = ' resnet18 ' checkpoint
= Mx.callback.do_checkpo Int (model_prefix)

mod = mx.mod.Module (SYMBOL=RESNET18,CONTEXT=MX.GPU (0))
Mod.fit (train,
        eval_data= Val,
        optimizer= ' sgd ',
        optimizer_params={' learning_rate ': 0.001, ' momentum ': 0.9},
        eval_metric= ' acc ',
        num_epoch=30, 
        epoch_end_callback=checkpoint)

5. Load Model Ready test:

Sym, arg_params, aux_params = Mx.model.load_checkpoint (' resnet18 ',)
mod = Mx.mod.Module (Symbol=sym, context= Mx.gpu (0), Label_names=none)
mod.bind (For_training=false, data_shapes=[(' Data ', (1,3,128,128))], 
         label_ Shapes=mod._label_shapes)
mod.set_params (Arg_params, Aux_params, Allow_missing=true)

6, the number of tests to predict:

%matplotlib inline import matplotlib.pyplot as PLT import cv2 import numpy as NP # define a Simpl
    e Data batch from collections import namedtuple batch = namedtuple (' Batch ', [' Data ']) def get_image (Path, show=false): # download and show the image img = Cv2.cvtcolor (cv2.imread (path), Cv2.
    COLOR_BGR2RGB) if IMG is None:return None if Show:plt.imshow (IMG) plt.axis ("Off")
    # Convert into format (batch, RGB, width, height) img = Cv2.resize (IMG, (128, 128)) img = np.swapaxes (IMG, 0, 2)  img = Np.swapaxes (IMG, 1, 2) img = Img[np.newaxis,:] return img def predict (PATH): img = get_image (PATH, SHOW=TRUE) # Compute the Predict probabilities Mod.forward (Batch ([Mx.nd.array (IMG))) prob = Mod.get_outputs (
        ) [0].asnumpy () # print the top-5 prob = Np.squeeze (prob) a = Np.argsort (prob) [:: -1] for I in A[0:5]: Print ((prob[i)) Predict ('/home/test_images/cat.jpg ')