#!/usr/bin/env python#-*-coding:utf-8-*-import struct# from BP import *from datetime import datetime# Data Loader base classes class Loa Der (object): def __init__ (self, Path, count): ' ' ' init loading Path: Data file path count: Number of samples in file ' ' Self.path = path Self.count = count def get_file_content (self ): ' ' ' read file content ' ' ' &NB Sp f = open (Self.path, ' RB ') content = F.read () Print content[:20]  ; F.close () return content def to_int (self,h): &nbs P return Struct.unpack (' B ', h) [0] # image Data Loader class Imageloader (Loader): def get_picture ( Self, Content, index): ' ' internal functions, get images from files ' ' ' &NBSP ; start = Index * * * + 16 picture = [] # print (content[16] ) for I in range: picture.append ([]) for J in range: Picture[i].append (# In python2.7, the red font part is right, but in python3.x, the blue font is right Self.to_int (Content[start + i * + J-1:start + i * + j])) &NB Sp Self.to_int (Content[start + i * + j])) return picture def get_one_sample (self, picture): ' intrinsics, converting images into sample input vectors ' ' &NBsp Sample = [] for I in range: for J in range: &NB Sp sample.append (Picture[i][j]) return Sample&nbs P def load (self): "' ' Load data file, get all sample input vector &N Bsp " content = self.get_file_content () Data_set = [] &NB Sp for index in range (Self.count): Data_set.append ( &NBSP ; Self.get_one_sample ( Self . get_picture (content, index))) return data_set# tag Data Loader class Labelloader (Loader): def load (self): "' ' Load data file, get all samples tagged vector &N Bsp ‘‘‘ content = self.get_file_content () # print content[:15] labels = [] for index in range (Self.count): #在python2.7, the Red font part is right, but in python3.x , the blue font is right Labels.append (self.norm (content[index + 7 :index + 8)) &NB Sp Labels.append (Self.norm (Content[index + 8)) return labels &N Bsp def norm (self, label): ' ' intrinsics, converting a value to a 10-D tag vector & nbsp " Label_vec = [] # print (' label is \ n ') # print (label[:20]) Label_value = self.to_int (label) for I in range (Ten): if i = = label_value: Label_vec. Append (0.9) else: LABEL_V Ec.append (0.1) return label_vecdef Get_training_data_set (): "' ' Get training data set ' filename1 = R ' E:\workspace\pythonpaper\importment\dataset\ Train-images.idx3-ubyte ' filename2 = R ' E:\workspace\pythonpaper\importment\dataset\ Train-labels.idx1-ubyte ' Image_loader = Imageloader (filename1, 60000) Label_loader = Labelloader (filename2, 60000) return image_loader.load (), Label_loader.load () def get_test_data_set (): ' get test data set ' ' Filename3 = R ' E:\workspace\pythonpaper\importment\ Dataset\t10k-images.idx3-ubyte ' filename4 = R ' E:\workspace\pythonpaper\importment\dataset\ T10k-labels.idx1-ubyte ' Image_loader = Imageloader (Filename3, 10000) Label_loader = Labelloader (Filename4, 10000) return image_loader.load (), Label_loader.load () def train_and_evaluate (): Train_data_set, train_labels = Get_training_data_set () Test_data_set, test_labels = Get_test_ Data_set () # print ' [DataSet train:]\n ' # print train_data_set[:10] if __name__ = = ' __main_ _ ': train_and_evaluate () 1, mnist Data set format of the introductionThe above code is an example of a tutorial I refer to, which itself is implemented with python2.7, but, for some reason, I use the python3.5 environment, in the implementation of this code, there are some problems, for this, I also explored a bit. Mnist data set is an IDX file format, downloaded from the Internet is four compressed files, two training samples of compressed files, two test samples of the compressed files, before the import code needs to extract them, extracted files are idx3-ubyte suffix file idx files, This file cannot be opened directly, so we need to write the program to process it into what we need. The training set IMAGE FILE (Train-images-idx3-ubyte) is described as an example of the train-images-idx3-ubyte of the Mnist dataset:[ offset] [type] [value] [description]
0000-bit integer 0x00000803 (2051) Magic number
0004-bit integer 60000 number of images
0008-bit integer number of rows
0012-bit integer number of columns
0016 unsigned byte?? Pixel
0017 unsigned byte?? Pixel
........ xxxx unsigned byte?? Pixel32bit is to say this data Book 32 bits, 8 bits =1b (1 bytes), so 32 bit =4b=4byte, we really want to read out is the value of this column, but 0000-0015 of the data is not what we need, the first 4B is the number of magic, The second 4B is how many images the file contains, and the third 4B is the number of lines of an image, and the fourth 4B is the number of columns that an image has, and a sample image of Mnist is 28*28. From 0016 onwards, we need the image content, 28*28=784, that is, we need 784 B to read an image, in the 0016 below the description, write is pixel, which is the meaning of pixels, that is, a pixel is a 1byte= 1 B, for example, with 1B (one byte) of the binary representation of a decimal 3, the binary is 0000 0011, in hexadecimal notation 3, is the \x03,3 abbreviation is ETX. So 784 pixels, 784 lines is an image sample. The following is the Mnist data set
train-labels-idx1-ubyte文件的结构TRAINING SET LABEL FILE (train-labels-idx1-ubyte):
[offset] [type] [value] [description]
0000 32 bit integer 0x00000801(2049) magic number (MSB first)
0004 32 bit integer 60000 number of items
0008 unsigned byte ?? label
0009 unsigned byte ?? label
........xxxx unsigned byte?? Label
train-labels-idx1-ubyte文件的结构的读法和Train-images-idx3-ubyte the same, the first 8 bytes are not the contents of the label, starting from 0008 is the content of a label, and by observing the offset and description fields of this table can be found, A byte is a label.2, pixel and binary, hexadecimal relationships, and the different output of print in PythonLet's start by explaining the binary and 16 binary and decimal relationships, and their abbreviated relationship ASCII control characters
| Binary
decimal |
hexadecimal |
Abbreviations |
notation that can be displayed |
name/meaning |
| 0000 0000 |
0 |
00 |
NUL |
? |
Null character (NULL) |
| 0000 0001 |
1 |
01 |
SOH |
? |
Title Start |
| 0000 0010 |
2 |
02 |
STX |
? |
This article begins |
| 0000 0011 |
3 |
03 |
ETX |
? |
This article concludes |
| 0000 0100 |
4 |
04 |
EOT |
? |
Transfer end |
| 0000 0101 |
5 |
05 |
ENQ |
? |
Request |
| 0000 0110 |
6 |
06 |
Ack |
? |
Confirm response |
| 0000 0111 |
7 |
07 |
BEL |
? |
Bell |
| 0000 1000 |
8 |
08 |
Bs |
? |
Backspace |
| 0000 1001 |
9 |
09 |
HT |
? |
Horizontal position Symbol |
| 0000 1010 |
10 |
0A |
LF |
? |
Line Break key |
| 0000 1011 |
11 |
0B |
Vt |
? |
Vertical positioning symbols |
| 0000 1100 |
12 |
0C |
Ff |
? |
Page Change key |
Read from open (filepath, ' RB ') is binary content, print Content[:5], display the first 5 elements of the content, an element is a pixel, so there are 5 pixels, and a pixel in a binary, a binary is a byte, A byte is a hexadecimal. A small example can further illustrate that an int contains 4 bytes, and a byte is a form of \x14. >>> a=20>>> b=400>>> t=struct.pack (' II ', A, b) >>> T ' \x14\x00\x00\x00\x90\x01\x00 \x00 ' >>> len (t) 8>>> type (a) <type ' int ' >a is of type int, the ' II ' in Pack (' II ', b) is the format, an I corresponds to an int, there are two I, corresponds to two int, an int of type A, accounting for 4 bytes (\x14\x00\x00\x00), Len output is a byte \x14 is one, all 8 \x such, Len (t) is 83, the introduction of the structA=20,b=400struct There are three methods, the Pack (Fmt,val) method is to convert the Val data in the format of FMT to binary data, T=struct.pack (' II ', A, b), convert a, B to binary form ' \x14\x00\ X00\x00\x90\x01\x00\x00 'The Unpack (Fmt,val) method is to convert the binary data to Python-readable data in the format of FMT, unpack (' II ', A, A, b), and convert a, B to 20,400
struct.unpack_from(
‘>IIII‘
, buf , index)‘>IIII‘是说使用大端法从index的位置读取4个unsinged int324. python2.7 and python3.5 problems with the format of the Mnist datasetIn python2.7, the content output is a 20-binary abbreviation, but in python3.5, print content[:20] outputs 20 hexadecimal digits. In python2.7, in Struct.unpack (' B ', byte) content[start+i*28+j], you can run, but in Python3, here you need to write [start+i*28+j-1:start+i* 28+J] to run successfully
Mnist format descriptions, as well as the differences in reading mnist datasets in python3.x and Python 2.x
