Mnist Data Format conversion implementation in Windows Caffe

Caffe source code in Src/caffe/caffe/examples/mnist/convert_mnist_ The implementation code provided by Data.cpp does not run directly under Windows, which is rewritten on the basis of the source code, allowing it to run directly on the Windows 64-bit, overwriting the codes as follows:

#include "stdafx.h" #include <gflags/gflags.h> #include <glog/logging.h> #include <google/protobuf/ text_format.h> #include <leveldb/db.h> #include <leveldb/write_batch.h> #include <lmdb.h># Include <stdint.h> #include <sys/stat.h> #include <fstream>//Nolint (Readability/streams) #include  <string> #include <iostream> #include "caffe/proto/caffe.pb.h" using namespace Caffe; Nolint (build/namespaces) The data type in the Using std::string;//gflags, C + + string, where to specify whether to convert to Lmdb or leveldbdefine_string (backend , "Lmdb", "the backend for storing the result"); uint32_t Swap_endian (uint32_t val) {val = (val << 8) & 0XFF00FF 00) | ((Val >> 8) & 0XFF00FF); return (Val << 16) | (Val >> 16);} void Convert_dataset (const char* image_filename, const char* label_filename,const char* db_path, const string& DB_BAC Kend) {//Open filesstd::ifstream image_file (image_filename, std::ios::in | std::ios::binary); Std::ifstream label_file (Label_fIlename, Std::ios::in | Std::ios::binary); CHECK (image_file) << "Unable to open file" << image_filename; CHECK (Label_file) << Unable to open file << label_filename;//Read the Magic and the Meta datauint32_t Magi c;uint32_t num_items;uint32_t num_labels;uint32_t rows;uint32_t cols;//reads the first n bytes of a file, gets the number of images, image width, image height Image_file.read ( Reinterpret_cast<char*> (&magic), 4); magic = Swap_endian (Magic); Check_eq (Magic, 2051) << "Incorrect image file magic."; Label_file.read (reinterpret_cast<char*> (&magic), 4); magic = Swap_endian (Magic); Check_eq (Magic, 2049) << "incorrect label file Magic."; Image_file.read (reinterpret_cast<char*> (&num_items), 4); num_items = Swap_endian (num_items); label_ File.read (reinterpret_cast<char*> (&num_labels), 4); num_labels = Swap_endian (num_labels); Check_eq (Num_items, num_labels); Image_file.read (Reinterpret_cast<char*> (&rows), 4); rows = Swap_endian ( rows); Image_file.read (REINTERPRET_CAST&LT;CHAR*&Gt; (&cols), 4); cols = Swap_endian (cols);//Lmdbmdb_env *mdb_env; MDB_DBI mdb_dbi; Mdb_val Mdb_key, Mdb_data; Mdb_txn *mdb_txn;//leveldbleveldb::D b* DB = null;leveldb::options options;options.error_if_exists = True;o ptions.create_if_missing = True;options.write_buffer_size = 268435456;leveldb::writebatch* Batch = NULL;//Open dbif (db _backend = = "Leveldb") {//Leveldblog (INFO) << "Opening leveldb" << db_path;leveldb::status Status = Leveld B::D b::open (Options, Db_path, &db); CHECK (Status.ok ()) << "Failed to open leveldb" << db_path<< ". Is it already existing? "; Batch = new Leveldb::writebatch ();} else if (db_backend = = "Lmdb") {//Lmdbint RC; LOG (INFO) << "Opening lmdb" << db_path;//create the specified storage directory//check_eq (mkdir (Db_path, 0744), 0) std::string strpath = s Td::string (Db_path); std::string delpath = "RM-RF" + Strpath;system (Delpath.c_str ()); strpath = "mkdir" + strpath;system ( Strpath.c_str ());//check_eq (System (STRPATH.C_STR ()), 0) << "MKDIR "<< db_path <<" failed ";//Create Lmdb Database Check_eq (mdb_env_create (&mdb_env), mdb_success) <<" mdb_ Env_create failed ";//check_eq (Mdb_env_set_mapsize (mdb_env, 1099511627776), mdb_success) <<" mdb_env_set_ Mapsize failed ";//1tbcheck_eq (Mdb_env_set_mapsize (mdb_env, 107374182), mdb_success) <<" Mdb_env_set_mapsize Failed ";//100mbcheck_eq (Mdb_env_open (mdb_env, Db_path, 0, 0664), mdb_success) <<" Mdb_env_open failed "; Check_eq (Mdb_txn_begin (mdb_env, NULL, 0, &mdb_txn), mdb_success) << "Mdb_txn_begin failed"; Check_eq (Mdb_open (MDB_TXN, NULL, 0, &AMP;MDB_DBI), mdb_success) << "Mdb_open failed. Does The Lmdb already exist? ";} else {LOG (FATAL) << "Unknown db backend" << db_backend;} Storing to Dbchar label;char* pixels = new Char[rows * Cols];int count = 0;const int kmaxkeylength = 10;char key_cstr[k Maxkeylength];string value;datum Datum; Caffe Data Class Datum.set_channels (1);d atum.set_height (rows);d atum.set_width (cols); LOG (INFO) &LT;&LT "A total of" << num_items << "items."; LOG (INFO) << rows: << rows << Cols: << cols;//writes data to Lmdb or LEVELDB database for (int item_id = 0; item_id < Num_items; ++item_id) {image_file.read (pixels, rows * cols), Label_file.read (&label, 1);d atum.set_data (pixels, rows*cols); Datum.set_label (label);//snprintf (Key_cstr, Kmaxkeylength, "%08d", item_id); int ret = _snprintf (KEY_CSTR, Kmaxkeylength, "%08d", item_id); if (ret = = Kmaxkeylength | | Ret < 0) {printf ("warning"); key_cstr[kmaxkeylength-1] = 0;} Datum. Serializetostring (&value); string Keystr (KEY_CSTR);//Put in dbif (db_backend = "Leveldb") {//Leveldbbatch->put (keystr, value);} else if (db_backend = = "Lmdb") {//lmdbmdb_data.mv_size = Value.size (); mdb_data.mv_data = Reinterpret_cast<void*> ( &value[0]); mdb_key.mv_size = Keystr.size (); mdb_key.mv_data = reinterpret_cast<void*> (&keystr[0]); Check_eq (Mdb_put (Mdb_txn, MDB_DBI, &mdb_key, &mdb_data, 0), mdb_success) << "Mdb_put failed";} else {LOG (FATAL) << "Unknown db backend" << db_backend;} if (++count% = = 0) {//Commit txnif (Db_backend = = "Leveldb") {//Leveldbdb->write (Leveldb::writeoptions (), Bat CH);d elete Batch;batch = new Leveldb::writebatch ();} else if (db_backend = = "Lmdb") {//Lmdbcheck_eq (Mdb_txn_commit (MDB_TXN), mdb_success) << "Mdb_txn_commit failed"; Check_eq (Mdb_txn_begin (mdb_env, NULL, 0, &mdb_txn), mdb_success) << "Mdb_txn_begin failed";} else {LOG (FATAL) << "Unknown db backend" << Db_backend;}}} Write the last Batchif (count%!! = 0) {if (Db_backend = = "Leveldb") {//Leveldbdb->write (leveldb::writeoption S (), batch);d elete Batch;delete db;} else if (db_backend = = "Lmdb") {//Lmdbcheck_eq (Mdb_txn_commit (MDB_TXN), mdb_success) << "Mdb_txn_commit failed"; m Db_close (mdb_env, MDB_DBI); Mdb_env_close (mdb_env);} else {LOG (FATAL) << "Unknown db backend" << db_backend;} LOG (ERROR) << "processed" << count << "files.";} delete[] pixels;} int main (int argc, char* argv[]) {#ifndef gflags_gflags_h_namespace GFLAGS = google; #endifargc = 4; #ifdef _debugargv[0] = " E:/gitcode/caffe/lib/dbg/x86_vc12/tools_convert_mnist_data[dbg_x86_vc12].exe "; #elseargv [0] =" E:/GitCode/Caffe/ Lib/rel/x86_vc12/tools_convert_mnist_data[rel_x86_vc12].exe "; #endif//argv[1] =" E:/gitcode/caffe/src/caffe/caffe /data/mnist/t10k-images.idx3-ubyte ";//argv[2] =" e:/gitcode/caffe/src/caffe/caffe/data/mnist/ T10k-labels.idx1-ubyte ";//argv[3] =" e:\\gitcode\\caffe\\src\\caffe\\caffe\\data\\mnist\\lmdb\\test "; argv[1] =" E :/gitcode/caffe/src/caffe/caffe/data/mnist/train-images.idx3-ubyte "; argv[2] =" e:/gitcode/caffe/src/caffe/caffe/ Data/mnist/train-labels.idx1-ubyte "; argv[3] =" E:\\gitcode\\caffe\\src\\caffe\\caffe\\data\\mnist\\lmdb\\train "  ;//used to set Usage description Gflags::setusagemessage ("This script converts the MNIST datasets to\n" "The LMDB/LEVELDB format used by Caffe To load data.\n "" usage:\n "" Convert_mnist_data [FLAGS] Input_image_file input_label_file "" output_db_file\n "" The MNIST dataset could be downloaded at\n "" Http://yann.lecun . com/exdb/mnist/\n "" should gunzip them after downloading, "" or directly use data/mnist/get_mnist.sh\n ");// Parse command-line arguments gflags::P arsecommandlineflags (&AMP;ARGC, &AMP;ARGV, true);//Gets the value of the flag parameter backend const string& Db_backend = Flags_backend;if (argc! = 4) {//Output Usage description gflags::showusagewithflagsrestrict (argv[0], "examples/mnist/convert_mnist_ Data ");} else {//Set the log file name in the "File name" field//initgooglelogging at least once per process, otherwise the log files google::initgooglelogging (argv[0]) will not be generated; Convert_ DataSet (Argv[1], argv[2], argv[3], db_backend);} Std::cout << "ok!" << Std::endl;return 0;}

Copyright NOTICE: This article for Bo Master original article, without Bo Master permission not reproduced.

Mnist Data Format conversion implementation in Windows Caffe