Caffe of the Neural network framework (convert picture to LEVELDB format)
This should be the older version of the Caffe, directly take it should not be used, but you can refer to the next
Caffe in charge of the entire network input datalayer is read from the Leveldb, is a Google implementation of a very efficient KV database.
Therefore, we must first turn the data into LEVELDB format for training the network.
What I do here is to turn all the pictures of a folder into a leveldb format.
Tool using the command grid format: Convert_imagedata src_dir dst_dir attach_dir channel width height
Example:./convert_imagedata.bin/home/linger/imdata/collar_train//home/linger/linger/testfile/crop_train_db//home/ LINGER/LINGER/TESTFILE/CROP_TRAIN_ATTACHMENT/3 50 50
Source:
[CPP]View Plaincopy
- #include <google/protobuf/text_format.h>
- #include <glog/logging.h>
- #include <leveldb/db.h>
- #include <stdint.h>
- #include <fstream>//Nolint (Readability/streams)
- #include <string>
- #include <set>
- #include <stdio.h>
- #include <string.h>
- #include <stdlib.h>
- #include <dirent.h>
- #include <sys/stat.h>
- #include <unistd.h>
- #include <sys/types.h>
- #include "Caffe/proto/caffe.pb.h"
- #include <opencv2/highgui/highgui.hpp>
- #include <opencv2/highgui/highgui_c.h>
- #include <opencv2/imgproc/imgproc.hpp>
- Using Std::string;
- Using namespace std;
- Set<string> All_class_name;
- Map<string,int> class2id;
- /**
- * Path: Directory
- * Files: vector for saving file names
- * R: Whether you need to traverse subdirectories
- * Return: File name, not including path
- */
- void List_dir (const char *path,vector<string>& files,bool R = false)
- {
- DIR *pdir;
- struct dirent *ent;
- Char childpath[512];
- Pdir = opendir (path);
- memset (Childpath, 0, sizeof (Childpath));
- While ((ent = Readdir (pdir)) = NULL)
- {
- if (Ent->d_type & Dt_dir)
- {
- if (strcmp (Ent->d_name, ".") = = 0 | | strcmp (ent->d_name, "..") = = 0)
- {
- continue;
- }
- if (r) //If you need to traverse subdirectories
- {
- sprintf (Childpath, "%s/%s", Path, ent->d_name);
- List_dir (Childpath,files);
- }
- }
- Else
- {
- Files.push_back (Ent->d_name);
- }
- }
- Sort (Files.begin (), Files.end ()); //Sort
- }
- String Get_classname (String path)
- {
- int index = path.find_last_of (' _ ');
- return path.substr (0, index);
- }
- int Get_labelid (String fileName)
- {
- String class_name_tmp = Get_classname (fileName);
- All_class_name.insert (CLASS_NAME_TMP);
- Map<string,int>::iterator name_iter_tmp = Class2id.find (class_name_tmp);
- if (name_iter_tmp = = Class2id.end ())
- {
- int id = class2id.size ();
- Class2id.insert (Name_iter_tmp, Std::make_pair (class_name_tmp, id));
- return ID;
- }
- Else
- {
- return name_iter_tmp->second;
- }
- }
- void Loadimg (string path,char* buffer)
- {
- Cv::mat img = cv::imread (path, cv_load_image_color);
- String Val;
- int rows = img.rows;
- int cols = Img.cols;
- int pos=0;
- For (int c = 0; c < 3; C + +)
- {
- For (int row = 0; row < rows; row++)
- {
- For (int col = 0; col < cols; col++)
- {
- Buffer[pos++]=img.at<cv::vec3b> (Row,col) [C];
- }
- }
- }
- }
- void Convert (String imgdir,string outputdb,string attachdir,int channel,int width,int height)
- {
- LEVELDB::D b* DB;
- Leveldb::options Options;
- Options.create_if_missing = true;
- Options.error_if_exists = true;
- Caffe::D Atum Datum;
- Datum.set_channels (channel);
- Datum.set_height (height);
- Datum.set_width (width);
- int image_size = channel*width*height;
- Char buffer[image_size];
- String value;
- CHECK (leveldb::D b::open (Options, Outputdb, &db). OK ());
- Vector<string> filenames;
- List_dir (Imgdir.c_str (), filenames);
- String img_log = attachdir+"Image_filename";
- Ofstream WriteFile (Img_log.c_str ());
- For (int i=0;i<filenames.size (); i++)
- {
- String Path= imgdir;
- Path.append (Filenames[i]); //Calculate the absolute path
- Loadimg (Path,buffer);
- int labelid = Get_labelid (Filenames[i]);
- Datum.add_label (Labelid);
- Datum.set_data (buffer,image_size);
- Datum. Serializetostring (&value);
- snprintf (buffer, image_size, "%05d", I);
- printf ("\nclassid:%d classname:%s abspath:%s", Labelid,get_classname (Filenames[i]). C_STR (), Path.c_str ());
- Db->put (Leveldb::writeoptions (), string (buffer), value);
- //printf ("%d%s\n", i,filenames[i].c_str ());
- ASSERT (Writefile.is_open ());
- writefile<<i<< "" <<filenames[i]<<"\ n";
- }
- Delete db;
- Writefile.close ();
- Img_log = attachdir+"Image_classname";
- Writefile.open (Img_log.c_str ());
- Set<string>::iterator iter = All_class_name.begin ();
- While (iter! = All_class_name.end ())
- {
- ASSERT (Writefile.is_open ());
- writefile<< (*iter) <<"\ n";
- //printf ("%s\n", (*iter). C_STR ());
- iter++;
- }
- Writefile.close ();
- }
- int main (int argc, char** argv)
- {
- if (argc < 6)
- {
- LOG (ERROR) << "Convert_imagedata src_dir dst_dir attach_dir channel width height";
- return 0;
- }
- ./convert_imagedata.bin/home/linger/imdata/collartest//home/linger/linger/testfile/dbtest//home/linger/ LINGER/TESTFILE/TEST_ATTACHMENT/3 250 250
- //./convert_imagedata.bin/home/linger/imdata/collar_train//home/linger/linger/testfile/crop_train_db//Home /LINGER/LINGER/TESTFILE/CROP_TRAIN_ATTACHMENT/3
- Google::initgooglelogging (Argv[0]);
- String src_dir = Argv[1];
- String src_dst = Argv[2];
- String attach_dir = Argv[3];
- int channel = Atoi (Argv[4]);
- int width = atoi (argv[5]);
- int height = atoi (argv[6]);
- //for Test
- /*
- Src_dir = "/home/linger/imdata/collartest/";
- SRC_DST = "/home/linger/linger/testfile/dbtest/";
- Attach_dir = "/home/linger/linger/testfile/";
- Channel = 3;
- width = 250;
- Height = 250;
- */
- Convert (src_dir,src_dst,attach_dir,channel,width,height);
- }
Caffe of the Neural network framework (convert picture to LEVELDB format)
