Libeio performance test

Preface:

I searched for it and it seems that there is no data about libeio's read/write performance testing. Therefore, I wrote a test program to test it. If you have not attempted to write, all interfaces are available. You can fill in the interface as needed. About libeio: this is the library Marc LehmannYou can search for this ox if you are interested.
Test: In the test environment, I use ABC to represent the test environment. I use a certain length a starting from a random reading position in the file to represent the test in my virtual machine. The test environment varies according to the data size in the test directory, it may cause a significant difference. B Indicates testing in Server 1. The testing directory is the root directory that stores more than 200 FLV files. Because the files are small, therefore, fast speed c Indicates testing in Server 2. The testing directory is the root directory of thousands of video data on the real online server. All video data occupies nearly 1 TB of disk space, due to the large file size, there is also the impact of data synchronization at a certain interval, and some other parameters in the system, so the number of test files is 1000. The test result of speed return intersection B is much slower. Test results: A: 1) Select the data of a large file. If there are not many files in the directory, the test structure can reach around 280 m/s. 2) Select a directory containing a compressed package, there are more than 9000 files in the directory, the file is small, and the test speed reaches 31.247337 Mb/s B: This environment is relatively simple, directly paste the test results all file list size: 223src file list size: 223
Dst_file_list_size: 223buf list is full outtest End, read size: 21766.363750 M cost: 83.639529, speed: 260.240152 m/s C: 1) Select a sub-directory of the real video data directory, the number of files is over 300. Test results: Use Pointer dir:/data/data2/test beginall file list size: 329src file list size: 329 dst_file_list_size: 329 test end, read size: 3942.934089 M cost: 14.385984, speed: 274.081640 m/s 2) Select the root directory of all video data, with nearly files. Due to the system relationship, open tens of thousands to test the card master, so select the 1000 files, the test results are as follows: first result: Use Pointer dir:/data/test beginall file list size: 7803src file list size: 7796 dst_file_list_size: 7796 test End, read size: 11804.820662 M cost: 65.706916, speed: 179.658724 m/s second result (due to the system page cache, so it will be large): Use Pointer dir:/data/test beginall file list size: 7803src file list size: 7796 dst_file_list_size: 7796 test End, read size: 11681.930182 M cost: 63.431233, speed: 184.166847 M/syou have new mail in/var/mail/rootdg35_r710_9sd6t2x:/WebService/Server/jiangwenlong/fileio # test code: Dependent Library: libeio, libapr, boost

   #include <apr.h>#include <apr_lib.h>#include <apr_general.h>#include <apr_poll.h>#include <apr_pools.h>#include <eio.h>#include <stdlib.h>#include <sys/types.h>#include <sys/stat.h>#include <fcntl.h>#include <stdlib.h>#include <errno.h>#include <apr_file_io.h>#include <apr_strings.h>#include <boost/filesystem.hpp>#include <vector>void want_poll();void done_poll();void apr_create_poll();void apr_create_pool();void apr_create_pipe();void event_loop();//TODO:read & write function for eio callback int write_cb(eio_req * req);int read_cb(eio_req * req);int open_cb(eio_req * req);int close_cb(eio_req * req);void read_dir();void random_read();void order_read();void random_write();void order_write();static apr_pool_t *p = NULL;static apr_pollset_t *pollset = NULL;static int MAX_WATCH_SIZE = 100;static apr_file_t *readp = NULL;static apr_file_t *writep = NULL;static long  read_size=0;static long  write_size=0;typedef struct _tag_FILE_INFO{int         fd;std::string file_path;intfile_size;bool        is_regular_file;time_t      last_write_time;char*buf[256];intpos ;_tag_FILE_INFO(){fd = 0;file_size = 0;last_write_time = 0;pos = 0;}}FILE_INFO;static char src_path[256] = "../";static char dst_path[256] = "test_dst_data";static std::vector<FILE_INFO>src_file_path_stat;static std::vector<FILE_INFO>src_file_path_list;static std::vector<std::string>dst_file_path_list;//TODO:for eio callback want_pollvoid want_poll(){char c;apr_size_t size=1;if(apr_file_write(writep,&c,&size)!= APR_SUCCESS){printf("write pipe error\n");exit(1);}}//TODO:for eio_callback done_pollvoid done_poll(){char c;apr_size_t size=1;if(apr_file_read(readp,&c,&size)!= APR_SUCCESS){printf("read pipe error\n");}}//TODO:use apr lib create pipevoid apr_create_pipe(){if(apr_file_pipe_create(&readp,&writep,p)!= APR_SUCCESS){abort();}}//TODO:use apr lib create pollsetvoid apr_create_poll(){if(apr_pollset_create(&pollset,MAX_WATCH_SIZE,p,0)!= APR_SUCCESS){abort();}}//TODO:clean all what apr need cleanvoid apr_clean_all(){if(apr_file_close(readp)!= APR_SUCCESS){printf("close pipe read fd error\n");}if(apr_file_close(writep)!= APR_SUCCESS){printf("close pipe write fd error\n");}}//TODO:use apr create poolvoid apr_create_pool(){if(apr_pool_create(&p,NULL) != APR_SUCCESS){abort();}}//TODO:for watch void event_loop(){apr_pollfd_t pollfd;pollfd.desc_type = APR_POLL_FILE;pollfd.reqevents = APR_POLLIN;pollfd.desc.f  = readp;if(apr_pollset_add(pollset,&pollfd)!= APR_SUCCESS){printf("add fd to pollset error\n");exit(1);}while(eio_nreqs()){int num=0;const apr_pollfd_t *outfd=NULL;if(apr_pollset_poll(pollset,-1,&num,&outfd)!= APR_SUCCESS){printf("apr_poll error\n");}if(num != 1){printf("incorrect events\n");}else{int rc = eio_poll();if(rc == -1)printf("eio_poll error:%d\n",eio_poll());}}apr_pollset_remove(pollset,&pollfd);}int write_cb(eio_req * req){int rc = EIO_RESULT(req);if( rc <= 0 ){printf("write file:%s rc:%d,error msg:%s\n",(char*)req->data,rc,strerror(req->errorno));abort();}return 0;}int read_cb(eio_req * req){int rc = EIO_RESULT(req);if ( rc < 0){printf("read file:%s rc:%d error msg:%s\n",(char*)req->data,rc,strerror(req->errorno));abort();}FILE_INFO* info = (FILE_INFO*)req->data;//printf("file:%-20s\tread size:%d\n",info->file_path.c_str(),rc);read_size+=rc;return 0;}int open_cb(eio_req * req){int rc = EIO_RESULT(req);FILE_INFO * info = (FILE_INFO*)req->data;if(rc <= 0){printf("open file :%s ,error msg:%s\n",info->file_path.c_str(),strerror(req->errorno));return 0;abort();}info->fd = rc;//printf("open file fd:%d\n",info->fd);return 0;}int close_cb(eio_req *req){int rc = EIO_RESULT(req);if( rc != 0){printf("close file error,error msg:%s\n",strerror(req->errorno));abort();}return 0;}void read_dir(){boost::filesystem::path spath(src_path);if(!boost::filesystem::is_directory(spath)){printf("error,src_path not directory\n");abort();}boost::filesystem::recursive_directory_iterator rcs_iter(spath);boost::filesystem::recursive_directory_iterator end;//boost::filesystem::directory_iterator rcs_iter(spath);//boost::filesystem::directory_iterator end;for(;rcs_iter != end; ++rcs_iter){FILE_INFO finfo;try{finfo.is_regular_file = boost::filesystem::is_regular_file(*rcs_iter);finfo.last_write_time = boost::filesystem::last_write_time(*rcs_iter);finfo.file_path  = (*rcs_iter).path().string();finfo.file_size  = boost::filesystem::file_size(*rcs_iter);src_file_path_stat.push_back(finfo);//printf("file:%s\n",finfo.file_path.c_str());}catch(...){}}printf("all file list size:%d\n",src_file_path_stat.size());std::vector<FILE_INFO>::iterator f_iter,f_end;f_iter = src_file_path_stat.begin();f_end  = src_file_path_stat.end();for(; f_iter != f_end; ++f_iter){if(f_iter->is_regular_file && f_iter->file_size > 0){src_file_path_list.push_back(*f_iter);boost::filesystem::path temp_path(f_iter->file_path);dst_file_path_list.push_back(temp_path.filename().string());//printf("filename:%s\n",temp_path.filename().string().c_str());}}printf("src file list size:%d\t dst_file_list_size:%d\n",src_file_path_list.size(),dst_file_path_list.size());}void random_read(){srand(time(NULL));bool exit_flag = false;//while(!exit_flag){for(int i=0; i<src_file_path_list.size(); ++i){if(src_file_path_list.at(i).fd == 0){continue;}int temp_size = src_file_path_list.at(i).file_size / 256*1024 ;if(temp_size == 0){continue;}int read_bytes=rand()%(temp_size>100?100*256*1024:100*1024);int offset    =rand()%(1024*1024);offset=(offset+read_bytes)>src_file_path_list.at(i).file_size ? (rand()%100):offset;char *buf = NULL;buf = (char*)apr_palloc(p,read_bytes+1);if(buf == NULL){printf("use apr poll alloc memory error\n");continue;}//printf("read file:%-20s,file size:%-10d,offset:%-5d,length:%-10d\n",//src_file_path_list.at(i).file_path.c_str(),//src_file_path_list.at(i).file_size,offset,read_bytes);src_file_path_list.at(i).buf[src_file_path_list.at(i).pos++] = buf;eio_read(src_file_path_list.at(i).fd,buf,read_bytes,offset,0,read_cb,&src_file_path_list.at(i));if(src_file_path_list.at(i).pos >= 10){printf("buf list is full out\n");exit_flag = true;break;}}//}}int main(int argc,char **argv){apr_initialize();apr_create_pool();apr_create_pipe();apr_create_poll();if(eio_init(want_poll,done_poll)){abort();}if(argc == 2){printf("use pointer dir:%s\n",argv[1]);memset(src_path,0,256);sprintf(src_path,"%s",argv[1]);}printf("test begin\n");read_dir();for(int i = 0; i < src_file_path_list.size(); ++i){if(i > 1000)break;eio_open(src_file_path_list.at(i).file_path.c_str(),O_RDONLY,0644,0,open_cb,&src_file_path_list.at(i));}event_loop();apr_time_t begin = apr_time_now();random_read();event_loop();apr_time_t end = apr_time_now();for(int i = 0; i < src_file_path_list.size(); ++i){if(src_file_path_list.at(i).fd > 0)eio_close(src_file_path_list.at(i).fd,0,close_cb,NULL);}event_loop();double total_size = read_size*1.0/(1024*1024);double cost_time  = (end-begin)/1000000.0;double speed = total_size/cost_time;printf("test end,read size:%lfM cost:%lf,speed:%lfM/s\n",read_size*1.0/(1024*1024),(end-begin)/1000000.0,speed);apr_terminate();return 0;}