Programming Skills for Mass Data Processing (C ++ code)

Moved to: http://www.wypblog.com/archives/158

The solution is as follows:

There are two files in the local disk: file1 and file2. Each file contains 5 million random integers (which can be repeated). The maximum value is 2147483648, which is an int value range. Write programs are required to output integers in both files to a new file.

Requirements:
1. The running time of the program cannot exceed 5 seconds.
2. No Memory leakage.
3. The code specification must take into account errors.

4. The code is highly reusable and scalable. You will need to change the requirements based on the job in the future.

On the first day, I thought it was very simple, not just to find the union of the two files, so I soon wrote the following code.

#include<iostream>#include<vector>#include<cstdlib>#include<algorithm>#include<fstream>using namespace std;void merge(const vector<int> &, const vector<int>&, vector<int> &);int main(){vector<int> v1, v2;vector<int> result;char buf[512];FILE *fp;fp = fopen("file1", "r");if(fp < 0){cout<<"Open file failed!\n";exit(1);}while(fgets(buf, 512, fp) != NULL){v1.push_back(atoi(buf));}sort(v1.begin(), v1.end());fclose(fp);fp = fopen("file2", "r");if(fp < 0){cout<<"Open file2 failed!\n";exit(1);}while(fgets(buf, 512, fp) != NULL){v2.push_back(atoi(buf));}sort(v2.begin(), v2.end());cout<<v1[v1.size() - 1]<<endl;cout<<v2[v2.size() - 1]<<endl;fclose(fp);merge(v1, v2, result);cout<<result.size();ofstream output;output.open("result");if(output.fail()){cerr<<"crete file failed!\n";exit(1);}vector<int>::const_iterator p = result.begin();for(; p != result.end(); p++){output<<*p<<endl;}output.close();return 0;}void merge(const vector<int>& v1, const vector<int>& v2, vector<int> &result){vector<int>::const_iterator p1, p2;p1 = v1.begin();p2 = v2.begin();while((p1 != v1.end()) && p2 != v2.end()){if(*p1 < *p2){p1++;}else if(*p1 > *p2){p2++;}else{result.push_back(*p1);p1++;p2++;}}}

Compile and run.

At first glance, No, it could not meet the above five seconds, so I thought for a long time. It didn't show that it took a long time to call sys, so I wrote a program, the Code is as follows:

#include <iostream>#include <fstream>#include <vector>#include <cstdlib>#include <cstdio>#define MAXLINE 32using namespace std;void qsort(vector<int>&, int, int);int partition(vector<int>&, int, int);bool binarySearch(const vector<int>&, int);int main(){vector<int> v1, result;int temp;char buf[MAXLINE];FILE *fd;fd = fopen("file1", "r");if(fd == NULL){cerr<<"Open file1 failed!\n";exit(1);}while(fgets(buf, MAXLINE, fd) != NULL){v1.push_back(atoi(buf));}fclose(fd);//cout<<v1.size()<<endl;qsort(v1, 0, v1.size() - 1);/*vector<int>::const_iterator p = v1.begin();for(; p != v1.end(); p++){cout<<*p<<endl;sleep(1);}*/fd = fopen("file2", "r");if(fd == NULL){cerr<<"open file2 failed!\n";exit(1);}while(fgets(buf, MAXLINE, fd) != NULL){temp = atoi(buf);if(binarySearch(v1, temp)){result.push_back(temp);}}cout<<result.size();return 0;}void qsort(vector<int> &v, int low, int hight){if(low < hight){int mid = partition(v, low, hight);qsort(v, low, mid - 1);qsort(v, mid + 1, hight);}}int  partition(vector<int> &v, int min, int max){int temp = v[min];while(min < max){while(min < max && v[max] >= temp)max--;v[min] = v[max];while(min < max && v[min] <= temp)min++;v[max] = v[min];}v[min] = temp;return min;}bool binarySearch(const vector<int> &v, int key){int low, hight, mid;low = 0; hight = v.size() - 1;while(low <= hight){mid = (low + hight) /2;if(v[mid] == key){return true;}else if(v[mid] < key){low = mid + 1;}else{hight = mid - 1;}}return false;}

It's fun to compile and run:

The result shows that the user time is 2.194 seconds, and the whole time is longer than before. Obviously, this method still does not work because the two files are too large and there are 5 million files, which are not generally small, the time spent above is mainly used for sorting, so I thought, can I do it without sorting? At this time, a friend and I talked about the bitmap method. As a result of inspiration, I wrote the code again:

#include <iostream>#include <cstdlib>#include <cstdio>#include <cstring>#include <fstream>#include <string>#include <vector>#include <algorithm>#include <iterator>#define SHIFT 5#define MAXLINE 32#define MASK 0x1Fusing namespace std;void setbit(int *bitmap, int i){bitmap[i >> SHIFT] |= (1 << (i & MASK));}bool getbit(int *bitmap1, int i){return bitmap1[i >> SHIFT] & (1 << (i & MASK));}size_t getFileSize(ifstream &in, size_t &size){in.seekg(0, ios::end);size = in.tellg();in.seekg(0, ios::beg);return size;}char * fillBuf(const char *filename){size_t size = 0;ifstream in(filename);if(in.fail()){cerr<< "open " << filename << " failed!" << endl;exit(1);}getFileSize(in, size);char *buf = (char *)malloc(sizeof(char) * size + 1);if(buf == NULL){cerr << "malloc buf error!" << endl;exit(1);}in.read(buf, size);in.close();buf[size] = '\0';return buf;}void setBitMask(const char *filename, int *bit){char *buf, *temp;temp = buf = fillBuf(filename);char *p = new char[11];int len = 0;while(*temp){if(*temp == '\n'){p[len] = '\0';len = 0;//cout<<p<<endl;setbit(bit, atoi(p));}else{p[len++] = *temp;}temp++;}delete buf;}void compareBit(const char *filename, int *bit, vector<int> &result){char *buf, *temp;temp = buf = fillBuf(filename);char *p = new char[11];int len = 0;while(*temp){if(*temp == '\n'){p[len] = '\0';len = 0;if(getbit(bit, atoi(p))){result.push_back(atoi(p));}}else{p[len++] = *temp;}temp++;}delete buf;}int main(){vector<int> result;unsigned int MAX = (unsigned int)(1 << 31);unsigned int size = MAX >> 5;int *bit1;bit1 = (int *)malloc(sizeof(int) * (size + 1));if(bit1 == NULL){cerr<<"Malloc bit1 error!"<<endl;exit(1);}memset(bit1, 0, size + 1);setBitMask("file1", bit1);compareBit("file2", bit1, result);delete bit1;cout<<result.size();sort(result.begin(), result.end());vector< int >::iteratorit = unique(result.begin(), result.end());ofstreamof("result");ostream_iterator<int>output(of, "\n");copy(result.begin(), it, output);return 0;}

This is a program implemented by the bitmap method and compiled and run.

The running time is obviously less than the first two, but the program exchanges space for time, and hundreds of megabytes of space is allocated when the program is running. It can be seen that the method is very important in programming. Select a method based on the conditions. However, I still think the previous two methods are okay because there is less space required.

(For more information, see http://www.wypblog.com/archives/158. Do not use it for commercial purposes .)