Java implementation of large file segmentation and merging instance code _java

Copy Code code as follows:

Package com.test;

Import Java.io.BufferedReader;


Import Java.io.BufferedWriter;


Import java.io.FileNotFoundException;


Import Java.io.FileReader;


Import Java.io.FileWriter;


Import java.io.IOException;


Import java.util.Collections;


Import Java.util.Iterator;


Import java.util.LinkedList;


Import Java.util.Random;





public class Largemappedfiles {


/**


* Large Data sorting merge


*


* @param args


*/


public static void Main (string[] args) throws IOException {


Path to write to file


String FilePath = "d:\\456";


The path of the Shard file


String Sqlitfilepath = "d:\\456\\123";


Number of data


int countnumbers=10000000;





Number of child files


int countfile=10;





Precision


int countaccuracy=30*countfile;





Long Startnumber=system.currenttimemillis ();


Writing Large data files


WriteData (filepath,countnumbers);


SYSTEM.OUT.PRINTLN ("Storage Complete");





Splitting large data files into 10 other small files


Sqlitfiledate (FilePath, sqlitfilepath,countfile);


System.out.println ("File cut complete!") ");


To sort the data of each file


Singlefiledatasort (Sqlitfilepath,countfile);


System.out.println ("Sorting each sub file!") ");





Precision adjustment, 10 file data for comparative integration


Deathdatafile (Filepath,sqlitfilepath,countaccuracy,countfile);


SYSTEM.OUT.PRINTLN ("complete integration");


Long Stopnumber=system.currenttimemillis ();


System.out.println ("time-consuming" + (Stopnumber-startnumber)/1000+ "milliseconds");


}


Writing Large data files


public static void WriteData (String path,int countnumbers) throws IOException {


Path = path + "\\12114.txt";


FileWriter fs = new FileWriter (path);


BufferedWriter fw=new BufferedWriter (FS);


for (int i = 0; i < countnumbers; i++) {


Fw.write (New Random (). Nextint (integer.max_value) + "\ r \ n");


}


Fw.close ();


Fs.close ();





}


Splitting large data files into 10 other small files


public static void Sqlitfiledate (string filepath, String Sqlitpath,


int countfile) throws IOException {


FileWriter fs = null;


BufferedWriter Fw=null;


FileReader FR = new FileReader (filepath + "\\12114.txt");


BufferedReader br = new BufferedReader (FR); Fetch entire row of data





int i = 1;


LinkedList writerlists=new LinkedList (); Initializing a collection of file stream objects


LinkedList fwlists=new LinkedList ();


for (int j = 1; J <= Countfile; j) {





declaring objects


FS = new FileWriter (Sqlitpath + "\\12" + j + ". txt", false);


Fw=new BufferedWriter (FS);








To load an object into a collection


Writerlists.add (FS);


Fwlists.add (FW);


}


Determine if there is any data return in the file stream


while (Br.ready ()) {





int count=1;//initialization First file stream


for (Iterator iterator = Fwlists.iterator (); Iterator.hasnext ();) {


BufferedWriter type = (bufferedwriter) iterator.next ();


if (i==count)//judgment turn to the first few file streams write data


{


Writes the data, jumps out, carries on the next file stream, writes the next data


Type.write (Br.readline () + "\ r \ n");


Break


}


count++;


}


To determine if the last file stream has been made.


if (i >= countfile) {


i = 1;


} else


i++;


}


Br.close ();


Fr.close ();


for (Iterator iterator = Fwlists.iterator (); Iterator.hasnext ();) {


BufferedWriter object = (bufferedwriter) iterator.next ();


Object.close ();


}


Traversal closes all child file streams


for (Iterator iterator = Writerlists.iterator (); Iterator.hasnext ();) {


FileWriter object = (FileWriter) iterator.next ();


Object.close ();


}


}


To sort the data of each file


public static void Singlefiledatasort (String path1,int countfile) throws IOException {


LinkedList nums = null;


for (int i = 1; I <= countfile; i++) {


Nums = new LinkedList ();


String Path = path1 + "\\12" + i + ". txt";


try {


FileReader FR = new FileReader (path);


BufferedReader br = new BufferedReader (FR);


while (Br.ready ()) {


To add a single read to the collection


Nums.add (Integer.parseint (Br.readline ()));


}


To sort a collection


Collections.sort (Nums);


Writes sorted data to the source file


Numbersort (nums, path);


Br.close ();


Fr.close ();


catch (NumberFormatException e) {


E.printstacktrace ();


catch (FileNotFoundException e) {


E.printstacktrace ();


catch (IOException e) {


E.printstacktrace ();


}


}


}


To sort each file data and then write to the source file


public static void Numbersort (LinkedList list, String path) {


try {


FileWriter fs = new FileWriter (path);


BufferedWriter fw=new BufferedWriter (FS);


for (Iterator iterator = List.iterator (); Iterator.hasnext ();) {


Object object = (object) iterator.next ();


Fw.write (object + "\ r \ n");


}


Fw.close ();


Fs.close ();


catch (IOException e) {


E.printstacktrace ();


}


}


Final integration of file data (precision tuning)


public static void Deathdatafile (string filepath, String sqlitFilePath1,


int countaccuracy, int countfile) throws IOException {


LinkedList nums = new LinkedList (); Add data, sort


Object temp = null; Record the last number that is left for each order


Boolean ispass = false;


LinkedList ispasses = null; Record status information for a data file


FileWriter fs = new FileWriter (filepath + "\\Sort.txt", false); Create a file stream so that consolidated data is written


BufferedWriter bw=new BufferedWriter (FS);


FileReader FR = null; Declaring read file streams


BufferedReader br = null; Statement BufferedReader


LinkedList writerlists = new LinkedList (); Initializing a collection of file stream objects


LinkedList writerlistfile = new LinkedList ();


for (int j = 1; J <= Countfile; j) {


Declaring the object, opening all the file streams to access all of the child files


FR = new FileReader (sqlitFilePath1 + "\\12" + j + ". txt");





Turn on all bufferedreader to facilitate the next full line of reading


br = new BufferedReader (FR);





Load all FileReader objects into the collection


Writerlistfile.add (FR);





Load all BufferedReader objects into the collection


Writerlists.add (BR);


}


for (;;) {


10 source files are stored in the collection of data, to facilitate the subsequent judgment


ispasses = new LinkedList ();





Read single data from 10 source files separately


for (Iterator iterator = Writerlists.iterator (); Iterator.hasnext ();) {


BufferedReader object = (BufferedReader) iterator.next ();


Object obj = null;


while (Object.ready ()) {


Add data for every file stream


Nums.add (Integer.parseint (Object.readline (). toString ()));


Break


}


if (object.ready () = = False)


Ispasses.add ("true"); Store data status in a collection in each file


}





Whether the decision is the first time to come in


if (nums.size ()% Countaccuracy = = 0 && Ispass = False) {


To sort a collection


Collections.sort (Nums);


Receives the largest data, other data writes to the total sort file


temp = Numbersortdata (Nums, filepath, false, countaccuracy, BW);





Reinitialize Collection


Nums = new LinkedList ();


Add the last set of data to compare the rest


Nums.add (temp);


Ispass = true;


Record the number of data in the source file so that the next traversal


Continue


}


if (Ispass) {


if (nums.size ()% countaccuracy = = 1 && nums.size () > 1) {


To sort a collection


Collections.sort (Nums);


Receives the largest data, other data writes to the total sort file


temp = Numbersortdata (Nums, filepath, true, Countaccuracy,


BW);


Nums = new LinkedList ();


Nums.add (temp);


Continue


}


}


Record the location of the next set of data


Judge whether 10 files have no data


if (ispasses.size () = = Countfile) {


Collections.sort (Nums);


temp = Numbersortdata (Nums, filepath, true, countaccuracy, BW);


Nums = new LinkedList ();


Break


}


}


Bw.close ();


Close Write Stream


Fs.close ();





Turn off all BufferedReader.


for (Iterator iterator = Writerlists.iterator (); Iterator.hasnext ();) {


BufferedReader object2 = (BufferedReader) iterator.next ();


Object2.close ();


}





Turn off all FileReader.


for (Iterator iterator = Writerlistfile.iterator (); Iterator.hasnext ();) {


FileReader object = (FileReader) iterator.next ();


Object.close ();


}


}


Sort data, write to final file (precision adjustment)


public static Object Numbersortdata (LinkedList list, String FilePath,


Boolean ispass, int countaccuracy,bufferedwriter fs) {


Object temp = 0; Record Last value


int tempcount = 0; Record where data is written


try {


for (Iterator iterator = List.iterator (); Iterator.hasnext ();) {


Object object = (object) iterator.next ();


Judge whether it is the last number


if (Tempcount = = List.size ()-1) {


Judging the collection is less than 100.


if (List.size () < Countaccuracy + 1 && ispass) {


temp = null;


} else {


Temp = object;


Break


}


}


Write to Data source


Fs.write (object + "\ r \ n");


Record the subscript of the data


tempcount++;


}


catch (IOException e) {


E.printstacktrace ();


}


return temp;


}


}