Java code for splitting and merging large files

Copy codeThe Code is as follows: package com. test;

Import java. io. BufferedReader;
Import java. io. BufferedWriter;
Import java. io. FileNotFoundException;
Import java. io. FileReader;
Import java. io. FileWriter;
Import java. io. IOException;
Import java. util. Collections;
Import java. util. Iterator;
Import java. util. Collections list;
Import java. util. Random;

Public class LargeMappedFiles {
/**
* Sort and merge Big Data
*
* @ Param args
*/
Public static void main (String [] args) throws IOException {
// Path of the file to be written
String filePath = "D :\\ 456 ";
// Path of the splitting File
String sqlitFilePath = "D :\\ 456 \ 123 ";
// Number of data
Int CountNumbers = 10000000;

// Number of subfiles
Int CountFile = 10;

// Accuracy
Int countAccuracy = 30 * CountFile;

Long startNumber = System. currentTimeMillis ();
// Write a big data file
WriteData (filePath, CountNumbers );
System. out. println ("Storage completed ");

// Split the big data file into ten other small files
SqlitFileDate (filePath, sqlitFilePath, CountFile );
System. out. println ("the file has been cut! ");
// Sort the data of each file
SingleFileDataSort (sqlitFilePath, CountFile );
System. out. println ("sorting of each sub-file is complete! ");

// Precision adjustment, comparison and integration of 10 file data
DeathDataFile (filePath, sqlitFilePath, countAccuracy, CountFile );
System. out. println ("integrated ");
Long stopNumber = System. currentTimeMillis ();
System. out. println ("Time consumed" + (stopNumber-startNumber)/1000 + "millisecond ");
}
// Write a big data file
Public static void WriteData (String path, int CountNumbers) throws IOException {
Path = path + "\ 12114.txt ";
FileWriter fs = new FileWriter (path );
BufferedWriter fw = new BufferedWriter (fs );
For (int I = 0; I <CountNumbers; I ++ ){
Fw. write (new Random (). nextInt (Integer. MAX_VALUE) + "\ r \ n ");
}
Fw. close ();
Fs. close ();

}
// Split the big data file into ten other small files
Public static void sqlitFileDate (String filepath, String sqlitPath,
Int CountFile) throws IOException {
FileWriter fs = null;
BufferedWriter fw = null;
FileReader fr = new FileReader (filepath + "\ 12114.txt ");
BufferedReader br = new BufferedReader (fr); // read and obtain the entire row of data

Int I = 1;
Jsonlist WriterLists = new jsonlist (); // initializes a collection of file stream objects
Listlist fwLists = new listlist ();
For (int j = 1; j <= CountFile; j ++ ){

// Declare the object
Fs = new FileWriter (sqlitPath + "\ 12" + j + ". txt", false );
Fw = new BufferedWriter (fs );

// Load the object into a set
WriterLists. add (fs );
FwLists. add (fw );
}
// Determine whether data is returned in the file stream.
While (br. ready ()){

Int count = 1; // initialize the first file stream
For (Iterator iterator = fwLists. iterator (); iterator. hasNext ();){
BufferedWriter type = (BufferedWriter) iterator. next ();
If (I = count) // determines the first file stream to write data.
{
// Write data, jump out, and write the next file stream and next data
Type. write (br. readLine () + "\ r \ n ");
Break;
}
Count ++;
}
// Determine whether the last file stream has arrived
If (I> = CountFile ){
I = 1;
} Else
I ++;
}
Br. close ();
Fr. close ();
For (Iterator iterator = fwLists. iterator (); iterator. hasNext ();){
BufferedWriter object = (BufferedWriter) iterator. next ();
Object. close ();
}
// Print and close all sub-file streams
For (Iterator iterator = WriterLists. iterator (); iterator. hasNext ();){
FileWriter object = (FileWriter) iterator. next ();
Object. close ();
}
}
// Sort the data of each file
Public static void singleFileDataSort (String path1, int CountFile) throws IOException {
Required list nums = null;
For (int I = 1; I <= CountFile; I ++ ){
Nums = new vertex list ();
String path = path1 + "\ 12" + I + ". txt ";
Try {
FileReader fr = new FileReader (path );
BufferedReader br = new BufferedReader (fr );
While (br. ready ()){
// Add a single read data to the set
Nums. add (Integer. parseInt (br. readLine ()));
}
// Sort the set
Collections. sort (nums );
// Write sorted data to the source file
NumberSort (nums, path );
Br. close ();
Fr. close ();
} Catch (NumberFormatException e ){
E. printStackTrace ();
} Catch (FileNotFoundException e ){
E. printStackTrace ();
} Catch (IOException e ){
E. printStackTrace ();
}
}
}
// Sort the data of each file before writing the source file
Public static void numberSort (shortlist list, String path ){
Try {
FileWriter fs = new FileWriter (path );
BufferedWriter fw = new BufferedWriter (fs );
For (Iterator iterator = list. iterator (); iterator. hasNext ();){
Object object = (Object) iterator. next ();
Fw. write (object + "\ r \ n ");
}
Fw. close ();
Fs. close ();
} Catch (IOException e ){
E. printStackTrace ();
}
}
// Final integration of file data (precision adjustment)
Public static void deathDataFile (String filepath, String sqlitFilePath1,
Int countAccuracy, int CountFile) throws IOException {
Sorted list nums = new sorted list (); // Add and sort data
Object temp = null; // record the last number after each sorting
Boolean ispass = false;
Optional list ispasses = null; // records the status information of the data file
FileWriter fs = new FileWriter (filepath + "\ Sort.txt", false); // create a file stream for Integrated Data Writing
BufferedWriter bw = new BufferedWriter (fs );
FileReader fr = null; // declare to read the file stream
BufferedReader br = null; // declare BufferedReader
Jsonlist WriterLists = new jsonlist (); // initializes a collection of file stream objects
Required list WriterListFile = new required list ();
For (int j = 1; j <= CountFile; j ++ ){
// Declare object, enable all sub-file streams to access data of all sub-Files
Fr = new FileReader (sqlitFilePath1 + "\ 12" + j + ". txt ");

// Enable all BufferedReader to facilitate the next full row read
Br = new BufferedReader (fr );

// Load all FileReader objects into a set
WriterListFile. add (fr );

// Load all BufferedReader objects into a set
WriterLists. add (br );
}
For (;;){
// Store the data of the ten source files to the set for later judgment.
Ispasses = new sort list ();

// Read the data of each of the 10 source files.
For (Iterator iterator = WriterLists. iterator (); iterator. hasNext ();){
BufferedReader object = (BufferedReader) iterator. next ();
Object obj = null;
While (object. ready ()){
// Add the data of all file streams each time
Nums. add (Integer. parseInt (object. readLine (). toString ()));
Break;
}
If (object. ready () = false)
Ispasses. add ("true"); // Save the data status of each file to the Collection
}

// Determine whether it is the first time you come in
If (nums. size () % countAccuracy = 0 & ispass = false ){
// Sort the set
Collections. sort (nums );
// Receives the largest data, and other data is written to the total sorting file.
Temp = numberSortData (nums, filepath, false, countAccuracy, bw );

// Reinitialize the set
Nums = new vertex list ();
// Add the previous group to compare the remaining data
Nums. add (temp );
Ispass = true;
// Record the number of data in the source file for next Traversal
Continue;
}
If (ispass ){
If (nums. size () % countAccuracy = 1 & nums. size ()> 1 ){
// Sort the set
Collections. sort (nums );
// Receives the largest data, and other data is written to the total sorting file.
Temp = numberSortData (nums, filepath, true, countAccuracy,
Bw );
Nums = new vertex list ();
Nums. add (temp );
Continue;
}
}
// Record the location of the next group of data
// Determine if there is no data in ten files
If (ispasses. size () = CountFile ){
Collections. sort (nums );
Temp = numberSortData (nums, filepath, true, countAccuracy, bw );
Nums = new vertex list ();
Break;
}
}
Bw. close ();
// Close the write stream
Fs. close ();

// Close all BufferedReader
For (Iterator iterator = WriterLists. iterator (); iterator. hasNext ();){
BufferedReader object2 = (BufferedReader) iterator. next ();
Object2.close ();
}

// Close all FileReader
For (Iterator iterator = WriterListFile. iterator (); iterator. hasNext ();){
FileReader object = (FileReader) iterator. next ();
Object. close ();
}
}
// Sort the data and write it into the final file (precision adjustment)
Public static Object numberSortData (writable list, String filePath,
Boolean ispass, int countAccuracy, BufferedWriter fs ){
Object temp = 0; // record the last value
Int tempCount = 0; // the location where the data is written.
Try {
For (Iterator iterator = list. iterator (); iterator. hasNext ();){
Object object = (Object) iterator. next ();
// Determine whether it is the last number
If (tempCount = list. size ()-1 ){
// Judge the number of less than one hundred data entries in the Set
If (list. size () <countAccuracy + 1 & ispass ){
Temp = null;
} Else {
Temp = object;
Break;
}
}
// Write data source
Fs. write (object + "\ r \ n ");
// Subscript of recorded data
TempCount ++;
}
} Catch (IOException e ){
E. printStackTrace ();
}
Return temp;
}
}