List set, txt file comparison tool class and file read/write tool class

At work, we often encounter big data processing problems. The following two tool classes are written when processing big data. We recommend that you use map to process two lists of data, if the method of listing is quite big data, it will play a role. At that time, two 100,000-level lists were processed, and the variant of the modified method was used to solve the problem.

1.txt file and list set comparison tool

 

Package com. hudong. util. other; import java. util. collection; import java. util. hashMap; import java. util. hashSet; import java. util. using list; import java. util. list; import java. util. map; import org. apache. commons. lang. stringUtils;/*** txt file comparison tool * list set comparison tool * @ Title: TxtCompareUtil. java * @ Copyright: Copyright (c) 2005 * @ author Yang Kai */public class TxtCompareUtil {/*** use two lists for comparison ** @ param input1 * @ param input2 * @ param output1 * @ param output2 */ public static void compareTxt (String input1, string input2, String output1, String output2) {List
 
  
ListInput1 = TxtReadWriteUtil. readTxt (input1); List
  
   
ListInput2 = TxtReadWriteUtil. readTxt (input2); for (String str: listInput1) {if (listInput2.contains (StringUtils. upperCase (str) | listInput2.contains (StringUtils. lowerCase (str) | listInput2.contains (str) {TxtReadWriteUtil. writerTXT (str, output1, true);} else {TxtReadWriteUtil. writerTXT (str, output2, true );}}} /*** calculate the intersection of two lists and find the common part of two txt files through the intersection method ** @ param input1 * @ param input2 * @ param output1 * @ param output2 */public static void interseTxt (String input1, string input2, String output1, String output2) {List
   
    
ListInput1 = TxtReadWriteUtil. readTxt (input1); List
    
     
ListInput2 = TxtReadWriteUtil. readTxt (input2); if (listInput2.retainAll (listInput1) {System. out. println (listInput2); // TxtReadWriteUtil. writerTXT (listInput2, output1, true);} else {// TxtReadWriteUtil. writerTXT (listInput2, output2, true );}} /*** calculate the union of two lists and merge two txt files by seeking the union of two lists ** @ param input1 * @ param input2 * @ param output1 * @ param output2 */public static void unionTxt (String input1, string input2, String output1, String output2) {List
     
      
ListInput1 = TxtReadWriteUtil. readTxt (input1); List
      
        ListInput2 = TxtReadWriteUtil. readTxt (input2); listInput2.removeAll (listInput1); // remove the same listInput2.addAll (listInput1); // put it in different systems. out. println (listInput2);}/*** get different elements of two sets ** @ param collmax * @ param collmin * @ return */@ SuppressWarnings ({"unchecked "}) public static Collection getDiffent (Collection collmax, Collection collmin) {// use LinkeList to prevent excessive differences, copy Collection csReturn = new sort list (); Collection max = collmax; collection min = collmin; // compare the size first, which will reduce the number of if judgments of the subsequent map if (collmax. size () <collmin. size () {max = collmin; min = collmax;} // specify the size directly to prevent the re-Hash Map
       
         Map = new HashMap
        
          (Max. size (); for (Object object: max) {map. put (object, 1) ;}for (Object object: min) {if (map. get (object) = null) {csReturn. add (object);} else {map. put (object, 2) ;}}for (Map. entry
         
           Entry: map. entrySet () {if (entry. getValue () = 1) {csReturn. add (entry. getKey () ;}} return csReturn;}/*** get different elements of the two sets, remove duplicate ** @ param collmax * @ param collmin * @ return */@ SuppressWarnings ({"unchecked"}) public static Collection getDiffentNoDuplicate (Collection collmax, Collection collmin) {return new HashSet (getDiffent (collmax, collmin);} public static void main (String [] args) {unionTxt ("e:/test/input1.txt", "e: /test/input2.txt "," e:/test/output1.txt "," e:/test/output1.txt ");}}
         
        
       
      
     
    
   
  
 

2. Tool class for reading and writing files

 

 

Package com. hudong. util. other; import java. io. bufferedReader; import java. io. bufferedWriter; import java. io. file; import java. io. fileOutputStream; import java. io. fileReader; import java. io. fileWriter; import java. util. arrayList; import java. util. list; import org. dom4j. io. outputFormat; import org. dom4j. io. XMLWriter;/*** txt file read/write tool ** @ Title: TxtReadWriteUtil. java * @ Copyright: Copyright (c) 2005 * @ author Yang Kai */public class TxtReadWriteUtil {/*** write a txt method ** @ param conent * @ param txtPath * @ param isAppend */public synchronized static void writerTXT (String conent, string txtPath, boolean isAppend) {try {File file = new File (txtPath); if (! File. getParentFile (). exists () {file. getParentFile (). mkdirs ();} if (! File. exists () {file. createNewFile ();} FileWriter fileWriter = new FileWriter (txtPath, isAppend); BufferedWriter bw = new BufferedWriter (fileWriter); bw. write (conent); bw. newLine (); fileWriter. flush (); bw. close (); fileWriter. close ();} catch (Exception e) {e. printStackTrace () ;}}/*** write txt method 2 ** @ param file * @ param sb */public static void createTxt (String file, StringBuffer sb) {try {OutputFormat format = OutputFormat. createPrettyPrint (); format. setEncoding ("gbk"); format. setExpandEmptyElements (true); format. setTrimText (false); FileOutputStream fos = new FileOutputStream (file); XMLWriter xmlWriter = new XMLWriter (fos, format); xmlWriter. write (sb. toString (); xmlWriter. close ();} catch (Exception e) {e. printStackTrace () ;}/ *** read txt *** @ param filePath * @ return */public static List
 
  
ReadTxt (String filePath) {List
  
   
List = new ArrayList
   
    
(); Try {BufferedReader br = new BufferedReader (new FileReader (filePath); String line = null; while (line = br. readLine ())! = Null) {list. add (line) ;}catch (Exception e) {e. printStackTrace () ;}return list ;}}