java讀寫檔案，讀超大檔案

標籤：

一直在處理爬蟲，經常能遇到讀寫檔案的操作，很多時候都是讀寫超大檔案，記錄如下：一、讀檔案    import java.io.BufferedOutputStream;    import  java.io.BufferedReader;    import java.io.File;    import java.io.FileOutputStream;   import java.io.FileReader;   import java.io.FileWriter;   import java.io.IOException;   import java.io.OutputStreamWriter;     public void ReadFile(String filename) {          File file=new File(filename);         BufferedReader reader=null;      try{               reader=new BufferedReader(new FileReader(file));   //如果是讀大檔案  則  new BufferedReader(new FileReader(file),5*1024*1024);  即，設定緩衝               String tempString=null;               while((tempString=reader.readLine())!=null)               {                    //進行操作.....              }            reader.close();       }catch(IOException e){             e.printStackTrace();      }finally{           if(reader!=null)         {               try{                      reader.close();               }catch(IOException e)             {             e.printStackTrace();            }          }       }}二、寫檔案    import java.io.BufferedOutputStream;    import  java.io.BufferedReader;   import  java.io.BufferedWriter;    import java.io.File;    import java.io.FileOutputStream;   import java.io.FileReader;   import java.io.FileWriter;   import java.io.IOException;   import java.io.OutputStreamWriter;     public void method1(String file,String content){     BufferedWriter out=null;     try{           out=new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file,true)));  //追加的方法          out.write(content);          out.write("\r\n");   }catch(IOException e){             e.printStackTrace();      }finally{     try{    out.close();  }catch(IOException e){             e.printStackTrace();      }}}public void main(String[] args){   String filename="D:\a.txt";  File f=new File(filename); if(f.exists()){    f.delete();}f.createNewFile();String ss="sssssss";method1(filename,ss);}===========================華麗麗的分隔字元=============================在實踐中發現上面的方法最多也就能處理10M以內的資料，從而編輯此文章。想要真正處理上百M，甚至上G的檔案，那就要用到java的nio包：下面的代碼轉自【http://www.oschina.net/code/snippet_54100_7938】import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; public class TestNio { public static void main(String args[]) throws Exception{ int bufSize = 100; File fin = new File("D:\\workspace\\test\\usagetracking.log"); File fout = new File("D:\\workspace\\test\\usagetracking2.log"); FileChannel fcin = new RandomAccessFile(fin, "r").getChannel(); ByteBuffer rBuffer = ByteBuffer.allocate(bufSize); FileChannel fcout = new RandomAccessFile(fout, "rws").getChannel(); ByteBuffer wBuffer = ByteBuffer.allocateDirect(bufSize); readFileByLine(bufSize, fcin, rBuffer, fcout, wBuffer); System.out.print("OK!!!"); } /*讀檔案同時寫檔案*/public static void readFileByLine(int bufSize, FileChannel fcin, ByteBuffer rBuffer, FileChannel fcout, ByteBuffer wBuffer){ String enterStr = "\n"; try{ byte[] bs = new byte[bufSize]; int size = 0; StringBuffer strBuf = new StringBuffer(""); //while((size = fcin.read(buffer)) != -1){ while(fcin.read(rBuffer) != -1){      int rSize = rBuffer.position();      rBuffer.rewind();      rBuffer.get(bs);      rBuffer.clear();      String tempString = new String(bs, 0, rSize);      //System.out.print(tempString);      //System.out.print("<200>");      int fromIndex = 0;      int endIndex = 0;      while((endIndex = tempString.indexOf(enterStr, fromIndex)) != -1){       String line = tempString.substring(fromIndex, endIndex);       line = new String(strBuf.toString() + line);       //System.out.print(line);       //System.out.print("</over/>");       //write to anthone file       writeFileByLine(fcout, wBuffer, line);             strBuf.delete(0, strBuf.length());       fromIndex = endIndex + 1;      }      if(rSize > tempString.length()){      strBuf.append(tempString.substring(fromIndex, tempString.length()));      }else{      strBuf.append(tempString.substring(fromIndex, rSize));      } } } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /*寫檔案*/public static void writeFileByLine(FileChannel fcout, ByteBuffer wBuffer, String line){ try { //write on file head //fcout.write(wBuffer.wrap(line.getBytes())); //wirte append file on foot fcout.write(wBuffer.wrap(line.getBytes()), fcout.size()); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } 

 

java讀寫檔案，讀超大檔案