大資料檔案處理

標籤：blog   io   ar   os   java   on   檔案   資料   div   

在處理大資料檔案時，利用"生產者-消費者"執行緒模式進行處理，代碼實現如下：

/** * 檔案處理類 * */public class FileProcessor {/**讀取檔案的路徑*/private String path = "";/**指定預設工作隊列的大小*/public static final int MAXWORKQUEUESIZE = 2 << 12;/**背景工作執行緒隊列*/private BlockingQueue<Runnable> workQueue = null; /**資料處理線程池*/private ThreadPoolExecutor excutor = null;public FileProcessor(String file) {this.path = file;workQueue = new LinkedBlockingQueue<Runnable>(MAXWORKQUEUESIZE);excutor = new ThreadPoolExecutor(10 , 15 , 5 * 60 * 1000L , TimeUnit.MILLISECONDS, workQueue);}public FileProcessor(BlockingQueue<Runnable> workQueue,String file) {this.path = file;excutor = new ThreadPoolExecutor(10 , 15 , 5 * 60 * 1000L , TimeUnit.MILLISECONDS, workQueue);}public void process() {/**開啟檔案讀取線程*/FileReaderProcessor fileReaderProcessor = new FileReaderProcessor(path,excutor);excutor.execute(fileReaderProcessor);}public static void main(String []args) {FileProcessor proc = new FileProcessor("D://test");proc.process();}}/********************************************************//**讀取檔案線程*/public class FileReaderProcessor implements Runnable {/**讀取檔案路徑*/private String path = "";private ThreadPoolExecutor excutor = null;public FileReaderProcessor(String file, ThreadPoolExecutor excutor) {this.path = file;this.excutor = excutor;}@Overridepublic void run() {// TODO Auto-generated method stubFileReader reader = null;BufferedReader br = null;int lineNumber = 0;try {reader = new FileReader(path);br = new BufferedReader(reader);String str = null;while((str = br.readLine()) != null) {++lineNumber;System.out.println("[" + Thread.currentThread().getName() + "] read " + lineNumber + " rows");/**防止讀入過快,導致工作隊列已滿無法接受任務,則超過工作隊列0.75時,暫停提交*/if(excutor.getQueue().size() >= FileProcessor.MAXWORKQUEUESIZE * 0.75) {System.out.println("[" + Thread.currentThread().getName() + "] sleep 5 seconds");TimeUnit.SECONDS.sleep(5); /**休眠五秒中*/}excutor.submit(new DateHandlerProcessor(str));}} catch (FileNotFoundException e) {// TODO Auto-generated catch blockSystem.out.println("File Not Find Error : " + e.getMessage());} catch (IOException e) {// TODO Auto-generated catch blockSystem.out.println("Read File Io Error : " + e.getMessage());} catch (InterruptedException e) {// TODO Auto-generated catch blockSystem.out.println("Thread Interrupt Error : " + e.getMessage());} finally {/**關閉資源*/this.close(br, reader, excutor);}}public void close(BufferedReader br, FileReader reader, ThreadPoolExecutor executor) {try {if(br != null) {br.close();}if(reader != null) {reader.close();}/**關閉線程池*/while(excutor.getQueue().size() != 0) {TimeUnit.SECONDS.sleep(1);}excutor.shutdown();if(!excutor.awaitTermination(5 * 60 * 1000L, TimeUnit.MILLISECONDS)) {excutor.shutdownNow();}} catch(Exception e) {System.out.println("Close Error : " + e.getMessage());}}}/*********************************************************//**資料處理類*/public class DateHandlerProcessor implements Runnable {/**處理檔案一行內容*/private String line = "";public DateHandlerProcessor(String line) {this.line = line;}@Overridepublic void run() {// TODO Auto-generated method stubtry {System.out.println("Thread[" + Thread.currentThread().getName() + "] Get Line " +  line);} catch (Exception e) {// TODO Auto-generated catch blockSystem.out.println("Thread[" + Thread.currentThread().getName() + "] Interrupt : " + e.getMessage());}}}

　　

大資料檔案處理