Java讀取操作大資料excel

標籤：excel java

工作需要，讀取大資料量的excel。用Apache poi的普通模式讀取，會拋記憶體溢出。查詢文檔得知有另外一種模式--使用者模式。該模式不會一下子整個檔案load進來放在記憶體裡，而是一行一行的讀取，這樣就能避免記憶體溢出了。

上碼：

package com.ism.excel.pkg07;import java.io.InputStream;import java.sql.SQLException;import java.util.ArrayList;import java.util.Iterator;import java.util.List;import org.apache.poi.xssf.eventusermodel.XSSFReader;import org.apache.poi.xssf.model.SharedStringsTable;import org.apache.poi.xssf.usermodel.XSSFRichTextString;import org.apache.poi.openxml4j.opc.OPCPackage;import org.xml.sax.Attributes;import org.xml.sax.InputSource;import org.xml.sax.SAXException;import org.xml.sax.XMLReader;import org.xml.sax.helpers.DefaultHandler;import org.xml.sax.helpers.XMLReaderFactory;/** * XSSF and SAX (Event API) */public abstract class XxlsAbstract extends DefaultHandler {private SharedStringsTable sst;private String lastContents;private boolean nextIsString;private int sheetIndex = -1;private List<String> rowlist = new ArrayList<String>();private int curRow = 0;//當前行private int curCol = 0;//當前列索引private int preCol = 0;//上一列列索引private int titleRow = 0;//標題列，一般情況下為0private int rowsize = 0;//列數//excel記錄行操作方法，以sheet索引，行索引和行元素列表為參數，對sheet的一行元素進行操作，元素為String類型public abstract void optRows(int sheetIndex,int curRow, List<String> rowlist) throws SQLException;//只遍曆一個sheet，其中sheetId為要遍曆的sheet索引，從1開始，1-3/** *  * @param filename * @param sheetId  sheetId為要遍曆的sheet索引，從1開始，1-3 * @throws Exception */public void processOneSheet(String filename,int sheetId) throws Exception {OPCPackage pkg = OPCPackage.open(filename);XSSFReader r = new XSSFReader(pkg);SharedStringsTable sst = r.getSharedStringsTable();XMLReader parser = fetchSheetParser(sst);// rId2 found by processing the Workbook// 根據 rId# 或 rSheet# 尋找sheetInputStream sheet2 = r.getSheet("rId"+sheetId);sheetIndex++;InputSource sheetSource = new InputSource(sheet2);parser.parse(sheetSource);sheet2.close();}/** * 遍曆 excel 檔案 */public void process(String filename) throws Exception {OPCPackage pkg = OPCPackage.open(filename);XSSFReader r = new XSSFReader(pkg);SharedStringsTable sst = r.getSharedStringsTable();XMLReader parser = fetchSheetParser(sst);Iterator<InputStream> sheets = r.getSheetsData();while (sheets.hasNext()) {curRow = 0;sheetIndex++;InputStream sheet = sheets.next();InputSource sheetSource = new InputSource(sheet);parser.parse(sheetSource);sheet.close();}}public XMLReader fetchSheetParser(SharedStringsTable sst)throws SAXException {XMLReader parser = XMLReaderFactory.createXMLReader();//.createXMLReader("org.apache.xerces.parsers.SAXParser");this.sst = sst;parser.setContentHandler(this);return parser;}public void startElement(String uri, String localName, String name,Attributes attributes) throws SAXException {// c => 儲存格if (name.equals("c")) {// 如果下一個元素是 SST 的索引，則將nextIsString標記為trueString cellType = attributes.getValue("t");String rowStr = attributes.getValue("r");curCol = this.getRowIndex(rowStr);if (cellType != null && cellType.equals("s")) {nextIsString = true;} else {nextIsString = false;}}// 置空lastContents = "";}public void endElement(String uri, String localName, String name)throws SAXException {// 根據SST的索引值的到儲存格的真正要儲存的字串// 這時characters()方法可能會被調用多次if (nextIsString) {try {int idx = Integer.parseInt(lastContents);lastContents = new XSSFRichTextString(sst.getEntryAt(idx)).toString();} catch (Exception e) {}}// v => 儲存格的值，如果儲存格是字串則v標籤的值為該字串在SST中的索引// 將儲存格內容加入rowlist中，在這之前先去掉字串前後的空白符if (name.equals("v")) {String value = lastContents.trim();value = value.equals("")?" ":value;int cols = curCol-preCol;if (cols>1){for (int i = 0;i < cols-1;i++){rowlist.add(preCol,"");}}preCol = curCol;rowlist.add(curCol-1, value);}else {//如果標籤名稱為 row ，這說明已到行尾，調用 optRows() 方法if (name.equals("row")) {int tmpCols = rowlist.size();if(curRow>this.titleRow && tmpCols<this.rowsize){for (int i = 0;i < this.rowsize-tmpCols;i++){rowlist.add(rowlist.size(), "");}}try {optRows(sheetIndex,curRow,rowlist);} catch (SQLException e) {e.printStackTrace();}if(curRow==this.titleRow){this.rowsize = rowlist.size();}rowlist.clear();curRow++;curCol = 0;preCol = 0;}}}public void characters(char[] ch, int start, int length)throws SAXException {//得到儲存格內容的值lastContents += new String(ch, start, length);}//得到列索引，每一列c元素的r屬性構成為字母加數位形式，字母組合為列索引，數字組合為行索引，//如AB45,表示為第（A-A+1）*26+（B-A+1）*26列，45行public int getRowIndex(String rowStr){rowStr = rowStr.replaceAll("[^A-Z]", "");byte[] rowAbc = rowStr.getBytes();int len = rowAbc.length;float num = 0;for (int i=0;i<len;i++){num += (rowAbc[i]-'A'+1)*Math.pow(26,len-i-1 );}return (int) num;}public int getTitleRow() {return titleRow;}public void setTitleRow(int titleRow) {this.titleRow = titleRow;}}

Java讀取操作大資料excel