基於檔案系統（及MySQL）使用Java實現MapReduce

標籤：com   .sql   user   amp   incr   format   reducer   try   odi   

實現這個代碼的原因是：

• 我會MapReduce，但是之前都是在AWS EMR上，自己搭過偽分布式的，但是感覺營運起來比較困難；
• 我就MySQL會一點（本來想用mongoDB的但是不太會啊）
• 資料量不是很大，至少對我來說。
• 希望不要出很麼問題，這方面檔案系統還是可以信任的。

設計思路如下：

• init階段：將所需的檔案添加到一個列表檔案input_file_list.txt中。
  
• Map階段：讀取input_file_list.txt中的每一個檔案的每一行，並將其映射成一個key-value對。
  考慮到key可能包含特殊字元，所以這裡使用MySQL儲存一個id到key的對應關係的資料。
• Reduce階段：針對每一個key，讀取對應的檔案，最終產生一個name-value列表，該name-value列表對應一個json對象，如：{ "name": "zifeiy", "age": 88 }，將所有的jsonObject Storage Service到一個結果檔案reduceResult.txt中。
• 處理結果階段，將reduceResult.txt檔案進行解析，最終產生結果的CSV檔案或者Excel檔案。

主要代碼：

package com.zifeiy.snowflake.tools.mapreduce.v1;import java.io.BufferedReader;import java.io.File;import java.io.FileInputStream;import java.io.FileOutputStream;import java.io.IOException;import java.io.InputStreamReader;import java.io.OutputStreamWriter;import java.sql.Connection;import java.sql.DriverManager;import java.sql.PreparedStatement;import java.sql.ResultSet;import java.sql.Statement;import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;import com.google.gson.Gson;import com.zifeiy.snowflake.assist.CsvOneLineParser;import com.zifeiy.snowflake.assist.FileHelper;import jxl.Workbook;import jxl.write.Label;import jxl.write.WritableSheet;import jxl.write.WritableWorkbook;public abstract class MapReduceBaseVersion1 {        private static final String APPENDED_DB_INFO = "?useUnicode=true&characterEncoding=UTF8"                                                     + "&rewriteBatchedStatements=true"                                                     + "&useLegacyDatetimeCode=false"                                                     + "&serverTimezone=Asia/Shanghai"                                                    + "&useSSL=false";    private static final String classname       = "com.mysql.cj.jdbc.Driver";    private static final String url             = "jdbc:mysql://localhost:3306/snowflake" + APPENDED_DB_INFO;    private static final String username            = "root";    private static final String password    = "password";        public static final String taskRootPath = "D:\\snowflake\\task";        private Connection connection = null;    private File inputListFile = null;    private File reduceResultFile = null;    private File resultFile = null;    private int taskId;        public void addInputPath(File file) throws IOException {        FileHelper.appendFile(inputListFile, file.getAbsolutePath() + "\r\n");    }        public void setKeyValuePair(String key, String value) throws Exception {        int id = -1;        Statement statement = connection.createStatement();        ResultSet resultSet = statement.executeQuery(String.format("select id from tmp" + taskId + " where kname=‘%s‘", key.replaceAll("‘", "‘‘")));        if (resultSet.next()) {            id = resultSet.getInt(1);        }        else {            statement.execute(String.format("insert into tmp" + taskId + " (kname) values (‘%s‘)", key.replaceAll("‘", key.replaceAll("‘", "‘‘"))));            resultSet = statement.executeQuery(String.format("select id from tmp" + taskId + " where kname=‘%s‘", key.replaceAll("‘", "‘‘")));            if (resultSet.next()) {                id = resultSet.getInt(1);            }        }        if (id == -1) throw new Exception("set key value pair failed: key = " + key + ", value = " + value);        File tmpFile = new File(taskRootPath + File.separator + taskId + File.separator + "tmp" + File.separator + id + ".txt");        if (tmpFile.exists() == false) {            tmpFile.createNewFile();        }        FileHelper.appendFile(tmpFile, value + "\r\n");    }        public void addParamList(List<Map<String, String>> paramList) throws Exception {        String content = "";        Gson gson = new Gson();        for (Map<String, String> params : paramList) {            String jsonString = gson.toJson(params);            content += jsonString + "\r\n";        }        FileHelper.appendFile(reduceResultFile, content);    }        public void generateFile(String[] columns, String[] nameColumns) throws Exception {        if (reduceResultFile == null || reduceResultFile.exists() == false) {            throw new Exception("[mapreduce.v1] in generateFile function: reduceResultFile do not exist!");        }//      if (false) {    // test        if (reduceResultFile.length() > 1 * 1024 * 1024) {  // 如果檔案大小超過1MB，匯出成csv            resultFile = new File(taskRootPath + File.separator + taskId + File.separator + "result.csv");                        Gson gson = new Gson();                        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(reduceResultFile), "UTF-8"));            FileOutputStream fos = new FileOutputStream(resultFile);            OutputStreamWriter osw = new OutputStreamWriter(fos, "UTF-8");                        String content = "";            for (int i = 0; i < nameColumns.length; i ++) {                if (i > 0)                    content += ",";                content += ‘"‘ + nameColumns[i] + ‘"‘;            }            osw.write(content + "\r\n");                        String line = null;            while ((line = br.readLine()) != null) {                content = "";                Map<String, String> map = gson.fromJson(line, Map.class);                if (map == null) { throw new Exception("map is null by parsing line: " + line); }                for (int i = 0; i < columns.length; i ++) {                    if (i > 0) content += ",";                    String c = columns[i];                    String v = map.get(c);                    if (v != null) {                        content += ‘"‘ + v + ‘"‘;                    }                }                osw.write(content + "\r\n");            }            br.close();            osw.write(content);            osw.flush();            osw.close();        } else {    // 如果檔案大小小於1MB，匯出成Excel檔案            resultFile = new File(taskRootPath + File.separator + taskId + File.separator + "result.xls");                        WritableWorkbook workbook = Workbook.createWorkbook(resultFile);            WritableSheet sheet = workbook.createSheet("sheet1", 0);                        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(reduceResultFile), "UTF-8"));            String line = null;                        for (int i = 0; i < nameColumns.length; i ++) {                sheet.addCell(new Label(i, 0, nameColumns[i]));            }                        int rowId = 1;            while ((line = br.readLine()) != null) {                Gson gson = new Gson();                List<String> rowList = new ArrayList<String>();                                Map<String, String> map = gson.fromJson(line, Map.class);                if (map == null) { throw new Exception("map is null by parsing line: " + line); }                for (int i = 0; i < columns.length; i ++) {                    String c = columns[i];                    String v = map.get(c);                    String innerContent = "";                    if (v != null) {                        innerContent = v;                    }                    sheet.addCell(new Label(i, rowId, innerContent));                }                rowId ++;                            }            br.close();                        workbook.write();            workbook.close();                    }    }        public abstract void init() throws Exception;        public abstract void map(String line) throws Exception;        public abstract void reduce(String key, ReduceReader reduceReader) throws Exception;        public abstract void generate() throws Exception;        public String mapreduce() {        try {            Class.forName(classname);            connection = DriverManager.getConnection(url, username, password);                        // generate taskId            PreparedStatement preparedStatement = connection.prepareStatement("insert into task () values ()");            preparedStatement.execute("insert into task () values ()", PreparedStatement.RETURN_GENERATED_KEYS);            ResultSet resultSet = preparedStatement.getGeneratedKeys();            if (resultSet.next()) {                taskId = resultSet.getInt(1);            }            else {                throw new Exception("[mapreduce.v1] Exception: can not generate taskId");            }            // generated task file path            String taskPath = taskRootPath + File.separator + taskId;            File taskPathDir = new File(taskPath);            if (taskPathDir.exists() == true) {                throw new Exception("[mapreduce.v1] Exception: task directory already exists");            }            taskPathDir.mkdirs();            String tmpDirPath = taskPath + File.separator + "tmp";            File tmpDir = new File(tmpDirPath);            tmpDir.mkdirs();            this.inputListFile = new File(taskPath + File.separator + "input_file_list.txt");            inputListFile.createNewFile();            // period. 1: init()            // during init period, we will use addInputPath function to add all the input files we need            init();                        // begin to read each line of each file            // peroid. 2: map(line)                        // db prepare            Statement statement = connection.createStatement();            statement.execute("create temporary table tmp" + taskId + " ( id int not null auto_increment primary key, kname varchar(200) )");                        // file content prepare                        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(inputListFile), "UTF-8"));            String inputFilename = null;               while ((inputFilename = br.readLine()) != null) {                File inputFile = new File(inputFilename);                if (inputFile.exists() == false) {                    throw new Exception("[mapreduce.v1] Exception: input file " + inputFilename + " do not exists!");                }                BufferedReader br2 = new BufferedReader(new InputStreamReader(new FileInputStream(inputFile), "GBK"));                String line = null;                while ((line = br2.readLine()) != null) {                    map(line);                }            }            br.close();                        // period. 3: reduce(key, valueList)            reduceResultFile = new File(taskPath + File.separator + "reduce.txt");            if (reduceResultFile.exists() == true) {                throw new Exception("[mapreduce.v1] reduce file already exists!");            }            reduceResultFile.createNewFile();                        resultSet = statement.executeQuery("select * from tmp" + taskId);            while (resultSet.next()) {                int id = resultSet.getInt(1);                String key = resultSet.getString(2);                File reduceFile = new File(tmpDirPath + File.separator + id + ".txt");                if (reduceFile.exists() == false) {                    throw new Exception("[mapreduce.v1] Exception: reduce file " + reduceFile.getName() + " not exists!");                }                ReduceReader reduceReader = new ReduceReader(reduceFile);                reduce(key, reduceReader);            }                        // period. 4: generate            // generate the result file            generate();                        connection.close();        } catch (Exception e) {            e.printStackTrace();        }                if (resultFile == null) return null;        else return resultFile.getAbsolutePath();            }            // main for test    public static void main(String[] args) {        MapReduceBaseVersion1 mapReduceBaseVersion1 = new MapReduceBaseVersion1() {                        @Override            public void reduce(String key, ReduceReader reduceReader) throws Exception {                // TODO Auto-generated method stub                List<Map<String, String>> paramList = new ArrayList<Map<String,String>>();                                String line;                while ( (line = reduceReader.next()) != null ) {                    List<String> rowList = CsvOneLineParser.parseLine(line);                    Map<String, String> tmpMap = new HashMap<String, String>();                    int idx = 0;                    for (String s : rowList) {                        idx ++;                        tmpMap.put("" + idx, s);                    }                    paramList.add(tmpMap);                }                addParamList(paramList);            }                        @Override            public void map(String line) throws Exception {                // TODO Auto-generated method stub                setKeyValuePair(line.substring(1, 3), line);            }                        @Override            public void init() throws Exception {                // TODO Auto-generated method stub                addInputPath(new File("D:\\test\\test.del"));            }                        @Override            public void generate() throws Exception {                // TODO Auto-generated method stub                generateFile(new String[] { "1", "2", "3", "4", "5", "6" }, new String[] { "一", "二", "三", "四", "五", "六" });            }        };        System.out.println(mapReduceBaseVersion1.mapreduce());    }}

基於檔案系統（及MySQL）使用Java實現MapReduce