Hadoop 檔案輸入和檔案輸出

本文完成對hadoop輸入、輸出檔案方式的控制，完成的功能如下：

1、改寫map讀取資料的格式：預設的<檔案位移量，行內容>----------->變為<檔案名稱，檔案內容>

2、改寫輸出的格式，輸出檔案時每個輸入檔案對應一個輸出檔案，輸出檔案的名字跟輸入檔案名稱字相同。

直接上代碼：

coAuInputFormat

package an.hadoop.code.audit;/** * The function of this class is revise the input format  * the <key,value > ---> map * <path,content> of the map * */import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.Text;import org.apache.hadoop.io.compress.CompressionCodec;import org.apache.hadoop.io.compress.CompressionCodecFactory;import org.apache.hadoop.mapred.FileSplit;import org.apache.hadoop.mapreduce.InputSplit;import org.apache.hadoop.mapreduce.RecordReader;import org.apache.hadoop.mapreduce.TaskAttemptContext;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;public class coAuInputFormat extends FileInputFormat<Text, Text>{private CompressionCodecFactory compressionCodecs = null;public void configure(Configuration conf) {compressionCodecs = new CompressionCodecFactory(conf);}/** * @brief isSplitable 不對檔案進行切分，必須對檔案整體進行處理 * * @param fs * @param file * * @return false */protected boolean isSplitable(FileSystem fs, Path file) {CompressionCodec codec = compressionCodecs.getCodec(file);return false;//以檔案為單位，每個單位作為一個split，即使單個檔案的大小超過了64M，也就是Hadoop一個塊得大小，也不進行分區}@Overridepublic RecordReader<Text, Text> createRecordReader(InputSplit split,TaskAttemptContext context) throws IOException,InterruptedException {// TODO Auto-generated method stubreturn new coAuRecordReader(context, split);}}

coAuRecordReader

package an.hadoop.code.audit;import java.io.IOException;import org.apache.commons.logging.Log;import org.apache.commons.logging.LogFactory;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.Text;import org.apache.hadoop.io.compress.CompressionCodec;import org.apache.hadoop.io.compress.CompressionCodecFactory;import org.apache.hadoop.mapreduce.InputSplit;import org.apache.hadoop.mapreduce.RecordReader;import org.apache.hadoop.mapreduce.TaskAttemptContext;import org.apache.hadoop.mapreduce.lib.input.FileSplit;public class coAuRecordReader extends RecordReader<Text, Text> {private static final Log LOG = LogFactory.getLog(coAuRecordReader.class.getName());private CompressionCodecFactory compressionCodecs = null;private long start;private long pos;private long end;private byte[] buffer;private String keyName;private FSDataInputStream fileIn;private Text key = null;    private Text value = null;public coAuRecordReader(TaskAttemptContext context, InputSplit genericSplit) throws IOException {// TODO Auto-generated constructor stubConfiguration job = context.getConfiguration();FileSplit split = (FileSplit) genericSplit;start = ((FileSplit) split).getStart(); //從中可以看出每個檔案是作為一個split的end = split.getLength() + start;final Path path = split.getPath();//keyName = path.toString();//key 的值是檔案路徑LOG.info("filename in hdfs is : " + keyName);//寫入記錄檔,去哪裡查看日誌呢？final FileSystem fs = path.getFileSystem(job);fileIn = fs.open(path);fileIn.seek(start);buffer = new byte[(int)(end - start)];this.pos = start;/*if(key == null){key = new Text();key.set(keyName);}if(value == null){value = new Text();value.set(utf8);}*/}//coAuRecordReader()@Overridepublic void initialize(InputSplit genericSplit, TaskAttemptContext context)throws IOException, InterruptedException {// TODO Auto-generated method stubFileSplit split = (FileSplit) genericSplit;    Configuration job = context.getConfiguration();    //this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength",Integer.MAX_VALUE);    start = split.getStart();    end = start + split.getLength();    final Path file = split.getPath();    compressionCodecs = new CompressionCodecFactory(job);    final CompressionCodec codec = compressionCodecs.getCodec(file);    keyName = file.toString();//key 的值是檔案路徑LOG.info("filename in hdfs is : " + keyName);//寫入記錄檔,去哪裡查看日誌呢？final FileSystem fs = file.getFileSystem(job);fileIn = fs.open(file);fileIn.seek(start);buffer = new byte[(int)(end - start)];this.pos = start;}@Overridepublic boolean nextKeyValue() throws IOException, InterruptedException {// TODO Auto-generated method stub//這個是需要做的if(key == null){key = new Text();}key.set(keyName);if(value == null){value = new Text();}key.clear();key.set(keyName);// set the keyvalue.clear();//clear the valuewhile(pos < end){fileIn.readFully(pos,buffer);value.set(buffer);pos += buffer.length;LOG.info("end is : " + end  + " pos is : " + pos);return true;}return false;}@Overridepublic Text getCurrentKey() throws IOException, InterruptedException {// TODO Auto-generated method stubreturn key;}@Overridepublic Text getCurrentValue() throws IOException, InterruptedException {// TODO Auto-generated method stubreturn value;}@Overridepublic float getProgress() throws IOException, InterruptedException {// TODO Auto-generated method stubif (start == end) {return 0.0f;} else {return Math.min(1.0f, (pos - start) / (float)(end - start));}}@Overridepublic void close() throws IOException {// TODO Auto-generated method stubif (fileIn != null) {        fileIn.close();     }}}

coAuOutputFormat

package an.hadoop.code.audit;/** * the name of the output file name *  * */import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;public class coAuOutputFormat extends MultipleOutputFormat<Text, Text> {private final static String suffix = "_its4";@Overrideprotected String generateFileNameForKeyValue(Text key, Text value,Configuration conf) {// TODO Auto-generated method stubString path =  key.toString(); //檔案的路徑及名字String[] dir = path.split("/");int length = dir.length; String filename = dir[length -1];return filename + suffix;//輸出的檔案名稱,輸出的檔案名稱}}

MultipleOutputFormat

package an.hadoop.code.audit;/** * the mutiply  * */import java.io.DataOutputStream;import java.io.IOException;import java.util.HashMap;import java.util.Iterator;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataOutputStream;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.Writable;import org.apache.hadoop.io.WritableComparable;import org.apache.hadoop.io.compress.CompressionCodec;import org.apache.hadoop.io.compress.GzipCodec;import org.apache.hadoop.mapreduce.OutputCommitter;import org.apache.hadoop.mapreduce.RecordWriter;import org.apache.hadoop.mapreduce.TaskAttemptContext;import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.util.ReflectionUtils;public abstract class MultipleOutputFormat<K extends WritableComparable<?>, V extends Writable>extends FileOutputFormat<K, V> { //預設的是TextOutputFormatprivate MultiRecordWriter writer = null;public RecordWriter<K, V> getRecordWriter(TaskAttemptContext job) throws IOException,InterruptedException {if (writer == null) {writer = new MultiRecordWriter(job, getTaskOutputPath(job));//job ,output path}return writer;}private Path getTaskOutputPath(TaskAttemptContext conf) throws IOException {//獲得輸出路徑Path workPath = null;OutputCommitter committer = super.getOutputCommitter(conf);if (committer instanceof FileOutputCommitter) {//如果是workPath = ((FileOutputCommitter) committer).getWorkPath();//工作路徑} else {Path outputPath = super.getOutputPath(conf);//獲得conf路徑if (outputPath == null) {throw new IOException("Undefined job output-path");}workPath = outputPath;}return workPath; //}/**通過key, value, conf來確定輸出檔案名（含副檔名）*/protected abstract String generateFileNameForKeyValue(K key, V value, Configuration conf);//抽象方法,被之後的方法重寫了public class MultiRecordWriter extends RecordWriter<K, V> {/**RecordWriter的緩衝*/private HashMap<String, RecordWriter<K, V>> recordWriters = null;private TaskAttemptContext job = null;/**輸出目錄*/private Path workPath = null;public MultiRecordWriter(TaskAttemptContext job, Path workPath) {//建構函式super();this.job = job;this.workPath = workPath;recordWriters = new HashMap<String, RecordWriter<K, V>>();}@Overridepublic void close(TaskAttemptContext context) throws IOException, InterruptedException {//多個writer都要關掉Iterator<RecordWriter<K, V>> values = this.recordWriters.values().iterator();while (values.hasNext()) {values.next().close(context);}this.recordWriters.clear();}@Overridepublic void write(K key, V value) throws IOException, InterruptedException {//得到輸出檔案名String baseName = generateFileNameForKeyValue(key, value, job.getConfiguration());//產生輸出檔案名RecordWriter<K, V> rw = this.recordWriters.get(baseName);//？？if (rw == null) {rw = getBaseRecordWriter(job, baseName);//this.recordWriters.put(baseName, rw);}rw.write(key, value);}// ${mapred.out.dir}/_temporary/_${taskid}/${nameWithExtension}private RecordWriter<K, V> getBaseRecordWriter(TaskAttemptContext job, String baseName)throws IOException, InterruptedException {Configuration conf = job.getConfiguration();boolean isCompressed = getCompressOutput(job);String keyValueSeparator = ",";RecordWriter<K, V> recordWriter = null;if (isCompressed) {Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job,GzipCodec.class);CompressionCodec codec = ReflectionUtils.newInstance(codecClass, conf);Path file = new Path(workPath, baseName + codec.getDefaultExtension());FSDataOutputStream fileOut = file.getFileSystem(conf).create(file, false);recordWriter = new LineRecordWriter<K, V>(new DataOutputStream(codec.createOutputStream(fileOut)), keyValueSeparator);} else {Path file = new Path(workPath, baseName);FSDataOutputStream fileOut = file.getFileSystem(conf).create(file, false);//file 是指的file name of the output filerecordWriter = new LineRecordWriter<K, V>(fileOut, keyValueSeparator);//這裡調用的LineRecordWriter}return recordWriter;}}}

LineRecordWriter

package an.hadoop.code.audit;/*RecordWriter的一個實現，用於把<Key, Value>轉化為一行文本。在Hadoop中，這個類作為TextOutputFormat的一個子類存在， * protected存取權限，因此普通程式無法訪問。這裡僅僅是把LineRecordWriter從TextOutputFormat抽取出來，作為一個獨立的公用*/import java.io.DataOutputStream;import java.io.IOException;import java.io.UnsupportedEncodingException;import org.apache.hadoop.io.NullWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.RecordWriter;import org.apache.hadoop.mapreduce.TaskAttemptContext;import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;/**摘自{@link TextOutputFormat}中的LineRecordWriter。 */public class LineRecordWriter<K, V> extends RecordWriter<K, V> {private static final String utf8 = "UTF-8";private static final byte[] newline;static {try {newline = "\n".getBytes(utf8);// 相當與分隔字元} catch (UnsupportedEncodingException uee) {throw new IllegalArgumentException("can't find " + utf8 + " encoding");}}protected DataOutputStream out;private final byte[] keyValueSeparator;public LineRecordWriter(DataOutputStream out, String keyValueSeparator) {this.out = out;try {this.keyValueSeparator = keyValueSeparator.getBytes(utf8);} catch (UnsupportedEncodingException uee) {throw new IllegalArgumentException("can't find " + utf8 + " encoding");}}public LineRecordWriter(DataOutputStream out) {this(out, "/t");//"/t"預設的分隔字元}private void writeObject(Object o) throws IOException {//被write函數調用if (o instanceof Text) {//Text to = (Text) o;out.write(to.getBytes(), 0, to.getLength());//將指定 byte 數組中從位移量 off 開始的 len 個位元組寫入基礎輸出資料流} else {out.write(o.toString().getBytes(utf8));}}public synchronized void write(K key, V value) throws IOException {//這個要修改成 只是寫成一個檔案的格式,boolean nullKey = key == null || key instanceof NullWritable;boolean nullValue = value == null || value instanceof NullWritable;//重點是要改寫Key，value，之類，value是一個文本，key是地址，這裡不寫入key了if (nullKey && nullValue) {return;}/*if (!nullKey) {//這個可以控制是否寫入key,seperate and valuewriteObject(key);}if (!(nullKey || nullValue)) {out.write(keyValueSeparator);}*/if (!nullValue) {writeObject(value);}out.write(newline);}public synchronized void close(TaskAttemptContext context) throws IOException {out.close();}}

CodeAudit

package an.hadoop.code.audit;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.NullWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.util.GenericOptionsParser;public class CodeAudit {public static void main(String[] args) throws Exception {    Configuration conf = new Configuration();    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();    if (otherArgs.length != 2) {      System.err.println("Usage: code audit <in> <out>");      System.exit(2);    }    Job job = new Job(conf, "code audit");    job.setJarByClass(CodeAudit.class);    job.setMapperClass(coAuMapper.class);    job.setInputFormatClass(coAuInputFormat.class);    //job.setOutputKeyClass(NullWritable.class);    job.setOutputKeyClass(Text.class);    job.setOutputValueClass(Text.class);        job.setOutputFormatClass(coAuOutputFormat.class);    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));    System.exit(job.waitForCompletion(true) ? 0 : 1);  }}