Hadoop MapReduce資料去重程式_Hadoop

    package com.hadoop.sample;            import java.io.IOException;      import java.util.StringTokenizer;            import org.apache.hadoop.conf.Configuration;      import org.apache.hadoop.fs.Path;      import org.apache.hadoop.io.IntWritable;      import org.apache.hadoop.io.Text;      import org.apache.hadoop.mapreduce.Job;      import org.apache.hadoop.mapreduce.Mapper;      import org.apache.hadoop.mapreduce.Reducer;      import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;      import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;      import org.apache.hadoop.util.GenericOptionsParser;            public class Dedup {          //map將輸入中的value複製到輸出資料的key上，並直接輸出          public static class Map extends Mapper<Object,Text,Text,Text>{              private static Text line = new Text();              public void map(Object key,Text value,Context context) throws IOException,InterruptedException{                  line = value;                  context.write(line, new Text(""));              }          }          //reduce將輸入中的key複製到輸出資料的key上，並直接輸出          public static class Reduce extends Reducer<Text,Text,Text,Text>{              public void reduce(Text key,Iterable<Text> values,Context context) throws IOException,InterruptedException{                  context.write(key, new Text(""));                                }          }          /**          * @param args          */          public static void main(String[] args) throws Exception{              // TODO Auto-generated method stub              Configuration conf = new Configuration();              String[] otherArgs = new GenericOptionsParser(conf,args).getRemainingArgs();              if(otherArgs.length != 2){                  System.err.println("Usage WordCount <int> <out>");                  System.exit(2);              }              Job job = new Job(conf,"Dedup");              job.setJarByClass(Dedup.class);              job.setMapperClass(Map.class);              job.setCombinerClass(Reduce.class);              job.setReducerClass(Reduce.class);              job.setOutputKeyClass(Text.class);              job.setOutputValueClass(Text.class);              FileInputFormat.addInputPath(job, new Path(otherArgs[0]));              FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));              System.exit(job.waitForCompletion(true) ? 0 : 1);          }            }