Hadoop MapReduce 編寫例子

編寫一個簡單的WordCount例子

WordCount.java

/** * 簡單的單詞計數器 *//** * * @author Neo neosfung_gmail_com * @version 1.0 2012-11-11 */import java.io.IOException;import java.util.Iterator;import java.util.StringTokenizer;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapred.FileInputFormat;import org.apache.hadoop.mapred.FileOutputFormat;import org.apache.hadoop.mapred.JobClient;import org.apache.hadoop.mapred.JobConf;import org.apache.hadoop.mapred.MapReduceBase;import org.apache.hadoop.mapred.Mapper;import org.apache.hadoop.mapred.OutputCollector;import org.apache.hadoop.mapred.Reducer;import org.apache.hadoop.mapred.Reporter;import org.apache.hadoop.mapred.TextInputFormat;import org.apache.hadoop.mapred.TextOutputFormat;public class WordCount {public static class Map extends MapReduceBase implementsMapper<LongWritable, Text, Text, IntWritable> {private final static IntWritable one = new IntWritable(1);private final Text word = new Text();/** * map函數繼承自MapReduceBase, 並且實現了Mapper介面, 此介面是一個泛型型別,它有4種形式的參數, * 分別用來指定map的輸入key類型, 輸入value實值型別, 輸出key實值型別和輸出value實值型別. * 在本例中，輸入使用的是TextInputFormat, 它的輸出key值是LongWritable類型, 輸出value值是Text類型. * 根據本例, 需要輸出的是<Text, IntWritable>的形式, 所以輸出的key實值型別是Text, * 輸出的value類型是IntWritable. */@Overridepublic void map(LongWritable key, Text value,OutputCollector<Text, IntWritable> output, Reporter reporter)throws IOException {String line = value.toString();StringTokenizer tokenizer = new StringTokenizer(line);while (tokenizer.hasMoreTokens()) {word.set(tokenizer.nextToken());output.collect(word, one);}}}public static class Reduce extends MapReduceBase implementsReducer<Text, IntWritable, Text, IntWritable> {/** * reduce函數的輸入以map的輸出作對應, 因此reduce的輸入類型是<Text, IntWritable>. */@Overridepublic void reduce(Text key, Iterator<IntWritable> values,OutputCollector<Text, IntWritable> output, Reporter reporter)throws IOException {int sum = 0;while (values.hasNext()) {sum += values.next().get();}output.collect(key, new IntWritable(sum));}}/** * @param args * @throws IOException */public static void main(String[] args) throws IOException {// TODO Auto-generated method stubJobConf conf = new JobConf(WordCount.class);conf.setJobName("wordcount");// 設定輸出的key和value類型conf.setOutputKeyClass(Text.class);conf.setOutputValueClass(IntWritable.class);// 設定各個作業的類conf.setMapperClass(Map.class);conf.setCombinerClass(Reduce.class);conf.setReducerClass(Reduce.class);// 設定輸入輸出的格式conf.setInputFormat(TextInputFormat.class);conf.setOutputFormat(TextOutputFormat.class);FileInputFormat.setInputPaths(conf, new Path(args[0]));FileOutputFormat.setOutputPath(conf, new Path(args[1]));JobClient.runJob(conf);}}

編譯:

javac -classpath ~/hadoop/hadoop-core-1.0.3.jar WordCount.java

打包:

jar -cvf WordCount.jar -C ./ .

運行，其中input為hdfs上的輸入檔案夾:

hadoop jar WordCount.jar WordCount input output