027_ Write MapReduce template classes Mapper, reducer, and Driver__mapper

Template class written after writing MapReduce program, the template class to write a good only need to change the parameters on the line, the code is as follows:

  1 package org.dragon.hadoop.mr.module;
  2 3 Import java.io.IOException;
  4 5 Import Org.apache.hadoop.conf.Configuration;
  6 Import org.apache.hadoop.conf.Configured;
  7 Import Org.apache.hadoop.fs.Path;
  8 Import org.apache.hadoop.io.LongWritable;
 9 Import Org.apache.hadoop.io.Text;
 Import Org.apache.hadoop.mapreduce.Job;
 Import Org.apache.hadoop.mapreduce.Mapper;
 Import Org.apache.hadoop.mapreduce.Reducer;
 Import Org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 Import Org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 Import Org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 Import Org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
 Import Org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;
 Import Org.apache.hadoop.util.Tool;
 Import Org.apache.hadoop.util.ToolRunner; /** * ########################################### * ############ MapReduce template class ##### ##### 25 *########################################### * @author Zhuxy * @time 2016-3-13 10:21:06 29 * 30 To public class Modulemapreduce extends configured implements Tool {/** * Mapper class 35 *
 /The public static class Modulemapper extends Mapper<longwritable, Text, longwritable, text> { @Override protected void Setup (context) throws IOException, Inter
 ruptedexception {super.setup (context);                 @Override protected void Map (longwritable key, Text value, context context) 47
 Throws IOException, interruptedexception {super.map (key, value, context);                 @Override protected void Cleanup (context) throws IOException, 53
 interruptedexception {super.cleanup (context); 
 55} 56} 57/** * Reducer class/The public static class Modulereducer extends reducer& Lt Longwritable, Text, longwritable, text> {@Override protected void Setup (context)             Throws IOException, interruptedexception {A/TODO auto-generated method stub 68
 Super.setup (context);
 @Override protected void reduce (longwritable key, iterable<text> values, Throws IOException, Interruptedexception {//TODO auto-generated Meth
 OD stub super.reduce (key, values, context);                 @Override protected void Cleanup (context) throws IOException, 80 interruptedexception {bayi//TODO auto-generated Method Stub super.cleanup (context)
 ; 83} 84 85} 86 87/** 88      * Driver Class 89 */90 91//Special extraction method for setting up the Parseinputandoutput public Job Tool Tool,config Uration conf,string[] args) throws IOException (args.length!=2) {System.err.prin
 TF ("usage:%s [generic options] <input> <output>\n", Tool.getclass (). Getsimplename ());
 Toolrunner.printgenericcommandusage (System.err);
 The return null; 98} 99 100//Create job and set configuration information and job name Job Job=new job (conf, ModuleMapReduce.class.getSim
Plename ()); 102 103//Set Job run class//step 3:set Job//1) set Run Jar class Job.set
Jarbyclass (Tool.getclass ());
108//) Job Output Path 109 fileoutputformat.setoutputpath (Job, New Path (args[1));
The return job; 112} 113 114 @Override The 117-up-to-public int run (string[] args) throws Exception {116/ SteP 1:get conf 118 Configuration conf = new Configuration ();
119//Step 2:create job 121 job = Parseinputandoutput (this, conf, args); 122 123 124//2) Set input format job.setinputformatclass (textinputformat.class);
Can province 126 127//3) Set input path 128 Fileinputformat.addinputpath (Job, New Path (args[0)); 129 130//4) Set Mapper class 131 Job.setmapperclass (Modulemapper.class); Can province 132//5) Set map input Key/value class 134 job.setmapoutputkeyclass (Longwritable.class); May province 135 Job.setmapoutputvalueclass (Text.class); Can province 136 137//6) Set Partitioner class 138 Job.setpartitionerclass (Hashpartitioner.class);         Can province 139 140//7) Set reducer number job.setnumreducetasks (1);//default 1//Can be 142 143 8) Set sort Comparator class 144//job.setsortcomparatorclass (LongWritable.Comparator.class); Can save 145
146//9) Set Group Comparator class 147//job.setgroupingcomparatorclass (LongWritable.Comparator.class);         148 149///Job.setcombinerclass (NULL) set combiner, default is NULL, but cannot be written here//can save 151 152 Set Reducer Class 153 Job.setreducerclass (Modulereducer.class); Can province 154//) Set output format 156 job.setoutputformatclass (Textoutputformat.class); Can province 157 158//) Job Output Key/value class 159 Job.setoutputkeyclass (Longwritable.class); May province 160 Job.setoutputvalueclass (Text.class);
Can province 161 162 163//Step 4:submit Job 164 Boolean issuccess = Job.waitforcompletion (true); 165 166//Step 5:return status 167 return issuccess?
0:1; 169 170 public static void main (string[] args) throws Exception {171 172 args = new Strin
G[] {173 "hdfs://hadoop-master.dragon.org:9000/wc/mininput/",174 "Hdfs://hadoop-master.dragon.org:9000/wc/minoutput" 175};
176 177//run MapReduce 178 int Status=toolrunner.run (new Modulemapreduce (), args);
179 180//exit 181 system.exit (status); 182} 183}

View Module Code

Template usage Steps:

1) Change name (MapReduce class name, Mapper class name, reducer class name)

2) Modify the type of key/value input and output parameters of the Mapper class and reducer class according to the actual business logic

3 Modify the parameter settings of the job that drives the driver section (Mapper class and Reducer class output type)

4 Write the actual business logic in the Mapper Class (Setup (), map (), Cleanup ())

5 Write the actual business logic in the Reducer Class (Setup (), map (), Cleanup ())

6 Check and modify the driver driver code (run () method in the template class)

7) Set the input and output path for the Mr Test.

Using Modulemapreduce to write WordCount programs

  1 package org.dragon.hadoop.mr.module;
  2 3 Import java.io.IOException;
  4 Import Java.util.StringTokenizer;
  5 6 Import Org.apache.hadoop.conf.Configuration;
  7 Import org.apache.hadoop.conf.Configured;
  8 Import Org.apache.hadoop.fs.Path;
 9 Import org.apache.hadoop.io.LongWritable;
 Import Org.apache.hadoop.io.Text;
 Import Org.apache.hadoop.mapreduce.Job;
 Import Org.apache.hadoop.mapreduce.Mapper;
 Import Org.apache.hadoop.mapreduce.Reducer;
 Import Org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 Import Org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 Import Org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 Import Org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
 Import Org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;
 Import Org.apache.hadoop.util.Tool;
 Import Org.apache.hadoop.util.ToolRunner;       21 22/** 23 * 24 * ########################################### 25 * ############ MapReduce template class ########## * ########################################### * @author Zhuxy * @tim E 2016-3-13 PM 10:21:06 * * Public class Wordcountbymodulemapreduce extends configured implements Tool {3 3/** * Mapper class/Notoginseng public static class Wordcountmapper extends Map per<longwritable, text, text, longwritable> {@Override protected void Setup (context Co
 ntext) throws IOException, Interruptedexception {super.setup (context);
 The private Text word = new text ();
 Private final static longwritable one = new longwritable (1);                 @Override protected void Map (longwritable key, Text value, context context) 51 Throws IOException, Interruptedexception {52 53//Get the value of each row of data a String Linevalue = Value.tos
 Tring ();      55 56       To divide the Stringtoken