hadoop2.2.0 MapReduce Partition

Package com.my.hadoop.mapreduce.partition;


Import Java.util.HashMap;
Import Java.util.Map;

Import org.apache.hadoop.conf.Configuration;
Import Org.apache.hadoop.fs.Path;
Import org.apache.hadoop.io.LongWritable;
Import Org.apache.hadoop.io.Text;
Import Org.apache.hadoop.mapreduce.Job;
Import Org.apache.hadoop.mapreduce.Mapper;
Import Org.apache.hadoop.mapreduce.Partitioner;
Import Org.apache.hadoop.mapreduce.Reducer;
Import Org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
Import Org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class Parcount {

public static class Parmap extends Mapper<longwritable, text, text, infobean>{
Private text key = new text ();
@Override
public void Map (longwritable key, Text value, Context context) throws Java.io.IOException, Interruptedexception {
string[] fields = Value.tostring (). Split ("\ t");
String Telno = fields[1];
Long uppayload = Long.parselong (fields[8]);
Long downpayload = Long.parselong (fields[9]);
Infobean bean = new Infobean (Telno, Uppayload, downpayload);
This.key.set (Telno);
Context.write (This.key, Bean);
}
}

public static class Parreduce extends Reducer<text, Infobean, Text, infobean>{
@Override
public void reduce (Text key, java.lang.iterable<infobean> value, Org.apache.hadoop.mapreduce.reducer<text, Infobean,text,infobean>. Context context) throws Java.io.IOException, Interruptedexception {
Long up_sum = 0;
Long down_sum = 0;
for (Infobean Bean:value) {
Up_sum + = Bean.getuppayload ();
Down_sum + = Bean.getdownpayload ();
}
Infobean bean = new Infobean ("", Up_sum, Down_sum);
Context.write (key, Bean);
}
}

/**
* partition, the output of the parameter map
* @author Yao
*
*/
public static class Mypar extends Partitioner<text, infobean>{

private static map<string, integer> Parflag = new hashmap<string, integer> ();
static {
Parflag.put ("135", 1);
Parflag.put ("136", 1);
Parflag.put ("137", 1);
Parflag.put ("138", 1);
Parflag.put ("139", 1);
Parflag.put ("150", 2);
Parflag.put ("159", 2);
Parflag.put ("182", 3);
Parflag.put ("183", 3);
}

@Override
public int getpartition (Text key, Infobean value, int arg2) {
String Telno = key.tostring (). substring (0, 3);
Integer code = parflag.get (Telno);
if (code = = null) {
Code = 0;
}

return code;
}

}

public static void Main (string[] args) throws Exception {
Configuration conf = new configuration ();
Job Job = job.getinstance (conf, ParCount.class.getSimpleName ());;
Job.setjarbyclass (Parcount.class);

Fileinputformat.setinputpaths (Job, New Path (Args[0]));
Job.setmapperclass (Parmap.class);
Job.setmapoutputkeyclass (Text.class);
Job.setmapoutputvalueclass (Infobean.class);

Job.setpartitionerclass (Mypar.class); Specifying a custom partition class
Job.setnumreducetasks (4); Need to set the number of reducer according to the number of partitions, more will appear empty files, less will be error

Job.setreducerclass (Parreduce.class);
Job.setoutputkeyclass (Text.class);
Job.setoutputvalueclass (Infobean.class);
Fileoutputformat.setoutputpath (Job, New Path (Args[1]));

System.exit (Job.waitforcompletion (true)? 0:1);

}

}

hadoop2.2.0 MapReduce Partition