MapReduce dependency Combination

Source: Internet
Author: User

Mport Java.io.IOException;
Import Java.util.StringTokenizer;

Import org.apache.hadoop.conf.Configuration;
Import Org.apache.hadoop.fs.Path;
Import org.apache.hadoop.io.IntWritable;
Import Org.apache.hadoop.io.Text;
Import Org.apache.hadoop.mapreduce.Job;
Import Org.apache.hadoop.mapreduce.Mapper;
Import Org.apache.hadoop.mapreduce.Reducer;
Import Org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
Import Org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob;
Import Org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl;
Import Org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
Import Org.apache.hadoop.util.GenericOptionsParser;

public class Driver {

public static class Tokenizermapper extends
Mapper<object, text, text, intwritable> {
Private final static intwritable one = new intwritable (1);
Private text Word = new text ();
public void Map (Object key, Text value, context context)
Throws IOException, Interruptedexception {
StringTokenizer ITR = new StringTokenizer (value.tostring ());
while (Itr.hasmoretokens ()) {
Word.set (Itr.nexttoken ());
Context.write (Word, one);
}
}
}

public static class Intsumreducer extends
Reducer<text, Intwritable, Text, intwritable> {
Private intwritable result = new intwritable ();
public void reduce (Text key, iterable<intwritable> values,
Context context) throws IOException, Interruptedexception {
int sum = 0;
for (intwritable val:values) {
Sum + = Val.get ();
}
Result.set (sum);
Context.write (key, result);
}
}

public static class Dependencemapper extends
Mapper<object, text, text, text> {
Private text Word = new text ();
public void Map (Object key, Text value, context context)
Throws IOException, Interruptedexception {
String []sep=value.tostring (). Split ("\ t");
Word.set (sep[1]+ "\ T" +sep[0]);
System.out.println (Value.tostring ());
Context.write (word,new Text (""));
}
}

public static class Dependencereducer extends
reducer<text,text,text,text> {
public void reduce (Text key, iterable<text> values,
Context context) throws IOException, Interruptedexception {
string[] Sep = key.tostring (). Split ("\ t");
System.out.println (sep[0]+ "++++++++=" + sep[1]);
Context.write (key,new Text (""));
}
}

public static void Main (string[] args) throws Exception {
Configuration conf = new configuration ();
string[] Otherargs = new Genericoptionsparser (conf, args)
. Getremainingargs ();
if (Otherargs.length < 2) {
System.err.println ("Usage:wordcount <in> <out>");
System.exit (2);
}
Job Job = new Job (conf, "word count");
Adding control containers
Controlledjob ctrljob1=new controlledjob (conf);
Ctrljob1.setjob (Job);
Job.setjarbyclass (Driver.class);
Job.setmapperclass (Tokenizermapper.class);
Job.setcombinerclass (Intsumreducer.class);
Job.setreducerclass (Intsumreducer.class);
Job.setoutputkeyclass (Text.class);
Job.setoutputvalueclass (Intwritable.class);
Fileinputformat.addinputpath (Job, New Path (Otherargs[0]));
Fileoutputformat.setoutputpath (Job, New Path (Otherargs[1]));
Job.waitforcompletion (TRUE);

Configuration Conf2 = new configuration ();
Job job2 = new Job (CONF2, "word count1");
Controlledjob ctrljob2=new controlledjob (conf);
Ctrljob2.setjob (JOB2);
Ctrljob2.adddependingjob (CTRLJOB1);
Job2.setjarbyclass (Driver.class);
Job2.setmapperclass (Dependencemapper.class);
Job2.setreducerclass (Dependencereducer.class);
Job2.setoutputkeyclass (Text.class);
Job2.setoutputvalueclass (Text.class);
Fileinputformat.addinputpath (JOB2, New Path (Otherargs[1]));
Fileoutputformat.setoutputpath (JOB2, New Path (otherargs[2]));
Job2.waitforcompletion (TRUE);
Jobcontrol jobctrl=new Jobcontrol ("Myctrl");

Added to the total Jobcontrol for control
Jobctrl.addjob (CTRLJOB1);
Jobctrl.addjob (CTRLJOB2);
Jobctrl.run ();

}
}

MapReduce dependency Combination

Related Article

Contact Us

The content source of this page is from Internet, which doesn't represent Alibaba Cloud's opinion; products and services mentioned on that page don't have any relationship with Alibaba Cloud. If the content of the page makes you feel confusing, please write us an email, we will handle the problem within 5 days after receiving your email.

If you find any instances of plagiarism from the community, please send an email to: info-contact@alibabacloud.com and provide relevant evidence. A staff member will contact you within 5 working days.

A Free Trial That Lets You Build Big!

Start building with 50+ products and up to 12 months usage for Elastic Compute Service

  • Sales Support

    1 on 1 presale consultation

  • After-Sales Support

    24/7 Technical Support 6 Free Tickets per Quarter Faster Response

  • Alibaba Cloud offers highly flexible support services tailored to meet your exact needs.