Mport Java.io.IOException;
Import Java.util.StringTokenizer;
Import org.apache.hadoop.conf.Configuration;
Import Org.apache.hadoop.fs.Path;
Import org.apache.hadoop.io.IntWritable;
Import Org.apache.hadoop.io.Text;
Import Org.apache.hadoop.mapreduce.Job;
Import Org.apache.hadoop.mapreduce.Mapper;
Import Org.apache.hadoop.mapreduce.Reducer;
Import Org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
Import Org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob;
Import Org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl;
Import Org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
Import Org.apache.hadoop.util.GenericOptionsParser;
public class Driver {
public static class Tokenizermapper extends
Mapper<object, text, text, intwritable> {
Private final static intwritable one = new intwritable (1);
Private text Word = new text ();
public void Map (Object key, Text value, context context)
Throws IOException, Interruptedexception {
StringTokenizer ITR = new StringTokenizer (value.tostring ());
while (Itr.hasmoretokens ()) {
Word.set (Itr.nexttoken ());
Context.write (Word, one);
}
}
}
public static class Intsumreducer extends
Reducer<text, Intwritable, Text, intwritable> {
Private intwritable result = new intwritable ();
public void reduce (Text key, iterable<intwritable> values,
Context context) throws IOException, Interruptedexception {
int sum = 0;
for (intwritable val:values) {
Sum + = Val.get ();
}
Result.set (sum);
Context.write (key, result);
}
}
public static class Dependencemapper extends
Mapper<object, text, text, text> {
Private text Word = new text ();
public void Map (Object key, Text value, context context)
Throws IOException, Interruptedexception {
String []sep=value.tostring (). Split ("\ t");
Word.set (sep[1]+ "\ T" +sep[0]);
System.out.println (Value.tostring ());
Context.write (word,new Text (""));
}
}
public static class Dependencereducer extends
reducer<text,text,text,text> {
public void reduce (Text key, iterable<text> values,
Context context) throws IOException, Interruptedexception {
string[] Sep = key.tostring (). Split ("\ t");
System.out.println (sep[0]+ "++++++++=" + sep[1]);
Context.write (key,new Text (""));
}
}
public static void Main (string[] args) throws Exception {
Configuration conf = new configuration ();
string[] Otherargs = new Genericoptionsparser (conf, args)
. Getremainingargs ();
if (Otherargs.length < 2) {
System.err.println ("Usage:wordcount <in> <out>");
System.exit (2);
}
Job Job = new Job (conf, "word count");
Adding control containers
Controlledjob ctrljob1=new controlledjob (conf);
Ctrljob1.setjob (Job);
Job.setjarbyclass (Driver.class);
Job.setmapperclass (Tokenizermapper.class);
Job.setcombinerclass (Intsumreducer.class);
Job.setreducerclass (Intsumreducer.class);
Job.setoutputkeyclass (Text.class);
Job.setoutputvalueclass (Intwritable.class);
Fileinputformat.addinputpath (Job, New Path (Otherargs[0]));
Fileoutputformat.setoutputpath (Job, New Path (Otherargs[1]));
Job.waitforcompletion (TRUE);
Configuration Conf2 = new configuration ();
Job job2 = new Job (CONF2, "word count1");
Controlledjob ctrljob2=new controlledjob (conf);
Ctrljob2.setjob (JOB2);
Ctrljob2.adddependingjob (CTRLJOB1);
Job2.setjarbyclass (Driver.class);
Job2.setmapperclass (Dependencemapper.class);
Job2.setreducerclass (Dependencereducer.class);
Job2.setoutputkeyclass (Text.class);
Job2.setoutputvalueclass (Text.class);
Fileinputformat.addinputpath (JOB2, New Path (Otherargs[1]));
Fileoutputformat.setoutputpath (JOB2, New Path (otherargs[2]));
Job2.waitforcompletion (TRUE);
Jobcontrol jobctrl=new Jobcontrol ("Myctrl");
Added to the total Jobcontrol for control
Jobctrl.addjob (CTRLJOB1);
Jobctrl.addjob (CTRLJOB2);
Jobctrl.run ();
}
}
MapReduce dependency Combination