MapReduce--—— ask 22 common friends

A:b,c,d,f,e,o
B:a,c,e,k
C:f,a,d,i
D:a,e,f,l
E:b,c,d,m,l
F:a,b,c,d,e,o,m
G:a,c,d,e,f
H:a,c,d,e,o
I:a,o
J:b,o
K:a,c,d
L:d,e,f
M:e,f,g
O:a,h,i,j,k

The above is the data:
A:b,c,d,f,e,o
Indicates: B,c,d,e,f,o is a user's friend.

1. Seek common friends among all 22 users

/** * @author: LPJ * @date: March 16, 2018 PM 7:16:47 * @Description:*/Package lpj.reducework; import java.io.ioexception;import java.util.Arrays; import Org.apache.hadoop.conf.configuration;import Org.apache.hadoop.fs.filesystem;import Org.apache.hadoop.fs.Path; Import Org.apache.hadoop.io.intwritable;import Org.apache.hadoop.io.longwritable;import Org.apache.hadoop.io.Text ; Import Org.apache.hadoop.mapreduce.job;import Org.apache.hadoop.mapreduce.mapper;import Org.apache.hadoop.mapreduce.reducer;import Org.apache.hadoop.mapreduce.lib.input.fileinputformat;import Org.apache.hadoop.mapreduce.lib.jobcontrol.controlledjob;import Org.apache.hadoop.mapreduce.lib.jobcontrol.jobcontrol;import Org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;/** * */ Public classSAMEFRIENDSMR { Public Static voidMain (string[] args) throws Exception {Configuration conf=NewConfiguration ();//Conf.addresource ("Hdfs-site.xml");//using configuration Files//system.setproperty ("Hadoop_user_name", "HADOOP");//using the cluster        ///////////////////First Step////////////////////////FileSystem fs = FileSystem.Get(conf);//default Use localJob Job=job.getinstance (conf); Job.setjarbyclass (SAMEFRIENDSMR.class); Job.setmapperclass (samefriendsmr_mapper.class); Job.setreducerclass (samefriendsmr_reducer.class); Job.setmapoutputkeyclass (Text.class); Job.setmapoutputvalueclass (Text.class); Job.setoutputkeyclass (Text.class); Job.setoutputvalueclass (Text.class); Path InputPath=NewPath ("D:/a/homework4.txt"); Path OutputPath=NewPath ("D:/a/homework4"); if(Fs.exists (InputPath)) {Fs.delete (OutputPath,true);        } fileinputformat.setinputpaths (Job, InputPath);        Fileoutputformat.setoutputpath (Job, OutputPath); ////////////////////Second step//////////////////////////////////FileSystem FS2 = FileSystem.Get(conf);//default Use localJob job2=job.getinstance (conf); Job2.setjarbyclass (SAMEFRIENDSMR.class); Job2.setmapperclass (samefriends2mr_mapper.class); Job2.setreducerclass (samefriends2mr_reducer.class); Job2.setmapoutputkeyclass (Text.class); Job2.setmapoutputvalueclass (Text.class); Job2.setoutputkeyclass (Text.class); Job2.setoutputvalueclass (Text.class); Path inputPath2=NewPath ("D:/a/homework4"); Path outputPath2=NewPath ("D:/a/homework4_1"); if(Fs2.exists (inputPath2)) {Fs2.delete (outputPath2,true);        } fileinputformat.setinputpaths (Job2, inputPath2);                Fileoutputformat.setoutputpath (JOB2, outputPath2); //2 mapreduce serial with JobcontrolControlledjob Ajob =NewControlledjob (Job.getconfiguration ()); Controlledjob bjob=NewControlledjob (Job2.getconfiguration ());        Ajob.setjob (Job);        Bjob.setjob (JOB2); Bjob.adddependingjob (ajob);//To specify a dependency relationshipJobcontrol JC=NewJobcontrol ("JcF");        Jc.addjob (Ajob);                Jc.addjob (bjob); Thread Thread=NewThread (JC);        Thread.Start ();  while(!jc.allfinished ()) {Thread.Sleep ( +);    } jc.stop (); }    ///////////////////////////////First Step/////////////////////////////////////////     Public Static classSamefriendsmr_mapper extends Mapper<longwritable, text, text, text>{Text Kout=NewText (); Text Valueout=NewText (); @Overrideprotected voidmap (longwritable key, Text Value,context Context) throws IOException, interruptedexception {//A:b,c,d,f,e,oString [] reads = Value.tostring (). Trim (). Split (":"); //The user is the value , the friend is the key value, to find a user as a common friend of the peopleString VV = reads[0]; Valueout.Set(VV); String [] Friends= reads[1].split (",");  for(inti =0; i < friends.length; i++) {String KK=Friends[i]; Kout.Set(KK);            Context.write (Kout, valueout); }        }    }     Public Static classSamefriendsmr_reducer extends Reducer<text, text, text, text>{Text Kout=NewText (); Text Valueout=NewText (); @Overrideprotected voidReduce (Text key, iterable<text>values, Context context) throws IOException, interruptedexception {//combine the people who share your friendsStringBuilder SB =NewStringBuilder ();  for(Text text:values) {sb.append (text.tostring ()). Append (","); } String VV= Sb.substring (0, Sb.length ()-1); Valueout.Set(VV);        Context.write (key, valueout); }            }    //////////////////////////////////////Second step///////////////////////////////////////////////////////////     Public Static classSamefriends2mr_mapper extends Mapper<longwritable, text, text, text>{Text Kout=NewText (); Text Valueout=NewText (); @Overrideprotected voidmap (longwritable key, Text Value,context Context) throws IOException, interruptedexception {//a f,i,o,k,g,d,c,h,b: means that f,i,o,k,g,d,c,h,b's common friend is a, the second step executes to get 22 friends, A is value, 22 pair is keyString [] reads = Value.tostring (). Trim (). Split ("\ t"); //A is ValueString VV = reads[0]; Valueout.Set(VV); //ask 22 friends to, in order to prevent duplication, need to sort friendsString [] Friends = reads[1].split (",");            Arrays.sort (Friends); //use a double loop to find a group of friends             for(inti =0; I < friends.length-1; i++){                 for(intj = i +1; J < Friends.length; J + +) {String KK= Friends[i] +"-"+Friends[j]; Kout.Set(KK);                Context.write (Kout, valueout); }            }        }    }     Public Static classSamefriends2mr_reducer extends Reducer<text, text, text, text>{Text Kout=NewText (); Text Valueout=NewText (); @Overrideprotected voidReduce (Text key, iterable<text>values, Context context) throws IOException, interruptedexception {//friends to group togetherStringBuilder SB =NewStringBuilder ();  for(Text text:values) {sb.append (text.tostring ()). Append (","); } String VV= Sb.substring (0, Sb.length ()-1); Valueout.Set(VV);        Context.write (key, valueout); }            }}

MapReduce--—— ask 22 common friends