---Custom grouping rawcomparator for Hadoop

Data

3 3
3 2
3 2
2 2
2 1
1 1

---------------------

Demand:

11
22
33

The lowest value of the second column when the first column is the same

 PackageGroup;ImportJava.io.DataInput;ImportJava.io.DataOutput;Importjava.io.IOException;ImportJava.net.URI;Importorg.apache.hadoop.conf.Configuration;ImportOrg.apache.hadoop.fs.FileSystem;ImportOrg.apache.hadoop.fs.Path;Importorg.apache.hadoop.io.LongWritable;ImportOrg.apache.hadoop.io.RawComparator;ImportOrg.apache.hadoop.io.Text;Importorg.apache.hadoop.io.WritableComparable;ImportOrg.apache.hadoop.io.WritableComparator;ImportOrg.apache.hadoop.mapreduce.Job;ImportOrg.apache.hadoop.mapreduce.Mapper;ImportOrg.apache.hadoop.mapreduce.Reducer;ImportOrg.apache.hadoop.mapreduce.lib.input.FileInputFormat;ImportOrg.apache.hadoop.mapreduce.lib.input.TextInputFormat;ImportOrg.apache.hadoop.mapreduce.lib.output.FileOutputFormat;/*** * Custom Group K2 **/ Public classGroupapp {Private Static FinalString inputpaths = "Hdfs://hadoop:9000/data"; Private Static FinalString Out_path = "Hdfs://hadoop:9000/out";  Public Static voidMain (string[] args)throwsException {Configuration conf=NewConfiguration (); FileSystem FileSystem= Filesystem.get (NewURI (Out_path), conf); Filesystem.delete (NewPath (Out_path),true); Job Job=NewJob (conf, Groupapp.class. Getsimplename ()); Job.setjarbyclass (Groupapp.class);        Fileinputformat.setinputpaths (Job, inputpaths); Job.setinputformatclass (Textinputformat.class); Job.setmapperclass (mymapper.class); Job.setmapoutputkeyclass (longwritable.class); Job.setmapoutputvalueclass (longwritable.class); //set a custom grouping keyJob.setgroupingcomparatorclass (Mygropcomparator.class); Job.setreducerclass (myreducer.class); Job.setoutputkeyclass (longwritable.class); Job.setoutputvalueclass (longwritable.class); Fileoutputformat.setoutputpath (Job,NewPath (Out_path)); Job.waitforcompletion (true); }             Public Static classMymapperextendsMapper<longwritable, Text, NewK2, longwritable>{@Overrideprotected voidmap (longwritable key, Text value, Org.apache.hadoop.mapreduce.Mapper<longwritable, Text, NewK2, longwritable>. Context context)throwsIOException, interruptedexception {string[] split= Value.tostring (). Split ("\ T"); Context.write (NewNewK2 (Long.parselong (split[0]), Long.parselong (Split[1])),NewLongwritable (Long.parselong (split[1]))); }    }         /*** * K2 is not equal at this time, meaning that reduce receives 6 packets **/     Public Static classMyreducerextendsREDUCER&LT;NEWK2, Longwritable, longwritable, longwritable>{@Overrideprotected voidReduce (NewK2 key, iterable<longwritable>values, Org.apache.hadoop.mapreduce.Reducer&LT;NEWK2, Longwritable, longwritable, longwritable>. Context context)throwsIOException, interruptedexception {Long min=Long.max_value;  for(longwritable longwritable:values) {if(Longwritable.get () <min) {min=Longwritable.get (); }} context.write (NewLongwritable (Key.frist),Newlongwritable (min)); }    }                /*** * Custom sort * **/     Public Static classNewK2ImplementsWritablecomparable<newk2>{        LongFrist; Longsecond;  PublicNewK2 () {} PublicNewK2 (LongFristLongsecond) {             This. Frist =Frist;  This. Second =second; } @Override Public voidWrite (DataOutput out)throwsIOException {Out.writelong ( This. Frist); Out.writelong ( This. Frist); } @Override Public voidReadFields (Datainput in)throwsIOException { This. Frist =In.readlong ();  This. Second =In.readlong (); }        /*** Compare, sort by first column, sort by second column when the first column is the same*/@Override Public intcompareTo (NewK2 o) {Longminus = This. Frist-o.frist; if(minus! = 0){                //Not Equal                return(int) minus; }            //The first column is equal so that the second column is processed            return(int)( This. Second-O.second); }    }            /*** Custom Grouping **/         Public Static classMygropcomparatorImplementsRawcomparator<newk2>{@Override Public intCompare (NewK2 O1, NewK2 O2) {return0; }                /*** B1 equivalent to this b2 equivalent to O * S1 and S2 represents the length from which to start reading values from a very long byte array, * L1 and L2 processing lengths*/@Override Public intComparebyte[] B1,intS1,intL1,byte[] B2,intS2,intL2) {                        returnWritablecomparator.comparebytes (B1, S1, 8, B2, S2, 8); }            }     }

---Custom grouping rawcomparator for Hadoop