Hadoop Two-time sort

Import Java.io.datainput;import java.io.dataoutput;import Java.io.file;import Java.io.ioexception;import Java.util.stringtokenizer;import Org.apache.hadoop.conf.configuration;import Org.apache.hadoop.fs.FileSystem; Import Org.apache.hadoop.fs.path;import Org.apache.hadoop.io.datainputbuffer;import Org.apache.hadoop.io.intwritable;import Org.apache.hadoop.io.longwritable;import Org.apache.hadoop.io.nullwritable;import Org.apache.hadoop.io.rawcomparator;import Org.apache.hadoop.io.Text; Import Org.apache.hadoop.io.writablecomparable;import Org.apache.hadoop.io.writablecomparator;import Org.apache.hadoop.io.writableutils;import Org.apache.hadoop.io.compress.compressioncodec;import Org.apache.hadoop.io.compress.lz4codec;//import Org.apache.hadoop.io.compress.gzipcodec;import Org.apache.hadoop.io.compress.snappycodec;import Org.apache.hadoop.mapreduce.lib.input.fileinputformat;import Org.apache.hadoop.mapreduce.lib.output.fileoutputformat;import org.apache.hadoop.mapreduce.lib.output.SequenceFileoutputformat;import Org.apache.hadoop.mapreduce.job;import Org.apache.hadoop.mapreduce.mapper;import Org.apache.hadoop.mapreduce.partitioner;import Org.apache.hadoop.mapreduce.reducer;import Org.apache.hadoop.util.genericoptionsparser;import com.hadoop.compression.lzo.lzocodec;//002484 18.29//600879 12.89public class Secondsotrstr {public static class Strpair implements writablecomparable<strpair> {private Text F Irst;private text second;private text third;private text Fourth;public Strpair () {Set (new text (), new text (), new text (), New Text ());} public void Set (text left, text right, text third, text fourth) {This.first = Left;this.second = Right;this.third = third; This.fourth = Fourth;} Public Text GetFirst () {return first;} Public Text Getsecond () {return second;} Public Text Getthird () {return third;} Public Text Getfourth () {return fourth;} @Overridepublic String toString () {return first + "\ T" + second + "\ T" + third + "\ T" + fourth;} @Overridepublic void ReadFields (datainput In) throws IOException {First.readfields (in), Second.readfields (in), Third.readfields (in), Fourth.readfields (in);} @Overridepublic void Write (DataOutput out), throws IOException {First.write (out), Second.write (out), Third.write (out); Fourth.write (out);} @Overridepublic int hashcode () {return First.hashcode () * 157 + second.hashcode () * + + Third.hashcode ();} @Overridepublic Boolean equals (Object right) {if instanceof Strpair) {Strpair r = (strpair) Right;return First.equa LS (r.first) && second.equals (R.second) && third.equals (R.third) && fourth.equals (R.fourth);} else {return false;}} /** A Comparator that compares serialized Strpair. */public Static class Comparator extends Writablecomparator {private static final text.comparator Text_comparator = new Te Xt. Comparator ();p ublic Comparator () {super (strpair.class);} Sort comparer, data all exist byte array public int compare (byte[] b1, int s1, int L1, byte[] B2, int s2,int L2) {//binary array read/** try {//system . OUT.PRINTLN ("--" + b1[s1]); IntegeR firstL1 =* writableutils.decodevintsize (b1[s1]) + Readvint (B1, S1); *//string str = READST//System.out.println ("Firstl 1 = "+* firstL1);} catch (IOException e) {//TODO auto-generated* catch block E.printstacktrace ();} *///int intvalue = ReadInt (B1, S1);/** int third = 0; for (int i =s1 + 9; i<= s1+; i++) {third +=* (b1[i]&0xff) << (24-8*i);} System.out.println ("third =" +* third); */system.out.println ("L1 =" + L1); return Comparebytes (B1, S1, L1, B2, S2, L2);/*t ry {int firstl1 = Writableutils.decodevintsize (b1[s1]) + Readvint (B1, s1); int firstl2 = Writableutils.decodevintsize (b2[ S2]) + readvint (B2, s2); int cmp = Text_comparator.compare (B1, S1, Firstl1, B2, S2,FIRSTL2); if (cmp! = 0) return Cmp;int firs tl12 = writableutils.decodevintsize (b1[s1 + FIRSTL1]) + Readvint (B1, S1 + firstl1); int firstl22 = Writableutils.decodevint Size (B2[s2 + firstl2]) + readvint (b2, s2 + firstl2), CMP = Text_comparator.compare (B1, S1 + firstl1, firstl12, B2, S2 + firs TL2,FIRSTL22); if (cmp! =0) return Cmp;int firstl13 = writableutils.decodevintsize (b1[s1+ firstl1 + firstl12]) + Readvint (B1, S1 + FIRSTL1 + firstl int firstl23 = writableutils.decodevintsize (b2[s2 + FIRSTL2 + firstl22]) + readvint (b2, s2 + firstl2 + firstl22); cmp = Text_comparator.compare (B1, s1+ Firstl1 + firstl12, firstl13, B2, s2 + firstl2 + firstl22,firstl23);//if (cmp! = 0) return Cmp;return Text_comparator.compare (B1, S1 + firstl1, L1-FIRSTL1, B2, S2 + firstl2, L1-FIRSTL2);} catch (IOException e) {throw new IllegalArgumentException (e);} */}}static {//register this comparatorwritablecomparator.define (Strpair.class, New Comparator ());} @Overridepublic int compareTo (Strpair o) {/** if (first! = O.first) {return first* < O.first? -1:1;} else if (s econd*! = O.second) {return second <* o.second? -1:1;} else if (third*! = O.third) {//return third <* O.third? -1:1;} * * return 0;*/return 0;}} /*** Partition based on the first part of the Pair.*/public static class Firstpartitioner Extends Partitioner<strpair, text> {@Override//public int getpartition (strpair key, Text value, int numpartitions) {return Math.Abs (Key.getfirst (). Hashcode () * 127)% Numpartitions;}} /*** Compare only the first part of the of the pair, so that reduce is called once* for each value of the first part.*///call here Pub LIC static class Firstgroupingcomparator implementsrawcomparator<strpair> {@Overridepublic int compare (byte[] B1 , int s1, int L1, byte[] b2, int s2, int l2) {return writablecomparator.comparebytes (B1, S1, integer.size/8,b2, S2, Inte Ger. SIZE/8);} @Overridepublic int Compare (Strpair O1, Strpair O2) {System.out.println ("-----group2-----"); Text L = O1.getfirst (); Text r = O2.getfirst (); return l.equals (R)? 0:1;//return L = = r? 0: (L < R -1:1);}} /*** Read integers from each line and generate a key, the value pair as* ((left, right), right). */public Static Class MAPC Lass extendsmapper<longwritable, Text, Strpair, nullwritable> {private final Strpair key = newStrpair ();p rivate final intwritable value = new intwritable ();p rivate text left = new text ();p rivate text right = new text ();p rivate Text third = new text ();p rivate Text Fourth = new text (); @Overridepublic void Map (longwritable inkey, Text Inva Lue, Context context) throws IOException, interruptedexception {System.out.println ("value" + invalue.tostring ()); StringTokenizer ITR = new StringTokenizer (invalue.tostring ()), if (Itr.hasmoretokens ()) {Left.set ((Itr.nexttoken ())); if (Itr.hasmoretokens ()) {Right.set (Itr.nexttoken ())), if (Itr.hasmoretokens ()) {Third.set (Itr.nexttoken ())), if ( Itr.hasmoretokens ()) {Fourth.set (Itr.nexttoken ());}}} Key.set (left, right, third, fourth);//Value.set (right); Context.write (Key, Nullwritable.get ());}}} /*** A Reducer class that just emits the sum of the input values.*/public static class Reduce Extendsreducer<strpair, N ullwritable, text, nullwritable> {private static final text SEPARATOR = new text ("----------------------------------- -------------");p rivate FInal text first = new text (); @Overridepublic void reduce (Strpair key, iterable<nullwritable> values,context Context ) throws IOException, interruptedexception {//Text Outkey = new Text (key.to);//Context.write (SEPARATOR, NULL);//First.s ET (Integer.tostring (Key.getfirst ()));//System.out.println ("Key1" + key); for (nullwritable value:values) {System.out . println ("Key2" + key), Context.write (New Text (Key.tostring ()), Nullwritable.get ());}}} private static Boolean Flag;public static Boolean DeleteFile (String spath) {flag = false; File File = new file (spath),//The path is a file and is not empty then delete if (File.isfile () && file.exists ()) {file.delete (); flag = true;} return flag;} public static Boolean deletedirectory (String spath) {//If spath does not end with a file delimiter, automatically adds the file delimiter if (!spath.endswith (file.separator) {spath = spath + file.separator;} File Dirfile = new file (spath); if the dir corresponding file does not exist or is not a directory, exit if (!dirfile.exists () | | |!dirfile.isdirectory ()) {return false;} Flag = true;//Delete all files (including subdirectories) under folder file[] files = dirfile. Listfiles (); for (int i = 0; i < files.length; i++) {//delete sub-file if (Files[i].isfile ()) {flag = DeleteFile (FILES[I].GETABSO Lutepath ()); if (!flag) break;} Delete subdirectory else {flag = DeleteDirectory (Files[i].getabsolutepath ())); if (!flag) break;}} if (!flag) return false;//Delete the current directory if (Dirfile.delete ()) {return true;} else {return false;}} public static void Main (string[] args) throws Exception {configuration conf = new Configuration ();/** Conf.setboolean ("Map Reduce.map.output.compress ", true); *//conf.setboolean (" Mapreduce.output.fileoutputformat.compress ", * false); * Conf.setclass ("Mapreduce.output.fileoutputformat.compress.codec", * gzipcodec.class, compressioncodec.class); */// gzip/** Conf.setboolean ("Mapreduce.map.output.compress", true); * Conf.setclass (" Mapreduce.map.output.compression.codec ", * gzipcodec.class, Compressioncodec.class); * Conf.setboolean (" Mapreduce.output.fileoutputformat.compress ", true); * Conf.setclass (" Mapreduce.output.fileoutputformat.compress.codec ", * gzipcodec.class, CompressioNcodec.class) */conf.set ("Mapreduce.map.log.level", "DEBUG");//snappy/** Conf.setboolean (" Mapreduce.map.output.compress ", true); * Conf.setclass (" Mapreduce.map.output.compression.codec ", * Snappycodec.class, Compressioncodec.class); * Conf.setboolean ("mapreduce.output.fileoutputformat.compress", false) ; * Conf.setclass ("Mapreduce.output.fileoutputformat.compress.codec", * snappycodec.class, Compressioncodec.class); */string[] Otherargs = new Genericoptionsparser (conf, args). Getremainingargs (); if (otherargs.length! = 2) { System.err.println ("Usage:secondsotrstr <in> <out>"); System.exit (2);} Path OutputDir = new Path (otherargs[1]); FileSystem fs = Filesystem.get (conf), if (Fs.exists (OutputDir)) {Fs.delete (OutputDir, True);} Job Job = new Job (conf, "secondary sort"); Job.setjarbyclass (Secondsotrstr.class); Job.setmapperclass (Mapclass.class); Job.setreducerclass (Reduce.class);/** Conf.setboolean ("Mapred.output.compress", true); * Conf.setclass ("Mapred.output.compression.codec", Gzipcodec. class,* compressioncodec.class); * Conf.setclass ("Mapred.output.compression.codec", snappycodec.class,* COMPRESSIONCODEC.CLASS); * * Conf.setboolean ("reduce.output.compress", true); * Conf.setclass ("Mapred.output.compression.codec", gzipcodec.class,* compressioncodec.class); * Conf.setclass (" Reduce.output.compression.codec ", snappycodec.class,* compressioncodec.class); * */* Conf.setboolean (" Mapreduce.output.compress ", true); * Conf.setclass (" Mapreduce.output.compression.codec ", gzipcodec.class,* Compressioncodec.class) *///Group and partition by the first int in the Pairjob.setpartitionerclass ( Firstpartitioner.class); Job.setgroupingcomparatorclass (firstgroupingcomparator.class);//The map output is StrPair, Intwritablejob.setmapoutputkeyclass (Strpair.class); Job.setmapoutputvalueclass (nullwritable.class);//The Reduce Output is Text, Intwritablejob.setoutputkeyclass (Text.class); Job.setoutputvalueclass (nullwritable.class);//lzo/** Conf.setboolean ("Mapreduce.map.output.compress", true);* Conf.setclass ("Mapreduce.map.output.compression.codec", * lzocodec.class, compressioncodec.class); * Conf.setboolean ("Mapreduce.output.fileoutputformat.compress", true); * Conf.setclass (" Mapreduce.output.fileoutputformat.compress.codec ", * lzocodec.class, Compressioncodec.class); *///block compression// Job.setoutputformatclass (Sequencefileoutputformat.class); Conf.set ("Mapred.output.compression.type", "BLOCK"); Fileinputformat.addinputpath (Job, New Path (Otherargs[0])); Fileoutputformat.setoutputpath (Job, New Path (Otherargs[1])); System.exit (Job.waitforcompletion (true)? 0:1);}}

　　

Hadoop Two-time sort