Hadoop Two-time sort

Hadoop Two-time sort

1 3
1 2
1 1
3 3
3 2
2 2
2 1
3 1

After sorting:

1 1
1 2
1 3
2 1
2 2
3 1
3 2
3 3

The code is:

Package com.hadoop.test.SecondSort;


Import Java.io.DataInput;
Import Java.io.DataOutput;
Import java.io.IOException;


Import org.apache.hadoop.io.WritableComparable;


public class SortKey implements writablecomparable<sortkey> {
Private Long first;
Private Long second;


Public SortKey () {
}


Public SortKey (long first, long second) {
Super ();
This.first = First;
This.second = second;
}


@Override
public void Write (DataOutput out) throws IOException {
Out.writelong (This.first);
Out.writelong (This.second);


}


@Override
public void ReadFields (Datainput in) throws IOException {
This.first = In.readlong ();
This.second = In.readlong ();


}


@Override
public int compareTo (SortKey o) {
if (this.first! = O.first) {
return (int) (This.first-o.first);
} else {
return (int) (this.second-o.second);
}


}


@Override
public int hashcode () {
TODO auto-generated Method Stub
Return This.first.hashCode () +this.second.hashcode ();
}


@Override
public boolean equals (Object obj) {
if (! ( obj instanceof SortKey)) {
return false;
} else {
SortKey key = (SortKey) obj;
return (This.first = = Key.first && This.second = = Key.second);
}
}


Public Long GetFirst () {
return first;
}


public void Setfirst (Long first) {
This.first = First;
}


Public Long Getsecond () {
return second;
}


public void Setsecond (Long second) {
This.second = second;
}





}

Package com.hadoop.test.SecondSort;


Import java.io.IOException;


Import org.apache.hadoop.io.LongWritable;
Import Org.apache.hadoop.io.Text;
Import Org.apache.hadoop.mapreduce.Mapper;


public class Sortmapper extends Mapper<longwritable, Text, SortKey, longwritable> {


@Override
protected void Map (longwritable key, Text value,context Context)
Throws IOException, Interruptedexception {
String line = value.tostring ();
string[] arr = line.split ("\ t");
SortKey temp = new SortKey (Long.parselong (arr[0]), Long.parselong (arr[1]));
if (arr.length==2) {
Context.write (temp, New longwritable (Long.parselong (arr[1)));
}
}


}

Package com.hadoop.test.SecondSort;


Import java.io.IOException;


Import org.apache.hadoop.io.LongWritable;
Import Org.apache.hadoop.mapreduce.Reducer;


public class Sortreducer extends Reducer<sortkey, longwritable, longwritable, longwritable> {


@Override
protected void reduce (SortKey K2, iterable<longwritable> values,context Context)
Throws IOException, Interruptedexception {
for (longwritable value:values) {
Context.write (New Longwritable (K2.getfirst ()), New Longwritable (K2.getsecond ()));
}
}


}

Package com.hadoop.test.SecondSort;


Import org.apache.hadoop.conf.Configuration;
Import Org.apache.hadoop.fs.FileSystem;
Import Org.apache.hadoop.fs.Path;
Import org.apache.hadoop.io.LongWritable;
Import Org.apache.hadoop.mapreduce.Job;
Import Org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
Import Org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
/**
* Default key sort install ascending sort
* @author Xiaoming
*
*/
public class Jobmain {
public static void Main (string[] args) throws exception{
Configuration configuration = new configuration ();
Job Job = new Job (configuration, "Sort-sort");
Job.setjarbyclass (Jobmain.class);

Job.setmapperclass (Sortmapper.class);
Job.setmapoutputkeyclass (Sortkey.class);
Job.setmapoutputvalueclass (Longwritable.class);

Job.setreducerclass (Sortreducer.class);
Job.setoutputkeyclass (Longwritable.class);
Job.setoutputvalueclass (Longwritable.class);

Fileinputformat.addinputpath (Job, New Path (Args[0]));

Path outputdir= New Path (args[1]);
FileSystem sys = filesystem.get (configuration);
if (sys.exists (OutputDir)) {
Sys.delete (OutputDir, true);
}
Fileoutputformat.setoutputpath (Job, OutputDir);

Job.waitforcompletion (TRUE);
}
}

Hadoop Two-time sort