Hadoop實戰之溫度排序__mapreduce

說明：輸入檔案為北京市2010年1月份到5月份每天每間隔3小時的溫度記錄，資料格式為yyyyMMddHHmm    temp，如下截圖

（圖中溫度為華氏溫度）

需求：求出每個月份溫度最高的5天

解決思路：1、以月份+溫度為key進行排序，月份升序，溫度降序

2、每個月份單獨產生一個檔案，讀取每個檔案前5條記錄，即為每個月份溫度最高的5天

程式如下

import java.io.DataInput;import java.io.DataOutput;import java.io.IOException;import java.text.ParseException;import java.text.SimpleDateFormat;import java.util.Calendar;import java.util.Date;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.hdfs.server.namenode.dfshealth_jsp;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.io.WritableComparable;import org.apache.hadoop.io.WritableComparator;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Partitioner;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import com.jcraft.jsch.jce.SHA1;import org.apache.hadoop.mapreduce.Job;/* * 根據月份分組 * 每月按照溫度降序排列 * 每個月單獨輸出一個檔案 */public class JobRun {public static void main(String[] args) {// TODO Auto-generated method stubConfiguration conf = new Configuration();try {Job job = new Job(conf);job.setJarByClass(JobRun.class);job.setMapperClass(LocalMap.class);job.setReducerClass(LocalReduce.class);job.setMapOutputKeyClass(DateKey.class);job.setMapOutputValueClass(Text.class);job.setSortComparatorClass(LocalSort.class);//設定成自己寫的排序類job.setGroupingComparatorClass(LocalGroup.class);//設定成自己寫的分組類job.setPartitionerClass(LocalPartitioner.class);//設定成自己寫的分區類job.setNumReduceTasks(5);//設定reduce個數為5,因為資料檔案包含5個月，每個月分產生一個檔案FileInputFormat.addInputPath(job, new Path("/user/root/temp/input"));FileOutputFormat.setOutputPath(job, new Path("/user/root/temp/output"));System.exit(job.waitForCompletion(true)?0:1);} catch (Exception e) {e.printStackTrace();}}//編寫mappublic static class LocalMap extends Mapper<LongWritable, Text, DateKey, Text> {@Overrideprotected void map(LongWritable key, Text value, Mapper<LongWritable, Text, DateKey, Text>.Context context)throws IOException, InterruptedException {String[] s = value.toString().split("\t");s[1]=s[1].trim();DateKey dk = new DateKey();SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmm");if(s.length==2){try {Date d = sdf.parse(s[0]);Calendar c =Calendar.getInstance();c.setTime(d);int month=c.get(Calendar.MONTH)+1;dk.setMonth(month);} catch (ParseException e) {e.printStackTrace();}dk.setTemp(Integer.parseInt(s[1]));context.write(dk, value);}}}//編寫reducepublic static class LocalReduce extends Reducer<DateKey, Text, Text,Text>{@Overrideprotected void reduce(DateKey key, Iterable<Text> value, Reducer<DateKey, Text, Text, Text>.Context context)throws IOException, InterruptedException {for(Text t:value){context.write(new Text(key.toString()), t);}}}//自訂封裝writable類public static class DateKey implements WritableComparable<DateKey>{public int getMonth() {return month;}public void setMonth(int month) {this.month = month;}public int getTemp() {return temp;}public void setTemp(int temp) {this.temp = temp;}private int month;private int temp;@Overridepublic void readFields(DataInput input) throws IOException {this.month = input.readInt();this.temp = input.readInt();}@Overridepublic void write(DataOutput out) throws IOException {out.writeInt(month);out.writeInt(temp);}@Overridepublic int compareTo(DateKey o) {int res = Integer.compare(month, o.getMonth());if(res!=0) return res;else{return Integer.compare(temp, o.getTemp());}}@Overridepublic String toString() {return month+"\t"+temp;}@Overridepublic int hashCode() {return new Integer(month+temp).hashCode();}}//自訂排序，月份相同按照溫度降序public static class LocalSort extends WritableComparator{@Overridepublic int compare(WritableComparable a, WritableComparable b) {DateKey d1 = (DateKey) a;DateKey d2 = (DateKey) b;int res = Integer.compare(d1.getMonth(),d2.getMonth());if(res!=0) return res;else{return -Integer.compare(d1.getTemp(),d2.getTemp());}}public LocalSort() {super(DateKey.class, true);}}//自訂partition，每個月份單獨產生一個reducepublic static class LocalPartitioner extends Partitioner<DateKey, Text>{@Overridepublic int getPartition(DateKey key, Text value, int num) {return key.getMonth()%num;}}//自訂分組，map後預設按照相同key合并，本例中需要按照key.month合并public static class LocalGroup extends WritableComparator{@Overridepublic int compare(WritableComparable a, WritableComparable b) {DateKey d1 = (DateKey) a;DateKey d2 = (DateKey) b;return Integer.compare(d1.getMonth(),d2.getMonth());}public LocalGroup(){super(DateKey.class, true);}}}

執行結果如下圖：

可以看到共產生了5個檔案，開啟檔案可以看到溫度都已經按照降序排列