Hadoop多目錄輸入，join，進入reduce，資料流分析

標籤：hadoop   多目錄輸入   資料流   

前言

在做需求時，經常遇到多個目錄，也就是多個維度進行join，這裡分析一下，資料是怎麼流動的。

1、多目錄輸入

使用MultipleInputs.addInputPath()  對多目錄製定格式和map

2、資料流分析

map按行讀入資料，需要對不同的輸入目錄，打上不同的標記（這個方法又叫reduce端串連），map在輸出後會進行partition和sort，按照key進行排序，然後輸出到reduce進行處理。

例子三個輸入檔案：a.txt：

500501

b.txt:

500501600 505

c.txt:

501500700 800

代碼

import java.io.IOException;import java.util.Iterator;import org.apache.hadoop.conf.Configured;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Partitioner;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.util.Tool;import org.apache.hadoop.util.ToolRunner;import util.TextPair;import com.sina.hadoop.MultipleInputs;public class Main extends Configured implements Tool{    public static void main(String[] args) throws Exception    {        int exitcode = ToolRunner.run(new Main(), args);        System.exit(exitcode);    }    /**     * 分區     */    static class TextPairKeyPartitioner extends Partitioner<TextPair, Text>    {        public int getPartition(TextPair key, Text value, int numPartitions)        {            return (key.getFirst().hashCode() & Integer.MAX_VALUE) % numPartitions;        }    }    public int run(String[] arg0) throws Exception    {        int exitcode = 0;        if (exitcode == 0)        {            Job job1 = new Job();            job1.setJobName("testMultipleInputs");            job1.setJarByClass(Main.class);            MultipleInputs.addInputPath(job1, new Path("xx/testMultipleInputs/input/a/"),                    TextInputFormat.class, AMapper.class);            MultipleInputs.addInputPath(job1, new Path("xx/testMultipleInputs/input/b/"),                    TextInputFormat.class, BMapper.class);            MultipleInputs.addInputPath(job1, new Path("xx/testMultipleInputs/input/c/"),                    TextInputFormat.class, CMapper.class);            job1.setReducerClass(TestReducer.class);            FileOutputFormat.setOutputPath(job1, new Path("xx/testMultipleInputs/output/"));            job1.setOutputKeyClass(Text.class);            job1.setOutputValueClass(Text.class);            job1.setPartitionerClass(TextPairKeyPartitioner.class);            job1.setGroupingComparatorClass(TextPair.FirstComparator.class);            job1.setMapOutputKeyClass(TextPair.class);            job1.setMapOutputValueClass(Text.class);            job1.setNumReduceTasks(1);            exitcode = job1.waitForCompletion(true) ? 0 : 1;        }        return exitcode;    }    public class AMapper extends Mapper<LongWritable, Text, TextPair, Text>    {        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException        {            String[] data = value.toString().split("\t", -1);            String id = "";            if (data.length >= 1)            {                id = data[0];                if (!"".equals(id))                {                    context.write(new TextPair(id, "1"), new Text("0"));                }            }        }    }    public class BMapper extends Mapper<LongWritable, Text, TextPair, Text>    {        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException        {            String[] data = value.toString().split("\t", -1);            String id1 = "";            String id2 = "";            if (data.length >= 2)            {                id1 = data[0];                id2 = data[1];                if (!"".equals(id1))                {                    context.write(new TextPair(id1, "2"), new Text(id2));                }            }        }    }    public class CMapper extends Mapper<LongWritable, Text, TextPair, Text>    {        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException        {            String[] data = value.toString().split("\t", -1);            String id1 = "";            String id2 = "";            if (data.length >= 2)            {                id1 = data[0];                id2 = data[1];                if (!"".equals(id1))                {                    context.write(new TextPair(id1, "3"), new Text(id2));                }            }        }    }    public class TestReducer extends Reducer<TextPair, Text, Text, Text>    {        public void reduce(TextPair key, Iterable<Text> values, Context context) throws IOException, InterruptedException        {            String data = "";            Iterator<Text> i = values.iterator();            while (i.hasNext())            {                data = i.next().toString();                context.write(key.getFirst(), new Text(data));            }        }    }}