Hadoop uses combiner to improve the efficiency of MAP/reduce programs

As we all know, the hadoop framework uses Mapper to process data into a <key, value> key-value pair, and then shuffle it between network nodes ), then, use CER to process the data and make the final output.

In the above process, we can see at least two performance bottlenecks:

1. If we have 1 billion million data records, mapper will generate 1 billion key-value pairs for transmission across the network, but if we only calculate the maximum value for the data, obviously, mapper only needs to output the maximum value it knows. This not only reduces the network pressure, but also greatly improves program efficiency.
2. This definition of data skew is elaborated using the country in the patent. Such data is far from consistent or balanced distribution. Because most patent countries belong to the United States, not only does the key-value pairs in the Mapper and the key-value pairs in the intermediate stage (shuffle) correspond to each other, most key-value pairs eventually converge on a single CER to overwhelm this CER, greatly reducing program performance.

Hadoop solves the above bottleneck by using a combiner step between Mapper and CER Cer. You can regard combiner as a helper of CER Cer. It is mainly used to reduce mapper output and network

Bandwidth and load on the reducer. If we define a combiner, The mapreducer framework uses it to process intermediate data multiple times.

If CER only runs Simple distributed methods, such as the maximum value, minimum value, or count, we can make the reducer itself as a combiner. However, many useful methods are not distributed. The following describes how to calculate the average value as an example:

Mapper outputs the key-value pairs it processes. In order to make a single datanode calculate the average reducer, it sorts and sums the <key, value> key-value pairs it receives.

Because CER treats the number of <key, value> key values received by reducer as the number of <key, value> key-value pairs in the input data, in this case, the main obstacle to using combiner is the count operation. We can rewrite the mapreduce program to clearly track the counting process.

The Code is as follows:

package com;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.conf.Configured;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.DoubleWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;import org.apache.hadoop.util.Tool;import org.apache.hadoop.util.ToolRunner;public class AveragingWithCombiner extends Configured implements Tool {public static class MapClass extends Mapper<LongWritable,Text,Text,Text> {static enum ClaimsCounters { MISSING, QUOTED };// Map Methodpublic void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {String fields[] = value.toString().split(",", -20);String country = fields[4];            String numClaims = fields[8];                        if (numClaims.length() > 0 && !numClaims.startsWith("\"")) {            context.write(new Text(country), new Text(numClaims + ",1"));            }}}public static class Reduce extends Reducer<Text,Text,Text,DoubleWritable> {// Reduce Methodpublic void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {double sum = 0;            int count = 0;            for (Text value : values) {                String fields[] = value.toString().split(",");                sum += Double.parseDouble(fields[0]);                count += Integer.parseInt(fields[1]);            }            context.write(key, new DoubleWritable(sum/count));}}public static class Combine extends Reducer<Text,Text,Text,Text> {// Reduce Methodpublic void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {double sum = 0;            int count = 0;            for (Text value : values) {                String fields[] = value.toString().split(",");                sum += Double.parseDouble(fields[0]);                count += Integer.parseInt(fields[1]);            }            context.write(key, new Text(sum+","+count));}}// run Methodpublic int run(String[] args) throws Exception {// Create and Run the JobJob job = new Job();job.setJarByClass(AveragingWithCombiner.class);FileInputFormat.addInputPath(job, new Path(args[0]));FileOutputFormat.setOutputPath(job, new Path(args[1]));job.setJobName("AveragingWithCombiner");job.setMapperClass(MapClass.class);job.setCombinerClass(Combine.class);job.setReducerClass(Reduce.class);job.setInputFormatClass(TextInputFormat.class);job.setOutputFormatClass(TextOutputFormat.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(Text.class);System.exit(job.waitForCompletion(true) ? 0 : 1);return 0;}public static void main(String[] args) throws Exception {int res = ToolRunner.run(new Configuration(), new AveragingWithCombiner(), args);System.exit(res);}}

 

 

package com;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.conf.Configured;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.DoubleWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;import org.apache.hadoop.util.Tool;import org.apache.hadoop.util.ToolRunner;public class AveragingWithCombiner extends Configured implements Tool {public static class MapClass extends Mapper<LongWritable,Text,Text,Text> {static enum ClaimsCounters { MISSING, QUOTED };// Map Methodpublic void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {String fields[] = value.toString().split(",", -20);String country = fields[4];            String numClaims = fields[8];                        if (numClaims.length() > 0 && !numClaims.startsWith("\"")) {            context.write(new Text(country), new Text(numClaims + ",1"));            }}}public static class Reduce extends Reducer<Text,Text,Text,DoubleWritable> {// Reduce Methodpublic void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {double sum = 0;            int count = 0;            for (Text value : values) {                String fields[] = value.toString().split(",");                sum += Double.parseDouble(fields[0]);                count += Integer.parseInt(fields[1]);            }            context.write(key, new DoubleWritable(sum/count));}}public static class Combine extends Reducer<Text,Text,Text,Text> {// Reduce Methodpublic void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {double sum = 0;            int count = 0;            for (Text value : values) {                String fields[] = value.toString().split(",");                sum += Double.parseDouble(fields[0]);                count += Integer.parseInt(fields[1]);            }            context.write(key, new Text(sum+","+count));}}// run Methodpublic int run(String[] args) throws Exception {// Create and Run the JobJob job = new Job();job.setJarByClass(AveragingWithCombiner.class);FileInputFormat.addInputPath(job, new Path(args[0]));FileOutputFormat.setOutputPath(job, new Path(args[1]));job.setJobName("AveragingWithCombiner");job.setMapperClass(MapClass.class);job.setCombinerClass(Combine.class);job.setReducerClass(Reduce.class);job.setInputFormatClass(TextInputFormat.class);job.setOutputFormatClass(TextOutputFormat.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(Text.class);System.exit(job.waitForCompletion(true) ? 0 : 1);return 0;}public static void main(String[] args) throws Exception {int res = ToolRunner.run(new Configuration(), new AveragingWithCombiner(), args);System.exit(res);}}