Bulkload for hbase mapreduce Performance Improvement Solution

We know that we will use bulkload for the first batch import of massive data.

A Brief Introduction to bulkload principles: (1) Using mapreduce, the output is formatted as the underlying storage file hfile of hbase on the map or reduce side. (2) Call bulkload to import the hfile generated by the first job to the corresponding hbase table.

PS: Note: (1) the hfile method is the fastest in all loading schemes, provided that the data must be first imported, indicating that it is empty! If data already exists in the table, the hbase table triggers the split cut operation when hfile is imported again. (2) Finally output the result. Whether it is map or reduce, we recommend that you only use <immutablebyteswritable, keyValue> for the output.

Now let's get started: bulkload is certainly the fastest way to write data into hbase. However, if we are doing business analysis and the data is already in hbase, we recommend that you use a common hbase-specific method, as shown in the following demo:

import com.yeepay.bigdata.bulkload.TableCreator;import org.apache.hadoop.hbase.HBaseConfiguration;import org.apache.hadoop.hbase.client.Put;import org.apache.hadoop.hbase.client.Result;import org.apache.hadoop.hbase.client.Scan;import org.apache.hadoop.hbase.io.ImmutableBytesWritable;import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;import org.apache.hadoop.hbase.mapreduce.TableMapper;import org.apache.hadoop.hbase.mapreduce.TableReducer;import org.apache.hadoop.mapreduce.Job;import org.apache.log4j.Logger;import java.io.IOException;public class HBaseMapReduceDemo {    static Logger LOG = Logger.getLogger(HBaseMapReduceDemo.class);    static class Mapper1 extends TableMapper<ImmutableBytesWritable, ImmutableBytesWritable> {        @Override        public void map(ImmutableBytesWritable row, Result values, Context context) throws IOException {            try {            //  context.write(key, value);            } catch (Exception e) {                LOG.error(e);            }        }    }    public static class Reducer1 extends TableReducer<ImmutableBytesWritable, ImmutableBytesWritable, ImmutableBytesWritable> {        public void reduce(ImmutableBytesWritable key, Iterable<ImmutableBytesWritable> values, Context context) throws IOException, InterruptedException {            try {                Put put = new Put(key.get());                // put.add();                context.write(key, put);            } catch (Exception e) {                LOG.error(e);                return ;            }  // catch        }  // reduce function    }  // reduce class    public static void main(String[] args) throws Exception {        HBaseConfiguration conf = new HBaseConfiguration();        conf.set("hbase.zookeeper.quorum", "yp-name02,yp-name01,yp-data01");        conf.set("hbase.zookeeper.property.clientPort", "2181");        // conf.set(TableInputFormat.INPUT_TABLE,"access_logs");        Job job = new Job(conf, "HBaseMapReduceDemo");        job.setJarByClass(HBaseMapReduceDemo.class);//        job.setNumReduceTasks(2);        Scan scan = new Scan();        scan.setCaching(2500);        scan.setCacheBlocks(false);        TableMapReduceUtil.initTableMapperJob("srcHBaseTableName", scan, Mapper1.class, ImmutableBytesWritable.class, ImmutableBytesWritable.class, job);//        TableCreator.createTable(20, true, "OP_SUM");        TableMapReduceUtil.initTableReducerJob("destHBasetableName", Reducer1.class, job);        System.exit(job.waitForCompletion(true) ? 0 : 1);    }}

In this case, a spliter is generated during the insertion of massive data, and the write speed is very high and slow. However, this situation is suitable for use when modifying existing hbase tables.

For example, hbase-> mapreduce analysis-> New table, we recommend using (hbase-> mapreduce analysis-> bulkload-> New table.

The demo is as follows:

Mapper example:

public class MyReducer extends Reducer<ImmutableBytesWritable, ImmutableBytesWritable, ImmutableBytesWritable, KeyValue> {    static Logger LOG = Logger.getLogger(MyReducer.class);    public void reduce(ImmutableBytesWritable key, Iterable<ImmutableBytesWritable> values, Context context) throws IOException, InterruptedException {        try {            context.write(key, kv);        } catch (Exception e) {            LOG.error(e);            return;        }  // catch    }  // reduce function}

Reducer is as follows:

public class MyReducer extends Reducer<ImmutableBytesWritable, ImmutableBytesWritable, ImmutableBytesWritable, KeyValue> {    static Logger LOG = Logger.getLogger(MyReducer.class);    public void reduce(ImmutableBytesWritable key, Iterable<ImmutableBytesWritable> values, Context context) throws IOException, InterruptedException {        try {            context.write(key, kv);        } catch (Exception e) {            LOG.error(e);            return;        }  // catch    }  // reduce function}

Job and bulkload:

public abstract class JobBulkLoad {    public void run(String[] args) throws Exception {        try {            if (args.length < 1) {                System.err.println("please set input dir");                System.exit(-1);                return;            }            String srcTableName = args[0];            String destTableName = args[1];            TableCreator.createTable(20, true, destTableName);            // 设置 HBase 參数            HBaseConfiguration conf = new HBaseConfiguration();            conf.set("hbase.zookeeper.quorum", "yp-name02,yp-name01,yp-data01");//          conf.set("hbase.zookeeper.quorum", "nn01, nn02, dn01");            conf.set("hbase.zookeeper.property.clientPort", "2181");            // 设置 Job 參数            Job job = new Job(conf, "hbase2hbase-bulkload");            job.setJarByClass(JobBulkLoad.class);            HTable htable = new HTable(conf, destTableName);  // 依据region的数量来决定reduce的数量以及每一个reduce覆盖的rowkey范围            // ----------------------------------------------------------------------------------------            Scan scan = new Scan();            scan.setCaching(2500);            scan.setCacheBlocks(false);            TableMapReduceUtil.initTableMapperJob(srcTableName, scan, MyMapper.class, ImmutableBytesWritable.class, ImmutableBytesWritable.class, job);//          TableMapReduceUtil.initTableReducerJob(destTableName, Common_Reducer.class, job);            job.setReducerClass(MyReducer.class);            Date now = new Date();            Path output = new Path("/output/" + destTableName + "/" + now.getTime());            System.out.println("/output/" + destTableName + "/" + now.getTime());            HFileOutputFormat.configureIncrementalLoad(job, htable);            FileOutputFormat.setOutputPath(job, output);            HFileOutputFormat.configureIncrementalLoad(job, htable);            job.waitForCompletion(true);//-----  运行BulkLoad  -------------------------------------------------------------------------------            HdfsUtil.chmod(conf, output.toString());            HdfsUtil.chmod(conf, output + "/" + YeepayConstant.COMMON_FAMILY);            htable = new HTable(conf, destTableName);            new LoadIncrementalHFiles(conf).doBulkLoad(output, htable);            System.out.println("HFile data load success!");        } catch (Throwable t) {            throw new RuntimeException(t);        }    }    }

Bulkload for hbase mapreduce Performance Improvement Solution