HBase with MapReduce (Summary)

We know that hbase is not like a relational database with powerful query functions and statistical functions, this article realizes how to use MapReduce to count the number of cell values in HBase, and bring the results to the table of the target,

(1) Realization of Mapper

Package Com.datacenter.hbasemapreduce.summary;import Java.io.ioexception;import Java.util.navigablemap;import Java.util.map.entry;import Org.apache.hadoop.hbase.client.result;import Org.apache.hadoop.hbase.io.immutablebyteswritable;import Org.apache.hadoop.hbase.mapreduce.tablemapper;import Org.apache.hadoop.hbase.util.bytes;import Org.apache.hadoop.io.intwritable;import Org.apache.hadoop.io.Text; Import Org.apache.hadoop.mapreduce.mapper;public class Summarymapper extends Tablemapper<text, intwritable> {// Here is the type of the context output in the specified map public static final byte[] CF = "CF". GetBytes ();p ublic static final byte[] ATTR1 = "attr1". GetBytes () ;p rivate final Intwritable one = new intwritable (1);p rivate text text = new text (); @Overrideprotected void Map (Immutableby Teswritable key, Result value, Context context) throws IOException, interruptedexception {//TODO auto-generated method Stu b/*byte[] ss = Value.getvalue (CF, ATTR1); Here is just a specific column family, the number of values for a particular column, or you can modify the string val = new String (ss) according to the actual situation; Text.set(Val); We can only emit writables. Context.write (text, one); *///counts the number of values for all column families and columns try {dealresult (value, context);} catch (Exception e) {//TODO auto-generate D catch Blocke.printstacktrace ();}} Counts the number of values for all column families and columns public void Dealresult (Result RS, Context context) throws Exception {if (Rs.isempty ()) {System.out.printl N ("result is empty!"); return;} Navigablemap<byte[], navigablemap<byte[], navigablemap<long, byte[]>>> tableResulrt = Rs.getMap (); String Rowkey = bytes.tostring (Rs.getrow ()); Actain rowkey///system.out.println ("rowkey->" + Rowkey); for (entry<byte[], navigablemap<byte[], Navigablemap<long, byte[]>>> FamilyResult:tableResulrt.entrySet ()) {//system.out.print ("\tfamily-> "+ bytes.tostring (Temp.getkey ())); for (entry<byte[], Navigablemap<long, byte[]>> Columnresult: Familyresult.getvalue (). EntrySet ()) {///system.out.print ("\tcol->" + bytes.tostring (Value.getkey ())); for (Entry <long, byte[]> Valueresult:columnresult.GetValue (). EntrySet ()) {//system.out.print ("\tvesion->" + Va.getkey ())//system.out.print ("\tvalue->" + Bytes.tostring (Va.getvalue ()));//system.out.println (); Text.set (New String (Valueresult.getvalue ())); Context.write (text, one);}}}}

  (2) Reduce implementation

Package Com.datacenter.hbasemapreduce.summary;import Java.io.ioexception;import Org.apache.hadoop.hbase.Cell; Import Org.apache.hadoop.hbase.client.mutation;import Org.apache.hadoop.hbase.client.put;import Org.apache.hadoop.hbase.io.immutablebyteswritable;import Org.apache.hadoop.hbase.mapreduce.tablereducer;import Org.apache.hadoop.hbase.util.bytes;import Org.apache.hadoop.io.intwritable;import Org.apache.hadoop.io.Text; Import Org.apache.hadoop.mapreduce.reducer;public class Summaryreducer Extendstablereducer<text, IntWritable, immutablebyteswritable> {public static final byte[] CF = "CF". GetBytes ();p ublic static final byte[] Count = "Count". Get Bytes (); @SuppressWarnings ("deprecation") @Overrideprotected void Reduce (Text key, iterable<intwritable> values , context context) throws IOException, interruptedexception {//TODO auto-generated method Stubint i = 0;for (intwritable V Al:values) {i + = Val.get ();} Put put = new put (Bytes.tobytes (key.tostring ()));//cell s=new Put.add (CF,COUNT, 100,bytes.tobytes (i)); Add a column count to the corresponding column family, record its number context.write (null, put);}}

  (3) Implementation of the main class loading information

Package Com.datacenter.hbasemapreduce.summary;import Java.io.ioexception;import Org.apache.hadoop.conf.configuration;import Org.apache.hadoop.hbase.hbaseconfiguration;import Org.apache.hadoop.hbase.client.hconnection;import Org.apache.hadoop.hbase.client.hconnectionmanager;import Org.apache.hadoop.hbase.client.scan;import Org.apache.hadoop.hbase.mapreduce.tablemapreduceutil;import Org.apache.hadoop.io.intwritable;import Org.apache.hadoop.io.text;import org.apache.hadoop.mapreduce.job;// In the Statistics HBase table, the number of values per row in the entire table is public class Summarymain {static string rootdir = "Hdfs://hadoop3:8020/hbase"; static string Zkserver = "HADOOP3"; static String port = "2181";p rivate static Configuration conf;private static hconnection hconn = null ;p ublic static void Hbaseutil (String rootdir, String zkserver, string port) {conf = Hbaseconfiguration.create ();//Get default configuration Letter Conf.set ("Hbase.rootdir", RootDir) Conf.set ("Hbase.zookeeper.quorum", Zkserver); Conf.set (" Hbase.zookeeper.property.clientPort ", port); try {hconn = HconNectionmanager.createconnection (conf);} catch (IOException e) {//TODO auto-generated catch Blocke.printstacktrace ();}} public static void Main (string[] args) throws Exception {//TODO auto-generated method Stubhbaseutil (RootDir, Zkserver, PO RT); Job Job = new Job (conf, "examplesummary"); Job.setjarbyclass (Summarymain.class); Class that contains mapper and//reducerscan scan = new scan (); scan.setcaching (500); 1 is the default in Scan, which would be a bad for//MapReduce jobsscan.setcacheblocks (false); Don ' t set to True for MR jobs//set other scan attrstablemapreduceutil.inittablemapperjob ("score",//input Tablescan, Scan instance to control CF and attribute Selectionsummarymapper.class,//Mapper Classtext.class,//Mapper output key Intwritable.class,//Mapper output Valuejob); Tablemapreduceutil.inittablereducerjob ("Test",//Output Tablesummaryreducer.class,//reducer classjob); Job.setnumreducetasks (1); At least one, adjust as requiredboolean B = job.waitforcompletion (trUE), if (!b) {throw new IOException ("Error with job!");}} 

HBase with MapReduce (Summary)