Create hbase secondary indexes and hbase Indexes

Create hbase secondary indexes and hbase Indexes

Run hbase secondary indexes on a single machine:

Import java. io. IOException; import java. util. hashMap; import java. util. map; import java. util. set; import org. apache. hadoop. conf. configuration; import org. apache. hadoop. fs. path; import org. apache. hadoop. hbase. HBaseConfiguration; import org. apache. hadoop. hbase. client. put; import org. apache. hadoop. hbase. client. result; import org. apache. hadoop. hbase. client. scan; import org. apache. hadoop. hbase. io. immutableBytesWr Itable; import org. apache. hadoop. hbase. mapreduce. multiTableOutputFormat; import org. apache. hadoop. hbase. mapreduce. tableInputFormat; import org. apache. hadoop. hbase. mapreduce. tableMapReduceUtil; import org. apache. hadoop. hbase. mapreduce. tableMapper; import org. apache. hadoop. hbase. util. bytes; import org. apache. hadoop. mapreduce. job; import org. apache. hadoop. mapreduce. mapper; import com. tansun. di. core. di. core. h Base. HBaseDaoPool; public class IndexCreateExample extendsTableMapper <ImmutableBytesWritable, Put> {public static Configuration conf = null; private String familyName; // original table private Map <byte [], immutableBytesWritable> indexes = new HashMap <byte [], ImmutableBytesWritable> (); static {conf = new Configuration (); String filePath = "hbase-site.xml "; path path = new Path (filePath); conf. addResource (pat H); conf = HBaseConfiguration. create (conf);} // public static class IndexMapper extends @ Overrideprotected void setup (Mapper <ImmutableBytesWritable, Result, ImmutableBytesWritable, Put>. context context) throws IOException, InterruptedException {String tableName = conf. get ("tableName"); // String family = conf. get ("familyName"); // obtain the family familyName = conf. get ("familyName"); // obtain the String [] qualifiers = Conf. getStrings ("qualifiers"); for (String qualifier: qualifiers) {// create a ing to create a table for each column, the table name is tableName + "-" + qualifier // The column index of the original table. The new table name is indexes. put (Bytes. toBytes (qualifier), new ImmutableBytesWritable (Bytes. toBytes (tableName + "-" + qualifier);} public static void main (String [] args) {if (args. length <3) {System. err. println ("Usage: IndexCreateExample <table_name> <cf> <cn>"); System. exit (-1);} // table name Str Ing tableName = args [0]; // columnFamily String columnFamily = args [1]; conf. set ("tableName", tableName); conf. set ("columnFamily", columnFamily); // The column may have multiple columns: String [] qualifiers = new String [args. length-2]; for (int I = 0; I <qualifiers. length; I ++) {qualifiers [I] = args [I + 2];} // sets the column conf. setStrings ("qualifiers", qualifiers); @ SuppressWarnings ("deprecation") Job job; try {job = new Job (conf, tableName); job. setJa RByClass (IndexCreateExample. class); job. setMapperClass (IndexCreateExample. class); job. setNumReduceTasks (0); // The reduce stage job does not need to be executed. setInputFormatClass (TableInputFormat. class); job. setOutputFormatClass (MultiTableOutputFormat. class); Scan scan = new Scan (); TableMapReduceUtil. initTableMapperJob (tableName, scan, IndexCreateExample. class, ImmutableBytesWritable. class, Put. class, job); job. waitForCompletion (true );} Catch (Exception e1) {e1.printStackTrace () ;}@ Overrideprotected void map (ImmutableBytesWritable key, Result value, Mapper <ImmutableBytesWritable, Result, ImmutableBytesWritable, Put>. context context) throws IOException, InterruptedException {// obtain tableName and Set <byte []> keys = indexes. keySet (); for (byte [] k: keys) {// index tableNameImmutableBytesWritable indexTableName = indexes. get (k); // byte [] Value2 = value. getValue (Bytes. toBytes (familyName), k); if (value2! = Null) {// index table Put put = new Put (value2); // index table row key // column Family column original table row key put. add (Bytes. toBytes ("f1"), Bytes. toBytes ("id"), key. get (); context. write (indexTableName, put );}}}}

Exception message: insufficient memory

or more detailed output, check application tracking page:http://ts.node2.com:8088/proxy/application_1472537544791_0007/Then, click on links to logs of each attempt.Diagnostics: Container [pid=3984,containerID=container_1472537544791_0007_02_000001] is running beyond physical memory limits. Current usage: 285.6 MB of 256 MB physical memory used; 1.5 GB of 537.6 MB virtual memory used. Killing container.Dump of the process-tree for container_1472537544791_0007_02_000001 :|- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE|- 3984 3982 3984 3984 (bash) 0 0 108617728 338 /bin/bash -c /usr/java/jdk1.7.0_79/bin/java -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/container/application_1472537544791_0007/container_1472537544791_0007_02_000001 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA  -Djava.net.preferIPv4Stack=true -Xmx825955249 org.apache.hadoop.mapreduce.v2.app.MRAppMaster 1>/var/log/hadoop-yarn/container/application_1472537544791_0007/container_1472537544791_0007_02_000001/stdout 2>/var/log/hadoop-yarn/container/application_1472537544791_0007/container_1472537544791_0007_02_000001/stderr  |- 3999 3984 3984 3984 (java) 2183 69 1504305152 72786 /usr/java/jdk1.7.0_79/bin/java -Dlog4j.configuration=container-log4j.properties -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/container/application_1472537544791_0007/container_1472537544791_0007_02_000001 -Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Djava.net.preferIPv4Stack=true -Xmx825955249 org.apache.hadoop.mapreduce.v2.app.MRAppMaster Container killed on request. Exit code is 143Container exited with a non-zero exit code 143Failing this attempt. Failing the application.16/09/01 11:07:56 INFO mapreduce.Job: Counters: 0

Exception 2:

 

16/08/31 20:40:03 ERROR mapreduce.TableInputFormat: java.io.IOException: java.lang.reflect.InvocationTargetExceptionat org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:240)at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:218)at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:119)at org.apache.hadoop.hbase.mapreduce.TableInputFormat.initialize(TableInputFormat.java:183)at org.apache.hadoop.hbase.mapreduce.TableInputFormatBase.getSplits(TableInputFormatBase.java:230)at org.apache.hadoop.hbase.mapreduce.TableInputFormat.getSplits(TableInputFormat.java:237)at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:597)at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:614)at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:492)at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1306)at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1303)at java.security.AccessController.doPrivileged(Native Method)at javax.security.auth.Subject.doAs(Subject.java:415)at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)at org.apache.hadoop.mapreduce.Job.submit(Job.java:1303)at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1324)at com.tansun.di.core.di.core.hbase.second.IndexCreateExample.main(IndexCreateExample.java:101)at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)at java.lang.reflect.Method.invoke(Method.java:606)at org.apache.hadoop.util.RunJar.run(RunJar.java:221)at org.apache.hadoop.util.RunJar.main(RunJar.java:136)Caused by: java.lang.reflect.InvocationTargetExceptionat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)at java.lang.reflect.Constructor.newInstance(Constructor.java:526)at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:238)... 22 moreCaused by: java.lang.UnsupportedOperationException: Unable to find org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactoryat org.apache.hadoop.hbase.util.ReflectionUtils.instantiateWithCustomCtor(ReflectionUtils.java:36)at org.apache.hadoop.hbase.ipc.RpcControllerFactory.instantiate(RpcControllerFactory.java:58)at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.createAsyncProcess(ConnectionManager.java:2220)at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.<init>(ConnectionManager.java:676)at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.<init>(ConnectionManager.java:618)... 27 moreCaused by: java.lang.ClassNotFoundException: org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactoryat java.net.URLClassLoader$1.run(URLClassLoader.java:366)at java.net.URLClassLoader$1.run(URLClassLoader.java:355)at java.security.AccessController.doPrivileged(Native Method)at java.net.URLClassLoader.findClass(URLClassLoader.java:354)at java.lang.ClassLoader.loadClass(ClassLoader.java:425)at java.lang.ClassLoader.loadClass(ClassLoader.java:358)at java.lang.Class.forName0(Native Method)at java.lang.Class.forName(Class.java:191)at org.apache.hadoop.hbase.util.ReflectionUtils.instantiateWithCustomCtor(ReflectionUtils.java:32)... 31 more16/08/31 20:40:03 INFO mapreduce.JobSubmitter: Cleaning up the staging area /tmp/hadoop-yarn/staging/root/.staging/job_1472695767533_000316/08/31 20:40:03 WARN security.UserGroupInformation: PriviledgedActionException as:root (auth:SIMPLE) cause:java.io.IOException: Cannot create a record reader because of a previous error. Please look at the previous logs lines from the task's full log for more details.java.io.IOException: Cannot create a record reader because of a previous error. Please look at the previous logs lines from the task's full log for more details.at org.apache.hadoop.hbase.mapreduce.TableInputFormatBase.getSplits(TableInputFormatBase.java:241)at org.apache.hadoop.hbase.mapreduce.TableInputFormat.getSplits(TableInputFormat.java:237)at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:597)at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:614)at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:492)at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1306)at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1303)at java.security.AccessController.doPrivileged(Native Method)at javax.security.auth.Subject.doAs(Subject.java:415)at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)at org.apache.hadoop.mapreduce.Job.submit(Job.java:1303)at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1324)at com.tansun.di.core.di.core.hbase.second.IndexCreateExample.main(IndexCreateExample.java:101)at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)at java.lang.reflect.Method.invoke(Method.java:606)at org.apache.hadoop.util.RunJar.run(RunJar.java:221)at org.apache.hadoop.util.RunJar.main(RunJar.java:136)Caused by: java.lang.IllegalStateException: The input format instance has not been properly initialized. Ensure you call initializeTable either in your constructor or initialize methodat org.apache.hadoop.hbase.mapreduce.TableInputFormatBase.getTable(TableInputFormatBase.java:389)at org.apache.hadoop.hbase.mapreduce.TableInputFormatBase.getSplits(TableInputFormatBase.java:236)

 

 

Create the original Table hbase (main): 002: 0> create 'studentinfo', 'f1' 0 row (s) in 0.6520 seconds => Hbase: Table-studentinfohbase (main ): 003: 0> put 'studentinfo', '1', 'f1: name', 'zhangsan' 0 row (s) in 0.1640 secondshbase (main): 004: 0> put 'studentinfo', '2', 'f1: name', 'lisi' 0 row (s) in 0.0240 secondshbase (main): 005: 0> put 'studentinfo', '3', 'f1: name', 'wangwu' 0 row (s) in 0.0290 secondshbase (main): 006: 0> scan 'studentinfo' row column + CELL 1 column = f1: name, timestamp = 1436262175823, value = zhangsan 2 column = f1: name, timestamp = 1436262183922, value = lisi 3 column = f1: name, timestamp = 1436262189250, value = wangwu3 row (s) in 0.0530 seconds

 

Create index table hbase (main): 007: 0> create 'studentinfo-name', 'f1'

0 row (s) in 0.7740 seconds => Hbase: Table-studentinfo-name