Tachyon basically uses 10 ----- running shark on tachyon

1. Install hive

1. Download and decompress hive

[[email protected] soft]# tar -zxf apache-hive-0.13.1-bin.tar.gz -C /usr/local/[[email protected] soft]# ln -s /usr/local/apache-hive-0.13.1-bin/ /usr/local/hive[[email protected] soft]#

2. Configure hive Environment Variables

[[email protected] soft]# cat /etc/profile.d/hive.shHIVE_HOME=/usr/local/hivePATH=$HIVE_HOME/bin:$PATH[[email protected] soft]# . /etc/profile.d/hive.sh[[email protected] soft]#

3. Install the MySQL database

[[email protected] soft]# yum -y install mysql-server[[email protected] soft]#chkconfig mysqld on[[email protected] soft]#service mysqld restart

4. Configure MySQL Permissions

mysql> createdatabase hive;Query OK, 1 row affected(0.00 sec) mysql> grant all on*.* to ‘hive‘@‘%‘ identified by ‘123456‘;Query OK, 0 rowsaffected (0.00 sec) mysql>

5. Copy the configuration file

[[email protected] conf]# pwd/usr/local/hive/conf[[email protected] conf]#  cp hive-env.sh.template hive-env.sh[[email protected] conf]# cphive-default.xml.template hive-site.xml[[email protected] conf]# cphive-exec-log4j.properties.template hive-exec-log4j.properties[[email protected] conf]# cphive-log4j.properties.template hive-log4j.properties[[email protected] conf]#

6. Configure hive-env.sh

[[email protected] conf]# pwd/usr/local/hive/conf[[email protected] conf]# cathive-env.sh |grep -v ^$ |grep -v ^#HADOOP_HOME=/usr/local/hadoop[[email protected] conf]#

7. Configure hive-site.sh

[[email protected] conf]#  pwd/usr/local/bigdata/hive/conf[[email protected] conf]#  cat hive-site.xml |grep -v ^$|grep -v ^#<?xmlversion="1.0"?><?xml-stylesheettype="text/xsl" href="configuration.xsl"?><configuration>   <property>     <name>javax.jdo.option.ConnectionURL</name>      <value>jdbc:mysql://node1:3306/hive?createDatabaseIfNotExist=true</value>      <description>JDBC connect stringfor a JDBC metastore</description>   </property>   <property>     <name>javax.jdo.option.ConnectionDriverName</name>      <value>com.mysql.jdbc.Driver</value>      <description>Driver class name fora JDBC metastore</description>   </property>   <property>     <name>javax.jdo.option.ConnectionUserName</name>      <value>hive</value>      <description>username to useagainst metastore database</description>   </property>   <property>     <name>javax.jdo.option.ConnectionPassword</name>      <value>123456</value>      <description>password to useagainst metastore database</description>   </property></configuration>[[email protected] conf]#

8. Copy the JDBC driver

[[email protected] soft]# cp mysql-connector-java-5.1.22-bin.jar /usr/local/hive/lib/[[email protected] soft]#

Ii. Install shark

1. Download and pressurize the shark

[[email protected] soft]# tar -zxf shark-0.9.1-bin-hadoop2.tgz -C /usr/local/ [[email protected] soft]# ln -s /usr/local/shark-0.9.1-bin-hadoop2/ /usr/local/shark[[email protected] soft]#

2. Configure the shark environment variable

[[email protected] soft]# cat /etc/profile.d/shark.shSHARK_HOME=/usr/local/sharkPATH=$SHARK_HOME/bin:$PATH[[email protected] soft]# . /etc/profile.d/shark.sh[[email protected] soft]#

3. Configure shark-env.sh

[[email protected] conf]# pwd/usr/local/shark/conf[[email protected] conf]# cat shark-env.sh|grep -v ^$|grep -v ^#export JAVA_HOME=/usr/java/latestexport SPARK_MEM=128mexport SHARK_MASTER_MEM=128mexport HADOOP_HOME="/usr/local/hadoop"export HIVE_HOME="/usr/local/hive"export HIVE_CONF_DIR="/usr/local/hive/conf"export SCALA_HOME="/usr/local/scala"export MASTER="spark://192.168.1.1:7077"export SPARK_HOME="/usr/local/spark"SPARK_JAVA_OPTS=" -Dspark.local.dir=/tmp "SPARK_JAVA_OPTS+="-Dspark.kryoserializer.buffer.mb=10"SPARK_JAVA_OPTS+="-verbose:gc -XX:-PrintGCDetails -XX:+PrintGCTimeStamps "export SPARK_JAVA_OPTS[[email protected] conf]#

4. Copy hive configurations to all slaves

[[email protected] ~]# scp -r/usr/local/apache-hive-0.13.1-bin/ node2:/usr/local/[[email protected] ~]# scp -r/usr/local/apache-hive-0.13.1-bin/ node3:/usr/local/[[email protected] ~]# sshnode2 ln -s /usr/local/apache-hive-0.13.1-bin/ /usr/local/hive[[email protected] ~]# sshnode3 ln -s /usr/local/apache-hive-0.13.1-bin/ /usr/local/hive[[email protected] ~]# scp/etc/profile.d/hive.sh node2:/etc/profile.d/hive.sh                                      100%   52     0.1KB/s  00:00    [[email protected] ~]# scp/etc/profile.d/hive.sh node3:/etc/profile.d/hive.sh                                      100%   52     0.1KB/s  00:00    [[email protected] ~]#

5. Copy the shark directory to all slaves

[[email protected] ~]# scp -r/usr/local/shark-0.9.1-bin-hadoop2/ node2:/usr/local/[[email protected] ~]# scp -r/usr/local/shark-0.9.1-bin-hadoop2/ node3:/usr/local/[[email protected] ~]# sshnode2 ln -s /usr/local/shark-0.9.1-bin-hadoop2/ /usr/local/shark[[email protected] ~]# sshnode3 ln -s /usr/local/shark-0.9.1-bin-hadoop2/ /usr/local/shark[[email protected] ~]# scp/etc/profile.d/shark.sh node2:/etc/profile.d/shark.sh                                      100%   55     0.1KB/s  00:00    [[email protected] ~]# scp/etc/profile.d/shark.sh node3:/etc/profile.d/shark.sh                                      100%   55    0.1KB/s   00:00    [[email protected] ~]#

6. Replace the jar package "hive-exec-0.11.0-shark-0.9.1.jar"

cd $SHARK_HOME/lib_managed/jars/edu.berkeley.cs.shark/hive-execunziphive-exec-0.11.0-shark-0.9.1.jarrm -fcom/google/protobuf/*rm  hive-exec-0.11.0-shark-0.9.1.jarzip -rhive-exec-0.11.0-shark-0.9.1.jar *rm -rf comhive-exec-log4j.properties javaewah/ javax/ javolution/ META-INF/ org/

Iii. Shark integration tachyon

1. Modifyshark-env.shAdd the following2Line

export TACHYON_MASTER="tachyon://node1:19998"export TACHYON_WAREHOUSE_PATH=/sharktables

2. Synchronize the modified configurations to other nodes.

[[email protected] conf]# scpshark-env.sh node2:/usr/local/shark/conf/shark-env.sh                                  100%2253     2.2KB/s   00:00   [[email protected] conf]# scpshark-env.sh node3:/usr/local/shark/conf/shark-env.sh                                  100%2253     2.2KB/s   00:00   [[email protected] conf]#

3. Start shark

[[email protected] conf]# sharkshark> show tables;OKssqTime taken: 0.841secondsshark>

4. Specify tblproperties ("Shark. cache" = "tachyon") to cache the table to tachyon.

shark> create table test1 TBLPROPERTIES("shark.cache" = "tachyon") as select * from ssq;

5. Specify the table name and end with _ tachyon to cache the table to tachyon.

shark> CREATE TABLEtest_tachyon AS SELECT * FROM ssq;

This article is from the "tachyon" blog, please be sure to keep this source http://ucloud.blog.51cto.com/3869454/1564230

Tachyon basically uses 10 ----- running shark on tachyon