Hadoop Hive Sqoop Zookeeper hbase production Environment Log Statistics application case (hive article)

3, hive installation configuration3.1install MySQL

Installing MySQL on the datanode5

# yum-y Installmysql-server MySQL

# MySQL

Mysql> Grant all privileges on * * [email protected] ' 10.40.214.% ' identified by ' hive ';

mysql> flush Privileges;

3.2Installing Hive

# tar-zxf Apache-hive-0.13.1-bin.tar.gz-c/var/data/; Mv/var/data/apache-hive-0.13.1/var/data/hive

# cd/var/data/hive

# vimbin/hive-config.sh # # Add the following at the beginning of the script

Exportjava_home=/usr/java/jdk1.7.0_71

Exporthive_home=/var/data/hive

Exporthadoop_home=/var/data/hadoop

# vim Hive-site.xml # # Configure Hive

<configuration>

<property>

<name>hive.exec.scratchdir</name>

<value>/tmp/hive-${user.name}</value>

<description>scratchspace for Hive jobs</description>

</property>

<property>

<name>hive.exec.local.scratchdir</name>

<value>/tmp/${user.name}</value>

<description>localscratch Space for Hive jobs</description>

</property>

<property>

<name>javax.jdo.option.ConnectionURL</name>

<value>jdbc:mysql://10.40.214.9:3306/hive?createDatabaseIfNotExist=true</value>

<description>jdbcconnect string for a JDBC metastore</description>

</property>

<property>

<name>javax.jdo.option.ConnectionDriverName</name>

<value>org.apache.derby.jdbc.EmbeddedDriver</value>

<description>driverclass name for a JDBC metastore</description>

</property>

<property>

<name>javax.jdo.PersistenceManagerFactoryClass</name>

<value>org.datanucleus.api.jdo.JDOPersistenceManagerFactory</value>

<description>classimplementing the JDO persistence</description>

</property>

<property>

<name>javax.jdo.option.DetachAllOnCommit</name>

<value>true</value>

<description>detachesall objects from session so, they can be used after transaction iscommitted</description >

</property>

<property>

<name>javax.jdo.option.NonTransactionalRead</name>

<value>true</value>

<description>readsoutside of Transactions</description>

</property>

<property>

<name>javax.jdo.option.ConnectionUserName</name>

<value>hive</value>

<description>usernameto use against Metastore database</description>

</property>

<property>

<name>javax.jdo.option.ConnectionPassword</name>

<value>hive</value>

<description>passwordto use against Metastore database</description>

</property>

<property>

<name>hive.metastore.warehouse.dir</name>

<value>/user/hive/warehouse</value>

<description>locationof default database for the warehouse</description>

</property>

</configuration>

# cpmysql-connector-java-5.1.18-bin.jar/var/data/hive/lib/ # # Add MySQL java driver jar Pack

# vim/etc/profile # # Configure hive Environment Variables

#hivepath

Exporthive_home=/var/data/hive

Exportpath= $PATH: $HIVE _home/bin

# Source/etc/profile

# hive # # start Hive

3.3hivethe Use

# # Create a library

# hive-e "CreateDatabase maillog;"

# # Create a table

# hive-e "CREATE Table Maillog.izhenxin (Mail_time string, message_id string,mail_to string, mail_domain string, Mail_rela Y string, Mail_delay string,mail_delays string,mail_dsn string, mail_status string) ROW FORMAT delimitedfields TERMINATED By ' | ' STORED as Textfile; "

# # Delete Table

# hive-e "Droptable maillog.izhenxin;"

# # Import data into hive table

# hive-e "LoadData LOCAL inpath '/opt/zhangdh/to_result.txt ' OVERWRITE into Tablemaillog.izhenxin;"

# # Simple data Query

# hive-e "Usemaillog;select * from Izhenxin_total limit 10;"

# # with Hive statistics, the mapreduce process is executed

# hive-e "Select Mail_domain,sum (case is mail_status= ' sent ' then 1 else 0 end) Sent,sum (case when mail_status= ' bounced ' Then 1 else 0 end) bounced,sum (Casewhen mail_status= ' deferred "then 1 else 0 end) deferred from Maillog.izhenxingroup by Mail_domain order by sent Desc; "

# # Add query results from hive table data to a new table

# hive-e "CREATE Table Maillog.izhenxin_total (Mail_domain string, sent_number int,bounced_number int, deferred int) ROW FO RMAT delimited fields TERMINATED by ' \ t ' STORED as textfile;

# hive-e "Usemaillog; Insert INTO table Izhenxin_total Select mail_domain,sum (case whenmail_status= ' sent ' then 1 else 0 end) Sent,sum Mail_status= ' bounced ' then 1 else 0 end] Bounced,sum (case if mail_status= ' deferred ' then 1 else 0end) deferred from Ma Illog.izhenxin GROUP BY Mail_domain Order by Sentdesc; "

This article is from the "Zhangdh Open Space" blog, so be sure to keep this source http://linuxblind.blog.51cto.com/7616603/1709749

Hadoop Hive Sqoop Zookeeper hbase production Environment Log Statistics application case (hive article)