Linux下Shell編程實現基於Hadoop的ETL(分析篇)__html5

加群：397706991，共同學習

conf/hive.xml

具體Shell代碼下載:http://download.csdn.net/detail/luo849278597/9490920

<?xml version="1.0" encoding="UTF-8"?><root><hive-shell type="analysis"><!-- 資料分析 --><hsql>create table if not exists think_statistics(date_type_name string,date_name string,type int,type_name string,count int) row format delimited fields terminated by '\\t' lines terminated by '\\n' stored as textfile</hsql><hsql>insert into think_statistics select concat(current_date,'-2'),current_date,2,'新任務數量',sum(case when status==2 then 1 else 0 end) from think_task</hsql><!-- hive 查詢語句 --></hive-shell></root>

bin/hive.sh

#!/bin/bash#########################以下命令的處理基於Hadoop環境以及Hive工具，並且相應的工具命令需要定義在環境變數中#sed命令是Shell編程中用於處理字串，過濾出所需字串的命令#tr -d '\r' 用於刪除每行中的分行符號#eval 用於執行字串命令#########################get the environmentif [ -f ~/.bashrc ];#進入Bash Shell命令，用於以下命令的執行then . ~/.bashrcfi#file exitif [ -z $1 ] ; then #判斷設定檔是否存在echo 'USAGE:COMMAND FILENAME'exit 0fi#預定義變數以及數組declare -a analysis_sqlsdeclare -i i=0FLAG=0content=`sed -e 's/\s*\(.*\)\s*$/\1/g' -e 's/\s*\(=\)\s*/\1/g' -e '/^\(\s\)*$/d' -e '/^$/d' -e 's/<!--.*-->//' $1`while read linedoif echo ${line}|grep -qE "^<hive-shell type=\"analysis\"" ; thenFLAG=1continuefiif echo ${line}|grep -qE "</hive-shell>" ;thenFLAG=0continuefiif [ ${FLAG} -ne 0 ] ; thensql=`echo ${line}|sed -n -e 's/<hsql>//' -e 's/<\/hsql>//p'|tr -d '\r'`fi#empty sql,then wo will not excute followif [ ${#sql} -eq 0 ] ; thencontinuefiif [ ${FLAG} -eq 1 ] ; thenanalysis_sqls[$i]=$sqllet i++continuefidone<<EOF$contentEOFshellsql="echo -e \"\033[41:33m excute hive.sh start analysis sqls \033[0m\""for ((i=0;i<${#analysis_sqls[*]};i++))doshellsql="${shellsql} && hive -e \"${analysis_sqls[$i]}\""doneshellsql="${shellsql} && echo -e \"\033[41:33m excute hive.sh finished analysis sqls \033[0m\""eval $shellsql