HADOOP2 Installation Scripts

#!/bin/bash
#
# Install Hadoop 2 using PDSH/PDCP where possible.
#
# Command can be interactive or file-based. This script sets up
# a Hadoop 2 cluster with basic configuration. Modify data, log, and PID
# directories as desired. Further configure your cluster with./conf-hadoop2.sh
#
#

# Basic Environment variables. Edit as necessary
Hadoop_version=2.0.5-alpha
Hadoop_home= "/opt/hadoop-${hadoop_version}"
Nn_data_dir=/var/data/hadoop/hdfs/nn
Snn_data_dir=/var/data/hadoop/hdfs/snn
Dn_data_dir=/var/data/hadoop/hdfs/dn
Yarn_log_dir=/var/log/hadoop/yarn
Hadoop_log_dir=/var/log/hadoop/hdfs
Hadoop_mapred_log_dir=/var/log/hadoop/mapred
Yarn_pid_dir=/var/run/hadoop/yarn
Hadoop_pid_dir=/var/run/hadoop/hdfs
Hadoop_mapred_pid_dir=/var/run/hadoop/mapred
Http_static_user=hdfs
yarn_proxy_port=8081

SOURCE hadoop-xml-conf.sh
cmd_options=$ (getopt-n "$"-O hif--long "Help,interactive,file"--"[email protected]")

# Take care of bad options in the command
If [$?-ne 0];
Then
Exit 1
Fi
Eval set--"$CMD _options"

All_hosts= "All_hosts"
Nn_host= "Nn_host"
Snn_host= "Snn_host"
Dn_hosts= "Dn_hosts"
Rm_host= "Rm_host"
Nm_hosts= "Nm_hosts"
Mr_history_host= "Mr_history_host"
Yarn_proxy_host= "Yarn_proxy_host"

Install ()
{
echo "Copying Hadoop $HADOOP _version to all hosts ..."
Pdcp-w ^all_hosts hadoop-"$HADOOP _version". tar.gz/opt

echo "Copying JDK 1.6.0_31 to all hosts ..."
Pdcp-w ^all_hosts jdk-6u31-linux-x64-rpm.bin/opt

echo "Installing JDK 1.6.0_31 on all hosts ..."
Pdsh-w ^all_hosts chmod A+x/opt/jdk-6u31-linux-x64-rpm.bin
Pdsh-w ^all_hosts/opt/jdk-6u31-linux-x64-rpm.bin-noregister 1>&-2>&-

echo "Setting java_home and Hadoop_home environment variables on all hosts ..."
Pdsh-w ^all_hosts ' echo export java_home=/usr/java/jdk1.6.0_31 >/etc/profile.d/java.sh '
Pdsh-w ^all_hosts "source/etc/profile.d/java.sh"
Pdsh-w ^all_hosts "Echo export hadoop_home= $HADOOP _home >/etc/profile.d/hadoop.sh"
Pdsh-w ^all_hosts ' echo export hadoop_prefix= $HADOOP _home >>/etc/profile.d/hadoop.sh '
Pdsh-w ^all_hosts "source/etc/profile.d/hadoop.sh"

echo "Extracting Hadoop $HADOOP _version distribution on all hosts ..."
Pdsh-w ^all_hosts tar-zxf/opt/hadoop-"$HADOOP _version". Tar.gz-c/opt

echo "Creating system accounts and groups on all hosts ..."
Pdsh-w ^all_hosts Groupadd Hadoop
Pdsh-w ^all_hosts useradd-g Hadoop yarn
Pdsh-w ^all_hosts useradd-g Hadoop HDFs
Pdsh-w ^all_hosts useradd-g Hadoop mapred

echo "Creating HDFS data directories on NameNode host, secondary NameNode host, and DataNode hosts ..."
Pdsh-w ^nn_host "Mkdir-p $NN _data_dir && chown hdfs:hadoop $NN _data_dir"
Pdsh-w ^snn_host "Mkdir-p $SNN _data_dir && chown hdfs:hadoop $SNN _data_dir"
Pdsh-w ^dn_hosts "Mkdir-p $DN _data_dir && chown hdfs:hadoop $DN _data_dir"

echo "Creating log directories on all hosts ..."
Pdsh-w ^all_hosts "Mkdir-p $YARN _log_dir && chown yarn:hadoop $YARN _log_dir"
Pdsh-w ^all_hosts "Mkdir-p $HADOOP _log_dir && chown hdfs:hadoop $HADOOP _log_dir"
Pdsh-w ^all_hosts "Mkdir-p $HADOOP _mapred_log_dir && chown mapred:hadoop $HADOOP _mapred_log_dir"

echo "Creating PID directories on all hosts ..."
Pdsh-w ^all_hosts "Mkdir-p $YARN _pid_dir && chown yarn:hadoop $YARN _pid_dir"
Pdsh-w ^all_hosts "Mkdir-p $HADOOP _pid_dir && chown hdfs:hadoop $HADOOP _pid_dir"
Pdsh-w ^all_hosts "Mkdir-p $HADOOP _mapred_pid_dir && chown mapred:hadoop $HADOOP _mapred_pid_dir"

echo "Editing Hadoop environment scripts for log directories on all hosts ..."
Pdsh-w ^all_hosts echo "Export hadoop_log_dir= $HADOOP _log_dir >> $HADOOP _home/etc/hadoop/hadoop-env.sh"
Pdsh-w ^all_hosts echo "Export yarn_log_dir= $YARN _log_dir >> $HADOOP _home/etc/hadoop/yarn-env.sh"
Pdsh-w ^all_hosts echo "Export hadoop_mapred_log_dir= $HADOOP _mapred_log_dir >> $HADOOP _home/etc/hadoop/ Mapred-env.sh "

echo "Editing Hadoop Environment scripts for PID directories on all hosts ..."
Pdsh-w ^all_hosts echo "Export hadoop_pid_dir= $HADOOP _pid_dir >> $HADOOP _home/etc/hadoop/hadoop-env.sh"
Pdsh-w ^all_hosts echo "Export yarn_pid_dir= $YARN _pid_dir >> $HADOOP _home/etc/hadoop/yarn-env.sh"
Pdsh-w ^all_hosts echo "Export hadoop_mapred_pid_dir= $HADOOP _mapred_pid_dir >> $HADOOP _home/etc/hadoop/ Mapred-env.sh "

echo "Creating base Hadoop XML config files ..."
Create_config--file Core-site.xml
Put_config--file core-site.xml--property fs.default.name--value "hdfs://$nn: 9000"
Put_config--file core-site.xml--property hadoop.http.staticuser.user--value "$HTTP _static_user"

Create_config--file Hdfs-site.xml
Put_config--file hdfs-site.xml--property dfs.namenode.name.dir--value "$NN _data_dir"
Put_config--file hdfs-site.xml--property fs.checkpoint.dir--value "$SNN _data_dir"
Put_config--file hdfs-site.xml--property fs.checkpoint.edits.dir--value "$SNN _data_dir"
Put_config--file hdfs-site.xml--property dfs.datanode.data.dir--value "$DN _data_dir"
Put_config--file hdfs-site.xml--property dfs.namenode.http-address--value "$nn: 50070"
Put_config--file hdfs-site.xml--property dfs.namenode.secondary.http-address--value "$SNN: 50090"

Create_config--file Mapred-site.xml
Put_config--file mapred-site.xml--property mapreduce.framework.name--value yarn
Put_config--file mapred-site.xml--property mapreduce.jobhistory.address--value "$mr _hist:10020"
Put_config--file mapred-site.xml--property mapreduce.jobhistory.webapp.address--value "$mr _hist:19888"
Put_config--file mapred-site.xml--property yarn.app.mapreduce.am.staging-dir--value/mapred

Create_config--file Yarn-site.xml
Put_config--file yarn-site.xml--property yarn.nodemanager.aux-services--value mapreduce.shuffle
Put_config--file yarn-site.xml--property yarn.nodemanager.aux-services.mapreduce.shuffle.class--value Org.apache.hadoop.mapred.ShuffleHandler
Put_config--file yarn-site.xml--property yarn.web-proxy.address--value "$yarn _proxy: $YARN _proxy_port"
Put_config--file yarn-site.xml--property yarn.resourcemanager.scheduler.address--value "$rmgr: 8030"
Put_config--file yarn-site.xml--property yarn.resourcemanager.resource-tracker.address--value "$rmgr: 8031"
Put_config--file yarn-site.xml--property yarn.resourcemanager.address--value "$rmgr: 8032"
Put_config--file yarn-site.xml--property yarn.resourcemanager.admin.address--value "$rmgr: 8033"
Put_config--file yarn-site.xml--property yarn.resourcemanager.webapp.address--value "$rmgr: 8088"

echo "Copying base Hadoop XML config files to all hosts ..."
Pdcp-w ^all_hosts core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml $HADOOP _home/etc/hadoop/

echo "Creating configuration, command, and script links on all hosts ..."
Pdsh-w ^all_hosts "Ln-s $HADOOP _home/etc/hadoop/etc/hadoop"
Pdsh-w ^all_hosts "Ln-s $HADOOP _home/bin/*/usr/bin"
Pdsh-w ^all_hosts "Ln-s $HADOOP _home/libexec/*/usr/libexec"

echo "Formatting the NameNode ..."
Pdsh-w ^nn_host "Su-hdfs-c ' $HADOOP _home/bin/hdfs namenode-format '"

echo "Copying startup scripts to all hosts ..."
Pdcp-w ^nn_host hadoop-namenode/etc/init.d/
Pdcp-w ^snn_host hadoop-secondarynamenode/etc/init.d/
Pdcp-w ^dn_hosts hadoop-datanode/etc/init.d/
Pdcp-w ^rm_host hadoop-resourcemanager/etc/init.d/
Pdcp-w ^nm_hosts hadoop-nodemanager/etc/init.d/
Pdcp-w ^mr_history_host hadoop-historyserver/etc/init.d/
Pdcp-w ^yarn_proxy_host hadoop-proxyserver/etc/init.d/

echo "Starting Hadoop $HADOOP _version services on all hosts ..."
Pdsh-w ^nn_host "chmod 755/etc/init.d/hadoop-namenode && chkconfig hadoop-namenode on && service hadoop- Namenode Start "
Pdsh-w ^snn_host "chmod 755/etc/init.d/hadoop-secondarynamenode && chkconfig hadoop-secondarynamenode on & & Service Hadoop-secondarynamenode Start "
Pdsh-w ^dn_hosts "chmod 755/etc/init.d/hadoop-datanode && chkconfig hadoop-datanode on && service Hadoop -datanode Start "
Pdsh-w ^rm_host "chmod 755/etc/init.d/hadoop-resourcemanager && chkconfig hadoop-resourcemanager on && Service Hadoop-resourcemanager Start "
Pdsh-w ^nm_hosts "chmod 755/etc/init.d/hadoop-nodemanager && chkconfig hadoop-nodemanager on && service Hadoop-nodemanager Start "

Pdsh-w ^yarn_proxy_host "chmod 755/etc/init.d/hadoop-proxyserver && chkconfig hadoop-proxyserver on && Service Hadoop-proxyserver Start "

echo "Creating MapReduce Job history directories ..."
Su-hdfs-c "Hadoop fs-mkdir-p/mapred/history/done_intermediate"
Su-hdfs-c "Hadoop fs-chown-r mapred:hadoop/mapred"
Su-hdfs-c "Hadoop fs-chmod-r g+rwx/mapred"

Pdsh-w ^mr_history_host "chmod 755/etc/init.d/hadoop-historyserver && chkconfig hadoop-historyserver on && Amp Service Hadoop-historyserver Start "

echo "Running YARN smoke test ..."
Pdsh-w ^all_hosts "Usermod-a-G Hadoop $ (WhoAmI)"
Su-hdfs-c "Hadoop fs-mkdir-p/user/$ (whoami)"
Su-hdfs-c "Hadoop Fs-chown $ (WHOAMI): $ (WhoAmI)/user/$ (WhoAmI)"
source/etc/profile.d/java.sh
source/etc/profile.d/hadoop.sh
source/etc/hadoop/hadoop-env.sh
source/etc/hadoop/yarn-env.sh
Hadoop jar $HADOOP _home/share/hadoop/mapreduce/hadoop-mapreduce-examples-$HADOOP _version.jar Pi- Dmapreduce.clientfactory.class.name=org.apache.hadoop.mapred.yarnclientfactory-libjars $HADOOP _home/share/ hadoop/mapreduce/hadoop-mapreduce-client-jobclient-$HADOOP _version.jar 16 10000
}

Interactive ()
{
Echo-n "Enter NameNode hostname:"
Read nn
Echo-n "Enter Secondary NameNode hostname:"
Read SNN
Echo-n "Enter ResourceManager hostname:"
Read Rmgr
Echo-n "Enter Job History Server hostname:"
Read Mr_hist
Echo-n "Enter YARN Proxy hostname:"
Read Yarn_proxy
Echo-n "Enter DataNode hostnames (comma separated or hostlist syntax):"
Read DNS
Echo-n "Enter NodeManager hostnames (comma separated or hostlist syntax):"
Read NMS

echo "$nn" > "$nn _host"
echo "$snn" > "$snn _host"
echo "$rmgr" > "$rm _host"
echo "$mr _hist" > "$MR _history_host"
echo "$yarn _proxy" > "$yarn _proxy_host"
dn_hosts_var=$ (sed ' s/\,/\n/g ' <<< $dns)
nm_hosts_var=$ (sed ' s/\,/\n/g ' <<< $nms)
echo "$dn _hosts_var" > "$dn _hosts"
echo "$nm _hosts_var" > "$nm _hosts"
echo "$ (echo" $nn $snn $rmgr $mr _hist $yarn _proxy $dn _hosts_var $nm _hosts_var "| Tr ' \ n ' | Sort-u) ">" $all _hosts "
}

File ()
{
nn=$ (Cat nn_host)
snn=$ (Cat snn_host)
rmgr=$ (Cat rm_host)
mr_hist=$ (Cat mr_history_host)
yarn_proxy=$ (Cat yarn_proxy_host)
dns=$ (Cat dn_hosts)
nms=$ (Cat nm_hosts)

echo "$ (echo" $nn $snn $rmgr $mr _hist $dns $nms "| Tr ' \ n ' | Sort-u) ">" $all _hosts "
}

Help ()
{
Cat << EOF
install-hadoop2.sh

This script installs Hadoop 2 with basic data, log, and PID directories.

usage:install-hadoop2.sh [Options]

OPTIONS:
-I,--interactive Prompt for fully qualified domain names (FQDN) of the NameNode,
Secondary NameNode, Datanodes, ResourceManager, Nodemanagers,
MapReduce Job History Server, and YARN Proxy server. Values
Entered is stored in files in the same directory as this command.

-F,--file use files with fully qualified domain names (FQDN), New-line
Separated. Place files in the same directory as this script.
Services and file name is as follows:
NameNode = Nn_host
Secondary NameNode = Snn_host
Datanodes = dn_hosts
ResourceManager = Rm_host
Nodemanagers = nm_hosts
MapReduce Job History Server = Mr_history_host
YARN Proxy Server = yarn_proxy_host

-H,--help Show this message.

EXAMPLES:
Prompt for Host names:
Install-hadoop2.sh-i
Install-hadoop2.sh--interactive

Use values from files in the same directory:
Install-hadoop2.sh-f
Install-hadoop2.sh--file

Eof
}

While true;
Do
Case "$" in

-H|--HELP)
Help
Exit 0
;;
-i|--interactive)
Interactive
Install
Shift
;;
-f|--file)
File
Install
Shift
;;
--)
Shift
Break
;;
Esac
Done

HADOOP2 Installation Scripts