hadoop2.0.5配置

配置zookeeper，NN的HA依賴它

配置

vi /opt/zookeeper/conf/zoo.cfg

dataDir=/home1/hadoop/zookeeper/data
dataLogDir=/home1/hadoop/zookeeper/zk_log
logDir=/home1/hadoop/zookeeper/logs

server.110=Server110:2888:3888
server.111=Server111:2888:3888
server.112=Server112:2888:3888

在所有zk機器上建立目錄和myid檔案

mkdir -p /home1/hadoop/zookeeper/data
mkdir -p /home1/hadoop/zookeeper/zk_log
mkdir -p /home1/hadoop/zookeeper/logs
ifconfig|grep 192|awk -F'.' '{print $4}'|awk '{print $1}' >/home1/hadoop/zookeeper/data/myid

在所有zk機器上啟動

 /opt/zookeeper/bin/zkServer.sh start

配置Hadoop

1. 配置環境變數

vi /etc/profile

vi /opt/hadoop/etc/hadoop/hadoop-env.sh
export JAVA_HOME=/usr/java/jdk1.7.0_25
export HADOOP_PREFIX="/opt/hadoop"
export PATH=$PATH:$HADOOP_PREFIX/bin
export PATH=$PATH:$HADOOP_PREFIX/sbin
export HADOOP_MAPRED_HOME=${HADOOP_PREFIX}
export HADOOP_COMMON_HOME=${HADOOP_PREFIX}
export HADOOP_HDFS_HOME=${HADOOP_PREFIX}
export YARN_HOME=${HADOOP_PREFIX}
export HADOOP_SSH_OPTS="-p 9922"
export HADOOP_CONF_DIR=${HADOOP_PREFIX}/etc/hadoop
export HDFS_CONF_DIR=${HADOOP_PREFIX}/etc/hadoop

export YARN_CONF_DIR=${HADOOP_PREFIX}/etc/hadoop

2. 配置 /opt/hadoop/etc/hadoop

配置core-site.xml檔案
<property>
   <name>hadoop.tmp.dir</name>
   <value>/home1/hadoop/tmp</value>
</property>
<property>
   <name>fs.default.name</name>
   <value>hdfs://hadoop110:8020</value>
</property>

---建立並配置slaves：vi slaves 並添加以下內容
Server110
Server111
Server112
Server113
Server114

---配置hdfs-site.xml
<configuration>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
  <name>dfs.nameservices</name>
  <value>ns1</value>
</property>
<property>
  <name>dfs.ha.namenodes.ns1</name>
  <value>ns1n1,ns1n2</value>
</property>
<property>
  <name>dfs.namenode.rpc-address.ns1.ns1n1</name>
  <value>Server110:8020</value>
</property>
<property>
  <name>dfs.namenode.rpc-address.ns1.ns1n2</name>
  <value>Server113:8020</value>
</property>
<property>
  <name>dfs.namenode.http-address.ns1.ns1n1</name>
  <value>Server110:50070</value>
</property>
<property>
  <name>dfs.namenode.http-address.ns1.ns1n2</name>
  <value>Server113:50070</value>
</property>
<property>
  <name>dfs.namenode.shared.edits.dir</name>
  <value>qjournal://Server110:8485;Server111:8485;Server112:8485/ns1</value>
</property>
<property>
  <name>dfs.client.failover.proxy.provider.ns1</name>
  <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
  <name>dfs.ha.fencing.methods</name>
  <value>sshfence(huanglq:9922)</value>
</property>
<property>
  <name>dfs.ha.fencing.ssh.private-key-files</name>
  <value>/home/hadoop/.ssh/id_rsa</value>
</property>
<property>
  <name>dfs.ha.fencing.ssh.connect-timeout</name>
  <value>30000</value>
</property>
<property>
  <name>dfs.journalnode.edits.dir</name>
  <value>/home1/hadoop/journal/node/local/data</value>
</property>
<property>
   <name>dfs.ha.automatic-failover.enabled</name>
   <value>true</value>
 </property>
 <property>
   <name>ha.zookeeper.quorum</name>
   <value>Server110:2181,Server111:2181,Server112:2181</value>
 </property>
 
<property>
   <name>dfs.namenode.name.dir</name>
   <value>file:/home1/hadoop/hdfs/name</value>
   <final>true</final>
</property>

<property>
   <name>dfs.federation.nameservice.id</name>
   <value>ns1</value>
</property>

<property>
   <name>dfs.federation.nameservices</name>
   <value>ns1</value>
   <!--<value>ns1,ns2</value>-->
</property>

<property>
   <name>dfs.namenode.rpc-address.ns1</name>
   <value>Server110:8020</value>
</property>

<property>
   <name>dfs.namenode.rpc-address.ns2</name>
   <value>Server111:8020</value>
</property>

<property>
   <name>dfs.namenode.http-address.ns1</name>
   <value>Server110:23001</value>
</property>

<property>
   <name>dfs.namenode.http-address.ns2</name>
   <value>Server111:13001</value>
</property>

<property>
   <name>dfs.dataname.data.dir</name>
   <value>file:/home1/hadoop/hdfs/data</value>
   <final>true</final>
</property>
<!--
使用了HA，backupnode和secondary namenode都不需要配置了
<property>
   <name>dfs.namenode.backup.address.ns1</name>
   <value>Server113:50100</value>
</property>

<property>
   <name>dfs.namenode.backup.http-address.ns1</name>
   <value>Server113:50105</value>
</property>
<property>
   <name>dfs.namenode.secondary.http-address.ns1</name>
   <value>Server110:23002</value>
</property>

<property>
   <name>dfs.namenode.secondary.http-address.ns2</name>
   <value>Server111:23002</value>
</property>

<property>
   <name>dfs.namenode.secondary.http-address.ns1</name>
   <value>Server110:23003</value>
</property>

<property>
   <name>dfs.namenode.secondary.http-address.ns2</name>
   <value>Server111:23003</value>
</property>

 -->
</configuration>

----配置yarn-site.xml
<configuration>

<!-- Site specific YARN configuration properties -->
<property>
   <name>yarn.resourcemanager.address</name>
   <value>Server110:18040</value>
</property>

<property>
   <name>yarn.resourcemanager.scheduler.address</name>
   <value>Server110:18030</value>
</property>

<property>
   <name>yarn.resourcemanager.webapp.address</name>
   <value>Server110:18088</value>
</property>

<property>
   <name>yarn.resourcemanager.resource-tracker.address</name>
   <value>Server110:18025</value>
</property>

<property>
   <name>yarn.resourcemanager.admin.address</name>
   <value>Server110:18141</value>
</property>

<property>
   <name>yarn.nodemanager.aux-services</name>
   <value>mapreduce.shuffle</value>
</property>
</configuration>

3. 初始化namenode

hdfs zkfc -formatZK

在所有的nn機器上執行

/opt/hadoop/bin/hdfs namenode -format ns1

4. 啟動
sbin/start-dfs.sh
sbin/start-yarn.sh