Install hadoop with cloudera manager 5 on centos 6.5

For partition consideration, do not use LVM
Root --> 20 GB
Swap -- 2x system memory

Ram --> 4 GB

Master node:
Raid 10, dual Ethernet cards, dual power supplies, etc.

Slave node:
1. Raid is not necessary

2. HDFS partition, not using LVM
/Etc/fstab -- ext3 defaults, noatime
Mount to/data/N, for n = 0, 1, 2... (one partition per disk)

Cloudera Repository:

Http://archive.cloudera.com/cdh5/

Http://archive-primary.cloudera.com/cm5/

On cloudera manager and all cluster nodes:

At least 3 servers for zookeeper, 3 datanodes for HDFS.

1. Disable SELinux and iptables
Service iptables stop
Chkconfig iptables off; chkconfig ip6tables off

Setenforce 0
VI/etc/sysconfig/SELinux
SELinux = disabled

2. Disable Ipv6
Echo "net. ipv6.conf. All. disable_ipv6 = 1">/etc/sysctl. conf
Sysctl-P

3. VI/etc/hosts to add all hosts FQDN, like below:
192.168.1.19 cm5.local cm5
192.168.1.20 master1.local master1 # HDFS namenode
192.168.1.21 master2.local master2 # yarn ResourceManager
192.168.1.22 slave1.local slave1
192.168.1.23 slave2.local slave2

VI/etc/sysconfig/network to set FQDN

Cat <EOF>/etc/yum. Repos. d/ISO. Repo
[ISO]
Name = ISO
Base url = http://mirrors.aliyun.com/centos/6.5/ OS /x86_64
Enable = 1
Gpgcheck = 0
EOF

Yum-y install OpenSSH-clients

Echo "VM. swappiness = 0">/etc/sysctl. conf

Sysctl-P

4.On cloudera manager, We will install MySQL 5.6 and Apache
Rpm-e -- nodeps mysql-libs
Yum-y install libaio Perl
Rpm-IVH MySQL-shared-compat-5.6.20-1.el6.x86_64.rpm
Rpm-IVH MySQL-shared-5.6.20-1.el6.x86_64.rpm
Rpm-IVH MySQL-server-5.6.20-1.el6.x86_64.rpm
Rpm-IVH MySQL-client-5.6.20-1.el6.x86_64.rpm

Service MySQL start; chkconfig MySQL on
Cat ~ /. Mysql_secret
Mysqladmin-uroot-P 'oldpassword' password newpassword
Mysql_secure_installation

MV/var/lib/MySQL/ib_logfile */backup

VI/etc/My. CNF
[Mysqld]
Transaction-isolation = read-committed
# Symbolic-links = 0

Key_buffer = 16 m
Key_buffer_size = 32 m
Max_allowed_packet = 32 m
Thread_stack = 256 K
Thread_cache_size = 64
Query_cache_limit = 8 m
Query_cache_size = 64 m
Query_cache_type = 1
Max_connections = 550

Log-bin = mysql-bin
Binlog_format = mixed
Expire_logs_days = 10
Max_binlog_size = 100 m

Read_buffer_size = 2 m
Read_rnd_buffer_size = 16 m
Sort_buffer_size = 8 m
Join_buffer_size = 8 m

# InnoDB settings
Innodb_file_per_table = 1
Innodb_flush_log_at_trx_commit = 2
Innodb_log_buffer_size = 64 m
Innodb_buffer_pool_size = 4G
Innodb_thread_concurrency = 8
Innodb_flush_method = o_direct
Innodb_log_file_size = 512 m

Service MySQL restart

Install MySQL JDBC ctor:

Tar zxf mysql-connector-java-5.1.33.tar.gz
Mkdir/usr/share/Java
CP mysql-connector-java-5.1.33/mysql-connector-java-5.1.33-bin.jar/usr/share/Java/mysql-connector-java.jar

Mysql-u root-P

# For activity monitor
Create Database Amon default Character Set utf8;
Grant all on Amon. * To 'amon' @ '%' identified by 'amon ';
Grant all on Amon. * To 'amon' @ 'localhost' identified by 'amon ';

# For reports Manager
Create Database RMAN default Character Set utf8;
Grant all on RMAN. * To 'rman '@' % 'identified by 'rman ';
Grant all on RMAN. * To 'rman '@ 'localhost' identified by 'rman ';

# For Hive MetaStore Server
Create Database MetaStore default Character Set utf8;
Grant all on MetaStore. * To 'hive' @ '%' identified by 'hive ';
Grant all on MetaStore. * To 'hive' @ 'localhost' identified by 'hive ';

# For sentry Server
Create Database sentry default Character Set utf8;
Grant all on sentry. * To 'sentry '@' % 'identified by 'sentry ';
Grant all on sentry. * To 'sentry '@ 'localhost' identified by 'sentry ';

# For cloudera navigator audit server
Create Database nav default Character Set utf8;
Grant all on NAV. * To 'nav' @ '%' identified by 'nav ';
Grant all on NAV. * To 'nav' @ 'localhost' identified by 'nav ';

Flush privileges;

Yum-y install httpd

Service httpd start; chkconfig httpd on

Mkdir/var/www/html/cm513

Mkdir/var/www/html/cdh513

Mount-o loop cm513.iso/var/www/html/cm513

Mount-o loop cdh513.iso/var/www/html/cdh513

Cat <EOF>/etc/yum. Repos. d/cm513.repo
[Cm513]
Name = cm513
Baseurl = http: // 192.168.1.19/cm513
Enable = 1
Gpgcheck = 0
EOF

Cat <EOF>/etc/yum. Repos. d/cdh513.repo
[Cdh513]
Name = cdh513
Baseurl = http: // 192.168.1.19/cdh513
Enable = 1
Gpgcheck = 0
EOF

Yum-y install oracle-j2sdk1.7 cloudera-Manager-daemons cloudera-Manager-server cloudera-Manager-agent

Ln-S/usr/Java/jdk1.7.0 _ 55-cloudera/usr/Java/Default
Echo 'export java_home =/usr/Java/default'>/etc/profile
Echo 'export Path = $ java_home/bin: $ path'>/etc/profile
Source/etc/profile

/Usr/share/CMF/Schema/scm_prepare_database.sh mysql-uroot-ppassword cm5 cm5 cm5

Service cloudera-SCM-Server start

Service cloudera-SCM-Agent start

Http: // 192.168.1.19: 7180

Username/password: admin/admin

5.On all cluster nodes

Cat <EOF>/etc/yum. Repos. d/cm513.repo
[Cm513]
Name = cm513
Baseurl = http: // 192.168.1.19/cm513
Enable = 1
Gpgcheck = 0
EOF

Cat <EOF>/etc/yum. Repos. d/cdh513.repo
[Cdh513]
Name = cdh513
Baseurl = http: // 192.168.1.19/cdh513
Enable = 1
Gpgcheck = 0
EOF

The yum-y install oracle-j2sdk1.7 cloudera-Manager-Agent cloudera-Manager-daemons

Ln-S/usr/Java/jdk1.7.0 _ 55-cloudera/usr/Java/Default
Echo 'export java_home =/usr/Java/default'>/etc/profile
Echo 'export Path = $ java_home/bin: $ path'>/etc/profile
Source/etc/profile

VI/etc/cloudera-SCM-agent/config. ini
Server_host = cm5.local
Server_port = 7182

Service cloudera-SCM-Agent start

Yum-y install Avro-tools crunch flume-ng hadoop-HDFS-fuse hadoop-hdfs-nfs3 hadoop-httpfs hbase-SOLR hive-hbase hive-webhcat hue-Beeswax hue-hbase hue Impala hue-pig hue-plugins hue-RDBMS hue-search hue-Spark hue-sqoop hue-Zookeeper Impala-shell kite llama mahout oozie pig-UDF-datafu search sentry SOLR-mapreduce spark-Python sqoop sqoop2 whirr

650) This. width = 650; "src =" http://s3.51cto.com/wyfs02/M00/4C/46/wKioL1Q6aDWz5dxUAAF82h7mSpE211.jpg "style =" float: none; "Title =" snap1.jpg "alt =" wkiol1q6adwz5dxuaaf821_mspe211.jpg "/>

650) This. width = 650; "src =" http://s3.51cto.com/wyfs02/M00/4C/44/wKiom1Q6Z_7SQF2DAAG3QgFrelw856.jpg "style =" float: none; "Title =" snap3.jpg "alt =" wkiom1q6z_7sqf2daag3qgfrelw856.jpg "/>

650) This. width = 650; "src =" http://s3.51cto.com/wyfs02/M00/4C/42/wKiom1Q6LYriiNt6AAgcQda6gcE346.jpg "style =" float: none; "Title =" snap10.jpg "alt =" wkiom1q6lyriint6aagcqda6gce346.jpg "/>

650) This. width = 650; "src =" http://s3.51cto.com/wyfs02/M01/4C/44/wKiom1Q6aHnyY8ZkAAeaPXr9aaM460.jpg "Title =" 2.jpg" alt = "wkiom1q6ahnyy8zkaaeapxr9aam460.jpg"/>

650) This. width = 650; "src =" http://s3.51cto.com/wyfs02/M01/4C/42/wKiom1Q6LYvTvWpKAAOd7mtGLuo403.jpg "style =" float: none; "Title =" snap19.jpg "alt =" wkiom1q6lyvtvwpkaaod7mtgluo403.jpg "/>

650) This. width = 650; "src =" http://s3.51cto.com/wyfs02/M02/4C/42/wKiom1Q6LY2RFRltAA7ozzeQlDo252.jpg "style =" float: none; "Title =" snapshotjpg "alt =" wkiom1q6ly2rfrltaa7ozzeqldo252.jpg "/>

OK

This article is from the ilovecat blog, be sure to keep this source http://hj192837.blog.51cto.com/655995/1562854

Install hadoop with cloudera manager 5 on centos 6.5