Installing Spark and Scala

Tag: Spark installs Scala

1. Download Spark

Http://mirrors.cnnic.cn/apache/spark/spark-1.3.0/spark-1.3.0-bin-hadoop2.3.tgz

2. Download Scala

Http://www.scala-lang.org/download/2.10.5.html

3. Install Scala

Mkdir/usr/lib/scala

TAR–ZXVF scala-2.10.5.tgz

MV Scala-2.10.5/usr/lib/scala

4. Set Scala path

Vim/etc/bashrc

Export scala_home=/usr/lib/scala/scala-2.10.5

Export path= $SCALA _home/bin: $PATH

Source/etc/bashrc

Scala–version

5. Distribution

Scp-r/usr/lib/scala/hd2:/usr/lib/scala

Scp-r/usr/lib/scala/hd3:/usr/lib/scala

Scp-r/usr/lib/scala/hd4:/usr/lib/scala

Scp-r/usr/lib/scala/hd5:/usr/lib/scala

SCP/ETC/BASHRC HD2:/ETC/BASHRC

SCP/ETC/BASHRC HD3:/ETC/BASHRC

SCP/ETC/BASHRC HD4:/ETC/BASHRC

SCP/ETC/BASHRC HD5:/ETC/BASHRC

6. Install Spark

TAR-ZXVF spark-1.3.0-bin-hadoop2.3.tgz

Mkdir/usr/local/spark

MV Spark-1.3.0-bin-hadoop2.3/usr/local/spark

Vim/etc/bashrc

Export spark_home=/usr/local/spark/spark-1.3.0-bin-hadoop2.3

Export path= $SCALA _home/bin: $SPARK _home/bin: $PATH

Source/etc/bashrc

cd/usr/local/spark/spark-1.3.0-bin-hadoop2.3/conf/

CP Spark-env.sh.template spark-env.sh

Vim spark-env.sh

Export Java_home=/java

Export scala_home=/usr/lib/scala/scala-2.10.5

Export spark_home=/usr/local/spark/spark-1.3.0-bin-hadoop2.3

Export spark_master_ip=192.168.137.101

Export SPARK_WORKER_MEMORY=1G

Export Hadoop_conf_dir=/home/hadoop/hadoop/etc/hadoop

Export Spark_library_path= $SPARK _home/lib

Export Scala_library_path= $SPARK _library_path

CP Slaves.template Slaves

Vim Slaves

Hd1

hd2

Hd3

Hd4

Hd5

7. Distribution

SCP/ETC/BASHRC hd2:/etc

SCP/ETC/BASHRC hd3:/etc

SCP/ETC/BASHRC hd4:/etc

SCP/ETC/BASHRC hd5:/etc

Scp-r/usr/local/spark/spark-1.3.0-bin-hadoop2.3 hd2:/usr/local/spark/

Scp-r/usr/local/spark/spark-1.3.0-bin-hadoop2.3 hd3:/usr/local/spark/

Scp-r/usr/local/spark/spark-1.3.0-bin-hadoop2.3 hd4:/usr/local/spark/

Scp-r/usr/local/spark/spark-1.3.0-bin-hadoop2.3 hd5:/usr/local/spark/

7. Start

In HD1, start

CD $SPARK _home/sbin

./start-all.sh

Installing Spark and Scala