This document provides instructions for installing Spark 2.0 on a Linux system using Cloudera's pre-built Spark jar file. It includes downloading the jar file, copying it to the correct directory, setting permissions, and restarting services to make Spark available. Additionally, it shows how to install Oracle Java JDK 1.8 by downloading the tar file, extracting it, setting it as the default Java version, and verifying the installation.

1. Installing SPARK 2.0:
https://www.cloudera.com/documentation/spark2/latest/topics/spark2_installing.html
[root@mugzy ~]# wget http://archive.cloudera.com/spark2/csd/SPARK2_ON_YARN-2.3.0.cloudera2.jar
--2018-05-12 05:12:50-- http://archive.cloudera.com/spark2/csd/SPARK2_ON_YARN-2.3.0.cloudera2.jar
Resolving archive.cloudera.com (archive.cloudera.com)... 151.101.0.167, 151.101.64.167,
151.101.128.167, ...
Connecting to archive.cloudera.com (archive.cloudera.com)|151.101.0.167|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 19010 (19K) [application/java-archive]
Saving to: ‘SPARK2_ON_YARN-2.3.0.cloudera2.jar’
100%[======================================>] 19,010 13.9KB/s in 1.3s
2018-05-12 05:12:52 (13.9 KB/s) - ‘SPARK2_ON_YARN-2.3.0.cloudera2.jar’ saved [19010/19010]

2. [root@mugzy ~]# cp SPARK2_ON_YARN-2.3.0.cloudera2.jar /opt/cloudera/csd
[root@mugzy ~]# cd /opt/cloudera/csd
[root@mugzy csd]# ls
SPARK2_ON_YARN-2.3.0.cloudera2.jar
[root@mugzy csd]# chmod 644 SPARK2_ON_YARN-2.3.0.cloudera2.jar
[root@mugzy csd]# chown cloudera-scm:cloudera-scm SPARK2_ON_YARN-2.3.0.cloudera2.jar
[root@mugzy csd]# ls -l
total 20
-rw-r--r-- 1 cloudera-scm cloudera-scm 19010 May 12 05:16 SPARK2_ON_YARN-2.3.0.cloudera2.jar
[root@mugzy csd]#
[root@mugzy csd]# service cloudera-scm-server restart
Restarting cloudera-scm-server (via systemctl): [ OK ]
[root@mugzy csd]#

12. Installing Oracle Java JDK 1.8
https://tecadmin.net/install-java-8-on-centos-rhel-and-fedora/
[root@mugzy bin]# cd /opt/
[root@mugzy opt]# wget --no-cookies --no-check-certificate --header "Cookie: gpw
_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "
http://download.oracle.com/otn-pub/java/jdk/8u171-b11/512cd62ec5174c3487ac17c61a
aa89e8/jdk-8u171-linux-x64.tar.gz"
--2018-05-12 06:46:24-- http://download.oracle.com/otn-pub/java/jdk/8u171-b11/5
12cd62ec5174c3487ac17c61aaa89e8/jdk-8u171-linux-x64.tar.gz
Resolving download.oracle.com (download.oracle.com)... 23.35.212.173
Connecting to download.oracle.com (download.oracle.com)|23.35.212.173|:80... con
nected.
HTTP request sent, awaiting response... 302 Moved Temporarily
Location: https://edelivery.oracle.com/otn-pub/java/jdk/8u171-b11/512cd62ec5174c
3487ac17c61aaa89e8/jdk-8u171-linux-x64.tar.gz [following]

13. --2018-05-12 06:46:25-- https://edelivery.oracle.com/otn-pub/java/jdk/8u171-b11
/512cd62ec5174c3487ac17c61aaa89e8/jdk-8u171-linux-x64.tar.gz
Resolving edelivery.oracle.com (edelivery.oracle.com)... 23.50.186.175, 2a02:26f
0:c000:186::2d3e, 2a02:26f0:c000:183::2d3e
Connecting to edelivery.oracle.com (edelivery.oracle.com)|23.50.186.175|:443...
connected.
HTTP request sent, awaiting response... 302 Moved Temporarily
Location: http://download.oracle.com/otn-pub/java/jdk/8u171-b11/512cd62ec5174c34
87ac17c61aaa89e8/jdk-8u171-linux-x64.tar.gz?AuthParam=1526100506_26535ed17999858
17a2f3c9440bf8c35 [following]
--2018-05-12 06:46:26-- http://download.oracle.com/otn-pub/java/jdk/8u171-b11/5
12cd62ec5174c3487ac17c61aaa89e8/jdk-8u171-linux-x64.tar.gz?AuthParam=1526100506_
26535ed1799985817a2f3c9440bf8c35
Connecting to download.oracle.com (download.oracle.com)|23.35.212.173|:80... con
nected.
HTTP request sent, awaiting response... 200 OK
Length: 190890122 (182M) [application/x-gzip]
Saving to: ‘jdk-8u171-linux-x64.tar.gz’
100%[================================================================>] 190,890,122
106KB/s in 29m 40s
2018-05-12 07:16:07 (105 KB/s) - ‘jdk-8u171-linux-x64.tar.gz’ saved [190890122/190890122]
[root@mugzy opt]# ls
cloudera jdk-8u171-linux-x64.tar.gz
[root@mugzy opt]# scp jdk-8u171-linux-x64.tar.gz moly:/opt
ssh: Could not resolve hostname moly: Name or service not known
lost connection

14. [root@mugzy opt]# scp jdk-8u171-linux-x64.tar.gz molly:/opt
jdk-8u171-linux-x64.tar.gz 100% 182MB 60.7MB/s 00:03
[root@mugzy opt]# tar xzf jdk-8u171-linux-x64.tar.gz
[root@mugzy opt]# cd /opt/jdk1.8.0_171/
[root@mugzy jdk1.8.0_171]# cd /opt/jdk1.8.0_171/
[root@mugzy jdk1.8.0_171]# alternatives --install /usr/bin/java java /opt/jdk1.8.0_171/bin/java 2
[root@mugzy jdk1.8.0_171]# alternatives --config java
There is 1 program that provides 'java'.
Selection Command
-----------------------------------------------
*+ 1 /opt/jdk1.8.0_171/bin/java
Enter to keep the current selection[+], or type selection number:
[root@mugzy jdk1.8.0_171]# java -version
java version "1.8.0_171"
Java(TM) SE Runtime Environment (build 1.8.0_171-b11)
Java HotSpot(TM) 64-Bit Server VM (build 25.171-b11, mixed mode)
[root@mugzy jdk1.8.0_171]#

17. /var/lib/hadoop-yarn/yarn-nm-recovery/yarn-nm-state/LOCK: Permission denied
https://community.cloudera.com/t5/Cloudera-Manager-Installation/CDH5-2-yarn-nm-state-LOCK-
Permission-denied/td-p/42828
https://community.cloudera.com/t5/Advanced-Analytics-Apache-Spark/Spark2-Error-deploying-and-
status-ok/td-p/61657

18. [root@mugzy conf]# alternatives --display spark2-conf
spark2-conf - status is auto.
link currently points to /etc/spark2/conf.cloudera.spark2_on_yarn
/opt/cloudera/parcels/SPARK2-2.3.0.cloudera2-1.cdh5.13.3.p0.316101/etc/spark2/conf.dist - priority 10
/etc/spark2/conf.cloudera.spark2_on_yarn - priority 51
Current `best' version is /etc/spark2/conf.cloudera.spark2_on_yarn.
[root@mugzy conf]# export JAVA_HOME=/opt/jdk1.8.0_171
[root@mugzy conf]# spark2-shell
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
Welcome to
____ __
/ __/__ ___ _____/ /__
_ / _ / _ `/ __/ '_/
/___/ .__/_,_/_/ /_/_ version 2.3.0.cloudera2
/_/
Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_171)
Type in expressions to have them evaluated.
Type :help for more information.
scala>