Install Hadoop on Ubuntu

Install java jdk 8

sudo apt install openjdk-8-jdk

Check for Jdk

cd /usr/lib/jvm

sudo nano .bashrc

paste below commands

export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 

export PATH=$PATH:/usr/lib/jvm/java-8-openjdk-amd64/bin 

export HADOOP_HOME=~/hadoop-3.3.6/ 

export PATH=$PATH:$HADOOP_HOME/bin 

export PATH=$PATH:$HADOOP_HOME/sbin 

export HADOOP_MAPRED_HOME=$HADOOP_HOME 

export YARN_HOME=$HADOOP_HOME 

export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop 

export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native 

export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native" 

export HADOOP_STREAMING=$HADOOP_HOME/share/hadoop/tools/lib/hadoop-streaming-3.3.6.jar

export HADOOP_LOG_DIR=$HADOOP_HOME/logs 

export PDSH_RCMD_TYPE=ssh

sudo apt-get install ssh

tar -zxvf ~/Downloads/hadoop-3.3.6.tar.gz 

(Extract the tar file)

cd hadoop-3.3.6/etc/hadoop

sudo nano hadoop-env.sh

JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 (set the path for JAVA_HOME)

core-site.xml

<configuration> 

 <property> 

 <name>fs.defaultFS</name> 

 <value>hdfs://localhost:9000</value>  </property> 

 <property> 

<name>hadoop.proxyuser.dataflair.groups</name> <value>*</value> 

 </property> 

 <property> 

<name>hadoop.proxyuser.dataflair.hosts</name> <value>*</value> 

 </property> 

 <property> 

<name>hadoop.proxyuser.server.hosts</name> <value>*</value> 

 </property> 

 <property> 

<name>hadoop.proxyuser.server.groups</name> <value>*</value> 

 </property> 

</configuration>

hdfs-site.xml

<configuration> 

 <property> 

 <name>dfs.replication</name> 

 <value>1</value> 

 </property> 

</configuration>

mapred-site.xml

<configuration> 

 <property> 

 <name>mapreduce.framework.name</name>  <value>yarn</value> 

 </property> 

 <property>

 <name>mapreduce.application.classpath</name> 

 <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value> 

 </property> 

</configuration>

yarn-site.xml

<configuration> 

 <property> 

 <name>yarn.nodemanager.aux-services</name> 

 <value>mapreduce_shuffle</value> 

 </property> 

 <property> 

 <name>yarn.nodemanager.env-whitelist</name> 

<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREP END_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value> 

 </property> 

</configuration>

ssh

ssh localhost 

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa 

cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys 

chmod 0600 ~/.ssh/authorized_keys 

hadoop-3.3.6/bin/hdfs namenode -format

export PDSH_RCMD_TYPE=ssh

To start hadoop

start-all.sh (Start NameNode daemon and DataNode daemon) 

browse - localhost:9870

$ hadoop fs -mkdir /sagar

$ hadoop fs -ls

stop-all.sh