CentOS 6.9 Hadoop 2.7.2

CentOS 6.9 Hadoop 2.7.2

master 192.168.1.130
slave 192.168.1.131

master

[root@localhost ~]# vi /etc/hosts

127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
127.0.0.1 localhost
192.168.1.130 master
192.168.1.131 slave

[root@localhost ~]# vi /etc/sysconfig/network

NETWORKING=yes
HOSTNAME=localhost.localdomain
NETWORKING=yes
HOSTNAME=master

slave

[root@localhost ~]# vi /etc/hosts

127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
127.0.0.1 localhost
192.168.1.130 master
192.168.1.131 slave

[root@localhost ~]# vi /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=localhost.localdomain
NETWORKING=yes
HOSTNAME=slave

hostname your-hostname

selinux

master

[root@master ~]# vim /etc/selinux/config
SELINUX=enforcing

SELINUX=disabled

firewall

CentOS iptables

master

[root@master ~]# iptables -F; /etc/init.d/iptables save
iptables: Saving firewall rules to /etc/sysconfig/iptables:[ OK ]
iptables -nvL

master

[root@master ~]# ssh-keygen

master

[root@master ~]# cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
[root@master ~]# scp ~/.ssh/authorized_keys slave:~/.ssh/
slave

[root@slave ~]# ls .ssh/
authorized_keys
master

[root@master ~]# ssh slave
[root@slave ~]# exit
[root@master ~]#

JDK

# java -version

[root@master ~]# wget http://download.Oracle.com/otn-pub/java/jdk/7u79-b15/jdk-7u79-linux-x64.tar.gz?AuthParam=1480051498_4f2fdb0325a457f4c7d33a69355b3560

[root@master ~]# mv jdk-7u79-linux-x64.tar.gz\?AuthParam\=1480051498_4f2fdb0325a457f4c7d33a69355b3560 jdk-7u79-linux-x64.tar.gz
[root@master ~]# tar zxvf jdk-7u79-linux-x64.tar.gz
[root@master ~]# mv jdk1.7.0_79 /usr/local/

[root@master ~]# vi /etc/profile.d/java.sh

export JAVA_HOME=/usr/local/jdk1.7.0_79
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
[root@master ~]# source /etc/profile.d/java.sh
[root@master ~]# java -version
java version “1.7.0_79”
Java(TM) SE Runtime Environment (build 1.7.0_79-b15)
Java HotSpot(TM) 64-Bit Server VM (build 24.79-b02, mixed mode)

[root@master ~]# scp jdk-7u79-linux-x64.tar.gz slave:/root/
[root@master ~]# scp /etc/profile.d/java.sh slave:/etc/profile.d/
slave

[root@slave ~]# tar zxvf jdk-7u79-linux-x64.tar.gz
[root@slave ~]# mv jdk1.7.0_79 /usr/local/

[root@slave ~]# source /etc/profile.d/java.sh
[root@slave ~]# java -version
java version “1.7.0_79”
Java(TM) SE Runtime Environment (build 1.7.0_79-b15)
Java HotSpot(TM) 64-Bit Server VM (build 24.79-b02, mixed mode)

Hadoop

master

[root@master ~]# wget https://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-2.7.2/hadoop-2.7.2.tar.gz
[root@master ~]# tar zxvf hadoop-2.7.2.tar.gz
[root@master ~]# mv hadoop-2.7.2 /usr/local/
[root@master ~]# ls /usr/local/
bin games include lib libexec share etc hadoop-2.7.2 jdk1.7.0_79 lib64 sbin src
[root@master ~]# ls /usr/local/hadoop-2.7.2/
bin include libexec NOTICE.txt sbin etc lib LICENSE.txt README.txt share
[root@master ~]# mkdir /usr/local/hadoop-2.7.2/tmp /usr/local/hadoop-2.7.2/dfs /usr/local/hadoop-2.7.2/dfs/data /usr/local/hadoop-2.7.2/dfs/name
/usr/local/hadoop-2.7.2/tmp
/usr/local/hadoop-2.7.2/dfs?

[root@master ~]# ls /usr/local/hadoop-2.7.2/
bin etc lib LICENSE.txt README.txt share dfs include libexec NOTICE.txt sbin tmp
[root@master ~]# rsync -av /usr/local/hadoop-2.7.2 slave:/usr/local
slave

[root@slave ~]# ls /usr/local/hadoop-2.7.2
bin etc lib LICENSE.txt README.txt share dfs include libexec NOTICE.txt sbin tmp

Hadoop

master

[root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/core-site.xml
fs.defaultFS
hdfs://master:9000 hadoop.tmp.dir
file:/usr/local/hadoop-2.7.2/tmp io.file.buffer.size
131072

fs.defaultFS
NameNode ?HDFS MapReduce core-site.xml hdfs-site.xml

[root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/hdfs-site.xml

dfs.namenode.name.dir
file:/usr/local/hadoop-2.7.2/dfs/name dfs.datanode.data.dir
file:/usr/local/hadoop-2.7.2/dfs/data dfs.replication
1 dfs.namenode.secondary.http-address
master:9001 dfs.webhdfs.enabled
true

[root@master ~]# mv /usr/local/hadoop-2.7.2/etc/hadoop/mapred-site.xml.template /usr/local/hadoop-2.7.2/etc/hadoop/mapred-site.xml
[root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/mapred-site.xml

mapreduce.framework.name
yarn mapreduce.jobhistory.address
master:10020 mapreduce.jobhistory.webapp.address
master:19888

[root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/yarn-site.xml

yarn.nodemanager.aux-services
mapreduce_shuffle yarn.nodemanager.auxservices.mapreduce.shuffle.class
org.apache.hadoop.mapred.ShuffleHandler yarn.resourcemanager.address
master:8032 yarn.resourcemanager.scheduler.address
master:8030 yarn.resourcemanager.resource-tracker.address
master:8031 yarn.resourcemanager.admin.address
master:8033 yarn.resourcemanager.webapp.address
master:8088

[root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/hadoop-env.sh
export JAVA_HOME=${JAVA_HOME}
export JAVA_HOME=/usr/local/jdk1.7.0_79

[root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/yarn-env.sh

export JAVA_HOME=/usr/local/jdk1.7.0_79

root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/mapred-env.sh
# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
export JAVA_HOME=/usr/local/jdk1.7.0_79

[root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/slaves
localhost
slave

[root@master ~]# rsync -av /usr/local/hadoop-2.7.2/etc/ slave:/usr/local/hadoop-2.7.2/etc/

[root@master ~]# vi /etc/profile.d/hadoop.sh

export HADOOP_HOME=/usr/local/hadoop-2.7.2
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

[root@master ~]# source /etc/profile.d/hadoop.sh
[root@master ~]# hadoop version
Hadoop 2.7.2
Subversion https://git-wip-us.apache.org/repos/asf/hadoop.git -r b165c4fe8a74265c792ce23f546c64604acf0e41
Compiled by jenkins on 2016-01-26T00:08Z
Compiled with protoc 2.5.0
From source with checksum d0fda26633fa762bff87ec759ebe689c
This command was run using /usr/local/hadoop-2.7.2/share/hadoop/common/hadoop-common-2.7.2.jar

[root@master ~]# scp /etc/profile.d/hadoop.sh slave:/etc/profile.d/
slave

hadoop

[root@slave ~]# source /etc/profile.d/hadoop.sh
[root@slave ~]# hadoop version
Hadoop 2.7.2
Subversion https://git-wip-us.apache.org/repos/asf/hadoop.git -r b165c4fe8a74265c792ce23f546c64604acf0e41
Compiled by jenkins on 2016-01-26T00:08Z
Compiled with protoc 2.5.0
From source with checksum d0fda26633fa762bff87ec759ebe689c
This command was run using /usr/local/hadoop-2.7.2/share/hadoop/common/hadoop-common-2.7.2.jar

Hadoop

master

[root@master ~]# /usr/local/hadoop-2.7.2/bin/hdfs namenode -format
[root@master ~]# echo $?
0

root@master ~]# /usr/local/hadoop-2.7.2/sbin/start-all.sh
[root@master ~]# jps
5560 ResourceManager
5239 NameNode
5631 Jps
5415 SecondaryNameNode
slave

[root@slave ~]# jps
5231 DataNode
5444 Jps
5320 NodeManager
master

web UI
master:50070 ?namenode datanode?
master:8088 ?Yarn?

[root@master ~]# cd /usr/local/hadoop-2.7.2/
[root@master hadoop-2.7.2]# bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.2.jar pi 1

[root@master ~]# /usr/local/hadoop-2.7.2/sbin/stop-all.sh

127.0.0.1 localhost
192.168.229.130 master
192.168.229.131 slave

copyFromLocal: Cannot create directory /123/. Name node is in safe mode

[root@master ~]# /usr/local/hadoop-2.7.2/bin/hdfs dfsadmin -safemode leave