CentOS 6.9 Hadoop 2.7.2
master 192.168.1.130
slave 192.168.1.131
master
[root@localhost ~]# vi /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
127.0.0.1 localhost
192.168.1.130 master
192.168.1.131 slave
[root@localhost ~]# vi /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=localhost.localdomain
NETWORKING=yes
HOSTNAME=master
slave
[root@localhost ~]# vi /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
127.0.0.1 localhost
192.168.1.130 master
192.168.1.131 slave
[root@localhost ~]# vi /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=localhost.localdomain
NETWORKING=yes
HOSTNAME=slave
hostname your-hostname
selinux
master
[root@master ~]# vim /etc/selinux/config
SELINUX=enforcing
SELINUX=disabled
firewall
CentOS iptables
master
[root@master ~]# iptables -F; /etc/init.d/iptables save
iptables: Saving firewall rules to /etc/sysconfig/iptables:[ OK ]
iptables -nvL
master
[root@master ~]# ssh-keygen
master
[root@master ~]# cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
[root@master ~]# scp ~/.ssh/authorized_keys slave:~/.ssh/
slave
[root@slave ~]# ls .ssh/
authorized_keys
master
[root@master ~]# ssh slave
[root@slave ~]# exit
[root@master ~]#
JDK
# java -version
[root@master ~]# wget http://download.Oracle.com/otn-pub/java/jdk/7u79-b15/jdk-7u79-linux-x64.tar.gz?AuthParam=1480051498_4f2fdb0325a457f4c7d33a69355b3560
[root@master ~]# mv jdk-7u79-linux-x64.tar.gz\?AuthParam\=1480051498_4f2fdb0325a457f4c7d33a69355b3560 jdk-7u79-linux-x64.tar.gz
[root@master ~]# tar zxvf jdk-7u79-linux-x64.tar.gz
[root@master ~]# mv jdk1.7.0_79 /usr/local/
[root@master ~]# vi /etc/profile.d/java.sh
export JAVA_HOME=/usr/local/jdk1.7.0_79
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
[root@master ~]# source /etc/profile.d/java.sh
[root@master ~]# java -version
java version “1.7.0_79”
Java(TM) SE Runtime Environment (build 1.7.0_79-b15)
Java HotSpot(TM) 64-Bit Server VM (build 24.79-b02, mixed mode)
[root@master ~]# scp jdk-7u79-linux-x64.tar.gz slave:/root/
[root@master ~]# scp /etc/profile.d/java.sh slave:/etc/profile.d/
slave
[root@slave ~]# tar zxvf jdk-7u79-linux-x64.tar.gz
[root@slave ~]# mv jdk1.7.0_79 /usr/local/
[root@slave ~]# source /etc/profile.d/java.sh
[root@slave ~]# java -version
java version “1.7.0_79”
Java(TM) SE Runtime Environment (build 1.7.0_79-b15)
Java HotSpot(TM) 64-Bit Server VM (build 24.79-b02, mixed mode)
Hadoop
master
[root@master ~]# wget https://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-2.7.2/hadoop-2.7.2.tar.gz
[root@master ~]# tar zxvf hadoop-2.7.2.tar.gz
[root@master ~]# mv hadoop-2.7.2 /usr/local/
[root@master ~]# ls /usr/local/
bin games include lib libexec share etc hadoop-2.7.2 jdk1.7.0_79 lib64 sbin src
[root@master ~]# ls /usr/local/hadoop-2.7.2/
bin include libexec NOTICE.txt sbin etc lib LICENSE.txt README.txt share
[root@master ~]# mkdir /usr/local/hadoop-2.7.2/tmp /usr/local/hadoop-2.7.2/dfs /usr/local/hadoop-2.7.2/dfs/data /usr/local/hadoop-2.7.2/dfs/name
/usr/local/hadoop-2.7.2/tmp
/usr/local/hadoop-2.7.2/dfs?
[root@master ~]# ls /usr/local/hadoop-2.7.2/
bin etc lib LICENSE.txt README.txt share dfs include libexec NOTICE.txt sbin tmp
[root@master ~]# rsync -av /usr/local/hadoop-2.7.2 slave:/usr/local
slave
[root@slave ~]# ls /usr/local/hadoop-2.7.2
bin etc lib LICENSE.txt README.txt share dfs include libexec NOTICE.txt sbin tmp
Hadoop
master
[root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/core-site.xml
fs.defaultFS
hdfs://master:9000
hadoop.tmp.dir
file:/usr/local/hadoop-2.7.2/tmp
io.file.buffer.size
131072
fs.defaultFS
NameNode ?HDFS MapReduce core-site.xml hdfs-site.xml
[root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/hdfs-site.xml
dfs.namenode.name.dir
file:/usr/local/hadoop-2.7.2/dfs/name
dfs.datanode.data.dir
file:/usr/local/hadoop-2.7.2/dfs/data
dfs.replication
1
dfs.namenode.secondary.http-address
master:9001
dfs.webhdfs.enabled
true
[root@master ~]# mv /usr/local/hadoop-2.7.2/etc/hadoop/mapred-site.xml.template /usr/local/hadoop-2.7.2/etc/hadoop/mapred-site.xml
[root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/mapred-site.xml
mapreduce.framework.name
yarn
mapreduce.jobhistory.address
master:10020
mapreduce.jobhistory.webapp.address
master:19888
[root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/yarn-site.xml
yarn.nodemanager.aux-services
mapreduce_shuffle
yarn.nodemanager.auxservices.mapreduce.shuffle.class
org.apache.hadoop.mapred.ShuffleHandler
yarn.resourcemanager.address
master:8032
yarn.resourcemanager.scheduler.address
master:8030
yarn.resourcemanager.resource-tracker.address
master:8031
yarn.resourcemanager.admin.address
master:8033
yarn.resourcemanager.webapp.address
master:8088
[root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/hadoop-env.sh
export JAVA_HOME=${JAVA_HOME}
export JAVA_HOME=/usr/local/jdk1.7.0_79
[root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/yarn-env.sh
export JAVA_HOME=/usr/local/jdk1.7.0_79
root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/mapred-env.sh
# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
export JAVA_HOME=/usr/local/jdk1.7.0_79
[root@master ~]# vi /usr/local/hadoop-2.7.2/etc/hadoop/slaves
localhost
slave
[root@master ~]# rsync -av /usr/local/hadoop-2.7.2/etc/ slave:/usr/local/hadoop-2.7.2/etc/
[root@master ~]# vi /etc/profile.d/hadoop.sh
export HADOOP_HOME=/usr/local/hadoop-2.7.2
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
[root@master ~]# source /etc/profile.d/hadoop.sh
[root@master ~]# hadoop version
Hadoop 2.7.2
Subversion https://git-wip-us.apache.org/repos/asf/hadoop.git -r b165c4fe8a74265c792ce23f546c64604acf0e41
Compiled by jenkins on 2016-01-26T00:08Z
Compiled with protoc 2.5.0
From source with checksum d0fda26633fa762bff87ec759ebe689c
This command was run using /usr/local/hadoop-2.7.2/share/hadoop/common/hadoop-common-2.7.2.jar
[root@master ~]# scp /etc/profile.d/hadoop.sh slave:/etc/profile.d/
slave
hadoop
[root@slave ~]# source /etc/profile.d/hadoop.sh
[root@slave ~]# hadoop version
Hadoop 2.7.2
Subversion https://git-wip-us.apache.org/repos/asf/hadoop.git -r b165c4fe8a74265c792ce23f546c64604acf0e41
Compiled by jenkins on 2016-01-26T00:08Z
Compiled with protoc 2.5.0
From source with checksum d0fda26633fa762bff87ec759ebe689c
This command was run using /usr/local/hadoop-2.7.2/share/hadoop/common/hadoop-common-2.7.2.jar
Hadoop
master
[root@master ~]# /usr/local/hadoop-2.7.2/bin/hdfs namenode -format
[root@master ~]# echo $?
0
root@master ~]# /usr/local/hadoop-2.7.2/sbin/start-all.sh
[root@master ~]# jps
5560 ResourceManager
5239 NameNode
5631 Jps
5415 SecondaryNameNode
slave
[root@slave ~]# jps
5231 DataNode
5444 Jps
5320 NodeManager
master
web UI
master:50070 ?namenode datanode?
master:8088 ?Yarn?
[root@master ~]# cd /usr/local/hadoop-2.7.2/
[root@master hadoop-2.7.2]# bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.2.jar pi 1
[root@master ~]# /usr/local/hadoop-2.7.2/sbin/stop-all.sh
127.0.0.1 localhost
192.168.229.130 master
192.168.229.131 slave
copyFromLocal: Cannot create directory /123/. Name node is in safe mode
[root@master ~]# /usr/local/hadoop-2.7.2/bin/hdfs dfsadmin -safemode leave
Recent Comments