{"id":7048,"date":"2017-10-10T16:49:46","date_gmt":"2017-10-10T08:49:46","guid":{"rendered":"http:\/\/rmohan.com\/?p=7048"},"modified":"2017-10-10T16:49:46","modified_gmt":"2017-10-10T08:49:46","slug":"centos-6-9-hadoop-2-7-2","status":"publish","type":"post","link":"https:\/\/mohan.sg\/?p=7048","title":{"rendered":"CentOS 6.9 Hadoop 2.7.2"},"content":{"rendered":"

CentOS 6.9 Hadoop 2.7.2<\/strong><\/p>\n

master\t192.168.1.130
\nslave\t192.168.1.131<\/p>\n

master<\/p>\n

[root@localhost ~]# vi \/etc\/hosts<\/p>\n

127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
\n::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
\n127.0.0.1 localhost
\n192.168.1.130 master
\n192.168.1.131 slave<\/p>\n

[root@localhost ~]# vi \/etc\/sysconfig\/network<\/p>\n

NETWORKING=yes
\nHOSTNAME=localhost.localdomain
\nNETWORKING=yes
\nHOSTNAME=master<\/p>\n

slave<\/p>\n

[root@localhost ~]# vi \/etc\/hosts<\/p>\n

127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
\n::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
\n127.0.0.1 localhost
\n192.168.1.130 master
\n192.168.1.131 slave<\/p>\n

[root@localhost ~]# vi \/etc\/sysconfig\/network
\nNETWORKING=yes
\nHOSTNAME=localhost.localdomain
\nNETWORKING=yes
\nHOSTNAME=slave<\/p>\n

hostname your-hostname<\/p>\n

selinux<\/p>\n

master<\/p>\n

[root@master ~]# vim \/etc\/selinux\/config
\nSELINUX=enforcing<\/p>\n

SELINUX=disabled<\/p>\n

firewall<\/p>\n

CentOS iptables <\/p>\n

master<\/p>\n

[root@master ~]# iptables -F; \/etc\/init.d\/iptables save
\niptables: Saving firewall rules to \/etc\/sysconfig\/iptables:[ OK ]
\niptables -nvL<\/p>\n

master<\/p>\n

[root@master ~]# ssh-keygen<\/p>\n

master<\/p>\n

[root@master ~]# cat ~\/.ssh\/id_rsa.pub >> ~\/.ssh\/authorized_keys
\n[root@master ~]# scp ~\/.ssh\/authorized_keys slave:~\/.ssh\/
\nslave<\/p>\n

[root@slave ~]# ls .ssh\/
\nauthorized_keys
\nmaster<\/p>\n

[root@master ~]# ssh slave
\n[root@slave ~]# exit
\n[root@master ~]#<\/p>\n

JDK<\/p>\n

# java -version<\/p>\n

[root@master ~]# wget http:\/\/download.Oracle.com\/otn-pub\/java\/jdk\/7u79-b15\/jdk-7u79-linux-x64.tar.gz?AuthParam=1480051498_4f2fdb0325a457f4c7d33a69355b3560<\/p>\n

[root@master ~]# mv jdk-7u79-linux-x64.tar.gz\\?AuthParam\\=1480051498_4f2fdb0325a457f4c7d33a69355b3560 jdk-7u79-linux-x64.tar.gz
\n[root@master ~]# tar zxvf jdk-7u79-linux-x64.tar.gz
\n[root@master ~]# mv jdk1.7.0_79 \/usr\/local\/<\/p>\n

[root@master ~]# vi \/etc\/profile.d\/java.sh<\/p>\n

export JAVA_HOME=\/usr\/local\/jdk1.7.0_79
\nexport PATH=$PATH:$JAVA_HOME\/bin
\nexport CLASSPATH=.:$JAVA_HOME\/jre\/lib\/rt.jar:$JAVA_HOME\/lib\/dt.jar:$JAVA_HOME\/lib\/tools.jar
\n[root@master ~]# source \/etc\/profile.d\/java.sh
\n[root@master ~]# java -version
\njava version “1.7.0_79”
\nJava(TM) SE Runtime Environment (build 1.7.0_79-b15)
\nJava HotSpot(TM) 64-Bit Server VM (build 24.79-b02, mixed mode)<\/p>\n

[root@master ~]# scp jdk-7u79-linux-x64.tar.gz slave:\/root\/
\n[root@master ~]# scp \/etc\/profile.d\/java.sh slave:\/etc\/profile.d\/
\nslave<\/p>\n

[root@slave ~]# tar zxvf jdk-7u79-linux-x64.tar.gz
\n[root@slave ~]# mv jdk1.7.0_79 \/usr\/local\/<\/p>\n

[root@slave ~]# source \/etc\/profile.d\/java.sh
\n[root@slave ~]# java -version
\njava version “1.7.0_79”
\nJava(TM) SE Runtime Environment (build 1.7.0_79-b15)
\nJava HotSpot(TM) 64-Bit Server VM (build 24.79-b02, mixed mode)<\/p>\n

Hadoop<\/p>\n

master<\/p>\n

[root@master ~]# wget https:\/\/mirrors.tuna.tsinghua.edu.cn\/apache\/hadoop\/common\/hadoop-2.7.2\/hadoop-2.7.2.tar.gz
\n[root@master ~]# tar zxvf hadoop-2.7.2.tar.gz
\n[root@master ~]# mv hadoop-2.7.2 \/usr\/local\/
\n[root@master ~]# ls \/usr\/local\/
\nbin games include lib libexec share etc hadoop-2.7.2 jdk1.7.0_79 lib64 sbin src
\n[root@master ~]# ls \/usr\/local\/hadoop-2.7.2\/
\nbin include libexec NOTICE.txt sbin etc lib LICENSE.txt README.txt share
\n[root@master ~]# mkdir \/usr\/local\/hadoop-2.7.2\/tmp \/usr\/local\/hadoop-2.7.2\/dfs \/usr\/local\/hadoop-2.7.2\/dfs\/data \/usr\/local\/hadoop-2.7.2\/dfs\/name
\n\/usr\/local\/hadoop-2.7.2\/tmp
\n\/usr\/local\/hadoop-2.7.2\/dfs?<\/p>\n

[root@master ~]# ls \/usr\/local\/hadoop-2.7.2\/
\nbin etc lib LICENSE.txt README.txt share dfs include libexec NOTICE.txt sbin tmp
\n[root@master ~]# rsync -av \/usr\/local\/hadoop-2.7.2 slave:\/usr\/local
\nslave<\/p>\n

[root@slave ~]# ls \/usr\/local\/hadoop-2.7.2
\nbin etc lib LICENSE.txt README.txt share dfs include libexec NOTICE.txt sbin tmp<\/p>\n

Hadoop<\/p>\n

master<\/p>\n

[root@master ~]# vi \/usr\/local\/hadoop-2.7.2\/etc\/hadoop\/core-site.xml
\n\n \n fs.defaultFS<\/name>
\n hdfs:\/\/master:9000<\/value>\n <\/property>\n \n hadoop.tmp.dir<\/name>
\n file:\/usr\/local\/hadoop-2.7.2\/tmp<\/value>\n <\/property>\n \n io.file.buffer.size<\/name>
\n 131072<\/value>\n <\/property>\n<\/configuration><\/p>\n

fs.defaultFS
\nNameNode ?HDFS MapReduce core-site.xml hdfs-site.xml<\/p>\n

[root@master ~]# vi \/usr\/local\/hadoop-2.7.2\/etc\/hadoop\/hdfs-site.xml<\/p>\n

\n \n dfs.namenode.name.dir<\/name>
\n file:\/usr\/local\/hadoop-2.7.2\/dfs\/name<\/value>\n <\/property>\n \n dfs.datanode.data.dir<\/name>
\n file:\/usr\/local\/hadoop-2.7.2\/dfs\/data<\/value>\n <\/property>\n \n dfs.replication<\/name>
\n 1<\/value>\n <\/property>\n \n dfs.namenode.secondary.http-address<\/name>
\n master:9001<\/value>\n <\/property>\n \n dfs.webhdfs.enabled<\/name>
\n true<\/value>\n <\/property>\n<\/configuration><\/p>\n

[root@master ~]# mv \/usr\/local\/hadoop-2.7.2\/etc\/hadoop\/mapred-site.xml.template \/usr\/local\/hadoop-2.7.2\/etc\/hadoop\/mapred-site.xml
\n[root@master ~]# vi \/usr\/local\/hadoop-2.7.2\/etc\/hadoop\/mapred-site.xml<\/p>\n

\n \n mapreduce.framework.name<\/name>
\n yarn<\/value>\n <\/property>\n \n mapreduce.jobhistory.address<\/name>
\n master:10020<\/value>\n <\/property>\n \n mapreduce.jobhistory.webapp.address<\/name>
\n master:19888<\/value>\n <\/property>\n<\/configuration><\/p>\n

[root@master ~]# vi \/usr\/local\/hadoop-2.7.2\/etc\/hadoop\/yarn-site.xml<\/p>\n

<\/p>\n

\n \n yarn.nodemanager.aux-services<\/name>
\n mapreduce_shuffle<\/value>\n <\/property>\n \n yarn.nodemanager.auxservices.mapreduce.shuffle.class<\/name>
\n org.apache.hadoop.mapred.ShuffleHandler<\/value>\n <\/property>\n \n yarn.resourcemanager.address<\/name>
\n master:8032<\/value>\n <\/property>\n \n yarn.resourcemanager.scheduler.address<\/name>
\n master:8030<\/value>\n <\/property>\n \n yarn.resourcemanager.resource-tracker.address<\/name>
\n master:8031<\/value>\n <\/property>\n \n yarn.resourcemanager.admin.address<\/name>
\n master:8033<\/value>\n <\/property>\n \n yarn.resourcemanager.webapp.address<\/name>
\n master:8088<\/value>\n <\/property>\n

<\/configuration><\/p>\n

[root@master ~]# vi \/usr\/local\/hadoop-2.7.2\/etc\/hadoop\/hadoop-env.sh
\nexport JAVA_HOME=${JAVA_HOME}
\nexport JAVA_HOME=\/usr\/local\/jdk1.7.0_79<\/p>\n

[root@master ~]# vi \/usr\/local\/hadoop-2.7.2\/etc\/hadoop\/yarn-env.sh<\/p>\n

export JAVA_HOME=\/usr\/local\/jdk1.7.0_79<\/p>\n

root@master ~]# vi \/usr\/local\/hadoop-2.7.2\/etc\/hadoop\/mapred-env.sh
\n# export JAVA_HOME=\/home\/y\/libexec\/jdk1.6.0\/
\nexport JAVA_HOME=\/usr\/local\/jdk1.7.0_79<\/p>\n

[root@master ~]# vi \/usr\/local\/hadoop-2.7.2\/etc\/hadoop\/slaves
\nlocalhost
\nslave<\/p>\n

[root@master ~]# rsync -av \/usr\/local\/hadoop-2.7.2\/etc\/ slave:\/usr\/local\/hadoop-2.7.2\/etc\/<\/p>\n

[root@master ~]# vi \/etc\/profile.d\/hadoop.sh<\/p>\n

export HADOOP_HOME=\/usr\/local\/hadoop-2.7.2
\nexport PATH=$HADOOP_HOME\/bin:$HADOOP_HOME\/sbin:$PATH<\/p>\n

[root@master ~]# source \/etc\/profile.d\/hadoop.sh
\n[root@master ~]# hadoop version
\nHadoop 2.7.2
\nSubversion https:\/\/git-wip-us.apache.org\/repos\/asf\/hadoop.git -r b165c4fe8a74265c792ce23f546c64604acf0e41
\nCompiled by jenkins on 2016-01-26T00:08Z
\nCompiled with protoc 2.5.0
\nFrom source with checksum d0fda26633fa762bff87ec759ebe689c
\nThis command was run using \/usr\/local\/hadoop-2.7.2\/share\/hadoop\/common\/hadoop-common-2.7.2.jar<\/p>\n

[root@master ~]# scp \/etc\/profile.d\/hadoop.sh slave:\/etc\/profile.d\/
\nslave<\/p>\n

hadoop<\/p>\n

[root@slave ~]# source \/etc\/profile.d\/hadoop.sh
\n[root@slave ~]# hadoop version
\nHadoop 2.7.2
\nSubversion https:\/\/git-wip-us.apache.org\/repos\/asf\/hadoop.git -r b165c4fe8a74265c792ce23f546c64604acf0e41
\nCompiled by jenkins on 2016-01-26T00:08Z
\nCompiled with protoc 2.5.0
\nFrom source with checksum d0fda26633fa762bff87ec759ebe689c
\nThis command was run using \/usr\/local\/hadoop-2.7.2\/share\/hadoop\/common\/hadoop-common-2.7.2.jar<\/p>\n

Hadoop<\/p>\n

master<\/p>\n

[root@master ~]# \/usr\/local\/hadoop-2.7.2\/bin\/hdfs namenode -format
\n[root@master ~]# echo $?
\n0<\/p>\n

root@master ~]# \/usr\/local\/hadoop-2.7.2\/sbin\/start-all.sh
\n[root@master ~]# jps
\n5560 ResourceManager
\n5239 NameNode
\n5631 Jps
\n5415 SecondaryNameNode
\nslave<\/p>\n

[root@slave ~]# jps
\n5231 DataNode
\n5444 Jps
\n5320 NodeManager
\nmaster<\/p>\n

web UI
\n master:50070 ?namenode datanode?
\n master:8088 ?Yarn?<\/p>\n

[root@master ~]# cd \/usr\/local\/hadoop-2.7.2\/
\n[root@master hadoop-2.7.2]# bin\/hadoop jar .\/share\/hadoop\/mapreduce\/hadoop-mapreduce-examples-2.7.2.jar pi 1<\/p>\n

[root@master ~]# \/usr\/local\/hadoop-2.7.2\/sbin\/stop-all.sh<\/p>\n

127.0.0.1 localhost
\n192.168.229.130 master
\n192.168.229.131 slave<\/p>\n

copyFromLocal: Cannot create directory \/123\/. Name node is in safe mode<\/p>\n

[root@master ~]# \/usr\/local\/hadoop-2.7.2\/bin\/hdfs dfsadmin -safemode leave<\/p>\n","protected":false},"excerpt":{"rendered":"

CentOS 6.9 Hadoop 2.7.2<\/p>\n

master 192.168.1.130 slave 192.168.1.131<\/p>\n

master<\/p>\n

[root@localhost ~]# vi \/etc\/hosts<\/p>\n

127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 127.0.0.1 localhost 192.168.1.130 master 192.168.1.131 slave<\/p>\n

[root@localhost ~]# vi \/etc\/sysconfig\/network<\/p>\n

NETWORKING=yes HOSTNAME=localhost.localdomain NETWORKING=yes HOSTNAME=master<\/p>\n

slave<\/p>\n

[root@localhost ~]# vi \/etc\/hosts<\/p>\n

127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 127.0.0.1 localhost 192.168.1.130 […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[5],"tags":[],"_links":{"self":[{"href":"https:\/\/mohan.sg\/index.php?rest_route=\/wp\/v2\/posts\/7048"}],"collection":[{"href":"https:\/\/mohan.sg\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/mohan.sg\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/mohan.sg\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/mohan.sg\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=7048"}],"version-history":[{"count":1,"href":"https:\/\/mohan.sg\/index.php?rest_route=\/wp\/v2\/posts\/7048\/revisions"}],"predecessor-version":[{"id":7049,"href":"https:\/\/mohan.sg\/index.php?rest_route=\/wp\/v2\/posts\/7048\/revisions\/7049"}],"wp:attachment":[{"href":"https:\/\/mohan.sg\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=7048"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/mohan.sg\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=7048"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/mohan.sg\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=7048"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}