Hadoop分布式集群部署

环境准备

IPHOSTNAMESYSTEM
192.168.131.129 hadoop-master CentOS 7.6
192.168.131.135 hadoop-slave1 CentOS 7.6
192.168.131.137 hadoop-slave2 CentOS 7.6 
[root@localhost ~]# cat /etc/redhat-release
CentOS Linux release 7.6.1810 (Core)
[root@localhost ~]# sestatus
SELinux status:                 disabled
[root@localhost ~]# systemctl status firewalld.service
● firewalld.service - firewalld - dynamic firewall daemon
   Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)
   Active: inactive (dead)
     Docs: man:firewalld(1)
[root@localhost ~]# hostnamectl set-hostname --static hadoop-master
[root@localhost ~]# hostnamectl set-hostname --static hadoop-slave1
[root@localhost ~]# hostnamectl set-hostname --static hadoop-slave2

配置免密

[root@hadoop-master ~]# cat >> /etc/hosts <<EOF
192.168.131.129 hadoop-master
192.168.131.135 hadoop-slave1
192.168.131.137 hadoop-slave2
EOF
[root@hadoop-master ~]# ssh-keygen
[root@hadoop-master ~]# ssh-copy-id hadoop-slave1
[root@hadoop-master ~]# ssh-copy-id hadoop-slave2
[root@hadoop-master ~]# scp /etc/hosts hadoop-slave1:/etc/hosts
hosts                                                              100%  248   151.9KB/s   00:00
[root@hadoop-master ~]# scp /etc/hosts hadoop-slave2:/etc/hosts
hosts                                                              100%  248   220.9KB/s   00:00

配置java环境

[root@hadoop-master ~]# tar xf jdk-8u211-linux-x64.tar.gz -C /usr/local/
[root@hadoop-master ~]# ln -s /usr/local/jdk1.8.0_211/ /usr/local/java
[root@hadoop-master ~]# ll /usr/local/
total 0
drwxr-xr-x. 2 root root   6 Apr 11  2018 bin
drwxr-xr-x. 2 root root   6 Apr 11  2018 etc
drwxr-xr-x. 2 root root   6 Apr 11  2018 games
drwxr-xr-x. 2 root root   6 Apr 11  2018 include
lrwxrwxrwx  1 root root  24 Sep  7 14:49 java -> /usr/local/jdk1.8.0_211/
drwxr-xr-x  7   10  143 245 Apr  2  2019 jdk1.8.0_211
drwxr-xr-x. 2 root root   6 Apr 11  2018 lib
drwxr-xr-x. 2 root root   6 Apr 11  2018 lib64
drwxr-xr-x. 2 root root   6 Apr 11  2018 libexec
drwxr-xr-x. 2 root root   6 Apr 11  2018 sbin
drwxr-xr-x. 5 root root  49 Mar 30  2019 share
drwxr-xr-x. 2 root root   6 Apr 11  2018 src
[root@hadoop-master ~]# vim /etc/profile
export JAVA_HOME=/usr/local/java/
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
[root@hadoop-master ~]# source /etc/profile
[root@hadoop-master ~]# java -version
java version "1.8.0_211"
Java(TM) SE Runtime Environment (build 1.8.0_211-b12)
Java HotSpot(TM) 64-Bit Server VM (build 25.211-b12, mixed mode)

部署hadoop

[root@hadoop-master ~]# wget https://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-2.7.7/hadoop-2.7.7.tar.gz
[root@hadoop-master ~]# tar xf hadoop-2.7.7.tar.gz -C /opt/
[root@hadoop-master ~]# vim /etc/profile
export HADOOP_HOME=/opt/hadoop-2.7.7
export PATH=$HADOOP_HOME/bin:$PATH
[root@hadoop-master ~]# source /etc/profile
[root@hadoop-master ~]# cd /opt/hadoop-2.7.7/
[root@hadoop-master hadoop-2.7.7]# ls
bin  etc  include  lib  libexec  LICENSE.txt  NOTICE.txt  README.txt  sbin  share
[root@hadoop-master hadoop-2.7.7]# cd etc/hadoop/
[root@hadoop-master hadoop]# ll
total 152
-rw-r--r-- 1 1000 ftp  4436 Jul 19  2018 capacity-scheduler.xml
-rw-r--r-- 1 1000 ftp  1335 Jul 19  2018 configuration.xsl
-rw-r--r-- 1 1000 ftp   318 Jul 19  2018 container-executor.cfg
-rw-r--r-- 1 1000 ftp   774 Jul 19  2018 core-site.xml
-rw-r--r-- 1 1000 ftp  3670 Jul 19  2018 hadoop-env.cmd
-rw-r--r-- 1 1000 ftp  4224 Jul 19  2018 hadoop-env.sh
-rw-r--r-- 1 1000 ftp  2598 Jul 19  2018 hadoop-metrics2.properties
-rw-r--r-- 1 1000 ftp  2490 Jul 19  2018 hadoop-metrics.properties
-rw-r--r-- 1 1000 ftp  9683 Jul 19  2018 hadoop-policy.xml
-rw-r--r-- 1 1000 ftp   775 Jul 19  2018 hdfs-site.xml
-rw-r--r-- 1 1000 ftp  1449 Jul 19  2018 httpfs-env.sh
-rw-r--r-- 1 1000 ftp  1657 Jul 19  2018 httpfs-log4j.properties
-rw-r--r-- 1 1000 ftp    21 Jul 19  2018 httpfs-signature.secret
-rw-r--r-- 1 1000 ftp   620 Jul 19  2018 httpfs-site.xml
-rw-r--r-- 1 1000 ftp  3518 Jul 19  2018 kms-acls.xml
-rw-r--r-- 1 1000 ftp  1527 Jul 19  2018 kms-env.sh
-rw-r--r-- 1 1000 ftp  1631 Jul 19  2018 kms-log4j.properties
-rw-r--r-- 1 1000 ftp  5540 Jul 19  2018 kms-site.xml
-rw-r--r-- 1 1000 ftp 11801 Jul 19  2018 log4j.properties
-rw-r--r-- 1 1000 ftp   951 Jul 19  2018 mapred-env.cmd
-rw-r--r-- 1 1000 ftp  1383 Jul 19  2018 mapred-env.sh
-rw-r--r-- 1 1000 ftp  4113 Jul 19  2018 mapred-queues.xml.template
-rw-r--r-- 1 1000 ftp   758 Jul 19  2018 mapred-site.xml.template
-rw-r--r-- 1 1000 ftp    10 Jul 19  2018 slaves
-rw-r--r-- 1 1000 ftp  2316 Jul 19  2018 ssl-client.xml.example
-rw-r--r-- 1 1000 ftp  2697 Jul 19  2018 ssl-server.xml.example
-rw-r--r-- 1 1000 ftp  2250 Jul 19  2018 yarn-env.cmd
-rw-r--r-- 1 1000 ftp  4567 Jul 19  2018 yarn-env.sh
-rw-r--r-- 1 1000 ftp   690 Jul 19  2018 yarn-site.xml

修改hadoop-env.sh和yarn-env.sh中JAVA_HOME参数

[root@hadoop-master hadoop]# sed -i 's#export JAVA_HOME=.*#export JAVA_HOME='/usr/local/java'#g' hadoop-env.sh
[root@hadoop-master hadoop]# sed -i 's#.*export JAVA_HOME=.*#export JAVA_HOME='/usr/local/java'#g' yarn-env.sh
[root@hadoop-master hadoop]# grep "export JAVA_HOME" hadoop-env.sh
export JAVA_HOME=/usr/local/java
[root@hadoop-master hadoop]# grep "export JAVA_HOME" yarn-env.sh
export JAVA_HOME=/usr/local/java

编辑core-site.xml

[root@hadoop-master hadoop]# vim core-site.xml
<configuration>
   <property>
      <name>fs.default.name</name>
      <value>hdfs://192.168.131.129:9000</value>
   </property>

   <property>
      <name>hadoop.tmp.dir</name>
      <value>/opt/hadoop/tmp</value>
   </property>
</configuration>

编辑hdfs-site.xml

[root@hadoop-master hadoop]# vim hdfs-site.xml
<configuration>
   <property>
      <name>dfs.replication</name>
      <value>2</value>
   </property>

   <property>
      <name>dfs.namenode.name.dir</name>
      <value>file:/opt/hadoop/dfs/name</value>
   </property>

   <property>
      <name>dfs.datanode.data.dir</name>
      <value>file:/opt/hadoop/dfs/data</value>
   </property>
</configuration>

编辑mapred-site.xml

[root@hadoop-master hadoop]# cp mapred-site.xml.template mapred-site.xml
[root@hadoop-master hadoop]# vim mapred-site.xml
<configuration>
   <property>
      <name>mapreduce.framework.name</name>
      <value>yarn</value>
   </property>

   <property>
      <name>mapreduce.jobhistory.address</name>
      <value>192.168.131.129:10020</value>
   </property>

   <property>
      <name>mapreduce.jobhistory.webapp.address</name>
      <value>192.168.131.129:19888</value>
   </property>
</configuration>

编辑yarn-site.xml

[root@hadoop-master hadoop]# vim yarn-site.xml
<configuration>

<!-- Site specific YARN configuration properties -->
   <property>
      <name>yarn.nodemanager.aux-services</name>
      <value>mapreduce_shuffle</value>
   </property>
   <property>
      <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
      <value>org.apache.hadoop.mapred.ShuffleHandler</value>
    </property>

   <property>
      <name>yarn.resourcemanager.address</name>
      <value>192.168.131.129:8032</value>
   </property>

   <property>
      <name>yarn.resourcemanager.scheduler.address</name>
      <value>192.168.131.129:8030</value>
   </property>

   <property>
      <name>yarn.resourcemanager.resource-tracker.address</name>
      <value>192.168.131.129:8031</value>
   </property>

   <property>
      <name>yarn.resourcemanager.admin.address</name>
      <value>192.168.131.129:8033</value>
   </property>

   <property>
      <name>yarn.resourcemanager.webapp.address</name>
      <value>192.168.131.129:8088</value>
   </property>
</configuration>

编辑slaves

[root@hadoop-master hadoop]# vim slaves
hadoop-slave1
hadoop-slave2

把配置分发到各个节点

[root@hadoop-master opt]# scp -r hadoop-2.7.7/ hadoop-slave1:/opt/
[root@hadoop-master opt]# scp -r hadoop-2.7.7/ hadoop-slave2:/opt/

在主节点启动hadoop

初始化namenode
[root@hadoop-master ~]# hdfs namenode -format
启动HDFS
[root@hadoop-master ~]# cd /opt/hadoop-2.7.7/sbin/
[root@hadoop-master sbin]# ./start-dfs.sh
启动YARN
[root@hadoop-master sbin]# ./start-yarn.sh

测试

浏览器访问
YARN WEB地址：http://192.168.131.129:8088/
HDFS WEB地址：http://192.168.131.129:50070/
posted @ 2020-09-07 16:24  月巴左耳东  阅读(109)  评论(0)    收藏  举报