4、hadoop 全分布配置

hadoop 全分布配置

分别有3台主机
10.10.10.128 master
10.10.10.129 slave1
10.10.10.130 slave2

1、配置网络信息（在所有节点上执行)

  vi /etc/sysconfig/network-scripts/ifcfg-网卡名

TYPE=Ethernet
NAME=网卡名
DEVICE=网卡名
BOOTPROTO=static
ONBOOT=yes
IPADDR=10.10.10.128
NETMASK=255.255.255.0
GATEWAY=10.10.10.2
DNS1=114.114.114.114

2、修改主机名, 配置域名解析

 systemctl restart network
 hostnamectl set-hostname 主机名.example.com
 bash

 vi /etc/hosts

10.10.10.128 master 
10.10.10.129 slave1
10.10.10.130 slave2 

3、关闭防火墙与SELinux(在所有节点上执行)

 systemctl disable --now firewalld
 setenforce 0
 vi /etc/selinux/config

 SELINUX=disabled

4、安装hadoop(在master节点上执行)

 tar xf jdk-8u152-linux-x64.tar.gz -C /usr/local/src/
 tar xf hadoop-2.7.1.tar.gz -C /usr/local/src/
 cd /usr/local/src/
 mv jdk1.8.0_152 jdk
 mv hadoop-2.7.1 hadoop
 vi /etc/profile.d/hadoop.sh

 export JAVA_HOME=/usr/local/src/jdk
 export HADOOP_HOME=/usr/local/src/hadoop
 export PATH=${JAVA_HOME}/bin:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:$PATH

# 保存以上配置后执行以下命令
 source /etc/profile.d/hadoop.sh
 echo $PATH
 vi /usr/local/src/hadoop/etc/hadoop/hadoop-env.sh

export JAVA_HOME=/usr/local/src/jdk

5、配置hdfs-site.xml文件参数(在master上执行)

 vi /usr/local/src/hadoop/etc/hadoop/hdfs-site.xml

<configuration>
	<property>
		<name>dfs.namenode.name.dir</name>
		<value>file:/usr/local/src/hadoop/dfs/name</value>
	</property>
	<property>
		<name>dfs.datanode.data.dir</name>
		<value>file:/usr/local/src/hadoop/dfs/data</value>
	</property>
	<property>
		<name>dfs.replication</name>
		<value>2</value>
	</property>
</configuration>
# 保存以上配置后执行以下命令
 mkdir -p /usr/local/src/hadoop/dfs/{name,data}

6、配置core-site.xml文件参数(在master上执行)

 vi /usr/local/src/hadoop/etc/hadoop/core-site.xml

<configuration>
	<property>
		<name>fs.defaultFS</name>
		<value>hdfs://master:9000</value>
	</property>
	<property>
		<name>io.file.buffer.size</name>
		<value>131072</value>
	</property>
	<property>
		<name>hadoop.tmp.dir</name>
		<value>file:/usr/local/src/hadoop/tmp</value>
	</property>
</configuration>
# 保存以上配置后执行以下命令
mkdir -p /usr/local/src/hadoop/tmp

7、配置mapred-site.xml文件参数(在master上执行)

 cd /usr/local/src/hadoop/etc/hadoop
 cp mapred-site.xml.template mapred-site.xml
 vi /usr/local/src/hadoop/etc/hadoop/mapred-site.xml

<configuration>
	<property>
		<name>mapreduce.framework.name</name>
		<value>yarn</value>
	</property>
	<property>
		<name>mapreduce.jobhistory.address</name>
		<value>master:10020</value>
	</property>
	<property>
		<name>mapreduce.jobhistory.webapp.address</name>
		<value>master:19888</value>
	</property>
</configuration>

8、配置yarn-site.xml文件参数(在master上执行)

 vi /usr/local/src/hadoop/etc/hadoop/yarn-site.xml

<configuration>
	<property>
		<name>arn.resourcemanager.address</name>
		<value>master:8032</value>
	</property>
	<property>
		<name>yarn.resourcemanager.scheduler.address</name>
		<value>master:8030</value>
	</property>
	<property>
		<name>yarn.resourcemanager.webapp.address</name>
		<value>master:8088</value>
	</property>
	<property>
		<name>yarn.resourcemanager.resource-tracker.address</name>
		<value>master:8031</value>
	</property>
	<property>
		<name>yarn.resourcemanager.admin.address</name>
		<value>master:8033</value>
	</property>
	<property>
		<name>yarn.nodemanager.aux-services</name>
		<value>mapreduce_shuffle</value>
	</property>
	<property>
		<name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
		<value>org.apache.hadoop.mapred.ShuffleHandler</value>
	</property>
</configuration>

9、hadoop的其它相关配置

# 在master上执行以下命令
 vi /usr/local/src/hadoop/etc/hadoop/masters

10.10.10.128

# 保存后执行以下命令
 vi /usr/local/src/hadoop/etc/hadoop/slaves

10.10.10.129
10.10.10.130

# 保存后执行以下命令
 useradd hadoop
 echo 'hadoop' | passwd --stdin hadoop
 chown -R hadoop.hadoop /usr/local/src
 ll /usr/local/src/

# 配置master能够免密登录所有slave节点
 ssh-keygen -t rsa
 ssh-copy-id root@slave1
 ssh-copy-id root@slave2

# 同步/usr/local/src/目录下所有文件至所有slave节点
 scp -r /usr/local/src/* root@slave1:/usr/local/src/
 scp -r /usr/local/src/* root@slave2:/usr/local/src/
 scp /etc/profile.d/hadoop.sh root@slave1:/etc/profile.d/
 scp /etc/profile.d/hadoop.sh root@slave2:/etc/profile.d/

# 在所有slave节点上执行以下命令
 useradd hadoop
 echo 'hadoop' | passwd --stdin hadoop
 chown -R hadoop.hadoop /usr/local/src
 ll /usr/local/src/
 source /etc/profile.d/hadoop.sh
 echo $PATH