Linux快速安装hadoop集群

准备：

192.168.25.121  hadoop121
192.168.25.122  hadoop122
192.168.25.123  hadoop123
hadoop121 安装namenode、datanode、nodeManager
hadoop122 安装datanode、nodeManager、resourceManager
hadoop123 安装datanode、secondaryNameNode、nodeManager

免密登录
ssh-keygen -t rsa
ssh-copy-id hadoop121
ssh-copy-id hadoop122
ssh-copy-id hadoop123

配置环境变量 
HOME的地方是 java和hadoop的目录
vi /etc/profile
export JAVA_HOME=/soft/jdk/jdk1.8
export PATH=$PATH:$JAVA_HOME/bin

export HADOOP_HOME=/soft/hadoop/hadoop-2.10.0
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin

增加hosts
vi /etc/hosts192.168.25.121 hadoop121
192.168.25.122 hadoop122
192.168.25.123 hadoop123

编写  xsrync

#!/bin/bash

pcount=$#
if((pcount==0));then
echo no args;
exit;
fi

p1=$1
fname=`basename $p1`
echo fname=$fname

pdir=`cd -P $(dirname $p1); pwd`
echo pdir=$pdir

user=`whoami`
# 这个地方要改成对应的ip地址
for((host=121;host<124;host++));do
        echo ----------hodoop$host-----------
        rsync -rvl $pdir/$fname $user@hadoop$host:$pdir
done

更改 hadoop的 core-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://hadoop121:9000</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/soft/hadoop/hadoop-2.10.0/data/tmp</value>
    </property>
</configuration>

更改 hdfs-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>3</value>
    </property>
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>hadoop123:50090</value>
    </property>
</configuration>

更改mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
<!--    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>hadoop120:10020</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>hadoop120:19888</value>
    </property>-->
</configuration>

更改  yarn-site.yml

<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>

<!-- Site specific YARN configuration properties -->
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>hadoop122</value>
    </property>
    <property>
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
    </property>
    <property>
        <name>yarn.log-aggregation.retain-seconds</name>
        <value>604800</value>
    </property>
</configuration>

更改环境变量
在 hadoop/etc/hadoop/下
修改hadoop-env.sh 的JAVA_HOME
修改yarn-env.sh 的JAVA_HOME

编写 slaves 文件
在 hadoop/etc/hadoop/slaves

　　hadoop121
　　hadoop122
　　hadoop123

在namenode上启动集群
在hadoop的sbin下启动
./start-dfs.sh 
在部署resourceManager的机器上启动如下命令
./start-yarn.sh 

 

访问
http://192.168.25.121:50070/