镜世界hadoop（自用）

步骤：

1、创建用户

sudo useradd -m hadoop -s /bin/bash   

sudo passwd hadoop                      

sudo adduser hadoop sudo              

 

 

2、注销当前用户并使用hadoop用户登陆

sudo su - hadoop

 

3、更新资源库

sudo apt-get update                               

 

4、安装vim

sudo apt-get install vim

 

 

 

 

5、安装ssh

 

cd ~

mkdir .ssh

cd ~/.ssh/

ssh-keygen -t rsa

cat id_rsa.pub>>authorized_keys                       

 

 

6、安装Java环境

sudo apt-get install openjdk-8-jre openjdk-8-jdk

 

 

cd

vim ~/.bashrc               

 

 

7、在文件第一行添加

export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64

export HADOOP_OPTS="-Djava.library.path=${HADOOP_HOME}/lib/native"

export JAVA_LIBRARY_PATH=/usr/local/hadoop/lib/native

 

 

8、使变量生效

source ~/.bashrc

 

 

9、安装hadoop 2

cd /usr/local

 

 

 

 

sudo tar -zxvf ./hadoop-2.10.1.tar.gz                           

 

 

sudo mv ./hadoop-2.10.1/ ./hadoop                             

sudo chown -R hadoop ./hadoop                               

 

 

10、修改配置文件

vim /usr/local/hadoop/etc/hadoop/core-site.xml

修改部分

<configuration>

    <property>

        <name>hadoop.tmp.dir</name>

        <value>file:/usr/local/hadoop/tmp</value>

    </property>

    <property>

        <name>fs.defaultFS</name>

        <value>hdfs://localhost:9000</value>

    </property>

</configuration>

 

配置

vim /usr/local/hadoop/etc/hadoop/hdfs-site.xml

<configuration>

    <property>

        <name>dfs.replication</name>

        <value>1</value>

    </property>

    <property>

        <name>dfs.namenode.name.dir</name>

        <value>file:/usr/local/hadoop/tmp/dfs/name</value>

    </property>

    <property>

        <name>dfs.datanode.data.dir</name>

        <value>file:/usr/local/hadoop/tmp/dfs/data</value>

    </property>

</configuration>

 

 

启动

cd /usr/local/hadoop                                             

bin/hdfs namenode -format       

 

 

sbin/start-dfs.sh               

 

 

 

jps

 

 

 

 

WORDCOUNT实例：

 bin/hdfs dfs -mkdir -p /user/hadoop

 bin/hdfs dfs -mkdir input

 bin/hdfs dfs -put etc/hadoop/*.xml input

 

bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.10.1.jar wordcount input output

 

bin/hdfs dfs -cat output/*