第一周进度总结

（1）本周做了什么，花在学习上多长时间，花在代码时间上多长时间，花在解决问题用了多长时间。

本周我完成了hadoop集群的配置。以下是部分配置用到的命令。

vim /etc/sysconfig/network-scripts/ifcfg-ens33

 

IPADDR=192.168.88.101

NETMASK=255.255.255.0

GATEWAY=192.168.88.2

DNS1=192.168.88.2

 

IPADDR=192.168.88.102

NETMASK=255.255.255.0

GATEWAY=192.168.88.2

DNS1=192.168.88.2

 

IPADDR=192.168.88.103

NETMASK=255.255.255.0

GATEWAY=192.168.88.2

DNS1=192.168.88.2

 

systemctl restart network

 

vim /etc/hosts

192.168.88.101 node1

192.168.88.102 node2

192.168.88.103 node3

 

ssh-keygen -t rsa -b 4096

ssh-copy-id node1

ssh-copy-id node2

ssh-copy-id node3

 

mkdir -p /export/server

tar -zxvf jdk-8u411-linux-x64.tar.gz -C /export/server/

 

cd /export/server/

ls

ln -s /export/server/jdk1.8.0_411 jdk

ls -l

 

vim /etc/profile

 

export JAVA_HOME=/export/server/jdk

export PATH=$PATH:$JAVA_HOME/bin

 

source /etc/profile

 

cd jdk

ll

cd bin

ll

rm -f /usr/bin/java

ln -s /epxort/server/jdk/bin/java /usr/bin/java

 

 

 

systemctl stop firewalld

systemctl disable firewalld

 

vim /etc/sysconfig/selinux

disabled

 

 

vi /etc/yum.repos.d/CentOS-Base.repo

 

CentOS-Base.repo

 

[base]

name=CentOS-$releasever - Base

baseurl=https://mirrors.aliyun.com/centos/$releasever/os/$basearch/

gpgcheck=1

gpgkey=https://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7

 

#released updates

[updates]

name=CentOS-$releasever - Updates

baseurl=https://mirrors.aliyun.com/centos/$releasever/updates/$basearch/

gpgcheck=1

gpgkey=https://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7

 

#additional packages that may be useful

[extras]

name=CentOS-$releasever - Extras

baseurl=https://mirrors.aliyun.com/centos/$releasever/extras/$basearch/

gpgcheck=1

gpgkey=https://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7

 

#additional packages that extend functionality of existing packages

[centosplus]

name=CentOS-$releasever - Plus

baseurl=https://mirrors.aliyun.com/centos/$releasever/centosplus/$basearch/

gpgcheck=1

enabled=0

gpgkey=https://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7

 

 

yum install -y ntp

 

ntpdate -u ntp.aliyun.com

 

systemctl start ntpd

systemctl enable ntpd

 

tar -zxvf hadoop-3.4.0.tar.gz -C /export/server/

 

cd /export/server/

ll

 

ln -s /export/server/hadoop-3.4.0 hadoop

 

cd hadoop

ll

 

cd etc

ll

 

cd hadoop

ll

 

vim workers

node1

node2

node3

 

vim hadoop-env.sh

export JAVA_HOME=/export/server/jdk

export HADOOP_HOME=/export/server/hadoop

export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop

export HADOOP_LOG_DIR=$HADOOP_HOME/logs

 

vim core-site.xml

<configuration>

  <property>

    <name>fs.defaultFS</name>

    <value>hdfs://node1:8020</value>

  </property>

  <property>

    <name>io.file.buffer.size</name>

    <value>131072</value>

  </property>

</configuration>

 

vim hdfs-site.xml

<configuration>

  <property>

    <name>dfs.datanode.data.dir.perm</name>

    <value>700</value></property>

  <property>

    <name>dfs.namenode.name.dir</name>

    <value>/data/nn</value>

  </property>

  <property>

    <name>dfs.namenode.hosts</name>

    <value>node1,node2,node3</value>

  </property>

  <property>

    <name>dfs.blocksize</name>

    <value>268435456</value>

  </property>

  <property>

    <name>dfs.namenode.handler.count</name>

    <value>100</value>

  </property>

  <property>

    <name>dfs.datanode.data.dir</name>

    <value>/data/dn</value>

  </property>

</configuration>

 

node1

mkdir -p /data/nn

mkdir -p /data/dn

node2 node3

mkdir -p /data/dn

 

scp -r hadoop-3.4.0 node2:`pwd`/

scp -r hadoop-3.4.0 node3:`pwd`/

 

ln -s /export/server/hadoop-3.4.0 hadoop

 

vim /etc/profile

export HADOOP_HOME=/export/server/hadoop

export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

 

chown -R leap:leap /data

chown -R leap:leap /export

 

su - leap

hadoop namenode -format

cd /data/

ll -h

 

start-dfs.sh

 

（2）下周准备做什么？

下周准备完成yarn集群的配置

（3）本周遇到的问题？

在配置ip地址时配置错了。在下载ntp时一直报错，后来修改为aliyun镜像后才解决。使用big data tools连接hdfs时一直超时。