0.Resources资料

1.镜像站

http://www.apache.org/mirrors/ #所有镜像站

http://mirrors.hust.edu.cn/ #华科的镜像站

2.集群管理

A：集群的环境

cat /etc/profile
# /etc/profile: system-wide .profile file for the Bourne shell (sh(1))
# and Bourne compatible shells (bash(1), ksh(1), ash(1), ...).

if [ "`id -u`" -eq 0 ]; then
  PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
else
  PATH="/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games"
fi
export PATH

if [ "${PS1-}" ]; then
  if [ "${BASH-}" ] && [ "$BASH" != "/bin/sh" ]; then
    # The file bash.bashrc already sets the default PS1.
    # PS1='\h:\w\$ '
    if [ -f /etc/bash.bashrc ]; then
      . /etc/bash.bashrc
    fi
  else
    if [ "`id -u`" -eq 0 ]; then
      PS1='# '
    else
      PS1='$ '
    fi
  fi
fi

if [ -d /etc/profile.d ]; then
  for i in /etc/profile.d/*.sh; do
    if [ -r $i ]; then
      . $i
    fi
  done
  unset i
fi

#Set Java Path
export JAVA_HOME=/home/hadoop/opt/jdk1.8.0_181
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

#Set Hadoop Path
export HADOOP_HOME=/home/hadoop/opt/hadoop-2.8.4
export PATH=$PATH:$HADOOP_HOME/sbin 
export PATH=$PATH:$HADOOP_HOME/bin

#set Zookeeper Path
export ZOOKEEPER_HOME=/home/hadoop/opt/zookeeper-3.4.10
export PATH=$PATH:$ZOOKEEPER_HOME/bin

#Set Hbase Path
export HBASE_HOME=/home/hadoop/opt/hbase-1.4.6
export PATH=$PATH:$HBASE_HOME/bin

#Set Hive Path
export HIVE_HOME=/home/hadoop/opt/hive-2.3.3
export PATH=$PATH:$HIVE_HOME/bin

#set Spark Path
export SPARK_HOME=/home/hadoop/opt/spark-2.3.1
export PATH=$PATH:$SPARK_HOME/bin

B：zookeeper的启动和关闭脚本

# zookeeper-start.sh

/home/hadoop/opt/zookeeper-3.4.10/bin/zkServer.sh start
ssh hadoop@slave1 "/home/hadoop/opt/zookeeper-3.4.10/bin/zkServer.sh start"
ssh hadoop@slave2 "/home/hadoop/opt/zookeeper-3.4.10/bin/zkServer.sh start"
ssh hadoop@slave3 "/home/hadoop/opt/zookeeper-3.4.10/bin/zkServer.sh start"

#zookeeper-stop.sh

ssh hadoop@slave1 "/home/hadoop/opt/zookeeper-3.4.10/bin/zkServer.sh stop"
ssh hadoop@slave2 "/home/hadoop/opt/zookeeper-3.4.10/bin/zkServer.sh stop"
ssh hadoop@slave3 "/home/hadoop/opt/zookeeper-3.4.10/bin/zkServer.sh stop"
/home/hadoop/opt/zookeeper-3.4.10/bin/zkServer.sh stop

C：整体的启动关闭顺序

备注：在使用hase时，执行hbase的start和stop指令，会自动执行zookeeper的相关start和stop指令，因此可以注释掉

#启动bigdata-start.sh

#!/bin/bash
source /etc/profile
echo ${JAVA_HOME}
echo ${HADOOP_HOME}
echo ${ZOOKEEPER_HOME}
echo ${HBASE_HOME}
echo ${HIVE_HOME}
echo ${SPARK_HOME}

echo "------首先启动hadoop集群------"
${HADOOP_HOME}/sbin/start-all.sh
echo "------然后启动Zookeeper------"
#${ZOOKEEPER_HOME}/bin/zookeeper-start.sh
echo "------然后启动Hbase------"
${HBASE_HOME}/bin/start-hbase.sh
echo "------然后启动Spark-History------"
${SPARK_HOME}/sbin/start-history-server.sh

 

#关闭bigdata-stop.sh

#!/bin/bash
source /etc/profile
echo ${JAVA_HOME}
echo ${HADOOP_HOME}
echo ${ZOOKEEPER_HOME}
echo ${HBASE_HOME}
echo ${HIVE_HOME}
echo ${SPARK_HOME}

echo "------关闭Spark-History------"
${SPARK_HOME}/sbin/stop-history-server.sh
echo "------然后关闭Hbase------"
${HBASE_HOME}/bin/stop-hbase.sh 
echo "------然后关闭Zookeeper------"             
#${ZOOKEEPER_HOME}/bin/zookeeper-stop.sh 
echo "------然后关闭hadoop集群------"
${HADOOP_HOME}/sbin/stop-all.sh