spark集群安装部署
修改master主机名字
[root@master]# hostname master
修改hosts文件
[root@master ~]# vim /etc/hosts 192.168.112.135 master 192.168.112.136 slave1 192.168.112.137 slave2
修改完成后保存执行如下命令
[root@master ~]# source /etc/hosts
配置master无密码登录所有Salve
在master节点上生成密码对
[root@master ~]# ssh-keygen -t rsa -P ''
把id_rsa.pub追加到授权的key里面去
[root@master ~]# cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
修改ssh配置文件"/etc/ssh/sshd_config"
[root@master ~]# vim /etc/ssh/sshd_config RSAAuthentication yes # 启用 RSA 认证 PubkeyAuthentication yes # 启用公钥私钥配对认证方式
重启ssh服务
[root@master ~]# systemctl restart ssh
验证无密码登录本机是否成功
[root@master ~]# ssh master
将公钥复制到所有的Slave机器上
[root@master ~]# scp /root/.ssh/id_rsa.pub root@slave1:/root/ [root@master ~]# scp /root/.ssh/id_rsa.pub root@slave2:/root/
接着配置slave节点，以下是在slave1、slave2节点的配置操作
在"/root/"下创建".ssh"文件夹
[root@slave1 ~]# mkdir /root/.ssh
将master的公钥追加到slave1、slave2的授权文件"authorized_keys"中
[root@slave1 ~]# cat /root/id_rsa.pub >> /root/.ssh/authorized_keys
修改"/etc/ssh/sshd_config"
[root@master ~]# vim /etc/ssh/sshd_config RSAAuthentication yes # 启用 RSA 认证 PubkeyAuthentication yes # 启用公钥私钥配对认证方式
重启ssh服务
[root@slave1 ~]# systemctl restart ssh
创建"slave1、slave2"自己的公钥和私钥，并把自己的公钥追加到"authorized_keys"文件中
[root@slave1 ~]# ssh-keygen -t rsa -P '' [root@slave1 ~]# cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys
将slave1、slave2节点的公钥"id_rsa.pub"复制到Master节点的"/root/"目录下
[root@slave1 ~]# scp /root/.ssh/id_rsa.pub root@Master:/root/
以下是在master节点的配置操作
将slave1、slave2的公钥追加到Master的授权文件"authorized_keys"中去。 [root@master ~]# cat ~/id_rsa.pub >> ~/.ssh/authorized_keys
删除slave1复制过来的"id_rsa.pub"文件
[root@master ~]# rm –r /root/id_rsa.pub
Java1.8.0_121环境搭建
上传并解压jdk # master,slave1,slave2
jdk-8u121-linux-x64.tar.gz [root@master ~]# tar -zxvf jdk-8u121-linux-x64.tar.gz
添加Java环境变量，在/etc/profile中添加
[root@master ~]# vim /etc/profile export JAVA_HOME=/usr/local/jdk1.8.0_121 export PATH=$JAVA_HOME/bin:$PATH export CLASSPATH=.:$JAVA_HOME/lib/rt.jar export JAVA_HOME PATH CLASSPATH
保存后刷新配置
[root@master ~]# source /etc/profile
Scala2.10.4环境搭建
上传并解压Scala
scala-2.10.4.rpm [root@master ~]# rpm -ivh scala-2.10.4.rpm
添加Scala环境变量，在/etc/profile中添加
[root@master ~]# vim /etc/profile export SCALA_HOME=/usr/share/scala export PATH=$SCALA_HOME/bin:$PATH
保存后刷新配置
[root@master ~]# source /etc/profile
Hadoop2.7.2完全分布式搭建
上传并解压Hadoop
hadoop-3.3.0.tar.gz [root@master ~]# tar -zxvf hadoop-2.7.2.tar.gz [root@master ~]# mv hadoop-2.7.2 /opt/
修改/etc/profile
export HADOOP_HOME=/opt/hadoop-2.7.2/ export PATH=$PATH:$HADOOP_HOME/bin export PATH=$PATH:$HADOOP_HOME/sbin export HADOOP_MAPRED_HOME=$HADOOP_HOME export HADOOP_COMMON_HOME=$HADOOP_HOME export HADOOP_HDFS_HOME=$HADOOP_HOME export YARN_HOME=$HADOOP_HOME export HADOOP_ROOT_LOGGER=INFO,console export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib"
保存后刷新配置
[root@master ~]# source /etc/profile
修改JAVA_HOME 如下
[root@master hadoop-2.7.2]# vim $HADOOP_HOME/etc/hadoop/hadoop-env.sh export JAVA_HOME=/usr/local/jdk1.8.0_121 [root@master hadoop-2.7.2]# vim $HADOOP_HOME/etc/hadoop/yarn-env.sh export JAVA_HOME=/usr/local/jdk1.8.0_121 [root@master hadoop-2.7.2]# vim $HADOOP_HOME/etc/hadoop/mapred-env.sh export JAVA_HOME=/usr/local/jdk1.8.0_121
/etc/hadoop/slaves，将原来的localhost删除，改成如下内容
[root@master ~]# vim $HADOOP_HOME/etc/hadoop/slaves slave1 Slave2
修改/opt/hadoop-2.7.2/etc/hadoop/core-site.xml
[root@master ~]# vim $HADOOP_HOME/etc/hadoop/core-site.xml
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://master:9000</value>
</property>
<property>
<name>io.file.buffer.size</name>
<value>131072</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/opt/hadoop-2.7.2/tmp</value>
</property>
</configuration>
修改/opt/hadoop-2.7.2/etc/hadoop/hdfs-site.xml
[root@master ~]# vim $HADOOP_HOME/etc/hadoop/hdfs-site.xml
<configuration>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>Master:50090</value>
</property>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/opt/hadoop-2.7.2/hdfs/name</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/opt/hadoop-2.7.2/hdfs/data</value>
</property>
</configuration>
修改/opt/hadoop-2.7.2/etc/hadoop/mapred-site.xml
[root@master ~]# cp mapred-site.xml.template mapred-site.xml
[root@master ~]# vim $HADOOP_HOME/etc/hadoop/mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>master:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>master:19888</value>
</property>
</configuration>
修改/opt/hadoop-2.7.2/etc/hadoop/yarn-site.xml
[root@master ~]# vim $HADOOP_HOME/etc/hadoop/yarn-site.xml
<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.resourcemanager.address</name>
<value>master:8032</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>master:8030</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>master:8031</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>master:8033</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>master:8088</value>
</property>
</configuration>
复制master节点的hadoop文件夹到Slave1和Slave2上
[root@master ~]# scp -r /opt/hadoop-2.7.2 root@slave1:/opt [root@master ~]# scp -r /opt/hadoop-2.7.2 root@slave2:/opt
在dlave1和dlave2上分别修改/etc/profile，过程同Master一样
在master节点启动集群，启动之前格式化一下namenode
[root@master ~]# hadoop namenode -format
启动
[root@master ~]# /opt/hadoop-2.7.2/sbin/start-all.sh
启动成功
This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh 20/12/02 05:20:25 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable Starting namenodes on [master] master: namenode running as process 21607. Stop it first. slave2: datanode running as process 2472. Stop it first. slave1: datanode running as process 2322. Stop it first. Starting secondary namenodes [Master] Master: secondarynamenode running as process 21846. Stop it first. 20/12/02 05:20:29 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable starting yarn daemons resourcemanager running as process 22027. Stop it first. slave2: nodemanager running as process 2559. Stop it first. slave1: nodemanager running as process 2410. Stop it first.
报错
Starting namenodes on [master]
ERROR: Attempting to operate on hdfs namenode as root
ERROR: but there is no HDFS_NAMENODE_USER defined. Aborting operation.
Starting datanodes
ERROR: Attempting to operate on hdfs datanode as root
ERROR: but there is no HDFS_DATANODE_USER defined. Aborting operation.
/opt/hadoop-3.3.0//libexec/hadoop-functions.sh: line 1776: /usr/local/jdk-15.0.1/bin/java: cannot execute binary file
/opt/hadoop-3.3.0//libexec/hadoop-functions.sh: line 1776: /usr/local/jdk-15.0.1/bin/java: Success
查看集群是否启动成功
[root@master ~]# jps 35441 Jps 21846 SecondaryNameNode 21607 NameNode 22027 ResourceManager
Spark2.1.0完全分布式环境搭建
上传压缩包并解压
spark-2.1.0-bin-hadoop2.7.tgz [root@master ~]# spark-2.1.0-bin-hadoop2.7.tgz [root@master ~]# mv park-2.1.0-bin-hadoop2.7 /opt
添加以下内容到/etc/profile
[root@master ~]# vim /etc/profile export SPARK_HOME=/opt/spark-2.1.0-bin-hadoop2.7/ export PATH=$PATH:$SPARK_HOME/bin
保存后刷新配置
[root@master ~]# source /etc/profile
复制spark-env.sh.template成spark-env.sh
[root@master ~]# cd /opt/spark-2.1.0-bin-hadoop2.7/conf [root@master conf]# vim $SPARK_HOME/conf/spark-env.sh export JAVA_HOME=/usr/local/jdk1.8.0_121 export SCALA_HOME=/usr/share/scala export HADOOP_HOME=/opt/hadoop-2.7.2 export HADOOP_CONF_DIR=/opt/hadoop-2.7.2/etc/hadoop export SPARK_MASTER_IP= masterIP地址 export SPARK_MASTER_HOST= masterIP地址 export SPARK_LOCAL_IP= masterIP地址 export SPARK_WORKER_MEMORY=1g export SPARK_WORKER_CORES=2 export SPARK_HOME=/opt/spark-2.1.0-bin-hadoop2.7 export SPARK_DIST_CLASSPATH=$(/opt/hadoop-2.7.2/bin/hadoop classpath)
复制slaves.template成slaves，并添加以下内容
[root@master conf]# cp slaves.template slaves [root@master conf]# vim $SPARK_HOME/conf/slaves master slave1 slave2
将配置好的spark文件复制到Slave1和Slave2节点
[root@master ~]# scp /opt/spark-2.1.0-bin-hadoop2.7 root@slave1:/opt [root@master ~]# scp /opt/spark-2.1.0-bin-hadoop2.7 root@slave2:/opt
修改slave1和slave2配置
在slave1和slave2上分别修改/etc/profile，增加Spark的配置，过程同Master一样。
在slave1和slave2修改$SPARK_HOME/conf/spark-env.sh，将export SPARK_LOCAL_IP=xxxxx改成slave1和slave2对应节点的IP
export JAVA_HOME=/usr/local/jdk1.8.0_121 export SCALA_HOME=/usr/share/scala export HADOOP_HOME=/opt/hadoop-2.7.2 export HADOOP_CONF_DIR=/opt/hadoop-2.7.2/etc/hadoop export SPARK_MASTER_IP=masterIP地址 export SPARK_MASTER_HOST= masterIP地址 export SPARK_LOCAL_IP=slaveIP地址 export SPARK_WORKER_MEMORY=1g export SPARK_WORKER_CORES=2 export SPARK_HOME=/opt/spark-2.1.0-bin-hadoop2.7 export SPARK_DIST_CLASSPATH=$(/opt/hadoop-2.7.2/bin/hadoop classpath) [root@master conf]# /opt/spark-2.1.0-bin-hadoop2.7/sbin/start-all.sh
启动成功
starting org.apache.spark.deploy.master.Master, logging to /opt/spark-2.1.0-bin-hadoop2.7/logs/spark-root-org.apache.spark.deploy.master.Master-1-master.out master: starting org.apache.spark.deploy.worker.Worker, logging to /opt/spark-2.1.0-bin-hadoop2.7/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-master.out slave1: starting org.apache.spark.deploy.worker.Worker, logging to /opt/spark-2.1.0-bin-hadoop2.7/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-slave1.out slave2: starting org.apache.spark.deploy.worker.Worker, logging to /opt/spark-2.1.0-bin-hadoop2.7/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-slave2.out
查看集群是否启动成功
[root@master ~]# jps 50704 Master 50837 Worker 50933 Jps 21846 SecondaryNameNode 21607 NameNode 22027 ResourceManager
浙公网安备 33010602011771号