环境配置
一。生成数据库表
1. 创建数据库和表
/* SQLyog Ultimate v12.08 (64 bit) MySQL - 5.6.42 : Database - zghPortrait ********************************************************************* */ /*!40101 SET NAMES utf8 */; /*!40101 SET SQL_MODE=''*/; /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; CREATE DATABASE /*!32312 IF NOT EXISTS*/`zghPortrait` /*!40100 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin */; USE `zghPortrait`; /*Table structure for table `userInfo` */ DROP TABLE IF EXISTS `userInfo`; CREATE TABLE `userInfo` ( `userid` int(20) DEFAULT NULL COMMENT '用户id', `username` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '名称', `password` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '密码', `sex` int(1) DEFAULT NULL COMMENT '性别', `telphone` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '手机号', `email` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '邮箱', `age` int(20) DEFAULT NULL COMMENT '年龄', `idCard` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '身份证', `registerTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '注册时间', `usetype` int(1) DEFAULT NULL COMMENT '使用终端：0 pc端：1 移动端：2 小程序端' ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; /*Data for the table `userInfo` */ /*Table structure for table `userdetail` */ DROP TABLE IF EXISTS `userdetail`; CREATE TABLE `userdetail` ( `userdetailid` int(20) NOT NULL COMMENT '详情id', `userid` int(20) DEFAULT NULL COMMENT '用户id', `edu` int(1) DEFAULT NULL COMMENT '学历', `profession` varchar(20) COLLATE utf8_bin DEFAULT NULL COMMENT '职业', `marriage` int(1) DEFAULT NULL COMMENT '婚姻状态:1 未婚：2 已婚：3 离异', `haschild` int(1) DEFAULT NULL COMMENT '1.没有小孩：2.有小孩 3.未知 4. 未知', `hascar` int(1) DEFAULT NULL COMMENT '1.有车；2.没车；3. 未知', `hashourse` int(1) DEFAULT NULL COMMENT '1.有房；2.没房；3. 未知', `telphonebrand` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '手机品牌', PRIMARY KEY (`userdetailid`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; /*Data for the table `userdetail` */ /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
二。创建maven工程
1. 生成父工程：zghPortraitMain ，并在此工程下生成工程：analySrv ，工程 analySrv 继承且聚合于工程 zghPortraitMain
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.zgh.portrait</groupId> <artifactId>zghPortraitMain</artifactId> <packaging>pom</packaging> <version>1.0-SNAPSHOT</version> <modules> <module>analySrv</module> </modules> </project>
2. 在工程 analySrv 下添加依赖，如果依赖下载无进度，停止任务并对应删除本地仓库的依赖，重新载入或者刷新下载
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.zgh.portrait</groupId> <!--依赖父版本--> <!--<version>1.0-SNAPSHOT</version>--> <artifactId>analySrv</artifactId> <parent> <groupId>org.apache.flink</groupId> <artifactId>flink-examples</artifactId> <version>1.7.0</version> </parent> <properties> <scala.version>2.11.12</scala.version> <scala.binary.version>2.11</scala.binary.version> </properties> <dependencies> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-streaming-java_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-scala_2.11</artifactId> <version>${project.version}</version> </dependency> </dependencies> </project>
3. 在 analySrv 下新建包：com.zgh.task ，并创建 Test 类
package com.zgh.task; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment; public class Test { public static void main(String[] args){ final ParameterTool params = ParameterTool.fromArgs(args); // set up the execution environment final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); // make parameters available in the web interface env.getConfig().setGlobalJobParameters(params); // get input data hdfs DataSet<String> text = env.readTextFile(params.get("input")); DataSet map = text.flatMap(null); DataSet reduce = map.groupBy("groupbyfield").reduce(null); try { env.execute("test"); } catch (Exception e) { e.printStackTrace(); } } }
三。搭建hadoop
1. 修改主机名为 master ，并对应 ip（dns），然后重新登录
[root@localhost ~]# vim /etc/hostname [root@localhost ~]# cat /etc/hostname master [root@localhost ~]# cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 10.20.0.129 master [root@localhost ~]# Last login: Fri Mar 1 10:16:35 2019 [root@master ~]#
2. 配置秘钥，回车默认，再配置秘钥登录
[root@master ~]# ssh-keygen -t rsa Generating public/private rsa key pair. Enter file in which to save the key (/root/.ssh/id_rsa): [root@master ~]# ssh-copy-id master /usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/.ssh/id_rsa.pub" [root@master ~]# ls ~/.ssh/ authorized_keys id_rsa id_rsa.pub known_hosts [root@master ~]# cat ~/.ssh/authorized_keys ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDDmisZgUIF36e10nYnff1YqoqXFabUW7FTEZ1uO2w6W2Xq9R95ywaRqElMUccNxaruI8T5NFJ0uGpD3Bbv7TOHr42DTalA7MYU3gLUCs5rFXQATKbRA3veQa9IN2KYSxwrvqfmJCVbJbLMEsTPvobWDaOvDAacWdwh5ppliY3ngI8tgevttXblFWPrfbThXdIws3/vu04plRSEYxUn0Dvp9y9s4fOBClSCCvAmo2y2D3wK3ut2qnwyM8jVL1Fy58tke/w4ZMtRb83Tt7lRFpY5InxGmxNH68sB1ql2YXzKrIPP3ShuLFldQkp8qK/uvLC8GwcHT7trL+X3wyDBrkX9 root@master [root@master ~]#
[root@master ~]# ssh master Last login: Fri Mar 1 10:17:55 2019 from 10.20.0.1 [root@master ~]#
3. 下载jdk，并解压
[root@master hadoop]# ls jdk-8u201-linux-x64.tar.gz [root@master hadoop]# tar -zxvf jdk-8u201-linux-x64.tar.gz
[root@master hadoop]# ls jdk1.8.0_201 jdk-8u201-linux-x64.tar.gz [root@master hadoop]# mv jdk1.8.0_201/ /usr/local/jdk [root@master hadoop]#
4. 下载 hadoop ：http://archive.apache.org/dist/hadoop/core/
[root@master hadoop]# tar -zxvf hadoop-2.6.0.tar.gz [root@master hadoop]# mv hadoop-2.6.0/ /usr/local/hadoop [root@master hadoop]#
5. 配置 hadoop jdk环境
[root@master ~]# cd /usr/local/hadoop/etc/hadoop/ [root@master hadoop]# vim hadoop-env.sh
export JAVA_HOME=/usr/local/jdk
6. 配置 core-site.xml
[root@master hadoop]# vim core-site.xml [root@master hadoop]#
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://master:9000</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/usr/local/hadoop/tmp</value>
</property>
</configuration>
7. 配置 hdfs-site.xml
[root@master hadoop]# vim hdfs-site.xml [root@master hadoop]#
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.permissions</name>
<value>true</value>
</property>
</configuration>
8. 配置 mapred-site.xml
[root@master hadoop]# cp mapred-site.xml.template mapred-site.xml [root@master hadoop]# vim mapred-site.xml [root@master hadoop]#
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>master:10020</value>
</property>
</configuration>
9. 配置 yarn-site.xml
[root@master hadoop]# vim yarn-site.xml [root@master hadoop]#
<configuration>
<!-- Site specific YARN configuration properties -->
<property>
<name>yarn.resourcemanager.hostname</name>
<value>master</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>mapreduce.job.ubertask.enable</name>
<value>true</value>
</property>
</configuration>
10. 配置环境变量
[root@master hadoop]# vim /etc/profile [root@master hadoop]#
在末尾加上
export JAVA_HOME=/usr/local/jdk export HADOOP_HOME=/usr/local/hadoop export PATH=.:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
让环境生效
[root@master hadoop]# source /etc/profile [root@master hadoop]# jps 31134 Jps [root@master hadoop]#
11. 启动 hadoop , 一键启动：start-all.sh
[root@master hadoop]# hadoop namenode -format [root@master hadoop]# start-dfs.sh [root@master hadoop]# start-yarn.sh [root@master ~]# jps 2518 DataNode 2886 NodeManager 2791 ResourceManager 2648 SecondaryNameNode 3180 Jps 2429 NameNode [root@master hadoop]# systemctl stop firewalld.service [root@master hadoop]#
12. 浏览器访问：http://10.20.0.129:50070/
四。安装hbase
1. 下载 zookeeper：http://mirror.bit.edu.cn/apache/zookeeper/
[root@master hadoop]# tar -zxvf zookeeper-3.4.10.tar.gz [root@master hadoop]# mv zookeeper-3.4.10/ /usr/local/zk
2. 配置 zookeeper 数据路径
[root@master hadoop]# cd /usr/local/zk/conf/ [root@master conf]# cp zoo_sample.cfg zoo.cfg [root@master conf]# vim zoo.cfg
dataDir=/usr/local/zk/data
3. 下载 hbase： http://archive.cloudera.com/cdh5/cdh/5/
[root@master hadoop]# tar -zxvf hbase-1.0.0-cdh5.5.1.tar.gz [root@master hadoop]# mv hbase-1.0.0-cdh5.5.1/ /usr/local/hbase
4. 配置 hbase jdk路径
[root@master hadoop]# cd /usr/local/hbase/conf/ [root@master conf]# vim hbase-env.sh
export JAVA_HOME=/usr/local/jdk
5. 配置 hbase-site.xml
[root@master conf]# vim hbase-site.xml
<configuration> <property> <name>hbase.rootdir</name> <value>hdfs://master:9000/hbase</value> </property> <property> <name>hbase.cluster.distributed</name> <value>true</value> </property> <property> <name>hbase.zookeeper.quorum</name> <value>master</value> </property> <property> <name>dfs.replication</name> <value>1</value> </property> </configuration>
6. 配置 regionservers 为 master
[root@master conf]# vim regionservers
7. 配置环境
[root@master conf]# vim /etc/profile
export JAVA_HOME=/usr/local/jdk export HADOOP_HOME=/usr/local/hadoop export ZK_HOME=/usr/local/zk export HBASE_HOME=/usr/local/hbase export PATH=.:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ZK_HOME/bin:$HBASE_HOME/bin:$PATH
[root@master conf]# source /etc/profile
8. 启动 hbase
[root@master conf]# zkServer.sh start ZooKeeper JMX enabled by default Using config: /usr/local/zk/bin/../conf/zoo.cfg Starting zookeeper ... STARTED [root@master conf]# zkServer.sh status ZooKeeper JMX enabled by default Using config: /usr/local/zk/bin/../conf/zoo.cfg Mode: standalone [root@master conf]# start-hbase.sh master: starting zookeeper, logging to /usr/local/hbase/bin/../logs/hbase-root-zookeeper-master.out starting master, logging to /usr/local/hbase/logs/hbase-root-master-master.out master: starting regionserver, logging to /usr/local/hbase/bin/../logs/hbase-root-regionserver-master.out [root@master conf]#
[root@master conf]# jps 4545 HMaster 5010 Jps 4197 QuorumPeerMain 2518 DataNode 2886 NodeManager 4678 HRegionServer 2791 ResourceManager 2648 SecondaryNameNode 4472 HQuorumPeer 2429 NameNode [root@master conf]#
9 . 浏览器访问hbase控制台：http://10.20.0.129:60010/
10. 浏览器访问 hadoop 控制台：http://10.20.0.129:50070
五。安装mongodb
1. 下载 mongodb ：https://www.mongodb.com/download-center/community
[root@master hadoop]# tar -zxvf mongodb-linux-x86_64-3.2.22.tgz [root@master hadoop]# mv mongodb-linux-x86_64-3.2.22/ /usr/local/mongodb [root@master hadoop]#
2. 配置 mongodb
[root@master hadoop]# cd /usr/local/mongodb/ [root@master mongodb]# mkdir data [root@master mongodb]# mkdir conf [root@master mongodb]# mkdir log [root@master mongodb]# vim conf/mongod.conf [root@master mongodb]# cat conf/mongod.conf #port = 12345 dbpath = /usr/local/mongodb/data logpath = /usr/local/mongodb/log/mongod.log fork = true logappend = true #bind_ip = 127.0.0.1 [root@master mongodb]#
3. 配置环境
[root@master mongodb]# vim /etc/profile [root@master mongodb]#
export JAVA_HOME=/usr/local/jdk export HADOOP_HOME=/usr/local/hadoop export ZK_HOME=/usr/local/zk export HBASE_HOME=/usr/local/hbase export MONGO_HOME=/usr/local/mongodb export PATH=.:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ZK_HOME/bin:$HBASE_HOME/bin:$MONGO_HOME/bin:$PATH
[root@master mongodb]# source /etc/profile [root@master mongodb]#
3. 启动 mongodb
[root@master mongodb]# mongod -f conf/mongod.conf about to fork child process, waiting until server is ready for connections. forked process: 5662 child process started successfully, parent exiting [root@master mongodb]#
4. 登录
> show dbs local 0.000GB > exit bye [root@master mongodb]#
