Hive3.1.2安装部署

一、安装Hive3.1.2

备注：在安装Hive3.1.2之前，请首先安装Hadoop3.1.3。

1. 下载并解压Hive安装包

tar -zxvf ./apache-hive-3.1.2-bin.tar.gz -C /usr/local

cd /usr/local/

mv apache-hive-3.1.2-bin hive       

chown -R hadoop:hadoop hive            

 

2. 配置环境变量

# vim /etc/profile

export HADOOP_HOME=/usr/local/hadoop
export HIVE_HOME=/usr/local/hive
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$HIVE_HOME/bin:$HADOOP_HOME/bin

使配置文件生效

# source /etc/profile

 

3. 修改hive-env.xml

cd /usr/local/hive/conf

cp hive-env.sh.template hive-env.sh   

vim hive-env.sh

export JAVA_HOME=/usr/local/jdk1.8.0_271
export HADOOP_HOME=/home/Hadoop
export HIVE_CONF_DIR=/home/hive/conf

3. 修改hive-site.xml

cd /usr/local/hive/conf

mv hive-default.xml.template hive-site.xml

vim hive-site.xml

<configuration>

<property>
        <name>system:java.io.tmpdir</name>
        <value>/user/hive/warehouse</value>
      </property>

      <property>
        <name>system:user.name</name>
        <value>${user.name}</value>
      </property>

     <property>
        <name>hive.metastore.db.type</name>
        <value>mysql</value>
     </property>

  <property>
    <name>javax.jdo.option.ConnectionURL</name>
    <value>jdbc:mysql://cancer01:3306/hive?createDatabaseIfNotExist=true</value>
  </property>

  <property>
    <name>javax.jdo.option.ConnectionDriverName</name>
    <value>com.mysql.jdbc.Driver</value>
  </property>

  <property>
    <name>javax.jdo.option.ConnectionUserName</name>
    <value>root</value>
  </property>

  <property>
    <name>javax.jdo.option.ConnectionPassword</name>
    <value>root</value>
  </property>

<property>
 <name>datanucleus.schema.autoCreateAll</name>
 <value>true</value>
</property>

<property>
 <name>hive.metastore.schema.verification</name>
 <value>false</value>
</property>

<!--以下配置为配合flume sink写入hive -->

<property>
<name>hive.metastore.uris</name>
<value>thrift://cancer01:9083</value>
</property>

<property>
<name>hive.txn.manager</name>
<value>org.apache.hadoop.hive.ql.lockmgr.DbTxnManager</value>
</property>

<property>
<name>hive.compactor.initiator.on</name>
<value>true</value>
</property>

<property>
<name>hive.compactor.worker.threads</name>
<value>1</value>
</property>

<property>
<name>hive.support.concurrency</name>
<value>true</value>
</property>

<property>
<name>hive.enforce.bucketing</name>
<value>true</value>
</property>

<property>
<name>hive.exec.dynamic.partition.mode</name>
<value>nonstrict</value>

</property>

<property>
<name>hive.in.test</name>
<value>true</value>
</property>

</configuration>

配置说明：

（1）报错：Caused by: org.apache.thrift.TApplicationException: Internal error processing open_txns

hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; -》 打开一部分事务支持

hive.compactor.initiator.on=true; -》运行启动程序和清除线程,用于打开所需参数的完整列表事务

hive.compactor.worker.threads=1; -》增加工作线程的数量将减少花费的时间

hive.support.concurrency=true; -》是否支持并发，默认是false

hive.enforce.bucketing=true; -》是否启用bucketing，写入table数据时会启动分桶

hive.exec.dynamic.partition.mode=nonstrict; -》设置非严格模式

(2)启动metastore时报错：

Table 'metastore.COMPACTION_QUEUE' doesn't exist

配置以下属性：这个是用来创建COMPACTION_QUEUE这张表的

(3)再启动metastore时报错：

Error rolling back: Can't call rollback when autocommit=true

去掉以下属性：之前没有安装，先安装：yum install -y nc

 

3. 修改hdfs目录

$HADOOP_HOME/bin/hadoop fs -mkdir -p /tmp

$HADOOP_HOME/bin/hadoop fs -chmod g+w   /tmp

$HADOOP_HOME/bin/hadoop fs -mkdir -p /user/hive/warehouse

$HADOOP_HOME/bin/hadoop fs -chmod g+w   /user/hive/warehouse

 

4. 配置日志

配置日志，复制一个模板

cp hive-log4j2.properties.template hive-log4j2.properties

vim hive-log4j2.properties

配置property.hive.log.dir
property.hive.log.dir = /usr/local/hive/logs

cd /usr/local/hive/

mkdir logs

 

二、安装mysql

1. mysql安装

sudo apt-get install mysql-server

sudo apt-get install libmysql-java

 

或者

删除原有mysql

rpm -qa | grep mysql 或 yum list installed | grep mysql

yum -y remove mysql-libs.x86_64

删除原有mariadb

rpm -qa | grep -i mariadb

rpm -ev  --nodeps mariadb-libs-5.5.65-1.el7.x86_64

yum install libaio

tar -xvf mysql-5.7.31-1.el7.x86_64.rpm-bundle.tar

rpm -ivh 顺序 common-》lib-》client-》server

 

2. mysql配置

安装完，启动服务

systemctl start mysqld

systemctl status mysqld

grep "password" /var/log/mysqld.log

A temporary password is generated for root@localhost: hilX0U!9i3_6

mysql -u root -p hilX0U!9i3_6

use mysql;

grant all privileges on *.* to root@"localhost" identified by "root";

flush privileges;

 

3. 创建Hive元数据库

mysql> create database hive

use hive;

mysql> source $HIVE_HOME/scripts/metastore/upgrade/mysql/hive-schema-3.1.2.mysql.sql;

 

4. 配置mysql 驱动包

tar -zxvf mysql-connector-java-5.1.48.tar.gz

#将mysql驱动包拷贝到/usr/local/hive/lib下

cp mysql-connector-java-5.1.48-bin.jar   /usr/local/hive/lib

 

或者

ln -s /usr/share/java/mysql-connector-java.jar $HIVE_HOME/lib/mysql-connector-java.jar

三、启动hive

1. 启动

初始化hive数据库

cd /usr/local/hive/bin

schematool -dbType mysql -initSchema

 

开启metastore（&表示后台进程）

./hive --service metastore &

 

启动hadoop集群。

start-dfs.sh

#启动hive

hive

 

2. 验证

创建库表

create database test;

use test;

create table user(

user_id int,

user_name string,

user_age int

)CLUSTERED BY (user_id) INTO 2 BUCKETS

row format delimited fields terminated by '\t'

stored as orc;

#创建表

    hive (default)> CREATE TABLE IF NOT EXISTS test_table

                 (col1 int COMMENT 'Integer Column',

                     col2 string COMMENT 'String Column'

                 )

                 COMMENT 'This is test table'

                 ROW FORMAT DELIMITED

                 FIELDS TERMINATED BY ','

                 STORED AS TEXTFILE;

    hive (default)> show tables;

        tab_name

        test_table

    #写入

    hive (default)> insert into test_table values(1,'aaa');

        MapReduce Jobs Launched:

        Stage-Stage-1: Map: 1  Reduce: 1   Cumulative CPU: 5.54 sec   HDFS Read: 15408 HDFS Write: 243 SUCCESS

        Total MapReduce CPU Time Spent: 5 seconds 540 msec

        OK

        col1 col2

        Time taken: 26.271 seconds

    #查询

    hive (default)> select * from test_table;

        test_table.col1 test_table.col2

            2 bbb

            3 ccc

            4 ddd

        Time taken: 0.205 seconds, Fetched: 3 row(s)

jps

    root@had001:/home/hive# jps

        6675 SecondaryNameNode

        6426 NameNode

        6908 ResourceManager

        8382 Jps

jps

    root@had002:~# jps

        3300 DataNode

        3430 NodeManager

        5610 Jps

    #查看是否能连接had001

    root@had002:~# /home/hadoop/bin/hdfs dfsadmin -report

    root@had003:~# /home/hadoop/bin/hdfs dfsadmin -report

    #正常有data目录

    root@had002:~# tree /usr/local/hadoop/tmp

    /usr/local/hadoop/tmp

    ├── dfs

    │   └── data

    │       ├── current

    │       │   ├── BP-1834162669-172.17.252.52-1532682436448

    │       │   │   ├── current

    │       │   │   │   ├── finalized