Hive参数优化

set hive.exec.dynamic.partition.mode=nonstrict;    
set hive.exec.dynamic.partition=true;
set hive.auto.convert.join = true;
set hive.mapjoin.smalltable.filesize=300000000;
set hive.exec.parallel=true;
set hive.auto.convert.join = true;30000000;
set mapreduce.map.memory.mb=4096;
set mapreduce.reduce.memory.mb=4096;
set yarn.nodemanager.vmem-pmem-ratio=3;
set hive.optimize.skewjoin=true;
set hive.skewjoin.key=1000000;
set mapred.reduce.tasks=10000;
set hive.exec.reducers.bytes.per.reducer=1000000000;

 

 

hive on tez:

SET hive.exec.dynamic.partition=true;
SET hive.exec.dynamic.partition.mode=nonstrict;
set hive.exec.max.dynamic.partitions=3000;
set hive.exec.max.dynamic.partitions.pernode=500;
SET hive.tez.container.size=6656;
SET hive.tez.java.opts=-Xmx5120m;
set hive.merge.tezfiles=true;
set hive.merge.smallfiles.avgsize=1280000000;
set hive.merge.size.per.task=1280000000;
set hive.execution.engine=tez;

set hive.merge.tezfiles=true;
set hive.merge.smallfiles.avgsize=128000000; --128MB
set hive.merge.size.per.task=128000000; -- 128MB

仅限参数，其中的值需要自己衡量。