UDF、UDAF、UDTF

UDF、UDAF、UDTF函数编写

        </h1>
        <div class="clear"></div>
        <div class="postBody">

一、UDF函数编写

1.步骤

1.继承UDF类
2.重写evalute方法

1、继承GenericUDF
2、实现initialize、evaluate、getDisplayString方法

2.案例

实现lower函数:

package com.xxx.udf;
import org.apache.hadoop.hive.ql.exec.UDF;

import org.apache.hadoop.io.Text;
public class LowerUDF extends UDF {

public Text evaluate(Text input){

if(null == input){

return null;

}

String inputValue = input.toString().trim() ;

if(null == inputValue){

return null ;

}

return new Text(inputValue.toLowerCase()) ;

}

}

package com.xxx.udf;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;

import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;

import org.apache.hadoop.hive.ql.metadata.HiveException;

import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;

import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;

import org.apache.hadoop.io.Text;
public class LowerUDF extends GenericUDF {
StringObjectInspector str ;
@Override
</span><span style="color: #0000ff;">public</span> ObjectInspector initialize(ObjectInspector[] arguments) <span style="color: #0000ff;">throws</span><span style="color: #000000;"> UDFArgumentException {
    </span><span style="color: #008000;">//</span><span style="color: #008000;">判断输入参数个数是否合法</span>
    <span style="color: #0000ff;">if</span> (arguments.length != 1<span style="color: #000000;">) {
        </span><span style="color: #0000ff;">throw</span> <span style="color: #0000ff;">new</span> UDFArgumentLengthException("输入参数长度不合法,应该为一个参数"<span style="color: #000000;">);
    }

    </span><span style="color: #008000;">//</span><span style="color: #008000;">判断输入参数类型是否合法</span>
    <span style="color: #0000ff;">if</span> (!(arguments[0] <span style="color: #0000ff;">instanceof</span><span style="color: #000000;"> StringObjectInspector)) {
        </span><span style="color: #0000ff;">throw</span> <span style="color: #0000ff;">new</span> UDFArgumentException("输入非法参数，应为字符串类型"<span style="color: #000000;">);
    }

    str</span>=(StringObjectInspector)arguments[0<span style="color: #000000;">];
    </span><span style="color: #008000;">//</span><span style="color: #008000;">确定返回值类型</span>
    <span style="color: #0000ff;">return</span><span style="color: #000000;"> PrimitiveObjectInspectorFactory.javaStringObjectInspector;
}

@Override
</span><span style="color: #0000ff;">public</span> Object  evaluate(DeferredObject[] arguments) <span style="color: #0000ff;">throws</span><span style="color: #000000;"> HiveException {
    String input </span>= str.getPrimitiveJavaObject(arguments[0<span style="color: #000000;">].get());
    </span><span style="color: #0000ff;">return</span> <span style="color: #0000ff;">new</span><span style="color: #000000;"> Text(input.toLowerCase());
}

@Override
</span><span style="color: #0000ff;">public</span><span style="color: #000000;"> String getDisplayString(String[] children) {
    </span><span style="color: #0000ff;">return</span> "方法的描述信息"<span style="color: #000000;">;
}

}

3.打成jar包上传

mvn clean package

4.在hive中创建临时函数

add jar /home/xxx/yf/to_lower.jar;
create temporary function to_lower as 'com.xxx.udf.LowerUDF';
select to_lower("DSJIFASD") from dual;
drop temporary function comparestringbysplit;

二、UDAF函数编写

1.步骤

1、继承AbstractGenericUDAFResolver
2、继承GenericUDAFEvaluator
3、Evaluator需要实现 init、iterate、terminatePartial、merge、terminate这几个函数 
   init初始化
   iterate函数处理读入的行数据
   terminatePartial返回iterate处理的中间结果
   merge合并上述处理结果
   terminate返回最终值

2.案例

实现avg

package com.xxx.udf;
import org.apache.hadoop.hive.ql.exec.UDAF;

import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
public class Avg extends UDAF {

public static class AvgState {

private long mCount;

private double mSum;

}
</span><span style="color: #0000ff;">public</span> <span style="color: #0000ff;">static</span> <span style="color: #0000ff;">class</span> AvgEvaluator <span style="color: #0000ff;">implements</span><span style="color: #000000;"> UDAFEvaluator {
    AvgState state;

    </span><span style="color: #0000ff;">public</span><span style="color: #000000;"> AvgEvaluator() {
        </span><span style="color: #0000ff;">super</span><span style="color: #000000;">();
        state </span>= <span style="color: #0000ff;">new</span><span style="color: #000000;"> AvgState();
        init();
    }

    </span><span style="color: #008000;">/**</span><span style="color: #008000;">
     * init函数类似于构造函数，用于UDAF的初始化
     </span><span style="color: #008000;">*/</span>
    <span style="color: #0000ff;">public</span> <span style="color: #0000ff;">void</span><span style="color: #000000;"> init() {
        state.mSum </span>= 0<span style="color: #000000;">;
        state.mCount </span>= 0<span style="color: #000000;">;
    }

    </span><span style="color: #008000;">/**</span><span style="color: #008000;">
     * iterate接收传入的参数，并进行内部的轮转。其返回类型为boolean * * </span><span style="color: #808080;">@param</span><span style="color: #008000;"> o * </span><span style="color: #808080;">@return</span>
     <span style="color: #008000;">*/</span>

    <span style="color: #0000ff;">public</span> <span style="color: #0000ff;">boolean</span><span style="color: #000000;"> iterate(Double o) {
        </span><span style="color: #0000ff;">if</span> (o != <span style="color: #0000ff;">null</span><span style="color: #000000;">) {
            state.mSum </span>+=<span style="color: #000000;"> o;
            state.mCount</span>++<span style="color: #000000;">;
        }
        </span><span style="color: #0000ff;">return</span> <span style="color: #0000ff;">true</span><span style="color: #000000;">;
    }

    </span><span style="color: #008000;">/**</span><span style="color: #008000;">
     * terminatePartial无参数，其为iterate函数遍历结束后，返回轮转数据， * terminatePartial类似于hadoop的Combiner * * </span><span style="color: #808080;">@return</span>
     <span style="color: #008000;">*/</span>

    <span style="color: #0000ff;">public</span><span style="color: #000000;"> AvgState terminatePartial() {
        </span><span style="color: #008000;">//</span><span style="color: #008000;"> combiner</span>
        <span style="color: #0000ff;">return</span> state.mCount == 0 ? <span style="color: #0000ff;">null</span><span style="color: #000000;"> : state;
    }

    </span><span style="color: #008000;">/**</span><span style="color: #008000;">
     * merge接收terminatePartial的返回结果，进行数据merge操作，其返回类型为boolean * * </span><span style="color: #808080;">@param</span><span style="color: #008000;"> o * </span><span style="color: #808080;">@return</span>
     <span style="color: #008000;">*/</span>

    <span style="color: #0000ff;">public</span> <span style="color: #0000ff;">boolean</span><span style="color: #000000;"> merge(AvgState avgState) {
        </span><span style="color: #0000ff;">if</span> (avgState != <span style="color: #0000ff;">null</span><span style="color: #000000;">) {
            state.mCount </span>+=<span style="color: #000000;"> avgState.mCount;
            state.mSum </span>+=<span style="color: #000000;"> avgState.mSum;
        }
        </span><span style="color: #0000ff;">return</span> <span style="color: #0000ff;">true</span><span style="color: #000000;">;
    }

    </span><span style="color: #008000;">/**</span><span style="color: #008000;">
     * terminate返回最终的聚集函数结果 * * </span><span style="color: #808080;">@return</span>
     <span style="color: #008000;">*/</span>
    <span style="color: #0000ff;">public</span><span style="color: #000000;"> Double terminate() {
        </span><span style="color: #0000ff;">return</span> state.mCount == 0 ? <span style="color: #0000ff;">null</span> : Double.valueOf(state.mSum /<span style="color: #000000;"> state.mCount);
    }
}

}

实现sum

package com.xxx.udf;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;

import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;

import org.apache.hadoop.hive.ql.metadata.HiveException;

import org.apache.hadoop.hive.ql.parse.SemanticException;

import org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver;

import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;

import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;

import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;

import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;

import org.apache.hadoop.io.DoubleWritable;

import org.apache.hadoop.io.LongWritable;
public class Test extends AbstractGenericUDAFResolver {
</span><span style="color: #008080;">/*</span><span style="color: #008080;">*
 * 获取处理逻辑类
 * @param info
 * @return
 * @throws SemanticException
 </span><span style="color: #008080;">*/</span>
<span style="color: #008000;">@Override</span>
<span style="color: #0000ff;">public</span> GenericUDAFEvaluator getEvaluator(TypeInfo<span style="color: #ff0000;">[]</span><span style="color: #000000;"> info) throws SemanticException {
    </span><span style="color: #808080;">//</span><span style="color: #000000;">判断输入参数是否合法,参数个数，参数类型
    </span><span style="color: #0000ff;">if</span> (info.length <span style="color: #808080;">!=</span> <span style="color: #800000; font-weight: bold;">1</span><span style="color: #000000;">) {
        throw new UDFArgumentLengthException("输入参数个数非法，一个参数");
    }

    </span><span style="color: #0000ff;">return</span><span style="color: #000000;"> new GenericEvaluate();
}


</span><span style="color: #808080;">//</span><span style="color: #000000;">处理逻辑类
</span><span style="color: #0000ff;">public</span><span style="color: #000000;"> static class GenericEvaluate extends GenericUDAFEvaluator {
    private PrimitiveObjectInspector input;
    private DoubleWritable result ;                   </span><span style="color: #808080;">//</span><span style="color: #000000;">保存最终结果
    private MyAggregationBuffer myAggregationBuffer;  </span><span style="color: #808080;">//</span><span style="color: #000000;">自定义聚合列，保存临时结果

    </span><span style="color: #808080;">//</span><span style="color: #000000;">自定义AggregationBuffer
    </span><span style="color: #0000ff;">public</span><span style="color: #000000;"> static class MyAggregationBuffer implements AggregationBuffer {
        </span><span style="color: #0000ff;">Double</span> <span style="color: #ff00ff;">sum</span><span style="color: #000000;">;
    }

    </span><span style="color: #008000;">@Override</span>  <span style="color: #808080;">//</span><span style="color: #000000;">指定返回类型
    </span><span style="color: #0000ff;">public</span> ObjectInspector init(Mode m, ObjectInspector<span style="color: #ff0000;">[]</span><span style="color: #000000;"> parameters) throws HiveException {
        super.init(m, parameters);
        result </span><span style="color: #808080;">=</span> new DoubleWritable(<span style="color: #800000; font-weight: bold;">0</span><span style="color: #000000;">);
        input </span><span style="color: #808080;">=</span> (PrimitiveObjectInspector) parameters<span style="color: #ff0000;">[</span><span style="color: #ff0000;">0</span><span style="color: #ff0000;">]</span><span style="color: #000000;">;
        </span><span style="color: #808080;">//</span><span style="color: #000000;"> 指定返回结果类型
        </span><span style="color: #0000ff;">return</span><span style="color: #000000;"> PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
    }

    </span><span style="color: #008000;">@Override</span>   <span style="color: #808080;">//</span><span style="color: #000000;">获得一个聚合的缓冲对象，每个map执行一次
    </span><span style="color: #0000ff;">public</span><span style="color: #000000;"> AggregationBuffer getNewAggregationBuffer() throws HiveException {
        MyAggregationBuffer myAggregationBuffer </span><span style="color: #808080;">=</span><span style="color: #000000;"> new MyAggregationBuffer();
        reset(myAggregationBuffer);  </span><span style="color: #808080;">//</span><span style="color: #000000;"> 重置聚合值
        </span><span style="color: #0000ff;">return</span><span style="color: #000000;"> myAggregationBuffer;
    }

    </span><span style="color: #008000;">@Override</span>
    <span style="color: #0000ff;">public</span><span style="color: #000000;"> void reset(AggregationBuffer agg) throws HiveException {
        MyAggregationBuffer newAgg </span><span style="color: #808080;">=</span><span style="color: #000000;"> (MyAggregationBuffer) agg;
        newAgg.</span><span style="color: #ff00ff;">sum</span> <span style="color: #808080;">=</span> <span style="color: #800000; font-weight: bold;">0.0</span><span style="color: #000000;">;
    }

    </span><span style="color: #008000;">@Override</span>  <span style="color: #808080;">//</span><span style="color: #000000;"> 传入参数值聚合
    </span><span style="color: #0000ff;">public</span> void iterate(AggregationBuffer agg, Object<span style="color: #ff0000;">[]</span><span style="color: #000000;"> parameters) throws HiveException {
        MyAggregationBuffer myAgg </span><span style="color: #808080;">=</span><span style="color: #000000;"> (MyAggregationBuffer) agg;
        </span><span style="color: #0000ff;">double</span> inputNum <span style="color: #808080;">=</span> PrimitiveObjectInspectorUtils.getDouble(parameters<span style="color: #ff0000;">[</span><span style="color: #ff0000;">0</span><span style="color: #ff0000;">]</span><span style="color: #000000;">, input);
        myAgg.</span><span style="color: #ff00ff;">sum</span> <span style="color: #808080;">+=</span><span style="color: #000000;"> inputNum;
    }

    </span><span style="color: #008000;">@Override</span>  <span style="color: #808080;">//</span>
    <span style="color: #0000ff;">public</span><span style="color: #000000;"> Object terminatePartial(AggregationBuffer agg) throws HiveException {
        MyAggregationBuffer newAgg </span><span style="color: #808080;">=</span><span style="color: #000000;"> (MyAggregationBuffer) agg;
        result.</span><span style="color: #0000ff;">set</span>(newAgg.<span style="color: #ff00ff;">sum</span><span style="color: #000000;">);
        </span><span style="color: #0000ff;">return</span><span style="color: #000000;"> result;
    }

    </span><span style="color: #008000;">@Override</span>  <span style="color: #808080;">//</span><span style="color: #000000;"> 合并  
    </span><span style="color: #0000ff;">public</span><span style="color: #000000;"> void merge(AggregationBuffer agg, Object partial) throws HiveException {
        </span><span style="color: #0000ff;">double</span> inputNum <span style="color: #808080;">=</span><span style="color: #000000;"> PrimitiveObjectInspectorUtils.getDouble(partial, input);
        MyAggregationBuffer newAgg </span><span style="color: #808080;">=</span><span style="color: #000000;"> (MyAggregationBuffer) agg;
        newAgg.</span><span style="color: #ff00ff;">sum</span> <span style="color: #808080;">+=</span><span style="color: #000000;"> inputNum;
    }

    </span><span style="color: #008000;">@Override</span>  <span style="color: #808080;">//</span><span style="color: #000000;">输出最终结果
    </span><span style="color: #0000ff;">public</span><span style="color: #000000;"> Object terminate(AggregationBuffer agg) throws HiveException {
        MyAggregationBuffer aggregationBuffer </span><span style="color: #808080;">=</span><span style="color: #000000;"> (MyAggregationBuffer) agg;
        result.</span><span style="color: #0000ff;">set</span>(aggregationBuffer.<span style="color: #ff00ff;">sum</span><span style="color: #000000;">);
        </span><span style="color: #0000ff;">return</span><span style="color: #000000;"> result;
    }
}

}

3.打包

mvn clean package

4.创建临时函数

add jar /home/xxx/yf/my_avg.jar;
create temporary function my_avg as 'com.xxx.udf.UDTFExplode';
select my_avg() from dual;
drop temporary function my_avg;

三、UDTF函数编写

1.步骤

1.继承GenericUDTF
2.重写initialize、process方法
  initialize初始化校验参数是否正确、
  process处理返回结果、
  forward将结果返回

2.案例

将字符串按照元素索引分别输出，如：‘a,c,b’ -- > a,1 c,2 b,3

package com.suning.udf;
import java.util.ArrayList;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;

import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;

import org.apache.hadoop.hive.ql.metadata.HiveException;

import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;

import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;

import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;

import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
public class UDTFExplode extends GenericUDTF {
</span><span style="color: #008000;">@Override</span>
<span style="color: #0000ff;">public</span> void <span style="color: #0000ff;">close</span><span style="color: #000000;">() throws HiveException {
    </span><span style="color: #808080;">//</span> TODO Auto<span style="color: #808080;">-</span><span style="color: #000000;">generated method stub

}


</span><span style="color: #008000;">@Override</span>
<span style="color: #0000ff;">public</span> StructObjectInspector initialize(ObjectInspector<span style="color: #ff0000;">[]</span><span style="color: #000000;"> args) throws UDFArgumentException {
    </span><span style="color: #0000ff;">if</span> (args.length <span style="color: #808080;">!=</span> <span style="color: #800000; font-weight: bold;">1</span><span style="color: #000000;">) {
        throw new UDFArgumentLengthException("ExplodeMap takes </span><span style="color: #0000ff;">only</span><span style="color: #000000;"> one argument");
    }
    </span><span style="color: #0000ff;">if</span> (args<span style="color: #ff0000;">[</span><span style="color: #ff0000;">0</span><span style="color: #ff0000;">]</span>.getCategory() <span style="color: #808080;">!=</span><span style="color: #000000;"> ObjectInspector.Category.PRIMITIVE) {
        throw new UDFArgumentException("ExplodeMap takes string </span><span style="color: #0000ff;">as</span><span style="color: #000000;"> a parameter");
    }

    ArrayList</span><span style="color: #808080;">&lt;</span>String<span style="color: #808080;">&gt;</span> fieldNames <span style="color: #808080;">=</span> new ArrayList<span style="color: #808080;">&lt;</span>String<span style="color: #808080;">&gt;</span><span style="color: #000000;">();
    ArrayList</span><span style="color: #808080;">&lt;</span>ObjectInspector<span style="color: #808080;">&gt;</span> fieldOIs <span style="color: #808080;">=</span> new ArrayList<span style="color: #808080;">&lt;</span>ObjectInspector<span style="color: #808080;">&gt;</span><span style="color: #000000;">();
    fieldNames.</span><span style="color: #0000ff;">add</span><span style="color: #000000;">("col1");
    fieldOIs.</span><span style="color: #0000ff;">add</span><span style="color: #000000;">(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    fieldNames.</span><span style="color: #0000ff;">add</span><span style="color: #000000;">("col2");
    fieldOIs.</span><span style="color: #0000ff;">add</span><span style="color: #000000;">(PrimitiveObjectInspectorFactory.javaStringObjectInspector);

    </span><span style="color: #0000ff;">return</span><span style="color: #000000;"> ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}

</span><span style="color: #008000;">@Override</span>
<span style="color: #0000ff;">public</span> void process(Object<span style="color: #ff0000;">[]</span><span style="color: #000000;"> args) throws HiveException {
    </span><span style="color: #808080;">//</span> TODO Auto<span style="color: #808080;">-</span><span style="color: #000000;">generated method stub
    String input </span><span style="color: #808080;">=</span> args<span style="color: #ff0000;">[</span><span style="color: #ff0000;">0</span><span style="color: #ff0000;">]</span><span style="color: #000000;">.toString();
    String</span><span style="color: #ff0000;">[]</span> test <span style="color: #808080;">=</span><span style="color: #000000;"> input.split(",");
    </span><span style="color: #0000ff;">for</span> (<span style="color: #0000ff;">int</span> i <span style="color: #808080;">=</span> <span style="color: #800000; font-weight: bold;">0</span>; i <span style="color: #808080;">&lt;</span> test.length; i<span style="color: #808080;">++</span><span style="color: #000000;">) {
        try {
            String</span><span style="color: #ff0000;">[]</span> result <span style="color: #808080;">=</span> (test<span style="color: #ff0000;">[</span><span style="color: #ff0000;">i</span><span style="color: #ff0000;">]</span><span style="color: #808080;">+</span>":"<span style="color: #808080;">+</span>String.valueOf(i<span style="color: #808080;">+</span><span style="color: #800000; font-weight: bold;">1</span><span style="color: #000000;">)).split(":");
            forward(result);
        } catch (Exception e) {
            </span><span style="color: #0000ff;">continue</span><span style="color: #000000;">;
        }
    }
}

}

3.打包

mvn clean package

4.创建临时函数

add jar /home/xxx/yf/str_index.jar;
create temporary function str_index as 'com.xxx.udf.UDTFExplode';
select str_index("a,c,b") from dual;
drop temporary function str_index;

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
<span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>spark<span style="color: #808080;">-</span>hive<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
<span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>spark<span style="color: #808080;">-</span>hive<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
<span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">1.0</span><span style="color: #808080;">-</span>SNAPSHOT<span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>

<span style="color: #808080;">&lt;</span>properties<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>project.build.sourceEncoding<span style="color: #808080;">&gt;</span>UTF<span style="color: #808080;">-</span><span style="color: #800000; font-weight: bold;">8</span><span style="color: #808080;">&lt;/</span>project.build.sourceEncoding<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>project.reporting.outputEncoding<span style="color: #808080;">&gt;</span>UTF<span style="color: #808080;">-</span><span style="color: #800000; font-weight: bold;">8</span><span style="color: #808080;">&lt;/</span>project.reporting.outputEncoding<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>scala.version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">2.11</span>.<span style="color: #800000; font-weight: bold;">8</span><span style="color: #808080;">&lt;/</span>scala.version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>spark.version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">2.1</span>.<span style="color: #800000; font-weight: bold;">0.9</span><span style="color: #808080;">&lt;/</span>spark.version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>spark.artifactId.version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">2.11</span><span style="color: #808080;">&lt;/</span>spark.artifactId.version<span style="color: #808080;">&gt;</span>
<span style="color: #808080;">&lt;/</span>properties<span style="color: #808080;">&gt;</span>
<span style="color: #808080;">&lt;</span>dependencies<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>commons<span style="color: #808080;">-</span>logging<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>commons<span style="color: #808080;">-</span>logging<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">1.1</span>.<span style="color: #800000; font-weight: bold;">1</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>type<span style="color: #808080;">&gt;</span>jar<span style="color: #808080;">&lt;/</span>type<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>

    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.commons<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>commons<span style="color: #808080;">-</span>lang3<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">3.1</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>

    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>log4j<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>log4j<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">1.2</span>.<span style="color: #800000; font-weight: bold;">17</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>

    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.hadoop<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>hadoop<span style="color: #808080;">-</span>common<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">2.6</span>.<span style="color: #800000; font-weight: bold;">2</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>mysql<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>mysql<span style="color: #808080;">-</span>connector<span style="color: #808080;">-</span>java<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">5.1</span>.<span style="color: #800000; font-weight: bold;">21</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.spark<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>spark<span style="color: #808080;">-</span>core_2.<span style="color: #800000; font-weight: bold;">11</span><span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">2.1</span>.<span style="color: #800000; font-weight: bold;">0</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.spark<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>spark<span style="color: #808080;">-</span>streaming_2.<span style="color: #800000; font-weight: bold;">11</span><span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">2.1</span>.<span style="color: #800000; font-weight: bold;">0</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>

    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.spark<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>spark<span style="color: #808080;">-</span>streaming<span style="color: #808080;">-</span>kafka<span style="color: #808080;">-</span><span style="color: #800000; font-weight: bold;">0</span><span style="color: #808080;">-</span>8_<span style="color: #800000; font-weight: bold;">2.11</span><span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">2.1</span>.<span style="color: #800000; font-weight: bold;">0</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>com.google.code.gson<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>gson<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">2.8</span>.<span style="color: #800000; font-weight: bold;">2</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>

    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.spark<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>spark<span style="color: #808080;">-</span>sql_2.<span style="color: #800000; font-weight: bold;">11</span><span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">2.1</span>.<span style="color: #800000; font-weight: bold;">0</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>

    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>com.alibaba<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>fastjson<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">1.2</span>.<span style="color: #800000; font-weight: bold;">29</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.spark<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>spark<span style="color: #808080;">-</span>hive_${spark.artifactId.version}<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span>${spark.version}<span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>scope<span style="color: #808080;">&gt;</span>provided<span style="color: #808080;">&lt;/</span>scope<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>!<span style="color: #008080;">--</span><span style="color: #008080;">flink dependency--&gt;</span>
    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.flink<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>flink<span style="color: #808080;">-</span>java<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">1.5</span>.<span style="color: #800000; font-weight: bold;">0</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.flink<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>flink<span style="color: #808080;">-</span>streaming<span style="color: #808080;">-</span>java_2.<span style="color: #800000; font-weight: bold;">11</span><span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">1.5</span>.<span style="color: #800000; font-weight: bold;">0</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.flink<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>flink<span style="color: #808080;">-</span>clients_2.<span style="color: #800000; font-weight: bold;">11</span><span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">1.5</span>.<span style="color: #800000; font-weight: bold;">0</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.flink<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>flink<span style="color: #808080;">-</span>connector<span style="color: #808080;">-</span>wikiedits_2.<span style="color: #800000; font-weight: bold;">11</span><span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">1.5</span>.<span style="color: #800000; font-weight: bold;">0</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>!<span style="color: #008080;">--</span><span style="color: #008080;">hbase dependency--&gt;</span>
    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.hbase<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>hbase<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">0.98</span>.<span style="color: #800000; font-weight: bold;">8</span><span style="color: #808080;">-</span>hadoop2<span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>type<span style="color: #808080;">&gt;</span>pom<span style="color: #808080;">&lt;/</span>type<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.hbase<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>hbase<span style="color: #808080;">-</span>client<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">0.98</span>.<span style="color: #800000; font-weight: bold;">8</span><span style="color: #808080;">-</span>hadoop2<span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.hbase<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>hbase<span style="color: #808080;">-</span>common<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">0.98</span>.<span style="color: #800000; font-weight: bold;">8</span><span style="color: #808080;">-</span>hadoop2<span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>dependency<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.hbase<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>hbase<span style="color: #808080;">-</span>server<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">0.98</span>.<span style="color: #800000; font-weight: bold;">8</span><span style="color: #808080;">-</span>hadoop2<span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;/</span>dependency<span style="color: #808080;">&gt;</span>
<span style="color: #808080;">&lt;/</span>dependencies<span style="color: #808080;">&gt;</span>
<span style="color: #808080;">&lt;</span>build<span style="color: #808080;">&gt;</span>
    <span style="color: #808080;">&lt;</span>plugins<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>plugin<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>maven<span style="color: #808080;">-</span>assembly<span style="color: #808080;">-</span>plugin<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>configuration<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;</span>descriptorRefs<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;</span>descriptorRef<span style="color: #808080;">&gt;</span>jar<span style="color: #808080;">-</span><span style="color: #0000ff;">with</span><span style="color: #808080;">-</span>dependencies<span style="color: #808080;">&lt;/</span>descriptorRef<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;/</span>descriptorRefs<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;/</span>configuration<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;/</span>plugin<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>plugin<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.codehaus.mojo<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>build<span style="color: #808080;">-</span>helper<span style="color: #808080;">-</span>maven<span style="color: #808080;">-</span>plugin<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">1.8</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>executions<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;</span>execution<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;</span>id<span style="color: #808080;">&gt;</span><span style="color: #0000ff;">add</span><span style="color: #808080;">-</span>source<span style="color: #808080;">&lt;/</span>id<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;</span>phase<span style="color: #808080;">&gt;</span>generate<span style="color: #808080;">-</span>sources<span style="color: #808080;">&lt;/</span>phase<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;</span>goals<span style="color: #808080;">&gt;</span>
                        <span style="color: #808080;">&lt;</span>goal<span style="color: #808080;">&gt;</span><span style="color: #0000ff;">add</span><span style="color: #808080;">-</span>source<span style="color: #808080;">&lt;/</span>goal<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;/</span>goals<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;</span>configuration<span style="color: #808080;">&gt;</span>
                        <span style="color: #808080;">&lt;</span>sources<span style="color: #808080;">&gt;</span>
                            <span style="color: #808080;">&lt;</span>source<span style="color: #808080;">&gt;</span>src<span style="color: #808080;">/</span>main<span style="color: #808080;">/</span>scala<span style="color: #808080;">&lt;/</span>source<span style="color: #808080;">&gt;</span>
                            <span style="color: #808080;">&lt;</span>source<span style="color: #808080;">&gt;</span>src<span style="color: #808080;">/</span>test<span style="color: #808080;">/</span>scala<span style="color: #808080;">&lt;/</span>source<span style="color: #808080;">&gt;</span>
                        <span style="color: #808080;">&lt;/</span>sources<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;/</span>configuration<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;/</span>execution<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;</span>execution<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;</span>id<span style="color: #808080;">&gt;</span><span style="color: #0000ff;">add</span><span style="color: #808080;">-</span>test<span style="color: #808080;">-</span>source<span style="color: #808080;">&lt;/</span>id<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;</span>phase<span style="color: #808080;">&gt;</span>generate<span style="color: #808080;">-</span>sources<span style="color: #808080;">&lt;/</span>phase<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;</span>goals<span style="color: #808080;">&gt;</span>
                        <span style="color: #808080;">&lt;</span>goal<span style="color: #808080;">&gt;</span><span style="color: #0000ff;">add</span><span style="color: #808080;">-</span>test<span style="color: #808080;">-</span>source<span style="color: #808080;">&lt;/</span>goal<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;/</span>goals<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;</span>configuration<span style="color: #808080;">&gt;</span>
                        <span style="color: #808080;">&lt;</span>sources<span style="color: #808080;">&gt;</span>
                            <span style="color: #808080;">&lt;</span>source<span style="color: #808080;">&gt;</span>src<span style="color: #808080;">/</span>test<span style="color: #808080;">/</span>scala<span style="color: #808080;">&lt;/</span>source<span style="color: #808080;">&gt;</span>
                        <span style="color: #808080;">&lt;/</span>sources<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;/</span>configuration<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;/</span>execution<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;/</span>executions<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;/</span>plugin<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>plugin<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.maven.plugins<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>maven<span style="color: #808080;">-</span>compiler<span style="color: #808080;">-</span>plugin<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">2.3</span>.<span style="color: #800000; font-weight: bold;">2</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>configuration<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;</span>source<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">1.7</span><span style="color: #808080;">&lt;/</span>source<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;</span>target<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">1.7</span><span style="color: #808080;">&lt;/</span>target<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;</span>encoding<span style="color: #808080;">&gt;</span>${project.build.sourceEncoding}<span style="color: #808080;">&lt;/</span>encoding<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;/</span>configuration<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;/</span>plugin<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>plugin<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.scala<span style="color: #808080;">-</span>tools<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>maven<span style="color: #808080;">-</span>scala<span style="color: #808080;">-</span>plugin<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>executions<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;</span>execution<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;</span>goals<span style="color: #808080;">&gt;</span>
                        <span style="color: #808080;">&lt;</span>goal<span style="color: #808080;">&gt;</span>compile<span style="color: #808080;">&lt;/</span>goal<span style="color: #808080;">&gt;</span>
                        <span style="color: #808080;">&lt;</span>goal<span style="color: #808080;">&gt;</span><span style="color: #0000ff;">add</span><span style="color: #808080;">-</span>source<span style="color: #808080;">&lt;/</span>goal<span style="color: #808080;">&gt;</span>
                        <span style="color: #808080;">&lt;</span>goal<span style="color: #808080;">&gt;</span>testCompile<span style="color: #808080;">&lt;/</span>goal<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;/</span>goals<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;/</span>execution<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;/</span>executions<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>configuration<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;</span>scalaVersion<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">2.11</span>.<span style="color: #800000; font-weight: bold;">8</span><span style="color: #808080;">&lt;/</span>scalaVersion<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;</span>sourceDir<span style="color: #808080;">&gt;</span>src<span style="color: #808080;">/</span>main<span style="color: #808080;">/</span>scala<span style="color: #808080;">&lt;/</span>sourceDir<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;</span>jvmArgs<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;</span>jvmArg<span style="color: #808080;">&gt;-</span>Xms64m<span style="color: #808080;">&lt;/</span>jvmArg<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;</span>jvmArg<span style="color: #808080;">&gt;-</span>Xmx1024m<span style="color: #808080;">&lt;/</span>jvmArg<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;/</span>jvmArgs<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;/</span>configuration<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;/</span>plugin<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>plugin<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.maven.plugins<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>maven<span style="color: #808080;">-</span>release<span style="color: #808080;">-</span>plugin<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">2.5</span>.<span style="color: #800000; font-weight: bold;">3</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;/</span>plugin<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>plugin<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.maven.plugins<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>maven<span style="color: #808080;">-</span>deploy<span style="color: #808080;">-</span>plugin<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>configuration<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;</span>skip<span style="color: #808080;">&gt;</span>false<span style="color: #808080;">&lt;/</span>skip<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;/</span>configuration<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;/</span>plugin<span style="color: #808080;">&gt;</span>
        <span style="color: #808080;">&lt;</span>plugin<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>groupId<span style="color: #808080;">&gt;</span>org.apache.maven.plugins<span style="color: #808080;">&lt;/</span>groupId<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>artifactId<span style="color: #808080;">&gt;</span>maven<span style="color: #808080;">-</span>shade<span style="color: #808080;">-</span>plugin<span style="color: #808080;">&lt;/</span>artifactId<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>version<span style="color: #808080;">&gt;</span><span style="color: #800000; font-weight: bold;">2.4</span>.<span style="color: #800000; font-weight: bold;">1</span><span style="color: #808080;">&lt;/</span>version<span style="color: #808080;">&gt;</span>
            <span style="color: #808080;">&lt;</span>executions<span style="color: #808080;">&gt;</span>
                <span style="color: #808080;">&lt;</span>execution<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;</span>phase<span style="color: #808080;">&gt;</span>package<span style="color: #808080;">&lt;/</span>phase<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;</span>goals<span style="color: #808080;">&gt;</span>
                        <span style="color: #808080;">&lt;</span>goal<span style="color: #808080;">&gt;</span>shade<span style="color: #808080;">&lt;/</span>goal<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;/</span>goals<span style="color: #808080;">&gt;</span>
                    <span style="color: #808080;">&lt;</span>configuration<span style="color: #808080;">&gt;</span>
                        <span style="color: #808080;">&lt;</span>filters<span style="color: #808080;">&gt;</span>
                            <span style="color: #808080;">&lt;</span>filter<span style="color: #808080;">&gt;</span>
                                <span style="color: #808080;">&lt;</span>artifact<span style="color: #808080;">&gt;*</span>:<span style="color: #808080;">*&lt;/</span>artifact<span style="color: #808080;">&gt;</span>
                                <span style="color: #808080;">&lt;</span>excludes<span style="color: #808080;">&gt;</span>
                                    <span style="color: #808080;">&lt;</span>exclude<span style="color: #808080;">&gt;</span>META<span style="color: #808080;">-</span>INF<span style="color: #008080;">/*</span><span style="color: #008080;">.SF&lt;/exclude&gt;
                                    org.apache.hive                          &lt;exclude&gt;META-INF/*.DSA&lt;/exclude&gt;
                                    &lt;exclude&gt;META-INF/*.RSA&lt;/exclude&gt;
                                &lt;/excludes&gt;
                            &lt;/filter&gt;
                        &lt;/filters&gt;
                        &lt;minimizeJar&gt;false&lt;/minimizeJar&gt;
                    &lt;/configuration&gt;
                &lt;/execution&gt;
            &lt;/executions&gt;
        &lt;/plugin&gt;
    &lt;/plugins&gt;
    &lt;resources&gt;
        &lt;resource&gt;
            &lt;directory&gt;src/main/resources&lt;/directory&gt;
            &lt;filtering&gt;true&lt;/filtering&gt;
        &lt;/resource&gt;
        &lt;resource&gt;
            &lt;directory&gt;src/main/resources/${profiles.active}&lt;/directory&gt;
        &lt;/resource&gt;
    &lt;/resources&gt;

    &lt;!-- 修复 Plugin execution not covered by lifecycle configuration --&gt;
    &lt;pluginManagement&gt;
        &lt;plugins&gt;
            &lt;plugin&gt;
                &lt;groupId&gt;org.eclipse.m2e&lt;/groupId&gt;
                &lt;artifactId&gt;lifecycle-mapping&lt;/artifactId&gt;
                &lt;version&gt;1.0.0&lt;/version&gt;
                &lt;configuration&gt;
                    &lt;lifecycleMappingMetadata&gt;
                        &lt;pluginExecutions&gt;
                            &lt;pluginExecution&gt;
                                &lt;pluginExecutionFilter&gt;
                                    &lt;groupId&gt;org.codehaus.mojo&lt;/groupId&gt;
                                    &lt;artifactId&gt;build-helper-maven-plugin&lt;/artifactId&gt;
                                    &lt;versionRange&gt;[1.8,)&lt;/versionRange&gt;
                                    &lt;goals&gt;
                                        &lt;goal&gt;add-source&lt;/goal&gt;
                                        &lt;goal&gt;add-test-source&lt;/goal&gt;
                                    &lt;/goals&gt;
                                &lt;/pluginExecutionFilter&gt;
                                &lt;action&gt;
                                    &lt;ignore&gt;&lt;/ignore&gt;
                                &lt;/action&gt;
                            &lt;/pluginExecution&gt;

                            &lt;pluginExecution&gt;
                                &lt;pluginExecutionFilter&gt;
                                    &lt;groupId&gt;org.scala-tools&lt;/groupId&gt;
                                    &lt;artifactId&gt;maven-scala-plugin&lt;/artifactId&gt;
                                    &lt;versionRange&gt;[1.8,)&lt;/versionRange&gt;
                                    &lt;goals&gt;
                                        &lt;goal&gt;compile&lt;/goal&gt;
                                        &lt;goal&gt;add-source&lt;/goal&gt;
                                        &lt;goal&gt;testCompile&lt;/goal&gt;
                                    &lt;/goals&gt;
                                &lt;/pluginExecutionFilter&gt;
                                &lt;action&gt;
                                    &lt;ignore&gt;&lt;/ignore&gt;
                                &lt;/action&gt;
                            &lt;/pluginExecution&gt;
                        &lt;/pluginExecutions&gt;
                    &lt;/lifecycleMappingMetadata&gt;
                &lt;/configuration&gt;
            &lt;/plugin&gt;
        &lt;/plugins&gt;
    &lt;/pluginManagement&gt;
&lt;/build&gt;

</project>

View Code

分类: hive

<div id="blog_post_info">

好文要顶已关注收藏该文

问题不大1
关注 - 10
粉丝 - 2

我在关注他取消关注

<div class="clear"></div>
<div id="post_next_prev">

<a href="https://www.cnblogs.com/yin-fei/p/10748505.html" class="p_n_p_prefix">« </a> 上一篇：    <a href="https://www.cnblogs.com/yin-fei/p/10748505.html" title="发布于 2019-04-22 09:52">使用sparksql往kafka推送数据</a>
<br>
<a href="https://www.cnblogs.com/yin-fei/p/10751608.html" class="p_n_p_prefix">» </a> 下一篇：    <a href="https://www.cnblogs.com/yin-fei/p/10751608.html" title="发布于 2019-04-22 17:56">hive提前过滤重要性</a>

posted @ 2019-04-22 09:55 问题不大1 阅读(522) 评论(0) 编辑收藏

posted @ 2020-04-01 14:45 Baron_ND 阅读(332) 评论(0) 收藏举报

刷新页面返回顶部

Baron_ND

UDF、UDAF、UDTF

一、UDF函数编写

1.步骤

2.案例

3.打成jar包上传

4.在hive中创建临时函数

二、UDAF函数编写

1.步骤

2.案例

3.打包

4.创建临时函数

三、UDTF函数编写

1.步骤

2.案例

3.打包

4.创建临时函数

公告