Spark基于自定义聚合函数实现【列转行、行转列】

一.分析

　　Spark提供了非常丰富的算子，可以实现大部分的逻辑处理，例如，要实现行转列，可以用hiveContext中支持的concat_ws(',', collect_set('字段'))实现。但是这有明显的局限性【sqlContext不支持】，因此，基于编码逻辑或自定义聚合函数实现相同的逻辑就显得非常重要了。

二.列转行代码实现　

package utils
import com.hankcs.hanlp.tokenizer.StandardTokenizer
import org.apache.log4j.{Level, Logger}
import org.apache.spark.sql.{SparkSession, Row}
import org.apache.spark.sql.types.{StringType, StructType, StructField}
/**
  * Created by Administrator on 2019/12/17.
  */
object Column2Row {
  /**
    * 设置日志级别
    */
  Logger.getLogger("org").setLevel(Level.WARN)
  def main(args: Array[String]) {
    val spark = SparkSession.builder().master("local[2]").appName(s"${this.getClass.getSimpleName}").getOrCreate()
    val sc = spark.sparkContext
    val sqlContext = spark.sqlContext

    val array : Array[String] = Array("spark-高性能大数据解决方案", "spark-机器学习图计算", "solr-搜索引擎应用广泛", "solr-ES灵活高效")
    val rdd = sc.parallelize(array)

    val termRdd = rdd.map(row => { // 标准分词，挂载Hanlp分词器
    var result = ""
      val type_content = row.split("-")
      val termList = StandardTokenizer.segment(type_content(1))
      for(i <- 0 until termList.size()){
        val term = termList.get(i)
        if(!term.nature.name.contains("w") && !term.nature.name().contains("u") && !term.nature.name().contains("m")){
          if(term.word.length > 1){
            result += term.word + " "
          }
        }
      }
      Row(type_content(0),result)
    })

    val structType = StructType(Array(
      StructField("arth_type", StringType, true),
      StructField("content", StringType, true)
    ))

    val termDF = sqlContext.createDataFrame(termRdd,structType)
    termDF.show(false)
    /**
      * 列转行
      */
    val termCheckDF = termDF.rdd.flatMap(row =>{
      val arth_type = row.getAs[String]("arth_type")
      val content = row.getAs[String]("content")
      var res = Seq[Row]()
      val content_array = content.split(" ")
      for(con <- content_array){
        res = res :+ Row(arth_type,con)
      }
      res
    }).collect()

    val termListDF = sqlContext.createDataFrame(sc.parallelize(termCheckDF), structType)
    termListDF.show(false)

    sc.stop()
  }
}

三.列转行执行结果

　　列转行之前：

　　

　　列转行：

　　

四.行转列代码实现

package test

import org.apache.log4j.{Level, Logger}
import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
import org.apache.spark.sql.types._
import org.apache.spark.sql.{Row, SparkSession}

/**
  * 自定义聚合函数实现行转列
  */
object AverageUserDefinedAggregateFunction extends UserDefinedAggregateFunction{
  //聚合函数输入数据结构
  override def inputSchema:StructType = StructType(StructField("input", StringType) :: Nil)

  //缓存区数据结构
  override def bufferSchema: StructType = StructType(StructField("result", StringType) :: Nil)

  //结果数据结构
  override def dataType : DataType = StringType

  // 是否具有唯一性
  override def deterministic : Boolean = true

  //初始化
  override def initialize(buffer : MutableAggregationBuffer) : Unit = {
    buffer(0) = ""
  }

  //数据处理 : 必写，其它方法可选，使用默认
  override def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
    if(input.isNullAt(0)) return
    if(buffer.getString(0) == null || buffer.getString(0).equals("")){
      buffer(0) = input.getString(0) //拼接字符串
    }else{
      buffer(0) = buffer.getString(0) + "," + input.getString(0) //拼接字符串
    }
  }

  //合并
  override def merge(bufferLeft: MutableAggregationBuffer, bufferRight: Row): Unit ={
    if(bufferLeft(0) == null || bufferLeft(0).equals("")){
      bufferLeft(0) = bufferRight.getString(0) //拼接字符串
    }else{
      bufferLeft(0) = bufferLeft(0) + "," + bufferRight.getString(0) //拼接字符串
    }
  }

  //计算结果
  override def evaluate(buffer: Row): Any  = buffer.getString(0)
}

/**
  * Created by Administrator on 2019/12/17.
  */
object Row2Columns {
  /**
    * 设置日志级别
    */
  Logger.getLogger("org").setLevel(Level.WARN)
  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder().master("local[2]").appName(s"${this.getClass.getSimpleName}").getOrCreate()
    val sc = spark.sparkContext
    val sqlContext = spark.sqlContext

    val array : Array[String] = Array("大数据-Spark","大数据-Hadoop","大数据-Flink","搜索引擎-Solr","搜索引擎-ES")

    val termRdd = sc.parallelize(array).map(row => { // 标准分词，挂载Hanlp分词器
      val content = row.split("-")
      Row(content(0), content(1))
    })

    val structType = StructType(Array(
      StructField("arth_type", StringType, true),
      StructField("content", StringType, true)
    ))

    val termDF = sqlContext.createDataFrame(termRdd,structType)
    termDF.show()
    termDF.createOrReplaceTempView("term")

    /**
      * 注册udaf
      */
    spark.udf.register("concat_ws", AverageUserDefinedAggregateFunction)
    spark.sql("select arth_type,concat_ws(content) content from term group by arth_type").show()
  }
}

五.行转列执行结果

　　行转列之前：

　　

　　行转列：