Spark的dataframe转rdd为实体类通用工具类

需求解决问题

当每次读取hive表或者其他数据源，获取数据，相对其进行rdd操作，遇到任何类都需要df.rdd(row>row.getstring(0))去获取，就很麻烦，所以可以实现个通用的转换方式

1.dataframe转为rdd通用方法

  /**
   * df转为rdd 通用方法
   *
   * @param frame
   * @return
   */
  def dataFrameToRdd(frame: DataFrame): RDD[Array[Any]] = {
    val fields: Array[StructField] = frame.schema.toArray
    val zipData: Array[(StructField, Int)] = fields.zipWithIndex
    val rdd: RDD[Array[Any]] = frame.rdd.map(row => {
      val res: Array[Row => Any] = zipData.map(structField => squareRowkey2(structField._1, structField._2))
      val array: Array[Any] = res.map(fun => {
        val value = fun(row)
        if (value == null) String.valueOf(value) else value
      })
      array
    })
    rdd
  }

　　对类型进行判断

/**
   * 根据schema信息进行判断与封装
   *
   * @param dataType
   * @return
   */
  //封装rowkey
  def squareRowkey2(dataType: (StructField, Int)): (Row) => Any = {
    val (structField, index) = dataType
    structField.dataType match {
      case StringType =>
        (row: Row) => if (row.isNullAt(index)) null else row.getString(index)
      case LongType =>
        (row: Row) =>if (row.isNullAt(index)) null else row.getLong(index)
      case FloatType =>
        (row: Row) => if (row.isNullAt(index)) null else row.getFloat(index)
      case DoubleType =>
        (row: Row) => if (row.isNullAt(index)) null else row.getDouble(index)
      case IntegerType =>
        (row: Row) => if (row.isNullAt(index)) null else row.getInt(index)
      case BooleanType =>
        (row: Row) => if (row.isNullAt(index)) null else row.getBoolean(index)
      case DateType =>
        (row: Row) => if (row.isNullAt(index)) null else row.getDate(index)
      case TimestampType =>
        (row: Row) => if (row.isNullAt(index)) null else row.getTimestamp(index)
      case BinaryType =>
        (row: Row) => if (row.isNullAt(index)) null else row.getAs[Array[Byte]](index)
      case ArrayType(elementType, containsNull) =>
        (row: Row) => {
          val value: mutable.WrappedArray[_ >: Integer with String <: io.Serializable with Comparable[_ >: Integer with String]] = elementType match {
            case IntegerType => {
              row.getAs[mutable.WrappedArray[Integer]](index)
            }
            case StringType => {
              row.getAs[mutable.WrappedArray[String]](index)
            }
            case _ => row.getAs[mutable.WrappedArray[String]](index)
          }
          //这儿必须转换为java的list 防止map转json字符串不符合要求
          if (value == null) {
            util.Collections.emptyList()
          }
          JavaConversions.bufferAsJavaList(value.toBuffer)
        }
      case StructType(fields) =>
        (row: Row) => row.getAs[mutable.Map[String, String]](index)
      case _ =>
        (row: Row) => row.getString(index)
    }
  }

　二、rdd转实体对象

大多数是都是讲数据分装为case calss或者对象

  def  dataFrameToEntity [U: ClassTag] (frame: DataFrame, clazz: Class[U], hiveRdd: RDD[Array[Any]]) = {
    val fields: Array[StructField] = frame.schema.toArray
    val rdd = hiveRdd.map(array => {
      val map = new util.HashMap[String, Any]()
      fields.map(_.name).zip(array)
        .foreach {
          case (k, v) => (map.put(k, v))
        }
      val str = GsonUtil.toJsonString(map)
    //这边转换工具类 就是gson的转为对象的方法
      val value: U = GsonUtil.GsonToBean(str, clazz)
      value
    })
    rdd
  }

　使用:

    lazy val df: DataFrame =spark.read.table("user")
    //将df转为rdd实体类
    val userRdd: RDD[Array[Any]] = RddUtils.dataFrameToRdd(df)
    val userRDD2: RDD[User] = RddUtils.dataFrameToEntity(df, classOf[User], userRdd)

　　

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.lang3.StringUtils;
import java.lang.reflect.Field;

import java.util.LinkedHashMap;

import java.util.List;

import java.util.Map;
/**

@ClassName: com.xxx.sbc.dw.spark.submit.service.GsonUtil
@Description: json工具类
@Author: imp
@Time: 2020/10/23 10:01
@Version: 1.0

*/

public class GsonUtil {
private static Gson gson = null;
//判断gson对象是否存在了,不存在则创建对象

static {

if (gson == null) {

//gson = new Gson();            //当使用GsonBuilder方式时属性为空的时候输出来的json字符串是有键值key的,显示形式是"key":null，而直接new出来的就没有"key":null的

gson = new GsonBuilder().setDateFormat("yyyy-MM-dd HH:mm:ss").create();

}

}
//无参的私有构造方法

private GsonUtil() {

}
/**

* 将对象转成json格式

*

* @param object

* @return String

*/

public static String GsonString(Object object) {

String gsonString = null;

if (gson != null) {

gsonString = gson.toJson(object);

}

return gsonString;

}
/**

* 将json转成特定的cls的对象

*

* @param gsonString

* @param cls

* @return

*/

public static <T> T GsonToBean(String gsonString, Class<T> cls) {

T t = null;

if (StringUtils.isNotEmpty(gsonString)) {

//传入json对象和对象类型,将json转成对象

t = JSONObject.parseObject(gsonString, cls);

}

return t;

}
/**

* json字符串转成list

*

* @param gsonString

* @param cls

* @return

*/

public static <T> List<T> GsonToList(String gsonString, Class<T> cls) {

List<T> list = null;

if (gson != null) {

//根据泛型返回解析指定的类型,TypeToken<List<T>>{}.getType()获取返回类型

list = gson.fromJson(gsonString, new TypeToken<List<T>>() {

}.getType());

}

return list;

}
/**

* json字符串转成list中有map的

*

* @param gsonString

* @return

*/

public static <T> List<Map<String, T>> GsonToListMaps(String gsonString) {

List<Map<String, T>> list = null;

if (gson != null) {

list = gson.fromJson(gsonString,

new TypeToken<List<Map<String, T>>>() {

}.getType());

}

return list;

}
public static <T> List<Map<String, T>> gsonToListMaps(String str) {

List<Map<String, T>> list = null;

if (gson != null) {

list = gson.fromJson(str,

new TypeToken<List<Map<String, T>>>() {

}.getType());

}

return list;

}
/**

* json字符串转成map的

*

* @param gsonString

* @return

*/

public static <T> Map<String, String> GsonToMaps(String gsonString) {

Map<String, String> map = null;

if (gson != null) {

map = gson.fromJson(gsonString, new TypeToken<Map<String, String>>() {

}.getType());

}

return map;

}
/**

* 判断是否是json

*

* @param object

* @return

*/

public static Boolean isJson(Object object) {

try {

gson.toJson(object);

return true;

} catch (Exception e) {

System.err.format("{} is not json", object.toString());

return false;

}

}
/**

* 对象转为json字符串

*

* @param o

* @return

*/

public static String toJsonString(Object o) {

return JSON.toJSONString(o, SerializerFeature.DisableCircularReferenceDetect);

}
/**

* json转为map

*

* @param json

* @return

*/

public static Map<String, String> jsonToMap(String json) {

return JSON.parseObject(json, new TypeReference<LinkedHashMap<String, String>>() {

});

}
public static Map<String, Object> entityToMap(Object obj) throws IllegalAccessException {

Map<String, Object> map = new LinkedHashMap<String, Object>();

Class<?> clazz = obj.getClass();

System.out.println(clazz);

for (Field field : clazz.getDeclaredFields()) {

field.setAccessible(true);

String fieldName = field.getName();

Object value = field.get(obj);

if (value == null) {

value = "";

}

map.put(fieldName, value);

}

return map;

}