GroupByKey，ReduceByKey

package com.shujia.spark.core

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object Demo6GroupByKey {
  def main(args: Array[String]): Unit = {

    val conf: SparkConf = new SparkConf()
      .setAppName("map")
      .setMaster("local")

    //spark  上下文对象
    val sc = new SparkContext(conf)

    val linesRDD: RDD[String] = sc.textFile("data/words.txt")

    val wordsRDD: RDD[String] = linesRDD.flatMap(_.split(","))
    //将rdd转换成kv格式
    val kvRDD: RDD[(String, Int)] = wordsRDD.map(word => (word, 1))

    /**
      * groupByKey: 通过key进行分组，将value 放到迭代器中
      * groupBy： 指定一个分组的列，
      *
      * 都会产生shuffle
      */

    val groupByKeyRDD: RDD[(String, Iterable[Int])] = kvRDD.groupByKey()

    val countyRDD: RDD[(String, Int)] = groupByKeyRDD.map {
      case (word: String, values: Iterable[Int]) =>
        (word, values.sum)
    }
    countyRDD.foreach(println)


    val groupByRDD: RDD[(String, Iterable[(String, Int)])] = kvRDD.groupBy(kv => kv._1)

  }
}

package com.shujia.spark.core

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD

object Demo7ReduceByKey {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf()
      .setAppName("map")
      .setMaster("local")

    //spark  上下文对象
    val sc = new SparkContext(conf)

    val linesRDD: RDD[String] = sc.textFile("data/words.txt")

    val wordsRDD: RDD[String] = linesRDD.flatMap(_.split(","))
    //将rdd转换成kv格式
    val kvRDD: RDD[(String, Int)] = wordsRDD.map(word => (word, 1))

    /**
      * reduceByKey: 对同一个key的value进行聚合处理
      *
      */

    val countRDD: RDD[(String, Int)] = kvRDD.reduceByKey((i: Int, j: Int) => i + j)

    countRDD.foreach(println)

    ////简写，如果参数只是用了一次，可以通过下划线代替
    val count2: RDD[(String, Int)] = kvRDD.reduceByKey(_ + _)


  }
}