编程模型：数据处理层
Basic相关API
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{HashPartitioner, SparkConf, SparkContext}
/**
* WordCount程序，Spark Streaming消费TCP Server发过来的实时数据的例子：
*
* 1、在master服务器上启动一个Netcat server
* `$ nc -lk 9998` (如果nc命令无效的话，我们可以用yum install -y nc来安装nc)
*
* 2、用下面的命令在在集群中将Spark Streaming应用跑起来
spark-submit --class com.dev.streaming.NetworkWordCount \
--master spark://master:7077 \
--deploy-mode client \
--driver-memory 512m \
--executor-memory 512m \
--total-executor-cores 4 \
--executor-cores 2 \
/home/hadoop-dev/spark-course/streaming/spark-streaming-basic-1.0-SNAPSHOT.jar
spark-shell --master spark://master:7077 --total-executor-cores 4 --executor-cores 2
*/
object BasicAPITest {
def main(args: Array[String]) {
val sparkConf = new SparkConf().setAppName("NetworkWordCount")
val sc = new SparkContext(sparkConf)
// StreamingContext 编程入口
val ssc = new StreamingContext(sc, Seconds(1))
//数据接收器(Receiver)
//创建一个接收器(ReceiverInputDStream)，这个接收器接收一台机器上的某个端口通过socket发送过来的数据并处理
val lines = ssc.socketTextStream("master", 9998, StorageLevel.MEMORY_AND_DISK_SER)
//数据处理(Process)
//处理的逻辑，就是简单的进行word count
val words = lines.flatMap(_.split(" ")).filter(_.contains("exception"))
val wordPairs = words.map(x => (x, 1))
// reduceByKey((a: Int, b: Int) => a + b, new HashPartitioner(10) 指定suffer后分区数量和分区算法（默认是HashPartitioner）
val wordCounts = wordPairs.repartition(100).reduceByKey((a: Int, b: Int) => a + b, new HashPartitioner(10))
//结果输出(Output)
//将结果输出到控制台
wordCounts.print()
//启动Streaming处理流
ssc.start()
//等待Streaming程序终止
ssc.awaitTermination()
}
}
Join相关API
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
/**
* Created by tangweiqun on 2018/1/6.
*/
object JoinAPITest {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setAppName("NetworkWordCount")
val sc = new SparkContext(sparkConf)
// Create the context with a 5 second batch size
val ssc = new StreamingContext(sc, Seconds(5))
val lines1 = ssc.socketTextStream("master", 9998, StorageLevel.MEMORY_AND_DISK_SER)
val kvs1 = lines1.map { line =>
val arr = line.split(" ")
(arr(0), arr(1))
}
val lines2 = ssc.socketTextStream("master", 9997, StorageLevel.MEMORY_AND_DISK_SER)
val kvs2 = lines2.map { line =>
val arr = line.split(" ")
(arr(0), arr(1))
}
kvs1.join(kvs2).print()
kvs1.fullOuterJoin(kvs2).print()
kvs1.leftOuterJoin(kvs2).print()
kvs1.rightOuterJoin(kvs2).print()
//启动Streaming处理流
ssc.start()
ssc.stop(false)
//等待Streaming程序终止
ssc.awaitTermination()
}
}
TransformAPI
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
/**
* Created by tangweiqun on 2018/1/6.
*/
object TransformAPITest {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setAppName("NetworkWordCount")
val sc = new SparkContext(sparkConf)
// Create the context with a 1 second batch size
val ssc = new StreamingContext(sc, Seconds(5))
val lines1 = ssc.socketTextStream("master", 9998, StorageLevel.MEMORY_AND_DISK_SER)
val kvs1 = lines1.map { line =>
val arr = line.split(" ")
(arr(0), arr(1))
}
/// 实时数据
val path = "hdfs://master:9999/user/hadoop-twq/spark-course/streaming/keyvalue.txt"
val keyvalueRDD =
sc.textFile(path).map { line =>
val arr = line.split(" ")
(arr(0), arr(1))
}
/// 静态数据
kvs1.transform { rdd =>
rdd.join(keyvalueRDD)
} print()
//启动Streaming处理流
ssc.start()
ssc.stop(false)
val lines2 = ssc.socketTextStream("master", 9997, StorageLevel.MEMORY_AND_DISK_SER)
val kvs2 = lines2.map { line =>
val arr = line.split(" ")
(arr(0), arr(1))
}
//（将实时数据与静态数据相关联）
kvs1.transformWith(kvs2, (rdd1: RDD[(String, String)], rdd2: RDD[(String, String)]) => rdd1.join(rdd2))
//等待Streaming程序终止
ssc.awaitTermination()
}
}
WindowAPI
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
/**
* Created by tangweiqun on 2018/1/6.
*/
object WindowAPITest {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setAppName("NetworkWordCount")
val sc = new SparkContext(sparkConf)
// Create the context with a 1 second batch size
val ssc = new StreamingContext(sc, Seconds(1)) //// 用来控制RDD的分区
val lines = ssc.socketTextStream("master", 9998, StorageLevel.MEMORY_AND_DISK_SER)
//每过2秒钟，然后显示前20秒的数据
val windowDStream = lines.window(Seconds(20), Seconds(2))
windowDStream.print()
//启动Streaming处理流
ssc.start()
//等待Streaming程序终止
ssc.awaitTermination()
ssc.stop(false)
}
}
batch interval - DStream产生的间隔，由StreamingContext指定 (这里设置为1s)，控制RDD分区
window length - 窗口的长度，即一个窗口包含的RDD的个数 (这里设置为20s，必须是batch interval的倍数)
sliding interval - 窗口滑动间隔，执行窗口操作的时间段(这里设置为2s，必须是batch interval的倍数)
ReduceByKeyAndWindowAPI
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
/**
* Created by tangweiqun on 2018/1/6.
*/
object ReduceByKeyAndWindowAPITest {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setAppName("NetworkWordCount")
val sc = new SparkContext(sparkConf)
// Create the context with a 1 second batch size
val ssc = new StreamingContext(sc, Seconds(1))
ssc.checkpoint("hdfs://master:9999/user/hadoop-twq/spark-course/streaming/checkpoint")
val lines = ssc.socketTextStream("master", 9998, StorageLevel.MEMORY_AND_DISK_SER)
val words = lines.flatMap(_.split(" "))
//每5秒中，统计前20秒内每个单词出现的次数
val wordPair = words.map(x => (x, 1))
val wordCounts =
wordPair.reduceByKeyAndWindow((a: Int, b: Int) => a + b, Seconds(20), Seconds(5))
wordCounts.print()
//启动Streaming处理流
ssc.start()
ssc.stop(false)
//接受一个ReduceFunc和一个invReduceFunc
//滑动时间比较短，窗口长度很长的场景
// 需要用checkpoint机制
val wordCountsOther =
wordPair.reduceByKeyAndWindow((a: Int, b: Int) => a + b,
(a: Int, b: Int) => a - b, Seconds(60), Seconds(2))
wordCountsOther.checkpoint(Seconds(12)) //窗口滑动间隔的5到10倍
wordCountsOther.print()
ssc.start()
//过滤掉value = 0的值
words.map(x => (x, 1)).reduceByKeyAndWindow((a: Int, b: Int) => a + b,
(a: Int, b: Int) => a - b,
Seconds(30), Seconds(10), 4,
(record: (String, Int)) => record._2 != 0)
//等待Streaming程序终止
ssc.awaitTermination()
}
}
1、分别对rdd2和rdd3进行reduceByKey
2、取在window内的rdd进行union，生成unionRDD
3、对unionRDD再次进行reduceByKey
（不需要 checkpoint机制，不需要依赖）
1、将两个window的所有rdd进行cogroup
（需要依赖前面的RDD，因此需要checkpoint机制）
2、对old rdds对应的value应用invReduceF
3、对new rdds对应的value应用reduceF
localCheckpoint() 存储在内存和磁盘中，但数据不可靠
checkpoint() 存储在HDFS中去，数据可靠，提高容错性能，需要设置文件目录
UpdateStateByKeyAPI
1、updateStateByKey，这个API根据一个key的之前的状态和新的接收到的数据来计算并且更新新状态。使用这个API需要做两步：第一就是为每一个key定义一个初始状态，这个状态的类型可以实任意类型；第二就是定义一个更新状态的函数，这个函数根据每一个key之前的状态和新接收到的数据计算新的状态。
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{HashPartitioner, SparkConf, SparkContext}
import scala.collection.mutable.ListBuffer
object UpdateStateByKeyAPITest {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setAppName("NetworkWordCount")
val sc = new SparkContext(sparkConf)
// Create the context with a 1 second batch size
val ssc = new StreamingContext(sc, Seconds(1))
ssc.checkpoint("hdfs://master:9999/user/hadoop-twq/spark-course/streaming/checkpoint")
val lines = ssc.socketTextStream("master", 9998, StorageLevel.MEMORY_AND_DISK_SER)
val words = lines.flatMap(_.split(" "))
val wordsDStream = words.map(x => (x, 1))
///values: Seq[Int] 在一定的时间段内收到的 当前key在这个时间段内收集到的value，
/// currentState: Option[Int] 当前key的状态
wordsDStream.updateStateByKey(
(values: Seq[Int], currentState: Option[Int]) => Some(currentState.getOrElse(0) + values.sum)).print()
//启动Streaming处理流
ssc.start()
ssc.stop(false)
//updateStateByKey的另一个API
/// 接收的函数是Iterator 三元组 String Key Seq[Int] 接收到的数据 Option[Int]) Key当前的状态
wordsDStream.updateStateByKey[Int]((iter: Iterator[(String, Seq[Int], Option[Int])]) => {
val list = ListBuffer[(String, Int)]()
while (iter.hasNext) {
val (key, newCounts, currentState) = iter.next
val state = Some(currentState.getOrElse(0) + newCounts.sum)
val value = state.getOrElse(0)
if (key.contains("error")) {
list += ((key, value)) // Add only keys with contains error
}
}
list.toIterator
}, new HashPartitioner(4), true).print()
ssc.start()
//等待Streaming程序终止
ssc.awaitTermination()
}
}
MapWithStateAPI
mapWithState，这个API的功能和updateStateByKey是一样的，只不过在性能方面做了很大的优化，这个函数对于没有接收到新数据的key是不会计算新状态的，而updateStateByKey是会重新计算任何的key的新状态的，由于这个原因所以导致mapWithState可以处理的key的数量比updateStateByKey多10倍多，性能也比updateStateByKey快很多。 支持促使状态mapWithState还支持timeout API
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming._
import org.apache.spark.{SparkConf, SparkContext}
object MapWithStateAPITest {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setAppName("NetworkWordCount")
val sc = new SparkContext(sparkConf)
// Create the context with a 1 second batch size
val ssc = new StreamingContext(sc, Seconds(5))
ssc.checkpoint("hdfs://master:9999/user/hadoop-twq/spark-course/streaming/checkpoint")
val lines = ssc.socketTextStream("master", 9998, StorageLevel.MEMORY_AND_DISK_SER)
val words = lines.flatMap(_.split(" "))
val wordsDStream = words.map(x => (x, 1))
val initialRDD = sc.parallelize(List(("dummy", 100L), ("source", 32L)))
// currentBatchTime : 表示当前的Batch的时间
// key: 表示需要更新状态的key
// value: 表示当前batch的对应的key的对应的值
// currentState: 对应key的当前的状态
val stateSpec = StateSpec.function((currentBatchTime: Time, key: String, value: Option[Int], currentState: State[Long]) => {
val sum = value.getOrElse(0).toLong + currentState.getOption.getOrElse(0L)
val output = (key, sum)
if (!currentState.isTimingOut()) {
currentState.update(sum)
}
Some(output)
}).initialState(initialRDD).numPartitions(2).timeout(Seconds(30)) //timeout: 当一个key超过这个时间没有接收到数据的时候，这个key以及对应的状态会被移除掉
val result = wordsDStream.mapWithState(stateSpec)
result.print()
// 从一开始显示所有数据，包含初始值
result.stateSnapshots().print()
//启动Streaming处理流
ssc.start()
ssc.stop(false)
//等待Streaming程序终止
ssc.awaitTermination()
}
}
浙公网安备 33010602011771号