Flink 流计算编程

//为了使用Scala字符特性 'x 来获取字段 (Table API)
import org.apache.flink.api.scala.extensions._
import org.apache.flink.api.scala._
import org.apache.flink.table.api.scala._

 

屏蔽日志输出

def init(): Unit = {
  org.apache.log4j.Logger.getLogger("org.apache.flink").setLevel(org.apache.log4j.Level.ERROR)
}

 

创建 Flink Stream

val senv = StreamExecutionEnvironment.getExecutionEnvironment  //StreamExecutionEnvironment

 

Flink Stream Table API

val senv = StreamExecutionEnvironment.getExecutionEnvironment //StreamExecutionEnvironment
val tableEnv = StreamTableEnvironment.create(senv)            //StreamTableEnvironment

 

 

创建 Blink Stream Table API

val senv = StreamExecutionEnvironment.getExecutionEnvironment //StreamExecutionEnvironment

val settings = EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build()
val tableEnv = StreamTableEnvironment.create(senv, settings) //StreamTableEnvironment

 

val senv = StreamExecutionEnvironment.getExecutionEnvironment

val settings = EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build()
val tableEnv = StreamTableEnvironment.create(senv, settings)

tableEnv.sqlQuery(
  """
    |SELECT
    |  store_id, channel, paid_date,
    |  order_id, gmv,
    |FROM order_line
  """.stripMargin)
  .printSchema()

 

并行度

senv.setParallelism(4) //并行度为:4

 

 使用事件时间

senv.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) //ProcessingTime(默认)

 

Flink状态管理(State Backends)

//MemoryStateBackend
//(数据持久化状态存储在内存中，state数据保存在Java堆内存中，执行checkpoint时会把state的快照数据保存到JobManager的内存中。生产环境不建议使用)
senv.setStateBackend(new MemoryStateBackend())

//FsStateBackend(state数据保存在TaskManager的内存中，执行checkpoint时会把state的快照数据保存到配置的文件系统中)
senv.setStateBackend((new FsStateBackend("hdfs://flink/checkpoints"))) //快照数据保存在HDFS，数据有备份很安全

//RocksDBStateBackend
//(使用一套日志结构的数据库引擎，它是Flink中内置的第三方状态管理器。在做checkpoint时会把本地的数据直接复制到HDFS文件系统)
senv.setStateBackend(new RocksDBStateBackend("hdfs://flink/checkpoint"), true)
//state.backend: rocksdb
//state.backend.incremental: true

 

用Checkpoint保存数据

//默认checkpoint功能是未启用的
//每隔1000毫秒启动一个检查点(即设置checkpoint的周期)
senv.enableCheckpointing(1000)
senv.getCheckpointConfig.setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE) //设置模式为:EXACTLY_ONCE(默认值)

//senv.enableCheckpointing(1000, CheckpointingMode.EXACTLY_ONCE)

//确保检查点之间有至少500毫秒的间隔(即checkpoint最小间隔)
senv.getCheckpointConfig.setMinPauseBetweenCheckpoints(500)

//检查点必须在一分钟内完成,或者被丢弃(即checkpoint的超时时间)
senv.getCheckpointConfig.setCheckpointTimeout(60000) // 1 * 60 * 1000 (ms)

//同一时间只允许一个检查点
senv.getCheckpointConfig.setMaxConcurrentCheckpoints(1)


//Retain_On_Cancellation:表示一旦Flink处理程序被取消,就会保留Checkpoint数据,以便后续根据实际需要恢复到指定的Checkpoint
//Delete_On_Cancellation:表示一旦Flink处理程序被取消,就会删除Checkpoint数据,只有Job执行失败的时候才会保存Checkpoint
senv.getCheckpointConfig.enableExternalizedCheckpoints(ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION)

//最大并行执行的检查点数量
//默认情况下只有一个检查点可以运行,用户可以指定同时触发多个Checkpoint,进而提升Checkpoint整体的效率
senv.setMaxParallelism(1)

 

 

故障率重启策略(Failure Rate Restart Strategy)

//restart-strategy: failure-rate
senv.setRestartStrategy(RestartStrategies.failureRateRestart(
  3,                            //restart-strategy.failure-rate.max-failures-per-interval: 3
  Time.of(5, TimeUnit.MINUTES), //restart-strategy.failure-rate.failure-rate-interval: 5 min
  Time.of(10, TimeUnit.SECONDS) //restart-strategy.failure-rate.delay: 10 s
))