flink启动脚本解析

本地环境编译flink源码 flink版本1.17

start-cluster.sh

1.加载config.sh函数以及配置属性

2.启动jobmanager

3.启动taskmanager

#判断执行的目录
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
#  加载配置属性
. "$bin"/config.sh

# Start the JobManager instance(s)
shopt -s nocasematch
#如果是HA高可用的话会读取zookeeper，高可用必须得配置zookeeper
if [[ $HIGH_AVAILABILITY == "zookeeper" ]]; then
    #调用config.sh中的readMasters方法，读取maters文件
　　 readMasters
　　 #输出有多少个masters，高可用
    echo "Starting HA cluster with ${#MASTERS[@]} masters."

    for ((i=0;i<${#MASTERS[@]};++i)); do
        master=${MASTERS[i]}
        webuiport=${WEBUIPORTS[i]}

        if [ ${MASTERS_ALL_LOCALHOST} = true ] ; then
            "${FLINK_BIN_DIR}"/jobmanager.sh start "${master}" "${webuiport}"
        else
            ssh -n $FLINK_SSH_OPTS $master -- "nohup /bin/bash -l \"${FLINK_BIN_DIR}/jobmanager.sh\" start ${master} ${webuiport} &"
        fi
    done

else
    echo "Starting cluster."
    # 启动standalone jobmanager
    "$FLINK_BIN_DIR"/jobmanager.sh start
fi
shopt -u nocasematch

# 启动taskmanager
TMWorkers start

jobmanager.sh脚本

USAGE="Usage: jobmanager.sh ((start|start-foreground) [args])|stop|stop-all"
#判断是start  stop 或者是stop-all参数
STARTSTOP=$1

if [ -z $2 ] || [[ $2 == "-D" ]]; then
    # start [-D ...]
    args=("${@:2}")
elif [ -z $3 ] || [[ $3 == "-D" ]]; then
    # legacy path: start <host> [-D ...]
    HOST=$2
    args=("${@:3}")
else
    # legacy path: start <host> <port> [-D ...]
    HOST=$2
    WEBUIPORT=$3
    args=("${@:4}")
fi

if [[ $STARTSTOP != "start" ]] && [[ $STARTSTOP != "start-foreground" ]] && [[ $STARTSTOP != "stop" ]] && [[ $STARTSTOP != "stop-all" ]]; then
  echo $USAGE
  exit 1
fi

bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/config.sh
#默认是standalonesession模式
ENTRYPOINT=standalonesession

if [[ $STARTSTOP == "start" ]] || [[ $STARTSTOP == "start-foreground" ]]; then
    # Add JobManager-specific JVM options
    export FLINK_ENV_JAVA_OPTS="${FLINK_ENV_JAVA_OPTS} ${FLINK_ENV_JAVA_OPTS_JM}"
    parseJmArgsAndExportLogs "${ARGS[@]}"

    args=("--configDir" "${FLINK_CONF_DIR}" "--executionMode" "cluster" "${args[@]}")
    if [ ! -z $HOST ]; then
        args+=("--host")
        args+=("${HOST}")
    fi

    if [ ! -z $WEBUIPORT ]; then
        args+=("--webui-port")
        args+=("${WEBUIPORT}")
    fi

    if [ ! -z "${DYNAMIC_PARAMETERS}" ]; then
        args=(${DYNAMIC_PARAMETERS[@]} "${args[@]}")
    fi
fi

if [[ $STARTSTOP == "start-foreground" ]]; then
    exec "${FLINK_BIN_DIR}"/flink-console.sh $ENTRYPOINT "${args[@]}"
else
　　　#启动flink 使用flink-daemon.sh 方式（后台启动）
    "${FLINK_BIN_DIR}"/flink-daemon.sh $STARTSTOP $ENTRYPOINT "${args[@]}"
fi

taskmanager.sh

# Start/stop a Flink TaskManager.
USAGE="Usage: taskmanager.sh (start|start-foreground|stop|stop-all)"

STARTSTOP=$1

ARGS=("${@:2}")

if [[ $STARTSTOP != "start" ]] && [[ $STARTSTOP != "start-foreground" ]] && [[ $STARTSTOP != "stop" ]] && [[ $STARTSTOP != "stop-all" ]]; then
  echo $USAGE
  exit 1
fi

bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/config.sh

ENTRYPOINT=taskexecutor

if [[ $STARTSTOP == "start" ]] || [[ $STARTSTOP == "start-foreground" ]]; then

    # if no other JVM options are set, set the GC to G1
    if [ -z "${FLINK_ENV_JAVA_OPTS}" ] && [ -z "${FLINK_ENV_JAVA_OPTS_TM}" ]; then
        export JVM_ARGS="$JVM_ARGS -XX:+UseG1GC"
    fi

    # Add TaskManager-specific JVM options
    export FLINK_ENV_JAVA_OPTS="${FLINK_ENV_JAVA_OPTS} ${FLINK_ENV_JAVA_OPTS_TM}"

    # Startup parameters

    parseTmArgsAndExportLogs "${ARGS[@]}"

    if [ ! -z "${DYNAMIC_PARAMETERS}" ]; then
        ARGS=(${DYNAMIC_PARAMETERS[@]} "${ARGS[@]}")
    fi

    ARGS=("--configDir" "${FLINK_CONF_DIR}" "${ARGS[@]}")
fi

if [[ $STARTSTOP == "start-foreground" ]]; then
    exec "${FLINK_BIN_DIR}"/flink-console.sh $ENTRYPOINT "${ARGS[@]}"
else
    if [[ $FLINK_TM_COMPUTE_NUMA == "false" ]]; then
        # Start a single TaskManager
        "${FLINK_BIN_DIR}"/flink-daemon.sh $STARTSTOP $ENTRYPOINT "${ARGS[@]}"
    else
        # Example output from `numactl --show` on an AWS c4.8xlarge:
        # policy: default
        # preferred node: current
        # physcpubind: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
        # cpubind: 0 1
        # nodebind: 0 1
        # membind: 0 1
        read -ra NODE_LIST <<< $(numactl --show | grep "^nodebind: ")
        for NODE_ID in "${NODE_LIST[@]:1}"; do
            # Start a TaskManager for each NUMA node
            numactl --membind=$NODE_ID --cpunodebind=$NODE_ID -- "${FLINK_BIN_DIR}"/flink-daemon.sh $STARTSTOP $ENTRYPOINT "${ARGS[@]}"
        done
    fi
fi

jobmanager.sh 和taskmanager.sh后台启动都是调用flink-damen.sh 来启动的

服务	入口类	描述
taskexecutor	org.apache.flink.runtime.taskexecutor.TaskManagerRunner	taskmanager服务
zookeeper	org.apache.flink.runtime.zookeeper.FlinkZooKeeperQuorumPeer	zookeeper高可用服务
historyserver	org.apache.flink.runtime.webmonitor.history.HistoryServer	历史服务器
standalonesession	org.apache.flink.runtime.entrypoint.StandaloneSessionClusterEntrypoint	Session模式启动
standalonejob	org.apache.flink.container.entrypoint.StandaloneApplicationClusterEntryPoint	application模式启动
sql-gateway	org.apache.flink.table.gateway.SqlGateway	于Netty实现的，允许其他应用程序通过REST API与Flink Cluster交互的服务

说明：Session模式：是多个应用共享一个集群，资源共享，所以可能会发生任务抢占的情况。

　　　 application模式：每一个提交的应用单独启动一个 JobManager，也就是创建一个集群。这个 JobManager 只为执行这一个应用而存在，执行结束之后 JobManager 也就关闭了，这就是所谓的应　　用模式。

　　 per_job模式：和application类似，每个提交的作业启动一个集群。但是flink本身不可以，需要使用大数据资源管理框架来控制。

flink-daemon.sh 脚本，所有的分析都要经过此脚本启动，部分代码截取

STARTSTOP=$1
DAEMON=$2
ARGS=("${@:3}") # get remaining arguments as array

bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/config.sh

case $DAEMON in
    (taskexecutor)
        CLASS_TO_RUN=org.apache.flink.runtime.taskexecutor.TaskManagerRunner
    ;;

    (zookeeper)
        CLASS_TO_RUN=org.apache.flink.runtime.zookeeper.FlinkZooKeeperQuorumPeer
    ;;

    (historyserver)
        CLASS_TO_RUN=org.apache.flink.runtime.webmonitor.history.HistoryServer
    ;;

    (standalonesession)
        CLASS_TO_RUN=org.apache.flink.runtime.entrypoint.StandaloneSessionClusterEntrypoint
    ;;

    (standalonejob)
        CLASS_TO_RUN=org.apache.flink.container.entrypoint.StandaloneApplicationClusterEntryPoint
    ;;

    (sql-gateway)
        CLASS_TO_RUN=org.apache.flink.table.gateway.SqlGateway
        SQL_GATEWAY_CLASSPATH=`findSqlGatewayJar`
    ;;

    (*)
        echo "Unknown daemon '${DAEMON}'. $USAGE."
        exit 1
    ;;
esac

posted on 2022-09-23 14:09 郝心人forever 阅读(513) 评论(0) 收藏举报

flink启动脚本解析

application模式启动