kafka之间同步topic脚本

注意： 该脚本适应低版本和高版本kafka之间同步topic

 

#!/usr/bin/env bash
# Kafka Topic 导入/导出：兼容 ZooKeeper 管理的旧集群 与 bootstrap-server / KRaft 新集群。
#
# 必填（二选一，也可同时给，由 KAFKA_ADMIN_PROTOCOL 决定）：
#   KAFKA_BOOTSTRAP_SERVER   例如 10.52.92.79:32518 或 b-1.xxx.kafka.amazonaws.com:9092
#   KAFKA_ZOOKEEPER          例如 zk1:2181,zk2:2181/chroot
#
# 常用可选（也可在下方「默认集群配置」里直接改）：
#   KAFKA_HOME               Kafka 安装目录
#   KAFKA_BIN                脚本目录，默认 $KAFKA_HOME/bin
#   KAFKA_ADMIN_PROTOCOL     auto | bootstrap | zookeeper   默认 auto
#   KAFKA_VERBOSE            1 时打印 Kafka CLI 错误详情
#   KAFKA_TOPIC_EXCLUDE      排除 topic 的 ERE，默认 ^(__|_confluent|connect-)；设为空字符串则不排除任何 topic
#   KAFKA_OUTPUT_FILE        导出文件名，默认 topic_configs_YYYYMMDD.csv
#   KAFKA_DEFAULT_PARTITIONS 导入时分区缺省值，默认 1
#   KAFKA_DEFAULT_RF         导入时副本缺省值，默认 1
#   KAFKA_ALTER_EXISTING     1 时对已存在 topic 同步动态配置（不改分区/副本）
#   KAFKA_INCREASE_PARTITIONS 1 时允许把已存在 topic 的分区数调大（不可调小）
#   KAFKA_CONFIG_SKIP        导入/导出时忽略的配置项（逗号分隔），默认 message.format.version
#   DRY_RUN                  1 时只打印将执行的命令
#
# 示例：
#   # 旧集群导出（ZK）
#   export KAFKA_BIN=/opt/kafka_2.11-1.1.1/bin
#   export KAFKA_ZOOKEEPER=zk-1:2181
#   ./sync-topics.sh export
#
#   # 新集群导入（bootstrap）
#   export KAFKA_BIN=/opt/lfx/kafka_2.13-4.1.1/bin
#   export KAFKA_BOOTSTRAP_SERVER=10.52.92.79:32518
#   ./sync-topics.sh import topic_configs_20260820.csv
#
set -uo pipefail
# 不使用 set -e：脚本里大量 [[ cond ]] && return/continue，条件为假时会误触发退出

# ===== 默认集群配置（按环境修改；运行时 export 同名变量可覆盖）=====
KAFKA_HOME="${KAFKA_HOME:-/home/ec2-user/kafka_2.12-3.7.2}"
KAFKA_BIN="${KAFKA_BIN:-${KAFKA_HOME}/bin}"
KAFKA_BOOTSTRAP_SERVER="${KAFKA_BOOTSTRAP_SERVER:-b-1.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:9092}"
KAFKA_ZOOKEEPER="${KAFKA_ZOOKEEPER:-}"
KAFKA_ADMIN_PROTOCOL="${KAFKA_ADMIN_PROTOCOL:-auto}"
# 用 - 而非 :-，这样 export KAFKA_TOPIC_EXCLUDE="" 表示「不排除任何 topic」
KAFKA_TOPIC_EXCLUDE="${KAFKA_TOPIC_EXCLUDE-^(__|_confluent|connect-)}"
KAFKA_OUTPUT_FILE="${KAFKA_OUTPUT_FILE:-topic_configs_$(date +%Y%m%d).csv}"
KAFKA_DEFAULT_PARTITIONS="${KAFKA_DEFAULT_PARTITIONS:-1}"
KAFKA_DEFAULT_RF="${KAFKA_DEFAULT_RF:-1}"
KAFKA_ALTER_EXISTING="${KAFKA_ALTER_EXISTING:-0}"
KAFKA_INCREASE_PARTITIONS="${KAFKA_INCREASE_PARTITIONS:-0}"
DRY_RUN="${DRY_RUN:-0}"
KAFKA_VERBOSE="${KAFKA_VERBOSE:-0}"
# 跨版本同步时常见只读/已废弃项；Kafka 3+ 不再支持 message.format.version 作为 topic 配置
KAFKA_CONFIG_SKIP="${KAFKA_CONFIG_SKIP:-message.format.version}"

TOPICS_SH="${KAFKA_BIN}/kafka-topics.sh"
CONFIGS_SH="${KAFKA_BIN}/kafka-configs.sh"

ADMIN_ARGS=()
PROTOCOL=""
HELP_CONFIGS=""

ensure_kafka_bins() {
    [[ -x "$TOPICS_SH" ]] || die "找不到可执行文件: $TOPICS_SH（请设置 KAFKA_HOME 或 KAFKA_BIN）"
    [[ -x "$CONFIGS_SH" ]] || die "找不到可执行文件: $CONFIGS_SH"
}

load_configs_help() {
    if [[ -n "$HELP_CONFIGS" ]]; then
        return 0
    fi
    ensure_kafka_bins
    HELP_CONFIGS="$("$CONFIGS_SH" --help 2>&1 || true)"
}

resolve_protocol() {
    local want="$KAFKA_ADMIN_PROTOCOL"
    case "$want" in
        auto|bootstrap|zookeeper) ;;
        *) die "KAFKA_ADMIN_PROTOCOL 只能是 auto / bootstrap / zookeeper" ;;
    esac

    if [[ "$want" == "auto" ]]; then
        if [[ -n "$KAFKA_BOOTSTRAP_SERVER" ]]; then
            want="bootstrap"
        elif [[ -n "$KAFKA_ZOOKEEPER" ]]; then
            want="zookeeper"
        else
            die "请设置 KAFKA_BOOTSTRAP_SERVER 或 KAFKA_ZOOKEEPER"
        fi
    fi

    if [[ "$want" == "bootstrap" ]]; then
        [[ -n "$KAFKA_BOOTSTRAP_SERVER" ]] || die "KAFKA_ADMIN_PROTOCOL=bootstrap 时必须设置 KAFKA_BOOTSTRAP_SERVER"
    else
        [[ -n "$KAFKA_ZOOKEEPER" ]] || die "KAFKA_ADMIN_PROTOCOL=zookeeper 时必须设置 KAFKA_ZOOKEEPER"
    fi

    PROTOCOL="$want"
    ADMIN_ARGS=()
    if [[ "$PROTOCOL" == "bootstrap" ]]; then
        ADMIN_ARGS+=(--bootstrap-server "$KAFKA_BOOTSTRAP_SERVER")
    else
        ADMIN_ARGS+=(--zookeeper "$KAFKA_ZOOKEEPER")
    fi
}

usage() {
    cat <<EOF
用法:
  $0 export [output.csv]
  $0 import <config.csv>
  $0 detect

环境变量见脚本头部注释。也可：
  DRY_RUN=1 $0 import file.csv
EOF
}

die() {
    echo "错误: $*" >&2
    exit 1
}

trim() {
    local s="${1-}"
    s="${s#"${s%%[![:space:]]*}"}"
    s="${s%"${s##*[![:space:]]}"}"
    printf '%s' "$s"
}

load_cli_help() {
    ensure_kafka_bins
}

run_cmd() {
    if [[ "$DRY_RUN" == "1" ]]; then
        printf '[dry-run]'
        printf ' %q' "$@"
        printf '\n'
        return 0
    fi
    "$@"
}

cmd_output() {
    if [[ "$DRY_RUN" == "1" ]]; then
        printf '[dry-run]' >&2
        printf ' %q' "$@" >&2
        printf '\n' >&2
        return 0
    fi
    "$@"
}

init_admin() {
    load_cli_help
    resolve_protocol
    echo "Kafka bin : $KAFKA_BIN"
    echo "协议      : $PROTOCOL"
    if [[ "$PROTOCOL" == "bootstrap" ]]; then
        echo "地址      : $KAFKA_BOOTSTRAP_SERVER"
    else
        echo "地址      : $KAFKA_ZOOKEEPER"
    fi
    if [[ "$DRY_RUN" == "1" ]]; then
        echo "DRY_RUN   : 开启"
    fi
}

show_kafka_error() {
    local label="$1" err_file="$2" rc="$3"
    echo "错误: $label 失败 (exit=$rc)" >&2
    if [[ -s "$err_file" ]]; then
        echo "--- Kafka 输出 ---" >&2
        cat "$err_file" >&2
        echo "------------------" >&2
    fi
}

export_one_header() {
    local out="$1" line="$2"
    parse_header_line "$line"
    local topic="$_topic"
    if is_excluded_topic "$topic"; then
        return 0
    fi
    if [[ -z "$topic" ]]; then
        return 0
    fi
    local parts="${_partitions:-$KAFKA_DEFAULT_PARTITIONS}"
    local rf="${_rf:-$KAFKA_DEFAULT_RF}"
    local semi retention
    semi="$(configs_to_semi "$_configs")"
    semi="$(sanitize_topic_configs "$semi")"
    retention="$(retention_from_configs "$semi")"
    echo "$(csv_escape "$topic"),$(trim "$parts"),$(trim "$rf"),$(trim "$retention"),$(csv_escape "$semi")" >> "$out"
    echo "  $topic  partitions=$parts rf=$rf retention.ms=${retention:-默认}"
}

is_excluded_topic() {
    local topic="$1"
    [[ -z "$topic" ]] && return 0
    [[ -z "$KAFKA_TOPIC_EXCLUDE" ]] && return 1
    [[ "$topic" =~ $KAFKA_TOPIC_EXCLUDE ]]
}

# 从 describe 首行解析 Configs: a=b,c=d （旧/新版格式都接近）
parse_header_line() {
    local line="$1"
    _topic=""
    _partitions=""
    _rf=""
    _configs=""

    _topic="$(printf '%s\n' "$line" | sed -n 's/.*Topic:[[:space:]]*\([^[:space:]]*\).*/\1/p')"
    _partitions="$(printf '%s\n' "$line" | sed -n 's/.*PartitionCount:[[:space:]]*\([0-9][0-9]*\).*/\1/p')"
    _rf="$(printf '%s\n' "$line" | sed -n 's/.*ReplicationFactor:[[:space:]]*\([0-9][0-9]*\).*/\1/p')"

    if printf '%s\n' "$line" | grep -q 'Configs:'; then
        _configs="$(printf '%s\n' "$line" | sed 's/.*Configs:[[:space:]]*//')"
        _configs="$(printf '%s' "$_configs" | tr -d '\r' | sed 's/[[:space:]]*$//')"
    fi
}

configs_to_semi() {
    local raw="$1"
    raw="$(trim "$raw")"
    [[ -z "$raw" ]] && { printf ''; return; }
    printf '%s' "$raw" | sed 's/,/;/g'
}

config_key_should_skip() {
    local key="$1" skip
    IFS=',' read -r -a _skips <<< "$KAFKA_CONFIG_SKIP"
    for skip in "${_skips[@]}"; do
        skip="$(trim "$skip")"
        [[ -z "$skip" ]] && continue
        if [[ "$key" == "$skip" ]]; then
            return 0
        fi
    done
    return 1
}

# 过滤源集群 describe 里带出、但目标集群 create/alter 不接受的配置
sanitize_topic_configs() {
    local semi="$1" pair key
    local -a kept=()
    semi="$(trim "$semi")"
    [[ -z "$semi" ]] && { printf ''; return; }
    IFS=';' read -r -a _pairs <<< "$semi"
    for pair in "${_pairs[@]}"; do
        pair="$(trim "$pair")"
        [[ -z "$pair" ]] && continue
        key="${pair%%=*}"
        key="$(trim "$key")"
        if config_key_should_skip "$key"; then
            continue
        fi
        kept+=("$pair")
    done
    local IFS=';'
    printf '%s' "${kept[*]}"
}

retention_from_configs() {
    local semi="$1"
    printf '%s\n' "$semi" | tr ';' '\n' | sed -n 's/^retention\.ms=//p' | head -1
}

csv_escape() {
    local s="$1"
    if [[ "$s" == *'"'* || "$s" == *','* || "$s" == *$'\n'* ]]; then
        s="${s//\"/\"\"}"
        printf '"%s"' "$s"
    else
        printf '%s' "$s"
    fi
}

list_topics() {
    cmd_output "$TOPICS_SH" "${ADMIN_ARGS[@]}" --list
}

describe_all() {
    cmd_output "$TOPICS_SH" "${ADMIN_ARGS[@]}" --describe
}

describe_one() {
    cmd_output "$TOPICS_SH" "${ADMIN_ARGS[@]}" --describe --topic "$1"
}

TOPIC_LIST_CACHE=""

refresh_topic_cache() {
    TOPIC_LIST_CACHE="$(mktemp)"
    local err_file rc
    err_file="$(mktemp)"
    set +e
    "$TOPICS_SH" "${ADMIN_ARGS[@]}" --list >"$TOPIC_LIST_CACHE" 2>"$err_file"
    rc=$?
    if [[ $rc -ne 0 ]]; then
        show_kafka_error "kafka-topics --list" "$err_file" "$rc"
        rm -f "$err_file"
        die "无法获取 topic 列表"
    fi
    rm -f "$err_file"
}

topic_exists() {
    local topic="$1"
    [[ -n "$TOPIC_LIST_CACHE" && -f "$TOPIC_LIST_CACHE" ]] || refresh_topic_cache
    grep -Fxq "$topic" "$TOPIC_LIST_CACHE" && return 0
    return 1
}

mark_topic_created() {
    local topic="$1"
    [[ -n "$TOPIC_LIST_CACHE" ]] && echo "$topic" >>"$TOPIC_LIST_CACHE"
}

# 创建时可带多个 --config；旧 CLI 也支持
create_topic() {
    local topic="$1" partitions="$2" rf="$3" semi_configs="$4"
    semi_configs="$(sanitize_topic_configs "$semi_configs")"
    local args=("$TOPICS_SH" "${ADMIN_ARGS[@]}" --create --topic "$topic"
        --partitions "$partitions" --replication-factor "$rf")
    local pair _pairs
    if [[ -n "$semi_configs" ]]; then
        IFS=';' read -r -a _pairs <<< "$semi_configs"
        for pair in "${_pairs[@]}"; do
            pair="$(trim "$pair")"
            [[ -z "$pair" ]] && continue
            args+=(--config "$pair")
        done
    fi
    run_cmd "${args[@]}"
}

alter_partitions() {
    local topic="$1" partitions="$2"
    run_cmd "$TOPICS_SH" "${ADMIN_ARGS[@]}" --alter --topic "$topic" --partitions "$partitions"
}

# 新旧 kafka-configs：优先 entity-type；失败再试 --topic（部分 2.x+）
alter_topic_configs() {
    local topic="$1" semi_configs="$2"
    local add
    semi_configs="$(sanitize_topic_configs "$semi_configs")"
    add="$(printf '%s' "$semi_configs" | sed 's/;/,/g')"
    [[ -z "$add" ]] && return 0

    load_configs_help
    if echo "$HELP_CONFIGS" | grep -q -- '--entity-type'; then
        run_cmd "$CONFIGS_SH" "${ADMIN_ARGS[@]}" \
            --entity-type topics --entity-name "$topic" \
            --alter --add-config "$add"
        return
    fi
    if echo "$HELP_CONFIGS" | grep -q -- '--topic'; then
        run_cmd "$CONFIGS_SH" "${ADMIN_ARGS[@]}" --alter --topic "$topic" --add-config "$add"
        return
    fi
    die "当前 kafka-configs.sh 无法识别 --entity-type / --topic，请更换 Kafka 工具版本"
}

export_topics() {
    local out="${1:-$KAFKA_OUTPUT_FILE}"
    local tmp err_file header_ok=0 topic_list_file

    echo "=== Kafka Topic 导出 ==="
    echo "模式      : 导出"
    echo "输出文件  : $out"

    init_admin

    # 先写 CSV 表头，避免后续 Kafka 命令失败时目录里没有任何文件
    echo "topic,partitions,replication_factor,retention_ms,configs" > "$out"
    echo "已创建: $out"

    tmp="$(mktemp)"
    err_file="$(mktemp)"
    topic_list_file="$(mktemp)"

    describe_all >"$tmp" 2>"$err_file"
    local rc=$?

    if [[ $rc -ne 0 && "$KAFKA_VERBOSE" == "1" ]]; then
        show_kafka_error "kafka-topics --describe" "$err_file" "$rc"
    fi

    if [[ $rc -eq 0 && -s "$tmp" ]] && grep -q 'PartitionCount' "$tmp"; then
        header_ok=1
    fi

    if [[ "$header_ok" -eq 1 ]]; then
        echo "使用一次 --describe 导出全部 topic"
        while IFS= read -r line || [[ -n "$line" ]]; do
            if printf '%s' "$line" | grep -q 'PartitionCount'; then
                export_one_header "$out" "$line"
            fi
        done < "$tmp"
    else
        echo "整集群 describe 不可用，回退为逐 topic describe"
        if [[ -s "$err_file" ]]; then
            echo "describe 失败原因:"
            cat "$err_file"
        fi

        "$TOPICS_SH" "${ADMIN_ARGS[@]}" --list >"$topic_list_file" 2>"$err_file"
        local list_rc=$?
        if [[ $list_rc -ne 0 ]]; then
            show_kafka_error "kafka-topics --list" "$err_file" "$list_rc"
            die "无法获取 topic 列表，CSV 仅有表头: $out"
        fi

        local topic line
        while IFS= read -r topic || [[ -n "$topic" ]]; do
            topic="$(trim "$topic")"
            [[ -z "$topic" ]] && continue
            is_excluded_topic "$topic" && continue
            echo "处理: $topic"
            line="$("$TOPICS_SH" "${ADMIN_ARGS[@]}" --describe --topic "$topic" 2>"$err_file" | grep 'PartitionCount' | head -1 || true)"
            if [[ -n "$line" ]]; then
                export_one_header "$out" "$line"
            else
                echo "  无法 describe，写入默认值"
                if [[ -s "$err_file" && "$KAFKA_VERBOSE" == "1" ]]; then
                    cat "$err_file"
                fi
                echo "$topic,$KAFKA_DEFAULT_PARTITIONS,$KAFKA_DEFAULT_RF,," >> "$out"
            fi
        done < "$topic_list_file"
    fi

    echo "导出完成: $out"
    echo "总行数(含表头): $(wc -l < "$out" | tr -d ' ')"
    echo "预览:"
    if command -v column >/dev/null 2>&1; then
        head -10 "$out" | column -t -s','
    else
        head -10 "$out"
    fi

    rm -f "$tmp" "$err_file" "$topic_list_file"
}

# 简易 CSV 一行解析：支持 configs 字段被双引号包裹
parse_csv_row() {
    local row="$1"
    _c_topic="" _c_parts="" _c_rf="" _c_ret="" _c_cfgs=""

    if [[ "$row" == '"'* ]]; then
        die "不支持 topic 字段带引号的 CSV 行: $row"
    fi

    _c_topic="${row%%,*}"
    row="${row#*,}"
    _c_parts="${row%%,*}"
    row="${row#*,}"
    _c_rf="${row%%,*}"
    row="${row#*,}"

    if [[ "$row" == '"'* ]]; then
        # retention 为空且 configs 被整体加引号的少见情况不处理；标准是 retention,configs
        _c_ret="${row%%,*}"
        row="${row#*,}"
    else
        _c_ret="${row%%,*}"
        if [[ "$row" == *","* ]]; then
            row="${row#*,}"
        else
            row=""
        fi
    fi

    _c_cfgs="$row"
    if [[ "$_c_cfgs" == '"'*'"' ]]; then
        _c_cfgs="${_c_cfgs#\"}"
        _c_cfgs="${_c_cfgs%\"}"
        _c_cfgs="${_c_cfgs//\"\"/\"}"
    fi

    _c_topic="$(trim "$_c_topic")"
    _c_parts="$(trim "$_c_parts")"
    _c_rf="$(trim "$_c_rf")"
    _c_ret="$(trim "$_c_ret")"
    _c_cfgs="$(trim "$_c_cfgs")"
}

import_topics() {
    local file="${1:-}"
    [[ -n "$file" && -f "$file" ]] || die "请提供配置文件。用法: $0 import <config.csv>"

    init_admin
    echo "模式      : 导入"
    echo "配置文件  : $file"

    local success=0 skip=0 error=0 altered=0
    local header first=1
    local topic parts rf retention configs line
    refresh_topic_cache

    while IFS= read -r line || [[ -n "$line" ]]; do
        line="${line%$'\r'}"
        [[ -z "$(trim "$line")" ]] && continue
        if [[ "$first" -eq 1 ]]; then
            first=0
            header="$(printf '%s' "$line" | tr 'A-Z' 'a-z')"
            if [[ "$header" == topic* ]]; then
                continue
            fi
        fi

        parse_csv_row "$line"
        topic="$_c_topic"
        parts="$_c_parts"
        rf="$_c_rf"
        retention="$_c_ret"
        configs="$_c_cfgs"

        [[ -z "$topic" || "$topic" == "topic" ]] && continue
        is_excluded_topic "$topic" && { echo "跳过内部 topic: $topic"; continue; }

        parts="${parts:-$KAFKA_DEFAULT_PARTITIONS}"
        rf="${rf:-$KAFKA_DEFAULT_RF}"
        if [[ -z "$configs" && -n "$retention" && "$retention" =~ ^[0-9]+$ ]]; then
            configs="retention.ms=$retention"
        elif [[ -n "$retention" && "$retention" =~ ^[0-9]+$ ]] && ! printf '%s' "$configs" | grep -q 'retention.ms='; then
            if [[ -n "$configs" ]]; then
                configs="${configs};retention.ms=${retention}"
            else
                configs="retention.ms=$retention"
            fi
        fi

        local raw_configs="$configs"
        configs="$(sanitize_topic_configs "$configs")"
        if [[ -n "$raw_configs" && "$configs" != "$raw_configs" ]]; then
            echo "  已忽略配置项: $KAFKA_CONFIG_SKIP"
        fi

        echo "处理: $topic  partitions=$parts rf=$rf configs=${configs:-默认}"

        if topic_exists "$topic"; then
            echo "  topic 已存在，跳过创建"
            skip=$((skip + 1))
            if [[ "$KAFKA_INCREASE_PARTITIONS" == "1" ]]; then
                echo "  尝试调整分区数 -> $parts"
                if alter_partitions "$topic" "$parts"; then
                    echo "  分区调整已提交（若目标更小或相等，broker 可能拒绝）"
                else
                    echo "  分区调整失败（分区只能增加）"
                fi
            fi
            if [[ "$KAFKA_ALTER_EXISTING" == "1" && -n "$configs" ]]; then
                if alter_topic_configs "$topic" "$configs"; then
                    echo "  已同步动态配置"
                    altered=$((altered + 1))
                else
                    echo "  动态配置同步失败"
                    error=$((error + 1))
                fi
            fi
            echo ""
            continue
        fi

        create_topic "$topic" "$parts" "$rf" "$configs"
        rc=$?
        if [[ $rc -eq 0 ]]; then
            echo "  创建成功"
            success=$((success + 1))
            mark_topic_created "$topic"
        else
            echo "  创建失败（常见原因：副本数大于 broker 数、topic 名非法）"
            error=$((error + 1))
            echo ""
            continue
        fi
        echo ""
    done < "$file"

    rm -f "$TOPIC_LIST_CACHE"
    echo "导入完成  成功=$success  已存在跳过=$skip  更新配置=$altered  失败=$error"
}

print_detect() {
    ensure_kafka_bins
    echo "KAFKA_BIN              = $KAFKA_BIN"
    echo "kafka-topics.sh        = $TOPICS_SH"
    echo "KAFKA_BOOTSTRAP_SERVER = ${KAFKA_BOOTSTRAP_SERVER:-未设置}"
    echo "KAFKA_ZOOKEEPER        = ${KAFKA_ZOOKEEPER:-未设置}"
    resolve_protocol
    echo "将使用协议             = $PROTOCOL"
}

main() {
    local cmd="${1:-}"
    shift || true
    case "$cmd" in
        export) export_topics "${1:-}" ;;
        import) import_topics "${1:-}" ;;
        detect) print_detect ;;
        -h|--help|help|"") usage ;;
        *) usage; die "未知命令: $cmd" ;;
    esac
}

main "$@"