k8s部署canal 1.1.8
说明：canal部署的是单实例，pvc挂载日志目录，因日志存储路径修改太过麻烦，所以把mysql的bin log点位信息存在日志目录下
推荐使用多实例
一、单实例
1、部署configmap，注意canal.destinations要和canal.instance.id一致
vim canal-config.yaml
apiVersion: v1 kind: ConfigMap metadata: name: canal-subotiz namespace: staging data: canal.properties: | # ========== Canal基础配置 ========== canal.id = 1 canal.port = 11111 canal.destinations = canal-subotiz-common # ========== 持久化路径 ========== canal.file.data.dir = /home/admin/canal-server/logs/db # ========== Zookeeper配置(可以注释掉不使用) ========== # canal.zkServers = z-2.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:2181,z-1.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:2181,z-3.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:2181 # ========== Server模式配置 ========== canal.serverMode = kafka # ========== Kafka生产者配置 ========== kafka.bootstrap.servers = b-2.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:9092,b-1.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:9092,b-3.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:9092 canal.mq.servers = b-2.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:9092,b-1.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:9092,b-3.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:9092 canal.mq.topic = subotiz-canal-merchantdb canal.mq.flatMessage = true canal.mq.partitionsNum = 1 # Kafka生产者高级配置 canal.mq.properties.max.request.size = 134217728 canal.mq.maxRequestSize = 134217728 canal.mq.bufferMemory = 134217728 canal.mq.kafka.producer.retries = 5 canal.mq.kafka.producer.retry.backoff.ms = 1000 canal.mq.kafka.producer.retry.backoff.max.ms = 5000 canal.mq.kafka.producer.metadata.fetch.timeout.ms = 120000 canal.mq.kafka.producer.metadata.max.age.ms = 300000 # ========== 自动扫描配置 ========== canal.auto.scan = true canal.auto.scan.interval = 5 # instance.properties 实例配置 instance.properties: | # ========== 核心新增：实例ID（必须和destinations一致） ========== canal.instance.id = canal-subotiz-common # ========== MySQL连接配置 ========== canal.instance.master.address = mypaydb1.cluster-cl20ky4iyqmn.ap-southeast-1.rds.amazonaws.com:3306 canal.instance.dbUsername = canal canal.instance.dbPassword = vGgRTZubA2qsI2mN canal.instance.connectionCharset = UTF-8 #========== 动态Topic配置 ========== canal.mq.topic = subotiz-canal-merchantdb canal.mq.dynamicTopic = subotiz-canal-developerdb:developerdb,subotiz-canal-merchantdb:merchantdb,subotiz-canal-paymentdb:paymentdb,subotiz-canal-tradingdb:tradingdb,subotiz-canal-pgsbilling:pgsbilling,subotiz-canal-promotiondb:promotiondb,subotiz-canal-robin:robin,subotiz-canal-email-server:email_server,subotiz-canal-notifydb:notifydb,shoplazza-canal-discount-limit:promotiondb\\.discount_used_total,subotiz-canal-tradingdb-trade:tradingdb\\.trade,subotiz-canal-tradingdb-refund-record:tradingdb\\.refund_record,subotiz-canal-tradingdb-checkout-session:tradingdb\\.checkout_session,subotiz-canal-merchantdb-customer:merchantdb\\.customer,subotiz-canal-merchantdb-merchant-setting:merchantdb\\.merchant_setting,subotiz-canal-paymentdb-pgs-i18n-message:paymentdb\\.pgs_i18n_message,subotiz-canal-paymentdb-pgs-subscription:paymentdb\\.pgs_subscription,subotiz-canal-paymentdb-pgs-invoice:paymentdb\\.pgs_invoice,subotiz-canal-tradingdb-trade-line-item:tradingdb\\.trade_line_item,subotiz-canal-paymentdb-pgs-subscription-period:paymentdb\\.pgs_subscription_period,subotiz-canal-paymentdb-pgs-payment-provider-method-attribute:paymentdb\\.pgs_payment_provider_method_attribute,subotiz-canal-paymentdb-pgs-payment-flow:paymentdb\\.pgs_payment_flow,subotiz-canal-paymentdb-pgs-prepayment:paymentdb\\.pgs_prepayment,subotiz-canal-llm-router-db-model-logs:llm_router_db\\.logs,subotiz-canal-llm-router-db-model-pricing:llm_router_db\\.model_pricing,subotiz-canal-llm-router-db-tokens:llm_router_db\\.tokens,subotiz-canal-llm-router-db-user-balances:llm_router_db\\.user_balances canal.mq.dynamicTopicPartitionNum = subotiz-canal-merchantdb:1 canal.mq.partitionHash = .*\..* # ========== 表过滤配置 ========== canal.instance.filter.regex = (developerdb|merchantdb|paymentdb|tradingdb|pgsbilling|promotiondb|robin|email_server|notifydb|llm_router_db)\\..* canal.instance.filter.black.regex = (mysql|dts|mqbone|oauth2_production|account|act|crab|crawfish|datafountain|email_api_production|exec|fortunecat|kirin|notify_api_production|pangu_db|leopard)\\..*,tiger\\._order_transactions_new,main\\.menus,main\\.menu_items canal.instance.filter.query = true canal.instance.filter.query.ddl = true canal.instance.filter.query.dml = true canal.instance.filter.query.regex = ^(SELECT|SHOW|DESC|EXPLAIN).*$ # ========== 心跳检测 ========== canal.instance.filter.heartbeat = true # ========== DDL解析 ========== canal.instance.parse.ddl = true # ========== 表元数据配置 ========== canal.instance.filter.table.meta = true canal.instance.tableMetaCache.enable = false # ========== TSDB表结构存储 ========== canal.instance.tsdb.enable = true # ========== 默认GTID配置 ========== canal.instance.gtidon = false # ========== 内存配置 ========== canal.instance.memory.buffer.size = 16384 canal.instance.memory.buffer.memunit = 1024 canal.instance.transaction.size = 1024 # ========== 并行解析 ========== canal.instance.parser.parallel = true
kubectl apply -f canal-config.yaml
2、部署statefulset，mysql bin点位信息放到logs目录下。不能挂载conf目录，不然pod首次启动时pvc此时目录为空导致canal启动失败
11111是canal的服务端口，11112是监控metrics端口
vim canal-statefulset.yaml
--- apiVersion: v1 kind: Service metadata: name: canal-subotiz namespace: staging spec: selector: app: canal-subotiz ports: - name: canal-server port: 11112 targetPort: 11112 type: ClusterIP --- apiVersion: apps/v1 kind: StatefulSet metadata: name: canal-subotiz namespace: staging spec: serviceName: canal-subotiz replicas: 1 selector: matchLabels: app: canal-subotiz template: metadata: labels: app: canal-subotiz spec: securityContext: runAsUser: 0 runAsGroup: 0 fsGroup: 1000 containers: - name: canal-server image: canal/canal-server:v1.1.8 imagePullPolicy: IfNotPresent ports: - containerPort: 11111 name: canal env: - name: JAVA_OPTS value: "-Xms2g -Xmx4g -XX:+UseG1GC" command: ["/bin/sh", "-c"] args: - | set -e # 初始化全局配置文件 [ ! -f /home/admin/canal-server/conf/canal.properties ] && cp /home/admin/canal-server/conf/canal.properties.example /home/admin/canal-server/conf/canal.properties # 定义update_config函数 update_config() { local config_file=$1 local custom_config=$2 while IFS='=' read -r key value; do if [[ -z "$key" || "$key" =~ ^# ]]; then continue fi local clean_key=$(echo "$key" | tr -d ' \t' | xargs) local clean_value=$(echo "$value" | xargs) local standard_line="$clean_key = $clean_value" if grep -q "^[ \t]*$clean_key[ \t]*=" "$config_file"; then sed -i "s@^\([ \t]*$clean_key[ \t]*=[ \t]*\).*@\1$clean_value@" "$config_file" echo "修改配置项：$clean_key = $clean_value" else echo "$standard_line" >> "$config_file" echo "新增配置项：$clean_key = $clean_value" fi done < "$custom_config" } # 处理全局配置 echo "===== 处理canal.properties =====" update_config "/home/admin/canal-server/conf/canal.properties" "/tmp/config/canal.properties" # 读取DEST DEST=$(grep "^canal.destinations" /home/admin/canal-server/conf/canal.properties | awk -F'=' '{print $2}' | tr -d ' \t' | xargs) echo "===== 动态识别实例名：$DEST =====" # 处理自定义实例配置 echo "===== 处理$DEST实例配置 =====" mkdir -p /home/admin/canal-server/conf/$DEST/ cp /home/admin/canal-server/conf/example/instance.properties /home/admin/canal-server/conf/$DEST/instance.properties update_config "/home/admin/canal-server/conf/${DEST}/instance.properties" "/tmp/config/${DEST}.properties" # 权限配置 chown -R admin:admin /home/admin/canal-server/conf/$DEST/ chmod -R 755 /home/admin/canal-server/conf/ mkdir -p /home/admin/canal-server/data/db /home/admin/canal-server/data/log chown -R admin:admin /home/admin/canal-server/data/db /home/admin/canal-server/data/log # 验证配置 echo "===== 最终配置验证 =====" grep "canal.destinations" /home/admin/canal-server/conf/canal.properties grep "canal.instance.master.address" /home/admin/canal-server/conf/$DEST/instance.properties echo "========================" # ===================== 核心新增：判断并删除example目录 ===================== if [ "$DEST" != "example" ]; then echo "===== 非example实例，删除原生example目录避免扫描 =====" rm -rf /home/admin/canal-server/conf/example else echo "===== 当前实例为example，保留原生example目录 =====" fi # ========================================================================== # 启动Canal /home/admin/app.sh resources: requests: memory: "1Gi" cpu: "500m" limits: memory: "4Gi" cpu: "2000m" volumeMounts: - name: canal-config mountPath: /tmp/config - name: canal-data mountPath: /home/admin/canal-server/logs volumes: - name: canal-config configMap: name: canal-subotiz volumeClaimTemplates: - metadata: name: canal-data spec: accessModes: ["ReadWriteOnce"] storageClassName: "gp2-ap-southeast-1b" resources: requests: storage: 20Gi
kubectl apply -f canal-statefulset.yaml
二、多实例
1、部署configmap，注意canal.destinations要和canal.instance.id一致
vim canal-saas-config.yaml
apiVersion: v1 kind: ConfigMap metadata: name: canal-saas namespace: staging data: # ====================== # 公共配置（所有实例共享） # ====================== canal.properties: | canal.id = 1 canal.port = 11111 canal.destinations = canal-saas,canal-ads canal.file.data.dir = /home/admin/canal-server/logs/db # ZK 公共 canal.zkServers = mse-f01be7312-zk.mse.aliyuncs.com:2181 # Kafka 公共地址 canal.serverMode = kafka kafka.bootstrap.servers = alikafka-post-cn-qzm4d4obv002-3-vpc.alikafka.aliyuncs.com:9092 canal.mq.servers = alikafka-post-cn-qzm4d4obv002-3-vpc.alikafka.aliyuncs.com:9092 # 公共 Kafka 高级参数 canal.mq.flatMessage = true canal.mq.partitionsNum = 1 canal.mq.maxRequestSize = 52428800 canal.mq.kafka.producer.retries = 5 canal.mq.kafka.producer.retry.backoff.ms = 1000 canal.mq.kafka.producer.retry.backoff.max.ms = 5000 canal.mq.kafka.producer.metadata.fetch.timeout.ms = 120000 canal.mq.kafka.producer.metadata.max.age.ms = 300000 canal.mq.partitionHash = .*\..* # 自动扫描 canal.auto.scan = true canal.auto.scan.interval = 5 # ====================== # 实例1：canal-saas（独立Topic） # ====================== canal-saas.properties: | canal.instance.id = canal-saas # ========== 实例独立 MQ 配置（核心！）========== canal.mq.topic = subotiz-canal-merchantdb canal.mq.dynamicTopic = subotiz-canal-merchantdb:product_api_dev\\..* canal.mq.dynamicTopicPartitionNum = subotiz-canal-merchantdb:1 # MySQL 连接 canal.instance.master.address = tidbdevob.aliyun-us-west-1.oceanbase.cloud:3306 canal.instance.dbUsername = canal canal.instance.dbPassword = vGgRTZubA2qsI2mN canal.instance.connectionCharset = UTF-8 # 过滤 canal.instance.filter.regex = (product_api_dev)\\..* canal.instance.filter.black.regex = (mysql|dts|mqbone|oauth2_production|account|act|crab|crawfish|datafountain|email_api_production|exec|fortunecat|kirin|notify_api_production|pangu_db|leopard)\\..*,tiger\\._order_transactions_new,main\\.menus,main\\.menu_items canal.instance.filter.query = true canal.instance.filter.query.ddl = true canal.instance.filter.query.dml = true canal.instance.filter.query.regex = ^(SELECT|SHOW|DESC|EXPLAIN).*$ # 基础 canal.mq.properties.max.request.size = 134217728 canal.mq.maxRequestSize = 134217728 canal.mq.bufferMemory = 134217728 canal.instance.filter.heartbeat = true canal.instance.parse.ddl = true canal.instance.filter.table.meta = true canal.instance.tableMetaCache.enable = false canal.instance.tsdb.enable = true canal.instance.gtidon = false canal.instance.memory.buffer.size = 16384 canal.instance.memory.buffer.memunit = 1024 canal.instance.transaction.size = 1024 canal.instance.parser.parallel = true # ====================== # 实例2：canal-ads（独立Topic） # ====================== canal-ads.properties: | canal.instance.id = canal-ads # ========== 实例独立 MQ 配置（核心！）========== canal.mq.topic = saas-canal-ads-google-categories canal.mq.dynamicTopic = saas-canal-ads-google-categories:ads_api_dev\\..* canal.mq.dynamicTopicPartitionNum = saas-canal-ads-google-categories:1 # MySQL 连接 canal.instance.master.address = tidbdevob.aliyun-us-west-1.oceanbase.cloud:3306 canal.instance.dbUsername = canal canal.instance.dbPassword = vGgRTZubA2qsI2mN canal.instance.connectionCharset = UTF-8 # 过滤 canal.instance.filter.regex = (ads_api_dev)\\..* canal.instance.filter.black.regex = (mysql|dts|mqbone)\\..* canal.instance.filter.query = true canal.instance.filter.query.ddl = true canal.instance.filter.query.dml = true canal.instance.filter.heartbeat = true canal.instance.parse.ddl = true canal.instance.tsdb.enable = true canal.instance.gtidon = false canal.instance.memory.buffer.size = 16384 canal.instance.transaction.size = 1024 canal.instance.parser.parallel = true
kc apply -f canal-saas-config.yaml
2、部署statefulset，mysql bin点位信息放到logs目录下。不能挂载conf目录，不然pod首次启动时pvc此时目录为空导致canal启动失败
11111是canal的服务端口，11112是监控metrics端口
vim canal-saas-sts.yaml
--- apiVersion: v1 kind: Service metadata: name: canal-saas namespace: staging spec: selector: app: canal-saas ports: - name: canal-server port: 11112 targetPort: 11112 type: ClusterIP --- apiVersion: apps/v1 kind: StatefulSet metadata: name: canal-saas namespace: staging spec: serviceName: canal-saas replicas: 1 selector: matchLabels: app: canal-saas template: metadata: labels: app: canal-saas spec: securityContext: runAsUser: 0 runAsGroup: 0 fsGroup: 1000 containers: - name: canal-server image: canal/canal-server:v1.1.8 imagePullPolicy: IfNotPresent ports: - containerPort: 11111 name: canal env: - name: JAVA_OPTS value: "-Xms2g -Xmx4g -XX:+UseG1GC" command: ["/bin/sh", "-c"] args: - | set -e # 初始化全局配置文件 [ ! -f /home/admin/canal-server/conf/canal.properties ] && cp /home/admin/canal-server/conf/canal.properties.example /home/admin/canal-server/conf/canal.properties # 定义update_config函数 update_config() { local config_file=$1 local custom_config=$2 while IFS='=' read -r key value; do if [[ -z "$key" || "$key" =~ ^# ]]; then continue fi local clean_key=$(echo "$key" | tr -d ' \t' | xargs) local clean_value=$(echo "$value" | xargs) local standard_line="$clean_key = $clean_value" if grep -q "^[ \t]*$clean_key[ \t]*=" "$config_file"; then sed -i "s@^\([ \t]*$clean_key[ \t]*=[ \t]*\).*@\1$clean_value@" "$config_file" echo "修改配置项：$clean_key = $clean_value" else echo "$standard_line" >> "$config_file" echo "新增配置项：$clean_key = $clean_value" fi done < "$custom_config" } # 处理全局canal.properties配置 echo "===== 处理canal.properties全局配置 =====" update_config "/home/admin/canal-server/conf/canal.properties" "/tmp/config/canal.properties" # 解析多实例列表 DEST_LIST=$(grep "^canal.destinations" /home/admin/canal-server/conf/canal.properties | awk -F'=' '{print $2}' | tr -d ' \t') echo "===== 识别多实例列表：$DEST_LIST =====" # 循环初始化每个实例 for DEST in $(echo "$DEST_LIST" | tr ',' ' '); do echo "===== 开始初始化实例：$DEST =====" mkdir -p /home/admin/canal-server/conf/${DEST}/ # 拷贝默认实例配置模板 cp /home/admin/canal-server/conf/example/instance.properties /home/admin/canal-server/conf/${DEST}/instance.properties # 加载对应实例独立配置 无多余前缀 直接${DEST}.properties update_config /home/admin/canal-server/conf/${DEST}/instance.properties /tmp/config/${DEST}.properties # 实例目录权限 chown -R admin:admin /home/admin/canal-server/conf/${DEST}/ done # 全局目录权限 chmod -R 755 /home/admin/canal-server/conf/ mkdir -p /home/admin/canal-server/data/db /home/admin/canal-server/data/log chown -R admin:admin /home/admin/canal-server/data/db /home/admin/canal-server/data/log # 删除原生example目录避免自动扫描干扰 rm -rf /home/admin/canal-server/conf/example # 启动Canal服务 /home/admin/app.sh resources: requests: memory: "1Gi" cpu: "500m" limits: memory: "4Gi" cpu: "2000m" volumeMounts: - name: canal-config mountPath: /tmp/config - name: canal-data mountPath: /home/admin/canal-server/logs volumes: - name: canal-config configMap: name: canal-saas volumeClaimTemplates: - metadata: name: canal-data spec: accessModes: ["ReadWriteOnce"] storageClassName: "alicloud-disk-ssd" resources: requests: storage: 20Gi
kc apply -f canal-saas-sts.yaml
浙公网安备 33010602011771号