k8s部署canal 1.1.8

 

canal部署的是单实例

1、部署configmap，注意canal.destinations要和canal.instance.id一致

vim canal-config.yaml

 

apiVersion: v1
kind: ConfigMap
metadata:
  name: canal-config
  namespace: canal
data:
  canal.properties: |
    # ========== Canal基础配置 ==========
    canal.id = 1
    canal.port = 11111
    canal.destinations = canal-subotiz-common
    
    # ========== 持久化路径 ==========
    canal.file.data.dir = /home/admin/canal-server/logs/db
    
    # ========== Zookeeper配置 ==========
    canal.zkServers = z-2.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:2181,z-1.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:2181,z-3.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:2181
    
    # ========== Server模式配置 ==========
    canal.serverMode = kafka
    
    # ========== Kafka生产者配置 ==========
    canal.mq.servers = b-2.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:9092,b-1.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:9092,b-3.subotizstgmsk.ck8tsh.c2.kafka.ap-southeast-1.amazonaws.com:9092
    canal.mq.topic = subotiz-canal-merchantdb
    canal.mq.flatMessage = true
    canal.mq.partitionsNum = 1
    canal.mq.maxRequestSize = 52428800
    
    # Kafka生产者高级配置
    canal.mq.kafka.producer.retries = 5
    canal.mq.kafka.producer.retry.backoff.ms = 1000
    canal.mq.kafka.producer.retry.backoff.max.ms = 5000
    canal.mq.kafka.producer.metadata.fetch.timeout.ms = 120000
    canal.mq.kafka.producer.metadata.max.age.ms = 300000
    
    # ========== 动态Topic配置 ==========
    canal.mq.dynamicTopic = canal-subotiz-test-table:test1_sre\\.test_users 
    canal.mq.dynamicTopicPartitionNum = subotiz-canal-merchantdb:1
    canal.mq.partitionHash = .*\..*
    
    # ========== 自动扫描配置 ==========
    canal.auto.scan = true
    canal.auto.scan.interval = 5
  
  # instance.properties 实例配置
  instance.properties: |
    # ========== 核心新增：实例ID（必须和destinations一致） ==========
    canal.instance.id = canal-subotiz-common

    # ========== MySQL连接配置 ==========
    canal.instance.master.address = mypaydb1.cluster-cl20ky4iyqmn.ap-southeast-1.rds.amazonaws.com:3306
    canal.instance.dbUsername = canal
    canal.instance.dbPassword = vGgRTZubA2qsI2mN
    canal.instance.connectionCharset = UTF-8
    
    # ========== 表过滤配置 ==========
    canal.instance.filter.regex = (test1_sre)\..*
    canal.instance.filter.black.regex = (mysql|dts|mqbone|oauth2_production|account|act|crab|crawfish|datafountain|email_api_production|exec|fortunecat|kirin|notify_api_production|pangu_db|leopard)\\..*,tiger\\._order_transactions_new,main\\.menus,main\\.menu_items
    canal.instance.filter.query = true
    canal.instance.filter.query.ddl = true
    canal.instance.filter.query.dml = true
    canal.instance.filter.query.regex = ^(SELECT|SHOW|DESC|EXPLAIN).*$
    
    # ========== 心跳检测 ==========
    canal.instance.filter.heartbeat = true
    
    # ========== DDL解析 ==========
    canal.instance.parse.ddl = true
    
    # ========== 表元数据配置 ==========
    canal.instance.filter.table.meta = true
    canal.instance.tableMetaCache.enable = false
    
    # ========== TSDB表结构存储 ==========
    canal.instance.tsdb.enable = true
    
    # ========== 默认GTID配置 ==========
    canal.instance.gtidon = false
    
    # ========== 内存配置 ==========
    canal.instance.memory.buffer.size = 16384
    canal.instance.memory.buffer.memunit = 1024
    canal.instance.transaction.size = 1024
    
    # ========== 并行解析 ==========
    canal.instance.parser.parallel = true

 

kubectl apply -f  canal-config.yaml

 

2、部署statefulset，mysql bin点位信息放到logs目录下。不能挂载conf目录，不然pod首次启动时pvc此时目录为空导致canal启动失败

vim canal-statefulset.yaml

 

---
apiVersion: v1
kind: Service
metadata:
  name: canal-service
  namespace: canal
spec:
  selector:
    app: canal-server
  ports:
  - name: canal-server
    port: 11111
    targetPort: 11111
  type: ClusterIP
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: canal-server
  namespace: canal
spec:
  serviceName: canal-service
  replicas: 1
  selector:
    matchLabels:
      app: canal-server
  template:
    metadata:
      labels:
        app: canal-server
    spec:
      securityContext:
        runAsUser: 0
        runAsGroup: 0
        fsGroup: 1000
      containers:
      - name: canal-server
        image: canal/canal-server:v1.1.8
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 11111
          name: canal
        env:
        - name: JAVA_OPTS
          value: "-Xms2g -Xmx4g -XX:+UseG1GC"
        command: ["/bin/sh", "-c"]
        args:
        - |
          set -e
          # 初始化全局配置文件
          [ ! -f /home/admin/canal-server/conf/canal.properties ] && cp /home/admin/canal-server/conf/canal.properties.example /home/admin/canal-server/conf/canal.properties
          
          # 定义update_config函数
          update_config() {
              local config_file=$1
              local custom_config=$2
              while IFS='=' read -r key value; do
                  if [[ -z "$key" || "$key" =~ ^# ]]; then
                      continue
                  fi
                  local clean_key=$(echo "$key" | tr -d ' \t' | xargs)
                  local clean_value=$(echo "$value" | xargs)
                  local standard_line="$clean_key = $clean_value"
                  if grep -q "^[ \t]*$clean_key[ \t]*=" "$config_file"; then
                      sed -i "s@^\([ \t]*$clean_key[ \t]*=[ \t]*\).*@\1$clean_value@" "$config_file"
                      echo "修改配置项：$clean_key = $clean_value"
                  else
                      echo "$standard_line" >> "$config_file"
                      echo "新增配置项：$clean_key = $clean_value"
                  fi
              done < "$custom_config"
          }
          
          # 处理全局配置
          echo "===== 处理canal.properties ====="
          update_config "/home/admin/canal-server/conf/canal.properties" "/tmp/config/canal.properties"
          
          # 读取DEST
          DEST=$(grep "^canal.destinations" /home/admin/canal-server/conf/canal.properties | awk -F'=' '{print $2}' | tr -d ' \t' | xargs)
          echo "===== 动态识别实例名：$DEST ====="
          
          # 处理自定义实例配置
          echo "===== 处理$DEST实例配置 ====="
          mkdir -p /home/admin/canal-server/conf/$DEST/
          cp /home/admin/canal-server/conf/example/instance.properties /home/admin/canal-server/conf/$DEST/instance.properties
          update_config "/home/admin/canal-server/conf/$DEST/instance.properties" "/tmp/config/instance.properties"
          
          # 权限���置
          chown -R admin:admin /home/admin/canal-server/conf/$DEST/
          chmod -R 755 /home/admin/canal-server/conf/
          mkdir -p /home/admin/canal-server/data/db /home/admin/canal-server/data/log
          chown -R admin:admin /home/admin/canal-server/data/db /home/admin/canal-server/data/log
          
          # 验证配置
          echo "===== 最终配置验证 ====="
          grep "canal.destinations" /home/admin/canal-server/conf/canal.properties
          grep "canal.instance.master.address" /home/admin/canal-server/conf/$DEST/instance.properties
          echo "========================"
          
          # ===================== 核心新增：判断并删除example目录 =====================
          if [ "$DEST" != "example" ]; then
              echo "===== 非example实例，删除原生example目录避免扫描 ====="
              rm -rf /home/admin/canal-server/conf/example
          else
              echo "===== 当前实例为example，保留原生example目录 ====="
          fi
          # ==========================================================================
          
          # 启动Canal
          /home/admin/app.sh
        resources:
          requests:
            memory: "1Gi"
            cpu: "500m"
          limits:
            memory: "3Gi"
            cpu: "2000m"
        volumeMounts:
        - name: canal-config
          mountPath: /tmp/config
        - name: canal-data
          mountPath: /home/admin/canal-server/data
      volumes:
      - name: canal-config
        configMap:
          name: canal-config
  volumeClaimTemplates:
  - metadata:
      name: canal-data
    spec:
      accessModes: ["ReadWriteOnce"]
      storageClassName: "gp2-ap-southeast-1b"
      resources:
        requests:
          storage: 20Gi

 

 

kubectl apply -f  canal-statefulset.yaml