strimzi-kafka 实现kafka集群之间同步数据（k8s集群）

 

 

vim kafka-mirrormaker2.yaml

 

一、4.3.1 版本

 

apiVersion: kafka.strimzi.io/v1
kind: KafkaMirrorMaker2
metadata:
  name: msk-mm2
  namespace: kafka
spec:
  replicas: 1
  version: 4.3.1
  resources:
    requests:
      cpu: 500m
      memory: 1Gi
    limits:
      cpu: "2000m"
      memory: 2Gi

  target:
    alias: local
    bootstrapServers: mq-kafka-bootstrap.kafka.svc.cluster.local:9092
    groupId: msk-mm2-group
    configStorageTopic: msk-mm2-configs
    offsetStorageTopic: msk-mm2-offsets
    statusStorageTopic: msk-mm2-status
    config:
      config.storage.replication.factor: 3
      offset.storage.replication.factor: 3
      status.storage.replication.factor: 3
      producer.max.request.size: 69206016
      producer.buffer.memory: 134217728

  mirrors:
    - source:
        alias: msk
        bootstrapServers: b-3.subotizprodmsk.hb7y9u.c1.kafka.us-west-2.amazonaws.com:9092,b-1.subotizprodmsk.hb7y9u.c1.kafka.us-west-2.amazonaws.com:9092,b-2.subotizprodmsk.hb7y9u.c1.kafka.us-west-2.amazonaws.com:9092
        config:
          consumer.fetch.max.bytes: 75497472
          consumer.max.partition.fetch.bytes: 73400320
      topicsPattern: ".*"
      topicsExcludePattern: "__.*,mm2-.*,local-to-msk-mm2-.*,.*\\.internal,.*\\.replica,mirrormaker2-.*"
      groupsPattern: ".*"
      sourceConnector:
        tasksMax: 4
        config:
          replication.policy.class: org.apache.kafka.connect.mirror.IdentityReplicationPolicy
          sync.topic.acls.enabled: "false"
          sync.topic.configs.enabled: "true"
          consumer.auto.offset.reset: "earliest"
          replication.factor: "3"
          offset-syncs.topic.replication.factor: "2"
          producer.override.max.request.size: "69206016"
          producer.override.buffer.memory: "134217728"
          consumer.fetch.max.bytes: "75497472"
          consumer.max.partition.fetch.bytes: "73400320"
      checkpointConnector:
        tasksMax: 1
        config:
          replication.policy.class: org.apache.kafka.connect.mirror.IdentityReplicationPolicy
          sync.group.offsets.enabled: "true"
          sync.group.offsets.interval.seconds: "10"
          emit.checkpoints.interval.seconds: "10"
          refresh.groups.interval.seconds: "30"
          checkpoints.topic.replication.factor: "3"

 

二、3.9.2 版本

apiVersion: kafka.strimzi.io/v1beta2
kind: KafkaMirrorMaker2
metadata:
  name: msk-to-strimzi
  namespace: kafka
spec:
  version: 3.9.2
  replicas: 1
  connectCluster: strimzi
  clusters:
    - alias: msk
      bootstrapServers: "b-3.subotizprodmsk.hb7y9u.c1.kafka.us-west-2.amazonaws.com:9092,b-1.subotizprodmsk.hb7y9u.c1.kafka.us-west-2.amazonaws.com:9092,b-2.subotizprodmsk.hb7y9u.c1.kafka.us-west-2.amazonaws.com:9092"
      config:
        consumer.fetch.max.bytes: 75497472
        consumer.max.partition.fetch.bytes: 73400320
    - alias: strimzi
      bootstrapServers: mq-kafka-bootstrap.kafka.svc.cluster.local:9092
      config:
        config.storage.replication.factor: 3
        offset.storage.replication.factor: 3
        status.storage.replication.factor: 3
        producer.max.request.size: 69206016
        producer.buffer.memory: 134217728
  mirrors:
    - sourceCluster: msk
      targetCluster: strimzi
      topicsPattern: "^(?!__.*|_confluent|connect-|local-to-msk-mm2-|mirrormaker2-).*"
      sourceConnector:
        tasksMax: 4
        config:
          replication.policy.class: org.apache.kafka.connect.mirror.IdentityReplicationPolicy
          sync.topic.acls.enabled: "false"
          sync.topic.configs.enabled: "true"
          refresh.topics.interval.seconds: "30"
          replication.factor: "3"
          offset-syncs.topic.replication.factor: "2"
          producer.override.max.request.size: "69206016"
          producer.override.buffer.memory: "134217728"
          consumer.override.fetch.max.bytes: "75497472"
          consumer.override.max.partition.fetch.bytes: "73400320"
      checkpointConnector:
        tasksMax: 1
        config:
          replication.policy.class: org.apache.kafka.connect.mirror.IdentityReplicationPolicy
          sync.group.offsets.enabled: "true"
          sync.group.offsets.interval.seconds: "10"
          emit.checkpoints.interval.seconds: "10"
          refresh.groups.interval.seconds: "30"
          checkpoints.topic.replication.factor: "3"
  resources:
    requests:
      cpu: 200m
      memory: 1Gi
    limits:
      cpu: "2000m"
      memory: 2Gi