milvus-cdc部署数据同步

环境:
milvus:2.4.23 部署方式:standalone,消息中间件不能使用自带的rocksmq,必须使用pulsar或是kafka,我这里使用的是kafka,kafka部署在与milvus同个机器
milvus-cdc:v1.0.0 发现v2有问题
源服务器:192.168.1.135
目标服务器:192.168.1.134

 

说明:
milvus-cdc部署在源服务器192.168.1.135

 

1.下载milvus-cdc
下载地址
https://github.com/zilliztech/milvus-cdc/releases/download/v1.0.0/milvus-cdc_Linux_x86_64.tar.gz

 

2.准备程序存放目录
mkdir -p /opt/milvus-cdc ##主目录
mkdir -p /opt/milvus-cdc/configs ##配置文件存放目录

 

3.解压
tar -xvf milvus-cdc_Linux_x86_64.tar.gz
mv milvus-cdc /opt/milvus-cdc

 

4.准备配置文件

[root@host135 configs]# more /opt/milvus-cdc/configs/cdc.yaml 
address: 0.0.0.0:8444
maxTaskNum: 200

metaStoreConfig:
  # the metastore type, available value: etcd, mysql
  storeType: etcd
  # etcd address
  etcdEndpoints:
    - localhost:2379
  # mysql connection address
  # mysqlSourceUrl: root:root@tcp(127.0.0.1:3306)/milvus-cdc?charset=utf8
  # meta data prefix, if multiple cdc services use the same store service, you can set different rootPaths to achieve multi-tenancy
  rootPath: cdc

sourceConfig:
  # etcd config
  etcdAddress:
    - localhost:2379
  etcdRootPath: by-dev
  etcdMetaSubPath: meta
  # default partition name
  defaultPartitionName: _default
  # read buffer length, mainly used for buffering if writing data to milvus-target is slow.
  readChanLen: 10
  replicateChan: by-dev-replicate-msg
  # milvus-source mq config, which is pulsar or kafka
  ##pulsar:
  ##  address: pulsar://localhost:6650
  ##  webAddress: localhost:80
  ##  maxMessageSize: 5242880
  ##  tenant: public
  ##  namespace: default
  ##   authPlugin: org.apache.pulsar.client.impl.auth.AuthenticationToken
  ##   authParams: token:xxx
  kafka:
    address: localhost:9092 ##这里写kafka ip好像不行

 

5.启动
./milvus-cdc server

 

6.创建任务

curl -X POST http://localhost:8444/cdc \
-H "Content-Type: application/json" \
-d '{
  "request_type": "create",
  "request_data": {
    "milvus_connect_param": {
      "host": "192.168.1.134",
      "port": 19530,
      "username":"root",
      "password":"Milvus",
      "connect_timeout": 10
    },
    "collection_infos": [
      {
        "name": "*"
      }
    ],
    "rpc_channel_info": {
      "name": "by-dev-replicate-msg"
    }    
  }
}'

 

7.查看任务

curl -X POST -H "Content-Type: application/json" -d '{
  "request_type": "list"
}' http://localhost:8444/cdc

 

8.恢复任务

curl -X POST -H "Content-Type: application/json" -d '{
  "request_type":"resume",
  "request_data": {
    "task_id": "e533062521d64ac0a83f730368c26ea4"
  }
}' http://localhost:8444/cdc