seatunnel单机部署(local 模式)

环境:
OS:Centos 7
seatunnel:2.3.13

 

1.查看java版本
[root@flink-134 config]# java -version
java version "1.8.0_361"
Java(TM) SE Runtime Environment (build 1.8.0_361-b09)
Java HotSpot(TM) 64-Bit Server VM (build 25.361-b09, mixed mode)

 

2.介质下载
下载地址:
https://archive.apache.org/dist/seatunnel/

cd /soft
wget https://archive.apache.org/dist/seatunnel/2.3.13/apache-seatunnel-2.3.13-bin.tar.gz

 

3.安装
[root@flink-134 soft]#cd /soft
[root@flink-134 soft]#tar -xvf apache-seatunnel-2.3.13-bin.tar.gz
[root@flink-134 soft]#mv apache-seatunnel-2.3.13 /opt/seatunnel-2.3.13

 

4.安装连接器
查看默认的链接器
[root@flink-134 connectors]#more /opt/seatunnel-2.3.13/config/plugin_config
[root@flink-134 connectors]#sh /opt/seatunnel-2.3.13/bin/install-plugin.sh 2.3.13
执行完,插件会放到 /opt/seatunnel-2.3.13/connectors 这个目录

若使用的链接器很多的话,这里下载要很久.

若不想全部下载的话,修改下这个配置文件,指定具体的链接器
vi /opt/seatunnel-2.3.13/config/plugin_config

--connectors-v2--
connector-doris
connector-jdbc
--end--

 

5.环境配置
vi /opt/seatunnel-2.3.13/config/seatunnel-env.sh
一般默认即可,可按需调整 JVM 参数:
export SEATUNNEL_HEAP_OPTS="-Xms512M -Xmx2G"

 

6.修改端口
避免与现有端口冲突

vi /opt/seatunnel-2.3.13/config/seatunnel.yaml

seatunnel:
  engine:
    classloader-cache-mode: true
    history-job-expire-minutes: 1440
    backup-count: 1
    queue-type: blockingqueue
    print-execution-info-interval: 60
    print-job-metrics-info-interval: 60
    slot-service:
      dynamic-slot: true
    checkpoint:
      interval: 10000
      timeout: 60000
      storage:
        type: hdfs
        max-retained: 3
        plugin-config:
          namespace: /tmp/seatunnel/checkpoint_snapshot
          storage.type: hdfs
          fs.defaultFS: file:///tmp/ # Ensure that the directory has written permission
    telemetry:
      metric:
        enabled: false
      log:
        scheduled-deletion-enable: true
    http:
      enable-http: true
      port: 8081
      enable-dynamic-port: false

 

8.同步doris数据到mysql
8.1 准备好配置文件
vi /opt/seatunnel-2.3.13/myconf/doris2mysql.conf

env {
  execution.parallelism = 2
  job.mode = "BATCH"
}

source {
  Doris {
    fenodes = "192.168.12.87:8030"
    query-port = 9030
    username = "root"
    password = "mysql"
    database = "db_test"
    table = "tb_hxl"
  }
}


sink {
  jdbc {
    url = "jdbc:mysql://192.168.1.14:3306/db_test?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai"
    driver = "com.mysql.cj.jdbc.Driver"
    username = "hxl"
    password = "mysql"
    query = "INSERT INTO tb_hxl(id,name1,name2,name3,name4,name5,name6,name7,name8,name9,name10,create_time,update_time) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)"
    save_mode = "NONE"
  }
}

 

vi /opt/seatunnel-2.3.13/myconf/doris2mysql_01.conf

env {
  execution.parallelism = 2
  job.mode = "BATCH"
}

source {
  Doris {
    fenodes = "192.168.12.87:8030"
    query-port = 9030
    username = "root"
    password = "mysql"
    database = "db_test"
    table = "tb_hxl"
  }
}

sink {
  jdbc {
    url = "jdbc:mysql://192.168.1.14:3306/db_test?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai"
    driver = "com.mysql.cj.jdbc.Driver"
    username = "hxl"
    password = "mysql"
    generate_sink_sql = true
    database = "db_test"
    table = "tb_hxl"
  }
}

 

mysql端需要提前创建好表,需要定义主键

 

8.2 需要下载相应的驱动
cd /opt/seatunnel-2.3.13/lib
wget https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar

 

将connectors下的插件拷贝到plugins目录
cd /opt/seatunnel-2.3.13
mkdir -p /opt/seatunnel-2.3.13/plugins/doris
mkdir -p /opt/seatunnel-2.3.13/plugins/jdbc

cp /opt/seatunnel-2.3.13/connectors/connector-doris-2.3.13.jar /opt/seatunnel-2.3.13/plugins/doris/
cp /opt/seatunnel-2.3.13/connectors/connector-jdbc-2.3.13.jar /opt/seatunnel-2.3.13/plugins/jdbc/

8.3 运行
/opt/seatunnel-2.3.13/bin/seatunnel.sh -m local -c /opt/seatunnel-2.3.13/myconf/doris2mysql.conf

/opt/seatunnel-2.3.13/bin/seatunnel.sh -m local -c /opt/seatunnel-2.3.13/myconf/doris2mysql_01.conf