seatunnel单机部署

环境:
OS:Centos 7
seatunnel:2.3.13

1.查看java版本
[root@flink-134 config]# java -version
java version "1.8.0_361"
Java(TM) SE Runtime Environment (build 1.8.0_361-b09)
Java HotSpot(TM) 64-Bit Server VM (build 25.361-b09, mixed mode)

 

2.介质下载
下载地址:
https://archive.apache.org/dist/seatunnel/

cd /soft
wget https://archive.apache.org/dist/seatunnel/2.3.13/apache-seatunnel-2.3.13-bin.tar.gz

 

3.安装
[root@flink-134 soft]#cd /soft
[root@flink-134 soft]#tar -xvf apache-seatunnel-2.3.13-bin.tar.gz
[root@flink-134 soft]#mv apache-seatunnel-2.3.13 /opt/seatunnel-2.3.13

4.安装连接器
查看默认的链接器
[root@flink-134 connectors]#more /opt/seatunnel-2.3.13/config/plugin_config
[root@flink-134 connectors]#sh /opt/seatunnel-2.3.13/bin/install-plugin.sh 2.3.13
执行完,插件会放到 /opt/seatunnel-2.3.13/connectors 这个目录

若使用的链接器很多的话,这里下载要很久.

若不想全部下载的话,修改下这个配置文件,指定具体的链接器
vi /opt/seatunnel-2.3.13/config/plugin_config

--connectors-v2--
connector-doris
connector-jdbc
--end--

 

5.准备相关驱动
cd /opt/seatunnel-2.3.13/lib
wget https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar

将connectors下的插件拷贝到plugins目录
cd /opt/seatunnel-2.3.13
mkdir -p /opt/seatunnel-2.3.13/plugins/doris
mkdir -p /opt/seatunnel-2.3.13/plugins/jdbc

cp /opt/seatunnel-2.3.13/connectors/connector-doris-2.3.13.jar /opt/seatunnel-2.3.13/plugins/doris/
cp /opt/seatunnel-2.3.13/connectors/connector-jdbc-2.3.13.jar /opt/seatunnel-2.3.13/plugins/jdbc/

 

6.修改默认端口

vi /opt/seatunnel-2.3.13/config/seatunnel.yaml

seatunnel:
  engine:
    classloader-cache-mode: true
    history-job-expire-minutes: 1440
    backup-count: 1
    queue-type: blockingqueue
    print-execution-info-interval: 60
    print-job-metrics-info-interval: 60
    slot-service:
      dynamic-slot: true
    checkpoint:
      interval: 10000
      timeout: 60000
      storage:
        type: hdfs
        max-retained: 3
        plugin-config:
          namespace: /tmp/seatunnel/checkpoint_snapshot
          storage.type: hdfs
          fs.defaultFS: file:///tmp/ # Ensure that the directory has written permission
    telemetry:
      metric:
        enabled: false
      logs:
        scheduled-deletion-enable: true
    http:
      enable-http: true
      port: 8081
      enable-dynamic-port: false
      # Uncomment the following lines to enable basic authentication for web UI
      # enable-basic-auth: true
      # basic-auth-username: admin
      # basic-auth-password: admin

 

7.启动
[root@flink-134 seatunnel-2.3.13]# cd /opt/seatunnel-2.3.13
[root@flink-134 seatunnel-2.3.13]# mkdir -p logs
#-d 代表daemon后台守护启动
[root@flink-134 seatunnel-2.3.13]# sh /opt/seatunnel-2.3.13/bin/seatunnel-cluster.sh -d

6.验证
# 看进程
jps | grep SeaTunnel
# 或者访问web
http://192.168.1.134:8081

 

#################################同步验证#####################################
1.准备同步脚本文件

[root@flink-134 myconf]# more /opt/seatunnel-2.3.13/myconf/doris2mysql_01.conf 
env {
  execution.parallelism = 2
  job.mode = "BATCH"
}

source {
  Doris {
    fenodes = "192.168.12.87:8030"
    query-port = 9030
    username = "root"
    password = "mysql"
    database = "db_test"
    table = "tb_hxl"
  }
}

sink {
  jdbc {
    url = "jdbc:mysql://192.168.1.113:3306/db_test?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai"
    driver = "com.mysql.cj.jdbc.Driver"
    username = "hxl"
    password = "mysql"
    generate_sink_sql = true
    database = "db_test"
    table = "tb_hxl"
  }
}

 

2.异步提交任务
/opt/seatunnel-2.3.13/bin/seatunnel.sh -c /opt/seatunnel-2.3.13/myconf/doris2mysql_01.conf --async
这个时候界面上可以查看得到正在运行的任务

3.任务管理
# 查看所有任务列表,拿到 jobId
[root@flink-134 config]#/opt/seatunnel-2.3.13/bin/seatunnel.sh --list

 

# 取消任务(把2换成你查到的 jobId)
[root@flink-134 config]#/opt/seatunnel-2.3.13/bin/seatunnel.sh --cancel-job 2

# 查看任务指标
[root@flink-134 config]#/opt/seatunnel-2.3.13/bin/seatunnel.sh --metrics 2

 

4.停掉seatunnel

/opt/seatunnel-2.3.13/bin/stop-seatunnel-cluster.sh