datax以k8s job形式运行

镜像制作

FROM centos/python-27-centos7
USER root
#安装jdk
ADD jdk-8u321-linux-x64.tar.gz /opt/local
ENV JAVA_HOME /opt/local/jdk1.8.0_321
ENV PATH $JAVA_HOME/bin:$PATH


# 添加datax文件到/opt/local 目录
RUN mkdir /opt/local/datax
COPY ./datax /opt/local/datax

# 设置运行的工作目录，可不添加,运行中使用 docker -w 指定
WORKDIR /opt/local/datax

# 执行脚本
COPY ./run.sh /opt/local/datax/run.sh
COPY ./job0.json /opt/local/datax/job0.json
RUN chmod 777 /opt/local/datax/run.sh
RUN rm -rf plugin/reader/.DS_Store
RUN rm -rf plugin/writer/.DS_Store

ENTRYPOINT ["sh","./run.sh","job/job.json"]
# docker run -it --rm datax_img:v1

datax运行脚本run.sh:

#! /bin/bash
python /opt/local/datax/bin/datax.py $1

将datax的json配置文件存储在configmap中

apiVersion: v1
kind: ConfigMap
metadata:
  name: datax-json-file
  namespace: test
data:
  job.json: |
    {
  "job": {
    "setting": {
      "speed": {
        "channel": 3,
        "byte": 1048576
      },
      "errorLimit": {
        "record": 0,
        "percentage": 0.02
      }
    },
    "content": [
      {
        "reader": {
          "name": "mysqlreader",
          "parameter": {
            "username": "root",
            "password": "root1234",
            "splitPk": "",
            "connection": [
              {
                "querySql": [
                  "select id, age from quality_people;"
                ],
                "jdbcUrl": [
                  "jdbc:mysql://192.168.29.156:3306/f442?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8&allowPublicKeyRetrieval=true"
                ]
              }
            ]
          }
        },
        "writer": {
          "name": "sqlserverwriter",
          "parameter": {
            "username": "sa",
            "password": "OcP2020123",
            "column": [
              "[id]",
              "[age]"
            ],
            "connection": [
              {
                "table": [
                  "dbo.people2"
                ],
                "jdbcUrl": "jdbc:sqlserver://192.168.29.156:1433;DatabaseName=demo"
              }
            ]
          }
        }
      }
    ]
  }
}

k8s job编排文件

kind: Job
apiVersion: batch/v1
metadata:
  name: datax-25
  namespace: test
  labels:
    job-name: datax-25
spec:
  template:
    metadata:
      name: datax-25
      labels:
        job-name: datax-25
    spec:
      volumes:
        - name: json-file
          configMap:
            name: datax-json-file
            defaultMode: 420
      containers:
        - name: datax-25
          image: 'datax_img:v1'
          env:
            - name: TZ
              value: Asia/Shanghai
          volumeMounts:
            - name: json-file
              mountPath: /opt/local/datax/job/job.json
              subPath: job.json              
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          imagePullPolicy: Always
      restartPolicy: Never
      terminationGracePeriodSeconds: 30
      dnsPolicy: ClusterFirst
      securityContext: {}
      schedulerName: default-scheduler