airflow dags 任务 python脚本
from datetime import datetime, timedelta
from textwrap import dedent
from airflow import DAG
from airflow.operators.bash import BashOperator
from airflow.contrib.operators.ssh_operator import SSHOperator
from airflow.sensors.external_task import ExternalTaskMarker, ExternalTaskSensor
from airflow.operators.dummy import DummyOperator
default_args = {
'owner': 'LinuxOperation',
'email': ['airflow@maetimes.com'],
'email_on_failure': True,
'email_on_retry': True,
'depends_on_past': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
'concurrency': 1,
'execution_timeout': timedelta(minutes=30),
}
with DAG(
dag_id='mongo_export',
description='pokekara_chat_messages doki_chat_messages export json file',
default_args=default_args,
schedule_interval=("10 3 * * *"),
start_date=datetime(2021, 12, 3,),
catchup=False,
tags=['mongo_export_task'],
) as dag:
poke_chat_exec = dedent(
"""
bash {{ params.dir }}/{{ params.exec_file }}
"""
)
doki_chat_exec = dedent(
"""
bash {{ params.dir }}/{{ params.exec_file }}
"""
)
mongo_export = SSHOperator(
ssh_conn_id='ssh-datanode04',
task_id='poke_chat_messages_export',
depends_on_past=False,
command=poke_chat_exec,
params={
'exec_file': 'mongo_export.sh',
'dir': '/usr/local/src/scripts'
},
)
mongo_export = SSHOperator(
ssh_conn_id='ssh-datanode04',
task_id='doki_chat_messages_export',
depends_on_past=False,
command=doki_chat_exec,
params={
'exec_file': 'doki_chat_messages.sh',
'dir': '/usr/local/src/scripts'
},
)
airflow会自动识别该脚本任务并执行。
浙公网安备 33010602011771号