Created
September 21, 2021 01:55
-
-
Save carlosverduzco/f839a9fc19684b9ea8135eb5ec038a9c to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from datetime import timedelta | |
| from textwrap import dedent | |
| from airflow import DAG | |
| from airflow.operators.bash import BashOperator | |
| from airflow.utils.dates import days_ago | |
| default_args = { | |
| 'owner': 'airflow', | |
| 'depends_on_past': False, | |
| 'email': ['carloseverduzco@hotmail.com','jefe@hotmail.com'], | |
| 'email_on_failure': ['carloseverduzco@hotmail.com'], | |
| 'email_on_retry': False, | |
| 'retries': 1, | |
| 'retry_delay': timedelta(minutes=5), | |
| } | |
| with DAG( | |
| 'tutorial', | |
| default_args=default_args, | |
| description='A simple tutorial DAG', | |
| schedule_interval=timedelta(hours=12), | |
| start_date=days_ago(2), | |
| tags=['example'], | |
| ) as dag: | |
| t1 = BashOperator( | |
| task_id='print_date', | |
| bash_command='date', | |
| ) | |
| t2 = BashOperator( | |
| task_id='sleep', | |
| depends_on_past=False, | |
| bash_command='sleep 5', | |
| retries=3, | |
| ) | |
| t1.doc_md = dedent( | |
| """\ | |
| #### Task Documentation | |
| You can document your task using the attributes `doc_md` (markdown), | |
| `doc` (plain text), `doc_rst`, `doc_json`, `doc_yaml` which gets | |
| rendered in the UI's Task Instance Details page. | |
|  | |
| """ | |
| ) | |
| dag.doc_md = __doc__ # providing that you have a docstring at the beggining of the DAG | |
| dag.doc_md = """ | |
| This is a documentation placed anywhere | |
| """ # otherwise, type it like this | |
| templated_command = dedent( | |
| """ | |
| {% for i in range(5) %} | |
| echo "{{ ds }}" | |
| echo "{{ macros.ds_add(ds, 7)}}" | |
| echo "{{ params.my_param }}" | |
| {% endfor %} | |
| """ | |
| ) | |
| t3 = BashOperator( | |
| task_id='templated', | |
| depends_on_past=False, | |
| bash_command=templated_command, | |
| params={'my_param': 'Parameter I passed in'}, | |
| ) | |
| t4 = BashOperator( | |
| task_id='save_db', | |
| depends_on_past=False, | |
| bash_command='py saveDB.py', | |
| ) | |
| t4.doc_md = dedent( | |
| """\ | |
| #### Save DB | |
| execute saveDB.py code, this code save the DB in CSV and upload to Amazon S3 | |
| """ | |
| ) | |
| t1 >> [t2, t3] >> t4 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment