from pathlib import Path
from airflow.utils.dates import days_ago
from airflow import DAG
import os

from airflow.operators.python import PythonVirtualenvOperator

dag = DAG('run-iati', start_date=days_ago(1), catchup=False)

def run_iati():

    import iatidata
    import os

    os.chdir('/tmp')
    os.environ["IATI_TABLES_SCHEMA"] = "iati" 
    os.environ["IATI_TABLES_OUTPUT"] = "/tmp" 
    iatidata.run_all(processes=6)

with dag:
    virtualenv_task = PythonVirtualenvOperator(
	task_id="runiati",
	python_callable=run_iati,
	requirements=["git+https://github.com/codeforIATI/iati-tables.git@main"],
	system_site_packages=False,
    )