Example Airflow DAG that shows how to use Google Dataprep.
This DAG relies on the following OS environment variables
Module Contents
-
tests.system.providers.google.cloud.dataprep.example_dataprep.ENV_ID[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.DAG_ID = 'example_dataprep'[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.CONNECTION_ID[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.DATAPREP_TOKEN[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.GCP_PROJECT_ID[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.GCS_BUCKET_NAME[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.GCS_BUCKET_PATH[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.DATASET_URI = 'gs://airflow-system-tests-resources/dataprep/dataset-00000.parquet'[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.DATASET_NAME[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.DATASET_WRANGLED_NAME[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.DATASET_WRANGLED_ID = "{{ task_instance.xcom_pull('create_wrangled_dataset')['id'] }}"[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.FLOW_ID = "{{ task_instance.xcom_pull('create_flow')['id'] }}"[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.FLOW_COPY_ID = "{{ task_instance.xcom_pull('copy_flow')['id'] }}"[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.RECIPE_NAME[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.WRITE_SETTINGS[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.log[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.create_bucket_task[source]
-
tests.system.providers.google.cloud.dataprep.example_dataprep.test_run[source]