Example Airflow DAG that demonstrates operators for the Google Vertex AI service in the Google
Cloud Platform.
This DAG relies on the following OS environment variables:
GCP_VERTEX_AI_BUCKET - Google Cloud Storage bucket where the model will be saved
after training process was finished.
CUSTOM_CONTAINER_URI - path to container with model.
PYTHON_PACKAGE_GSC_URI - path to test model in archive.
LOCAL_TRAINING_SCRIPT_PATH - path to local training script.
DATASET_ID - ID of dataset which will be used in training process.
MODEL_ID - ID of model which will be used in predict process.
MODEL_ARTIFACT_URI - The artifact_uri should be the path to a GCS directory containing saved model
artifacts.
Module Contents
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.PROJECT_ID[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.REGION[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.BUCKET[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.STAGING_BUCKET[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.DISPLAY_NAME[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.CONTAINER_URI = gcr.io/cloud-aiplatform/training/tf-cpu.2-2:latest[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.CUSTOM_CONTAINER_URI[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.MODEL_SERVING_CONTAINER_URI = gcr.io/cloud-aiplatform/prediction/tf2-cpu.2-2:latest[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.REPLICA_COUNT = 1[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.MACHINE_TYPE = n1-standard-4[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.ACCELERATOR_TYPE = ACCELERATOR_TYPE_UNSPECIFIED[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.ACCELERATOR_COUNT = 0[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.TRAINING_FRACTION_SPLIT = 0.7[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.TEST_FRACTION_SPLIT = 0.15[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.VALIDATION_FRACTION_SPLIT = 0.15[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.PYTHON_PACKAGE_GCS_URI[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.PYTHON_MODULE_NAME = aiplatform_custom_trainer_script.task[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.LOCAL_TRAINING_SCRIPT_PATH[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.TRAINING_PIPELINE_ID = test-training-pipeline-id[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.CUSTOM_JOB_ID = test-custom-job-id[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.IMAGE_DATASET[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.TABULAR_DATASET[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.TEXT_DATASET[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.VIDEO_DATASET[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.TIME_SERIES_DATASET[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.DATASET_ID[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.TEST_EXPORT_CONFIG[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.TEST_IMPORT_CONFIG[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.DATASET_TO_UPDATE[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.TEST_UPDATE_MASK[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.TEST_TIME_COLUMN = date[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.TEST_TIME_SERIES_IDENTIFIER_COLUMN = store_name[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.TEST_TARGET_COLUMN = sale_dollars[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.COLUMN_SPECS[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.COLUMN_TRANSFORMATIONS[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.MODEL_ID[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.MODEL_ARTIFACT_URI[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.MODEL_NAME[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.JOB_DISPLAY_NAME[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.BIGQUERY_SOURCE[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.GCS_DESTINATION_PREFIX = gs://test-vertex-ai-bucket-us/output[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.MODEL_PARAMETERS[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.ENDPOINT_CONF[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.DEPLOYED_MODEL[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.MODEL_OUTPUT_CONFIG[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.MODEL_OBJ[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.create_custom_container_training_job[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.create_image_dataset_job[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.create_auto_ml_forecasting_training_job[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.create_batch_prediction_job[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.create_endpoint[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.create_hyperparameter_tuning_job[source]
-
airflow.providers.google.cloud.example_dags.example_vertex_ai.upload_model[source]
