Source code for airflow.providers.google.cloud.links.dataproc

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
"""This module contains Google Dataproc links."""

from __future__ import annotations

import warnings
from typing import TYPE_CHECKING, Any

import attr

from airflow.exceptions import AirflowProviderDeprecationWarning
from airflow.models import BaseOperatorLink, XCom
from airflow.providers.google.cloud.links.base import BASE_LINK, BaseGoogleLink

if TYPE_CHECKING:
    from airflow.models import BaseOperator
    from airflow.models.taskinstancekey import TaskInstanceKey
    from airflow.utils.context import Context


[docs]def __getattr__(name: str) -> Any:
    # PEP-562: deprecate module-level variable
    if name == "DATAPROC_JOB_LOG_LINK":
        # TODO: remove DATAPROC_JOB_LOG_LINK alias in the next major release
        # For backward-compatibility, DATAPROC_JOB_LINK was DATAPROC_JOB_LOG_LINK.
        warnings.warn(
            (
                "DATAPROC_JOB_LOG_LINK has been deprecated and will be removed in the next MAJOR release."
                " Please use DATAPROC_JOB_LINK instead"
            ),
            AirflowProviderDeprecationWarning,
            stacklevel=2,
        )
        return DATAPROC_JOB_LINK
    raise AttributeError(f"module {__name__} has no attribute {name}")


[docs]DATAPROC_BASE_LINK = BASE_LINK + "/dataproc"
[docs]DATAPROC_JOB_LINK = DATAPROC_BASE_LINK + "/jobs/{job_id}?region={region}&project={project_id}"

[docs]DATAPROC_CLUSTER_LINK = (
    DATAPROC_BASE_LINK + "/clusters/{cluster_id}/monitoring?region={region}&project={project_id}"
)
[docs]DATAPROC_WORKFLOW_TEMPLATE_LINK = (
    DATAPROC_BASE_LINK + "/workflows/templates/{region}/{workflow_template_id}?project={project_id}"
)
[docs]DATAPROC_WORKFLOW_LINK = (
    DATAPROC_BASE_LINK + "/workflows/instances/{region}/{workflow_id}?project={project_id}"
)

[docs]DATAPROC_BATCH_LINK = DATAPROC_BASE_LINK + "/batches/{region}/{batch_id}/monitoring?project={project_id}"
[docs]DATAPROC_BATCHES_LINK = DATAPROC_BASE_LINK + "/batches?project={project_id}"
[docs]DATAPROC_JOB_LINK_DEPRECATED = DATAPROC_BASE_LINK + "/jobs/{resource}?region={region}&project={project_id}"
[docs]DATAPROC_CLUSTER_LINK_DEPRECATED = (
    DATAPROC_BASE_LINK + "/clusters/{resource}/monitoring?region={region}&project={project_id}"
)


@attr.s(auto_attribs=True)
[docs]class DataprocLink(BaseOperatorLink):
    """
    Helper class for constructing Dataproc resource link.

    .. warning::
       This link is pending to deprecate.
    """

[docs]    name = "Dataproc resource"
[docs]    key = "conf"

    @staticmethod
[docs]    def persist(
        context: Context,
        task_instance,
        url: str,
        resource: str,
    ):
        task_instance.xcom_push(
            context=context,
            key=DataprocLink.key,
            value={
                "region": task_instance.region,
                "project_id": task_instance.project_id,
                "url": url,
                "resource": resource,
            },
        )

[docs]    def get_link(
        self,
        operator: BaseOperator,
        *,
        ti_key: TaskInstanceKey,
    ) -> str:
        conf = XCom.get_value(key=self.key, ti_key=ti_key)
        return (
            conf["url"].format(
                region=conf["region"], project_id=conf["project_id"], resource=conf["resource"]
            )
            if conf
            else ""
        )

[docs]    def __attrs_post_init__(self):
        # This link is still used into the selected operators
        # - airflow.providers.google.cloud.operators.dataproc.DataprocScaleClusterOperator
        # - airflow.providers.google.cloud.operators.dataproc.DataprocJobBaseOperator
        # - airflow.providers.google.cloud.operators.dataproc.DataprocSubmitPigJobOperator
        # As soon as we remove reference to this link we might deprecate it by add warning message
        # with `stacklevel=3` below in this method.
        ...


@attr.s(auto_attribs=True)
[docs]class DataprocListLink(BaseOperatorLink):
    """
    Helper class for constructing list of Dataproc resources link.

    .. warning::
       This link is deprecated.
    """

[docs]    name = "Dataproc resources"
[docs]    key = "list_conf"

    @staticmethod
[docs]    def persist(
        context: Context,
        task_instance,
        url: str,
    ):
        task_instance.xcom_push(
            context=context,
            key=DataprocListLink.key,
            value={
                "project_id": task_instance.project_id,
                "url": url,
            },
        )

[docs]    def get_link(
        self,
        operator: BaseOperator,
        *,
        ti_key: TaskInstanceKey,
    ) -> str:
        list_conf = XCom.get_value(key=self.key, ti_key=ti_key)
        return (
            list_conf["url"].format(
                project_id=list_conf["project_id"],
            )
            if list_conf
            else ""
        )

[docs]    def __attrs_post_init__(self):
        warnings.warn(
            "This DataprocListLink is deprecated.",
            AirflowProviderDeprecationWarning,
            stacklevel=3,
        )


[docs]class DataprocClusterLink(BaseGoogleLink):
    """Helper class for constructing Dataproc Cluster Link."""

[docs]    name = "Dataproc Cluster"
[docs]    key = "dataproc_cluster"
[docs]    format_str = DATAPROC_CLUSTER_LINK

    @staticmethod
[docs]    def persist(
        context: Context,
        operator: BaseOperator,
        cluster_id: str,
        region: str,
        project_id: str,
    ):
        operator.xcom_push(
            context,
            key=DataprocClusterLink.key,
            value={"cluster_id": cluster_id, "region": region, "project_id": project_id},
        )


[docs]class DataprocJobLink(BaseGoogleLink):
    """Helper class for constructing Dataproc Job Link."""

[docs]    name = "Dataproc Job"
[docs]    key = "dataproc_job"
[docs]    format_str = DATAPROC_JOB_LINK

    @staticmethod
[docs]    def persist(
        context: Context,
        operator: BaseOperator,
        job_id: str,
        region: str,
        project_id: str,
    ):
        operator.xcom_push(
            context,
            key=DataprocJobLink.key,
            value={"job_id": job_id, "region": region, "project_id": project_id},
        )


[docs]class DataprocWorkflowLink(BaseGoogleLink):
    """Helper class for constructing Dataproc Workflow Link."""

[docs]    name = "Dataproc Workflow"
[docs]    key = "dataproc_workflow"
[docs]    format_str = DATAPROC_WORKFLOW_LINK

    @staticmethod
[docs]    def persist(context: Context, operator: BaseOperator, workflow_id: str, project_id: str, region: str):
        operator.xcom_push(
            context,
            key=DataprocWorkflowLink.key,
            value={"workflow_id": workflow_id, "region": region, "project_id": project_id},
        )


[docs]class DataprocWorkflowTemplateLink(BaseGoogleLink):
    """Helper class for constructing Dataproc Workflow Template Link."""

[docs]    name = "Dataproc Workflow Template"
[docs]    key = "dataproc_workflow_template"
[docs]    format_str = DATAPROC_WORKFLOW_TEMPLATE_LINK

    @staticmethod
[docs]    def persist(
        context: Context,
        operator: BaseOperator,
        workflow_template_id: str,
        project_id: str,
        region: str,
    ):
        operator.xcom_push(
            context,
            key=DataprocWorkflowTemplateLink.key,
            value={"workflow_template_id": workflow_template_id, "region": region, "project_id": project_id},
        )


[docs]class DataprocBatchLink(BaseGoogleLink):
    """Helper class for constructing Dataproc Batch Link."""

[docs]    name = "Dataproc Batch"
[docs]    key = "dataproc_batch"
[docs]    format_str = DATAPROC_BATCH_LINK

    @staticmethod
[docs]    def persist(
        context: Context,
        operator: BaseOperator,
        batch_id: str,
        project_id: str,
        region: str,
    ):
        operator.xcom_push(
            context,
            key=DataprocBatchLink.key,
            value={"batch_id": batch_id, "region": region, "project_id": project_id},
        )


[docs]class DataprocBatchesListLink(BaseGoogleLink):
    """Helper class for constructing Dataproc Batches List Link."""

[docs]    name = "Dataproc Batches List"
[docs]    key = "dataproc_batches_list"
[docs]    format_str = DATAPROC_BATCHES_LINK

    @staticmethod
[docs]    def persist(
        context: Context,
        operator: BaseOperator,
        project_id: str,
    ):
        operator.xcom_push(
            context,
            key=DataprocBatchesListLink.key,
            value={"project_id": project_id},
        )