# -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import re
from googleapiclient.errors import HttpError
from airflow import AirflowException
from airflow.contrib.utils.gcp_field_validator import GcpBodyFieldValidator, \
GcpFieldValidationException
from airflow.version import version
from airflow.models import BaseOperator
from airflow.contrib.hooks.gcp_function_hook import GcfHook
from airflow.utils.decorators import apply_defaults
def _validate_available_memory_in_mb(value):
if int(value) <= 0:
raise GcpFieldValidationException("The available memory has to be greater than 0")
def _validate_max_instances(value):
if int(value) <= 0:
raise GcpFieldValidationException(
"The max instances parameter has to be greater than 0")
CLOUD_FUNCTION_VALIDATION = [
dict(name="name", regexp="^.+$"),
dict(name="description", regexp="^.+$", optional=True),
dict(name="entryPoint", regexp=r'^.+$', optional=True),
dict(name="runtime", regexp=r'^.+$', optional=True),
dict(name="timeout", regexp=r'^.+$', optional=True),
dict(name="availableMemoryMb", custom_validation=_validate_available_memory_in_mb,
optional=True),
dict(name="labels", optional=True),
dict(name="environmentVariables", optional=True),
dict(name="network", regexp=r'^.+$', optional=True),
dict(name="maxInstances", optional=True, custom_validation=_validate_max_instances),
dict(name="source_code", type="union", fields=[
dict(name="sourceArchiveUrl", regexp=r'^.+$'),
dict(name="sourceRepositoryUrl", regexp=r'^.+$', api_version='v1beta2'),
dict(name="sourceRepository", type="dict", fields=[
dict(name="url", regexp=r'^.+$')
]),
dict(name="sourceUploadUrl")
]),
dict(name="trigger", type="union", fields=[
dict(name="httpsTrigger", type="dict", fields=[
# This dict should be empty at input (url is added at output)
]),
dict(name="eventTrigger", type="dict", fields=[
dict(name="eventType", regexp=r'^.+$'),
dict(name="resource", regexp=r'^.+$'),
dict(name="service", regexp=r'^.+$', optional=True),
dict(name="failurePolicy", type="dict", optional=True, fields=[
dict(name="retry", type="dict", optional=True)
])
])
]),
]
[docs]class GcfFunctionDeployOperator(BaseOperator):
"""
Creates a function in Google Cloud Functions.
:param project_id: Google Cloud Platform Project ID where the function should
be created.
:type project_id: str
:param location: Google Cloud Platform region where the function should be created.
:type location: str
:param body: Body of the Cloud Functions definition. The body must be a
Cloud Functions dictionary as described in:
https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions
. Different API versions require different variants of the Cloud Functions
dictionary.
:type body: dict or google.cloud.functions.v1.CloudFunction
:param gcp_conn_id: The connection ID to use to connect to Google Cloud Platform.
:type gcp_conn_id: str
:param api_version: API version used (for example v1 or v1beta1).
:type api_version: str
:param zip_path: Path to zip file containing source code of the function. If the path
is set, the sourceUploadUrl should not be specified in the body or it should
be empty. Then the zip file will be uploaded using the upload URL generated
via generateUploadUrl from the Cloud Functions API.
:type zip_path: str
:param validate_body: If set to False, body validation is not performed.
:type validate_body: bool
"""
@apply_defaults
def __init__(self,
project_id,
location,
body,
gcp_conn_id='google_cloud_default',
api_version='v1',
zip_path=None,
validate_body=True,
*args, **kwargs):
self.project_id = project_id
self.location = location
self.full_location = 'projects/{}/locations/{}'.format(self.project_id,
self.location)
self.body = body
self.gcp_conn_id = gcp_conn_id
self.api_version = api_version
self.zip_path = zip_path
self.zip_path_preprocessor = ZipPathPreprocessor(body, zip_path)
self._field_validator = None
if validate_body:
self._field_validator = GcpBodyFieldValidator(CLOUD_FUNCTION_VALIDATION,
api_version=api_version)
self._hook = GcfHook(gcp_conn_id=self.gcp_conn_id, api_version=self.api_version)
self._validate_inputs()
super(GcfFunctionDeployOperator, self).__init__(*args, **kwargs)
def _validate_inputs(self):
if not self.project_id:
raise AirflowException("The required parameter 'project_id' is missing")
if not self.location:
raise AirflowException("The required parameter 'location' is missing")
if not self.body:
raise AirflowException("The required parameter 'body' is missing")
self.zip_path_preprocessor.preprocess_body()
def _validate_all_body_fields(self):
if self._field_validator:
self._field_validator.validate(self.body)
def _create_new_function(self):
self._hook.create_new_function(self.full_location, self.body)
def _update_function(self):
self._hook.update_function(self.body['name'], self.body, self.body.keys())
def _check_if_function_exists(self):
name = self.body.get('name')
if not name:
raise GcpFieldValidationException("The 'name' field should be present in "
"body: '{}'.".format(self.body))
try:
self._hook.get_function(name)
except HttpError as e:
status = e.resp.status
if status == 404:
return False
raise e
return True
def _upload_source_code(self):
return self._hook.upload_function_zip(parent=self.full_location,
zip_path=self.zip_path)
def _set_airflow_version_label(self):
if 'labels' not in self.body.keys():
self.body['labels'] = {}
self.body['labels'].update(
{'airflow-version': 'v' + version.replace('.', '-').replace('+', '-')})
[docs] def execute(self, context):
if self.zip_path_preprocessor.should_upload_function():
self.body[SOURCE_UPLOAD_URL] = self._upload_source_code()
self._validate_all_body_fields()
self._set_airflow_version_label()
if not self._check_if_function_exists():
self._create_new_function()
else:
self._update_function()
SOURCE_ARCHIVE_URL = 'sourceArchiveUrl'
SOURCE_UPLOAD_URL = 'sourceUploadUrl'
SOURCE_REPOSITORY = 'sourceRepository'
ZIP_PATH = 'zip_path'
class ZipPathPreprocessor:
"""
Pre-processes zip path parameter.
Responsible for checking if the zip path parameter is correctly specified in
relation with source_code body fields. Non empty zip path parameter is special because
it is mutually exclusive with sourceArchiveUrl and sourceRepository body fields.
It is also mutually exclusive with non-empty sourceUploadUrl.
The pre-process modifies sourceUploadUrl body field in special way when zip_path
is not empty. An extra step is run when execute method is called and sourceUploadUrl
field value is set in the body with the value returned by generateUploadUrl Cloud
Function API method.
:param body: Body passed to the create/update method calls.
:type body: dict
:param zip_path: path to the zip file containing source code.
:type body: dict
"""
upload_function = None
def __init__(self, body, zip_path):
self.body = body
self.zip_path = zip_path
@staticmethod
def _is_present_and_empty(dictionary, field):
return field in dictionary and not dictionary[field]
def _verify_upload_url_and_no_zip_path(self):
if self._is_present_and_empty(self.body, SOURCE_UPLOAD_URL):
if not self.zip_path:
raise AirflowException(
"Parameter '{}' is empty in the body and argument '{}' "
"is missing or empty. You need to have non empty '{}' "
"when '{}' is present and empty.".
format(SOURCE_UPLOAD_URL, ZIP_PATH, ZIP_PATH, SOURCE_UPLOAD_URL))
def _verify_upload_url_and_zip_path(self):
if SOURCE_UPLOAD_URL in self.body and self.zip_path:
if not self.body[SOURCE_UPLOAD_URL]:
self.upload_function = True
else:
raise AirflowException("Only one of '{}' in body or '{}' argument "
"allowed. Found both."
.format(SOURCE_UPLOAD_URL, ZIP_PATH))
def _verify_archive_url_and_zip_path(self):
if SOURCE_ARCHIVE_URL in self.body and self.zip_path:
raise AirflowException("Only one of '{}' in body or '{}' argument "
"allowed. Found both."
.format(SOURCE_ARCHIVE_URL, ZIP_PATH))
def should_upload_function(self):
if self.upload_function is None:
raise AirflowException('validate() method has to be invoked before '
'should_upload_function')
return self.upload_function
def preprocess_body(self):
self._verify_archive_url_and_zip_path()
self._verify_upload_url_and_zip_path()
self._verify_upload_url_and_no_zip_path()
if self.upload_function is None:
self.upload_function = False
FUNCTION_NAME_PATTERN = '^projects/[^/]+/locations/[^/]+/functions/[^/]+$'
FUNCTION_NAME_COMPILED_PATTERN = re.compile(FUNCTION_NAME_PATTERN)
[docs]class GcfFunctionDeleteOperator(BaseOperator):
"""
Deletes the specified function from Google Cloud Functions.
:param name: A fully-qualified function name, matching
the pattern: `^projects/[^/]+/locations/[^/]+/functions/[^/]+$`
:type name: str
:param gcp_conn_id: The connection ID to use to connect to Google Cloud Platform.
:type gcp_conn_id: str
:param api_version: API version used (for example v1 or v1beta1).
:type api_version: str
"""
@apply_defaults
def __init__(self,
name,
gcp_conn_id='google_cloud_default',
api_version='v1',
*args, **kwargs):
self.name = name
self.gcp_conn_id = gcp_conn_id
self.api_version = api_version
self._validate_inputs()
self.hook = GcfHook(gcp_conn_id=self.gcp_conn_id, api_version=self.api_version)
super(GcfFunctionDeleteOperator, self).__init__(*args, **kwargs)
def _validate_inputs(self):
if not self.name:
raise AttributeError('Empty parameter: name')
else:
pattern = FUNCTION_NAME_COMPILED_PATTERN
if not pattern.match(self.name):
raise AttributeError(
'Parameter name must match pattern: {}'.format(FUNCTION_NAME_PATTERN))
[docs] def execute(self, context):
try:
return self.hook.delete_function(self.name)
except HttpError as e:
status = e.resp.status
if status == 404:
self.log.info('The function does not exist in this project')
else:
self.log.error('An error occurred. Exiting.')
raise e