Source code for tests.system.providers.amazon.aws.example_appflow_run
# Licensed to the Apache Software Foundation (ASF) under one# or more contributor license agreements. See the NOTICE file# distributed with this work for additional information# regarding copyright ownership. The ASF licenses this file# to you under the Apache License, Version 2.0 (the# "License"); you may not use this file except in compliance# with the License. You may obtain a copy of the License at## http://www.apache.org/licenses/LICENSE-2.0## Unless required by applicable law or agreed to in writing,# software distributed under the License is distributed on an# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY# KIND, either express or implied. See the License for the# specific language governing permissions and limitations# under the License.from__future__importannotationsimportjsonfromdatetimeimportdatetimeimportboto3fromairflow.decoratorsimporttaskfromairflow.models.baseoperatorimportchainfromairflow.models.dagimportDAGfromairflow.providers.amazon.aws.operators.appflowimport(AppflowRunOperator,)fromairflow.providers.amazon.aws.operators.s3import(S3CreateBucketOperator,S3CreateObjectOperator,S3DeleteBucketOperator,)fromairflow.utils.trigger_ruleimportTriggerRulefromtests.system.providers.amazon.aws.utilsimportSystemTestContextBuilder
[docs]defcreate_s3_to_s3_flow(flow_name:str,bucket_name:str,source_folder:str):"""creates a flow that takes a CSV and converts it to a json containing the same data"""client=boto3.client("appflow")client.create_flow(flowName=flow_name,triggerConfig={"triggerType":"OnDemand"},sourceFlowConfig={"connectorType":"S3","sourceConnectorProperties":{"S3":{"bucketName":bucket_name,"bucketPrefix":source_folder,"s3InputFormatConfig":{"s3InputFileType":"CSV"},},},},destinationFlowConfigList=[{"connectorType":"S3","destinationConnectorProperties":{"S3":{"bucketName":bucket_name,"s3OutputFormatConfig":{"fileType":"JSON","aggregationConfig":{"aggregationType":"None",},},}},},],tasks=[{"sourceFields":["col1","col2"],"connectorOperator":{"S3":"PROJECTION"},"taskType":"Filter",},{"sourceFields":["col1"],"connectorOperator":{"S3":"NO_OP"},"destinationField":"col1","taskType":"Map","taskProperties":{"DESTINATION_DATA_TYPE":"string","SOURCE_DATA_TYPE":"string"},},{"sourceFields":["col2"],"connectorOperator":{"S3":"NO_OP"},"destinationField":"col2","taskType":"Map","taskProperties":{"DESTINATION_DATA_TYPE":"string","SOURCE_DATA_TYPE":"string"},},],)
env_id=test_context["ENV_ID"]flow_name=f"{env_id}-flow"bucket_name=f"{env_id}-for-appflow"source_folder="source"create_bucket=S3CreateBucketOperator(task_id="create_bucket",bucket_name=bucket_name)upload_csv=S3CreateObjectOperator(task_id="upload_csv",s3_bucket=bucket_name,s3_key="source_folder/data.csv",data="""col1,col2\n"data1","data2"\n""",replace=True,)# [START howto_operator_appflow_run]run_flow=AppflowRunOperator(task_id="run_flow",flow_name=flow_name,)# [END howto_operator_appflow_run]run_flow.poll_interval=1delete_bucket=S3DeleteBucketOperator(task_id="delete_bucket",trigger_rule=TriggerRule.ALL_DONE,bucket_name=bucket_name,force_delete=True,)chain(# TEST SETUPtest_context,create_bucket,setup_bucket_permissions(bucket_name),upload_csv,create_s3_to_s3_flow(flow_name,bucket_name,source_folder),# TEST BODYrun_flow,# TEARDOWNdelete_flow(flow_name),delete_bucket,)fromtests.system.utils.watcherimportwatcher# This test needs watcher in order to properly mark success/failure# when "tearDown" task with trigger rule is part of the DAGlist(dag.tasks)>>watcher()fromtests.system.utilsimportget_test_run# noqa: E402# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)