Source code for tests.system.providers.amazon.aws.example_comprehend
# Licensed to the Apache Software Foundation (ASF) under one# or more contributor license agreements. See the NOTICE file# distributed with this work for additional information# regarding copyright ownership. The ASF licenses this file# to you under the Apache License, Version 2.0 (the# "License"); you may not use this file except in compliance# with the License. You may obtain a copy of the License at## http://www.apache.org/licenses/LICENSE-2.0## Unless required by applicable law or agreed to in writing,# software distributed under the License is distributed on an# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY# KIND, either express or implied. See the License for the# specific language governing permissions and limitations# under the License.from__future__importannotationsimportjsonfromdatetimeimportdatetimefromairflowimportDAGfromairflow.decoratorsimporttask_groupfromairflow.models.baseoperatorimportchainfromairflow.providers.amazon.aws.operators.comprehendimportComprehendStartPiiEntitiesDetectionJobOperatorfromairflow.providers.amazon.aws.operators.s3import(S3CreateBucketOperator,S3CreateObjectOperator,S3DeleteBucketOperator,)fromairflow.providers.amazon.aws.sensors.comprehendimport(ComprehendStartPiiEntitiesDetectionJobCompletedSensor,)fromairflow.utils.trigger_ruleimportTriggerRulefromtests.system.providers.amazon.aws.utilsimportSystemTestContextBuilder
env_id=test_context["ENV_ID"]bucket_name=f"{env_id}-comprehend"input_data_configurations={"S3Uri":f"s3://{bucket_name}/{INPUT_S3_KEY_START_PII_ENTITIES_DETECTION_JOB}","InputFormat":"ONE_DOC_PER_LINE",}output_data_configurations={"S3Uri":f"s3://{bucket_name}/redacted_output/"}pii_entities_kwargs={"RedactionConfig":{"PiiEntityTypes":["NAME","ADDRESS"],"MaskMode":"REPLACE_WITH_PII_ENTITY_TYPE",}}create_bucket=S3CreateBucketOperator(task_id="create_bucket",bucket_name=bucket_name,)upload_sample_data=S3CreateObjectOperator(task_id="upload_sample_data",s3_bucket=bucket_name,s3_key=INPUT_S3_KEY_START_PII_ENTITIES_DETECTION_JOB,data=json.dumps(SAMPLE_DATA),)delete_bucket=S3DeleteBucketOperator(task_id="delete_bucket",trigger_rule=TriggerRule.ALL_DONE,bucket_name=bucket_name,force_delete=True,)chain(# TEST SETUPtest_context,create_bucket,upload_sample_data,# TEST BODYpii_entities_detection_job_workflow(),# TEST TEARDOWNdelete_bucket,)fromtests.system.utils.watcherimportwatcher# This test needs watcher in order to properly mark success/failure# when "tearDown" task with trigger rule is part of the DAGlist(dag.tasks)>>watcher()fromtests.system.utilsimportget_test_run# noqa: E402# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)