# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
#
# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
[docs]
def get_provider_info():
return {
"package-name": "apache-airflow-providers-common-io",
"name": "Common IO",
"description": "Common IO Provider\n",
"integrations": [
{
"integration-name": "Common IO",
"external-doc-url": "https://filesystem-spec.readthedocs.io/en/latest/index.html",
"how-to-guide": [
"/docs/apache-airflow-providers-common-io/operators.rst",
"/docs/apache-airflow-providers-common-io/transfer.rst",
],
"tags": ["software"],
}
],
"operators": [
{
"integration-name": "Common IO",
"python-modules": ["airflow.providers.common.io.operators.file_transfer"],
}
],
"asset-uris": [
{
"schemes": ["file"],
"handler": "airflow.providers.common.io.assets.file.sanitize_uri",
"to_openlineage_converter": "airflow.providers.common.io.assets.file.convert_asset_to_openlineage",
"factory": "airflow.providers.common.io.assets.file.create_asset",
}
],
"dataset-uris": [
{
"schemes": ["file"],
"handler": "airflow.providers.common.io.assets.file.sanitize_uri",
"to_openlineage_converter": "airflow.providers.common.io.assets.file.convert_asset_to_openlineage",
"factory": "airflow.providers.common.io.assets.file.create_asset",
}
],
"config": {
"common.io": {
"description": "Common IO configuration section",
"options": {
"xcom_objectstorage_path": {
"description": "Path to a location on object storage where XComs can be stored in url format.\n",
"version_added": "1.3.0",
"type": "string",
"example": "s3://conn_id@bucket/path",
"default": "",
},
"xcom_objectstorage_threshold": {
"description": "Threshold in bytes for storing XComs in object storage. -1 means always store in the\ndatabase. 0 means always store in object storage. Any positive number means\nit will be stored in object storage if the size of the value is greater than the threshold.\n",
"version_added": "1.3.0",
"type": "integer",
"example": "1000000",
"default": "-1",
},
"xcom_objectstorage_compression": {
"description": "Compression algorithm to use when storing XComs in object storage. Supported algorithms\nare a.o.: snappy, zip, gzip, bz2, and lzma. If not specified, no compression will be used.\nNote that the compression algorithm must be available in the Python installation (e.g.\npython-snappy for snappy). Zip, gz, bz2 are available by default.\n",
"version_added": "1.3.0",
"type": "string",
"example": "gz",
"default": "",
},
"state_store_objectstorage_path": {
"description": "Base path on object storage for the task/asset state store backend, in URL format.\nWhen set, StateStoreObjectStorageBackend will persist task and asset state under this\nprefix, organized as <dag_id>/<run_id>/<task_id>/<map_index>/<key> for tasks and\nassets/<asset_identifier>/<key> for assets.\n",
"version_added": "1.8.0",
"type": "string",
"example": "s3://conn_id@bucket/task-state/",
"default": "",
},
"state_store_objectstorage_threshold": {
"description": "Threshold in bytes for offloading serialized state store values to object storage. 0 means\nalways offload to object storage. Any positive number means values will be offloaded\nonly when their serialized size is at least that many bytes. Must be non-negative.\n",
"version_added": "1.8.0",
"type": "integer",
"example": "1000000",
"default": "0",
},
"state_store_objectstorage_compression": {
"description": "Compression algorithm to use when writing task/asset state store values to object storage.\nSupported algorithms are a.o.: ``gzip``, ``bz2``, ``lzma``, and ``xz``. If not specified,\nno compression will be used. The same algorithm must be available on all workers.\n",
"version_added": "1.8.0",
"type": "string",
"example": "gzip",
"default": "",
},
},
}
},
}