# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
#
# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
[docs]
def get_provider_info():
return {
"package-name": "apache-airflow-providers-edge3",
"name": "Edge Executor",
"description": "Handle edge workers on remote sites via HTTP(s) connection and orchestrates work over distributed sites.\n\nWhen tasks need to be executed on remote sites where the connection need to pass through\nfirewalls or other network restrictions, the Edge Worker can be deployed. The Edge Worker\nis a lightweight process with reduced dependencies. The worker only needs to be able to\ncommunicate with the central Airflow site via HTTPS.\n\nIn the central Airflow site the EdgeExecutor is used to orchestrate the work. The EdgeExecutor\nis a custom executor which is used to schedule tasks on the edge workers. The EdgeExecutor can co-exist\nwith other executors (for example CeleryExecutor or KubernetesExecutor) in the same Airflow site.\n\nAdditional REST API endpoints are provided to distribute tasks and manage the edge workers. The endpoints\nare provided by the API server.\n",
"plugins": [
{
"name": "edge_executor",
"plugin-class": "airflow.providers.edge3.plugins.edge_executor_plugin.EdgeExecutorPlugin",
}
],
"cli": ["airflow.providers.edge3.cli.definition.get_edge_cli_commands"],
"executors": ["airflow.providers.edge3.executors.EdgeExecutor"],
"db-managers": ["airflow.providers.edge3.models.db.EdgeDBManager"],
"config": {
"edge": {
"description": "This section only applies if you are using the EdgeExecutor in\n``[core]`` section above\n",
"options": {
"api_enabled": {
"description": "Flag if the plugin endpoint is enabled to serve Edge Workers.\n",
"version_added": None,
"type": "boolean",
"example": "True",
"default": "False",
},
"api_url": {
"description": "URL endpoint on which the Airflow code edge API is accessible from edge worker.\n",
"version_added": None,
"type": "string",
"example": "https://airflow.hosting.org/edge_worker/v1/rpcapi",
"default": None,
},
"job_poll_interval": {
"description": "Edge Worker currently polls for new jobs via HTTP. This parameter defines the number\nof seconds it should sleep between polls for new jobs.\nJob polling only happens if the Edge Worker seeks for new work. Not if busy.\n",
"version_added": None,
"type": "integer",
"example": "5",
"default": "5",
},
"heartbeat_interval": {
"description": "Edge Worker continuously reports status to the central site. This parameter defines\nhow often a status with heartbeat should be sent.\nDuring heartbeat status is reported as well as it is checked if a running task is to be terminated.\n",
"version_added": None,
"type": "integer",
"example": "10",
"default": "30",
},
"drain_timeout_sec": {
"description": "Maximum seconds the worker waits for running jobs to finish after entering drain\n(shutdown requested via SIGINT / SIGTERM / SHUTDOWN_REQUEST / version mismatch).\nOnce this timeout elapses the worker sends SIGTERM to each running job, then SIGKILL\nafter ``drain_kill_grace_sec``, and exits regardless of remaining jobs.\nSet to 0 (the default) to wait indefinitely and preserve legacy behavior.\n",
"version_added": "3.6.0",
"type": "integer",
"example": "3600",
"default": "0",
},
"drain_kill_grace_sec": {
"description": "Grace period in seconds after SIGTERM before the worker escalates to SIGKILL and exits.\nOnly used when ``drain_timeout_sec`` is greater than 0.\n",
"version_added": "3.6.0",
"type": "integer",
"example": "30",
"default": "30",
},
"worker_concurrency": {
"description": "The concurrency defines the default max parallel running task instances and can also be set during\nstart of worker with the ``airflow edge worker`` command parameter. The size of the workers\nand the resources must support the nature of your tasks. The parameter\nworks together with the concurrency_slots parameter of a task.\n",
"version_added": None,
"type": "integer",
"example": None,
"default": "8",
},
"job_success_purge": {
"description": "Minutes after which successful jobs for EdgeExecutor are purged from database\n",
"version_added": None,
"type": "integer",
"example": None,
"default": "5",
},
"job_fail_purge": {
"description": "Minutes after which failed jobs for EdgeExecutor are purged from database\n",
"version_added": None,
"type": "integer",
"example": None,
"default": "60",
},
"push_log_chunk_size": {
"description": "Edge Worker uploads log files in chunks. If the log file part which is uploaded\nexceeds the chunk size it creates a new request. The application gateway can\nlimit the max body size see:\nhttps://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size\nA HTTP 413 issue can point to this value to fix the issue.\nThis value must be defined in Bytes.\n",
"version_added": None,
"type": "integer",
"example": None,
"default": "524288",
},
"push_logs": {
"description": "Flag to enable or disable pushing of log files from edge worker to the central site.\nWhen enabled, edge workers will upload task log files in chunks to the central Airflow site.\nWhen disabled, logs will only be available locally on the edge worker.\n",
"version_added": "1.5.0",
"type": "boolean",
"example": "True",
"default": "True",
},
"worker_umask": {
"description": "The default umask to use for edge worker when run in daemon mode\n\nThis controls the file-creation mode mask which determines the initial value of file permission bits\nfor newly created files.\n\nThis value is treated as an octal-integer.\n",
"version_added": None,
"type": "string",
"default": None,
"example": None,
},
"minimum_acceptable_core_version_for_workers": {
"description": "The minimum acceptable version of the central Airflow site and edge worker to work with.\n\nThis is used to make sure that the central Airflow site and edge workers are compatible with each\nother. When an edge worker connects to the central site, it reports its installed airflow\ncore version. If the version is lower than the minimum acceptable version, the central site will\nnotify the worker during heartbeat and raise an alert. The worker will gracefully terminate and\ndrain jobs and shut down.\n\nIf you know your workers are still compatible with older versions of the central site, you can set\nthis to a lower version to avoid having to update all workers at the same time as the central site.\nDefault is to force matching versions, so relaxing is optional to be configured.\n\nIf the airflow core version between edge worker and central site is not matching, the worker will\nreport warning status.\n",
"version_added": "3.6.0",
"type": "string",
"default": None,
"example": "3.2.1",
},
"minimum_acceptable_edge_version_for_workers": {
"description": "The minimum acceptable version of the edge worker to work with the central site.\n\nThis is used to make sure that the central Airflow site and edge workers are compatible with each\nother. When an edge worker connects to the central site, it reports its edge version. If the\nversion is lower than the minimum acceptable version, the central site will notify the worker\nduring heartbeat and raise an alert. The worker will gracefully terminate and drain jobs and shut\ndown.\n\nIf you know your edge workers are still compatible with older versions of the central site, you\ncan set this to a lower version to avoid having to update all workers at the same time as the\ncentral site. Default is to force matching versions, so relaxing is optional to be configured.\n\nIf the edge version between edge worker and central site is not matching, the worker will report\nwarning status.\n",
"version_added": "3.6.0",
"type": "string",
"default": None,
"example": "3.5.0",
},
"extended_system_info_function": {
"description": "The function to call to get extended system information for the worker.\n\nThe function must be async and return a ``dict[str, str | int | float | datetime]``.\nThe information will be sent to the central site with each heartbeat and can be used for monitoring\nand debugging purposes. All int and float values will also be published to metric collection systems\nlike statsd or otel.\n\nFunction must be provided as a string with the full path to the function. See\nhttps://github.com/apache/airflow/blob/main/providers/edge3/src/airflow/providers/edge3/cli/example_extended_sysinfo.py\nfor an example implementation.\n",
"version_added": "3.5.0",
"type": "string",
"default": None,
"example": "airflow.providers.edge3.cli.example_extended_sysinfo.get_example_extended_sysinfo",
},
"automatic_maintenance_on": {
"description": 'Enable automatic maintenance mode for edge workers based on reported system info status.\n\nWhen configured, edge workers will automatically enter maintenance mode when the reported system\ninfo status indicates a problem on the worker. So in conjunction with a\n``extended_system_info_function`` that reports system status, this can be used to automatically\ntake workers out when they are unhealthy.\n\nPossible options are "Error", "Warning" and "Off". When set to "Error", workers will enter\nmaintenance mode when the system info function reports a status of "Error". When set to "Warning",\nworkers will enter maintenance mode when the system info function reports a status of "Error" or\n"Warning".\n\nWhen set to "Off", automatic maintenance mode is disabled and workers will not automatically enter\nmaintenance mode based on system info status.\n',
"version_added": "3.6.0",
"type": "string",
"example": "Error",
"default": "Off",
},
"automatic_maintenance_exit": {
"description": 'Enable automatic exit from maintenance mode for edge workers based on reported system info status.\n\nWhen configured, edge workers will automatically exit maintenance mode that was set by automatic\nmaintenance mode when the reported system info status indicates that the worker is healthy (again).\nSo in conjunction with a `extended_system_info_function` that reports system status, this can be\nused to automatically bring workers back when they are healthy.\n\nPossible options are "Warning", "Info" and "Off". When set to "Warning", workers will exit\nmaintenance mode when the system info function reports a status of "Warning" or "Info".\nWhen set to "Info", workers will exit maintenance mode when the system info function reports a\nstatus of "Info".\nWhen set to "Off", automatic maintenance exit is disabled and workers will not automatically exit\nmaintenance mode based on system info status.\n',
"version_added": "3.6.0",
"type": "string",
"example": "Info",
"default": "Off",
},
},
}
},
}