Kubernetes¶
Kubernetes Executor¶
The Kubernetes Executor allows you to run all the Airflow tasks on Kubernetes as separate Pods.
KubernetesPodOperator¶
The KubernetesPodOperator
allows you to create Pods on Kubernetes. It works with
any type of executor.
from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
from airflow.kubernetes.secret import Secret
from airflow.kubernetes.volume import Volume
from airflow.kubernetes.volume_mount import VolumeMount
from airflow.kubernetes.pod import Port
secret_file = Secret('volume', '/etc/sql_conn', 'airflow-secrets', 'sql_alchemy_conn')
secret_env = Secret('env', 'SQL_CONN', 'airflow-secrets', 'sql_alchemy_conn')
secret_all_keys = Secret('env', None, 'airflow-secrets-2')
volume_mount = VolumeMount('test-volume',
mount_path='/root/mount_file',
sub_path=None,
read_only=True)
port = Port('http', 80)
configmaps = ['test-configmap-1', 'test-configmap-2']
volume_config= {
'persistentVolumeClaim':
{
'claimName': 'test-volume'
}
}
volume = Volume(name='test-volume', configs=volume_config)
affinity = {
'nodeAffinity': {
'preferredDuringSchedulingIgnoredDuringExecution': [
{
"weight": 1,
"preference": {
"matchExpressions": {
"key": "disktype",
"operator": "In",
"values": ["ssd"]
}
}
}
]
},
"podAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": [
{
"labelSelector": {
"matchExpressions": [
{
"key": "security",
"operator": "In",
"values": ["S1"]
}
]
},
"topologyKey": "failure-domain.beta.kubernetes.io/zone"
}
]
},
"podAntiAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": [
{
"labelSelector": {
"matchExpressions": [
{
"key": "security",
"operator": "In",
"values": ["S2"]
}
]
},
"topologyKey": "kubernetes.io/hostname"
}
]
}
}
tolerations = [
{
'key': "key",
'operator': 'Equal',
'value': 'value'
}
]
k = KubernetesPodOperator(namespace='default',
image="ubuntu:16.04",
cmds=["bash", "-cx"],
arguments=["echo", "10"],
labels={"foo": "bar"},
secrets=[secret_file, secret_env, secret_all_keys],
ports=[port]
volumes=[volume],
volume_mounts=[volume_mount]
name="test",
task_id="task",
affinity=affinity,
is_delete_operator_pod=True,
hostnetwork=False,
tolerations=tolerations,
configmaps=configmaps
)
Pod Mutation Hook¶
The Airflow local settings file (airflow_local_settings.py
) can define a pod_mutation_hook
function
that has the ability to mutate pod objects before sending them to the Kubernetes client
for scheduling. It receives a single argument as a reference to pod objects, and
is expected to alter its attributes.
This could be used, for instance, to add sidecar or init containers to every worker pod launched by KubernetesExecutor or KubernetesPodOperator.
def pod_mutation_hook(pod: Pod):
pod.annotations['airflow.apache.org/launched-by'] = 'Tests'