`airflow.contrib.hooks.gcp_dataflow_hook`¶

Module Contents¶

airflow.contrib.hooks.gcp_dataflow_hook.DEFAULT_DATAFLOW_LOCATION = us-central1[source]¶

class airflow.contrib.hooks.gcp_dataflow_hook._DataflowJob(dataflow, project_number, name, location, poll_sleep=10, job_id=None, num_retries=None)[source]¶

Bases: airflow.utils.log.logging_mixin.LoggingMixin

_get_job_id_from_name(self)[source]¶

_get_job(self)[source]¶

wait_for_done(self)[source]¶

get(self)[source]¶

class airflow.contrib.hooks.gcp_dataflow_hook._Dataflow(cmd)[source]¶

Bases: airflow.utils.log.logging_mixin.LoggingMixin

_line(self, fd)[source]¶

static _extract_job(line)[source]¶

wait_for_done(self)[source]¶

class airflow.contrib.hooks.gcp_dataflow_hook.DataFlowHook(gcp_conn_id='google_cloud_default', delegate_to=None, poll_sleep=10)[source]¶

Bases: airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook

get_conn(self)[source]¶: Returns a Google Cloud Dataflow service object.

_start_dataflow(self, variables, name, command_prefix, label_formatter)[source]¶

static _set_variables(variables)[source]¶

start_java_dataflow(self, job_name, variables, dataflow, job_class=None, append_job_name=True)[source]¶

start_template_dataflow(self, job_name, variables, parameters, dataflow_template, append_job_name=True)[source]¶

start_python_dataflow(self, job_name, variables, dataflow, py_options, append_job_name=True)[source]¶

static _build_dataflow_job_name(job_name, append_job_name=True)[source]¶

static _build_cmd(variables, label_formatter)[source]¶

_start_template_dataflow(self, name, variables, parameters, dataflow_template)[source]¶

airflow.contrib.hooks.gcp_dataflow_hook¶

Module Contents¶

`airflow.contrib.hooks.gcp_dataflow_hook`¶