src/sdk/bundle.tstypescript
/*!
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

// The bundle: what a TypeScript bundle process provides, and how it serves it.

import { brand, DUPLICATE_COPY_HINT, hasBrand } from "./brand.js";
import { Dag, finalizeDag, getDagTaskRecords, isDag } from "./dag.js";
import { getTaskHandlerFunction, isTaskHandler, TaskHandler } from "./task-handler.js";
import type { Operator } from "./operator.js";
import type { TaskFunction } from "./task.js";

// Assigned inside Bundle's static block, as Dag does for its tasks.
let dagsOf: (bundle: Bundle) => ReadonlyMap<string, Dag>;
let taskHandlersOf: (bundle: Bundle) => ReadonlyMap<string, ReadonlyMap<string, TaskFunction>>;

/**
 * What {@link Bundle.register} and the {@link Bundle} constructor accept: a
 * {@link TaskHandler} for a task that a Python Dag declares, or a {@link Dag}
 * declared in TypeScript.
 */
// `never` for the handler's argument type: a TaskHandler is contravariant in it
// (the handler takes it), so this is the one instantiation every typed handler
// is assignable to, `TaskHandler<TransformArgs>` included.
export type Registerable = Dag | TaskHandler<never, unknown>;

/** Internal: whether `value` is a Bundle built by any copy of this package. */
export function isBundle(value: unknown): value is Bundle {
  return hasBrand(value, "Bundle");
}

/**
 * What a bundle process provides to Airflow, and the thing that serves it.
 *
 * A bundle entry point builds one, registers what it provides, and awaits
 * `serve()` at module top level:
 *
 * ```ts
 * const dag = new Dag("my_dag");
 * dag.task("extract", extractFn);
 *
 * const bundle = new Bundle();
 * bundle.register(dag);
 * await bundle.serve();
 * ```
 *
 * Registering holds no sockets and starts nothing, so a test can build a bundle
 * and invoke a handler through {@link getTaskHandler} without any runtime in
 * scope. Only `serve()` connects to Airflow.
 *
 * Lookups delegate live to each Dag's task map, so tasks added to a Dag
 * after registration are visible: the bundle records Dag identity, not
 * a snapshot of its tasks.
 */
export class Bundle {
  // Dags declared in TypeScript, keyed by dag_id in registration order.
  #dags = new Map<string, Dag>();
  // Handlers for the tasks a Python Dag declares, keyed by dag_id and then by
  // task_id, both in registration order.
  #taskHandlers = new Map<string, Map<string, TaskFunction>>();

  static {
    dagsOf = (bundle) => bundle.#dags;
    taskHandlersOf = (bundle) => bundle.#taskHandlers;
  }

  /** Registers `items`, on the same terms as {@link register}. */
  constructor(...items: Registerable[]) {
    brand(this, "Bundle");
    this.register(...items);
  }

  /** Register what this bundle provides. Registering an already-registered
   *  `dagId` throws, and a call that throws registers none of its items.
   *
   *  The constructor covers the common case; this is for a bundle that
   *  collects what it provides across several modules. */
  register(...items: Registerable[]): void {
    // Staged on copies and committed once every item is accepted, so a call
    // that throws registers none of its items and a bundle never
    // half-provides what its author listed in one call.
    const dags = new Map(this.#dags);
    const taskHandlers = new Map(
      [...this.#taskHandlers].map(([dagId, handlers]) => [dagId, new Map(handlers)] as const),
    );
    for (const item of items) {
      // Typed as Registerable, so narrowing it would collapse to never; these
      // guard callers reaching this from plain JavaScript.
      const candidate: unknown = item;
      // Another copy's value cannot be registered, since both kinds read
      // private state keyed to this copy's class, so it is rejected by its cause.
      if (candidate instanceof Dag) {
        stageDag(dags, taskHandlers, candidate);
      } else if (candidate instanceof TaskHandler) {
        stageTaskHandler(dags, taskHandlers, candidate);
      } else if (isDag(candidate)) {
        throw new Error(`Dag "${(candidate as Dag).dagId}" ${DUPLICATE_COPY_HINT}`);
      } else if (isTaskHandler(candidate)) {
        const foreign = candidate as TaskHandler;
        throw new Error(
          `Task handler for Dag "${foreign.dagId}" task "${foreign.taskId}" ${DUPLICATE_COPY_HINT}`,
        );
      } else {
        throw new Error("only Dag and TaskHandler instances can be registered");
      }
    }
    this.#dags = dags;
    this.#taskHandlers = taskHandlers;
  }

  /**
   * Serve this bundle to Airflow. The entry point of a TypeScript Dag bundle.
   *
   * A bundle process serves one supervisor request, so a second call, which
   * would connect a second pair of sockets, is rejected. A call that fails is
   * not a serve, and may be retried. Resolves once the supervisor has been sent
   * the terminal frame for the work this process was started for, and the same
   * call answers the build-time `--airflow-metadata` query `airflow-ts-pack`
   * makes.
   *
   * Everything this bundle provides must be registered before `serve()` is
   * awaited: what is left out is not part of the bundle, and its tasks are
   * marked removed at runtime.
   */
  async serve(): Promise<void> {
    // `const { serve } = bundle` detaches the method, which would otherwise
    // fail deep in the runtime on a missing private field rather than here.
    validateOwnBundle(this, "bundle.serve()");
    // Imported here rather than at module top: the coordinator reads a bundle,
    // so a static import would make the authoring surface and the coordinator
    // mutually dependent for the sake of one call.
    const { serveBundle } = await import("../coordinator/serve.js");
    await serveBundle(this);
  }

  /** Look up a registered handler, the way the runtime dispatches a task.
   *  Returns `undefined` when no handler exists. */
  getTaskHandler(dagId: string, taskId: string): TaskFunction | undefined {
    const dag = this.#dags.get(dagId);
    return dag === undefined
      ? this.#taskHandlers.get(dagId)?.get(taskId)
      : getDagTaskRecords(dag).get(taskId)?.fn;
  }
}

// A Dag declared in TypeScript owns its own tasks, so a dag_id it holds and a
// dag_id task handlers hold are two disagreeing sources for one task list.
// Each kind is therefore rejected for a dag_id the other one already holds,
// whichever order they were registered in.

function stageDag(
  dags: Map<string, Dag>,
  taskHandlers: ReadonlyMap<string, ReadonlyMap<string, TaskFunction>>,
  dag: Dag,
): void {
  if (taskHandlers.has(dag.dagId)) {
    throw new Error(
      `Dag "${dag.dagId}" already has registered task handlers; a Dag declared in ` +
        "TypeScript owns its own tasks, so one Dag ID cannot have both",
    );
  }
  if (dags.has(dag.dagId)) {
    throw new Error(`Dag "${dag.dagId}" is already registered`);
  }
  dags.set(dag.dagId, dag);
}

function stageTaskHandler(
  dags: ReadonlyMap<string, Dag>,
  taskHandlers: Map<string, Map<string, TaskFunction>>,
  handler: TaskHandler,
): void {
  if (dags.has(handler.dagId)) {
    throw new Error(
      `Dag "${handler.dagId}" is declared in TypeScript; attach its tasks with ` +
        "dag.task(...) rather than registering task handlers for them",
    );
  }
  const forDag = taskHandlers.get(handler.dagId) ?? new Map<string, TaskFunction>();
  if (forDag.has(handler.taskId)) {
    throw new Error(
      `A handler for Dag "${handler.dagId}" task "${handler.taskId}" is already registered`,
    );
  }
  forDag.set(handler.taskId, getTaskHandlerFunction(handler));
  taskHandlers.set(handler.dagId, forDag);
}

/** Internal: reject a `this` that is not a Bundle built by this copy, naming
 *  the cause. */
export function validateOwnBundle(value: unknown, accessor: string): asserts value is Bundle {
  if (value instanceof Bundle) return;
  throw new Error(
    isBundle(value)
      ? `The bundle ${accessor} was called on ${DUPLICATE_COPY_HINT}`
      : `${accessor} must be called on a Bundle; build one with new Bundle(...)`,
  );
}

/**
 * Internal: what kind of task the runtime is to run, and what it needs to run it.
 *
 * A `handler` runs the author's function. An `operator` is one the SDK runs itself,
 * with no handler: its options come from the factory the task was declared with.
 */
export type BundleTask =
  | { readonly kind: "handler"; readonly fn: TaskFunction }
  | { readonly kind: "operator"; readonly operator: Operator<never, unknown>; readonly dag: Dag };

/** Internal: the task `taskId` of Dag `dagId`, or `undefined` when this bundle does not provide it. */
export function getBundleTask(
  bundle: Bundle,
  dagId: string,
  taskId: string,
): BundleTask | undefined {
  const dag = dagsOf(bundle).get(dagId);
  if (dag === undefined) {
    // A Dag declared in Python: the bundle holds only handlers for its stub tasks.
    const fn = taskHandlersOf(bundle).get(dagId)?.get(taskId);
    return fn === undefined ? undefined : { kind: "handler", fn };
  }
  // A Dag declared in TypeScript: the task's record says what runs it.
  const record = getDagTaskRecords(dag).get(taskId);
  if (record === undefined) return undefined;
  if (record.operator) return { kind: "operator", operator: record.operator, dag };
  return { kind: "handler", fn: record.fn };
}

/** Internal: finalize every Dag this bundle declared in TypeScript, so no task
 *  can be added or wired afterwards. */
export function finalizeBundleDags(bundle: Bundle): void {
  for (const dag of dagsOf(bundle).values()) {
    finalizeDag(dag);
  }
}

/** Internal: the Dags declared in TypeScript that this bundle owns, keyed by
 *  dag_id, in registration order. Excludes Dags named only by task handlers,
 *  since those Dags live in Python and have no TypeScript source. */
export function bundleDags(bundle: Bundle): ReadonlyMap<string, Dag> {
  return dagsOf(bundle);
}

/** Internal: the task IDs this bundle can dispatch, per Dag: the Dags declared
 *  in TypeScript first, then the Dags its task handlers name, each in
 *  registration order. A Dag declared in TypeScript with no tasks is included. */
export function bundleDagTaskIds(bundle: Bundle): Map<string, string[]> {
  const byDag = new Map<string, string[]>();
  for (const [dagId, dag] of dagsOf(bundle)) {
    byDag.set(dagId, [...dag.taskIds]);
  }
  for (const [dagId, handlers] of taskHandlersOf(bundle)) {
    byDag.set(dagId, [...handlers.keys()]);
  }
  return byDag;
}