Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

14.0 KB
Raw Blame
import type { HandoffRequest } from "@openbot/shared/computer-control";
import type { NavigateResult } from "./schema";
import { checkNavigationTarget } from "./target";

/** The computer did not accept or answer a request. */
export class ComputerUnavailableError extends Error {
  constructor(reason: string) {
    super(reason);
    this.name = "ComputerUnavailableError";
  }
}

/**
 * A person pressed Stop and the action was aborted mid-flight.
 *
 * A subclass of `ComputerUnavailableError` on purpose: everything downstream that catches an
 * unavailable computer to tell the model still catches this unchanged. What it adds is a type the
 * gateway can see, so the audit row it writes is `computer.action_stopped` rather than
 * `computer.action_failed` -- a stop is not an outage, and a count of failures that includes every
 * Stop reports one where there was none.
 */
export class ComputerStoppedError extends ComputerUnavailableError {
  constructor(reason: string) {
    super(reason);
    this.name = "ComputerStoppedError";
  }
}

/**
 * What every caller is told when a computer did not finish starting in time.
 *
 * Written for the model and for a person at once, because both read it: the headless tools hand it
 * to the model as the result, and the routes send it to the browser, which shows it on the line.
 */
export const STILL_STARTING_MESSAGE =
  "The computer is still starting. This call was not performed. Try it again in a moment.";

/**
 * The wait for a cold computer ran out before it could take the call.
 *
 * A NORMAL WAIT THAT RAN LONG, NOT AN OUTAGE. A computer that was asleep costs a pod schedule and a
 * browser launch to wake, and the first start pulls an image as well, so a minute is ordinary and
 * two is not unheard of. Reporting that as a failure put red text and "did not become ready" in
 * front of somebody whose computer was, at that moment, finishing its start.
 *
 * A subclass of `ComputerUnavailableError`, so everything that already catches an unavailable
 * computer still does. What the type adds is that every surface can say the gentle thing instead:
 * the route marks the body `starting`, the headless tools answer with a result rather than a throw,
 * and the browser renders it as a wait rather than a failure. The message says the call was NOT
 * performed, which is true by construction: this is only thrown before anything reached the
 * computer, so a retry cannot do the thing twice.
 */
export class ComputerStillStartingError extends ComputerUnavailableError {
  constructor(reason: string = STILL_STARTING_MESSAGE) {
    super(reason);
    this.name = "ComputerStillStartingError";
  }
}

/** The requested element is not on the current page. */
export class ElementNotFoundError extends Error {
  constructor(reason: string) {
    super(reason);
    this.name = "ElementNotFoundError";
  }
}

/** The navigation target is not permitted. */
export class NavigationRefusedError extends Error {
  constructor(reason: string) {
    super(reason);
    this.name = "NavigationRefusedError";
  }
}

/** The computer refused access to a path outside its workspace. */
export class WorkspaceRefusedError extends Error {
  constructor(reason: string) {
    super(reason);
    this.name = "WorkspaceRefusedError";
  }
}

/** The workspace request names a path or value that cannot be used. */
export class WorkspaceRequestError extends Error {
  constructor(reason: string) {
    super(reason);
    this.name = "WorkspaceRequestError";
  }
}

/** The workspace has no file at the requested path. */
export class WorkspaceNotFoundError extends Error {
  constructor(reason: string) {
    super(reason);
    this.name = "WorkspaceNotFoundError";
  }
}

/** The requested workspace file is larger than the download limit. */
export class WorkspaceTooLargeError extends Error {
  constructor(reason: string) {
    super(reason);
    this.name = "WorkspaceTooLargeError";
  }
}

/** Raw bytes and their size, kept as a stream until the caller sends them onward. */
export type ComputerDownload = {
  body: ReadableStream<Uint8Array>;
  bytes: number;
};

/** The page changed after the caller received its element references. */
export class StaleSnapshotError extends Error {
  constructor(
    reason: string,
    readonly snapshotRequired = false,
  ) {
    super(reason);
    this.name = "StaleSnapshotError";
  }
}

/**
 * A person has the wheel, so the Bot's action was not carried out.
 *
 * Its own condition rather than a stale snapshot, though both arrive as 409, because what the caller
 * should do next is the opposite in each case. Stale refs mean take a fresh snapshot and go again;
 * this means stop and leave the browser to the person holding it. Told apart by the flag the computer
 * puts on the body, which is the only thing in the response that distinguishes them.
 */
export class HumanHasControlError extends Error {
  constructor(
    reason: string,
    readonly requestId?: string,
    readonly handoff?: HandoffRequest,
  ) {
    super(reason);
    this.name = "HumanHasControlError";
  }
}

/** Invalid, unknown, or superseded handoff identity is distinct from page freshness. */
export class HandoffRequestError extends Error {
  constructor(
    reason: string,
    readonly status: 400 | 404 | 409,
  ) {
    super(reason);
    this.name = "HandoffRequestError";
  }
}

/**
 * Transport options used inside the computer gateway.
 *
 * This is an internal seam. Application code uses ComputerGateway and does not
 * use this interface directly.
 */
export type ComputerTransportOptions = {
  token?: string;
  allowPrivateHosts?: boolean;
  timeoutMs?: number;
  fetchImpl?: typeof fetch;
};

/**
 * Which seat on the computer a request is for.
 *
 * The Bot alone on a computer of one person's own (or an isolated Bot's), where each Bot is a seat;
 * the Bot and the person on the computer many people share, where the seat is the pair. A string is
 * the Bot alone.
 */
export type SeatAddress = string | { botId: string; personId?: string };

/** Internal HTTP interface used only by ComputerGateway. */
export interface ComputerTransport {
  call<T>(
    baseUrl: string,
    botId: SeatAddress,
    path: string,
    init?: RequestInit,
    caller?: AbortSignal,
    /** Overrides the transport's own deadline for this one call. */
    timeoutMs?: number,
  ): Promise<T>;
  post<T>(
    baseUrl: string,
    botId: SeatAddress,
    path: string,
    payload: unknown,
    caller?: AbortSignal,
    /** Overrides the transport's own deadline for this one call. */
    timeoutMs?: number,
  ): Promise<T>;
  navigate(
    baseUrl: string,
    botId: SeatAddress,
    url: string,
    toolCallId?: string,
  ): Promise<NavigateResult>;
  download(
    baseUrl: string,
    botId: SeatAddress,
    path: string,
    caller?: AbortSignal,
    timeoutMs?: number,
  ): Promise<ComputerDownload>;
}

/** The headers that name a seat to the computer. See {@link SeatAddress}. */
export function seatHeaders(seat: SeatAddress): Record<string, string> {
  const { botId, personId } =
    typeof seat === "string" ? { botId: seat, personId: undefined } : seat;
  return {
    "x-openbot-bot-id": botId,
    ...(personId ? { "x-openbot-person-id": personId } : {}),
  };
}

/**
 * Send authenticated HTTP requests to one located agent-computer process.
 *
 * Lifecycle and location are deliberately absent. ComputerGateway owns those
 * operations through ComputerProvider.
 */
export function createComputerTransport(
  options: ComputerTransportOptions,
): ComputerTransport {
  const doFetch = options.fetchImpl ?? fetch;
  const defaultTimeoutMs = options.timeoutMs ?? 45_000;

  async function request(
    baseUrl: string,
    botId: SeatAddress,
    path: string,
    init?: RequestInit,
    caller?: AbortSignal,
    timeoutMsOverride?: number,
  ): Promise<Response> {
    if (caller?.aborted) {
      throw new ComputerStoppedError("The action was stopped.");
    }

    /*
     * A browser action either happens in seconds or has gone wrong, so 45s is the right deadline for
     * it. A command is not that: the shell's own budget is 120s by default and up to 600s, and the
     * tool description tells the model to install packages. Giving up here first reported failure to
     * the person while the command carried on running to completion inside the container, and made
     * the shell's own limit unreachable. A caller with a longer limit of its own passes it in, and
     * this becomes the backstop rather than the limit.
     */
    const timeoutMs = timeoutMsOverride ?? defaultTimeoutMs;
    const target = baseUrl.replace(/\/$/, "");
    let response: Response;
    try {
      response = await doFetch(`${target}${path}`, {
        ...init,
        headers: {
          ...(init?.headers as Record<string, string> | undefined),
          ...seatHeaders(botId),
          ...(options.token
            ? { "x-openbot-computer-token": options.token }
            : {}),
        },
        signal: caller
          ? AbortSignal.any([caller, AbortSignal.timeout(timeoutMs)])
          : AbortSignal.timeout(timeoutMs),
      });
    } catch (error) {
      /*
       * The caller's own abort is answered first, and says what the check above the fetch says.
       *
       * The signal is handed to fetch precisely so a Stop can land mid-flight, and a fetch aborted
       * that way rejects with an AbortError, which is neither a TimeoutError nor a computer that is
       * not running. Both of the other answers are statements about the infrastructure, and this
       * message is not only read by the model: the gateway writes it into the action's audit row,
       * and the type below is what keeps that row a stop rather than an outage.
       */
      if (caller?.aborted) {
        throw new ComputerStoppedError("The action was stopped.");
      }
      throw new ComputerUnavailableError(
        error instanceof Error && error.name === "TimeoutError"
          ? "The Bot's computer didn't respond in time."
          : "The Bot's computer isn't running.",
      );
    }
    return response;
  }

  async function call<T>(
    baseUrl: string,
    botId: SeatAddress,
    path: string,
    init?: RequestInit,
    caller?: AbortSignal,
    timeoutMsOverride?: number,
  ): Promise<T> {
    const response = await request(
      baseUrl,
      botId,
      path,
      init,
      caller,
      timeoutMsOverride,
    );
    const body = (await response.json().catch(() => null)) as Record<
      string,
      unknown
    > | null;
    if (!response.ok) {
      throwMappedError(response.status, body);
    }
    return body as T;
  }

  async function download(
    baseUrl: string,
    botId: SeatAddress,
    path: string,
    caller?: AbortSignal,
    timeoutMs?: number,
  ): Promise<ComputerDownload> {
    const response = await request(
      baseUrl,
      botId,
      path,
      { method: "GET" },
      caller,
      timeoutMs,
    );
    if (!response.ok) {
      const body = (await response.json().catch(() => null)) as Record<
        string,
        unknown
      > | null;
      throwMappedError(response.status, body);
    }

    const length = response.headers.get("content-length");
    const bytes =
      length !== null && /^\d+$/.test(length) ? Number(length) : Number.NaN;
    if (!response.body || !Number.isSafeInteger(bytes) || bytes < 0) {
      // Nothing else will ever read this body, so release it here rather than
      // leaving the whole download budget checked out of the pool. A cancel that
      // fails must not replace the refusal the caller is told about.
      await response.body?.cancel().catch(() => undefined);
      throw new ComputerUnavailableError(
        "The Bot's computer sent back a file that couldn't be read.",
      );
    }
    return { body: response.body, bytes };
  }

  function post<T>(
    baseUrl: string,
    botId: SeatAddress,
    path: string,
    payload: unknown,
    caller?: AbortSignal,
    timeoutMs?: number,
  ): Promise<T> {
    return call<T>(
      baseUrl,
      botId,
      path,
      {
        method: "POST",
        headers: { "content-type": "application/json" },
        body: JSON.stringify(payload),
      },
      caller,
      timeoutMs,
    );
  }

  async function navigate(
    baseUrl: string,
    botId: SeatAddress,
    url: string,
    toolCallId?: string,
  ): Promise<NavigateResult> {
    const verdict = checkNavigationTarget(url, {
      allowPrivateHosts: options.allowPrivateHosts,
    });
    if (!verdict.allowed) {
      throw new NavigationRefusedError(verdict.reason);
    }
    return post<NavigateResult>(baseUrl, botId, "/navigate", {
      url: verdict.url,
      ...(toolCallId ? { toolCallId } : {}),
    });
  }

  return { call, post, navigate, download };
}

/** Map agent-computer responses to errors that a caller can act on. */
function throwMappedError(
  status: number,
  body: Record<string, unknown> | null,
): never {
  const detail =
    typeof body?.error === "string" ? body.error : `HTTP ${status}`;
  if (
    body?.controlRequestError === true &&
    (status === 400 || status === 404 || status === 409)
  )
    throw new HandoffRequestError(detail, status);
  if (status === 409) {
    // The computer says which kind of 409 this is. Absent, it is the ordinary one.
    if (body?.humanHasControl === true) {
      throw new HumanHasControlError(
        detail,
        typeof body.requestId === "string" ? body.requestId : undefined,
        body.handoff && typeof body.handoff === "object"
          ? (body.handoff as HandoffRequest)
          : undefined,
      );
    }
    throw new StaleSnapshotError(detail, body?.snapshotRequired === true);
  }
  if (status === 403) {
    throw new WorkspaceRefusedError(detail);
  }
  if (status === 400) {
    throw new WorkspaceRequestError(detail);
  }
  if (status === 404) {
    throw new WorkspaceNotFoundError(detail);
  }
  if (status === 413) {
    throw new WorkspaceTooLargeError(detail);
  }
  if (/waiting for locator|Timeout .* exceeded/i.test(detail)) {
    const ref = detail.match(/aria-ref=([A-Za-z0-9_-]+)/)?.[1];
    throw new ElementNotFoundError(
      `${ref ? `Element ${ref} is` : "That element is"} not on the page any more. Take a fresh snapshot and use the refs from it.`,
    );
  }
  throw new ComputerUnavailableError(detail);
}