Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

5.5 KB
Raw Blame
import { tryClient } from "@/lib/client";
import {
  reportComputerActivity,
  withdrawComputerActivity,
} from "@/lib/copilot/computer-activity";

export type ToolOutcome = Record<string, unknown> & { ok: boolean };

/**
 * The calls that never reach the Bot's screen.
 *
 * Reading a file or running a command happens on the computer, not on the display, so announcing
 * them opened the live screen on whatever still page was already there — a pane that arrived for an
 * action nobody could see, which is indistinguishable from one that arrived for no reason.
 *
 * A LIST OF EXCEPTIONS, NOT A LIST OF PERMITS, and deliberately so: a browser path added later and
 * never added here goes on announcing itself, which is what every browser path already does. The
 * failure mode of forgetting this list is a pane that opens, not one that silently stops.
 */
const OFF_SCREEN_PATHS = new Set([
  "/files/list",
  "/files/read",
  "/files/write",
  "/exec",
]);

/**
 * Exported for the test that covers what a Bot is told when a call is refused.
 *
 * The distinctions this draws from a status and a body decide the model's next step, and they are
 * drawn nowhere else, so they are worth pinning without standing up the tool registrations and the
 * runtime around them.
 */
export async function callComputer(
  botId: string,
  path: string,
  /*
   * A body, not a `RequestInit`. The client serialises it, so a caller that stringified first would
   * send a JSON string of a JSON string — which is what happened, briefly, when this moved over.
   */
  init?: { method?: string; body?: unknown },
  signal?: AbortSignal,
): Promise<ToolOutcome> {
  // Announce before the call so the screen can open while the action is running.
  const run = OFF_SCREEN_PATHS.has(path) ? null : reportComputerActivity(botId);
  let response: Response;
  try {
    response = await tryClient(`/api/computers/${botId}${path}`, {
      method: init?.method,
      body: init?.body,
      // Abort cancels the request and prevents later actions, but cannot undo browser work already executing.
      signal,
    });
  } catch (error) {
    // An abort is a stopped run, not a computer failure.
    if (error instanceof DOMException && error.name === "AbortError") {
      return { ok: false, reason: "Stopped.", stopped: true };
    }
    return {
      ok: false,
      reason: "The assistant's computer could not be reached.",
    };
  }

  const body = (await response.json().catch(() => null)) as Record<
    string,
    unknown
  > | null;

  if (!response.ok) {
    /*
     * A REFUSAL IS AN INTENTION, NOT AN ACTION. 403 is a policy, a capability switch or a network
     * rule saying no, and it says so before the browser is touched: there is nothing on the screen
     * to see, because nothing was done to it. Every other failure reached the computer and may have
     * moved it, so those keep the screen they opened.
     */
    if (response.status === 403 && run !== null)
      withdrawComputerActivity(botId, run);
    return failedComputerCall(response.status, body);
  }

  return { ok: true, ...(body ?? {}) };
}

/**
 * What a Bot is told when a computer call came back refused or failed.
 *
 * A result, never a thrown error. The SDK starts no follow-up run after a frontend tool throws, so a
 * throw ends the Bot's turn on the spot with nothing said; a result reaches the model, which can take
 * a fresh snapshot, pick another field, or tell the person what stopped it. Shared with the approval
 * gate's wrapper, which reaches the same computer by another route and must answer the same way.
 */
export function failedComputerCall(
  status: number,
  body: Record<string, unknown> | null,
): ToolOutcome {
  /*
   * A cold start that outlasted the server's wait. Said as what it is: the computer is starting, this
   * call did nothing, and asking again shortly is the right next step. Kept apart from the failures
   * below so the line in the transcript reads as a wait, and so the model is not handed something
   * that looks like a broken computer and gives up on one seconds from ready.
   */
  if (body?.starting === true) {
    return {
      ok: false,
      starting: true,
      notPerformed: true,
      reason:
        typeof body.error === "string"
          ? body.error
          : "The computer is still starting. This call was not performed. Try it again in a moment.",
    };
  }
  /*
   * A rule wants to ask about this call, and it went straight to the computer without its
   * conversation. Nothing was done. `useFrontendTool` in approval-tools sends it again through the
   * approval route, so this result only reaches the Bot from a surface with no conversation at all.
   */
  if (body?.needsConversation === true) {
    return {
      ok: false,
      needsConversation: true,
      notPerformed: true,
      reason:
        typeof body.error === "string"
          ? body.error
          : "This action needs your approval in a chat with this Bot.",
    };
  }
  return {
    ok: false,
    reason: typeof body?.error === "string" ? body.error : "That did not work.",
    // Preserve refusal/stale-ref/control distinctions for the model's next step.
    ...(status === 403 ? { refused: true, rule: body?.rule ?? null } : {}),
    ...(status === 409
      ? body?.humanHasControl === true
        ? {
            humanHasControl: true,
            requestId: body?.requestId,
            handoff: body?.handoff,
          }
        : body?.snapshotRequired === true
          ? { staleRefs: true, snapshotRequired: true }
          : { staleRefs: true }
      : {}),
  };
}