Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

10.5 KB
Raw Blame
import type {
  BrowserChallenge,
  HandoffRequest,
} from "@openbot/shared/computer-control";
import { callComputer, type ToolOutcome } from "./call";
import { cancelControl, readControl } from "./control";

export type PendingHandoff = { requestId: string; toolCallId: string };
const active = new Map<string, number>();
/**
 * Requests whose answer a waiting tool call in this tab has already taken, so the turn it belongs to carried on with
 * it. A conversation opened again after its person left (while they held control) is a new chat, and the call it
 * restores looks unanswered there; this is how it knows the answer went to the turn that was still running rather than
 * nowhere, and that there is nothing to resume.
 */
const answered = new Set<string>();
const key = (botId: string) => `openbot:pending-handoff:${botId}`;

/** Bookkeeping only. Never persist page text, credentials, or the human's input. */
export function pendingHandoff(botId: string): PendingHandoff | null {
  if (typeof sessionStorage === "undefined") return null;
  const text = sessionStorage.getItem(key(botId));
  if (!text) return null;
  try {
    const value: unknown = JSON.parse(text);
    return value &&
      typeof value === "object" &&
      "requestId" in value &&
      typeof value.requestId === "string" &&
      "toolCallId" in value &&
      typeof value.toolCallId === "string"
      ? { requestId: value.requestId, toolCallId: value.toolCallId }
      : null;
  } catch {
    return null;
  }
}
export function rememberHandoff(
  botId: string,
  requestId: string,
  toolCallId?: string,
) {
  if (toolCallId && typeof sessionStorage !== "undefined")
    sessionStorage.setItem(
      key(botId),
      JSON.stringify({ requestId, toolCallId }),
    );
}
export function forgetHandoff(botId: string, requestId: string) {
  if (pendingHandoff(botId)?.requestId === requestId)
    sessionStorage.removeItem(key(botId));
}
export function hasActiveHandoff(botId: string, requestId: string) {
  return (active.get(`${botId}:${requestId}`) ?? 0) > 0;
}
/** Whether a tool call in this tab took this request's answer and carried its turn on. See `answered`. */
export function wasHandoffAnswered(botId: string, requestId: string) {
  return answered.has(`${botId}:${requestId}`);
}

/**
 * What the Bot is told when the person hands control back after doing what it asked.
 *
 * Said so the Bot has something to say. "The person handed control back" and a snapshot left the next turn with no
 * news in it, and the model often ended that turn without a word: the conversation showed a blank turn under the help
 * request. It names what was done, that the page in this result is as they left it, and asks for the next step said
 * out loud before it is taken. First in the result, ahead of the page, because it is what the Bot must read first.
 */
export function handedBack(reason: string): string {
  return (
    `A person took control to do what you asked: "${reason}" They have finished and handed control back to you, ` +
    "and the snapshot here is the page as they left it. Carry on with the task now. Start your reply by telling the " +
    "person, in one short sentence, what you are doing next; if nothing is left to do, tell them it is done."
  );
}

function pause(ms: number, signal?: AbortSignal): Promise<void> {
  return new Promise((resolve) => {
    if (signal?.aborted) {
      resolve();
      return;
    }
    const finish = () => {
      clearTimeout(timer);
      signal?.removeEventListener("abort", finish);
      resolve();
    };
    const timer = setTimeout(finish, ms);
    signal?.addEventListener("abort", finish, { once: true });
  });
}

/** No human-control deadline: waiting expiry and all terminal outcomes belong to the server. */
export async function awaitHandoff(
  botId: string,
  requestId: string,
  signal?: AbortSignal,
  pollMs = 1000,
): Promise<ToolOutcome> {
  const identity = `${botId}:${requestId}`;
  active.set(identity, (active.get(identity) ?? 0) + 1);
  let lastRequest: HandoffRequest | undefined;
  let failures = 0;
  try {
    while (!signal?.aborted) {
      const state = await readControl(botId, requestId, signal).catch(
        () => null,
      );
      if (signal?.aborted) break;
      if (!state) {
        if (++failures >= 5)
          return {
            ok: false,
            requestId,
            reconnecting: true,
            reason:
              "The handoff could not be reached. Reopen this conversation and use Resume task to check this request again.",
          };
        await pause(pollMs, signal);
        continue;
      }
      failures = 0;
      if (state.request?.id !== requestId)
        return {
          ok: false,
          requestId,
          reason:
            "The computer did not return the requested handoff. Nothing has been marked complete.",
        };
      lastRequest = state.request;
      if (lastRequest.status === "completed") {
        const snapshot = await callComputer(
          botId,
          "/snapshot",
          { method: "POST" },
          signal,
        );
        if (!snapshot.ok)
          return {
            ...snapshot,
            requestId,
            handoffStatus: "completed",
            snapshotRequired: true,
          };
        const nextChallenge = challengeFrom(snapshot);
        if (nextChallenge) {
          if (nextChallenge.requestId === requestId)
            return {
              ok: false,
              requestId,
              reason:
                "The completed request still reports an active challenge. Check the browser before continuing.",
            };
          rememberHandoff(
            botId,
            nextChallenge.requestId,
            pendingHandoff(botId)?.toolCallId,
          );
          const next = await awaitHandoff(
            botId,
            nextChallenge.requestId,
            signal,
            pollMs,
          );
          return { ...next, challenge: nextChallenge };
        }
        answered.add(identity);
        return {
          result: handedBack(lastRequest.reason),
          handoffStatus: "completed",
          ...snapshot,
          requestId,
        };
      }
      if (lastRequest.status !== "waiting" && lastRequest.status !== "taken") {
        answered.add(identity);
        return {
          ok: false,
          requestId,
          handoffStatus: lastRequest.status,
          reason:
            lastRequest.interruption ??
            `The handoff was ${lastRequest.status}. The person has not completed this request.`,
        };
      }
      await pause(pollMs, signal);
    }
    // Stop detaches a driver. Only a still-waiting request may be cancelled.
    if (signal?.reason === "detached")
      return {
        ok: false,
        stopped: true,
        requestId,
        reason: "The waiting view disconnected.",
      };
    const current = await readControl(
      botId,
      requestId,
      AbortSignal.timeout(2000),
    ).catch(() => null);
    if (
      current?.request?.id === requestId &&
      current.request.status === "waiting"
    )
      await cancelControl(botId, requestId).catch(() => null);
    return {
      ok: false,
      stopped: true,
      requestId,
      reason:
        "Stopped waiting. A person who has control keeps it until they hand back.",
    };
  } finally {
    const remaining = (active.get(identity) ?? 1) - 1;
    if (remaining) active.set(identity, remaining);
    else active.delete(identity);
  }
}

function requestFrom(result: ToolOutcome): HandoffRequest | undefined {
  const request = result.request;
  return request &&
    typeof request === "object" &&
    "id" in request &&
    typeof request.id === "string"
    ? (request as HandoffRequest)
    : undefined;
}

export async function runHelpRequest(
  botId: string,
  reason: string,
  toolCallId?: string,
  signal?: AbortSignal,
): Promise<ToolOutcome> {
  const stored = pendingHandoff(botId);
  if (toolCallId && stored?.toolCallId === toolCallId)
    return awaitHandoff(botId, stored.requestId, signal);
  const asked = await callComputer(
    botId,
    "/control/request",
    { method: "POST", body: { reason, ...(toolCallId ? { toolCallId } : {}) } },
    signal,
  );
  if (!asked.ok) return asked;
  const request = requestFrom(asked);
  if (!request)
    return {
      ok: false,
      reason: "The computer did not identify the handoff request.",
    };
  rememberHandoff(botId, request.id, toolCallId);
  return awaitHandoff(botId, request.id, signal);
}

export async function runNavigation(
  botId: string,
  url: string,
  toolCallId?: string,
  signal?: AbortSignal,
): Promise<ToolOutcome> {
  const stored = pendingHandoff(botId);
  if (toolCallId && stored?.toolCallId === toolCallId)
    return awaitHandoff(botId, stored.requestId, signal);
  const result = await callComputer(
    botId,
    "/navigate",
    { method: "POST", body: { url, ...(toolCallId ? { toolCallId } : {}) } },
    signal,
  );
  const challenge = challengeFrom(result);
  if (!challenge?.requestId) return result;
  rememberHandoff(botId, challenge.requestId, toolCallId);
  const resumed = await awaitHandoff(botId, challenge.requestId, signal);
  // Only the resumed snapshot describes the current page. The initial response belongs to
  // the challenge page and may carry text/truncation fields absent from a snapshot.
  return { ...resumed, challenge, challengeResolved: resumed.ok };
}

function challengeFrom(result: ToolOutcome): BrowserChallenge | undefined {
  const value = result.challenge;
  if (
    !value ||
    typeof value !== "object" ||
    !("requestId" in value) ||
    typeof value.requestId !== "string" ||
    !("kind" in value) ||
    (value.kind !== "cloudflare" && value.kind !== "visible-challenge") ||
    !("reason" in value) ||
    typeof value.reason !== "string"
  )
    return undefined;
  return { requestId: value.requestId, kind: value.kind, reason: value.reason };
}

export async function runBrowserRead(
  botId: string,
  path: "/read" | "/snapshot",
  toolCallId?: string,
  signal?: AbortSignal,
): Promise<ToolOutcome> {
  const stored = pendingHandoff(botId);
  if (toolCallId && stored?.toolCallId === toolCallId)
    return awaitHandoff(botId, stored.requestId, signal);
  const result = await callComputer(
    botId,
    path,
    path === "/snapshot" ? { method: "POST" } : undefined,
    signal,
  );
  const challenge = challengeFrom(result);
  if (!challenge) return result;
  rememberHandoff(botId, challenge.requestId, toolCallId);
  const resumed = await awaitHandoff(botId, challenge.requestId, signal);
  return { ...resumed, challenge, challengeResolved: resumed.ok };
}