Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

41.5 KB
Raw Blame
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { BotBrowserState } from "../../shared/bot-browser";
import type { ComputerControlState } from "../../shared/computer-control";
import type { LauncherConnection } from "../../shared/openbot-browser";
import type { MiddlewareHandler } from "hono";
import type { AuditEventInput, AuditStore } from "../../server/src/audit";
import type {
  AppVariables,
  AuthenticatedActor,
} from "../../server/src/auth/guards";
import { createComputerGateway } from "../../server/src/computer/gateway";
import type { PolicyStore } from "../../server/src/computer/policy-store";
import type { ComputerProvider } from "../../server/src/computer/provider";
import { createComputerRoutes } from "../../server/src/computer/routes";
import {
  NOT_ITS_OWN_COMPUTER,
  NOT_ITS_OWN_COMPUTER_RECORD,
} from "../src/person-access";

/**
 * A Bot's browser worked by a person, through the computer itself: the real process, its token, its dispatcher, its
 * control and the Bot's own routes, with bundled Chromium headless and the fixture Recorder (`fixtures/recorder`),
 * which reports every call it receives to the OpenBot it was connected to, so what these tests read about the
 * Recorder is the Recorder's side of each call.
 *
 * What it pins: the Bot works as it did with the Recorder in its browser; the bar and Record refuse anyone without the
 * wheel; a recording starts only for the person holding it; and the computer ends the recording, then disconnects the
 * Recorder, whenever the wheel leaves them.
 *
 *   OPENBOT_COMPUTER_BROWSER=1 bun test tests/bot-browser-computer.test.ts
 */
const asked = process.env.OPENBOT_COMPUTER_BROWSER === "1";
const TOKEN = "bot-browser-computer-token";
const BOT = "sales-bot";
const PERSON = "person-fedcba9876543210fedcba9876543210";
const FIXTURE = join(import.meta.dir, "fixtures", "recorder");

type FixtureCall = {
  method: string;
  input: unknown;
  front: string;
  /** Whether the last of a stopped recording was still on its way to Intelligence when the call came. */
  sending?: boolean;
};
const calls: FixtureCall[] = [];
/** How long the last of a stopped recording takes to reach Intelligence, as this OpenBot tells the fixture Recorder. */
let tailMs = 0;

type Computer = { base: string; child: ReturnType<typeof Bun.spawn> };
const computers: Computer[] = [];
let root = "";
let site: ReturnType<typeof Bun.serve> | undefined;
let origin = "";

async function freePort(): Promise<number> {
  const probe = Bun.serve({
    hostname: "127.0.0.1",
    port: 0,
    fetch: () => new Response(),
  });
  const { port } = probe;
  await probe.stop(true);
  if (port === undefined) throw new Error("The port probe did not listen.");
  return port;
}

async function startComputer(
  name: string,
  env: Record<string, string>,
): Promise<Computer> {
  const port = await freePort();
  const base = `http://127.0.0.1:${port}`;
  const child = Bun.spawn([process.execPath, "src/index.ts"], {
    cwd: join(import.meta.dir, ".."),
    env: {
      ...process.env,
      COMPUTER_TOKEN: TOKEN,
      PORT: String(port),
      COMPUTER_BROWSER_BACKEND: "managed",
      COMPUTER_BROWSER_MODE: "headless",
      // No API server pushes this computer a network policy, so it browses before one arrives.
      EGRESS_POLICY_REQUIRED: "0",
      PROFILES_DIR: join(root, name, "profiles"),
      WORKSPACE_DIR: join(root, name, "workspace"),
      ...env,
    },
    stdout: "ignore",
    stderr: "inherit",
  });
  const computer = { base, child };
  computers.push(computer);
  const deadline = Date.now() + 15_000;
  for (;;) {
    if (child.exitCode !== null)
      throw new Error(`The computer exited with ${child.exitCode}.`);
    const ready = await fetch(`${base}/health`)
      .then((response) => response.ok)
      .catch(() => false);
    if (ready) return computer;
    if (Date.now() > deadline) throw new Error("The computer did not start.");
    await Bun.sleep(50);
  }
}

let computer: Computer;

async function call<T = Record<string, unknown>>(
  path: string,
  options: { body?: unknown; botId?: string; on?: Computer } = {},
): Promise<{ status: number; json: T }> {
  const response = await fetch(`${(options.on ?? computer).base}${path}`, {
    method: options.body === undefined ? "GET" : "POST",
    headers: {
      "content-type": "application/json",
      "x-openbot-computer-token": TOKEN,
      "x-openbot-bot-id": options.botId ?? BOT,
    },
    ...(options.body === undefined
      ? {}
      : { body: JSON.stringify(options.body) }),
  });
  return { status: response.status, json: (await response.json()) as T };
}

const page = (name: string) => `${origin}/page/${name}`;
const state = async (on?: Computer) =>
  (await call<BotBrowserState>("/bot-browser/state", { on })).json;
const activeTab = (current: BotBrowserState) =>
  current.tabs.find((tab) => tab.active);
const callsFor = (method: string) =>
  calls.filter((each) => each.method === method);

const connection = (): LauncherConnection => ({
  openbotUrl: origin,
  token: "obr_bot-browser-fixture",
  connectionId: "connection-bot-1",
  user: { id: "user-1", name: "Pat Example", email: "[email protected]" },
});

/** Ask for help and take the wheel as `by`, as the server does for the signed-in person. */
async function takeWheel(by: string, on?: Computer): Promise<string> {
  const asked = await call<ComputerControlState>("/control/request", {
    body: { reason: "I want to take control of the browser." },
    on,
  });
  const requestId = asked.json.request?.id;
  if (!requestId) throw new Error("The request was not raised.");
  const taken = await call<ComputerControlState>("/control/take", {
    body: { requestId, by },
    on,
  });
  expect(taken.status).toBe(200);
  return requestId;
}

const start = (by: string, title = "File an expense report", on?: Computer) =>
  call<BotBrowserState>("/bot-browser/recording", {
    body: { action: "start", title, by, connection: connection() },
    on,
  });

beforeAll(async () => {
  if (!asked) return;
  root = await mkdtemp(join(tmpdir(), "openbot-bot-browser-computer-"));
  // The OpenBot the fixture Recorder reports to, and the pages the Bot and the person browse.
  site = Bun.serve({
    hostname: "127.0.0.1",
    port: 0,
    async fetch(request) {
      const url = new URL(request.url);
      if (url.pathname === "/fixture/calls") {
        calls.push((await request.json()) as FixtureCall);
        return new Response("");
      }
      if (url.pathname === "/api/recorder/me")
        return Response.json({ user: connection().user });
      if (url.pathname === "/fixture/tail")
        return Response.json({ ms: tailMs });
      // A cookie kept for an hour, as a site keeps a sign-in: it outlives the browser closing.
      if (url.pathname === "/set-cookie")
        return new Response("<!doctype html><title>set</title>", {
          headers: {
            "content-type": "text/html; charset=utf-8",
            "set-cookie": "signed-in=yes; Max-Age=3600; Path=/",
          },
        });
      // The cookies the browser sent, as the page's title.
      if (url.pathname === "/cookies")
        return new Response(
          `<!doctype html><title>cookies:${request.headers.get("cookie") ?? ""}</title>`,
          { headers: { "content-type": "text/html; charset=utf-8" } },
        );
      // Busy 20 ms on every key down, and writes every key it receives into its title, in the order it received them.
      if (url.pathname === "/keys")
        return new Response(
          '<!doctype html><title>none</title><input autofocus><script>const seen=[];const show=e=>{seen.push(e.type.slice(3)+":"+e.key);document.title=seen.join(" ")};addEventListener("keydown",e=>{const until=Date.now()+20;while(Date.now()<until){}show(e)});addEventListener("keyup",show)</script>',
          { headers: { "content-type": "text/html; charset=utf-8" } },
        );
      // A form a Bot fills part of and hands to a person for the rest: a field, a dropdown and an editor.
      if (url.pathname === "/contact")
        return new Response(
          '<!doctype html><title>Contact</title><label>Name <input name="name"></label><label>Telephone <input name="tel"></label><label>Size <select name="size"><option>Small</option><option>Large</option></select></label><div role="textbox" aria-label="Notes" contenteditable="true"></div>',
          { headers: { "content-type": "text/html; charset=utf-8" } },
        );
      const name = url.pathname.split("/").at(-1) || "home";
      return new Response(
        `<!doctype html><title>${name}</title><label>Amount <input name="amount" autofocus></label>`,
        { headers: { "content-type": "text/html; charset=utf-8" } },
      );
    },
  });
  origin = `http://127.0.0.1:${site.port}`;
  // A Bot's own computer, as a supervisor starts each container and a Sandbox each pod: Record runs only on one.
  computer = await startComputer("with-recorder", {
    OPENBOT_RECORDER_DIR: FIXTURE,
    COMPUTER_BOT_ID: BOT,
  });
}, 30_000);

afterAll(async () => {
  if (!asked) return;
  for (const running of computers) {
    for (const botId of [
      BOT,
      "other-bot",
      "source-bot",
      "typed-bot",
      "clear-bot",
    ])
      await call("/computers/stop", { body: {}, botId, on: running }).catch(
        () => undefined,
      );
    running.child.kill();
    await running.child.exited;
  }
  site?.stop(true);
  if (root) await rm(root, { recursive: true, force: true });
}, 60_000);

describe.skipIf(!asked)(
  "a Bot's browser on a computer with the Recorder",
  () => {
    test("launches with the Recorder idle in it, and the Bot reads, snapshots and screenshots its own page as before", async () => {
      const navigated = await call<{ url: string; title: string }>(
        "/navigate",
        {
          body: { url: page("start") },
        },
      );
      expect(navigated.status).toBe(200);
      expect(navigated.json).toMatchObject({
        url: page("start"),
        title: "start",
      });

      const snapshot = await call<{
        url: string;
        elements: { role: string; name: string }[];
      }>("/snapshot", { body: {} });
      expect(snapshot.status).toBe(200);
      expect(snapshot.json.url).toBe(page("start"));
      expect(snapshot.json.elements).toContainEqual(
        expect.objectContaining({ role: "textbox", name: "Amount" }),
      );
      const screenshot = await call<{ url: string; base64: string }>(
        "/screenshot",
      );
      expect(screenshot.status).toBe(200);
      expect(screenshot.json.url).toBe(page("start"));
      expect(screenshot.json.base64.length).toBeGreaterThan(0);

      // One tab, the Bot's: the Recorder's setup page, which it opens at every install, never became one.
      const current = await state();
      expect(current.open).toBe(true);
      expect(current.tabs).toEqual([
        expect.objectContaining({
          url: page("start"),
          title: "start",
          active: true,
        }),
      ]);
      // Loaded and answering, connected to nobody, recording nothing.
      expect(current.recorder).toMatchObject({
        connected: false,
        phase: "idle",
      });
      expect(current.recordingBy).toBeUndefined();
      expect(calls).toEqual([]);
    }, 60_000);

    test("refuses a person's browser id: on a computer Bots share before any route, on the person's own at the Bot's", async () => {
      // A computer Bots run on runs no person's browser at all (person-access.ts), so the id goes no further than that.
      for (const [path, body] of [
        ["/bot-browser/state", undefined],
        ["/bot-browser/tabs", { action: "new" }],
      ] as const) {
        const refused = await call(path, { body, botId: PERSON });
        expect([path, refused.status, refused.json]).toEqual([
          path,
          503,
          { error: NOT_ITS_OWN_COMPUTER },
        ]);
      }
      // A person's own computer takes their id, and the Bot's routes still refuse it: their browser has its own routes.
      const own = await startComputer("persons-own", {
        OPENBOT_RECORDER_DIR: FIXTURE,
        COMPUTER_BOT_ID: PERSON,
      });
      for (const [path, body] of [
        ["/bot-browser/state", undefined],
        ["/bot-browser/tabs", { action: "new" }],
      ] as const)
        expect([
          path,
          (await call(path, { body, botId: PERSON, on: own })).status,
        ]).toEqual([path, 400]);
    }, 30_000);

    test("refuses Record on the computer every Bot shares, before the person's connection reaches the Bot's profile", async () => {
      const shared = await startComputer("shared", {
        OPENBOT_RECORDER_DIR: FIXTURE,
      });
      await takeWheel("user-1", shared);
      const before = calls.length;
      const refused = await start("user-1", "File an expense report", shared);
      expect([refused.status, refused.json]).toEqual([
        403,
        { error: NOT_ITS_OWN_COMPUTER_RECORD },
      ]);
      // The Recorder there was never handed the connection, nor asked to record.
      expect(calls.slice(before)).toEqual([]);
      // The bar is still the person's while they hold the wheel.
      expect(
        (
          await call("/bot-browser/navigate", {
            body: { url: page("shared") },
            on: shared,
          })
        ).status,
      ).toBe(200);
    }, 60_000);

    test("records on a person's own computer, where only their own Bots run", async () => {
      // How the supervisor and a Sandbox start a person's computer: named for the person, no Bot.
      const own = await startComputer("persons-computer", {
        OPENBOT_RECORDER_DIR: FIXTURE,
        COMPUTER_PERSON_ID: "user-1",
      });
      await takeWheel("user-1", own);
      const before = calls.length;
      const started = await start("user-1", "File an expense report", own);
      expect(started.status).toBe(200);
      expect(started.json.recordingBy).toBe("user-1");
      // The Recorder there was handed the person's connection and asked to record.
      expect(
        calls
          .slice(before)
          .map((each) => each.method)
          .filter((method) => method === "connect" || method === "record"),
      ).toEqual(["connect", "record"]);
      expect(
        (
          await call("/bot-browser/recording", {
            body: { action: "stop" },
            on: own,
          })
        ).status,
      ).toBe(200);
      // The fixture Recorder's calls are one list for the file; the tests after this one start from none.
      calls.length = 0;
    }, 60_000);

    test("refuses the bar, Record and narration while the Bot holds the wheel", async () => {
      for (const [path, body] of [
        ["/bot-browser/tabs", { action: "new", url: page("sneaky") }],
        ["/bot-browser/navigate", { url: page("sneaky") }],
        ["/bot-browser/navigate", { action: "back" }],
      ] as const)
        expect((await call(path, { body })).status).toBe(409);
      expect((await start("user-1")).status).toBe(409);
      expect(
        (await call("/bot-browser/recording", { body: { action: "stop" } }))
          .status,
      ).toBe(409);
      expect(
        (
          await call("/bot-browser/narration", {
            body: {
              text: "Here",
              startedAt: 1_760_000_000_000,
              endedAt: 1_760_000_000_500,
              sentAt: 1_760_000_001_000,
            },
          })
        ).status,
      ).toBe(409);
      // Nothing moved, and the Recorder was never handed anybody's connection.
      expect((await state()).tabs.map((tab) => tab.url)).toEqual([
        page("start"),
      ]);
      expect(calls).toEqual([]);
    });

    test("names the person who takes the wheel, and the bar then works for them, with the Bot following the tab they leave in front", async () => {
      const requestId = await takeWheel("user-1");
      const control = (await call<ComputerControlState>("/control")).json;
      expect(control).toMatchObject({ holder: "human", heldBy: "user-1" });

      const two = await call<BotBrowserState>("/bot-browser/tabs", {
        body: { action: "new", url: page("two") },
      });
      expect(two.status).toBe(200);
      expect(two.json.tabs).toHaveLength(2);
      expect(activeTab(two.json)).toMatchObject({
        url: page("two"),
        title: "two",
      });
      const first = two.json.tabs.find((tab) => tab.url === page("start"));
      const second = activeTab(two.json);
      if (!first || !second) throw new Error("Both tabs should be listed.");

      const switched = await call<BotBrowserState>("/bot-browser/tabs", {
        body: { action: "activate", tabId: first.id },
      });
      expect(activeTab(switched.json)?.id).toBe(first.id);
      // The Bot's page is the tab in front now: what it reads next is that tab.
      expect((await call<{ url: string }>("/read")).json.url).toBe(
        page("start"),
      );

      // The address bar, back, forward and reload, in the tab in front.
      const there = await call<BotBrowserState>("/bot-browser/navigate", {
        body: { url: page("three") },
      });
      expect(activeTab(there.json)).toMatchObject({
        id: first.id,
        url: page("three"),
      });
      const back = await call<BotBrowserState>("/bot-browser/navigate", {
        body: { action: "back" },
      });
      expect(activeTab(back.json)?.url).toBe(page("start"));
      const forward = await call<BotBrowserState>("/bot-browser/navigate", {
        body: { action: "forward" },
      });
      expect(activeTab(forward.json)?.url).toBe(page("three"));
      expect(
        (await call("/bot-browser/navigate", { body: { action: "reload" } }))
          .status,
      ).toBe(200);
      expect(
        (
          await call("/bot-browser/navigate", {
            body: { url: "javascript:alert(1)" },
          })
        ).status,
      ).toBe(400);

      const closed = await call<BotBrowserState>("/bot-browser/tabs", {
        body: { action: "close", tabId: second.id },
      });
      expect(closed.json.tabs.map((tab) => tab.id)).toEqual([first.id]);

      // Handed back: nobody is named, and the bar refuses again.
      expect(
        (await call("/control/release", { body: { requestId } })).status,
      ).toBe(200);
      expect(
        (await call<ComputerControlState>("/control")).json.heldBy,
      ).toBeUndefined();
      expect(
        (await call("/bot-browser/tabs", { body: { action: "new" } })).status,
      ).toBe(409);
      // And the Bot carries on in the tab the person left in front.
      expect((await call<{ url: string }>("/read")).json.url).toBe(
        page("three"),
      );
    }, 60_000);

    test("records the person holding the wheel and nobody else, narrates, and Stop ends it and disconnects the Recorder", async () => {
      const requestId = await takeWheel("user-1");
      calls.length = 0;
      // Somebody who does not hold the wheel cannot record, and the Recorder is never handed their connection.
      expect((await start("user-2")).status).toBe(409);
      expect(calls).toEqual([]);

      const started = await start("user-1");
      expect(started.status).toBe(200);
      expect(started.json.recordingBy).toBe("user-1");
      expect(started.json.recorder).toMatchObject({
        connected: true,
        userId: "user-1",
        phase: "recording",
        title: "File an expense report",
        narration: "external",
      });
      // The Recorder's side: connected with the person's connection for the app's narration, then recording the tab
      // the person has in front.
      expect(callsFor("connect").map((each) => each.input)).toEqual([
        { connection: connection(), narration: "external" },
      ]);
      const [record] = callsFor("record");
      expect(record?.input).toEqual({
        action: "start",
        title: "File an expense report",
      });
      expect(record?.front).toBe(activeTab(started.json)?.url ?? "missing");
      expect((await start("user-1", "Again")).status).toBe(409);

      const sentAt = 1_760_000_003_000;
      const phrase = {
        text: "This is where the receipts go",
        startedAt: 1_760_000_000_000,
        endedAt: 1_760_000_002_500,
      };
      expect(
        (await call("/bot-browser/narration", { body: { ...phrase, sentAt } }))
          .status,
      ).toBe(200);
      expect(
        (
          await call("/bot-browser/narration", {
            body: { speaking: true, since: 1_760_000_004_000, sentAt },
          })
        ).status,
      ).toBe(200);
      expect(callsFor("narrate").map((each) => each.input)).toEqual([phrase]);
      expect(callsFor("speaking").map((each) => each.input)).toEqual([
        { since: 1_760_000_004_000 },
      ]);
      expect((await state()).recordingBy).toBe("user-1");

      const stopped = await call<BotBrowserState>("/bot-browser/recording", {
        body: { action: "stop" },
      });
      expect(stopped.status).toBe(200);
      expect(stopped.json.recordingBy).toBeUndefined();
      // Whose it was, beside the recording the Recorder names, and still so when the bar is read again.
      expect(stopped.json.lastStoppedBy).toBe("user-1");
      expect((await state()).lastStoppedBy).toBe("user-1");
      expect(stopped.json.recorder).toMatchObject({
        connected: false,
        phase: "idle",
        lastStopped: {
          recordingId: started.json.recorder?.recordingId,
          title: "File an expense report",
        },
      });
      // Stopped, then disconnected, in that order.
      expect(calls.map((each) => each.method).slice(-2)).toEqual([
        "record",
        "disconnect",
      ]);
      expect(callsFor("record").at(-1)?.input).toEqual({ action: "stop" });
      expect(
        (
          await call("/bot-browser/narration", {
            body: { ...phrase, sentAt },
          })
        ).status,
      ).toBe(409);
      await call("/control/release", { body: { requestId } });
    }, 60_000);

    test("handing the wheel back ends the recording, then disconnects the Recorder, before the hand-back answers", async () => {
      const requestId = await takeWheel("user-1");
      expect((await start("user-1", "Handed back")).status).toBe(200);
      calls.length = 0;
      const released = await call<ComputerControlState>("/control/release", {
        body: { requestId },
      });
      expect(released.status).toBe(200);
      expect(calls.map((each) => each.method)).toEqual([
        "record",
        "disconnect",
      ]);
      expect(calls[0]?.input).toEqual({ action: "stop" });
      const after = await state();
      expect(after.recordingBy).toBeUndefined();
      expect(after.recorder).toMatchObject({
        connected: false,
        phase: "idle",
        lastStopped: { title: "Handed back" },
      });
      expect(after.lastStoppedBy).toBe("user-1");
    }, 60_000);

    test("a cancelled request ends the recording though the person still has the page, and they cannot start another", async () => {
      const requestId = await takeWheel("user-1");
      expect((await start("user-1", "Cancelled")).status).toBe(200);
      calls.length = 0;
      expect(
        (await call("/control/cancel", { body: { requestId } })).status,
      ).toBe(200);
      expect(calls.map((each) => each.method)).toEqual([
        "record",
        "disconnect",
      ]);
      expect((await state()).recordingBy).toBeUndefined();
      expect((await start("user-1", "After the cancel")).status).toBe(409);
      await call("/control/release", { body: { requestId } });
    }, 60_000);

    test("stopping the computer ends the recording while the Recorder can still be told", async () => {
      await takeWheel("user-1");
      expect((await start("user-1", "Stopped computer")).status).toBe(200);
      calls.length = 0;
      expect((await call("/computers/stop", { body: {} })).json).toMatchObject({
        stopped: true,
        wasRunning: true,
      });
      expect(calls.map((each) => each.method)).toEqual([
        "record",
        "disconnect",
      ]);
      expect(await state()).toEqual({ open: false, tabs: [] });
      expect(
        (await call<ComputerControlState>("/control")).json.heldBy,
      ).toBeUndefined();
    }, 60_000);
  },
);

describe.skipIf(!asked)("clearing a Bot's browser", () => {
  const botId = "clear-bot";
  const on = (path: string, body?: unknown) =>
    call<BotBrowserState>(path, { body, botId });
  const clear = (by: string) => on("/bot-browser/clear", { by });
  /** The cookies the Bot's browser sends, read by the Bot itself. */
  const botSees = async () =>
    (
      await call<{ title: string }>("/navigate", {
        body: { url: `${origin}/cookies` },
        botId,
      })
    ).json.title;
  /** The cookies the Bot's browser sends, read through the bar by the person holding the wheel. */
  const personSees = async () => {
    const there = await on("/bot-browser/navigate", {
      url: `${origin}/cookies`,
    });
    expect(there.status).toBe(200);
    return there.json.tabs.find((tab) => tab.active)?.title;
  };

  test("only the person holding the wheel clears it: their recording ends first, the profile goes and the Bot's files stay, and the browser starts fresh under them", async () => {
    // The Bot signs in somewhere (a cookie kept for an hour) and keeps a note in its files.
    expect(
      (
        await call("/navigate", {
          body: { url: `${origin}/set-cookie` },
          botId,
        })
      ).status,
    ).toBe(200);
    expect(await botSees()).toBe("cookies:signed-in=yes");
    expect(
      (
        await call("/files/write", {
          body: { path: "notes.md", contents: "Keep me." },
          botId,
        })
      ).status,
    ).toBe(200);

    // Never under the Bot, and never by someone who is only watching.
    expect((await clear("user-1")).status).toBe(409);
    const asked = await call<ComputerControlState>("/control/request", {
      body: { reason: "I want to take control of the browser." },
      botId,
    });
    const requestId = asked.json.request?.id;
    expect(
      (
        await call("/control/take", {
          body: { requestId, by: "user-1" },
          botId,
        })
      ).status,
    ).toBe(200);
    const refused = await clear("user-2");
    expect(refused.status).toBe(409);
    expect(refused.json as unknown).toEqual({
      error:
        "Only the person holding control of this Bot's browser can clear it. Take control first.",
    });
    expect(await personSees()).toBe("cookies:signed-in=yes");

    // A recording of theirs is running, and the last of it takes a moment to reach Intelligence after Stop.
    expect(
      (
        await on("/bot-browser/recording", {
          action: "start",
          title: "Before the clear",
          by: "user-1",
          connection: connection(),
        })
      ).status,
    ).toBe(200);
    tailMs = 1_500;
    calls.length = 0;
    const cleared = await clear("user-1");
    tailMs = 0;
    expect(cleared.status).toBe(200);
    // Stopped, waited for, and let go, before the browser was wiped.
    expect(calls.map((each) => each.method)).toEqual(["record", "disconnect"]);
    expect(calls[0]?.input).toEqual({ action: "stop" });
    expect(calls[1]?.sending).toBe(false);
    // Fresh: one empty tab, nothing recording, the Recorder in it again and connected to nobody.
    expect(cleared.json.open).toBe(true);
    expect(cleared.json.recordingBy).toBeUndefined();
    expect(cleared.json.tabs).toEqual([
      expect.objectContaining({ url: "about:blank", active: true }),
    ]);
    expect(cleared.json.recorder).toMatchObject({
      connected: false,
      phase: "idle",
    });
    // Still theirs: the wheel stayed with them, and the bar works the fresh browser.
    expect(
      (await call<ComputerControlState>("/control", { botId })).json,
    ).toMatchObject({
      holder: "human",
      heldBy: "user-1",
      recoveryRequired: false,
    });
    // Signed out: the cookie went with the profile, where only closing the browser would have kept it.
    expect(await personSees()).toBe("cookies:");
    // The Bot's files are its own, and stay.
    expect(
      await Bun.file(
        join(root, "with-recorder", "workspace", "notes.md"),
      ).text(),
    ).toBe("Keep me.");

    expect(
      (await call("/control/release", { body: { requestId }, botId })).status,
    ).toBe(200);
  }, 90_000);
});

describe.skipIf(!asked)("a person's keys on a Bot's live screen", () => {
  test("reach the page in the order they were typed, however long each takes to land", async () => {
    // Applying a key is a trip to the browser, and on a page busy 20 ms per key down, keys applied as each was ready
    // rather than one after another reached it with every key up of a word before any of its key downs.
    const botId = "other-bot";
    expect(
      (await call("/navigate", { body: { url: `${origin}/keys` }, botId }))
        .status,
    ).toBe(200);
    const asked = await call<ComputerControlState>("/control/request", {
      body: { reason: "I want to take control of the browser." },
      botId,
    });
    await call("/control/take", {
      body: { requestId: asked.json.request?.id, by: "user-1" },
      botId,
    });
    const socket = new WebSocket(
      `${computer.base.replace("http:", "ws:")}/stream?bot=${botId}&token=${TOKEN}`,
    );
    const errors: string[] = [];
    const casting = new Promise<void>((resolve) => {
      socket.addEventListener("message", (event) => {
        const message = JSON.parse(String(event.data)) as {
          type: string;
          error?: string;
        };
        if (message.type === "frame") resolve();
        if (message.type === "error") errors.push(message.error ?? "");
      });
    });
    try {
      await casting;
      const typed = "abcdef";
      for (const key of typed) {
        const code = `Key${key.toUpperCase()}`;
        socket.send(
          JSON.stringify({ type: "key", event: "down", key, code, text: key }),
        );
        socket.send(JSON.stringify({ type: "key", event: "up", key, code }));
      }
      const expected = [...typed]
        .flatMap((key) => [`down:${key}`, `up:${key}`])
        .join(" ");
      let title = "";
      const deadline = Date.now() + 10_000;
      while (Date.now() < deadline) {
        title = (await call<{ title: string }>("/read", { botId })).json.title;
        if (title.split(" ").length >= expected.split(" ").length) break;
        await Bun.sleep(50);
      }
      expect(title).toBe(expected);
      expect(errors).toEqual([]);
    } finally {
      socket.close();
      await call("/control/release", {
        body: { requestId: asked.json.request?.id },
        botId,
      });
    }
  }, 60_000);
});

describe.skipIf(!asked)("what a person types while they hold the wheel", () => {
  type Snapshot = {
    snapshotId: number;
    elements: { ref: string; role: string; name: string; value?: string }[];
  };
  const snapshot = async (botId: string) =>
    (await call<Snapshot>("/snapshot", { body: {}, botId })).json;
  const field = (taken: Snapshot, name: string) =>
    taken.elements.find((element) => element.name === name);
  const FILLED = "[hidden: filled in by the person]";

  test("reaches the Bot as filled, never as what was typed, in its snapshot and its page text, until the Bot types there itself", async () => {
    // The run that found it: after a help step the Bot's next snapshot read `textbox "Telephone": 555-0100`, though
    // the person took over so they could enter what the Bot must not be told.
    const botId = "typed-bot";
    expect(
      (await call("/navigate", { body: { url: `${origin}/contact` }, botId }))
        .status,
    ).toBe(200);
    // The Bot fills the name itself: its own value, which it goes on reading.
    const before = await snapshot(botId);
    const name = field(before, "Name");
    if (!name) throw new Error("The form should have a Name field.");
    expect(
      (
        await call("/type", {
          body: {
            ref: name.ref,
            snapshotId: before.snapshotId,
            text: "Ada Lovelace",
          },
          botId,
        })
      ).status,
    ).toBe(200);

    const asked = await call<ComputerControlState>("/control/request", {
      body: { reason: "Enter the telephone number." },
      botId,
    });
    const requestId = asked.json.request?.id;
    await call("/control/take", { body: { requestId, by: "user-1" }, botId });
    const socket = new WebSocket(
      `${computer.base.replace("http:", "ws:")}/stream?bot=${botId}&token=${TOKEN}`,
    );
    const errors: string[] = [];
    const casting = new Promise<void>((resolve) => {
      socket.addEventListener("message", (event) => {
        const message = JSON.parse(String(event.data)) as {
          type: string;
          error?: string;
        };
        if (message.type === "frame") resolve();
        if (message.type === "error") errors.push(message.error ?? "");
      });
    });
    const key = (key: string, code: string, windowsVirtualKeyCode: number) => {
      for (const event of ["down", "up"])
        socket.send(
          JSON.stringify({
            type: "key",
            event,
            key,
            code,
            windowsVirtualKeyCode,
          }),
        );
    };
    try {
      await casting;
      // From the name the Bot filled, on through the form as a person would: Tab, type, Tab, choose, Tab, type.
      key("Tab", "Tab", 9);
      socket.send(JSON.stringify({ type: "text", text: "555-0100" }));
      key("Tab", "Tab", 9);
      key("ArrowDown", "ArrowDown", 40);
      key("Tab", "Tab", 9);
      socket.send(JSON.stringify({ type: "text", text: "call after six" }));
      // Landed once the editor holds something, as the page's text says it either way.
      const deadline = Date.now() + 10_000;
      for (;;) {
        const text = (await call<{ text: string }>("/read", { botId })).json
          .text;
        if (text.includes("call after six") || text.includes(FILLED)) break;
        if (Date.now() > deadline)
          throw new Error("The person's typing did not land.");
        await Bun.sleep(50);
      }
      expect(errors).toEqual([]);
    } finally {
      socket.close();
    }
    expect(
      (await call("/control/release", { body: { requestId }, botId })).status,
    ).toBe(200);

    // Back with the Bot: it reads that the person filled each, and never what with.
    const after = await snapshot(botId);
    expect(field(after, "Name")?.value).toBe("Ada Lovelace");
    expect(field(after, "Telephone")?.value).toBe(FILLED);
    expect(field(after, "Size")?.value).toBe(FILLED);
    expect(JSON.stringify(after)).not.toContain("555-0100");
    const read = (await call<{ text: string }>("/read", { botId })).json.text;
    expect(read).toContain(FILLED);
    expect(read).not.toContain("call after six");
    expect(read).not.toContain("555-0100");
    expect((await call("/screenshot", { botId })).status).toBe(200);

    // The Bot typing into a field the person filled makes the value its own again.
    const telephone = field(after, "Telephone");
    if (!telephone) throw new Error("The form should have a Telephone field.");
    expect(
      (
        await call("/type", {
          body: {
            ref: telephone.ref,
            snapshotId: after.snapshotId,
            text: "020 7946 0000",
          },
          botId,
        })
      ).status,
    ).toBe(200);
    expect(field(await snapshot(botId), "Telephone")?.value).toBe(
      "020 7946 0000",
    );
  }, 60_000);
});

describe.skipIf(!asked)(
  "a Bot's browser on a computer without the Recorder",
  () => {
    let without: Computer;

    beforeAll(async () => {
      without = await startComputer("without-recorder", {
        OPENBOT_RECORDER_DIR: "",
        COMPUTER_BOT_ID: "other-bot",
      });
    }, 30_000);

    test("launches as it always has, its bar works for the person holding the wheel, and Record says it cannot", async () => {
      expect(
        (
          await call("/navigate", {
            body: { url: page("plain") },
            botId: "other-bot",
            on: without,
          })
        ).status,
      ).toBe(200);
      const plain = (
        await call<BotBrowserState>("/bot-browser/state", {
          botId: "other-bot",
          on: without,
        })
      ).json;
      expect(plain).toEqual({
        open: true,
        tabs: [expect.objectContaining({ url: page("plain"), active: true })],
      });

      const asked = await call<ComputerControlState>("/control/request", {
        body: { reason: "I want to take control of the browser." },
        botId: "other-bot",
        on: without,
      });
      await call("/control/take", {
        body: { requestId: asked.json.request?.id, by: "user-1" },
        botId: "other-bot",
        on: without,
      });
      const opened = await call<BotBrowserState>("/bot-browser/tabs", {
        body: { action: "new", url: page("second") },
        botId: "other-bot",
        on: without,
      });
      expect(opened.status).toBe(200);
      expect(activeTab(opened.json)?.url).toBe(page("second"));

      const record = await call<{ error: string }>("/bot-browser/recording", {
        body: {
          action: "start",
          title: "Nothing to record with",
          by: "user-1",
          connection: connection(),
        },
        botId: "other-bot",
        on: without,
      });
      expect(record).toEqual({
        status: 503,
        json: {
          error:
            "This Bot's browser cannot record: the OpenBot Recorder is not installed on its computer.",
        },
      });
    }, 60_000);
  },
);

/**
 * The server's control routes and gateway, against this computer: the path a person's take and a Bot's request for help
 * travel in a deployment, minus only the sign-in in front of the routes.
 */
function serverOn(on: Computer) {
  const provider: ComputerProvider = {
    name: "this-test",
    isolation: "per-person",
    locate: async () => on.base,
    status: async (seat) => ({ botId: seat.botId, state: "ready" }),
    stop: async () => ({ wasRunning: true }),
    reset: async () => ({ cleared: true }),
    list: async () => [],
  };
  const gateway = createComputerGateway({
    provider,
    auditStore: {
      insert: async (_event: AuditEventInput) => {},
    } as unknown as AuditStore,
    policy: () => ({ mode: "enforce", deny: [], allow: ["true"] }),
    token: TOKEN,
  });
  const actor: AuthenticatedActor = {
    id: "user-1",
    email: "[email protected]",
    role: "user",
  };
  const signedIn: MiddlewareHandler<{ Variables: AppVariables }> = async (
    context,
    next,
  ) => {
    context.set("actor", actor);
    await next();
  };
  const routes = createComputerRoutes(
    gateway,
    {} as PolicyStore,
    signedIn,
    async () => true,
  );
  return {
    gateway,
    actor,
    /** A POST to the server's route under `/api/computers`, as the app makes it. */
    async post(botId: string, path: string, body: unknown) {
      const response = await routes.request(
        `http://openbot.test/${botId}${path}`,
        {
          method: "POST",
          headers: { "content-type": "application/json" },
          body: JSON.stringify(body),
        },
      );
      return {
        status: response.status,
        json: (await response.json()) as ComputerControlState,
      };
    },
  };
}

describe.skipIf(!asked)("who asked for the wheel", () => {
  test("a person's own take is recorded as theirs (manual) through the server's route, and the Bot's request for help as the Bot's (model)", async () => {
    const botId = "source-bot";
    const server = serverOn(computer);
    // What the app's Take control sends: its own words for asking, and that it is a person's take.
    const asked = await server.post(botId, "/control/request", {
      reason: "I want to take control of the browser.",
      source: "manual",
    });
    expect(asked.status).toBe(200);
    expect(asked.json.request).toMatchObject({
      source: "manual",
      reason: "I want to take control of the browser.",
      status: "waiting",
    });
    const requestId = asked.json.request?.id ?? "";
    const taken = await server.post(botId, "/control/take", { requestId });
    expect(taken.status).toBe(200);
    // The computer's own record, read past the server: the person's take, held by them.
    const held = await call<ComputerControlState>(
      `/control?requestId=${requestId}`,
      { botId },
    );
    expect(held.json).toMatchObject({
      holder: "human",
      heldBy: "user-1",
      request: { source: "manual", status: "taken" },
    });
    expect(
      (await server.post(botId, "/control/release", { requestId })).status,
    ).toBe(200);

    // The Bot asking, as the headless tools and the password service do: no source, so the Bot's.
    // On the seat the routes reach for this person, as the headless tools resolve it.
    const bots = await server.gateway.requestHelp(
      await server.gateway.seatFor(botId, server.actor.id),
      server.actor,
      "Sign in to the bank, then hand back.",
      "call-sign-in",
    );
    expect(bots.request).toMatchObject({
      source: "model",
      reason: "Sign in to the bank, then hand back.",
    });
    expect(
      (
        await server.post(botId, "/control/cancel", {
          requestId: bots.request?.id,
        })
      ).status,
    ).toBe(200);
    // And as the app's own help request does (`runHelpRequest`), through the route with no source: the Bot's too.
    const viaRoute = await server.post(botId, "/control/request", {
      reason: "Solve the puzzle on this page.",
      toolCallId: "call-puzzle",
    });
    expect(viaRoute.json.request).toMatchObject({
      source: "model",
      reason: "Solve the puzzle on this page.",
    });
    // A caller cannot claim a challenge the computer did not see.
    expect(
      (
        await call("/control/request", {
          body: { reason: "Nothing", source: "cloudflare" },
          botId,
        })
      ).status,
    ).toBe(400);
  }, 60_000);
});