Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

19.5 KB
Raw Blame
// What OpenBot's own page asks the Recorder in the same browser (shared/recorder-page.ts), end to end through the
// Recorder's own code: the real service worker (connection, recording, router, setup, launcher) and the real offscreen
// document side by side over a fake OpenBot and Intelligence, driven through `externally_connectable` messages as
// Chrome delivers them, with the sender's origin, tab and frame as Chrome reports them. The steps are rows of the full
// end-to-end run (fixtures/e2e-held-steps.json).
import { afterEach, beforeEach, expect, test } from "bun:test";
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { LAUNCHER_GLOBAL } from "@openbot/shared/openbot-browser";
import {
  RECORDER_SIGNED_OUT,
  recorderPaths,
} from "@openbot/shared/recorder-api";
import type { CapturedEvent } from "@openbot/shared/recorder-masking";
import { IDBKeyRange as FakeIDBKeyRange, IDBFactory } from "fake-indexeddb";
import { installConnection } from "../src/background/connection";
import { installLauncher } from "../src/background/launcher";
import {
  installRecording,
  type RecordingDeps,
} from "../src/background/recording";
import { installRouter } from "../src/background/router";
import { installSetup } from "../src/background/setup";
import { createOffscreenController } from "../src/offscreen/index";
import { openOutbox } from "../src/offscreen/outbox";
import {
  PORT_CONTENT,
  PORT_OFFSCREEN,
  type RecorderState,
} from "../src/protocol/messages";
import type { ConnectionRecord, DrainRecord } from "../src/shared/storage";
import { type ChromeFake, installChromeFake } from "./support/chrome-fake";
import { type FakeGateway, startFakeGateway } from "./support/fake-gateway";
import { fakeMicrophone, fakeSpeech } from "./support/fake-speech";

const EXTENSION_ID = "test";
const ada = { id: "p1", name: "Ada", email: "[email protected]" };
const ORDER = "http://127.0.0.1:7420/orders/1043";
const TOKEN = "obr_openbot-page-token";
const NOT_CONNECTED = "Connect this browser to OpenBot first.";

const rows = JSON.parse(
  readFileSync(
    join(import.meta.dir, "fixtures", "e2e-held-steps.json"),
    "utf8",
  ),
) as { archiveClick: CapturedEvent };

type Globals = { indexedDB?: IDBFactory; IDBKeyRange?: unknown };
const globals = globalThis as unknown as Globals;
let saved: Globals = {};

const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
const settle = () => sleep(20);
async function until(check: () => boolean | Promise<boolean>, label: string) {
  const deadline = Date.now() + 8_000;
  while (!(await check())) {
    if (Date.now() > deadline) throw new Error(`Timed out waiting: ${label}`);
    await sleep(5);
  }
}

// ---- The fake OpenBot and Intelligence --------------------------------------------------------

let gateway: FakeGateway;
/** The tokens OpenBot still knows; the codes it took, and the tokens the Recorder asked it to revoke. */
let liveTokens = new Set<string>();
let exchanged: string[] = [];
let revokedTokens: string[] = [];
/** OpenBot taking the code OpenBot's page handed the Recorder. */
async function exchange(request: Request): Promise<Response> {
  const { code } = (await request.json()) as { code: string };
  exchanged.push(code);
  liveTokens.add(TOKEN);
  return Response.json({
    token: TOKEN,
    connection: { id: `c-${TOKEN}` },
    user: ada,
  });
}
function route(request: Request): Response | Promise<Response> | null {
  const path = new URL(request.url).pathname;
  const token =
    request.headers.get("authorization")?.replace(/^Bearer /, "") ?? "";
  if (path === recorderPaths.connections && request.method === "POST")
    return exchange(request);
  if (path === recorderPaths.currentConnection && request.method === "DELETE") {
    revokedTokens.push(token);
    liveTokens.delete(token);
    return new Response(null, { status: 204 });
  }
  if (path === recorderPaths.me) {
    if (!liveTokens.has(token))
      return Response.json(
        {
          error: "This browser is no longer connected to OpenBot.",
          code: RECORDER_SIGNED_OUT,
          reason: "revoked",
        },
        { status: 401 },
      );
    return Response.json({ user: ada, connectionId: `c-${token}` });
  }
  if (path.startsWith("/api/recorder/"))
    return Response.json({ error: "Not found." }, { status: 404 });
  const pass = /^\/api\/copilotkit\/trajectory\/([^/]+)\/connect$/.exec(path);
  if (!pass || request.method !== "POST") return null;
  const trajectoryId = decodeURIComponent(pass[1] ?? "");
  return Response.json({
    joinToken: gateway.mintToken(trajectoryId),
    realtime: {
      clientUrl: `${gateway.url}/client`,
      topic: `trajectory:${trajectoryId}`,
    },
  });
}
const sent = (trajectoryId: string) =>
  gateway.events(trajectoryId).map((entry) => entry.raw as CapturedEvent);
const names = (trajectoryId: string) =>
  sent(trajectoryId).map((event) => event.name);

// ---- Time, and the two contexts ---------------------------------------------------------------

let clock = 0;
type Due = { id: number; at: number; fn: () => void };
let due: Due[] = [];
let nextTimer = 0;
const workerTimers = {
  set(fn: () => void, ms: number) {
    nextTimer += 1;
    due.push({ id: nextTimer, at: clock + Math.max(0, ms), fn });
    return nextTimer;
  },
  clear(handle: unknown) {
    due = due.filter((timer) => timer.id !== handle);
  },
};
/** Moves the clock to `to`, firing the service worker's held-step timers that fall due on the way, in order. */
async function advanceTo(to: number) {
  for (;;) {
    const next = due
      .filter((timer) => timer.at <= to)
      .sort((a, b) => a.at - b.at)[0];
    if (!next) break;
    due = due.filter((timer) => timer !== next);
    clock = Math.max(clock, next.at);
    next.fn();
    await settle();
  }
  clock = Math.max(clock, to);
  await settle();
}

let fake: ChromeFake;
let databaseName = "";
let databaseCounter = 0;
let documentTimers: ReturnType<typeof setTimeout>[] = [];

function runDocument(): void {
  fake.runAs("offscreen-document", () =>
    createOffscreenController({
      openOutbox: () => openOutbox(databaseName),
      connectPort: () => chrome.runtime.connect({ name: PORT_OFFSCREEN }),
      now: () => clock,
      timers: {
        set: (fn, ms) => {
          const handle = setTimeout(fn, ms);
          documentTimers.push(handle);
          return handle;
        },
        clear: (handle) =>
          clearTimeout(handle as ReturnType<typeof setTimeout>),
      },
      voice: {
        Recognition: fakeSpeech().Recognition,
        getUserMedia: fakeMicrophone().getUserMedia,
      },
    }),
  );
}

/** The service worker as background/index.ts starts it, on the test's clock. */
function startWorker(): void {
  const deps: Partial<RecordingDeps> = {
    createNetworkCapture: () => ({
      start: async () => "debugger",
      addTab: async () => {},
      onCommitted: async () => {},
      resume: async () => "debugger",
      recordPageEvent: async () => {},
      recordPageAgent: () => {},
      stop: async () => {},
    }),
    createPageFallback: () => ({
      start: async () => false,
      addTab: async () => {},
      onCommitted: async () => {},
      stop: async () => {},
      active: () => false,
    }),
    acceptPageNetworkEvent: () => null,
    hasHostPermission: async () => true,
    now: () => clock,
    timers: { set: () => 0, clear: () => {} },
    stepTimers: workerTimers,
  };
  const recording = installRecording(deps);
  installRouter(recording);
  installConnection({
    connectionLost: recording.connectionLost,
    connected: recording.connected,
  });
  installSetup();
  installLauncher(recording, { now: () => clock });
}

// ---- The tabs, and who asks ---------------------------------------------------------------------

const OPENBOT_TAB = 7;
const ORDER_TAB = 8;
const openbotPage = () => `${gateway.httpUrl}/`;

/** OpenBot in tab 7, the tab the person is on, and an order already open in tab 8 behind it. */
function openTabs(): void {
  fake.tabs.set(OPENBOT_TAB, {
    id: OPENBOT_TAB,
    url: openbotPage(),
    title: "OpenBot",
    active: true,
    windowId: 1,
  } as chrome.tabs.Tab);
  fake.tabs.set(ORDER_TAB, {
    id: ORDER_TAB,
    url: ORDER,
    title: "Order 1043 - Corner Shop admin",
    active: false,
    windowId: 1,
  } as chrome.tabs.Tab);
}

/** OpenBot's page (OpenBot Desktop's, on its loopback address), as Chrome reports it to the Recorder: its top frame. */
const fromOpenBot = (
  overrides: Partial<chrome.runtime.MessageSender> = {},
): chrome.runtime.MessageSender => ({
  origin: gateway.httpUrl,
  url: openbotPage(),
  tab: fake.tabs.get(OPENBOT_TAB),
  frameId: 0,
  ...overrides,
});

/** Pages that are not the connected OpenBot, each as Chrome would report it. */
const others = (): chrome.runtime.MessageSender[] => [
  {
    origin: "https://evil.example",
    url: "https://evil.example/",
    tab: { id: 9 } as chrome.tabs.Tab,
    frameId: 0,
  },
  // Another server on this computer, on another port: not the OpenBot this copy is connected to.
  {
    origin: "http://127.0.0.1:1",
    url: "http://127.0.0.1:1/",
    tab: { id: 10 } as chrome.tabs.Tab,
    frameId: 0,
  },
  // A sandboxed frame's opaque origin, and a sender Chrome gives no origin.
  {
    origin: "null",
    url: "about:blank",
    tab: { id: 9 } as chrome.tabs.Tab,
    frameId: 2,
  },
  { url: openbotPage(), tab: { id: 9 } as chrome.tabs.Tab, frameId: 0 },
];

const ask = (message: unknown, sender = fromOpenBot()) =>
  fake.sendExternal(message, sender);
const stored = () =>
  fake.storage.local.get("connection") as ConnectionRecord | undefined;
async function recorderState(): Promise<RecorderState> {
  const answer = (await fake.sendFromPage(
    { type: "page.state" },
    { url: "chrome-extension://test/popup.html" },
  )) as { state: RecorderState };
  return answer.state;
}

/** OpenBot's page connects this browser: OpenBot Desktop's own address, so with no Connect page. */
async function connectFromOpenBot(): Promise<void> {
  expect(
    await ask({ type: "openbot.connect", code: "obc_from-openbot" }),
  ).toMatchObject({ ok: true, connected: true });
  await settle();
}

beforeEach(async () => {
  saved = { indexedDB: globals.indexedDB, IDBKeyRange: globals.IDBKeyRange };
  globals.indexedDB = new IDBFactory();
  globals.IDBKeyRange = FakeIDBKeyRange;
  databaseName = `openbot-page-${Date.now()}-${databaseCounter++}`;
  due = [];
  documentTimers = [];
  liveTokens = new Set();
  exchanged = [];
  revokedTokens = [];
  gateway = await startFakeGateway({ extraRoutes: route });
  fake = installChromeFake({
    extensionId: EXTENSION_ID,
    manifestVersion: "0.3.0",
  });
  // OpenBot Desktop runs on this computer, and its native host confirms the address its OpenBot answers on
  // (recorder_host.rs `address`): that is what makes OpenBot's page there one the Recorder connects with no click.
  fake.native.reply = (message) =>
    (message as { type?: string }).type === "address"
      ? { ok: true, openbotUrl: gateway.httpUrl }
      : { ok: false, error: "unknown-message" };
  fake.offscreen.onDocumentCreated.addListener(() => {
    setTimeout(runDocument, 0);
  });
  clock = rows.archiveClick.timestamp - 7_000;
  startWorker();
  openTabs();
  await settle();
}, 15_000);

afterEach(async () => {
  await settle();
  for (const handle of documentTimers) clearTimeout(handle);
  await gateway.stop();
  fake.uninstall();
  Reflect.deleteProperty(globalThis, LAUNCHER_GLOBAL);
  for (const key of ["indexedDB", "IDBKeyRange"] as const) {
    if (saved[key] === undefined) Reflect.deleteProperty(globals, key);
    else Object.assign(globals, { [key]: saved[key] });
  }
}, 15_000);

test("OpenBot's page connects with no Connect page, records its own tab and then a tab the person switches to, and stops", async () => {
  expect(await ask({ type: "openbot.state" })).toEqual({ connected: false });
  await connectFromOpenBot();
  expect(exchanged).toEqual(["obc_from-openbot"]);
  expect(fake.opened).toEqual([]);
  expect(stored()).toMatchObject({
    openbotUrl: gateway.httpUrl,
    token: TOKEN,
    via: "web",
  });
  expect(await ask({ type: "openbot.state" })).toEqual({
    connected: true,
    phase: "idle",
    actions: 0,
  });

  const startedAt = clock;
  expect(
    await ask({ type: "openbot.record", title: "  Archive order 1043  " }),
  ).toEqual({
    ok: true,
    state: {
      connected: true,
      phase: "recording",
      title: "Archive order 1043",
      startedAt,
      actions: 0,
    },
  });
  // It records the tab OpenBot asked from, through the popup's own Record.
  expect((await recorderState()).tabIds).toEqual([OPENBOT_TAB]);
  const active = fake.storage.local.get("activeRecording") as DrainRecord;
  const trajectoryId = active.trajectoryIds[0] ?? "";
  await until(
    () => names(trajectoryId).includes("outcome"),
    "the recording's start in Intelligence",
  );

  // The person goes to the order, open in a tab behind OpenBot's, and archives it there.
  fake.person.switchTo(ORDER_TAB);
  await until(
    async () => (await recorderState()).tabIds.includes(ORDER_TAB),
    "the tab the person switched to followed",
  );
  const port = fake.connectFrom(PORT_CONTENT, {
    id: EXTENSION_ID,
    tab: fake.tabs.get(ORDER_TAB),
    frameId: 0,
    url: ORDER,
  });
  const click = rows.archiveClick;
  clock = click.timestamp;
  port.postMessage({ type: "content.event", event: click });
  await settle();
  expect(await ask({ type: "openbot.state" })).toEqual({
    connected: true,
    phase: "recording",
    title: "Archive order 1043",
    startedAt,
    actions: 1,
  });
  // A second Record while one runs is refused with the Recorder's own sentence, and changes nothing.
  expect(await ask({ type: "openbot.record", title: "Again" })).toEqual({
    ok: false,
    error: "A recording is already running.",
  });

  await advanceTo(click.timestamp + 5_000);
  const stoppedAt = clock;
  expect(await ask({ type: "openbot.stop" })).toEqual({
    ok: true,
    state: {
      connected: true,
      phase: "idle",
      actions: 0,
      lastStopped: {
        recordingId: active.recordingId,
        title: "Archive order 1043",
        stoppedAt,
        actions: 1,
        phrases: 0,
      },
    },
  });
  await until(
    () => names(trajectoryId).filter((name) => name === "outcome").length === 2,
    "the recording stopped",
  );
  // OpenBot's page as the recording's first tab, then the order as its second, with the click in it.
  const steps = sent(trajectoryId).filter(
    (event) => event.name === "navigation" || event.name === "click",
  );
  expect(
    steps.map((event) => {
      // A navigation names the page it went `to`; a click the page it was on (`url`).
      const value = event.value as { tab?: number; to?: string; url?: string };
      return [event.name, value.tab, value.to ?? value.url];
    }),
  ).toEqual([
    ["navigation", 1, openbotPage()],
    ["navigation", 2, ORDER],
    ["click", 2, ORDER],
  ]);
  expect(sent(trajectoryId).at(-1)?.value).toMatchObject({
    phase: "succeeded",
  });
  // State after Stop says the same, and Stop again changes nothing.
  expect(await ask({ type: "openbot.state" })).toMatchObject({
    phase: "idle",
    lastStopped: { recordingId: active.recordingId, actions: 1 },
  });
  expect(await ask({ type: "openbot.stop" })).toMatchObject({
    ok: true,
    state: { phase: "idle", lastStopped: { recordingId: active.recordingId } },
  });
}, 20_000);

test("any other page gets not connected and a refusal, the same whether this copy is connected or not, and learns nothing", async () => {
  const answers = async () => {
    const all: unknown[] = [];
    for (const sender of others())
      for (const message of [
        { type: "openbot.state" },
        { type: "openbot.record", title: "Steal it" },
        { type: "openbot.stop" },
      ])
        all.push(await ask(message, sender));
    return all;
  };
  const expected = others().flatMap(() => [
    { connected: false },
    { ok: false, error: NOT_CONNECTED },
    { ok: false, error: NOT_CONNECTED },
  ]);
  // Nothing connected.
  expect(await answers()).toEqual(expected);

  // Connected to OpenBot, and recording from it.
  await connectFromOpenBot();
  expect(
    await ask({ type: "openbot.record", title: "Archive order 1043" }),
  ).toMatchObject({ ok: true, state: { phase: "recording" } });
  const active = fake.storage.local.get("activeRecording") as DrainRecord;
  expect(await answers()).toEqual(expected);
  // The recording OpenBot started runs on, untouched, and no other started.
  expect(fake.storage.local.get("activeRecording")).toEqual(active);
  expect(await ask({ type: "openbot.state" })).toMatchObject({
    connected: true,
    phase: "recording",
    title: "Archive order 1043",
  });
  expect((await recorderState()).tabIds).toEqual([OPENBOT_TAB]);
}, 20_000);

test("OpenBot's origin in a frame of another page may read the state and stop, but records only from its own top frame", async () => {
  await connectFromOpenBot();
  const framed = fromOpenBot({
    url: openbotPage(),
    tab: { id: 9, url: "https://evil.example/" } as chrome.tabs.Tab,
    frameId: 4,
  });
  expect(await ask({ type: "openbot.record", title: "x" }, framed)).toEqual({
    ok: false,
    error: "Record from OpenBot's own tab.",
  });
  expect(fake.storage.local.get("activeRecording")).toBeUndefined();
  expect(await ask({ type: "openbot.state" }, framed)).toEqual({
    connected: true,
    phase: "idle",
    actions: 0,
  });
});

test("once the person disconnects this browser, OpenBot's page is told it is not connected and can no longer record", async () => {
  await connectFromOpenBot();
  expect(
    await fake.sendFromPage(
      { type: "page.disconnect" },
      { url: "chrome-extension://test/popup.html" },
    ),
  ).toEqual({ ok: true });
  expect(revokedTokens).toEqual([TOKEN]);
  expect(await ask({ type: "openbot.state" })).toEqual({ connected: false });
  expect(await ask({ type: "openbot.record", title: "x" })).toEqual({
    ok: false,
    error: NOT_CONNECTED,
  });
  expect(fake.storage.local.get("activeRecording")).toBeUndefined();
});

test("Record from OpenBot's page carries the person's narration choice from their account, and without one the Recorder's own", async () => {
  // OpenBot keeps the choice on the person's account, and its own Record form passes it on; a Record from its page
  // in the person's everyday browser used to leave it out, so the Recorder went by its own setting instead.
  await connectFromOpenBot();
  const voice = async () => (await recorderState()).capture.voice?.state;
  const recordAndStop = async (message: object) => {
    expect(await ask(message)).toMatchObject({ ok: true });
    await until(
      async () => (await voice()) !== undefined,
      "the recording's voice",
    );
    const heard = await voice();
    await advanceTo(clock + 5_000);
    expect(await ask({ type: "openbot.stop" })).toMatchObject({ ok: true });
    await until(
      async () => (await recorderState()).phase === "idle",
      "the recording stopped",
    );
    return heard;
  };
  // Nothing chosen in the Recorder, which is no microphone, while the account says to narrate.
  expect(
    await recordAndStop({
      type: "openbot.record",
      title: "Archive order 1043",
      narrate: true,
    }),
  ).not.toBe("muted");
  // The account says no, while the Recorder was set to narrate: the account's choice wins.
  fake.storage.local.set("narration", "on");
  expect(
    await recordAndStop({
      type: "openbot.record",
      title: "Archive order 1044",
      narrate: false,
    }),
  ).toBe("muted");
  // No choice from the account: the Recorder's own.
  expect(
    await recordAndStop({
      type: "openbot.record",
      title: "Archive order 1045",
    }),
  ).not.toBe("muted");
}, 20_000);