Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

17.9 KB
Raw Blame
// What the Recorder tells OpenBot it is recording (the Browser button's red dot), end to end through the Recorder's own
// code: the real service worker (connection, recording, router, setup, launcher) and the real offscreen document side
// by side over a fake OpenBot and Intelligence, on the test's clock, for a connection the popup records with and one
// an OpenBot browser's launcher made. Every report is read as OpenBot's route reads it (`recorderActivitySchema`).
import { afterEach, beforeEach, expect, spyOn, test } from "bun:test";
import {
  LAUNCHER_GLOBAL,
  type LauncherApi,
} from "@openbot/shared/openbot-browser";
import {
  RECORDER_SIGNED_OUT,
  type RecorderActivity,
  recorderActivitySchema,
  recorderPaths,
} from "@openbot/shared/recorder-api";
import type { CapturedEvent } from "@openbot/shared/recorder-masking";
import { IDBKeyRange as FakeIDBKeyRange, IDBFactory } from "fake-indexeddb";
import { ACTIVITY_REPORT_MS } from "../src/background/activity";
import { installConnection } from "../src/background/connection";
import { installLauncher } from "../src/background/launcher";
import {
  installRecording,
  type RecordingDeps,
} from "../src/background/recording";
import { installRouter } from "../src/background/router";
import { installSetup } from "../src/background/setup";
import { createOffscreenController } from "../src/offscreen/index";
import { openOutbox } from "../src/offscreen/outbox";
import {
  PORT_CONTENT,
  PORT_OFFSCREEN,
  type RecorderState,
} from "../src/protocol/messages";
import type { ConnectionRecord } from "../src/shared/storage";
import {
  type ChromeFake,
  type FakePort,
  installChromeFake,
} from "./support/chrome-fake";
import { type FakeGateway, startFakeGateway } from "./support/fake-gateway";
import { fakeMicrophone, fakeSpeech } from "./support/fake-speech";

const EXTENSION_ID = "test";
const POPUP = { url: `chrome-extension://${EXTENSION_ID}/popup.html` };
const ada = { id: "p1", name: "Ada", email: "[email protected]" };
const ORDER = "http://127.0.0.1:7420/orders/1043";
const DESKTOP_TOKEN = "obr_desktop-connection";
const LAUNCHER_TOKEN = "obr_openbot-browser";
const SIGNED_OUT =
  "This browser was disconnected from OpenBot. Your recording stopped, and everything in it is kept in this browser for 7 days, until you connect again.";

type Globals = { indexedDB?: IDBFactory; IDBKeyRange?: unknown };
const globals = globalThis as unknown as Globals;
let saved: Globals = {};

const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
const settle = () => sleep(20);
async function until(check: () => boolean | Promise<boolean>, label: string) {
  const deadline = Date.now() + 8_000;
  while (!(await check())) {
    if (Date.now() > deadline) throw new Error(`Timed out waiting: ${label}`);
    await sleep(5);
  }
}

// ---- The fake OpenBot -------------------------------------------------------------------------

let gateway: FakeGateway;
let liveTokens = new Set<string>();
/** Every report OpenBot's activity route received, with the bearer it came with. */
let reports: { token: string; body: RecorderActivity }[] = [];
/** How OpenBot's activity route answers: as it should (204), signed out (401), refusing the report (400, or 415 for a
 * body that is not JSON), broken, or never. */
let activityAnswer:
  | "ok"
  | "signed-out"
  | "bad-report"
  | "not-json"
  | "error"
  | "hang" = "ok";

const signedOut = () =>
  Response.json(
    {
      error: "This browser is no longer connected to OpenBot.",
      code: RECORDER_SIGNED_OUT,
      reason: "revoked",
    },
    { status: 401 },
  );

async function activity(request: Request, token: string): Promise<Response> {
  // OpenBot's route takes only what the contract types: anything else would be a 400 there.
  reports.push({
    token,
    body: recorderActivitySchema.parse(await request.json()),
  });
  if (!liveTokens.has(token) || activityAnswer === "signed-out")
    return signedOut();
  if (activityAnswer === "bad-report")
    return Response.json(
      { error: "That is not a Recorder report.", code: "INVALID_REQUEST" },
      { status: 400 },
    );
  if (activityAnswer === "not-json")
    return Response.json(
      { error: "Send the report as JSON.", code: "UNSUPPORTED_MEDIA_TYPE" },
      { status: 415 },
    );
  if (activityAnswer === "error")
    return Response.json({ error: "OpenBot broke." }, { status: 500 });
  if (activityAnswer === "hang") await new Promise(() => {});
  return new Response(null, { status: 204 });
}

function route(request: Request): Response | Promise<Response> | null {
  const path = new URL(request.url).pathname;
  const token =
    request.headers.get("authorization")?.replace(/^Bearer /, "") ?? "";
  if (path === recorderPaths.activity && request.method === "POST")
    return activity(request, token);
  if (path === recorderPaths.me)
    return liveTokens.has(token)
      ? Response.json({ user: ada, connectionId: `c-${token}` })
      : signedOut();
  if (path.startsWith("/api/recorder/"))
    return Response.json({ error: "Not found." }, { status: 404 });
  const pass = /^\/api\/copilotkit\/trajectory\/([^/]+)\/connect$/.exec(path);
  if (!pass || request.method !== "POST") return null;
  const trajectoryId = decodeURIComponent(pass[1] ?? "");
  return Response.json({
    joinToken: gateway.mintToken(trajectoryId),
    realtime: {
      clientUrl: `${gateway.url}/client`,
      topic: `trajectory:${trajectoryId}`,
    },
  });
}

// ---- Time, and the two contexts ---------------------------------------------------------------

let clock = 0;
type Due = { id: number; at: number; fn: () => void };
let due: Due[] = [];
let nextTimer = 0;
/** The service worker's timers, on the test's clock: only `advanceTo` fires them. */
const workerTimers = {
  set(fn: () => void, ms: number) {
    nextTimer += 1;
    due.push({ id: nextTimer, at: clock + Math.max(0, ms), fn });
    return nextTimer;
  },
  clear(handle: unknown) {
    due = due.filter((timer) => timer.id !== handle);
  },
};
async function advanceTo(to: number) {
  for (;;) {
    const next = due
      .filter((timer) => timer.at <= to)
      .sort((a, b) => a.at - b.at)[0];
    if (!next) break;
    due = due.filter((timer) => timer !== next);
    clock = Math.max(clock, next.at);
    next.fn();
    await settle();
  }
  clock = Math.max(clock, to);
  await settle();
}

let fake: ChromeFake;
let databaseName = "";
let databaseCounter = 0;
let documentTimers: ReturnType<typeof setTimeout>[] = [];

function runDocument(): void {
  fake.runAs("offscreen-document", () =>
    createOffscreenController({
      openOutbox: () => openOutbox(databaseName),
      connectPort: () => chrome.runtime.connect({ name: PORT_OFFSCREEN }),
      now: () => clock,
      timers: {
        set: (fn, ms) => {
          const handle = setTimeout(fn, ms);
          documentTimers.push(handle);
          return handle;
        },
        clear: (handle) =>
          clearTimeout(handle as ReturnType<typeof setTimeout>),
      },
      voice: {
        Recognition: fakeSpeech().Recognition,
        getUserMedia: fakeMicrophone().getUserMedia,
      },
    }),
  );
}

/** The service worker as background/index.ts starts it, on the test's clock and timers. */
function startWorker(): void {
  const deps: Partial<RecordingDeps> = {
    createNetworkCapture: () => ({
      start: async () => "debugger",
      addTab: async () => {},
      onCommitted: async () => {},
      resume: async () => "debugger",
      recordPageEvent: async () => {},
      recordPageAgent: () => {},
      stop: async () => {},
    }),
    createPageFallback: () => ({
      start: async () => false,
      addTab: async () => {},
      onCommitted: async () => {},
      stop: async () => {},
      active: () => false,
    }),
    acceptPageNetworkEvent: () => null,
    hasHostPermission: async () => true,
    now: () => clock,
    timers: workerTimers,
    stepTimers: workerTimers,
  };
  const recording = installRecording(deps);
  installRouter(recording);
  installConnection({
    connectionLost: recording.connectionLost,
    connected: recording.connected,
  });
  installSetup();
  installLauncher(recording, { now: () => clock });
}

const launcher = () =>
  (globalThis as unknown as Record<string, LauncherApi>)[
    LAUNCHER_GLOBAL
  ] as LauncherApi;

/** A connection OpenBot Desktop made for the person's own browser, as the popup records with it. */
function connectedThroughDesktop(): void {
  liveTokens.add(DESKTOP_TOKEN);
  fake.storage.local.set("connection", {
    openbotUrl: gateway.httpUrl,
    token: DESKTOP_TOKEN,
    connectionId: `c-${DESKTOP_TOKEN}`,
    user: ada,
    via: "desktop",
    connectedAt: clock,
  } satisfies ConnectionRecord);
}

function openOrder(): void {
  fake.tabs.set(7, {
    id: 7,
    url: ORDER,
    title: "Order 1043 - Corner Shop admin",
    active: true,
    windowId: 1,
  } as chrome.tabs.Tab);
}

const page = (message: object) => fake.sendFromPage(message, POPUP);
const state = async () =>
  ((await page({ type: "page.state" })) as { state: RecorderState }).state;

/** Record from the popup, as the person presses it. */
async function recordFromPopup(title: string): Promise<void> {
  openOrder();
  expect(
    await page({ type: "page.record", title, tabId: 7, language: "en-GB" }),
  ).toEqual({ ok: true });
}

function contentPort(): FakePort {
  return fake.connectFrom(PORT_CONTENT, {
    id: EXTENSION_ID,
    tab: { id: 7 } as chrome.tabs.Tab,
    frameId: 0,
    url: ORDER,
  });
}

const click = (at: number): CapturedEvent => ({
  type: "CUSTOM",
  name: "click",
  timestamp: at,
  value: { tag: "button", text: "Archive", url: ORDER, frame: 0 },
});

const phases = () => reports.map((report) => report.body.phase);
const last = () => reports.at(-1)?.body;

beforeEach(async () => {
  saved = { indexedDB: globals.indexedDB, IDBKeyRange: globals.IDBKeyRange };
  globals.indexedDB = new IDBFactory();
  globals.IDBKeyRange = FakeIDBKeyRange;
  databaseName = `activity-${Date.now()}-${databaseCounter++}`;
  due = [];
  documentTimers = [];
  liveTokens = new Set([LAUNCHER_TOKEN]);
  reports = [];
  activityAnswer = "ok";
  gateway = await startFakeGateway({ extraRoutes: route });
  fake = installChromeFake({
    extensionId: EXTENSION_ID,
    manifestVersion: "0.3.0",
  });
  fake.offscreen.onDocumentCreated.addListener(() => {
    setTimeout(runDocument, 0);
  });
  clock = 1_760_000_000_000;
  startWorker();
  await settle();
}, 15_000);

afterEach(async () => {
  await settle();
  for (const handle of documentTimers) clearTimeout(handle);
  await gateway.stop();
  fake.uninstall();
  Reflect.deleteProperty(globalThis, LAUNCHER_GLOBAL);
  for (const key of ["indexedDB", "IDBKeyRange"] as const) {
    if (saved[key] === undefined) Reflect.deleteProperty(globals, key);
    else Object.assign(globals, { [key]: saved[key] });
  }
}, 15_000);

test("a recording made from the popup tells OpenBot what it records: each phase, every 30 s with the actions so far, then idle", async () => {
  connectedThroughDesktop();
  // Connected and idle, nothing changes: nothing is reported.
  await advanceTo(clock + 3 * ACTIVITY_REPORT_MS);
  expect(reports).toEqual([]);

  const startedAt = clock;
  await recordFromPopup("Archive order 1043");
  await until(() => last()?.phase === "recording", "the recording reported");
  const { recordingId } = await state();
  expect(last()).toEqual({
    phase: "recording",
    recordingId,
    title: "Archive order 1043",
    startedAt,
    actions: 0,
  });
  // Starting comes first when OpenBot hears it before the recording is live; either way it is never heard after.
  expect(phases().filter((phase) => phase !== "starting")[0]).toBe("recording");

  // The person archives the order; the next report, 30 s on, carries it.
  const port = contentPort();
  port.postMessage({ type: "content.event", event: click(clock + 1_000) });
  await until(async () => (await state()).actions === 1, "the click counted");
  const before = reports.length;
  await advanceTo(startedAt + ACTIVITY_REPORT_MS);
  await until(() => reports.length > before, "the 30 s report");
  expect(last()).toEqual({
    phase: "recording",
    recordingId,
    title: "Archive order 1043",
    startedAt,
    actions: 1,
  });
  await advanceTo(startedAt + 2 * ACTIVITY_REPORT_MS);
  await until(() => reports.length > before + 1, "the 60 s report");

  expect(await page({ type: "page.stop" })).toEqual({ ok: true });
  await until(() => last()?.phase === "idle", "idle reported");
  expect(last()).toEqual({ phase: "idle", actions: 0 });
  // Every report went with the connection's own bearer, and stopped when the recording did.
  expect(new Set(reports.map((report) => report.token))).toEqual(
    new Set([DESKTOP_TOKEN]),
  );
  const after = reports.length;
  await advanceTo(clock + 3 * ACTIVITY_REPORT_MS);
  expect(reports).toHaveLength(after);
}, 20_000);

test("an OpenBot browser's connection reports too, with the bearer its launcher handed in, and Start over reports the new recording", async () => {
  expect(
    await launcher().connect({
      connection: {
        openbotUrl: gateway.httpUrl,
        token: LAUNCHER_TOKEN,
        connectionId: `c-${LAUNCHER_TOKEN}`,
        user: ada,
      },
      narration: "external",
    }),
  ).toEqual({ ok: true });
  openOrder();
  const started = await launcher().record({
    action: "start",
    title: "Refund an order",
  });
  expect(started.ok).toBe(true);
  await until(() => last()?.phase === "recording", "the recording reported");
  const first = last()?.recordingId;

  expect(await page({ type: "page.restart" })).toEqual({ ok: true });
  await until(
    () => last()?.phase === "recording" && last()?.recordingId !== first,
    "Start over's recording reported",
  );
  expect(last()).toMatchObject({ title: "Refund an order", actions: 0 });

  expect(await launcher().record({ action: "stop" })).toMatchObject({
    ok: true,
  });
  await until(() => last()?.phase === "idle", "idle reported");
  expect(new Set(reports.map((report) => report.token))).toEqual(
    new Set([LAUNCHER_TOKEN]),
  );
}, 20_000);

test("OpenBot answering a report signed out ends the recording as any sign-out does, everything kept for its person", async () => {
  connectedThroughDesktop();
  await recordFromPopup("Archive order 1043");
  await until(() => last()?.phase === "recording", "the recording reported");

  // OpenBot revoked this browser: the next report hears it (the connection check, 30 s apart too, still says ok).
  activityAnswer = "signed-out";
  await advanceTo(clock + ACTIVITY_REPORT_MS);
  await until(async () => (await state()).phase === "idle", "the stop");
  expect(await state()).toMatchObject({ phase: "idle", error: SIGNED_OUT });
  expect(fake.storage.local.get("connection")).toBeUndefined();
  expect(fake.storage.local.get("notice")).toMatchObject({
    kind: "signed-out",
    reason: "revoked",
  });
  expect(
    (fake.storage.local.get("drains") as { kept?: object }[])[0]?.kept,
  ).toMatchObject({ waitingForReconnect: true });
}, 20_000);

test("a report OpenBot cannot take, or never answers, holds up and fails nothing: Record, the 30 s reports and Stop go on", async () => {
  connectedThroughDesktop();
  activityAnswer = "error";
  await recordFromPopup("Archive order 1043");
  await until(() => reports.length > 0, "a report sent");
  expect((await state()).phase).toBe("recording");

  activityAnswer = "hang";
  const before = reports.length;
  await advanceTo(clock + ACTIVITY_REPORT_MS);
  await until(() => reports.length > before, "the 30 s report sent");
  // Stop answers at once though OpenBot is still holding the report.
  const stopping = page({ type: "page.stop" });
  expect(await stopping).toEqual({ ok: true });
  expect((await state()).phase).toBe("idle");
  expect(fake.storage.local.get("connection")).toMatchObject({
    token: DESKTOP_TOKEN,
  });
}, 20_000);

test("a report OpenBot refuses as malformed (400, 415) is said in the worker's console and stops nothing: the next one still goes", async () => {
  connectedThroughDesktop();
  const said: unknown[][] = [];
  const spy = spyOn(console, "error").mockImplementation((...args) => {
    said.push(args);
  });
  try {
    activityAnswer = "bad-report";
    await recordFromPopup("Archive order 1043");
    await until(() => reports.length > 0, "a report sent");
    await until(
      () =>
        said.some(
          (args) => args[0] === "Recorder: OpenBot refused its activity report",
        ),
      "the refusal said",
    );
    activityAnswer = "not-json";
    const before = reports.length;
    await advanceTo(clock + ACTIVITY_REPORT_MS);
    await until(() => reports.length > before, "the 30 s report sent");
    await until(
      () =>
        said.filter(
          (args) => args[0] === "Recorder: OpenBot refused its activity report",
        ).length >= 2,
      "the second refusal said",
    );
  } finally {
    spy.mockRestore();
  }
  expect((await state()).phase).toBe("recording");
  expect(fake.storage.local.get("connection")).toMatchObject({
    token: DESKTOP_TOKEN,
  });
  expect(fake.storage.local.get("notice")).toBeUndefined();

  activityAnswer = "ok";
  expect(await page({ type: "page.stop" })).toEqual({ ok: true });
  await until(() => last()?.phase === "idle", "idle reported");
}, 20_000);

test("a service worker that restarts while a recording runs reports it again, and goes on every 30 s", async () => {
  connectedThroughDesktop();
  await recordFromPopup("Archive order 1043");
  await until(() => last()?.phase === "recording", "the recording reported");
  const { recordingId } = await state();

  // Chrome stops the worker and wakes it again: its timers and memory go, storage stays.
  fake.restartServiceWorker();
  due = [];
  const before = reports.length;
  startWorker();
  await until(() => reports.length > before, "the restarted worker's report");
  expect(last()).toMatchObject({ phase: "recording", recordingId });
  await advanceTo(clock + ACTIVITY_REPORT_MS);
  await until(() => reports.length > before + 1, "its 30 s report");
  expect(last()).toMatchObject({ phase: "recording", recordingId });
}, 20_000);