Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

19.0 KB
Raw Blame
// The page hook's side of AG-UI, when Chrome refuses the debugger: the same traffic real Chrome recorded (tests/fixtures/
// agui), served again byte for byte, chunk by chunk, by a local server to the page's own fetch and WebSocket, must give
// the agent events the same observations the debugger capture gives from the CDP of that traffic, and the page must get
// its responses and sockets untouched.
import {
  afterAll,
  afterEach,
  beforeAll,
  beforeEach,
  expect,
  test,
} from "bun:test";
import { GlobalRegistrator } from "@happy-dom/global-registrator";
import type { CaptureObservation } from "../src/background/agent/observation";
import { createNetworkCapture } from "../src/background/network";
import type { PageObservation } from "../src/shared/agui";
import {
  type Fixture,
  loadFixture,
  marks,
  replay,
  responseChunks,
} from "./support/cdp-replay";
import { installChromeFake } from "./support/chrome-fake";

// Bun.serve, fetch and WebSocket are the runtime's own: happy-dom's registration replaces them on globalThis.
const NativeResponse = Response;
const nativeFetch = fetch;
const NativeWebSocket = WebSocket;
const NativeAbortController = AbortController;

type Json = Record<string, unknown>;
type Sent = {
  id: string;
  message: { type: string; observation?: PageObservation; event?: Json };
};

/** What one recorded fixture asks the server to answer: each request by method, path and body, and each socket by path. */
type HttpAnswer = {
  status: number;
  contentType: string;
  chunks: Uint8Array[];
  /** The page stopped this one: its answer never ends. */
  aborted: boolean;
};
type SocketScript = {
  /** In recorded order: what the page sent, and what the socket received. */
  frames: { sent: boolean; data: string }[];
  closed: boolean;
};

let server: ReturnType<typeof Bun.serve<{ path: string }, never>>;
let answers = new Map<string, HttpAnswer[]>();
let scripts = new Map<string, SocketScript[]>();
let sent: Sent[] = [];

const requestKey = (method: string, path: string, body: string) =>
  `${method} ${path} ${body}`;

beforeAll(() => {
  server = Bun.serve<{ path: string }, never>({
    hostname: "127.0.0.1",
    port: 0,
    async fetch(request, srv) {
      const url = new URL(request.url);
      const path = `${url.pathname}${url.search}`;
      if (srv.upgrade(request, { data: { path } })) return;
      const body = request.method === "GET" ? "" : await request.text();
      const answer = answers
        .get(requestKey(request.method, path, body))
        ?.shift();
      if (!answer) return new NativeResponse("not recorded", { status: 404 });
      const stream = new ReadableStream<Uint8Array>({
        async start(controller) {
          for (const chunk of answer.chunks) {
            controller.enqueue(chunk);
            await Bun.sleep(1);
          }
          if (!answer.aborted) controller.close();
        },
      });
      return new NativeResponse(stream, {
        status: answer.status,
        headers: { "content-type": answer.contentType },
      });
    },
    websocket: {
      open(ws) {
        const script = scripts.get(ws.data.path)?.[0];
        if (!script) return;
        // What the socket received before the page sent anything.
        while (script.frames[0] && !script.frames[0].sent)
          ws.send(script.frames.shift()?.data ?? "");
      },
      message(ws) {
        const script = scripts.get(ws.data.path)?.[0];
        if (!script) return;
        script.frames.shift();
        while (script.frames[0] && !script.frames[0].sent)
          ws.send(script.frames.shift()?.data ?? "");
        if (script.frames.length === 0) {
          scripts.get(ws.data.path)?.shift();
          if (script.closed) ws.close(1000, "recorded");
        }
      },
    },
  });
  GlobalRegistrator.register({ url: `http://127.0.0.1:${server.port}/page` });
});
afterAll(async () => {
  await GlobalRegistrator.unregister();
  await server.stop(true);
});

const hookKey = (name: "configure" | "stop") =>
  Symbol.for(`openbot.recorder.${name}`);
const globals = globalThis as unknown as Record<symbol, unknown>;
const configure = (id: string) =>
  (globals[hookKey("configure")] as (id: string) => void)(id);
const stop = () => (globals[hookKey("stop")] as () => void)();

beforeEach(async () => {
  sent = [];
  answers = new Map();
  scripts = new Map();
  globalThis.fetch = nativeFetch;
  globalThis.WebSocket = NativeWebSocket;
  (globalThis as unknown as { chrome: unknown }).chrome = {
    runtime: {
      sendMessage: (id: string, message: Sent["message"]) => {
        sent.push({ id, message });
        return Promise.resolve();
      },
    },
  };
  await import(`../src/main-world/network-hook.ts?run=${crypto.randomUUID()}`);
});
afterEach(() => {
  stop();
  globalThis.fetch = nativeFetch;
  globalThis.WebSocket = NativeWebSocket;
  delete (globalThis as { chrome?: unknown }).chrome;
  delete globals[hookKey("configure")];
  delete globals[hookKey("stop")];
});

const pathOf = (url: string) => {
  const parsed = new URL(url);
  return `${parsed.pathname}${parsed.search}`;
};

/** The OpenBot a recording of OpenBot's own page is connected to: the page's origin. The page hook knows nothing of
 * OpenBot, and the debugger capture follows only OpenBot's chat and its runtime's `info`, so the two still agree. */
const openbotOf = (fixture: Fixture): string | null =>
  fixture === loadFixture("openbot-chat")
    ? new URL(String(marks(fixture, "navigate")[0]?.data?.url)).origin
    : null;

/** The observations the debugger capture makes from the fixture's CDP. */
async function debuggerObservations(
  fixture: Fixture,
): Promise<CaptureObservation[]> {
  const page = (globalThis as { chrome?: unknown }).chrome;
  const fake = installChromeFake({ extensionId: "test" });
  const observations: CaptureObservation[] = [];
  const capture = createNetworkCapture(
    {
      openbotUrl: () => openbotOf(fixture),
      call: () => {},
      writeStarted: () => {},
      writeEnded: () => {},
      status: () => {},
      digests: () => new Set(),
      agent: (observation) => observations.push(observation),
    },
    {},
  );
  try {
    await capture.start(7, String(marks(fixture, "navigate")[0]?.data?.url));
    await replay(fake, 7, fixture).run();
  } finally {
    await capture.stop();
    fake.uninstall();
    (globalThis as { chrome?: unknown }).chrome = page;
  }
  return observations;
}

type Recorded = {
  kind: "fetch" | "socket";
  id: string;
  url: string;
  method: string;
  body?: string;
  headers: Record<string, string>;
  /** A fetch's recorded answer. */
  answer?: HttpAnswer;
};

/** What the page did, in order: each fetch it made (the page's own requests, never a script or a document) and each
 * socket it opened, with what the server is to answer. */
function plan(fixture: Fixture): Recorded[] {
  const order: Recorded[] = [];
  const failed = new Set<string>();
  const status = new Map<string, { status: number; type: string }>();
  for (const entry of fixture.entries) {
    if (entry.kind !== "event") continue;
    const params = entry.params as Json & { requestId: string };
    if (entry.method === "Network.loadingFailed") failed.add(params.requestId);
    if (entry.method === "Network.responseReceived") {
      const response = params.response as { status: number; mimeType: string };
      status.set(params.requestId, {
        status: response.status,
        type: response.mimeType,
      });
    }
  }
  const postData = (id: string) => {
    for (const entry of fixture.entries)
      if (
        entry.kind === "command" &&
        entry.method === "Network.getRequestPostData" &&
        entry.params.requestId === id
      )
        return entry.result?.postData as string | undefined;
    return undefined;
  };
  for (const entry of fixture.entries) {
    if (entry.kind !== "event") continue;
    const params = entry.params as Json & { requestId: string };
    if (
      entry.method === "Network.requestWillBeSent" &&
      params.type === "Fetch"
    ) {
      const request = params.request as {
        url: string;
        method: string;
        headers: Record<string, string>;
        postData?: string;
      };
      const body =
        request.method === "GET"
          ? undefined
          : (request.postData ?? postData(params.requestId) ?? "");
      const answer = status.get(params.requestId);
      if (!answer) continue;
      const key = requestKey(request.method, pathOf(request.url), body ?? "");
      const recorded: HttpAnswer = {
        status: answer.status,
        contentType: answer.type,
        chunks: responseChunks(fixture, params.requestId).map(
          ({ bytes }) => bytes,
        ),
        aborted: failed.has(params.requestId),
      };
      answers.set(key, [...(answers.get(key) ?? []), recorded]);
      order.push({
        answer: recorded,
        kind: "fetch",
        id: params.requestId,
        url: request.url,
        method: request.method,
        ...(body !== undefined ? { body } : {}),
        headers: Object.fromEntries(
          Object.entries(request.headers).filter(([name]) =>
            /^(content-type|accept)$/i.test(name),
          ),
        ),
      });
    }
    if (entry.method === "Network.webSocketCreated") {
      const url = String(params.url);
      const frames: SocketScript["frames"] = [];
      let closed = false;
      for (const other of fixture.entries) {
        if (
          other.kind !== "event" ||
          other.params.requestId !== params.requestId
        )
          continue;
        if (
          other.method === "Network.webSocketFrameSent" ||
          other.method === "Network.webSocketFrameReceived"
        )
          frames.push({
            sent: other.method === "Network.webSocketFrameSent",
            data: (other.params.response as { payloadData: string })
              .payloadData,
          });
        if (other.method === "Network.webSocketClosed") closed = true;
      }
      const path = pathOf(url.replace(/^ws/, "http"));
      scripts.set(path, [...(scripts.get(path) ?? []), { frames, closed }]);
      order.push({
        kind: "socket",
        id: params.requestId,
        url,
        method: "GET",
        headers: {},
      });
    }
  }
  return order;
}

/** Plays the page's side against the server: each fetch read to its end (one the page stopped, aborted once its
 * recorded bytes came), each socket sending what the page sent once the socket had received what came before it. */
async function playPage(steps: Recorded[]) {
  const scriptsBefore = new Map(
    [...scripts].map(([path, list]) => [
      path,
      list.map((script) => [...script.frames]),
    ]),
  );
  const opened: WebSocket[] = [];
  const pageGot = new Map<string, Uint8Array>();
  for (const step of steps) {
    if (step.kind === "fetch") {
      const controller = new NativeAbortController();
      const path = pathOf(step.url);
      const answer = step.answer;
      const response = await fetch(`http://127.0.0.1:${server.port}${path}`, {
        method: step.method,
        headers: step.headers,
        ...(step.body !== undefined ? { body: step.body } : {}),
        signal: controller.signal,
      });
      const reader = response.body?.getReader();
      const parts: Uint8Array[] = [];
      const expected =
        answer?.chunks.reduce((sum, chunk) => sum + chunk.length, 0) ?? 0;
      let received = 0;
      for (;;) {
        if (answer?.aborted && received >= expected) {
          controller.abort();
          break;
        }
        const read = await reader?.read().catch(() => undefined);
        if (!read?.value) break;
        parts.push(read.value);
        received += read.value.length;
      }
      const whole = new Uint8Array(received);
      let offset = 0;
      for (const part of parts) {
        whole.set(part, offset);
        offset += part.length;
      }
      pageGot.set(step.id, whole);
      continue;
    }
    const path = pathOf(step.url.replace(/^ws/, "http"));
    const frames = scriptsBefore.get(path)?.shift() ?? [];
    const socket = new WebSocket(`ws://127.0.0.1:${server.port}${path}`);
    opened.push(socket);
    let received = 0;
    const done = new Promise<void>((resolve) => {
      let index = 0;
      const pump = () => {
        // Send what the page sent next, once everything recorded before it has arrived.
        while (index < frames.length) {
          const frame = frames[index];
          if (!frame) break;
          if (!frame.sent) {
            const before = frames
              .slice(0, index + 1)
              .filter((f) => !f.sent).length;
            if (received < before) return;
            index++;
            continue;
          }
          socket.send(frame.data);
          index++;
        }
        resolve();
      };
      socket.addEventListener("open", pump);
      socket.addEventListener("message", () => {
        received++;
        pump();
      });
      socket.addEventListener("close", () => resolve());
    });
    await Promise.race([done, Bun.sleep(3_000)]);
  }
  await Bun.sleep(50);
  for (const socket of opened)
    if (socket.readyState === NativeWebSocket.OPEN) socket.close();
  await Bun.sleep(50);
  return pageGot;
}

/** An observation as both captures should make it, without what differs by design: the tab and stream ids (each
 * capture names its own), when it saw it, and the server's address. The runtime's `info` answer is a stream of its
 * own. */
function signature(observations: (CaptureObservation | PageObservation)[]) {
  const streams = new Map<string, unknown[]>();
  for (const observation of observations) {
    const {
      streamId = "runtime",
      at: _at,
      ...rest
    } = observation as Json & { streamId?: string; at: number };
    const { tabId: _tab, ...kept } = rest as Json;
    const shown =
      kept.kind === "request" || kept.kind === "runtime"
        ? { ...kept, url: pathOf(String(kept.url)) }
        : kept;
    streams.set(streamId, [...(streams.get(streamId) ?? []), shown]);
  }
  return [...streams.values()].map((stream) => JSON.stringify(stream)).sort();
}

const pageObservations = () =>
  sent.flatMap(({ message }) =>
    message.type === "recorder.page-agent" && message.observation
      ? [message.observation]
      : [],
  );

/** The fixtures of a CopilotKit page: its runtime's `info` call is in their traffic (single route, REST, Intelligence). */
const COPILOTKIT_PAGES = new Map([
  ["single-route-sse", 1],
  ["rest-sse", 1],
  ["intelligence-websocket", 1],
  // OpenBot's own chat asks when it loads and again after the run.
  ["openbot-chat", 2],
]);

for (const name of [
  "single-route-sse",
  "rest-sse",
  "intelligence-websocket",
  "generic-proto-ws-ticker",
  "late-and-aborted",
  "openbot-chat",
])
  test(`the page hook gives the same observations as the debugger for the same traffic: ${name}`, async () => {
    const fixture = loadFixture(name);
    const expected = await debuggerObservations(fixture);
    const steps = plan(fixture);
    configure("extension-id");
    const recorded = new Map(
      steps
        .filter((step) => step.kind === "fetch")
        .map((step) => [step.id, step.answer] as const),
    );
    const pageGot = await playPage(steps);
    const observed = pageObservations();
    expect(observed.length).toBeGreaterThan(0);
    expect(signature(observed)).toEqual(signature(expected));
    for (const observation of observed) {
      expect(observation.at).toBeGreaterThan(0);
      if (observation.kind !== "runtime")
        expect(typeof observation.streamId).toBe("string");
    }
    // A CopilotKit page asked its runtime what it serves and the runtime answered: both captures say the page runs
    // CopilotKit, once; the other servers' pages never do.
    const runtimeAnswers = (list: (CaptureObservation | PageObservation)[]) =>
      list.filter((observation) => observation.kind === "runtime").length;
    expect(runtimeAnswers(expected)).toBe(COPILOTKIT_PAGES.get(name) ?? 0);
    expect(runtimeAnswers(observed)).toBe(runtimeAnswers(expected));
    // The page got every response untouched, byte for byte, read through to its end while the hook read its copy.
    for (const [id, answer] of recorded) {
      const bytes = answer?.chunks.reduce(
        (sum, chunk) => sum + chunk.length,
        0,
      );
      expect(pageGot.get(id)?.length).toBe(bytes);
    }
    // An agent run is never a network event, there as in the debugger's capture.
    for (const { message } of sent)
      if (message.type === "recorder.page-network")
        expect(JSON.stringify(message.event)).not.toContain("agent/run");
  });

test("the page's WebSocket stays the browser's own: instanceof, prototype, constants and subclassing, and stop puts it back", async () => {
  configure("extension-id");
  expect(globalThis.WebSocket).not.toBe(NativeWebSocket);
  const socket = new WebSocket(`ws://127.0.0.1:${server.port}/nothing`);
  expect(socket instanceof WebSocket).toBe(true);
  expect(socket instanceof NativeWebSocket).toBe(true);
  expect(Object.getPrototypeOf(socket)).toBe(NativeWebSocket.prototype);
  expect(WebSocket.prototype).toBe(NativeWebSocket.prototype);
  expect([
    WebSocket.CONNECTING,
    WebSocket.OPEN,
    WebSocket.CLOSING,
    WebSocket.CLOSED,
  ]).toEqual([0, 1, 2, 3]);
  class Named extends WebSocket {
    name = "mine";
  }
  const named = new Named(`ws://127.0.0.1:${server.port}/nothing`);
  expect(named).toBeInstanceOf(Named);
  expect(named.name).toBe("mine");
  socket.close();
  named.close();
  stop();
  expect(globalThis.WebSocket).toBe(NativeWebSocket);
});

test("an event stream that is not AG-UI gives the agent events nothing, even answering a run-shaped POST, which stays a network event; the page reads it whole", async () => {
  const fixture = loadFixture("generic-proto-ws-ticker");
  const ticker = fixture.entries.find(
    (entry) =>
      entry.kind === "event" &&
      entry.method === "Network.requestWillBeSent" &&
      entry.params.type === "EventSource",
  );
  const chunks = responseChunks(
    fixture,
    String(ticker?.kind === "event" ? ticker.params.requestId : ""),
  ).map(({ bytes }) => bytes);
  const length = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
  expect(chunks.length).toBeGreaterThan(1);
  const body = JSON.stringify({
    threadId: "t-1",
    runId: "r-1",
    messages: [{ id: "m-1", role: "user", content: "prices?" }],
  });
  const answer = (): HttpAnswer => ({
    status: 200,
    contentType: "text/event-stream",
    chunks,
    aborted: false,
  });
  answers.set(requestKey("GET", "/prices", ""), [answer()]);
  answers.set(requestKey("POST", "/prices", body), [answer()]);
  configure("extension-id");
  const read = async (response: Response) =>
    new Uint8Array(await response.arrayBuffer()).length;
  expect(
    await read(await fetch(`http://127.0.0.1:${server.port}/prices`)),
  ).toBe(length);
  expect(
    await read(
      await fetch(`http://127.0.0.1:${server.port}/prices`, {
        method: "POST",
        headers: { "content-type": "application/json" },
        body,
      }),
    ),
  ).toBe(length);
  await Bun.sleep(100);
  expect(pageObservations()).toEqual([]);
  expect(
    sent.filter(({ message }) => message.type === "recorder.page-network"),
  ).toHaveLength(2);
});