Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

11.1 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import type { MiddlewareHandler } from "hono";
import type { AppVariables, AuthenticatedActor } from "../src/auth/guards";
import { personSeat } from "../src/computer/computer-key";
import type { ComputerGateway } from "../src/computer/gateway";
import type { PageFrameStore } from "../src/computer/page-frames";
import type { PolicyStore } from "../src/computer/policy-store";
import { createComputerRoutes } from "../src/computer/routes";

/**
 * The frame is taken where the navigation happens.
 *
 * The surface used to capture it after the turn and file it under the tool call, which lost a race it
 * could not win: the same computer is driven by other conversations between the turn ending and the
 * tile asking, and a resumed computer starts blank. So the transcript showed the wrong page, or none.
 *
 * What these cover is the seam that replaced it: navigating photographs the page it just opened, and
 * a screenshot that cannot be taken never fails the navigation the Bot was actually asked to do.
 */

const actor: AuthenticatedActor = {
  id: "user-1",
  email: "[email protected]",
  role: "user",
};

/** Who is asking, which a test can change to be somebody else on the same Bot. */
let asking: AuthenticatedActor = actor;

const asActor: MiddlewareHandler<{ Variables: AppVariables }> = async (
  context,
  next,
) => {
  context.set("actor", asking);
  await next();
};

function harness(options?: {
  screenshot?: () => Promise<{ base64: string; url?: string }>;
  navigate?: () => Promise<{ url: string; title: string }>;
  status?: (botId: string) => Promise<{ botId: string; state: string }>;
  isolation?: "per-person" | "shared";
}) {
  const saved: Array<{
    computerId: string;
    toolCallId: string;
    url: string;
    frame: string;
  }> = [];
  const gateway = {
    seatFor: async (botId: string, personId: string) =>
      personSeat(personId, botId),
    provider: { isolation: options?.isolation ?? "per-person" },
    navigate:
      options?.navigate ??
      (async () => ({
        url: "https://example.com/story",
        title: "A story",
        text: "",
        truncated: false,
        elapsedMs: 1,
      })),
    status:
      options?.status ??
      (async (botId: string) => ({ botId, state: "ready" as const })),
    screenshot:
      options?.screenshot ??
      (async () => ({ base64: "PNGBYTES", url: "https://example.com/story" })),
  } as unknown as ComputerGateway;

  const pageFrames: PageFrameStore = {
    async save(frame) {
      saved.push({
        computerId: frame.computerId,
        toolCallId: frame.toolCallId,
        url: frame.url,
        frame: frame.frame,
      });
    },
    async load(computerId, toolCallId) {
      const found = saved.find(
        (row) => row.computerId === computerId && row.toolCallId === toolCallId,
      );
      return found ? { url: found.url, title: null, frame: found.frame } : null;
    },
    async clear() {
      return saved.splice(0).length;
    },
    async purge() {
      return 0;
    },
  };

  const routes = createComputerRoutes(
    gateway,
    {} as PolicyStore,
    asActor,
    async () => true,
    pageFrames,
  );
  return { routes, saved };
}

function navigate(
  routes: ReturnType<typeof harness>["routes"],
  url: string,
  // `null` means "send no turn at all". `undefined` would take the default, which is the opposite.
  toolCallId: string | null = "call-1",
) {
  return routes.request("http://openbot.test/bot-9/navigate", {
    method: "POST",
    headers: { "content-type": "application/json" },
    body: JSON.stringify({ url, ...(toolCallId ? { toolCallId } : {}) }),
  });
}

describe("the frame a page was opened on", () => {
  test("navigating keeps a frame of the page it landed on", async () => {
    const { routes, saved } = harness();

    expect((await navigate(routes, "https://example.com/story")).status).toBe(
      200,
    );

    // Filed under the person's seat on the Bot, not the Bot alone.
    expect(saved).toEqual([
      {
        computerId: "person:user-1/bot-9",
        toolCallId: "call-1",
        url: "https://example.com/story",
        frame: "PNGBYTES",
      },
    ]);
  });

  /*
   * The address the browser ended on, not the one that was asked for. A redirect, a canonical host or
   * an added trailing slash all mean the two differ, and the transcript asks by the page the tool
   * reported, so filing under the request would file under a key nothing ever looks up.
   */
  test("the frame is filed under the page the browser reached", async () => {
    const { routes, saved } = harness({
      navigate: async () => ({
        url: "https://www.example.com/story/",
        title: "A story",
      }),
      screenshot: async () => ({
        base64: "PNGBYTES",
        url: "https://www.example.com/story/",
      }),
    });

    await navigate(routes, "https://example.com/story");

    expect(saved[0]?.url).toBe("https://www.example.com/story/");
  });

  test("the kept frame is read back by the page it was taken of", async () => {
    const { routes } = harness();
    await navigate(routes, "https://example.com/story");

    const response = await routes.request(
      "http://openbot.test/bot-9/page-frame/call-1",
    );

    expect(await response.json()).toEqual({
      frame: {
        url: "https://example.com/story",
        title: null,
        frame: "PNGBYTES",
      },
    });
  });

  /*
   * Two people on one public Bot. Filed under the Bot alone, anybody who could use it could read
   * another person's signed-in page by naming their turn.
   */
  test("another person on the same Bot cannot read the frame by naming the turn", async () => {
    const { routes } = harness();
    await navigate(routes, "https://example.com/story");

    asking = { id: "user-2", email: "[email protected]", role: "user" };
    try {
      const response = await routes.request(
        "http://openbot.test/bot-9/page-frame/call-1",
      );
      expect(await response.json()).toEqual({ frame: null });
    } finally {
      asking = actor;
    }
  });

  test("a frame kept before computers were per person still reads back", async () => {
    const { routes, saved } = harness();
    saved.push({
      computerId: "bot-9",
      toolCallId: "call-old",
      url: "https://example.com/old",
      frame: "OLDPNG",
    });

    const response = await routes.request(
      "http://openbot.test/bot-9/page-frame/call-old",
    );
    expect(await response.json()).toEqual({
      frame: { url: "https://example.com/old", title: null, frame: "OLDPNG" },
    });
  });

  test("a turn nobody photographed reads back as no frame", async () => {
    const { routes } = harness();

    const response = await routes.request(
      "http://openbot.test/bot-9/page-frame/call-never",
    );

    expect(await response.json()).toEqual({ frame: null });
  });

  /*
   * The picture is a convenience for reading the conversation back. Failing the navigation the Bot
   * was asked to do because the convenience failed would be the wrong trade every time.
   */
  test("a screenshot that cannot be taken does not fail the navigation", async () => {
    const { routes, saved } = harness({
      screenshot: async () => {
        throw new Error("computer is suspended");
      },
    });

    const response = await navigate(routes, "https://example.com/story");

    expect(response.status).toBe(200);
    expect(await response.json()).toMatchObject({
      url: "https://example.com/story",
    });
    expect(saved).toEqual([]);
  });

  /*
   * The screenshot is a second round trip and nothing holds the browser still between them. With one
   * computer shared by every Bot, another Bot's navigation lands in that gap, and this used to file
   * its page under this turn.
   */
  test("a frame of a different page than the one opened is refused", async () => {
    const { routes, saved } = harness({
      screenshot: async () => ({
        base64: "SOMEBODY-ELSES-PAGE",
        url: "https://payroll.example/salaries",
      }),
    });

    expect((await navigate(routes, "https://example.com/story")).status).toBe(
      200,
    );

    expect(saved).toEqual([]);
  });

  /*
   * A convenience picture must not wake a machine the culler has just put to sleep, nor hold a
   * navigation open for the length of a pod schedule while it does.
   */
  test("a suspended computer is not resumed to photograph it", async () => {
    let asked = false;
    const { routes, saved } = harness({
      status: async (botId: string) => ({ botId, state: "suspended" }),
      screenshot: async () => {
        asked = true;
        return { base64: "PNGBYTES", url: "https://example.com/story" };
      },
    });

    expect((await navigate(routes, "https://example.com/story")).status).toBe(
      200,
    );

    expect(asked).toBe(false);
    expect(saved).toEqual([]);
  });

  /*
   * A computer built before screenshots said which page they were of.
   *
   * Refusing on a missing url did not fail safe, it failed silently and completely: a fleet part-way
   * through a rollout kept no frames at all. With a computer each there is nobody to race with, so
   * the picture can only be this turn's.
   */
  test("an old computer's unlabelled frame is kept when the Bot has its own", async () => {
    const { routes, saved } = harness({
      isolation: "per-person",
      screenshot: async () => ({ base64: "PNGBYTES" }),
    });

    await navigate(routes, "https://example.com/story");

    expect(saved).toHaveLength(1);
  });

  /* And on one shared browser it cannot be told apart from another Bot's, so it is refused. */
  test("an old computer's unlabelled frame is refused when the computer is shared", async () => {
    const { routes, saved } = harness({
      isolation: "shared",
      screenshot: async () => ({ base64: "PNGBYTES" }),
    });

    await navigate(routes, "https://example.com/story");

    expect(saved).toEqual([]);
  });

  /* A caller that does not know which turn it is still navigates, and simply keeps no frame. */
  test("a navigation with no turn named keeps no frame", async () => {
    const { routes, saved } = harness();

    expect(
      (await navigate(routes, "https://example.com/story", null)).status,
    ).toBe(200);

    expect(saved).toEqual([]);
  });

  /*
   * The store is optional so a deployment can be wired without one. Navigation must not notice.
   */
  test("a deployment keeping no frames still navigates", async () => {
    const gateway = {
      seatFor: async (botId: string, personId: string) =>
        personSeat(personId, botId),
      navigate: async () => ({ url: "https://example.com/story", title: "T" }),
      status: async (botId: string) => ({ botId, state: "ready" as const }),
      screenshot: async () => {
        throw new Error("should not be asked");
      },
    } as unknown as ComputerGateway;
    const routes = createComputerRoutes(
      gateway,
      {} as PolicyStore,
      asActor,
      async () => true,
    );

    expect((await navigate(routes, "https://example.com/story")).status).toBe(
      200,
    );
    expect(
      await (
        await routes.request("http://openbot.test/bot-9/page-frame/call-1")
      ).json(),
    ).toEqual({ frame: null });
  });
});