Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

19.6 KB
Raw Blame
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { createServer } from "node:net";
import { tmpdir } from "node:os";
import { join } from "node:path";

/**
 * The live screen, driven against the real process with a real browser.
 *
 * `viewer.test.ts` covers who owns the screen and what that lets them do, and it cannot cover
 * whether the handlers ask. That gap is the whole of this bug: every failure here reads as correct
 * in a unit test of the decision, because the decision was never the part that was wrong. So this
 * one imports `index.ts`, opens real sockets against the port it listens on, and lets it launch
 * Chromium.
 *
 * ASKED FOR BY NAME, like `tests/smoke/journey.test.ts`, and for a related reason. `index.ts` imports
 * Playwright at module scope, `playwright` is declared only in this directory's own `package.json`,
 * and CI installs the root workspaces plus the two Bots and never this one. An ungated file would
 * therefore throw on import there, and `bun run test:ci` asserts a floor on the number of tests
 * executed, so that failure reddens the build rather than skipping quietly. Reading the flag before
 * the dynamic import below is what keeps the default suite honest on a machine where Playwright was
 * never installed:
 *
 *   bun run test:live-screen
 *
 * Everything here is timing against a browser that has to start, so the waits are generous. They are
 * not the thing under test; what is under test is whether anything relaunches a browser nobody asked
 * to start, and whether input reaches a page it does not belong to.
 */

const asked = process.env.OPENBOT_LIVE_SCREEN === "1";

const TOKEN = "test-computer-token";

/**
 * A port the operating system says is free, rather than one picked in advance.
 *
 * A fixed number here fails the whole file at import when anything else holds it, and reports as one
 * broken test rather than as a port clash. 41641 in particular is Tailscale's default, so on a host
 * running it this file could never have run at all.
 */
async function freePort(): Promise<number> {
  return new Promise((resolve, reject) => {
    const probe = createServer();
    probe.on("error", reject);
    probe.listen(0, "127.0.0.1", () => {
      const found = (probe.address() as { port: number }).port;
      probe.close(() => resolve(found));
    });
  });
}

let BASE = "";
let WS = "";

/**
 * How long "it did not come back" has to keep being true.
 *
 * A single check one tick later cannot tell a relaunch that never happened from one still in flight:
 * `evict` reads only the browsers already running, and a launch is not one of those until it
 * finishes, so both answer `wasRunning: false`. Holding the answer across several ticks and a whole
 * cold start is what makes it mean the loop is gone rather than merely slow.
 */
const STAYS_STOPPED_MS = 9_000;

/** Long enough for a close to land and the follow loop to tick once after it. */
const AFTER_ONE_FOLLOW_TICK_MS = 1_600;
/** A cold Chromium launch here takes under two seconds; this leaves room on a slower machine. */
const LAUNCH_MS = 8_000;

/** A page that writes every key it receives into the body, so input that lands is readable back. */
const TYPING_PAGE =
  "data:text/html," +
  encodeURIComponent(
    "<body>start</body><script>addEventListener('keydown',e=>{document.body.textContent+=e.key})</script>",
  );

/** A focused field whose value becomes page text, so character insertion is observable via /read. */
const TEXT_FIELD_PAGE =
  "data:text/html," +
  encodeURIComponent(
    "<input autofocus><output>empty</output><script>const input=document.querySelector('input');const output=document.querySelector('output');input.addEventListener('input',()=>output.textContent=input.value||'empty')</script>",
  );

let root = "";
let closing: Array<() => void> = [];

function api(path: string, botId: string, init?: RequestInit) {
  return fetch(`${BASE}${path}`, {
    ...init,
    headers: {
      "content-type": "application/json",
      "x-openbot-bot-id": botId,
      "x-openbot-computer-token": TOKEN,
      ...(init?.headers ?? {}),
    },
  });
}

async function takeControl(botId: string) {
  const requested = await api("/control/request", botId, {
    method: "POST",
    body: JSON.stringify({ reason: "Manual control for the screen test" }),
  });
  const state: { request: { id: string } } = await requested.json();
  const taken = await api("/control/take", botId, {
    method: "POST",
    body: JSON.stringify({ requestId: state.request.id }),
  });
  expect(taken.status).toBe(200);
}

type Frames = {
  socket: WebSocket;
  /** Every error the server sent this socket, in order. */
  errors: string[];
  /**
   * Resolves when the connection is up, which is when the server's `open` handler starts running.
   *
   * The cold-launch window opens here, not when the socket is constructed. A socket closed before it
   * has connected never reaches the handler at all, so nothing is ever stranded and the failure this
   * file exists to catch cannot happen. Waiting for this is what puts the close inside the launch.
   */
  connected: Promise<void>;
  /** Resolves once a frame of the page has arrived, which means this socket is the one casting. */
  casting: Promise<void>;
  close: () => void;
};

function watch(botId: string): Frames {
  const socket = new WebSocket(`${WS}/stream?bot=${botId}&token=${TOKEN}`);
  const errors: string[] = [];
  let sawFrame = () => {};
  const casting = new Promise<void>((resolve) => {
    sawFrame = resolve;
  });
  let opened = () => {};
  const connected = new Promise<void>((resolve) => {
    opened = resolve;
  });
  socket.addEventListener("open", () => opened());
  socket.addEventListener("message", (event) => {
    const message = JSON.parse(String(event.data)) as {
      type?: string;
      error?: string;
    };
    if (message.type === "frame") sawFrame();
    if (message.type === "error") errors.push(message.error ?? "");
  });
  const close = () => {
    try {
      socket.close();
    } catch {
      // Already gone.
    }
  };
  closing.push(close);
  return { socket, errors, connected, casting, close };
}

const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));

/** Wait for something to become true, rather than sleeping a guessed amount and hoping. */
async function until(
  what: () => boolean,
  budgetMs: number,
  why: string,
  refresh?: () => Promise<void>,
): Promise<void> {
  const deadline = Date.now() + budgetMs;
  while (Date.now() < deadline) {
    await refresh?.();
    if (what()) return;
    await wait(25);
  }
  throw new Error(`Timed out waiting for ${why}`);
}

/** Keep asking, and fail the moment the browser comes back rather than only at the end. */
async function staysStopped(botId: string): Promise<void> {
  const deadline = Date.now() + STAYS_STOPPED_MS;
  while (Date.now() < deadline) {
    if (await stopped(botId)) {
      throw new Error(
        "Something started the browser again after it was stopped",
      );
    }
    await wait(250);
  }
}

async function stopped(botId: string): Promise<boolean> {
  const response = await api("/computers/stop", botId, { method: "POST" });
  const body = (await response.json()) as { wasRunning?: boolean };
  return body.wasRunning === true;
}

beforeAll(async () => {
  if (!asked) return;
  root = await mkdtemp(join(tmpdir(), "agent-computer-live-screen-"));
  const port = await freePort();
  BASE = `http://127.0.0.1:${port}`;
  WS = `ws://127.0.0.1:${port}`;
  process.env.COMPUTER_TOKEN = TOKEN;
  process.env.PORT = String(port);
  process.env.PROFILES_DIR = join(root, "profiles");
  process.env.WORKSPACE_DIR = join(root, "workspace");
  await mkdir(join(root, "profiles"), { recursive: true });
  // After the environment is set, because the module reads it while it loads, and behind the flag,
  // because this is the import that needs Playwright present.
  await import("../src/index");
});

afterAll(async () => {
  if (!asked) return;
  for (const close of closing) close();
  closing = [];
  // Every browser this file started, so the rest of the suite does not inherit a stray Chromium.
  for (const botId of [
    "cold-close",
    "supersede",
    "late-close",
    "wheel",
    "stop-viewer",
    "reset-viewer",
    "wont-launch",
    "still-starting",
    "punctuation",
  ]) {
    await api("/computers/stop", botId, { method: "POST" }).catch(
      () => undefined,
    );
  }
  await rm(root, { recursive: true, force: true });
  // Generous, because it is stopping real browsers: the default hook budget is shorter than a
  // Chromium shutdown and the file would fail on its own cleanup rather than on anything it tested.
}, 60_000);

describe.skipIf(!asked)(
  "a socket that closes while the browser is starting",
  () => {
    test("leaves nothing behind that starts the browser again", async () => {
      // Failure 1, and the observable is deliberately not "is there a viewer": it is whether anything
      // relaunches Chromium after somebody stopped it. The orphaned follow interval called
      // `currentPage` every second, which is a launch path, so a stopped computer came back up.
      const botId = "cold-close";
      const viewer = watch(botId);
      // Connected first, so the server's `open` is running and awaiting a page, then closed straight
      // away. That is the window: closing before the connection is up never reaches the handler.
      await viewer.connected;
      viewer.close();

      await wait(LAUNCH_MS);
      // The launch really did produce a browser. Without this the whole case passes vacuously on a
      // machine where the launch failed or is still going: nothing was cast, no follow loop existed,
      // and the orphaned interval this exists to catch was never created.
      expect(await stopped(botId)).toBe(true);

      await staysStopped(botId);
    }, 30_000);
  },
);

describe.skipIf(!asked)("a socket that another connection replaced", () => {
  test("cannot type into the screen that replaced it, and is told so", async () => {
    // Failure 3. The input handler dispatched through whatever the session held, so the replaced
    // window's keys went into the page the current viewer was watching, and the sender heard nothing
    // because the old check returned before reaching anything that could report.
    const botId = "supersede";
    await api("/navigate", botId, {
      method: "POST",
      body: JSON.stringify({ url: TYPING_PAGE }),
    });

    const first = watch(botId);
    await first.casting;
    const second = watch(botId);
    await second.casting;

    await takeControl(botId);
    first.socket.send(JSON.stringify({ type: "key", key: "z" }));

    // The exact refusal, not merely some error. Dispatching through a cast the sender does not own
    // also fails, and fails loudly, so "an error arrived" passes just as well when the ownership
    // check is gone. Only the wording separates being refused from blundering into a null.
    await until(
      () => first.errors.some((e) => /no longer live/i.test(e)),
      5_000,
      "the replaced socket to be told its screen is no longer live",
    );

    const read = await api("/read", botId);
    const { text } = (await read.json()) as { text: string };
    expect(text).not.toContain("z");
  }, 30_000);
});

describe.skipIf(!asked)("a superseded socket closing later", () => {
  test("does not take the screen down with it", async () => {
    // What #191 fixed, at the level where it can actually go wrong again. Replacement does not close
    // the socket it superseded, so that socket closes on its own schedule, which on an ordinary
    // make-before-break reconnect is after the replacement is already casting. A close that stopped
    // whatever the slot held rather than naming its own socket would leave the person who just
    // reconnected watching a still image with input going nowhere, and nothing would say so.
    const botId = "late-close";
    await api("/navigate", botId, {
      method: "POST",
      body: JSON.stringify({ url: TYPING_PAGE }),
    });

    const first = watch(botId);
    await first.casting;
    const second = watch(botId);
    await second.casting;

    // The replaced socket goes away now, after its replacement is live.
    first.close();
    await wait(AFTER_ONE_FOLLOW_TICK_MS);

    // The survivor still owns the screen, and the proof is that its typing arrives: a cast that was
    // stopped underneath it, or an ownership it quietly lost, would refuse this instead.
    await takeControl(botId);
    second.socket.send(JSON.stringify({ type: "key", key: "k" }));

    let landed = "";
    await until(
      () => landed.includes("k"),
      5_000,
      "the surviving viewer's key to reach the page",
      async () => {
        const read = await api("/read", botId);
        landed = ((await read.json()) as { text: string }).text;
      },
    );

    expect(second.errors).toEqual([]);
  }, 30_000);
});

describe.skipIf(!asked)("printable punctuation from the live screen", () => {
  test("inserts a period using the browser key code sent by the surface", async () => {
    const botId = "punctuation";
    await api("/navigate", botId, {
      method: "POST",
      body: JSON.stringify({ url: TEXT_FIELD_PAGE }),
    });
    const viewer = watch(botId);
    await viewer.casting;
    await takeControl(botId);

    viewer.socket.send(
      JSON.stringify({
        type: "key",
        event: "down",
        key: ".",
        code: "Period",
        text: ".",
        windowsVirtualKeyCode: 190,
        modifiers: 0,
      }),
    );
    viewer.socket.send(
      JSON.stringify({
        type: "key",
        event: "up",
        key: ".",
        code: "Period",
        windowsVirtualKeyCode: 190,
        modifiers: 0,
      }),
    );
    // Older OpenBot surfaces did not send the browser keyCode. Keep their punctuation usable while
    // a deployment rolls the frontend and computer images independently.
    viewer.socket.send(
      JSON.stringify({
        type: "key",
        event: "down",
        key: ".",
        code: "Period",
        text: ".",
        modifiers: 0,
      }),
    );
    viewer.socket.send(
      JSON.stringify({
        type: "key",
        event: "up",
        key: ".",
        code: "Period",
        modifiers: 0,
      }),
    );

    let landed = "";
    await until(
      () => landed.includes(".."),
      5_000,
      "periods from current and legacy surfaces to be inserted into the focused field",
      async () => {
        const read = await api("/read", botId);
        landed = ((await read.json()) as { text: string }).text;
      },
    );

    expect(viewer.errors).toEqual([]);
  }, 30_000);
});

describe.skipIf(!asked)(
  "the wheel, with the ownership check in front of it",
  () => {
    test("still refuses the casting socket while the Bot holds it", async () => {
      // The control half of the reordering. The superseded case above runs with a person already
      // holding the wheel, so the control check is passive there and a rewiring that dropped it would
      // still pass. Here the socket genuinely owns the screen and nobody has taken control, which is
      // the only arrangement where that check is the one doing the refusing.
      const botId = "wheel";
      await api("/navigate", botId, {
        method: "POST",
        body: JSON.stringify({ url: TYPING_PAGE }),
      });
      // A fresh Bot starts with Bot ownership; no handoff exists to release.

      const viewer = watch(botId);
      await viewer.casting;

      viewer.socket.send(JSON.stringify({ type: "key", key: "q" }));

      await until(
        () => viewer.errors.length > 0,
        5_000,
        "the owner to be told to take control first",
      );
      expect(viewer.errors.some((e) => /control/i.test(e))).toBe(true);

      const read = await api("/read", botId);
      const { text } = (await read.json()) as { text: string };
      expect(text).not.toContain("q");
    }, 30_000);
  },
);

describe.skipIf(!asked)("stopping the computer out from under a viewer", () => {
  test("takes the screen down and does not come back", async () => {
    // Failure 2. Stopping released the wheel and left the viewer alone, so the follow loop's next
    // tick asked for a page, which starts a browser, and the computer somebody had just stopped was
    // running again a second later, refreshing its own idle timestamp every tick while it did.
    const botId = "stop-viewer";
    const viewer = watch(botId);
    await viewer.casting;

    expect(await stopped(botId)).toBe(true);

    await until(
      () => viewer.errors.some((e) => /stopped/i.test(e)),
      5_000,
      "the viewer to be told the computer stopped",
    );

    await staysStopped(botId);
  }, 30_000);

  test("the same holds when the computer is reset rather than stopped", async () => {
    // Reset wipes the profile as well, and had the identical hole: it released the wheel and never
    // touched the viewer. Its own response carries no `wasRunning`, so the relaunch is observed
    // through a following stop rather than through what reset itself answers.
    const botId = "reset-viewer";
    const viewer = watch(botId);
    await viewer.casting;

    await api("/computers/reset", botId, { method: "POST" });

    await until(
      () => viewer.errors.some((e) => /stopped/i.test(e)),
      5_000,
      "the viewer to be told its computer went away",
    );

    await staysStopped(botId);
  }, 30_000);
});

describe.skipIf(!asked)("typing into a screen that is still opening", () => {
  test("is told the screen is starting, not that it ended", async () => {
    // The reason there are three standings rather than two. A socket mid-launch owns a claim and no
    // cast, exactly like a socket that was superseded, and answering both the same way tells somebody
    // whose screen is seconds from live that their session is over.
    const botId = "still-starting";
    const viewer = watch(botId);
    // Only connected, deliberately: the browser is still starting, so no cast exists yet.
    await viewer.connected;
    viewer.socket.send(JSON.stringify({ type: "key", key: "s" }));

    await until(
      () => viewer.errors.length > 0,
      LAUNCH_MS,
      "the socket to be answered while its screen is still starting",
    );

    expect(viewer.errors[0]).toMatch(/still starting/i);
  }, 40_000);
});

describe.skipIf(!asked)("a screen whose browser will not start", () => {
  test("says so and leaves nothing holding the session", async () => {
    // `open`'s catch. A file sits where this Bot's profile directory would go, so Chromium cannot
    // launch and `currentPage` throws while the claim is already held. The socket is told and closed,
    // which is the observable part; the claim being released with it is what keeps the session
    // sweepable, and `viewer.test.ts` pins that half because a leaked claim changes nothing a caller
    // outside this process can see until the map has grown.
    const botId = "wont-launch";
    await writeFile(join(root, "profiles", botId), "not a directory");

    const viewer = watch(botId);
    await viewer.connected;

    await until(
      () => viewer.errors.length > 0,
      15_000,
      "the socket to be told its screen could not be started",
    );

    // What it says is Playwright's to word, so this pins only that the socket was told why its
    // screen never arrived rather than handed one of the lifecycle refusals, which would mean the
    // failure had been reported as somebody else taking the screen.
    expect(viewer.errors[0]).not.toMatch(
      /no longer live|still starting|computer stopped/i,
    );

    // Closed by the handler rather than left open against a browser that does not exist.
    await until(
      () => viewer.socket.readyState === WebSocket.CLOSED,
      5_000,
      "the socket to be closed after the failure",
    );
  }, 40_000);
});