Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type {
BrowserState,
LauncherConnection,
} from "../../shared/openbot-browser";
import { NOT_ITS_OWN_COMPUTER, ONLY_THE_PERSONS } from "../src/person-access";
/**
* A person's browser through the computer itself: the real process, its token, its dispatcher and
* its live screen, with bundled Chromium headless and the fixture Recorder.
*
* `person-browser.test.ts` drives the browser routes over every kind of person's browser. This is
* what only the running computer decides: who may call them, what a person's browser id may reach,
* and that a person's input reaches their browser through the live screen without anybody taking
* control first.
*
* OPENBOT_COMPUTER_BROWSER=1 bun test tests/person-browser-computer.test.ts
*
* One case reaches the public internet (Google, through the computer's own egress filter), so it
* runs only when that is asked for as well:
*
* OPENBOT_COMPUTER_BROWSER=1 OPENBOT_LIVE_NETWORK=1 bun test tests/person-browser-computer.test.ts
*/
const asked = process.env.OPENBOT_COMPUTER_BROWSER === "1";
const online = asked && process.env.OPENBOT_LIVE_NETWORK === "1";
const TOKEN = "person-browser-computer-token";
const PERSON = "person-fedcba9876543210fedcba9876543210";
const FIXTURE = join(import.meta.dir, "fixtures", "recorder");
type Computer = { base: string; child: ReturnType<typeof Bun.spawn> };
const computers: Computer[] = [];
let root = "";
let site: ReturnType<typeof Bun.serve> | undefined;
async function freePort(): Promise<number> {
const probe = Bun.serve({
hostname: "127.0.0.1",
port: 0,
fetch: () => new Response(),
});
const { port } = probe;
await probe.stop(true);
if (port === undefined) throw new Error("The port probe did not listen.");
return port;
}
async function startComputer(
name: string,
env: Record<string, string>,
): Promise<Computer> {
const port = await freePort();
const base = `http://127.0.0.1:${port}`;
const child = Bun.spawn([process.execPath, "src/index.ts"], {
cwd: join(import.meta.dir, ".."),
env: {
...process.env,
COMPUTER_TOKEN: TOKEN,
PORT: String(port),
COMPUTER_BROWSER_BACKEND: "managed",
COMPUTER_BROWSER_MODE: "headless",
// No API server pushes this computer a network policy, so it browses before one arrives.
EGRESS_POLICY_REQUIRED: "0",
PROFILES_DIR: join(root, name, "profiles"),
WORKSPACE_DIR: join(root, name, "workspace"),
...env,
},
stdout: "ignore",
stderr: "inherit",
});
const computer = { base, child };
computers.push(computer);
const deadline = Date.now() + 15_000;
for (;;) {
if (child.exitCode !== null)
throw new Error(`The computer exited with ${child.exitCode}.`);
const ready = await fetch(`${base}/health`)
.then((response) => response.ok)
.catch(() => false);
if (ready) return computer;
if (Date.now() > deadline) throw new Error("The computer did not start.");
await Bun.sleep(50);
}
}
let computer: Computer;
/** What the fixture Recorder said it was called with, oldest first. */
const calls: { method: string; input: unknown }[] = [];
function call(
path: string,
options: {
body?: unknown;
botId?: string;
token?: string;
/** Another computer than the person's own. */
on?: Computer;
} = {},
) {
return fetch(`${(options.on ?? computer).base}${path}`, {
method: options.body === undefined ? "GET" : "POST",
headers: {
"content-type": "application/json",
"x-openbot-computer-token": options.token ?? TOKEN,
...(options.botId === undefined
? { "x-openbot-bot-id": PERSON }
: options.botId
? { "x-openbot-bot-id": options.botId }
: {}),
},
...(options.body === undefined
? {}
: { body: JSON.stringify(options.body) }),
});
}
const connection = (): LauncherConnection => ({
openbotUrl: `http://127.0.0.1:${site?.port}`,
token: "obr_fixture-token",
connectionId: "connection-1",
user: { id: "user-1", name: "Pat Example", email: "[email protected]" },
});
beforeAll(async () => {
if (!asked) return;
root = await mkdtemp(join(tmpdir(), "openbot-person-computer-"));
// The OpenBot the fixture Recorder reports to, and a page whose title is what the person types.
site = Bun.serve({
hostname: "127.0.0.1",
port: 0,
async fetch(request) {
const { pathname } = new URL(request.url);
if (pathname === "/fixture/calls") {
calls.push((await request.json()) as (typeof calls)[number]);
return new Response("");
}
// A cookie kept for an hour, as a site keeps a sign-in: it outlives the browser closing.
if (pathname === "/set-cookie")
return new Response("<!doctype html><title>set</title>", {
headers: {
"content-type": "text/html; charset=utf-8",
"set-cookie": "signed-in=yes; Max-Age=3600; Path=/",
},
});
// The cookies the browser sent, as the page's title.
if (pathname === "/cookies")
return new Response(
`<!doctype html><title>cookies:${request.headers.get("cookie") ?? ""}</title>`,
{ headers: { "content-type": "text/html; charset=utf-8" } },
);
return new Response(
"<!doctype html><title>empty</title><input autofocus><script>document.querySelector('input').addEventListener('input',e=>document.title=e.target.value)</script>",
{ headers: { "content-type": "text/html; charset=utf-8" } },
);
},
});
// Started for the person's browser alone, as a supervisor starts the container it makes for it.
computer = await startComputer("with-recorder", {
OPENBOT_RECORDER_DIR: FIXTURE,
COMPUTER_BOT_ID: PERSON,
});
}, 30_000);
afterAll(async () => {
if (!asked) return;
if (computer)
await call("/browser/close", { body: {} }).catch(() => undefined);
for (const running of computers) {
running.child.kill();
await running.child.exited;
}
site?.stop(true);
if (root) await rm(root, { recursive: true, force: true });
}, 30_000);
describe.skipIf(!asked)("a person's browser on the computer", () => {
test("is behind the computer token like every other route", async () => {
expect((await call("/browser/state", { token: "wrong" })).status).toBe(401);
});
test("takes only its own person's browser id: no Bot's, and nobody else's", async () => {
for (const botId of [
"shared",
"sales-bot",
"person-00000000000000000000000000000000",
]) {
const response = await call("/browser/open", {
body: { connection: connection() },
botId,
});
expect(response.status).toBe(403);
expect(await response.json()).toEqual({ error: ONLY_THE_PERSONS });
}
});
test("reaches nothing a Bot acts or reads through", async () => {
for (const [path, body] of [
["/navigate", { url: "https://example.com" }],
["/click", { ref: "e1" }],
["/exec", { command: "env" }],
["/snapshot", {}],
["/control/request", { reason: "help" }],
["/files/read", { path: "notes.md" }],
] as const) {
const response = await call(path, { body });
expect(response.status).toBe(403);
}
expect((await call("/read")).status).toBe(403);
expect((await call("/screenshot")).status).toBe(403);
});
test("opens connected, and starts with the person holding the wheel", async () => {
const opening = await call("/browser/open", {
body: { connection: connection() },
});
expect(opening.status).toBe(200);
const state = (await opening.json()) as BrowserState;
expect(state).toMatchObject({
open: true,
browser: "chromium",
window: "screen",
recorder: { connected: true, narration: "external" },
});
const control = (await (await call("/control")).json()) as {
holder: string;
requested: boolean;
};
expect(control).toMatchObject({ holder: "human", requested: false });
}, 60_000);
test("takes the person's input through the live screen without a takeover", async () => {
const page = `http://127.0.0.1:${site?.port}/form`;
expect(
(
await call("/browser/tabs", {
body: { action: "new", url: page },
})
).status,
).toBe(200);
const socket = new WebSocket(
`${computer.base.replace("http:", "ws:")}/stream?bot=${PERSON}&token=${TOKEN}`,
);
const errors: string[] = [];
const casting = new Promise<void>((resolve, reject) => {
socket.addEventListener("message", (event) => {
const message = JSON.parse(String(event.data)) as {
type: string;
error?: string;
};
if (message.type === "frame") resolve();
if (message.type === "error") errors.push(message.error ?? "");
});
socket.addEventListener("error", () =>
reject(new Error("The live screen failed.")),
);
});
try {
await casting;
socket.send(JSON.stringify({ type: "text", text: "typed live" }));
const deadline = Date.now() + 10_000;
let title = "";
while (Date.now() < deadline) {
const state = (await (
await call("/browser/state")
).json()) as BrowserState;
title = state.tabs.find((tab) => tab.active)?.title ?? "";
if (title === "typed live") break;
await Bun.sleep(100);
}
expect(title).toBe("typed live");
expect(errors).toEqual([]);
} finally {
socket.close();
}
// And by the person's input route, the same way.
const typed = await call("/human/type", { body: { text: " and posted" } });
expect(typed.status).toBe(200);
}, 60_000);
test("closes, and the live screen then finds nothing to show", async () => {
expect(await (await call("/browser/close", { body: {} })).json()).toEqual({
open: false,
tabs: [],
});
const socket = new WebSocket(
`${computer.base.replace("http:", "ws:")}/stream?bot=${PERSON}&token=${TOKEN}`,
);
const error = await new Promise<string>((resolve) => {
socket.addEventListener("message", (event) => {
const message = JSON.parse(String(event.data)) as {
type: string;
error?: string;
};
if (message.type === "error") resolve(message.error ?? "");
});
});
socket.close();
expect(error).toContain("not open");
// Nothing reopened it behind the person's back.
const state = (await (await call("/browser/state")).json()) as BrowserState;
expect(state.open).toBe(false);
}, 30_000);
test("clears: a recording in it ends first, the whole profile goes and nothing else does, and it opens again signed out of everything", async () => {
const origin = `http://127.0.0.1:${site?.port}`;
const open = () =>
call("/browser/open", { body: { connection: connection() } });
const sees = async () => {
const there = await call("/browser/navigate", {
body: { url: `${origin}/cookies` },
});
expect(there.status).toBe(200);
return ((await there.json()) as BrowserState).tabs.find(
(tab) => tab.active,
)?.title;
};
expect((await open()).status).toBe(200);
expect(
(
await call("/browser/navigate", {
body: { url: `${origin}/set-cookie` },
})
).status,
).toBe(200);
expect(await sees()).toBe("cookies:signed-in=yes");
// Something of the computer's that is not the browser's.
const workspace = join(root, "with-recorder", "workspace");
await mkdir(workspace, { recursive: true });
await writeFile(join(workspace, "notes.md"), "Keep me.");
const profiles = join(root, "with-recorder", "profiles");
expect(await readdir(profiles)).toContain(PERSON);
expect(
(
await call("/browser/recording", {
body: { action: "start", title: "Before the clear" },
})
).status,
).toBe(200);
calls.length = 0;
const cleared = await call("/browser/clear", { body: {} });
expect(cleared.status).toBe(200);
expect(await cleared.json()).toEqual({ open: false, tabs: [] });
// The recording was stopped while the Recorder could still be told.
expect(calls.map((each) => [each.method, each.input])).toEqual([
["record", { action: "stop" }],
]);
// The profile is gone, and left gone: only the server can hand the Recorder a connection, so it opens it again.
expect(await readdir(profiles)).not.toContain(PERSON);
expect(await Bun.file(join(workspace, "notes.md")).text()).toBe("Keep me.");
expect(
((await (await call("/browser/state")).json()) as BrowserState).open,
).toBe(false);
// Opened again, it is fresh: signed out of the site, with only closing the browser it would have stayed signed in.
expect((await open()).status).toBe(200);
expect(await sees()).toBe("cookies:");
// A browser that is closed is cleared all the same.
await call("/browser/close", { body: {} });
expect((await call("/browser/clear", { body: {} })).status).toBe(200);
expect(await readdir(profiles)).not.toContain(PERSON);
}, 90_000);
test("a computer without a built Recorder refuses to open one, and says why", async () => {
const without = await startComputer("without-recorder", {
OPENBOT_RECORDER_DIR: "",
COMPUTER_BOT_ID: PERSON,
});
const response = await fetch(`${without.base}/browser/open`, {
method: "POST",
headers: {
"x-openbot-computer-token": TOKEN,
"x-openbot-bot-id": PERSON,
},
body: JSON.stringify({ connection: connection() }),
});
expect(response.status).toBe(503);
expect(((await response.json()) as { error: string }).error).toContain(
"OPENBOT_RECORDER_DIR",
);
const state = await fetch(`${without.base}/browser/state`, {
headers: {
"x-openbot-computer-token": TOKEN,
"x-openbot-bot-id": PERSON,
},
});
expect(((await state.json()) as BrowserState).open).toBe(false);
}, 30_000);
});
/**
* A computer whose image lacks Playwright's Chromium: the person's browser cannot start, which is OpenBot's setup and
* nothing the person did or can do. Run 4 saw a launch failure reach the person as a 502 reading "The browser could
* not be opened:" and Playwright's own text, its install box included. No browser starts, so this runs everywhere.
*/
describe("a person's browser on a computer whose browser is not installed", () => {
test("is a 503 the server turns into its sentence for the person, and Playwright's text stays in the computer's log", async () => {
const dir = await mkdtemp(join(tmpdir(), "openbot-no-chromium-"));
const port = await freePort();
const base = `http://127.0.0.1:${port}`;
const child = Bun.spawn([process.execPath, "src/index.ts"], {
cwd: join(import.meta.dir, ".."),
env: {
...process.env,
COMPUTER_TOKEN: TOKEN,
PORT: String(port),
COMPUTER_BROWSER_BACKEND: "managed",
COMPUTER_BROWSER_MODE: "headless",
EGRESS_POLICY_REQUIRED: "0",
PROFILES_DIR: join(dir, "profiles"),
WORKSPACE_DIR: join(dir, "workspace"),
OPENBOT_RECORDER_DIR: FIXTURE,
COMPUTER_BOT_ID: PERSON,
// Where Playwright looks for its browsers: an empty folder, as an image built without them is.
PLAYWRIGHT_BROWSERS_PATH: join(dir, "browsers"),
},
stdout: "pipe",
stderr: "inherit",
});
try {
const deadline = Date.now() + 15_000;
while (
!(await fetch(`${base}/health`).then(
(response) => response.ok,
() => false,
))
) {
if (Date.now() > deadline || child.exitCode !== null)
throw new Error("The computer did not start.");
await Bun.sleep(50);
}
const response = await fetch(`${base}/browser/open`, {
method: "POST",
headers: {
"content-type": "application/json",
"x-openbot-computer-token": TOKEN,
"x-openbot-bot-id": PERSON,
},
body: JSON.stringify({
connection: {
openbotUrl: "http://127.0.0.1:7829",
token: "obr_fixture-token",
connectionId: "connection-1",
user: {
id: "user-1",
name: "Pat Example",
email: "[email protected]",
},
},
}),
});
expect(response.status).toBe(503);
const { error } = (await response.json()) as { error: string };
expect(error).toContain("Executable doesn't exist at");
child.kill();
await child.exited;
const logged = (await new Response(child.stdout).text())
.split("\n")
.flatMap((line) => {
try {
return [JSON.parse(line) as Record<string, unknown>];
} catch {
return [];
}
})
.filter((line) => line.type === "computer-browser-open-failed");
expect(logged).toEqual([
{
type: "computer-browser-open-failed",
botId: PERSON,
step: "launch",
status: 503,
error,
ms: expect.any(Number),
},
]);
} finally {
child.kill();
await child.exited;
await rm(dir, { recursive: true, force: true });
}
}, 30_000);
});
/**
* Where a person's browser may run at all, as the running computer decides it from how it was started. A Bot on the
* computer every Bot shares may run a shell as the same user the browser's profile belongs to, so a person's browser
* runs only on a computer of its own: one started for it (`COMPUTER_BOT_ID`, as a supervisor or a Sandbox starts each),
* or one started for people's browsers alone (`COMPUTER_PEOPLE_ONLY`, OpenBot Desktop's). Refused before anything is
* launched, so no browser starts and this runs everywhere.
*/
describe("a person's browser on a computer that is not its own", () => {
let dir = "";
/** The computers this block starts, which it stops itself: the file's own teardown runs only with a browser. */
let first = 0;
beforeAll(async () => {
dir = await mkdtemp(join(tmpdir(), "openbot-person-placement-"));
root ||= dir;
first = computers.length;
});
afterAll(async () => {
for (const running of computers.splice(first)) {
running.child.kill();
await running.child.exited;
}
if (dir) await rm(dir, { recursive: true, force: true });
}, 30_000);
const open = (on: Computer, botId = PERSON) =>
call("/browser/open", {
body: { connection: connection() },
botId,
on,
});
test("is refused on the computer every Bot shares, before its profile, policy or live screen exist", async () => {
const shared = await startComputer("shared", {
OPENBOT_RECORDER_DIR: FIXTURE,
PROFILES_DIR: join(dir, "shared", "profiles"),
});
const opening = await open(shared);
expect(opening.status).toBe(503);
expect(await opening.json()).toEqual({ error: NOT_ITS_OWN_COMPUTER });
for (const path of ["/browser/state", "/egress-policy", "/run"])
expect((await call(path, { on: shared })).status).toBe(503);
expect(
(await fetch(`${shared.base}/stream?token=${TOKEN}&bot=${PERSON}`))
.status,
).toBe(503);
expect(await readdir(join(dir, "shared", "profiles"))).toEqual([]);
// A Bot's computer, started for that Bot, refuses it as well.
const bots = await startComputer("a-bot", {
OPENBOT_RECORDER_DIR: FIXTURE,
COMPUTER_BOT_ID: "sales-bot",
});
expect((await open(bots)).status).toBe(503);
// Bots are served on both as before: who holds the wheel is a Bot's to read.
for (const on of [shared, bots])
expect((await call("/control", { botId: "sales-bot", on })).status).toBe(
200,
);
}, 30_000);
test("on OpenBot Desktop's computer, which runs people's browsers and no Bot, a Bot is refused", async () => {
const desktop = await startComputer("people-only", {
OPENBOT_RECORDER_DIR: FIXTURE,
COMPUTER_PEOPLE_ONLY: "on",
});
for (const botId of ["sales-bot", "shared"]) {
const asked = await call("/exec", {
body: { command: "cat /profiles/*/Default/Cookies" },
botId,
on: desktop,
});
expect(asked.status).toBe(403);
expect(await asked.json()).toEqual({ error: ONLY_THE_PERSONS });
}
// The person's own routes go on to the browser host, which says the browser is not open yet.
expect((await call("/browser/state", { on: desktop })).status).toBe(200);
}, 30_000);
});
describe.skipIf(!online)(
"the open internet through the computer's egress filter",
() => {
/*
* Google answers with an HTTP/2 ACCEPT_CH frame, which makes Chromium restart the page's request,
* and Playwright, answering the filter's proxy credentials through request interception, never
* sent the restarted one: every browser on a computer timed out on Google while Bing loaded.
*/
const loads = async (response: Response) => {
expect(response.status).toBe(200);
return (await response.json()) as BrowserState & { url?: string };
};
test("a person's browser loads Google", async () => {
await loads(
await call("/browser/open", { body: { connection: connection() } }),
);
for (const url of [
"https://www.google.com/",
"https://accounts.google.com/",
]) {
const state = await loads(
await call("/browser/navigate", { body: { url } }),
);
expect(state.tabs.find((tab) => tab.active)?.url).toStartWith(
new URL(url).origin,
);
}
await call("/browser/close", { body: {} });
}, 90_000);
test("a Bot's browser loads Google", async () => {
// On a computer of the Bot's: the person's runs nothing else.
const on = await startComputer("live-network-bot", {});
const answer = await loads(
await call("/navigate", {
body: { url: "https://www.google.com/" },
botId: "live-network-bot",
on,
}),
);
expect(answer.url).toStartWith("https://www.google.com");
await call("/computers/stop", {
body: {},
botId: "live-network-bot",
on,
});
}, 90_000);
},
);