Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
/**
* Launching a person's browser with the Recorder in it.
*
* One browser per person, a computer named for them (`person-` and 32 hex digits). On OpenBot
* Desktop it is the person's installed Chrome, or Edge, as a native window on a dedicated OpenBot
* profile; hosted, it is this computer's bundled Chromium, which the person sees through the live
* screen. Either way the Recorder is in it before anybody looks, its microphone is granted, and
* none of its setup pages show: `person-launch.ts` has the switches and why, this file starts the
* browser and puts the Recorder in.
*
* Proven on real browsers before this was written (2026-10-21):
* - Bundled Chromium loads the Recorder from `--load-extension` with `--disable-extensions-except`.
* - Branded Chrome ignores `--load-extension` since Chrome 137, and installs it through DevTools
* `Extensions.loadUnpacked` on a browser-level session, when launched with
* `--enable-unsafe-extension-debugging` and without Playwright's `--disable-extensions`. With that
* switch still on the command line the call answers an id and the extension is silently absent.
* Edge accepts both routes.
* - An extension loaded through DevTools is gone after a restart, so it is loaded at every launch;
* its `chrome.storage.local` survives, so the Recorder keeps what it stored.
* - `Browser.grantPermissions` with the extension's origin grants it the microphone without a
* prompt, in Chrome and in Chromium.
* https://playwright.dev/docs/chrome-extensions
* https://chromedevtools.github.io/devtools-protocol/tot/Extensions/#method-loadUnpacked
* https://chromedevtools.github.io/devtools-protocol/tot/Browser/#method-grantPermissions
*/
import { join } from "node:path";
import {
type BrowserContext,
type CDPSession,
chromium,
type Page,
} from "playwright";
import type {
BrowserProduct,
BrowserWindow,
} from "../../shared/openbot-browser";
import type { BrowserRuntime } from "./browser-runtime";
import type { Egress } from "./egress";
import { launchNativeBrowser } from "./native-browser";
import {
copyNativeMessagingHosts,
defaultNativeMessagingHosts,
forgetRecorderPins,
installedBrowserAt,
NO_INSTALLED_BROWSER,
type PersonDisplay,
personNativeLaunch,
personWindow,
recorderLaunchOptions,
} from "./person-launch";
import { isRecorderPage, type Recorder } from "./recorder";
import { fitWindowsToViewport, holdRecorder } from "./recorder-in-browser";
import type { RecorderLauncher } from "./recorder-launcher";
/** No Chrome and no Edge on this machine: nothing to open a person's window in. */
export class NoInstalledBrowserError extends Error {
constructor() {
super(NO_INSTALLED_BROWSER);
this.name = "NoInstalledBrowserError";
}
}
/**
* The steps of opening a person's browser, in the order they run. A failed open names the one it
* stopped at, because the same sentence ("the Recorder did not answer in time") can come from waking
* its worker, reloading it or connecting it, and each has a different cause.
*/
export type PersonOpenStep =
/** The built Recorder this computer is configured with. */
| "recorder-files"
/** Copying the person's native messaging hosts into the profile (`copyNativeMessagingHosts`). */
| "native-hosts"
/** Finding and starting the browser, and opening its first tab. */
| "launch"
/** Installing the Recorder over DevTools, which a native window needs at every launch. */
| "recorder-load"
/** Reaching the Recorder's service worker, waking it when it is asleep. */
| "worker"
/** Loading the Recorder again because its worker ran an earlier build (`keepRecorderCurrent`). */
| "recorder-reload"
/** Granting the Recorder the microphone. */
| "microphone"
/** Handing downloads back to the browser's own setting. */
| "downloads"
/** Handing the Recorder its connection through the launcher API. */
| "connect"
/** Reading the open browser's tabs and the Recorder's report, which the open answers with. */
| "state";
/**
* Playwright's own words for a browser this computer cannot start at all: the browser it bundles is not installed
* (`playwright install`), the system libraries it needs are missing (`playwright install-deps`), or the installed browser
* a launch named is not there. These are the computer's setup, which only whoever runs OpenBot can see to, never anything
* the person did. Matched on playwright-core 1.62.1's messages (its browser registry and its dependency check), which
* `person-browser-rules.test.ts` pins against the real text, so a Playwright that words them differently fails there.
*/
const BROWSER_NOT_INSTALLED = [
/Executable doesn't exist at /,
/Host system is missing dependencies/,
/Chromium distribution '[^']+' is not found at /,
];
/** Whether a launch failed because this computer's browser is not installed, by Playwright's message. */
export function isBrowserNotInstalled(message: string): boolean {
return BROWSER_NOT_INSTALLED.some((pattern) => pattern.test(message));
}
/**
* Opening a person's browser failed at `step`, for the reason its message gives. `setup` when the reason is this
* computer's setup rather than anything about this open: the built Recorder's files unreadable, or a browser that is not
* installed or cannot run here. The routes answer those 503, which the server tells the person is for an admin.
*/
export class PersonOpenError extends Error {
readonly step: PersonOpenStep;
readonly setup: boolean;
constructor(step: PersonOpenStep, cause: unknown, setup?: boolean) {
const message = cause instanceof Error ? cause.message : String(cause);
super(message, { cause });
this.name = "PersonOpenError";
this.step = step;
this.setup =
setup ?? (step === "recorder-files" || isBrowserNotInstalled(message));
}
}
/** Run one step of an open, so a failure says which step it was. A failure already named keeps its name. */
async function step<T>(
name: PersonOpenStep,
work: () => Promise<T>,
setup?: boolean,
): Promise<T> {
try {
return await work();
} catch (error) {
if (error instanceof PersonOpenError) throw error;
throw new PersonOpenError(name, error, setup);
}
}
/** What a person's running browser is, beyond what any browser here is. */
export type PersonBrowser = {
product: BrowserProduct;
window: BrowserWindow;
/** The Recorder's id in this browser, which says which pages and which worker are its own. */
recorderId: string;
launcher: RecorderLauncher;
/** The browser-level DevTools session, which the tab list reads titles from. */
session: CDPSession;
};
export type PersonLaunch = {
botId: string;
/** The profile directory, which is also the browser's user data directory. */
dir: string;
runtime: BrowserRuntime;
recorder: Recorder;
/** The egress filter for a hosted browser. A native window has none: it is the person's own. */
proxy: Egress | null;
sandbox: boolean;
botArgs: readonly string[];
viewport: { width: number; height: number };
/** The person's own display, which a native window opens on. See `personBrowserEnv`. */
display: PersonDisplay;
};
/**
* Every page in the browser's windows: the person's tabs and any of the Recorder's own pages a tab
* shows (its options, say), but not DevTools. The Recorder's offscreen document is not one: Chromium
* reports it as a background page, which Playwright does not count among the context's pages.
*
* A person's browser is open while there is any: a window on the screen is an open browser, whatever
* it shows.
*/
export function windowPages(context: BrowserContext): Page[] {
return context
.pages()
.filter(
(page) => !page.isClosed() && !page.url().startsWith("devtools://"),
);
}
/** The person's pages: page targets that are not the Recorder's own and not DevTools. */
export function personPages(
context: BrowserContext,
recorderId: string,
): Page[] {
return context
.pages()
.filter(
(page) =>
!page.isClosed() &&
!isRecorderPage(page.url(), recorderId) &&
!page.url().startsWith("devtools://"),
);
}
/**
* Start a person's browser, with the Recorder loaded and its microphone granted.
*
* All or nothing: a browser whose Recorder could not be put in is closed again rather than handed
* back, because a person's browser without the Recorder is the one thing this must never open.
*
* A failure is a `PersonOpenError` naming the step it stopped at, and whether it is this computer's
* setup, except that no installed browser is a `NoInstalledBrowserError`, which the routes answer
* differently.
*/
export async function launchPersonBrowser(
input: PersonLaunch,
): Promise<{ context: BrowserContext; person: PersonBrowser }> {
const window = personWindow(input.runtime);
let product: BrowserProduct = "chromium";
let executable: string | undefined;
if (window === "native") {
// Looking for Chrome or Edge fails only on a platform with no place to look, or a folder this computer cannot read.
const installed = await step(
"launch",
() => installedBrowserAt(process.env),
true,
);
if (!installed) throw new NoInstalledBrowserError();
product = installed.browser;
executable = installed.executable;
/*
* The person's password manager and other desktop hookups, copied from their own profile of the
* same browser into this profile's folder, which is where a browser on a dedicated user data
* directory reads them. Manifests only, at every launch. Windows reads them from the registry.
*/
const from = defaultNativeMessagingHosts(installed.browser, process.env);
if (from) {
const copied = await step("native-hosts", () =>
copyNativeMessagingHosts(from, join(input.dir, "NativeMessagingHosts")),
);
console.info(
JSON.stringify({
type: "computer-person-native-hosts",
botId: input.botId,
hosts: copied,
}),
);
}
}
await step("launch", () => forgetRecorderPins(input.dir, input.recorder.id));
const headless = input.runtime.mode === "headless";
/*
* A native window is started here and attached to, with none of Playwright's test defaults and none
* of its focus emulation (`native-browser.ts`): its tabs behind the one in front are behind it, as
* in any browser. The hosted browser is Playwright's launch, as a Bot's is.
*/
const context = await step("launch", () =>
executable
? launchNativeBrowser({
executable,
...personNativeLaunch({
dir: input.dir,
headless,
display: input.display,
}),
})
: chromium.launchPersistentContext(input.dir, {
channel: product,
...recorderLaunchOptions(
{
recorderDir: input.recorder.dir,
headless,
sandbox: input.sandbox,
botArgs: input.botArgs,
},
input.viewport,
),
// This process owns shutdown, as for a Bot's browser: Playwright's own handlers would kill
// the browser before its profile is written.
handleSIGTERM: false,
handleSIGINT: false,
handleSIGHUP: false,
...(input.proxy ? { proxy: input.proxy } : {}),
}),
);
try {
const session = await step("launch", async () => {
const browser = context.browser();
if (!browser) throw new Error("The browser did not start.");
return browser.newBrowserCDPSession();
});
let recorderId = input.recorder.id;
if (window === "native") {
const { id } = await step("recorder-load", () =>
session.send("Extensions.loadUnpacked", { path: input.recorder.dir }),
);
if (id !== recorderId) {
// The browser's answer is the truth about which pages and worker are the Recorder's. The
// manifest key decides it in every Chromium, so a difference is worth saying out loud.
console.info(
JSON.stringify({
type: "computer-recorder-id-differs",
expected: recorderId,
loaded: id,
}),
);
recorderId = id;
}
}
const launcher = await holdRecorder({
session,
context,
recorder: input.recorder,
recorderId,
botId: input.botId,
step,
});
if (window === "screen")
void fitWindowsToViewport(context, input.viewport, input.botId);
// Narration in a native window is the Recorder hearing the person, and it asks nobody first.
// The hosted browser has no microphone; the grant costs nothing there and keeps one launch.
await step("microphone", () =>
session.send("Browser.grantPermissions", {
origin: `chrome-extension://${recorderId}`,
permissions: ["audioCapture"],
}),
);
if (window === "native") {
/*
* Downloads where the person's browser puts them. A browser Playwright launches sends every
* download to a temporary folder of its own under a random name and deletes it when the
* browser closes, which is right for a test and loses a person's files. Attached with
* `noDefaults`, Playwright leaves downloads alone; they are handed back here all the same, so
* no change in Playwright's defaults can take a person's files. The browser's own behaviour is
* the Downloads folder its settings name.
*/
await step("downloads", () =>
session.send("Browser.setDownloadBehavior", { behavior: "default" }),
);
}
return {
context,
person: { product, window, recorderId, launcher, session },
};
} catch (error) {
await context.close().catch(() => undefined);
throw error;
}
}