Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import { z } from "zod";
/**
* THE OPENBOT BROWSER: a real browser a person opens from OpenBot and uses as their own, with the OpenBot Recorder
* already in it and already connected. This file is the contract between the four places that build it: the computer
* that launches the browser (`agent-computer`), the server that opens it for a signed-in person, the Recorder that
* records in it, and the app's Browser page. Design: Notion "OpenBot Browser: BUILD (2026-10-21)".
*
* One browser per person. It is a computer named for that person, so every way of reaching a computer that gives it a
* computer of its own (a supervisor container, a Kubernetes Sandbox, or the host-side local Chrome computer on OpenBot
* Desktop) carries it unchanged: the id goes in `x-openbot-bot-id` as a Bot's does. Never the one computer every Bot
* shares: a Bot there runs a shell beside the browser's profile, and the profile holds what the person signed in to by
* hand. The server refuses it there, and so does the computer (`PEOPLE_ONLY_ENV`, `COMPUTER_BOT_ID`).
*/
export const PERSON_BROWSER_PREFIX = "person-";
/** A person's browser id: the prefix and 32 lowercase hex digits the server derives from the OpenBot user id. */
const PERSON_BROWSER_ID = /^person-[0-9a-f]{32}$/;
export function isPersonBrowserId(id: string): boolean {
return PERSON_BROWSER_ID.test(id);
}
/**
* Where the browser shows. `native`: a window on the person's own desktop (OpenBot Desktop, the person's installed
* Chrome or Edge). `screen`: a cloud browser the app shows through the live screen (hosted OpenBot, bundled Chromium).
*/
export type BrowserWindow = "native" | "screen";
/** The person's own everyday browsers OpenBot Desktop can put the Recorder in and bring forward. */
export type RealBrowser = "chrome" | "msedge";
export type BrowserProduct = "chrome" | "msedge" | "chromium";
/**
* Who hears the person. `own`: the Recorder transcribes the person's microphone itself, as it does in any browser
* (native). `external`: the browser has no microphone (screen), so the app transcribes the person in their own browser
* and each final phrase reaches the Recorder through `narrate`.
*/
export type Narration = "own" | "external";
/* THE RECORDER'S LAUNCHER API.
*
* What the process that launched the browser, and alone holds its DevTools pipe, calls inside the Recorder's service
* worker as `globalThis[LAUNCHER_GLOBAL]`. No web page and no other extension can reach it: there is no message route
* to it from outside the Recorder. Every method resolves; a refusal is `{ ok: false, error }`, never a throw. */
export const LAUNCHER_GLOBAL = "openbotLauncher";
export type LauncherConnection = {
/** The OpenBot this browser belongs to: an https origin, or http on a loopback IP with a port (OpenBot Desktop). */
openbotUrl: string;
/** A Recorder bearer (`obr_…`) the server minted for this person and this browser. */
token: string;
connectionId: string;
user: { id: string; name: string; email: string };
};
export type LauncherState = {
connected: boolean;
/** The connected person, when there is one. */
userId?: string;
/**
* The connection it holds (`LauncherConnection.connectionId`), when there is one: OpenBot keeps a connection that
* stands for this browser rather than making another each time the browser is opened.
*/
connectionId?: string;
phase: "idle" | "starting" | "recording" | "stopping";
recordingId?: string;
title?: string;
startedAt?: number;
/** The person's actions so far in this recording, as the Recorder's badge counts them. */
actions: number;
narration: Narration;
/**
* Whether the Recorder is hearing the person itself, for narration `own`: `listening` while a recording runs and the
* microphone is live, `muted` when the person muted it, `unavailable` when it cannot hear (no permission, no device,
* the recogniser failed), `off` when nothing is being recorded. Absent for narration `external`.
*/
voice?: "off" | "listening" | "muted" | "unavailable";
/**
* The recording Stop ended last, so OpenBot can confirm it went to the person's Learning space: how many of the
* person's actions it held (as the badge counted them) and how many narrated phrases. Both 0: nothing was recorded.
*/
lastStopped?: {
recordingId: string;
title: string;
stoppedAt: number;
actions: number;
phrases: number;
/**
* True while some of this recording is still on its way to Intelligence after Stop. Absent once all of it is sent,
* or once what is left is kept for later (held by Intelligence's refusal, or waiting for its person to connect this
* browser again) or discarded. Whoever disconnects the Recorder after a Stop waits for it, because a disconnect
* keeps what is not sent yet for the person's next connection here instead of sending it now.
*/
sending?: boolean;
};
};
export type LauncherResult<T = Record<never, never>> =
| ({ ok: true } & T)
| { ok: false; error: string };
export type LauncherApi = {
/** Store this connection (replacing any other) and take narration from `narration`. Idempotent. */
connect(input: {
connection: LauncherConnection;
narration: Narration;
}): Promise<LauncherResult>;
/** Start recording the tab the person is on (or `tabId`), named `title`; or stop the live recording. */
record(
input:
| {
action: "start";
title: string;
tabId?: number;
language?: string;
narrate?: boolean;
}
| { action: "stop" },
): Promise<LauncherResult<{ state: LauncherState }>>;
/** One final phrase the app heard, with when the person started and stopped saying it (epoch ms). */
narrate(input: {
text: string;
startedAt: number;
endedAt: number;
}): Promise<LauncherResult>;
/**
* The app heard the person start a phrase at `since` (epoch ms), or stop speaking (`null`). While a phrase is being
* spoken the Recorder holds the steps taken from `since` on until that phrase's `narrate` arrives (at most 15 s), as
* it does for its own recogniser, so a phrase is placed before the steps it was spoken over.
*/
speaking(input: { since: number | null }): Promise<LauncherResult>;
/**
* OpenBot revoked this browser's connection (the person disconnected it in Settings, Browsers): forget it now and
* report signed out, as the Recorder does when OpenBot answers that its token is gone. A recording in progress ends
* the way a sign-out ends one. Idempotent.
*/
disconnect(): Promise<LauncherResult>;
state(): Promise<LauncherState>;
};
/* THE COMPUTER'S ROUTES (agent-computer), under the computer token like every other route, with the person's browser
* id in `x-openbot-bot-id`. A non-person id is refused with 400. */
export const browserComputerPaths = {
open: "/browser/open",
state: "/browser/state",
recording: "/browser/recording",
narration: "/browser/narration",
/** POST, no body: the Recorder forgets its connection now (`LauncherApi.disconnect`). Answers `BrowserState`. */
disconnect: "/browser/disconnect",
tabs: "/browser/tabs",
navigate: "/browser/navigate",
close: "/browser/close",
/**
* POST, no body: wipe the person's browser, open or closed, as a Bot's is wiped (`shared/bot-browser.ts`, CLEARING A
* BROWSER), and leave it closed: only the server can hand its Recorder a connection again. Answers `BrowserState`.
*/
clear: "/browser/clear",
} as const;
/**
* An open. With `connection`, the Recorder is handed it. Without one, the browser opens (or comes forward) and its
* Recorder keeps the connection it holds, which the answer's `recorder` names: OpenBot opens it so when it holds a
* connection that stands for this browser, and hands it a new one only when the Recorder turns out not to hold that
* one. Minting one at every open revoked the last each time, and wrote a `recorder.revoked` and a `recorder.connected`
* row for every visit to the Browser page.
*/
export const browserOpenRequestSchema = z
.object({
connection: z
.object({
openbotUrl: z.string().url().max(2000),
token: z.string().min(1).max(200),
connectionId: z.string().min(1).max(200),
user: z
.object({
id: z.string().min(1).max(200),
name: z.string().max(200),
email: z.string().max(320),
})
.strict(),
})
.strict()
.optional(),
})
.strict();
export type BrowserOpenRequest = z.infer<typeof browserOpenRequestSchema>;
export type BrowserTab = {
id: string;
url: string;
title: string;
active: boolean;
};
export type BrowserState = {
open: boolean;
browser?: BrowserProduct;
window?: BrowserWindow;
/** The Recorder's own report; absent while the browser is closed or the Recorder has not answered yet. */
recorder?: LauncherState;
tabs: BrowserTab[];
};
export const browserRecordingRequestSchema = z.discriminatedUnion("action", [
z
.object({
action: z.literal("start"),
title: z.string().trim().min(1).max(200),
language: z.string().max(35).optional(),
/** Whether the person narrates this recording, as OpenBot's Record form asked them: the Recorder opens the
* microphone only when it is true. Absent: the choice made in the Recorder itself, and none is no. */
narrate: z.boolean().optional(),
})
.strict(),
z.object({ action: z.literal("stop") }).strict(),
]);
export type BrowserRecordingRequest = z.infer<
typeof browserRecordingRequestSchema
>;
/**
* A final phrase, or the person starting (`speaking: true`) or stopping (`speaking: false`) a phrase. `sentAt` is the
* sender's clock when it sent this: the page's times come from the person's computer, so the server shifts every time by
* (its own clock on arrival minus `sentAt`) before forwarding, and the Recorder sees them on the server's clock, which is
* the clock its steps are on. The computer receives the corrected times and a `sentAt` of the server's own clock.
*/
const epochMs = z.number().int().positive();
export const browserNarrationRequestSchema = z.union([
z
.object({
text: z.string().trim().min(1).max(4000),
startedAt: epochMs,
endedAt: epochMs,
sentAt: epochMs,
})
.strict(),
z
.object({ speaking: z.literal(true), since: epochMs, sentAt: epochMs })
.strict(),
z.object({ speaking: z.literal(false), sentAt: epochMs }).strict(),
]);
export type BrowserNarrationRequest = z.infer<
typeof browserNarrationRequestSchema
>;
export const browserTabsRequestSchema = z.discriminatedUnion("action", [
z
.object({ action: z.literal("new"), url: z.string().max(2000).optional() })
.strict(),
z
.object({ action: z.literal("activate"), tabId: z.string().min(1) })
.strict(),
z.object({ action: z.literal("close"), tabId: z.string().min(1) }).strict(),
]);
export type BrowserTabsRequest = z.infer<typeof browserTabsRequestSchema>;
export const browserNavigateRequestSchema = z.union([
z.object({ url: z.string().trim().min(1).max(2000) }).strict(),
z.object({ action: z.enum(["back", "forward", "reload"]) }).strict(),
]);
export type BrowserNavigateRequest = z.infer<
typeof browserNavigateRequestSchema
>;
/* THE SERVER'S ROUTES, for the signed-in person's own browser only. The app calls these; the server finds the person's
* browser, mints its Recorder connection, and forwards to the computer routes above. */
export const browserApiPaths = {
/** POST: open (or bring forward) the person's browser. Answers `BrowserOpenResponse`. */
open: "/api/browser/open",
/** GET: `BrowserApiState`. */
state: "/api/browser",
/** POST `BrowserRecordingRequest`: answers `BrowserApiState`. */
recording: "/api/browser/recording",
/** POST `BrowserNarrationRequest`. */
narration: "/api/browser/narration",
/** POST `BrowserTabsRequest`: answers `BrowserApiState`. */
tabs: "/api/browser/tabs",
/** POST `BrowserNavigateRequest`: answers `BrowserApiState`. */
navigate: "/api/browser/navigate",
/** POST: close the person's browser. */
close: "/api/browser/close",
/**
* POST, no body: clear the person's browser (`shared/bot-browser.ts`, CLEARING A BROWSER). One that was open opens
* again, fresh. Answers `BrowserApiState`. Audited as `computer.browser_cleared`.
*/
clear: "/api/browser/clear",
} as const;
/* THE PERSON'S REAL BROWSER (OpenBot Desktop). Where the person's own everyday Chrome or Edge can hold the Recorder,
* the Browser button works with that browser instead of a separate one: OpenBot Desktop registers the Recorder with it
* (from the store), the Recorder connects through OpenBot Desktop's native host with no Connect page, and the host-side
* computer brings that browser forward, or opens the Recorder's store page in it while the Recorder is not there yet.
* These computer routes act on the person's desktop and exist only on the local Chrome backend. */
export const realBrowserComputerPaths = {
/** GET: `{ browsers: RealBrowser[] }`, the person's own browsers installed on this computer. Never launches one. */
installed: "/browser/real",
/** POST `{ browser }`: bring the person's own browser forward (launching it if it is not running). */
front: "/browser/real/front",
/** POST `{ browser, url }`: open `url` (the Recorder's store page) in the person's own browser, not the default one. */
store: "/browser/real/store",
} as const;
export const realBrowserRequestSchema = z
.object({
browser: z.enum(["chrome", "msedge"]),
url: z.string().url().max(2000).optional(),
})
.strict();
export type RealBrowserRequest = z.infer<typeof realBrowserRequestSchema>;
/** What `POST /api/browser/open` did on OpenBot Desktop when the person's own browser is the Browser button's. */
export type RealBrowserOpenResponse = {
ok: true;
/** `real`: brought their browser forward, its Recorder connected. `store`: opened the Recorder's store page in it. */
opened: "real" | "store";
browser: RealBrowser;
};
export type BrowserOpenResponse = {
ok: true;
/** The computer id of the person's browser: the live screen streams `/api/computers/<computerId>/stream`. */
computerId: string;
browser: BrowserProduct;
window: BrowserWindow;
narration: Narration;
};
export type BrowserApiState = BrowserState & {
/**
* On OpenBot Desktop: the person's own browsers installed on this computer, and whether a Recorder in one of them is
* connected to this OpenBot as this person (a live connection made through OpenBot Desktop). Absent elsewhere.
*/
real?: { connected: boolean; browsers: RealBrowser[] };
/** What the person's own Recorders are doing, wherever they record (the newest live report): the red dot. */
activity?: import("./recorder-api").RecorderActivity;
/** False when this OpenBot has no way to run a person's browser; the app then hides the Browser button. */
available: boolean;
/**
* Why the person's browser is not the one they would expect, in one plain sentence for them: why this OpenBot cannot
* run one at all (with `available` false), or, on OpenBot Desktop, why it is not their own installed Chrome or Edge.
* Absent when there is nothing to explain.
*/
note?: string;
/**
* The person's browser is there and is not answering, said for them in one plain sentence: nothing of it can be shown
* until it answers again. Absent while it answers.
*/
problem?: string;
/**
* Where this OpenBot shows the person's browser, known before it is opened, so the sidebar's Browser button knows
* what one click does: `native` opens (or brings forward) a window on the person's desktop and leaves them where
* they are; `screen` takes them to the browser in the page. Present whenever `available` is.
*/
shows?: BrowserWindow;
computerId: string;
narration?: Narration;
};
/* CONFIGURATION, by name. */
/** On the computer: the built Recorder (`extension/dist`), absolute. Without it `/browser/open` refuses. */
export const RECORDER_DIR_ENV = "OPENBOT_RECORDER_DIR";
/** On the server: the computer that runs people's browsers here, when it is not the Bots' computer (OpenBot Desktop's
* host-side local Chrome computer, `http://127.0.0.1:4101`). Unset: the normal computer provider locates it. */
export const BROWSER_URL_ENV = "OPENBOT_BROWSER_URL";
/**
* On the computer: `on` when it runs people's own browsers and nothing else (OpenBot Desktop's host-side local Chrome
* computer). Such a computer refuses every Bot, and a computer without it runs a person's browser only when it was
* started for that one browser (`COMPUTER_BOT_ID` names it: a supervisor's container, a Sandbox).
*/
export const PEOPLE_ONLY_ENV = "COMPUTER_PEOPLE_ONLY";
/**
* On the server: why OpenBot Desktop runs no browser of the person's own on this machine, as one of
* {@link BROWSER_ABSENCES}' names. Desktop writes it at every Start where it found none, and takes it out where it
* found one. The server tells the person in the sentence beside it.
*/
export const BROWSER_ABSENT_ENV = "OPENBOT_BROWSER_ABSENT";
/** Each reason OpenBot Desktop can give, and what the person is told. */
export const BROWSER_ABSENCES = {
"no-chrome-or-edge":
"OpenBot can't find Google Chrome or Microsoft Edge on this computer, so it can't open your own browser.",
"flatpak-chrome":
"OpenBot can't open the Flatpak version of Google Chrome, so install Chrome from google.com/chrome to use your own browser.",
"flatpak-edge":
"OpenBot can't open the Flatpak version of Microsoft Edge, so install Edge from microsoft.com/edge to use your own browser.",
"recorder-not-built":
"Your own browser isn't set up yet, so quit OpenBot and open it again.",
"no-home-folder":
"OpenBot can't find your home folder, so it has nowhere to keep your own browser's profile.",
} satisfies Record<string, string>;
export type BrowserAbsence = keyof typeof BROWSER_ABSENCES;
export function isBrowserAbsence(value: string): value is BrowserAbsence {
return Object.hasOwn(BROWSER_ABSENCES, value);
}