Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import type {
LearningSpaceState,
RecorderConnectVia,
RecorderMe,
} from "@openbot/shared/recorder-api";
import type { CapturedEvent } from "@openbot/shared/recorder-masking";
import { MAX_TIMESTAMP_MS } from "@openbot/shared/recorder-masking";
import {
RECORDER_CONNECT_STATUS,
type RecorderConnectWait,
type RecorderPageResult,
type RecorderPageState,
recorderPageMessageSchema,
} from "@openbot/shared/recorder-page";
import { z } from "zod";
import { normaliseOpenBotOrigin } from "../shared/origin";
import type { DrainRecord, RecorderNotice } from "../shared/storage";
export const PORT_CONTENT = "recorder:content";
export const PORT_OFFSCREEN = "recorder:offscreen";
export type CaptureStatus = {
intelligence:
| "off"
| "connecting"
| "streaming"
| "retrying"
| "held"
| "parked"; // held / parked: stopped with every event kept (Task 11's table)
intelligenceReason?: string;
events?: { sent: number; pending: number; rejected: number; dropped: number };
network: "off" | "waiting" | "debugger" | "page" | "stopped";
networkReason?: string;
/** The person's narration, transcribed in this browser; absent when the recording has no voice. Replaced whole on
* every update, so a reason or the words heard never outlive the state they belong to. */
voice?: VoiceCapture;
};
/** What the Voice row shows. `heard` is the last words recognised (interim words too), masked as the transcript events
* are; it lives in the popup's state only. `problem` says what went wrong when `state` is failed, so the row can offer
* the fix (the setup page for `microphone`). `reason` is the sentence the row shows: why voice stopped when failed,
* or, while listening, that the browser hears speech but hands back no words for it. */
export type VoiceCapture = {
state: "starting" | "listening" | "muted" | "failed";
reason?: string;
heard?: string;
problem?: VoiceProblem;
/** For `speech-pack`: the language the browser has to download, which the popup's Download asks for. */
language?: string;
};
/** `on-device`: this browser cannot recognise speech on the person's computer at all. `speech-pack`: it can, once the
* narration's language is downloaded, which takes the person's click. `language`: it cannot recognise this language on
* the computer. Recognition never leaves the computer (offscreen/voice.ts), so each of these turns voice off. */
export type VoiceProblem =
| "microphone"
| "no-microphone"
| "no-recognition"
| "on-device"
| "speech-pack"
| "service"
| "network"
| "language";
export type RecorderState = {
phase: "idle" | "starting" | "recording" | "stopping";
recordingId?: string;
title?: string;
tabIds: number[];
startedAt?: number;
/** What the person did so far: clicks, inputs and page loads in a followed tab's top frame. The 50-action notice and
* the badge count these. */
actions: number;
noticeShown: boolean;
/** How many actions there were when the 50-action notice showed: the popup and the page both say that number "or
* more", while the count goes on. */
noticeAt?: number;
/** The 50-action notice is on screen: from when it shows until any choice (in a tab or the popup), Stop, Start over
* or the end of the recording. The popup shows its card from this. */
noticeOpen: boolean;
capture: CaptureStatus;
/** Narrated phrases this recording has kept so far: each final phrase placed among its steps, heard by the
* Recorder itself or passed in by its launcher. Absent while there are none. */
phrases?: number;
/** The recording Stop ended last: the popup says it went to Intelligence, and the launcher tells OpenBot its name,
* when it stopped and what it held (the person's actions as the badge counted them, and its narrated phrases), so
* OpenBot can confirm it too and say when it held nothing. */
lastStopped?: {
recordingId: string;
title?: string;
stoppedAt?: number;
actions?: number;
phrases?: number;
};
draining: number; // stopped recordings still sending
learning?: LearningSpaceState; // the person's own Learning space at Record (Decision 10)
topUrls: Record<string, string>; // followed tab id -> its top frame's current URL
/** Tabs the person used during the recording (switched to or opened) while they showed no website: a new tab page,
* one of the browser's own pages or the Recorder's. Each joins `tabIds` at its first website (recording.ts). */
usedTabIds?: number[];
/** While phase is "stopping": persisted so a restarted service worker finishes the stop (Task 9). */
stopping?: { since: number };
/** Stopped recordings whose events are kept (summed over drains with `kept`), for the popup's Discard offer;
* `waitingFor` names the person whose recordings wait for them to connect this browser again (Decision 19). */
kept?: {
recordings: number;
events: number;
reason: string;
parked: boolean;
waitingFor?: { name: string; recordings: number };
};
error?: string;
};
export const IDLE_STATE: RecorderState = {
phase: "idle",
tabIds: [],
actions: 0,
noticeShown: false,
noticeOpen: false,
draining: 0,
topUrls: {},
capture: { intelligence: "off", network: "off" },
};
const event = z
.object({
type: z.literal("CUSTOM"),
name: z.string().min(1).max(200),
timestamp: z.number().positive().max(MAX_TIMESTAMP_MS),
value: z.unknown(),
})
.strict();
const contentToBackgroundSchema = z.discriminatedUnion("type", [
z
.object({
type: z.literal("content.hello"),
top: z.boolean(),
url: z.string().max(4096),
})
.strict(),
z.object({ type: z.literal("content.event"), event }).strict(),
z
.object({
type: z.literal("content.sensitive"),
digest: z.string().regex(/^[0-9a-f]{64}$/),
})
.strict(),
z
.object({
type: z.literal("content.notice"),
choice: z.enum(["stop", "restart", "continue"]),
})
.strict(),
// The person's click on the recording mark in a tab: leave the site the tab shows out of every recording, or record it
// again. The service worker takes the site from the tab, never from the page.
z.object({ type: z.literal("content.site"), recorded: z.boolean() }).strict(),
]);
export type ContentToBackground = z.infer<typeof contentToBackgroundSchema>;
/**
* `background.capture` says whether a frame captures, and, for a tab the recording follows, what its recording mark
* shows (content/indicator.ts): `site` is the site the tab shows, `excluded` that the person keeps it out of their
* recordings (nothing there is captured), and `narrating` whether the Recorder hears them now (absent where it does not
* hear them itself). Without `site`, no mark: nothing is being recorded in the tab.
*/
export type BackgroundToContent =
| {
type: "background.capture";
active: boolean;
recordingId?: string;
site?: string;
excluded?: boolean;
narrating?: boolean;
}
| { type: "background.notice"; show: boolean; actions: number };
export type StreamSession = {
recordingId: string;
title: string;
openbotUrl: string;
token: string;
trajectoryId: string;
startedAt: number;
allowedIntelligenceUrls: string[] | null;
version: string;
/** The narration's language (BCP 47) and whether the person muted it. Absent: this recording has no voice. */
voice?: { language: string; muted: boolean };
/** Narration the Recorder's launcher passes in (`external`, launcher.ts): no voice here, and each phrase it is handed
* is placed as the document's own recogniser's are, in this language. The document never reads it. */
narration?: { language: string };
/** e2e variant only: the service worker copies storage.local.RECORDER_E2E_HOLD_RETRY_MS here (an offscreen document can use
* no extension API but runtime, so it cannot read storage itself); the offscreen document reads it only inside its e2e branch. */
e2eHoldRetryMs?: number;
};
/** Why a recording ended without the person's Stop, sent to Intelligence with its failed outcome: `disconnected` for a
* Disconnect (the person's own, or the organization's policy moving the Recorder), `signed-out` for a connection OpenBot
* signed out. Events streamed live before it ended are already in Intelligence. */
export type StopFailure = "disconnected" | "signed-out";
export type BackgroundToOffscreen =
| { type: "offscreen.start"; session: StreamSession }
| { type: "offscreen.resume"; session: StreamSession } // the service worker restarted and found no document: same outbox
| { type: "offscreen.event"; event: CapturedEvent; key?: string } // `key`: a held step's (steps.ts), stored once
| {
type: "offscreen.rotate";
recordingId: string;
trajectoryId: string;
/** When Start over began: the new trajectory's recording.started is stamped then, before its first step. */
startedAt?: number;
} // "Start over"
| { type: "offscreen.voice.end"; recordingId: string } // Stop: the phrase in progress goes to the service worker, placed among the steps it still holds, before they are sent; answered with offscreen.voice.ended
| { type: "offscreen.stop"; outcome: "succeeded" } // the person's Stop; idempotent per recording (Task 11)
| { type: "offscreen.stop"; outcome: "failed"; reason: StopFailure } // the browser was disconnected or signed out mid-recording
| {
type: "offscreen.recover";
openbotUrl: string;
token: string;
/** The managed policy's pinned Intelligence addresses when it was sent (null: none pinned); the recovered
* streamers are held to it, since a document opened after a browser start was never told a policy. */
allowedIntelligenceUrls: string[] | null;
/** The extension's version for the recovered trajectories' metadata: an offscreen document has no
* chrome.runtime.getManifest (Chrome gives it only runtime messaging, getURL and id), so the service worker names it. */
version: string;
drains: DrainRecord[];
} // after a browser restart, and when the person who recorded kept drains connects again
| { type: "offscreen.discard"; recordingIds: string[] } // the person's own "Discard kept events" or Disconnect, or a Record that never went live: clear their outbox
| { type: "offscreen.token"; recordingId: string; token: string } // the live recording's person connected this browser again: its passes use the new token
| { type: "offscreen.halt"; token: string } // that connection is gone (signed out, or moved away by policy): every stream sending with its token stops at once, its events kept
| { type: "offscreen.policy"; allowedIntelligenceUrls: string[] | null } // the managed policy changed: streams to a gateway it no longer allows are parked
| { type: "offscreen.voice"; muted: boolean } // the person muted or unmuted the live recording's voice
| { type: "offscreen.digests"; digests: string[] }; // every sensitive-value digest of the live recording, so a spoken secret is masked too
/** The one offscreen message the service worker parses: it signs a connection out, so it names the bearer the failing
* pass used, and only that connection is signed out (a pass taken with an older token after the person connected again
* signs out nothing). */
const offscreenSignedOutSchema = z
.object({
type: z.literal("offscreen.signed-out"),
reason: z.enum(["revoked", "removed", "organization"]),
token: z.string(),
})
.strict();
export type OffscreenSignedOut = z.infer<typeof offscreenSignedOutSchema>;
export type OffscreenToBackground =
| { type: "offscreen.ready" }
| {
type: "offscreen.status";
capture: Partial<CaptureStatus>;
trajectoryId: string;
}
| {
type: "offscreen.trajectory";
recordingId: string;
trajectoryId: string;
} // a long recording rolled over to another trajectory
| {
type: "offscreen.stopped";
recordingId: string;
trajectoryIds: string[];
/** Final phrases this document put straight into the stream after Stop reached it (the service worker had sent
* its held steps by then), which the service worker did not place and so has not counted. */
phrases?: number;
} // final outcome queued
| { type: "offscreen.done"; recordingId: string; failed?: string } // every event sent, every trajectory finished; `failed`: none of it could be sent, and why
| {
type: "offscreen.kept";
recordingId: string;
reason: string;
parked: boolean;
events: number;
} // a stopped recording's events are kept (Decision 19)
| { type: "offscreen.transcript"; recordingId: string; event: CapturedEvent } // a final phrase, stamped with its start: the service worker places it among the steps it holds
| { type: "offscreen.speaking"; recordingId: string; since: number | null } // when the phrase being spoken began (null: none is): steps from then on wait for its transcript
| { type: "offscreen.voice.ended"; recordingId: string } // the answer to offscreen.voice.end: every phrase was sent before it
| { type: "offscreen.held"; recordingId: string } // the steps a recovered drain carried (`held`) are in the outbox, before its outcome
| OffscreenSignedOut; // a pass came back RECORDER_SIGNED_OUT, with the guard's reason; its events stay kept (Decision 19)
/** The connect page's Connect or Cancel. `origin` is the one the page showed (its page.connect.pending answer), in the
* form a pending connect stores (normaliseOpenBotOrigin's): the service worker refuses a decide whose origin is not the
* pending connect's, so a connect tab only ever confirms the site it shows. */
const pageConnectDecideSchema = z
.object({
type: z.literal("page.connect.decide"),
approve: z.boolean(),
label: z.string(),
origin: z.string().refine((value) => {
const checked = normaliseOpenBotOrigin(value);
return checked.ok && checked.origin === value;
}),
})
.strict();
type PageConnectDecide = z.infer<typeof pageConnectDecideSchema>;
export type PageToBackground =
| { type: "page.state" }
| { type: "page.learning" }
| {
type: "page.record";
title: string;
tabId: number;
/** The narration's language: the page's own, else the browser's (the service worker's browser language when
* absent or unusable). */
language?: string;
/** Whether the person narrates this recording, when whoever started it asked them (OpenBot's own Record form, by
* the launcher). Absent: the choice they made in the Recorder, and no microphone until they made one. */
narrate?: boolean;
}
| { type: "page.stop" }
| { type: "page.voice"; muted: boolean }
/** The person's choice to narrate their recordings, or not: kept for every later recording. */
| { type: "page.narration"; on: boolean }
/** A site left out of every recording, or recorded again (the options page). */
| { type: "page.site"; site: string; recorded: boolean }
| { type: "page.restart" }
| { type: "page.notice"; choice: "stop" | "restart" | "continue" }
| { type: "page.notice.dismiss" }
| { type: "page.connect.pending" }
| PageConnectDecide
| { type: "page.disconnect" }
| { type: "page.desktop.available" }
| { type: "page.connect.desktop" }
| { type: "page.kept.discard" } // the person discards every stopped recording's kept events
| { type: "RECORDER_E2E_FLOOD"; count: number }; // e2e variant only: the router ignores it in a release build (Task 9), and a type emits no code
export type PageStateResponse = {
ok: true;
state: RecorderState;
connection: {
openbotUrl: string;
user: { name: string; email: string };
via: RecorderConnectVia;
} | null;
me: RecorderMe | null;
policy: { openbotUrl: string | null; invalid: boolean };
notice: RecorderNotice | null;
/** Whether the person narrates their recordings; null until they chose. */
narration: "on" | "off" | null;
/** The sites the person keeps out of every recording, sorted. */
excludedSites: string[];
};
export type PageResponse =
| PageStateResponse
| {
ok: true;
pending: { origin: string; via: "web" | "desktop" } | null;
/** The connection approving it replaces, when this browser is connected: its OpenBot and the person it is for. */
replaces?: { origin: string; email: string };
}
| { ok: true; learning: LearningSpaceState }
| { ok: true; available: boolean }
| { ok: true; connected: true } // OpenBot Desktop connected this browser, with no Connect page
| { ok: true; connected: false; reason: string } // why OpenBot Desktop did not connect this browser, as a code (DesktopConnect)
| { ok: true }
| { ok: false; error: string };
/** An AG-UI observation from the MAIN-world hook (shared/agui.ts PageObservation): raw, as the debugger's are; the
* service worker adds the tab and prefixes a stream id with it. `runtime` says the page's CopilotKit runtime answered its
* `info` call. */
const streamId = z.string().min(1).max(512);
const at = z.number().positive().max(MAX_TIMESTAMP_MS);
const pageAgentObservation = z.discriminatedUnion("kind", [
z
.object({
kind: z.literal("request"),
streamId,
at,
url: z.string().max(8192),
agentId: z.string().max(512).nullable(),
method: z.enum(["agent/run", "agent/connect"]),
input: z.unknown(),
})
.strict(),
z
.object({
kind: z.literal("event"),
streamId,
at,
agentId: z.string().max(512).nullable(),
threadId: z.string().max(512).nullable(),
event: z.looseObject({ type: z.string().min(1).max(100) }),
})
.strict(),
z
.object({
kind: z.literal("end"),
streamId,
at,
outcome: z.enum(["complete", "failed"]),
})
.strict(),
z
.object({
kind: z.literal("runtime"),
at,
url: z.string().max(8192),
})
.strict(),
]);
const externalToBackgroundSchema = z.discriminatedUnion("type", [
z.object({ type: z.literal("openbot.ping") }).strict(),
z
.object({
type: z.literal("openbot.connect"),
code: z.string().startsWith("obc_").max(80),
})
.strict(),
// Where the asking origin's own connect has got to (shared/recorder-page.ts).
z.object({ type: z.literal(RECORDER_CONNECT_STATUS) }).strict(),
// From the MAIN-world fallback (Task 18), masked and trimmed to fit one event in the page; accepted only from a
// recorded tab.
z.object({ type: z.literal("recorder.page-network"), event }).strict(),
// From the same hook: what it saw of AG-UI, raw (the agent events mask it); accepted only from a recorded tab.
z
.object({
type: z.literal("recorder.page-agent"),
observation: pageAgentObservation,
})
.strict(),
// What OpenBot's own page asks the Recorder in the same browser (shared/recorder-page.ts): openbot.state,
// openbot.record and openbot.stop, answered only for the OpenBot this copy is connected to (openbot-page.ts).
...recorderPageMessageSchema.options,
]);
export type ExternalToBackground = z.infer<typeof externalToBackgroundSchema>;
export type ExternalResponse =
| { ok: true } // a ping from any origin this copy is not connected to (Decision 22)
// A connect waiting for the person's click on the Connect page. Nothing about this copy goes with it: any site can
// ask to connect, and it learns this copy's version and its install id for it only once the person approves.
| { ok: true; awaitingConfirmation: true }
// Where the asking origin's own connect has got to: nothing about this copy goes with it either.
| { ok: true; connect: RecorderConnectWait }
| {
ok: true;
version: string;
installId: string;
/** A trusted OpenBot's connect (connection.ts `trustedConnectRule`): connected now, with no Connect page. */
connected?: true;
} // the connected origin's ping, and a trusted OpenBot's connect; installId = installIdFor(sender.origin)
| {
ok: false;
error:
| "not-https"
| "pinned-elsewhere"
| "policy-invalid"
| "busy"
| "invalid"
| "not-recording"
// A trusted OpenBot's code that OpenBot refused, or an OpenBot that did not answer its exchange.
| "exchange-refused"
| "openbot-unreachable";
}
| { ok: false; error: "throttled"; retryAfterMs: number }; // too many connects from this origin lately, and how long until it may ask again
/** Every answer onMessageExternal gives: openbot.state's is a RecorderPageState, openbot.record's and openbot.stop's a
* RecorderPageResult. */
export type ExternalAnswer =
| ExternalResponse
| RecorderPageState
| RecorderPageResult;
const MAX_MESSAGE_BYTES = 64 * 1024;
/** An AG-UI observation is not cut in the page: a run's input carries its whole Thread, a snapshot the agent's state. */
const MAX_AGENT_MESSAGE_BYTES = 8 * 1024 * 1024;
function jsonBytes(raw: unknown): number | null {
try {
const text = JSON.stringify(raw);
return text === undefined ? null : new TextEncoder().encode(text).length;
} catch {
return null;
}
}
/** safeParse + 64 KiB JSON size cap. */
export function parseContentMessage(raw: unknown): ContentToBackground | null {
const bytes = jsonBytes(raw);
if (bytes === null || bytes > MAX_MESSAGE_BYTES) return null;
const parsed = contentToBackgroundSchema.safeParse(raw);
return parsed.success ? parsed.data : null;
}
/** safeParse + 64 KiB JSON size cap: a recorder.page-network event arrives already trimmed to fit one Intelligence
* event (16 KiB), so nothing larger is ever a real one. A recorder.page-agent observation is raw, up to 8 MiB. */
export function parseExternalMessage(
raw: unknown,
): ExternalToBackground | null {
const bytes = jsonBytes(raw);
const agent =
typeof raw === "object" &&
raw !== null &&
(raw as { type?: unknown }).type === "recorder.page-agent";
if (
bytes === null ||
bytes > (agent ? MAX_AGENT_MESSAGE_BYTES : MAX_MESSAGE_BYTES)
)
return null;
const parsed = externalToBackgroundSchema.safeParse(raw);
return parsed.success ? parsed.data : null;
}
/** The one page message the router parses, as it carries the origin the decide handler checks. */
export function parsePageConnectDecide(raw: unknown): PageConnectDecide | null {
const parsed = pageConnectDecideSchema.safeParse(raw);
return parsed.success ? parsed.data : null;
}
/** A sign-out without the token it is about is refused: forgetting without one would forget whichever connection is
* stored. */
export function parseOffscreenSignedOut(
raw: unknown,
): OffscreenSignedOut | null {
const parsed = offscreenSignedOutSchema.safeParse(raw);
return parsed.success ? parsed.data : null;
}