Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
// The Recorder's launcher API: what the process that launched this browser, and alone holds its DevTools pipe, calls in
// this service worker as `globalThis[LAUNCHER_GLOBAL]` (shared/openbot-browser.ts). OpenBot opens a person's own
// browser with the Recorder in it, and hands it its connection and its Record, Stop and narration through here.
//
// Reachable only by code running in this service worker itself: the launcher evaluates it over the DevTools protocol
// (Runtime.evaluate in the worker's target). There is no chrome.runtime message route to it, internal or external, so
// no web page, content script or other extension can reach it, and none of the Recorder's own pages does either: each
// of them is its own global scope. A connection taken here is checked as a Connect would check it (the OpenBot address
// rule and the organization's policy), and every command goes through the same paths the popup's own do. A revoke
// OpenBot tells it of goes through the path a signed-out answer from OpenBot takes.
import {
LAUNCHER_GLOBAL,
type LauncherApi,
type LauncherConnection,
type LauncherResult,
type LauncherState,
type Narration,
} from "@openbot/shared/openbot-browser";
import {
joinedLearningContainer,
type RecorderMe,
recorderPaths,
} from "@openbot/shared/recorder-api";
import type { PageResponse, RecorderState } from "../protocol/messages";
import { OpenBotError, openbotJson } from "../shared/openbot-client";
import { normaliseOpenBotOrigin } from "../shared/origin";
import { allowsOpenBot, readPolicy } from "../shared/policy";
import { narrationLanguage } from "../shared/speech-language";
import {
type ConnectionRecord,
readLocal,
writeLocal,
writeSession,
} from "../shared/storage";
import { showBadge } from "./badge";
import {
type ConnectionLoss,
connectFromLauncher,
currentConnection,
} from "./connection";
import type { LauncherSide } from "./recording";
import type { RouterHandlers } from "./router";
/** What the launcher drives: the recording controller the popup drives (recording.ts). */
export type LauncherControls = Pick<RouterHandlers, "page"> &
LauncherSide & {
connected(): Promise<void>;
connectionLost(loss: ConnectionLoss): Promise<void>;
};
type LauncherDeps = {
readPolicy: typeof readPolicy;
/** The tab the person is on: the active tab of the window they used last. */
activeTab: () => Promise<chrome.tabs.Tab | undefined>;
/** The page's own language, read as the popup reads it when Record is pressed. */
pageLanguage: (tabId: number) => Promise<string | null>;
now: () => number;
};
/** How long a connection waits on OpenBot to say who it is for: it is stored either way, and the keep-alive asks again. */
const ME_WAIT_MS = 5_000;
const TITLE_MAX = 200;
const PHRASE_MAX = 4_000;
const SENTENCES = {
invalid: "The Recorder could not read that request.",
address:
"That OpenBot address is not one the Recorder connects to. It must be https, or http on 127.0.0.1 or [::1] with a port.",
policyInvalid:
"Your organization's Recorder policy names an OpenBot address the Recorder cannot use. Ask your IT team.",
pinnedElsewhere:
"Your organization's Recorder policy does not allow this OpenBot. Ask your IT team.",
refused:
"OpenBot did not accept this browser's connection. Open the browser from OpenBot again.",
notConnected: "Connect this browser to OpenBot first.",
noTab: "There is no tab to record. Open one and try again.",
failed: "The Recorder could not do that. Try again.",
} as const;
const refuse = (error: string): { ok: false; error: string } => ({
ok: false,
error,
});
const isRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value);
const isText = (value: unknown, max: number): value is string =>
typeof value === "string" && value.trim() !== "" && value.length <= max;
const isTime = (value: unknown): value is number =>
typeof value === "number" && Number.isFinite(value) && value > 0;
/** The launcher's connection, read as the contract types it: anything else is refused, whatever sent it. */
function connectionFrom(value: unknown): LauncherConnection | null {
if (!isRecord(value)) return null;
const { openbotUrl, token, connectionId, user } = value;
if (
!isText(openbotUrl, 2000) ||
!isText(token, 200) ||
!token.startsWith("obr_") ||
!isText(connectionId, 200) ||
!isRecord(user) ||
!isText(user.id, 200) ||
typeof user.name !== "string" ||
typeof user.email !== "string"
)
return null;
return {
openbotUrl,
token,
connectionId,
user: { id: user.id, name: user.name, email: user.email },
};
}
const NARRATIONS: readonly Narration[] = ["own", "external"];
async function activeTab(): Promise<chrome.tabs.Tab | undefined> {
const [focused] = await chrome.tabs.query({
active: true,
lastFocusedWindow: true,
});
if (focused) return focused;
// A browser on a screen may never have had a window in focus: any window's active tab is the one on show.
const [shown] = await chrome.tabs.query({ active: true });
return shown;
}
async function pageLanguage(tabId: number): Promise<string | null> {
try {
const [frame] = await chrome.scripting.executeScript({
target: { tabId },
func: () => document.documentElement.lang,
});
return typeof frame?.result === "string" && frame.result
? frame.result
: null;
} catch {
return null;
}
}
const defaultDeps: LauncherDeps = {
readPolicy,
activeTab,
pageLanguage,
now: () => Date.now(),
};
/** The same connection, as far as anything the Recorder keeps is concerned: connecting it again changes nothing. */
function sameConnection(
stored: ConnectionRecord | null,
next: ConnectionRecord,
): boolean {
return (
stored !== null &&
stored.via === next.via &&
stored.openbotUrl === next.openbotUrl &&
stored.token === next.token &&
stored.connectionId === next.connectionId &&
stored.user.id === next.user.id &&
stored.user.name === next.user.name &&
stored.user.email === next.user.email &&
(stored.narration ?? "own") === (next.narration ?? "own")
);
}
/**
* Whether the Recorder hears the person itself, from its own recogniser's report (the popup's Voice row): listening
* while the microphone is open or opening, muted as the person set it, unavailable when it failed (no permission, no
* microphone, no speech service, or one it cannot reach just now and tries again), and off while nothing is recorded
* (a Record still starting has opened no microphone yet). The contract has no word for a microphone still opening; it
* is reported as listening, as the popup's Voice row shows it on.
*/
function voiceOf(state: RecorderState): NonNullable<LauncherState["voice"]> {
if (state.phase === "idle") return "off";
switch (state.capture.voice?.state) {
case "starting":
case "listening":
return "listening";
case "muted":
return "muted";
case "failed":
return "unavailable";
case undefined:
return "off";
}
}
/** The recording Stop ended last, by name and with what it held, once it went to Intelligence (a Disconnect or a
* failed send leaves none), and not when the person's own Learning space is known not to take it: OpenBot confirms it
* went there, or says it held nothing. One stopped by an earlier build, which did not count, is not reported: OpenBot
* would say it held nothing. OpenBot's own page reads it the same way (openbot-page.ts). */
export function lastStopped(
state: RecorderState,
): LauncherState["lastStopped"] {
const stopped = state.lastStopped;
if (
stopped?.title === undefined ||
stopped.stoppedAt === undefined ||
stopped.actions === undefined ||
stopped.phrases === undefined
)
return undefined;
if (state.learning && joinedLearningContainer(state.learning) === null)
return undefined;
return {
recordingId: stopped.recordingId,
title: stopped.title,
stoppedAt: stopped.stoppedAt,
actions: stopped.actions,
phrases: stopped.phrases,
};
}
/**
* Whether some of a stopped recording is still on its way to Intelligence: its drain is still in storage (it goes on
* offscreen.done, once all of it is sent), and it is neither kept for later (a hold or park row, or waiting for its
* person to connect again) nor being discarded.
*/
async function stillSending(recordingId: string): Promise<boolean> {
const drains = (await readLocal("drains")) ?? [];
return drains.some(
(drain) =>
drain.recordingId === recordingId &&
!drain.discarding &&
drain.kept === undefined,
);
}
export function createLauncher(
controls: LauncherControls,
overrides: Partial<LauncherDeps> = {},
): LauncherApi {
const deps: LauncherDeps = { ...defaultDeps, ...overrides };
async function launcherState(): Promise<LauncherState> {
const connection = await currentConnection();
const state: RecorderState = await controls.current();
const narration = connection?.narration ?? "own";
const stopped = lastStopped(state);
const sending =
stopped !== undefined && (await stillSending(stopped.recordingId));
return {
connected: connection !== null,
...(connection
? { userId: connection.user.id, connectionId: connection.connectionId }
: {}),
phase: state.phase,
...(state.recordingId ? { recordingId: state.recordingId } : {}),
...(state.title ? { title: state.title } : {}),
...(state.startedAt !== undefined ? { startedAt: state.startedAt } : {}),
actions: state.actions,
narration,
...(narration === "own" ? { voice: voiceOf(state) } : {}),
...(stopped
? { lastStopped: sending ? { ...stopped, sending } : stopped }
: {}),
};
}
/** Who OpenBot says a connection is for, as the popup would ask when it opens, asked BEFORE the connection is taken:
* `signed-out` when OpenBot revoked it already, `unknown` when OpenBot cannot answer for now (the connection is taken,
* and the keep-alive asks again). */
async function ask(connection: ConnectionRecord): Promise<
| { kind: "ok"; value: RecorderMe }
| {
kind: "signed-out";
reason: Extract<ConnectionLoss, { kind: "signed-out" }>["reason"];
}
| { kind: "unknown" }
> {
try {
const value = await openbotJson<RecorderMe>(
connection,
recorderPaths.me,
{
signal: AbortSignal.timeout(ME_WAIT_MS),
},
);
return { kind: "ok", value };
} catch (error) {
if (error instanceof OpenBotError && error.signedOut)
return { kind: "signed-out", reason: error.body?.reason ?? "revoked" };
return { kind: "unknown" };
}
}
/** Every method resolves, and a refusal is `{ ok: false }`: whatever goes wrong inside is said as a sentence, never
* thrown back down the DevTools pipe. */
function guarded<A extends unknown[], R>(
work: (...args: A) => Promise<R>,
failed: () => R | Promise<R>,
): (...args: A) => Promise<R> {
return async (...args) => {
try {
return await work(...args);
} catch (error) {
console.error("Recorder: the launcher's request failed", error);
return failed();
}
};
}
const answer = async (
response: PageResponse,
): Promise<LauncherResult<{ state: LauncherState }>> =>
response.ok
? { ok: true, state: await launcherState() }
: refuse(response.error);
return {
connect: guarded(
async (input: unknown): Promise<LauncherResult> => {
if (!isRecord(input)) return refuse(SENTENCES.invalid);
const connection = connectionFrom(input.connection);
const narration = input.narration;
if (!connection || !NARRATIONS.includes(narration as Narration))
return refuse(SENTENCES.invalid);
const checked = normaliseOpenBotOrigin(connection.openbotUrl);
if (!checked.ok) return refuse(SENTENCES.address);
const policy = await deps.readPolicy();
if (policy.openbotUrlInvalid) return refuse(SENTENCES.policyInvalid);
if (!allowsOpenBot(policy, checked.origin))
return refuse(SENTENCES.pinnedElsewhere);
const record: ConnectionRecord = {
openbotUrl: checked.origin,
token: connection.token,
connectionId: connection.connectionId,
user: connection.user,
via: "openbot-browser",
connectedAt: deps.now(),
narration: narration as Narration,
};
const current = await currentConnection();
if (sameConnection(current, record)) return { ok: true };
/*
* Asked before it is taken. Two opens of one browser can cross, and the older one's connection, revoked by the
* newer one's, can arrive last: taking it first would sign out the connection that stands and stop its
* recording. A revoked connection is refused and the one held stays; with none held (or this one), it ends as
* OpenBot's own signed-out answer does, with the notice.
*/
const asked = await ask(record);
if (asked.kind === "signed-out") {
if (current && current.token !== record.token)
return refuse(SENTENCES.refused);
// Taken and lost at once, so the popup says why this browser is not connected.
await connectFromLauncher(record, { readPolicy: deps.readPolicy });
await controls.connectionLost({
kind: "signed-out",
reason: asked.reason,
connection: record,
});
return refuse(SENTENCES.refused);
}
if (
!(await connectFromLauncher(record, { readPolicy: deps.readPolicy }))
)
return refuse(SENTENCES.pinnedElsewhere);
if (asked.kind === "ok") {
await writeSession("me", {
value: asked.value,
fetchedAt: deps.now(),
});
await writeLocal("lastKeepAliveAt", deps.now());
}
// What an approved Connect does next: a recording of this person goes on under the new token, and their kept
// recordings are sent.
await controls.connected();
await showBadge(await controls.current());
return { ok: true };
},
() => refuse(SENTENCES.failed),
),
record: guarded(
async (
input: unknown,
): Promise<LauncherResult<{ state: LauncherState }>> => {
if (!isRecord(input)) return refuse(SENTENCES.invalid);
if (input.action === "stop")
return answer(await controls.page({ type: "page.stop" }, {}));
if (input.action !== "start" || !isText(input.title, TITLE_MAX))
return refuse(SENTENCES.invalid);
if (
input.tabId !== undefined &&
!(typeof input.tabId === "number" && Number.isInteger(input.tabId))
)
return refuse(SENTENCES.invalid);
if (input.narrate !== undefined && typeof input.narrate !== "boolean")
return refuse(SENTENCES.invalid);
if (!(await currentConnection())) return refuse(SENTENCES.notConnected);
const tabId =
typeof input.tabId === "number"
? input.tabId
: (await deps.activeTab())?.id;
if (tabId === undefined) return refuse(SENTENCES.noTab);
// The person's own language when the launcher hears them, else the page's, as the popup reads it.
const language =
typeof input.language === "string" && input.language.trim()
? input.language
: narrationLanguage(
await deps.pageLanguage(tabId),
navigator.language,
);
return answer(
await controls.page(
{
type: "page.record",
title: input.title.trim(),
tabId,
language,
// Whether the person narrates, as OpenBot's own Record form asked them.
...(typeof input.narrate === "boolean"
? { narrate: input.narrate }
: {}),
},
{},
),
);
},
() => refuse(SENTENCES.failed),
),
narrate: guarded(
async (input: unknown): Promise<LauncherResult> => {
if (
!isRecord(input) ||
!isText(input.text, PHRASE_MAX) ||
!isTime(input.startedAt) ||
!isTime(input.endedAt) ||
input.endedAt < input.startedAt
)
return refuse(SENTENCES.invalid);
const response = await controls.narrate({
text: input.text,
startedAt: input.startedAt,
endedAt: input.endedAt,
});
return response.ok ? { ok: true } : refuse(response.error);
},
() => refuse(SENTENCES.failed),
),
speaking: guarded(
async (input: unknown): Promise<LauncherResult> => {
if (!isRecord(input) || !(input.since === null || isTime(input.since)))
return refuse(SENTENCES.invalid);
const response = await controls.speaking(input.since);
return response.ok ? { ok: true } : refuse(response.error);
},
() => refuse(SENTENCES.failed),
),
// OpenBot revoked this browser's connection (the person disconnected it in Settings, Browsers) and says so now,
// rather than leaving the Recorder to hear it on its next call to OpenBot, which on a quiet browser is the keep-alive,
// hours away. It goes the way OpenBot answering that the token is gone goes: the connection is forgotten with the
// signed-out notice, a recording under it stops as signed out and is kept for its person, and the next connection
// the launcher hands in sends it. OpenBot has revoked the token already, so it is not asked to again.
disconnect: guarded(
async (): Promise<LauncherResult> => {
const connection = await currentConnection();
if (connection)
await controls.connectionLost({
kind: "signed-out",
reason: "revoked",
connection,
});
return { ok: true };
},
() => refuse(SENTENCES.failed),
),
state: guarded(launcherState, async () => ({
connected: false,
phase: "idle",
actions: 0,
narration: "own",
})),
};
}
/** Puts the launcher API where its launcher reaches it: this service worker's own global scope, and nowhere else. */
export function installLauncher(
controls: LauncherControls,
overrides?: Partial<LauncherDeps>,
): LauncherApi {
const api = createLauncher(controls, overrides);
Object.defineProperty(globalThis, LAUNCHER_GLOBAL, {
value: api,
configurable: true,
enumerable: false,
writable: false,
});
return api;
}