Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import type { Narration } from "@openbot/shared/openbot-browser";
import type {
LearningSpaceState,
RecorderConnectVia,
RecorderMe,
RecorderUser,
} from "@openbot/shared/recorder-api";
import type { CapturedEvent } from "@openbot/shared/recorder-masking";
import type { AgentSnapshot } from "../background/agent/events";
import type { StepSnapshot } from "../background/steps";
import type { RecorderState, StreamSession } from "../protocol/messages";
import { normaliseSites } from "./sites";
export type ConnectionRecord = {
openbotUrl: string;
token: string;
connectionId: string;
user: RecorderUser;
via: RecorderConnectVia;
connectedAt: number;
/** Who hears the person, for a connection the Recorder's launcher made (launcher.ts): `external` when the browser has
* no microphone and OpenBot passes each phrase in. Absent: the Recorder hears the microphone itself, as always. */
narration?: Narration;
};
/** What a stopped recording's streamer reported when it stopped with events kept (Decision 19, Task 11's offscreen.kept),
* or, with `waitingForReconnect`, that the connection was signed out and the recording waits for its person to connect again. */
export type KeptEvents = {
reason: string;
parked: boolean;
events: number;
at: number;
waitingForReconnect?: boolean;
};
/** A recording whose events may still need sending to Intelligence. A recording exists only in this browser and in
* Intelligence: `recordingId` is the Recorder's own id for it, and `trajectoryIds` its trajectories in Intelligence,
* each in its person's own Learning space. */
export type DrainRecord = {
recordingId: string;
title: string;
/** The OpenBot that hands out its capture passes. */
openbotUrl: string;
/** The person who recorded it: its events are sent only while that person is connected to `openbotUrl` (Decision 19). */
userId: string;
userName: string;
trajectoryIds: string[];
startedAt: number;
/** The person chose to discard it (Discard kept events, Disconnect): nothing of it is ever sent, and a restart that
* interrupted the clearing finishes it. */
discarding?: boolean;
/** Set while its events are kept after a stop (hold) or stop (park) row, or while it waits for its person to connect
* again; cleared on offscreen.done. */
kept?: KeptEvents;
/** The steps its service worker still held when the browser quit (`heldSteps`), as they are sent: the document that
* recovers it puts them in its outbox before the interrupted outcome, and says so (offscreen.held). */
held?: HeldEvent[];
};
/** A held step as it is sent, with its key (steps.ts `Pending.key`): the document stores a key once, so a step that left
* before the browser quit, which the held steps in storage may still list, is never sent again. */
export type HeldEvent = { event: CapturedEvent; key?: string };
/** The steps the live recording's service worker holds (steps.ts), written as they change: a service worker that
* restarts takes them up again, and after a browser quit they go with the recording's drain. The events are masked
* already, as the outbox stores them. */
export type HeldSteps = {
recordingId: string;
steps: StepSnapshot;
/** What the recording's agent events know that a click's Thread context and the links depend on (agent/events.ts),
* in the same write as the steps, so the two never disagree on what has left: ids and times only. */
agent?: AgentSnapshot;
};
export type RecorderNotice =
| { kind: "interrupted"; at: number }
| {
kind: "signed-out";
at: number;
reason?: "revoked" | "removed" | "organization";
}
| { kind: "policy-moved"; at: number; origin: string };
type LocalState = {
/** Random per installed copy (each browser and profile has its own), written on install or first use (Decision 18). */
installSecret?: string;
connection?: ConnectionRecord;
activeRecording?: DrainRecord; // the recording in progress; moved into `drains` at Stop, Start over and on recovery
drains?: DrainRecord[]; // stopped recordings still sending; removed on offscreen.done
heldSteps?: HeldSteps; // the live recording's held steps and agent context; absent while it holds neither
notice?: RecorderNotice;
/** When the organization-session keep-alive last reached OpenBot (Decision 2, Task 8). */
lastKeepAliveAt?: number;
/** Whether the person narrates their recordings, as they chose on the setup page or in the Record form: the Recorder
* opens the microphone only when it is `on`. Absent until they chose, which leaves it closed. */
narration?: "on" | "off";
/** The sites the person keeps out of every recording (sites.ts), sorted: nothing in a tab on one is recorded. */
excludedSites?: string[];
};
export type PendingConnect = {
origin: string;
code: string;
/** Only a page or OpenBot Desktop asks to connect; the launcher's connection is never pending. */
via: "web" | "desktop";
requestedAt: number;
tabId?: number;
};
type SessionState = {
recorder: RecorderState;
pendingConnect?: PendingConnect;
stream?: StreamSession; // so a restarted service worker can resume the offscreen document
me?: { value: RecorderMe; fetchedAt: number };
/** The person's own Learning space as OpenBot last described it to the popup, and the connection it was asked with:
* Record copies it into the recording's state. Chrome stops a service worker idle for 30 s, so the one that answers
* Record is often not the one the popup asked; it takes this up for the same connection only. */
learning?: { value: LearningSpaceState; connectionId: string };
/** Sensitive-value digests of the recording in progress. Memory only (storage.session is never written to disk
* and is not readable by content scripts by default), so they survive a service-worker restart and vanish with
* the browser; cleared at Stop. Never written to storage.local or IndexedDB. */
digests?: string[];
/** The digests of what the person typed into sensitive fields in each followed tab (by tab id), latest last: a save
* from that tab whose body held one carries their values. Kept as `digests` are, and cleared with them. */
typedSecrets?: [number, string[]][];
/** openbot.connect requests per origin in the last 10 minutes (timestamps), for Decision 22's limit (Task 8). */
connectRequests?: Record<string, number[]>;
/** The last web connect the person cancelled on the Connect page, so the page that asked hears so
* (`openbot.connect.status`); gone once that origin asks to connect again. */
cancelledConnect?: { origin: string; at: number };
/** The setup tab the install opened (setup.ts), so the launcher's connection can close it: a browser OpenBot opened
* has its microphone granted already and is never shown a setup step. */
setupTabId?: number;
};
const isString = (value: unknown): value is string => typeof value === "string";
function isKeptEvents(value: unknown): value is KeptEvents {
if (typeof value !== "object" || value === null) return false;
const kept = value as Record<string, unknown>;
return (
isString(kept.reason) &&
typeof kept.parked === "boolean" &&
typeof kept.events === "number" &&
typeof kept.at === "number" &&
(kept.waitingForReconnect === undefined ||
typeof kept.waitingForReconnect === "boolean")
);
}
/** A recording record in this build's shape. A build from before the Recorder sent to Intelligence only stored its
* recordings by demonstration, which nothing can send any more. */
export function isDrainRecord(value: unknown): value is DrainRecord {
if (typeof value !== "object" || value === null) return false;
const drain = value as Record<string, unknown>;
return (
isString(drain.recordingId) &&
isString(drain.title) &&
isString(drain.openbotUrl) &&
isString(drain.userId) &&
isString(drain.userName) &&
Array.isArray(drain.trajectoryIds) &&
drain.trajectoryIds.every(isString) &&
typeof drain.startedAt === "number" &&
(drain.discarding === undefined || typeof drain.discarding === "boolean") &&
(drain.kept === undefined || isKeptEvents(drain.kept)) &&
(drain.held === undefined ||
(Array.isArray(drain.held) && drain.held.every(isHeldEvent)))
);
}
function isEvent(value: unknown): value is CapturedEvent {
if (typeof value !== "object" || value === null) return false;
const event = value as Record<string, unknown>;
return (
event.type === "CUSTOM" &&
isString(event.name) &&
typeof event.timestamp === "number"
);
}
function isHeldEvent(value: unknown): value is HeldEvent {
if (typeof value !== "object" || value === null) return false;
const held = value as Record<string, unknown>;
return isEvent(held.event) && (held.key === undefined || isString(held.key));
}
function isAgentSnapshot(value: unknown): value is AgentSnapshot {
if (typeof value !== "object" || value === null) return false;
const snapshot = value as Record<string, unknown>;
return Array.isArray(snapshot.tabs) && Array.isArray(snapshot.recordedRuns);
}
function isHeldSteps(value: unknown): value is HeldSteps {
if (typeof value !== "object" || value === null) return false;
const held = value as Record<string, unknown>;
const steps = held.steps as Record<string, unknown> | null | undefined;
return (
isString(held.recordingId) &&
typeof steps === "object" &&
steps !== null &&
Array.isArray(steps.queue) &&
steps.queue.every(
(step) =>
typeof step === "object" &&
step !== null &&
isEvent((step as Record<string, unknown>).event),
) &&
Array.isArray(steps.calls) &&
Array.isArray(steps.typed) &&
(held.agent === undefined || isAgentSnapshot(held.agent))
);
}
/** The recordings in storage.local are read through a shape check: one of another shape is left out (and goes with
* the next write of the list), never handed to the code that sends recordings. */
function checked<K extends keyof LocalState>(
key: K,
value: unknown,
): LocalState[K] {
if (value === undefined) return undefined as LocalState[K];
if (key === "drains")
return (
Array.isArray(value) ? value.filter(isDrainRecord) : []
) as LocalState[K];
if (key === "activeRecording")
return (isDrainRecord(value) ? value : undefined) as LocalState[K];
if (key === "heldSteps")
return (isHeldSteps(value) ? value : undefined) as LocalState[K];
if (key === "narration")
return (
value === "on" || value === "off" ? value : undefined
) as LocalState[K];
if (key === "excludedSites") return normaliseSites(value) as LocalState[K];
return value as LocalState[K];
}
export async function readLocal<K extends keyof LocalState>(
key: K,
): Promise<LocalState[K]> {
const items = await chrome.storage.local.get(key);
return checked(key, items[key]);
}
export async function writeLocal<K extends keyof LocalState>(
key: K,
value: NonNullable<LocalState[K]>,
): Promise<void> {
await chrome.storage.local.set({ [key]: value });
}
export async function removeLocal(key: keyof LocalState): Promise<void> {
await chrome.storage.local.remove(key);
}
export async function readSession<K extends keyof SessionState>(
key: K,
): Promise<SessionState[K] | undefined> {
const items = await chrome.storage.session.get(key);
return items[key] as SessionState[K] | undefined;
}
export async function writeSession<K extends keyof SessionState>(
key: K,
value: NonNullable<SessionState[K]>,
): Promise<void> {
await chrome.storage.session.set({ [key]: value });
}
export async function removeSession(key: keyof SessionState): Promise<void> {
await chrome.storage.session.remove(key);
}
/**
* Keeps the bearer token out of content scripts: storage.local defaults to readable by them.
*
* Never rejects, as nothing the service worker starts at its top level may (recovery.ts says the same of recovery): it is
* called there with nobody waiting on it, where a rejection was an unhandled one, said nowhere the Recorder's own
* failures are. A failure is logged as theirs are, and the restriction is asked for again at the next start of the
* service worker, which Chrome stops and starts many times a day.
*/
export async function restrictStorageToTrustedContexts(): Promise<void> {
try {
await chrome.storage.local.setAccessLevel({
accessLevel: "TRUSTED_CONTEXTS",
});
} catch (error) {
console.error(
"Recorder: keeping its stored connection from content scripts failed; it asks again at its next start",
error,
);
}
}
/** Reads `installSecret`, creating it with crypto.randomUUID() when absent. */
let pendingSecret: Promise<string> | null = null;
/** Concurrent callers share one in-flight read-or-create, so two first calls never mint two secrets;
* the share clears on settle, so later calls read storage again and a failed write can retry. */
export function installSecret(): Promise<string> {
pendingSecret ??= (async () => {
const existing = await readLocal("installSecret");
if (existing) return existing;
const created = crypto.randomUUID();
await writeLocal("installSecret", created);
return created;
})().finally(() => {
pendingSecret = null;
});
return pendingSecret;
}
/** This copy's install id for one OpenBot origin: the first 32 hex digits of SHA-256(installSecret + "\n" + origin),
* so it is stable per (copy, origin) and links nothing across sites (Decision 18). */
export async function installIdFor(origin: string): Promise<string> {
const secret = await installSecret();
const digest = await crypto.subtle.digest(
"SHA-256",
new TextEncoder().encode(`${secret}\n${origin}`),
);
const hex = [...new Uint8Array(digest)]
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
return hex.slice(0, 32);
}