Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
/*
* The OpenBot Recorder's launcher API, standing in for the Recorder in agent-computer's tests while
* the Recorder's own is built alongside it. It answers exactly as `LauncherApi` in
* shared/openbot-browser.ts types it, and keeps what it holds in `chrome.storage.local` as the
* Recorder does, so it survives its worker being stopped and started again.
*
* Like the Recorder, it attaches `chrome.debugger` to the tab it records for as long as it records:
* that is what makes a browser show its "started debugging this browser" bar, which the live screen
* must not lose the page to.
*
* What it was called with goes to the OpenBot it was connected to, at `/fixture/calls`, with what it
* can see from inside the browser at that moment (its tabs, the one in front, its microphone
* permission, whether it opened its setup page), so a test reads the Recorder's side of each call
* rather than taking the computer's word for it.
*
* Like the Recorder, it asks OpenBot whether a connection stands before it takes it, and refuses one
* OpenBot says is signed out.
*
* Like the Recorder, the last of a recording reaches Intelligence after Stop answers: for as long as the OpenBot it is
* connected to says at `/fixture/tail` (none when it says nothing), `lastStopped.sending` is true. A disconnect in that
* time keeps the rest for the person's next connection, as the Recorder does, so it is sending no longer; every report
* says whether it was still sending when the call came.
*/
const LAUNCHER_GLOBAL = "openbotLauncher";
/**
* Which build of this script is running, sent with every call. A test copies the fixture, changes this
* line, and reads back which build answered, as a person's browser would run a Recorder rebuilt since.
*/
const BUILD = "fixture";
async function held() {
const stored = await chrome.storage.local.get([
"connection",
"narration",
"recording",
"lastStopped",
"setupOpened",
"sendingUntil",
]);
return {
connection: stored.connection,
narration: stored.narration ?? "own",
recording: stored.recording,
lastStopped: stored.lastStopped,
setupOpened: stored.setupOpened === true,
sendingUntil: stored.sendingUntil ?? 0,
};
}
/** Whether the last of the recording Stop ended is still on its way to Intelligence. */
const sending = (now) =>
Boolean(now.lastStopped) && Date.now() < now.sendingUntil;
/** How long the OpenBot this is connected to says the last of a stopped recording takes to send. */
async function tailMs(connection) {
const answer = await fetch(new URL("/fixture/tail", connection.openbotUrl))
.then((response) => response.json())
.catch(() => ({}));
return typeof answer.ms === "number" ? answer.ms : 0;
}
function stateOf(now) {
const { tabId: _recorded, ...recording } = now.recording ?? {};
return {
connected: Boolean(now.connection),
...(now.connection
? {
userId: now.connection.user.id,
connectionId: now.connection.connectionId,
}
: {}),
phase: now.recording ? "recording" : "idle",
...recording,
actions: 0,
narration: now.narration,
// Hearing the person itself only for `own`: while it records, and not at all otherwise.
...(now.narration === "own"
? { voice: now.recording ? "listening" : "off" }
: {}),
...(now.lastStopped
? {
lastStopped: sending(now)
? { ...now.lastStopped, sending: true }
: now.lastStopped,
}
: {}),
};
}
/**
* The tab the person is on, as the Recorder finds it: in the window they used last, or, in a browser
* on a screen that never had a window in focus, any window's.
*/
async function frontTab() {
const [focused] = await chrome.tabs.query({
active: true,
lastFocusedWindow: true,
});
if (focused) return focused;
const [shown] = await chrome.tabs.query({ active: true });
return shown;
}
async function report(method, input) {
const now = await held();
if (!now.connection) return;
const tabs = await chrome.tabs.query({});
const [front] = await chrome.tabs.query({
active: true,
lastFocusedWindow: true,
});
const microphone = (await navigator.permissions.query({ name: "microphone" }))
.state;
await fetch(new URL("/fixture/calls", now.connection.openbotUrl), {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
method,
input,
tabs: tabs.map((tab) => tab.url ?? tab.pendingUrl ?? ""),
front: front?.url ?? "",
microphone,
setupOpened: now.setupOpened,
sending: sending(now),
build: BUILD,
}),
});
}
/**
* Whether OpenBot says this connection is signed out, which the Recorder asks before it takes one
* (`confirm` in extension/src/background/launcher.ts). Only that answer refuses it: an OpenBot that
* cannot be reached leaves the connection taken, as the Recorder's does.
*/
async function signedOut(connection) {
const me = await fetch(new URL("/api/recorder/me", connection.openbotUrl), {
headers: { authorization: `Bearer ${connection.token}` },
}).catch(() => undefined);
return me?.status === 401;
}
globalThis[LAUNCHER_GLOBAL] = {
async connect(input) {
// The connection it holds already, handed again, is taken at once: the Recorder asks only of a new one.
const now = await held();
const holding =
JSON.stringify(now.connection) === JSON.stringify(input.connection) &&
now.narration === input.narration;
if (!holding && (await signedOut(input.connection))) {
// As the Recorder ends after OpenBot says its connection is gone: signed out, and a recording
// under it stopped.
if (now.recording)
await chrome.debugger
.detach({ tabId: now.recording.tabId })
.catch(() => undefined);
await chrome.storage.local.set({ connection: null, recording: null });
return {
ok: false,
error:
"OpenBot did not accept this browser's connection. Open the browser from OpenBot again.",
};
}
await chrome.storage.local.set({
connection: input.connection,
narration: input.narration,
});
await report("connect", input);
return { ok: true };
},
async record(input) {
const now = await held();
if (!now.connection) return { ok: false, error: "Not connected." };
if (input.action === "start") {
if (now.recording) return { ok: false, error: "Already recording." };
const front = await frontTab();
if (front?.id === undefined)
return { ok: false, error: "There is no tab to record." };
await chrome.debugger.attach({ tabId: front.id }, "1.3");
now.recording = {
recordingId: crypto.randomUUID(),
title: input.title,
startedAt: Date.now(),
tabId: front.id,
};
} else {
if (!now.recording)
return { ok: false, error: "Nothing is being recorded." };
await chrome.debugger
.detach({ tabId: now.recording.tabId })
.catch(() => undefined);
now.lastStopped = {
recordingId: now.recording.recordingId,
title: now.recording.title,
stoppedAt: Date.now(),
};
now.recording = undefined;
now.sendingUntil = Date.now() + (await tailMs(now.connection));
}
await chrome.storage.local.set({
recording: now.recording ?? null,
lastStopped: now.lastStopped ?? null,
sendingUntil: now.sendingUntil,
});
await report("record", input);
return { ok: true, state: stateOf(now) };
},
async narrate(input) {
const now = await held();
if (!now.recording)
return { ok: false, error: "Nothing is being recorded." };
await report("narrate", input);
return { ok: true };
},
async speaking(input) {
const now = await held();
if (!now.recording)
return { ok: false, error: "Nothing is being recorded." };
await report("speaking", input);
return { ok: true };
},
async disconnect() {
const now = await held();
// Said while it still knows where to say it, then forgotten. A recording ends as a sign-out ends
// one: stopped, with nothing kept as the last one Stop ended.
await report("disconnect", undefined);
if (now.recording)
await chrome.debugger
.detach({ tabId: now.recording.tabId })
.catch(() => undefined);
// What had not gone is kept for the person's next connection: nothing is on its way any more.
await chrome.storage.local.set({
connection: null,
recording: null,
sendingUntil: 0,
});
return { ok: true };
},
async state() {
return stateOf(await held());
},
};
// As the Recorder does on install: open its setup page. A browser OpenBot opened must never show it.
chrome.runtime.onInstalled.addListener(async () => {
await chrome.storage.local.set({ setupOpened: true });
await chrome.tabs.create({ url: chrome.runtime.getURL("setup.html") });
});