Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
// The AG-UI traffic Chrome's debugger shows in a followed tab, as the agent events read it (agent/observation.ts): each
// agent run a page posts, the events of its SSE or protobuf answer read live as they arrive (Network.streamResourceContent,
// then each Network.dataReceived carries its bytes), the AG-UI events its WebSockets receive, and CopilotKit's runtime
// `info` call when its runtime answers it (the page runs CopilotKit). The request tracker
// (debugger.ts) asks this which requests are runs: a run is never a step's save, nor a write a step waits for.
import {
createChannels,
createHttpStream,
createSocketReader,
isRuntimeInfoAnswer,
matchRun,
matchRuntimeInfoRequest,
type PageObservation,
type RunMatch,
runChannel,
streamFormat,
} from "../../shared/agui";
import type { CaptureObservation } from "../agent/observation";
import type { Debuggee } from "./debugger";
type Send = (
target: Debuggee,
method: string,
params?: Record<string, unknown>,
) => Promise<unknown>;
type Json = Record<string, unknown>;
const isRecord = (value: unknown): value is Json =>
typeof value === "object" && value !== null && !Array.isArray(value);
const string = (value: unknown): string | undefined =>
typeof value === "string" ? value : undefined;
const number = (value: unknown): number | undefined =>
typeof value === "number" && Number.isFinite(value) ? value : undefined;
/** The key the request tracker and this capture share for one request. */
export const requestKey = (target: Debuggee, requestId: string) =>
`${target.tabId}/${target.sessionId ?? ""}/${requestId}`;
function headerValue(headers: unknown, name: string): string | undefined {
if (!isRecord(headers)) return undefined;
for (const [key, value] of Object.entries(headers))
if (key.toLowerCase() === name && typeof value === "string") return value;
return undefined;
}
function bytesOf(base64: string): Uint8Array {
const binary = atob(base64);
const bytes = new Uint8Array(binary.length);
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
return bytes;
}
const encoder = new TextEncoder();
/** A body Chrome hands over, as base64 or as text. */
const bodyBytes = (body: string, base64: boolean) =>
base64 ? bytesOf(body) : encoder.encode(body);
const bodyText = (body: string, base64: boolean) =>
base64 ? new TextDecoder().decode(bytesOf(body)) : body;
/** The resource types an AG-UI run or stream can be: the page's own requests and event sources, never a script,
* a style sheet, an image or a document. */
const FOLLOWED_TYPES = new Set(["Fetch", "XHR", "EventSource", "Other"]);
/** How many verdicts wait for the tracker at most: it asks for each POST it follows, once. */
const MAX_VERDICTS = 512;
/** CDP's monotonic seconds against the wall clock (epoch ms) at one moment, to read later timestamps as wall time. */
type Clock = { monotonic: number; wall: number };
const wallAt = (clock: Clock, monotonic: number | undefined) =>
monotonic === undefined
? Date.now()
: Math.round(clock.wall + (monotonic - clock.monotonic) * 1000);
type Request = {
target: Debuggee;
requestId: string;
url: string;
clock: Clock;
/** A POST whose body Chrome left out of requestWillBeSent: what it is waits for Network.getRequestPostData. */
match: RunMatch | null | "pending";
stream?: ReturnType<typeof createHttpStream>;
/** Everything that happens to the request, in the order Chrome reported it: a body still being read holds what
* came after it. */
work: Promise<void>;
/** The answer's binding, once its headers came: an SSE or protobuf answer is read for events. */
format: "sse" | "proto" | null;
/** live: the answer streams through dataReceived; waiting: streamResourceContent has not answered yet. */
reading: "off" | "waiting" | "live";
/** dataReceived bytes that came while streamResourceContent had not answered, after what it had buffered. */
early: { bytes: Uint8Array; at: number }[];
/** Whether the request is an agent run, for the tracker. */
verdict: Verdict;
/** CopilotKit's runtime `info` call: its answer is read once it finished. */
info: boolean;
/** Followed for its `info` call alone (`infoOnly`): never read as AG-UI or as a run. */
infoOnly: boolean;
/** The answer's status, once its headers came. */
status: number | null;
};
/** Whether a request is an agent run: known at once for most, after its answer's first event for a candidate. */
export type RunVerdict = {
readonly value?: boolean;
readonly promise: Promise<boolean>;
/** Chrome's answer to Network.getRequestPostData for a body it left out of requestWillBeSent: asked once, by the
* AG-UI capture, for the tracker too. */
readonly postData?: Promise<unknown>;
};
type Verdict = {
value?: boolean;
promise: Promise<boolean>;
decide(run: boolean): void;
postData?: Promise<unknown>;
};
function verdict(): Verdict {
let decide: (run: boolean) => void = () => {};
const result: Verdict = {
promise: new Promise<boolean>((resolve) => {
decide = resolve;
}),
decide(run) {
if (result.value !== undefined) return;
result.value = run;
decide(run);
},
};
return result;
}
type Socket = {
tabId: number;
clock?: Clock;
reader: ReturnType<typeof createSocketReader>;
/** Its frames, in order: each waits for the run answers its tab is still reading, which name the channels. */
work: Promise<void>;
};
export type AgentCapture = ReturnType<typeof createAgentCapture>;
/**
* Follows the AG-UI traffic per debuggee from the Network domain's events, and hands each observation to `observe`.
* `ignore` names URLs never followed (OpenBot's own); `infoOnly` names URLs followed only when the request is
* CopilotKit's runtime `info` call, and then only for its answer (OpenBot's own runtime); `onRun` says a request whose
* body was read late is a run, so the tracker stops holding steps for it.
*/
export function createAgentCapture(options: {
send: Send;
observe(observation: CaptureObservation): void;
ignore(url: string): boolean;
infoOnly(url: string): boolean;
onRun(key: string): void;
}) {
const requests = new Map<string, Request>();
const sockets = new Map<string, Socket>();
/** Each tab's Intelligence channels, from its run answers, for the sockets that join them. */
const channels = new Map<number, ReturnType<typeof createChannels>>();
const channelsOf = (tabId: number) => {
let known = channels.get(tabId);
if (!known) {
known = createChannels();
channels.set(tabId, known);
}
return known;
};
/** What the tracker asks about a request after this capture let it go: its verdict, until the tracker had it. */
const verdicts = new Map<string, Verdict>();
/** Each tab's run answers still being read: a socket's frames wait for them, so a channel is known before its events. */
const reading = new Map<number, Promise<unknown>>();
const emitFor =
(tabId: number) =>
(observation: PageObservation): void => {
try {
options.observe({ ...observation, tabId } as CaptureObservation);
} catch (error) {
console.error("Recorder: an agent observation failed", error);
}
};
/** Runs `step` after everything the request had queued, so its observations keep Chrome's order. */
const then = (request: Request, step: () => Promise<void> | void) => {
request.work = request.work.then(step).catch((error) => {
console.error("Recorder: an agent stream could not be read", error);
});
};
/** A post body's text from Chrome's answer to Network.getRequestPostData. */
const postText = (answer: unknown): string | undefined =>
isRecord(answer) && typeof answer.postData === "string"
? bodyText(answer.postData, answer.base64Encoded === true)
: undefined;
function begin(request: Request, match: RunMatch | null, at: number) {
request.match = match;
// Not a run, the request is still followed: an event stream it answers with is AG-UI from its agent.
request.stream = createHttpStream({
streamId: requestKey(request.target, request.requestId),
url: request.url,
match,
at,
emit: emitFor(request.target.tabId),
});
if (match?.kind === "run") request.verdict.decide(true);
else if (match?.kind !== "candidate") request.verdict.decide(false);
}
function onRequest(target: Debuggee, raw: Json): void {
const requestId = string(raw.requestId);
const sent = isRecord(raw.request) ? raw.request : null;
const url = string(sent?.url);
if (requestId === undefined || !sent || url === undefined) return;
const key = requestKey(target, requestId);
// A redirect reuses the request id: the one it lands on is followed from here.
requests.delete(key);
const type = string(raw.type);
if (
options.ignore(url) ||
(type !== undefined && !FOLLOWED_TYPES.has(type))
)
return;
const method = (string(sent.method) ?? "GET").toUpperCase();
const inline = string(sent.postData);
const infoOnly = options.infoOnly(url);
// A URL followed for the info call alone: any other request to it is not followed at all.
if (infoOnly && !matchRuntimeInfoRequest(url, method, inline)) return;
const wallTime = number(raw.wallTime);
const clock: Clock = {
monotonic: number(raw.timestamp) ?? 0,
wall: wallTime === undefined ? Date.now() : Math.round(wallTime * 1000),
};
const request: Request = {
target,
requestId,
url,
clock,
match: null,
work: Promise.resolve(),
format: null,
reading: "off",
early: [],
verdict: verdict(),
info: infoOnly,
infoOnly,
status: null,
};
requests.set(key, request);
if (infoOnly) {
request.verdict.decide(false);
return;
}
if (method === "POST") {
verdicts.set(key, request.verdict);
for (const oldest of verdicts.keys()) {
if (verdicts.size <= MAX_VERDICTS) break;
verdicts.delete(oldest);
}
}
request.info = matchRuntimeInfoRequest(url, method, inline);
if (
method !== "POST" ||
inline !== undefined ||
sent.hasPostData !== true
) {
begin(request, matchRun(url, method, inline), clock.wall);
return;
}
// CDP leaves out a body over Network.enable's maxPostDataSize (CopilotKit's run bodies grow with the Thread).
request.match = "pending";
const answer = options.send(target, "Network.getRequestPostData", {
requestId,
});
request.verdict.postData = answer;
// What the body is decides the tracker's verdict as soon as it is read; the stream begins in its turn.
const matched = answer
.then(postText, () => undefined)
.then((body) => {
request.info = matchRuntimeInfoRequest(url, method, body);
const match = matchRun(url, method, body);
if (match?.kind !== "run" && match?.kind !== "candidate")
request.verdict.decide(false);
return match;
});
then(request, async () => {
const match = await matched;
begin(request, match, clock.wall);
if (match && match.kind !== "none") options.onRun(key);
});
}
/** Reads the answer live from here on: what Chrome buffered so far comes back, every later dataReceived carries
* its bytes. A response that already finished cannot stream; its body is read at loadingFinished instead. */
function stream(request: Request, at: number): void {
request.reading = "waiting";
const answer = options.send(
request.target,
"Network.streamResourceContent",
{ requestId: request.requestId },
);
then(request, async () => {
let buffered: string | undefined;
try {
const result = await answer;
buffered = isRecord(result) ? string(result.bufferedData) : undefined;
} catch {
request.reading = "off";
request.early = [];
return;
}
request.reading = "live";
if (buffered) request.stream?.data(bytesOf(buffered), at);
for (const chunk of request.early.splice(0))
request.stream?.data(chunk.bytes, chunk.at);
});
}
function onResponse(target: Debuggee, raw: Json): void {
const requestId = string(raw.requestId);
const response = isRecord(raw.response) ? raw.response : null;
if (requestId === undefined || !response) return;
const request = requests.get(requestKey(target, requestId));
if (!request) return;
const contentType =
string(response.mimeType) ||
headerValue(response.headers, "content-type");
const status = number(response.status) ?? 0;
const at = wallAt(request.clock, number(raw.timestamp));
request.status = status;
request.format = streamFormat(contentType);
then(request, () => {
request.stream?.response(contentType, status);
if (request.stream?.isRun() === false) request.verdict.decide(false);
});
if (request.format && !request.infoOnly) stream(request, at);
}
function onData(target: Debuggee, raw: Json): void {
const requestId = string(raw.requestId);
const data = string(raw.data);
if (requestId === undefined || data === undefined) return;
const request = requests.get(requestKey(target, requestId));
// A stream that showed it is not AG-UI (another app's event stream) keeps coming: its bytes are not even decoded.
if (!request || request.reading === "off" || request.stream?.ignored())
return;
const chunk = {
bytes: bytesOf(data),
at: wallAt(request.clock, number(raw.timestamp)),
};
if (request.reading === "waiting") {
request.early.push(chunk);
return;
}
then(request, () => {
request.stream?.data(chunk.bytes, chunk.at);
const run = request.stream?.isRun();
if (run !== null && run !== undefined) request.verdict.decide(run);
});
}
function onDone(target: Debuggee, raw: Json, ok: boolean): void {
const requestId = string(raw.requestId);
if (requestId === undefined) return;
const key = requestKey(target, requestId);
const request = requests.get(key);
if (!request) return;
requests.delete(key);
const at = wallAt(request.clock, number(raw.timestamp));
const { tabId } = target;
let done: () => void = () => {};
const finished = new Promise<void>((resolve) => {
done = resolve;
});
reading.set(tabId, Promise.all([reading.get(tabId), finished]));
then(request, async () => {
try {
await finish(request, ok, at);
} finally {
done();
}
});
}
/** CopilotKit's runtime `info` call answered: when its answer is CopilotKit's RuntimeInfo, the page runs CopilotKit. */
async function runtimeInfo(request: Request, at: number) {
const body = await options
.send(request.target, "Network.getResponseBody", {
requestId: request.requestId,
})
.catch(() => null);
if (
isRecord(body) &&
typeof body.body === "string" &&
isRuntimeInfoAnswer(
request.status,
bodyText(body.body, body.base64Encoded === true),
)
)
emitFor(request.target.tabId)({ kind: "runtime", url: request.url, at });
}
async function finish(request: Request, ok: boolean, at: number) {
if (ok && request.info) await runtimeInfo(request, at);
const { stream, match } = request;
if (!stream || match === "pending") return;
const run = match?.kind === "run";
// An event stream that could not be read live (it finished first) is read whole now; a run's JSON answer may
// name the Intelligence channel its events stream on.
if (ok && request.reading !== "live" && (run || request.format)) {
const body = await options
.send(request.target, "Network.getResponseBody", {
requestId: request.requestId,
})
.catch(() => null);
if (isRecord(body) && typeof body.body === "string") {
const base64 = body.base64Encoded === true;
if (request.format) stream.data(bodyBytes(body.body, base64), at);
const channel = run ? runChannel(bodyText(body.body, base64)) : null;
if (channel)
channelsOf(request.target.tabId).set(channel.topic, {
agentId: match?.agentId ?? null,
threadId: channel.threadId ?? match?.threadId ?? null,
});
}
}
stream.finish(ok, at);
request.verdict.decide(stream.isRun() === true);
}
function socketFor(target: Debuggee, requestId: string): Socket | undefined {
return sockets.get(requestKey(target, requestId));
}
function onSocketCreated(target: Debuggee, raw: Json): void {
const requestId = string(raw.requestId);
const url = string(raw.url);
if (requestId === undefined || url === undefined) return;
let followed = true;
try {
const parsed = new URL(url);
parsed.protocol = parsed.protocol === "wss:" ? "https:" : "http:";
followed = !options.ignore(parsed.href);
} catch {
followed = false;
}
const key = requestKey(target, requestId);
if (!followed) {
sockets.delete(key);
return;
}
sockets.set(key, {
tabId: target.tabId,
work: Promise.resolve(),
reader: createSocketReader({
streamId: (topic, n) =>
topic === null ? `${key}/ws/${n}` : `${key}/ws/${n}/${topic}`,
emit: emitFor(target.tabId),
channels: channelsOf(target.tabId),
}),
});
}
/** Frames carry only CDP's monotonic time: the handshake ties it to the wall clock. */
function onHandshake(target: Debuggee, raw: Json): void {
const socket = socketFor(target, string(raw.requestId) ?? "");
const monotonic = number(raw.timestamp);
const wall = number(raw.wallTime);
if (socket && monotonic !== undefined && wall !== undefined)
socket.clock = { monotonic, wall: Math.round(wall * 1000) };
}
function onFrame(target: Debuggee, raw: Json, received: boolean): void {
const socket = socketFor(target, string(raw.requestId) ?? "");
const frame = isRecord(raw.response) ? raw.response : null;
// Text frames only (opcode 1): a binary frame's payload is base64 and never a JSON AG-UI event.
if (!socket || !frame || frame.opcode !== 1) return;
const data = string(frame.payloadData);
if (data === undefined) return;
const timestamp = number(raw.timestamp);
const at = socket.clock ? wallAt(socket.clock, timestamp) : Date.now();
queue(socket, () => {
if (received) socket.reader.received(data, at);
else socket.reader.sent(data, at);
});
}
function queue(socket: Socket, step: () => void): void {
const pending = reading.get(socket.tabId);
socket.work = socket.work
.then(() => pending)
.then(step)
.catch((error) => {
console.error("Recorder: a WebSocket frame could not be read", error);
});
}
function onSocketClosed(target: Debuggee, raw: Json): void {
const key = requestKey(target, string(raw.requestId) ?? "");
const socket = sockets.get(key);
if (!socket) return;
sockets.delete(key);
const timestamp = number(raw.timestamp);
const at = socket.clock ? wallAt(socket.clock, timestamp) : Date.now();
queue(socket, () => socket.reader.closed(at));
}
return {
onEvent(target: Debuggee, method: string, raw: unknown): void {
if (!isRecord(raw)) return;
switch (method) {
case "Network.requestWillBeSent":
onRequest(target, raw);
return;
case "Network.responseReceived":
onResponse(target, raw);
return;
case "Network.dataReceived":
onData(target, raw);
return;
case "Network.loadingFinished":
onDone(target, raw, true);
return;
case "Network.loadingFailed":
onDone(target, raw, false);
return;
case "Network.webSocketCreated":
onSocketCreated(target, raw);
return;
case "Network.webSocketWillSendHandshakeRequest":
onHandshake(target, raw);
return;
case "Network.webSocketFrameReceived":
onFrame(target, raw, true);
return;
case "Network.webSocketFrameSent":
onFrame(target, raw, false);
return;
case "Network.webSocketClosed":
onSocketClosed(target, raw);
return;
}
},
/** What the tracker reads at requestWillBeSent: a run or a candidate is never a write a step waits for; a body
* still being read is, until onRun says otherwise. */
kind(key: string): "run" | "candidate" | "pending" | null {
const match = requests.get(key)?.match;
if (match === "pending") return "pending";
if (match?.kind === "run" || match?.kind === "candidate")
return match.kind;
return null;
},
/** Whether the request is an agent run (`value` once known, `promise` until then), and the body this capture read
* for it when Chrome left it out. The tracker never makes a run a step's call. */
inspect(key: string): RunVerdict | undefined {
return verdicts.get(key);
},
/** The tracker had the request: its verdict is no longer kept. */
release(key: string): void {
verdicts.delete(key);
},
/** A tab that closed or navigated away: its requests and sockets are dropped, their streams unended. */
forgetTab(tabId: number): void {
for (const [key, request] of requests)
if (request.target.tabId === tabId) {
requests.delete(key);
request.verdict.decide(false);
}
for (const [key, socket] of sockets)
if (socket.tabId === tabId) sockets.delete(key);
for (const [key, known] of verdicts)
if (key.startsWith(`${tabId}/`)) {
known.decide(false);
verdicts.delete(key);
}
channels.delete(tabId);
reading.delete(tabId);
},
clear(): void {
for (const known of verdicts.values()) known.decide(false);
requests.clear();
sockets.clear();
verdicts.clear();
channels.clear();
reading.clear();
},
};
}