Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
// The agent's side of a recording: the AG-UI traffic the network capture observes (observation.ts), turned into exactly
// the events the CopilotKit SDK emits for the same traffic, so Intelligence cannot tell a Recorder trajectory from a
// CopilotKit one: `thread.linked`, `agent.run`, `agent.message` and `tool.call` as CopilotKit's LearningBridge emits them
// through a legacy sink, the agent context CopilotKit adds to a click, and `agent.event` for every piece of AG-UI the
// others leave out (agent-event.ts).
//
// LearningBridge links a Thread when a chat opens it (`thread.linked`, reason "open") and when the same chat moves to
// another Thread (reason "switch"), once per change of Thread in the page. The network shows a chat's Thread when the
// page first talks to its agent about it, so the page's chat for an agent is on that agent's latest Thread in the page:
// its first Thread in a page view is linked "open", a later one "switch", just before its first event; `hasMessages`
// says whether the Thread held messages before the page's new ones. A Thread the chat moved away from is no longer open.
//
// Each Thread of each agent in a tab is an AG-UI agent of its own (thread-agent.ts): a run's request adds the page's new
// messages to it as CopilotKit adds them before it runs, and the run's events are fed to it as they arrive, whatever the
// transport (an SSE or protobuf response, or the Intelligence WebSocket). A stream the recording joined mid-run waits for
// its next RUN_STARTED, and a run that never ends is left open, never finished for it: AG-UI forbids inventing a
// RUN_FINISHED. When a Thread's history is replayed (a page that reloads, a channel that joins again), a run already
// recorded for the Thread, or an event already recorded (Intelligence's event id), is not recorded again; a replay of a
// run cut short carries on where the recording left it. Runs CopilotKit makes for its suggestions are left out, as
// CopilotKit's own capture leaves them out. Whatever is dropped is counted in `recorder.dropped`.
//
// What a click's context and the links depend on survives a service-worker restart (`snapshot`, kept with the held
// steps, and `restore`): the page's open Threads, its run requests, the Thread linked last and each agent's last run.
// The AG-UI streams and the Threads' conversations are taken up again from the page's next traffic.
import type { Message, RunAgentInput } from "@ag-ui/client";
import type { BaseEvent } from "@ag-ui/core";
import type {
CapturedEvent,
JsonValue,
} from "@openbot/shared/recorder-masking";
import { ReplaySubject } from "rxjs";
import {
AGENT_TARGET_FIELD,
type AgentTarget,
} from "../../shared/agent-target";
import { createExpressed, type Expressed } from "./agent-event";
import { finishAgentEvent } from "./mask";
import type {
AgentRunRequest,
AgentStreamEnd,
AgentStreamEvent,
CaptureObservation,
} from "./observation";
import { type FedEvent, type PageAgent, ThreadAgent } from "./thread-agent";
export type AgentEvents = {
/** One observation from the network capture, in the order the capture saw it. */
observe(observation: CaptureObservation): void;
/** A click from a followed tab, before the steps see it: the agent context CopilotKit adds to a click. */
enrichClick(tabId: number, click: CapturedEvent): CapturedEvent;
/** A click as it leaves the steps, by what the page did after it. `joins` says whether a run the page asked for at a
* time is the click's doing (after it, before the person's next step, within the steps' join): a click with the
* open-Thread context takes the Thread of the first such run, the run it sent; one whose Thread the page has since
* left, by no doing of the click, names none, since the page may have left it before the click. */
settleClick(
tabId: number,
click: CapturedEvent,
joins: (at: number) => boolean,
): CapturedEvent;
/** Whether a click held in the steps still waits for the run it may have sent: one with the open-Thread context in a
* tab whose page runs CopilotKit, whose join holds no run yet. */
awaitsRun(
tabId: number,
click: CapturedEvent,
joins: (at: number) => boolean,
): boolean;
/** The tab's top frame reached another page: its open Threads close. */
tabNavigated(tabId: number): void;
/** What a click's context and the links depend on, as JSON, for a service worker that restarts to take up again. */
snapshot(): AgentSnapshot;
/** Takes up what another service worker's agent events knew (their `snapshot`), before any traffic or click. */
restore(snapshot: AgentSnapshot): void;
/** The recording ended: forget everything held. */
stop(): void;
};
/** What the agent events know that a click's agent context and the Thread links depend on: per tab, that its page runs
* CopilotKit, the Thread each chat has open and the Threads it left, the Thread linked last, the latest run requests and
* each agent's last run; and the runs recorded per Thread, so a replay of one is never recorded again. Ids and times
* only, never a message. The AG-UI streams and the Threads' conversations are not kept: a service worker that restarts
* takes them up from the page's next traffic. */
export type AgentSnapshot = {
tabs: (LinkState & {
tabId: number;
runs: { at: number; threadId: string }[];
lastRunIds: [string, string][];
})[];
recordedRuns: [string, string[]][];
};
/** How many of a tab's latest run requests (and Threads its chats left) are kept for the clicks still held in their
* steps. */
const MAX_RUNS_KEPT = 20;
/** CopilotKit's suggestion tool (`@copilotkit/core` suggestion-engine.ts): a run that forces it is a suggestion run. */
const SUGGEST_TOOL = "copilotkitSuggest";
/** The routing fields Intelligence's runner stamps on every event it relays, removed as CopilotKit's IntelligenceAgent
* removes them (`stripIntelligenceRoutingFields`) and as AG-UI's enforcement removes the rest before anyone sees them. */
const ROUTING_FIELDS = ["thread_id", "run_id", "organization_id", "project_id"];
const RUN_IDENTITY_EVENTS = new Set(["RUN_STARTED", "RUN_FINISHED"]);
/** What the page sent for a run, as far as the agent events read it. */
type RunInput = Pick<
RunAgentInput,
"threadId" | "tools" | "context" | "forwardedProps" | "state" | "resume"
> & { runId: string | undefined; messages: Message[]; raw: unknown };
type PageRun = { input: RunInput; at: number; followUp: boolean };
type Run = {
thread: Thread;
runId: string | undefined;
events: ReplaySubject<FedEvent>;
};
type Thread = {
agentId: string | null;
threadId: string;
/** The recording it belongs to: what it emits after that recording stopped goes nowhere. */
born: number;
agent: ThreadAgent;
/** Each step of the Thread (a request's messages, a run) waits for the one before it to be applied. */
chain: Promise<void>;
/** The run being fed, or one cut short that a replay may carry on. */
open: Run | null;
/** A run the page asked for whose RUN_STARTED has not arrived. */
pending: PageRun | null;
/** The run id CopilotKit reports the last run under (RunHandler's logical run id), which a follow-up continues. */
lastLogicalRunId: string | undefined;
};
type Stream = {
agentId: string | null;
threadId: string | null;
/** A request of the page opened it, so it starts at its beginning. */
requested: boolean;
suggestion: boolean;
events: number;
thread: Thread | null;
run: Run | null;
/** Inside a replayed run that is already recorded. */
skipping: boolean;
/** The RUN_STARTED of a stream no request opened, held until the next event shows whether it is a suggestion run. */
held: { fed: FedEvent; id: string | null } | null;
};
type Tab = {
/** The tab's page talks to a CopilotKit runtime: its clicks carry the open-Thread context. */
traffic: boolean;
/** The Thread each agent's chat in the page has open (CopilotKit's open-Thread registrations), in the order the chats
* opened; a new document starts with none. */
views: Map<string, string>;
/** Threads a chat moved away from, and when the page showed the Thread it moved to. */
left: { threadId: string; at: number }[];
/** The Thread the page linked last (LearningBridge.lastLinkedThreadId); a new document starts with none. */
lastLinked: string | null;
/** The page's latest run requests, when each was seen and for which Thread: a click's step window may hold one. */
runs: { at: number; threadId: string }[];
/** Links decided whose `thread.linked` the Thread's steps have not emitted yet, oldest first, each with what the tab's
* Threads were before it: `snapshot` gives the Threads as they were before the oldest, so a service worker that
* restarts before a link is emitted makes it again, and never takes up a link that went nowhere. */
pendingLinks: { before: LinkState; done: boolean }[];
/** CopilotKit's agent in the page, per agent id. */
agents: Map<string, PageAgent>;
streams: Map<string, Stream>;
threads: Map<string, Thread>;
drops: Map<string, number>;
};
/** A tab's open Threads, the Threads its chats left and the Thread linked last, as a snapshot keeps them. */
type LinkState = {
views: [string, string][];
left: { threadId: string; at: number }[];
lastLinked: string | null;
};
const isRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value);
const isMessage = (value: unknown): value is Message =>
isRecord(value) &&
typeof value.id === "string" &&
typeof value.role === "string";
function readInput(raw: unknown): RunInput | null {
if (!isRecord(raw) || typeof raw.threadId !== "string" || raw.threadId === "")
return null;
return {
threadId: raw.threadId,
runId: typeof raw.runId === "string" ? raw.runId : undefined,
messages: Array.isArray(raw.messages) ? raw.messages.filter(isMessage) : [],
tools: Array.isArray(raw.tools) ? (raw.tools as RunInput["tools"]) : [],
context: Array.isArray(raw.context)
? (raw.context as RunInput["context"])
: [],
forwardedProps: raw.forwardedProps,
state: raw.state,
...(Array.isArray(raw.resume)
? { resume: raw.resume as RunInput["resume"] }
: {}),
raw,
};
}
/** A run CopilotKit's suggestion engine makes: it forces the suggestion tool, and offers no other. */
function isSuggestion(input: unknown): boolean {
if (!isRecord(input)) return false;
const choice = isRecord(input.forwardedProps)
? input.forwardedProps.toolChoice
: undefined;
const forced =
isRecord(choice) && isRecord(choice.function)
? choice.function.name
: undefined;
const tools = Array.isArray(input.tools) ? input.tools : [];
return (
forced === SUGGEST_TOOL ||
(tools.length > 0 &&
tools.every((tool) => isRecord(tool) && tool.name === SUGGEST_TOOL))
);
}
/** The event the library applies: a copy, since applying a state patch can change the values it inserts, and the
* capture's own observation is never changed. */
function normalise(raw: unknown): BaseEvent | null {
if (!isRecord(raw) || typeof raw.type !== "string") return null;
const event: Record<string, unknown> = structuredClone(raw);
for (const field of ROUTING_FIELDS) delete event[field];
if (!RUN_IDENTITY_EVENTS.has(event.type as string)) {
delete event.threadId;
delete event.runId;
}
return event as unknown as BaseEvent;
}
/** Intelligence's durable id of an event it relays (IntelligenceAgent.readEventId); an id it has not stored yet
* (`cpki_ingested...`) is none. */
function eventIdOf(event: BaseEvent): string | null {
const metadata = (event as { metadata?: unknown }).metadata;
const id = isRecord(metadata) ? metadata.cpki_event_id : undefined;
return typeof id === "string" &&
id.trim() !== "" &&
!id.startsWith("cpki_ingested")
? id
: null;
}
const isTerminal = (event: BaseEvent) =>
event.type === "RUN_FINISHED" || event.type === "RUN_ERROR";
function readTarget(raw: unknown): AgentTarget | null {
if (!isRecord(raw)) return null;
const text = (value: unknown) => (typeof value === "string" ? value : null);
return { messageId: text(raw.messageId), toolCallId: text(raw.toolCallId) };
}
export function createAgentEvents(options: {
/** A masked event ready for the trajectory, in its tab's order. */
emit(tabId: number, event: CapturedEvent): void;
/** The recording's sensitive-value digests. */
digests(): ReadonlySet<string>;
now(): number;
/** What `snapshot` reads changed: called as each change is made. */
onChange?(): void;
}): AgentEvents {
const tabs = new Map<number, Tab>();
/** Per Thread, across tabs: the runs recorded, and what has been carried. */
const recordedRuns = new Map<string, Set<string>>();
const expressedByThread = new Map<string, Expressed>();
/** Intelligence event ids already recorded. */
const recordedEvents = new Set<string>();
/** Bumped when the recording stops: what the library still applies afterwards belongs to no recording and is not
* emitted. */
let recording = 0;
const changed = () => options.onChange?.();
/** CopilotKit's agent in a page: its last run id, which says when it changes. */
const pageAgent = (lastRunId: string | undefined): PageAgent => {
let current = lastRunId;
return {
get lastRunId() {
return current;
},
set lastRunId(next) {
if (next === current) return;
current = next;
changed();
},
};
};
const tabOf = (tabId: number): Tab => {
let tab = tabs.get(tabId);
if (!tab) {
tab = {
traffic: false,
views: new Map(),
left: [],
lastLinked: null,
runs: [],
pendingLinks: [],
agents: new Map(),
streams: new Map(),
threads: new Map(),
drops: new Map(),
};
tabs.set(tabId, tab);
}
return tab;
};
const drop = (tab: Tab, reason: string, count = 1) =>
tab.drops.set(reason, (tab.drops.get(reason) ?? 0) + count);
const reportDrops = (tabId: number, tab: Tab) => {
for (const [reason, count] of tab.drops)
options.emit(tabId, {
type: "CUSTOM",
name: "recorder.dropped",
timestamp: options.now(),
value: { count, reason },
});
tab.drops.clear();
};
const runsOf = (threadId: string): Set<string> => {
let runs = recordedRuns.get(threadId);
if (!runs) {
runs = new Set();
recordedRuns.set(threadId, runs);
}
return runs;
};
const expressedOf = (threadId: string): Expressed => {
let expressed = expressedByThread.get(threadId);
if (!expressed) {
expressed = createExpressed();
expressedByThread.set(threadId, expressed);
}
return expressed;
};
const keyOf = (agentId: string | null) => JSON.stringify(agentId);
const linkStateOf = (tab: Tab): LinkState => ({
views: [...tab.views],
left: tab.left.map((entry) => ({ ...entry })),
lastLinked: tab.lastLinked,
});
/** The Threads open in the tab, each once, in the order their chats opened (LearningBridge.distinctOpenThreadIds). */
const openIds = (tab: Tab): string[] => [...new Set(tab.views.values())];
const pageAgentOf = (tab: Tab, agentId: string | null): PageAgent => {
const key = keyOf(agentId);
let agent = tab.agents.get(key);
if (!agent) {
agent = pageAgent(undefined);
tab.agents.set(key, agent);
}
return agent;
};
/** The page shows the Thread in its agent's chat: CopilotKit's registerOpenThread for the chat's first Thread in the
* page, its registration's update when the chat moves to another. Says how LearningBridge links it: "open" for a
* chat's first Thread, "switch" for a later one, null when that changes nothing or the Thread is open in another
* chat. */
const showThread = (
tab: Tab,
thread: Thread,
at: number,
): "open" | "switch" | null => {
const key = keyOf(thread.agentId);
const previous = tab.views.get(key);
if (previous === thread.threadId) return null;
const elsewhere = openIds(tab).includes(thread.threadId);
tab.views.set(key, thread.threadId);
if (previous !== undefined && !openIds(tab).includes(previous)) {
tab.left.push({ threadId: previous, at });
if (tab.left.length > MAX_RUNS_KEPT) tab.left.shift();
}
changed();
if (elsewhere) return null;
return previous === undefined ? "open" : "switch";
};
/** The Thread an agent id and Thread id name in a tab; a stream that names no agent takes the tab's Thread of that id. */
const threadOf = (
tabId: number,
tab: Tab,
agentId: string | null,
threadId: string,
): Thread => {
const key = JSON.stringify([agentId, threadId]);
const existing =
tab.threads.get(key) ??
(agentId === null
? [...tab.threads.values()].find(
(thread) => thread.threadId === threadId,
)
: undefined);
if (existing) return existing;
const born = recording;
const thread: Thread = {
agentId,
threadId,
born,
agent: new ThreadAgent(
agentId,
threadId,
{
emit: (name, value, at) => {
if (born !== recording) return;
options.emit(
tabId,
finishAgentEvent(name, value, at, options.digests()),
);
},
invalid: () => drop(tab, "agent-refused"),
},
expressedOf(threadId),
pageAgentOf(tab, agentId),
),
chain: Promise.resolve(),
open: null,
pending: null,
lastLogicalRunId: undefined,
};
tab.threads.set(key, thread);
return thread;
};
const streamOf = (
tab: Tab,
streamId: string,
agentId: string | null,
): Stream => {
let stream = tab.streams.get(streamId);
if (!stream) {
stream = {
agentId,
threadId: null,
requested: false,
suggestion: false,
events: 0,
thread: null,
run: null,
skipping: false,
held: null,
};
tab.streams.set(streamId, stream);
}
stream.agentId ??= agentId;
return stream;
};
/** LearningBridge.emitThreadLinked for the Thread the page now talks to its agent about: nothing when its chat already
* shows it, or it is the Thread linked last in the page. Decided in the order the page's traffic came; the function
* it returns sends the link from inside the Thread's step, so it leaves before the Thread's events that follow. */
const linkThread = (
tabId: number,
tab: Tab,
thread: Thread,
at: number,
): ((hasMessages: boolean, at: number) => void) => {
const before = linkStateOf(tab);
const reason = showThread(tab, thread, at);
if (reason === null || tab.lastLinked === thread.threadId) return () => {};
tab.lastLinked = thread.threadId;
const pending = { before, done: false };
tab.pendingLinks.push(pending);
changed();
return (hasMessages, at) => {
// The link is decided for good as it goes to the steps.
pending.done = true;
while (tab.pendingLinks[0]?.done) tab.pendingLinks.shift();
changed();
if (thread.born !== recording) return;
options.emit(
tabId,
finishAgentEvent(
"thread.linked",
{
threadId: thread.threadId,
agentId: thread.agentId,
hasMessages,
reason,
},
at,
options.digests(),
),
);
};
};
/** A step of the Thread, after the steps before it; a step the library refuses is counted, never thrown. */
const step = (tab: Tab, thread: Thread, work: () => void | Promise<void>) => {
thread.chain = thread.chain.then(work).catch(() => {
drop(tab, "agent-refused");
});
};
/** The Thread's open run is over without its end (cut short, or replaced by a new run): its events stop, and nothing is
* invented for it. */
const endOpenRun = (tab: Tab, thread: Thread) => {
const run = thread.open;
if (!run) return;
thread.open = null;
run.events.complete();
for (const stream of tab.streams.values())
if (stream.run === run) stream.run = null;
};
const feed = (tab: Tab, run: Run, event: BaseEvent, at: number) => {
run.events.next({ event, at });
if (!isTerminal(event)) return;
run.events.complete();
if (run.thread.open === run) run.thread.open = null;
for (const stream of tab.streams.values())
if (stream.run === run) stream.run = null;
};
/** CopilotKit adds the person's new messages with `addMessage` before it runs (an `agent.message` each), and a frontend
* tool's result straight into the history: the request shows both, after what the Thread already holds. The page's
* `link` goes first, saying whether the Thread held messages before the person's new ones. */
const addPageMessages = (
thread: Thread,
page: PageRun,
link: (hasMessages: boolean, at: number) => void,
) => {
const held = new Set(thread.agent.messages.map(({ id }) => id));
const carried = expressedOf(thread.threadId).messages;
const { messages } = page.input;
let start = messages.length;
while (start > 0) {
const message = messages[start - 1] as Message;
if (
message.role !== "user" ||
held.has(message.id) ||
carried.has(message.id)
)
break;
start--;
}
link(start > 0, page.at);
const typed = messages.slice(start);
const added = messages
.slice(0, start)
.map((message, index) => ({
message: structuredClone(message),
after: index > 0 ? (messages[index - 1] as Message).id : null,
}))
.filter(({ message }) => !held.has(message.id));
page.followUp =
typed.length === 0 &&
added.some(({ message }) => message.role === "tool");
thread.agent.addFromPage(
added,
typed.map((message) => structuredClone(message)),
page.at,
);
};
const startRun = (
tabId: number,
tab: Tab,
stream: Stream,
thread: Thread,
first: FedEvent,
id: string | null,
) => {
endOpenRun(tab, thread);
const page = thread.pending;
thread.pending = null;
// A run no request of the page's showed (a channel joined on its own): its Thread shows itself here.
const link = page ? null : linkThread(tabId, tab, thread, first.at);
const started = first.event as BaseEvent & {
runId?: string;
input?: unknown;
};
const runId = started.runId ?? page?.input.runId;
const run: Run = { thread, runId, events: new ReplaySubject<FedEvent>() };
if (runId !== undefined && !runsOf(thread.threadId).has(runId)) {
runsOf(thread.threadId).add(runId);
changed();
}
thread.open = run;
stream.run = run;
const input =
page?.input ??
readInput(started.input) ??
({
threadId: thread.threadId,
runId,
messages: [],
tools: [],
context: [],
forwardedProps: undefined,
state: undefined,
raw: undefined,
} satisfies RunInput);
step(tab, thread, async () => {
link?.(thread.agent.messages.length > 0, first.at);
const resumed = thread.agent.continuationRunIdFor(input);
const followUp = page?.followUp === true;
const continuation =
resumed !== undefined || followUp
? { expectedRunId: resumed ?? thread.lastLogicalRunId }
: null;
const runIdOnInput = input.runId ?? runId ?? crypto.randomUUID();
thread.lastLogicalRunId = continuation
? continuation.expectedRunId
: runIdOnInput;
await thread.agent.applyRun(
run.events,
{ ...input, runId: runIdOnInput },
continuation,
page?.input.raw,
);
});
if (id !== null) recordedEvents.add(id);
feed(tab, run, first.event, first.at);
};
const onRequest = (tabId: number, tab: Tab, request: AgentRunRequest) => {
const input = readInput(request.input);
const stream = streamOf(tab, request.streamId, request.agentId);
stream.requested = true;
if (!input) {
drop(tab, "agent-unreadable");
return;
}
stream.threadId = input.threadId;
if (isSuggestion(request.input)) {
stream.suggestion = true;
return;
}
const thread = threadOf(tabId, tab, request.agentId, input.threadId);
stream.thread = thread;
const link = linkThread(tabId, tab, thread, request.at);
if (request.method !== "agent/run") {
step(tab, thread, () => link(input.messages.length > 0, request.at));
return;
}
tab.runs.push({ at: request.at, threadId: input.threadId });
if (tab.runs.length > MAX_RUNS_KEPT) tab.runs.shift();
changed();
// The page started a new run: a run of the Thread still open was cut short.
endOpenRun(tab, thread);
const page: PageRun = { input, at: request.at, followUp: false };
thread.pending = page;
step(tab, thread, () => addPageMessages(thread, page, link));
};
const onRunStarted = (
tabId: number,
tab: Tab,
stream: Stream,
fed: FedEvent,
id: string | null,
) => {
const event = fed.event as BaseEvent & {
threadId?: string;
runId?: string;
input?: unknown;
};
const threadId = event.threadId || stream.threadId;
if (!threadId) {
drop(tab, "agent-unreadable");
return;
}
if (!stream.requested && isSuggestion(event.input)) {
stream.suggestion = true;
return;
}
const thread = threadOf(tabId, tab, stream.agentId, threadId);
stream.thread = thread;
stream.skipping = false;
const replayed =
(event.runId !== undefined && runsOf(threadId).has(event.runId)) ||
(id !== null && recordedEvents.has(id));
if (replayed) {
showThread(tab, thread, fed.at);
// The page's agent applies the replay, so the run it saw last is this one, though the recording has it already.
if (event.runId !== undefined)
pageAgentOf(tab, thread.agentId).lastRunId = event.runId;
drop(tab, "agent-replayed");
// A run cut short whose replay names each event can carry on: its recorded events are skipped one by one.
if (
id !== null &&
thread.open !== null &&
thread.open.runId === event.runId
)
stream.run = thread.open;
else stream.skipping = true;
return;
}
if (
!stream.requested &&
event.input === undefined &&
thread.pending === null
) {
stream.held = { fed, id };
return;
}
startRun(tabId, tab, stream, thread, fed, id);
};
const onEvent = (tabId: number, tab: Tab, observation: AgentStreamEvent) => {
const stream = streamOf(tab, observation.streamId, observation.agentId);
if (observation.threadId) stream.threadId ??= observation.threadId;
if (stream.suggestion) return;
const event = normalise(observation.event);
if (!event) {
drop(tab, "agent-unreadable");
return;
}
stream.events++;
const fed: FedEvent = { event, at: observation.at };
if (stream.held) {
const { held } = stream;
stream.held = null;
const toolCallName = (event as { toolCallName?: unknown }).toolCallName;
if (event.type === "TOOL_CALL_START" && toolCallName === SUGGEST_TOOL) {
stream.suggestion = true;
return;
}
if (stream.thread)
startRun(tabId, tab, stream, stream.thread, held.fed, held.id);
}
const id = eventIdOf(event);
if (event.type === "RUN_STARTED") {
onRunStarted(tabId, tab, stream, fed, id);
return;
}
if (stream.skipping) {
if (id !== null) recordedEvents.add(id);
drop(tab, "agent-replayed");
if (isTerminal(event)) stream.skipping = false;
return;
}
if (id !== null && recordedEvents.has(id)) {
drop(tab, "agent-replayed");
return;
}
if (!stream.run) {
// A run the page asked for that failed before it started still ends in CopilotKit's `agent.run` error.
if (
event.type === "RUN_ERROR" &&
stream.requested &&
stream.events === 1 &&
stream.thread?.pending
)
startRun(tabId, tab, stream, stream.thread, fed, id);
else drop(tab, "agent-mid-run");
return;
}
if (id !== null) recordedEvents.add(id);
feed(tab, stream.run, event, observation.at);
};
const onEnd = (tabId: number, tab: Tab, end: AgentStreamEnd) => {
const stream = tab.streams.get(end.streamId);
if (!stream) return;
if (stream.held && stream.thread)
startRun(
tabId,
tab,
stream,
stream.thread,
stream.held.fed,
stream.held.id,
);
tab.streams.delete(end.streamId);
reportDrops(tabId, tab);
};
/** The agent context CopilotKit adds to a click (LearningBridge.enrich): the exact message first, then the open
* Threads. */
const clickContext = (
tab: Tab,
target: AgentTarget | null,
): Record<string, JsonValue> => {
if (target?.messageId) {
for (const thread of tab.threads.values()) {
const message = thread.agent.messages.find(
({ id }) => id === target.messageId,
);
if (!message) continue;
const toolCall =
target.toolCallId !== null && message.role === "assistant"
? message.toolCalls?.find(({ id }) => id === target.toolCallId)
: undefined;
const answered =
toolCall !== undefined &&
thread.agent.messages.some(
(other) =>
other.role === "tool" && other.toolCallId === toolCall.id,
);
return {
threadId: thread.agent.threadId,
agentId: thread.agent.id,
messageId: message.id,
runId: thread.agent.runIdForMessage(message.id),
...(toolCall === undefined
? {}
: {
toolCallId: toolCall.id,
toolName: toolCall.function.name,
toolStatus: thread.agent.isExecuting(toolCall.id)
? "executing"
: answered
? "complete"
: "inProgress",
}),
};
}
}
const open = openIds(tab);
if (open.length > 1)
return {
threadId: null,
threadAmbiguity: { reason: "multiple-open", candidates: open },
};
return { threadId: open[0] ?? null };
};
/** A click with the open-Thread context (no message of its own, no ambiguity) in a tab whose page runs CopilotKit. */
const openContextOf = (
tabId: number,
click: CapturedEvent,
): { tab: Tab; value: Record<string, unknown> } | null => {
const value = click.value;
const tab = tabs.get(tabId);
if (
!tab?.traffic ||
!isRecord(value) ||
!Object.hasOwn(value, "threadId") ||
value.messageId !== undefined ||
value.threadAmbiguity !== undefined
)
return null;
return { tab, value };
};
return {
observe(observation) {
const tab = tabOf(observation.tabId);
if (!tab.traffic) {
tab.traffic = true;
changed();
}
// The page asked its CopilotKit runtime what it serves: it runs CopilotKit, which is all this says.
if (observation.kind === "runtime") return;
try {
if (observation.kind === "request")
onRequest(observation.tabId, tab, observation);
else if (observation.kind === "event")
onEvent(observation.tabId, tab, observation);
else onEnd(observation.tabId, tab, observation);
} catch {
drop(tab, "agent-unreadable");
}
},
enrichClick(tabId, click) {
const value = click.value;
if (!isRecord(value)) return click;
const { [AGENT_TARGET_FIELD]: target, ...rest } = value;
const tab = tabs.get(tabId);
if (!tab?.traffic)
return target === undefined ? click : { ...click, value: rest };
return {
...click,
value: { ...rest, ...clickContext(tab, readTarget(target)) },
};
},
settleClick(tabId, click, joins) {
const context = openContextOf(tabId, click);
if (!context) return click;
const { tab, value } = context;
// The run the click sent: a run goes out on its chat's open Thread, so that is the Thread the click was made in.
const run = tab.runs.find(({ at }) => joins(at));
if (run)
return run.threadId === value.threadId
? click
: { ...click, value: { ...value, threadId: run.threadId } };
// The page showed the click's chat on another Thread after the click, by no doing of the click's: the chat may
// have left the click's Thread at any moment since that Thread's own traffic (a New thread button calls nothing),
// so which Thread the click was made in is not known, and a stale one is never given.
const left = tab.left.some(
({ threadId, at }) =>
threadId === value.threadId && at >= click.timestamp && !joins(at),
);
return left ? { ...click, value: { ...value, threadId: null } } : click;
},
awaitsRun(tabId, click, joins) {
const context = openContextOf(tabId, click);
return context !== null && !context.tab.runs.some(({ at }) => joins(at));
},
tabNavigated(tabId) {
const tab = tabs.get(tabId);
if (!tab) return;
// A new document: its chats open their Threads anew, and CopilotKit's agents in it have run nothing yet.
tab.views.clear();
tab.lastLinked = null;
changed();
for (const agent of tab.agents.values()) agent.lastRunId = undefined;
for (const thread of tab.threads.values()) thread.agent.pageGone();
reportDrops(tabId, tab);
},
snapshot() {
return {
tabs: [...tabs]
.filter(([, tab]) => tab.traffic)
.map(([tabId, tab]) => ({
tabId,
...(tab.pendingLinks[0]?.before ?? linkStateOf(tab)),
runs: tab.runs.map((run) => ({ ...run })),
lastRunIds: [...tab.agents].flatMap(([key, agent]) =>
agent.lastRunId === undefined
? []
: [[key, agent.lastRunId] as [string, string]],
),
})),
recordedRuns: [...recordedRuns].map(([threadId, runs]) => [
threadId,
[...runs],
]),
};
},
restore(snapshot) {
for (const kept of snapshot.tabs) {
const tab = tabOf(kept.tabId);
tab.traffic = true;
tab.views = new Map(kept.views);
tab.left = kept.left.map((entry) => ({ ...entry }));
tab.lastLinked = kept.lastLinked;
tab.runs = kept.runs.map((run) => ({ ...run }));
tab.agents = new Map(
kept.lastRunIds.map(([key, runId]) => [key, pageAgent(runId)]),
);
}
for (const [threadId, runs] of snapshot.recordedRuns)
for (const runId of runs) runsOf(threadId).add(runId);
},
stop() {
for (const [tabId, tab] of tabs) {
for (const thread of tab.threads.values()) {
thread.agent.flushHeld();
if (thread.open) thread.open.events.complete();
}
reportDrops(tabId, tab);
}
recording++;
tabs.clear();
recordedRuns.clear();
expressedByThread.clear();
recordedEvents.clear();
},
};
}