Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import { expect, spyOn, test } from "bun:test";
import type { AbstractAgent, BaseEvent, RunAgentInput } from "@ag-ui/client";
import { EventType, HttpAgent } from "@ag-ui/client";
import { LLMock } from "@copilotkit/aimock";
import { BuiltInAgent } from "@copilotkit/runtime/v2";
import type { GrantedTool } from "@openbot/toolkit/granted-tool";
import { z } from "zod";
import {
buildAgents,
type HandoffForRun,
type RuntimeModel,
type ToolSelection,
} from "../src/copilot";
import { createModelCompleter } from "../src/routing/model";
const model: RuntimeModel = { provider: "openai", defaultModel: "gpt-5.5" };
const input: RunAgentInput = {
threadId: "cancellation-fixture-thread",
runId: "cancellation-fixture-run",
messages: [
{ id: "user", role: "user", content: "Read the fixture document." },
],
tools: [],
context: [],
state: {},
forwardedProps: {},
};
const skills = [
{
slug: "read",
title: "Read",
summary: "Read a fixture.",
tools: ["fixture/read"],
},
];
function deferred<T>() {
return Promise.withResolvers<T>();
}
async function bounded<T>(
promise: Promise<T>,
boundary = "operation",
): Promise<T> {
let timer: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
promise,
new Promise<never>((_, reject) => {
timer = setTimeout(
() => reject(new Error(`Fixture ${boundary} timed out`)),
1500,
);
}),
]);
} finally {
clearTimeout(timer);
}
}
function observe(agent: AbstractAgent) {
const events: BaseEvent[] = [];
const settled = deferred<void>();
let completions = 0;
const subscription = agent.run(input).subscribe({
next: (event) => events.push(event),
error: settled.reject,
complete: () => {
completions++;
settled.resolve();
},
});
return {
events,
settled: settled.promise,
subscription,
completions: () => completions,
};
}
async function fixture() {
const llm = new LLMock();
const originalBase = process.env.OPENAI_BASE_URL;
process.env.OPENAI_BASE_URL = await llm.start();
llm.onMessage(/.*/, { content: "Fixture completed." });
let executions = 0;
const granted: GrantedTool[] = [
{
ref: "fixture/read",
name: "mcp__fixture__read",
description: "Read a fixture",
parameters: z.object({}),
execute: async () => {
executions++;
return "fixture";
},
},
];
return {
llm,
executions: () => executions,
async agent(
choose?: ToolSelection["choose"],
handoff?: HandoffForRun,
record?: ToolSelection["record"],
) {
const agents = await buildAgents(
[
{
id: "fixture",
name: "Fixture",
type: "built_in",
systemPrompt: "Answer the fixture.",
},
],
model,
"synthetic-fixture-key",
undefined,
async () => granted,
undefined,
undefined,
undefined,
choose
? { choose, loadSkills: async () => skills, floor: 0, record }
: undefined,
undefined,
handoff,
);
if (!agents.fixture) throw new Error("Fixture agent missing");
return agents.fixture.clone();
},
async [Symbol.asyncDispose]() {
if (originalBase === undefined) delete process.env.OPENAI_BASE_URL;
else process.env.OPENAI_BASE_URL = originalBase;
await llm.stop();
},
};
}
for (const phase of ["selector", "handoff"] as const) {
for (const outcome of ["resolve", "reject"] as const) {
test(`Stop during ${phase} blocks a late ${outcome} and completes once`, async () => {
await using f = await fixture();
const gate = deferred<void>();
const entered = deferred<void>();
let records = 0;
const wait = async () => {
entered.resolve();
await gate.promise;
};
const agent = await f.agent(
phase === "selector"
? async () => {
await wait();
return '{"skills":["read"]}';
}
: undefined,
phase === "handoff"
? async () => {
await wait();
return [];
}
: undefined,
async () => {
records++;
},
);
const run = observe(agent);
try {
await bounded(entered.promise);
agent.abortRun();
if (outcome === "resolve") gate.resolve();
else gate.reject(new Error("Synthetic late build rejection"));
await bounded(run.settled);
// Drain the released build's continuation, not merely the abort notification.
await new Promise<void>((resolve) => setImmediate(resolve));
expect(f.llm.getRequests()).toHaveLength(0);
expect(run.events).toHaveLength(0);
expect(f.executions()).toBe(0);
expect(records).toBe(0);
expect(run.completions()).toBe(1);
} finally {
gate.resolve();
run.subscription.unsubscribe();
}
});
}
}
/** A real selector connection held by this test. Final requests go only to LLMock. */
async function selectorEndpoint(
llm: LLMock,
hold: "selector" | "final" = "selector",
) {
const entered = deferred<void>();
const closed = deferred<void>();
const release = deferred<void>();
const handled = deferred<void>();
let selectorRequests = 0;
let finalRequests = 0;
const server = Bun.serve({
hostname: "127.0.0.1",
port: 0,
async fetch(request) {
const body = await request.text();
const selector = body.includes("You choose which capabilities to load");
if (selector) selectorRequests++;
else finalRequests++;
if (selector === (hold === "selector")) {
request.signal.addEventListener("abort", () => closed.resolve(), {
once: true,
});
entered.resolve();
await release.promise;
handled.resolve();
return Response.json({
choices: [{ message: { content: '{"skills":["read"]}' } }],
});
}
return fetch(`${llm.url}${new URL(request.url).pathname}`, {
method: "POST",
headers: { "content-type": "application/json" },
body,
});
},
});
return {
url: `${server.url.origin}/v1`,
entered,
closed,
release,
handled,
counts: () => ({ selectorRequests, finalRequests }),
async [Symbol.asyncDispose]() {
release.resolve();
await server.stop(true);
},
};
}
test("Stop cancels the production selector HTTP connection before a late reply can start the final model", async () => {
await using f = await fixture();
await using endpoint = await selectorEndpoint(f.llm);
process.env.OPENAI_BASE_URL = endpoint.url;
let records = 0;
const agent = await f.agent(
createModelCompleter({
model,
resolveApiKey: async () => "synthetic-fixture-key",
}),
undefined,
async () => {
records++;
},
);
const run = observe(agent);
try {
await bounded(endpoint.entered.promise, "request arrival");
agent.abortRun();
// The server observes the request abort while its response is still held.
await bounded(endpoint.closed.promise, "server request abort");
await bounded(run.settled);
endpoint.release.resolve();
await bounded(endpoint.handled.promise, "late response handler");
await new Promise<void>((resolve) => setImmediate(resolve));
expect(endpoint.counts()).toEqual({
selectorRequests: 1,
finalRequests: 0,
});
expect(f.llm.getRequests()).toHaveLength(0);
expect(records).toBe(0);
expect(run.events).toHaveLength(0);
expect(run.completions()).toBe(1);
console.log(
JSON.stringify({
proof: "production-selector-HTTP-stop",
...endpoint.counts(),
serverRequestAbortedBeforeRelease: true,
selectionRecords: records,
completions: run.completions(),
}),
);
} finally {
endpoint.release.resolve();
run.subscription.unsubscribe();
}
});
test("Stop after the build still aborts the inner agent and its model HTTP connection", async () => {
await using f = await fixture();
await using endpoint = await selectorEndpoint(f.llm, "final");
process.env.OPENAI_BASE_URL = endpoint.url;
const agent = await f.agent(undefined, async () => []);
const abort = spyOn(BuiltInAgent.prototype, "abortRun");
const run = observe(agent);
try {
await bounded(endpoint.entered.promise, "request arrival");
agent.abortRun();
await bounded(endpoint.closed.promise, "server request abort");
await bounded(run.settled);
expect(abort).toHaveBeenCalledTimes(1);
expect(endpoint.counts()).toEqual({
selectorRequests: 0,
finalRequests: 1,
});
expect(run.completions()).toBe(1);
console.log(
JSON.stringify({
proof: "post-build-model-HTTP-stop",
...endpoint.counts(),
serverRequestAbortedBeforeRelease: true,
innerAborts: abort.mock.calls.length,
}),
);
} finally {
endpoint.release.resolve();
await bounded(endpoint.handled.promise, "late response handler");
run.subscription.unsubscribe();
abort.mockRestore();
}
});
test("an aborted pending run does not cancel its clone or the next run on the same wrapper", async () => {
await using f = await fixture();
const gate = deferred<string | null>();
const entered = deferred<void>();
let selections = 0;
const agent = await f.agent(async () => {
selections++;
entered.resolve();
return selections === 1 ? gate.promise : '{"skills":["read"]}';
});
const first = observe(agent);
const clone = agent.clone();
try {
// Learning acquisition may precede selection. Abort the build held by this fixture,
// rather than racing to stop before its first selector call has even begun.
await bounded(entered.promise, "selector entry");
agent.abortRun();
await bounded(first.settled);
// Finish the next run before the cancelled build resolves.
for (const next of [agent, clone]) {
next.setMessages(input.messages);
await bounded(next.runAgent());
expect(next.messages.at(-1)).toMatchObject({
role: "assistant",
content: "Fixture completed.",
});
}
gate.resolve('{"skills":["read"]}');
await new Promise<void>((resolve) => setImmediate(resolve));
expect(f.llm.getRequests()).toHaveLength(2);
expect(first.events).toHaveLength(0);
expect(first.completions()).toBe(1);
expect(selections).toBe(3);
} finally {
agent.abortRun();
clone.abortRun();
gate.resolve(null);
first.subscription.unsubscribe();
}
});
async function remoteSseEndpoint() {
const entered = deferred<void>();
const aborted = deferred<void>();
const release = deferred<void>();
const finished = deferred<void>();
let requests = 0;
const bodies: RunAgentInput[] = [];
const encoder = new TextEncoder();
const server = Bun.serve({
hostname: "127.0.0.1",
port: 0,
async fetch(request) {
requests++;
const body = (await request.clone().json()) as RunAgentInput;
bodies.push(body);
request.signal.addEventListener("abort", () => aborted.resolve(), {
once: true,
});
entered.resolve();
const stream = new ReadableStream<Uint8Array>({
async start(controller) {
controller.enqueue(
encoder.encode(
`data: ${JSON.stringify({
type: EventType.RUN_STARTED,
threadId: body.threadId,
runId: body.runId,
input: body,
})}\n\n`,
),
);
await release.promise;
controller.enqueue(
encoder.encode(
`data: ${JSON.stringify({
type: EventType.RUN_FINISHED,
threadId: body.threadId,
runId: body.runId,
result: "released",
})}\n\n`,
),
);
controller.close();
finished.resolve();
},
cancel() {
aborted.resolve();
},
});
return new Response(stream, {
headers: { "content-type": "text/event-stream" },
});
},
});
return {
server,
entered,
aborted,
release,
finished,
url: server.url.href,
counts: () => ({ requests, bodies: [...bodies] }),
async [Symbol.asyncDispose]() {
release.resolve();
await server.stop(true);
},
};
}
type RemoteSseEndpoint = Awaited<ReturnType<typeof remoteSseEndpoint>>;
async function resolvesWithin<T>(
promise: Promise<T>,
ms: number,
): Promise<boolean> {
let timer: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
promise.then(() => true),
new Promise<boolean>((resolve) => {
timer = setTimeout(() => resolve(false), ms);
}),
]);
} finally {
clearTimeout(timer);
}
}
async function runRemoteAndStop(
label: string,
agent: AbstractAgent,
endpoint: RemoteSseEndpoint,
) {
const started = deferred<void>();
const run = agent
.runAgent(input, {
onRunStartedEvent() {
started.resolve();
},
})
.catch((error: unknown) => ({
error: String(error instanceof Error ? error.message : error),
}));
await bounded(endpoint.entered.promise, `${label} request arrival`);
await bounded(started.promise, `${label} RUN_STARTED event`);
const closedBeforeStop = await resolvesWithin(endpoint.aborted.promise, 100);
agent.abortRun();
const closedAfterStop = await resolvesWithin(endpoint.aborted.promise, 1000);
endpoint.release.resolve();
await Promise.race([run, resolvesWithin(endpoint.finished.promise, 1500)]);
return { closedBeforeStop, closedAfterStop, ...endpoint.counts() };
}
test("Stop aborts a production wrapped remote AG-UI HTTP stream", async () => {
const never = Promise.withResolvers<void>();
expect(await resolvesWithin(never.promise, 20)).toBe(false);
expect(await resolvesWithin(Promise.resolve(), 1500)).toBe(true);
await using directEndpoint = await remoteSseEndpoint();
const direct = await runRemoteAndStop(
"direct remote",
new HttpAgent({ url: directEndpoint.url }),
directEndpoint,
);
expect(direct.closedBeforeStop).toBe(false);
expect(direct.closedAfterStop).toBe(true);
expect(direct.requests).toBe(1);
expect(direct.bodies[0]?.runId).toBe(input.runId);
expect(typeof direct.bodies[0]?.threadId).toBe("string");
await using wrappedEndpoint = await remoteSseEndpoint();
const agents = await buildAgents(
[
{
id: "remote-fixture",
name: "Remote Fixture",
type: "remote_ag_ui",
endpoint: wrappedEndpoint.url,
standingMessage: {
id: "standing-role:remote-fixture",
role: "system",
content: "You are Remote Fixture.",
},
},
],
model,
"unused-synthetic-key",
);
const wrapped = agents["remote-fixture"];
if (!wrapped) throw new Error("remote fixture not built");
const wrappedResult = await runRemoteAndStop(
"production wrapped remote",
wrapped,
wrappedEndpoint,
);
expect(wrappedResult.closedBeforeStop).toBe(false);
expect(wrappedResult.closedAfterStop).toBe(true);
expect(wrappedResult.requests).toBe(1);
expect(wrappedResult.bodies[0]?.runId).toBe(input.runId);
expect(typeof wrappedResult.bodies[0]?.threadId).toBe("string");
expect(wrappedResult.bodies[0]?.messages[0]).toMatchObject({
id: "standing-role:remote-fixture",
role: "system",
});
console.log(
JSON.stringify({
proof: "production-wrapped-remote-http-stop",
direct: {
closedBeforeStop: direct.closedBeforeStop,
closedAfterStop: direct.closedAfterStop,
requests: direct.requests,
},
wrapped: {
closedBeforeStop: wrappedResult.closedBeforeStop,
closedAfterStop: wrappedResult.closedAfterStop,
requests: wrappedResult.requests,
},
}),
);
});