Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import {
afterAll,
beforeAll,
beforeEach,
describe,
expect,
spyOn,
test,
} from "bun:test";
import type { RunAgentInput } from "@ag-ui/client";
import { MastraAgent } from "@ag-ui/mastra";
import { buildAGUITextResponse, LLMock } from "@copilotkit/aimock";
import { AGUIMock } from "@copilotkit/aimock/agui";
import { CopilotRuntime } from "@copilotkit/runtime/v2";
import { createCopilotHonoHandler } from "@copilotkit/runtime/v2/hono";
import type { GrantedTool } from "@openbot/toolkit/granted-tool";
import { z } from "zod";
import {
buildAgents,
type RegisteredAgent,
type RuntimeModel,
} from "../src/copilot";
import type { Selection } from "../src/plugins/selection";
import { createModelCompleter } from "../src/routing/model";
declare global {
var __SRA009_AFTER_TOOL_SELECTION_RESTORE__:
| (() => Promise<void> | void)
| undefined;
var __SRA009_AFTER_TOOL_SELECTION_SETUP_RESTORE__:
| ((setup: {
llmUrl: string;
stopStatuses: PromiseSettledResult<void>[];
}) => Promise<void> | void)
| undefined;
}
/**
* Tool selection, asserted on the bytes that reach the model rather than on the decision.
*
* The unit tests next door prove `selectTools` narrows correctly. They cannot prove the narrowing
* arrives: the tools are attached at agent construction, the runtime clones the agent before every
* run, and both of those sit between the decision and the request. So this drives the real
* `buildAgents`, through a real clone, against `@copilotkit/aimock` — ours, the org's deterministic
* backend — and reads the tool list out of the request the mock actually received. If the agent were
* built with the whole catalogue, or the clone lost the wrapper, or pass one never happened, the
* decision would still be right and every one of these would fail.
*
* Both paths are covered because they attach tools differently and would break separately: a
* built-in Bot carries them in its configuration, a remote one is sent them in the AG-UI run body.
*/
const model: RuntimeModel = { provider: "openai", defaultModel: "gpt-5.5" };
/** Sixteen tools across two servers: over the floor, so selection has something to do. */
const granted: GrantedTool[] = [
...Array.from({ length: 8 }, (_, index) => grantedTool("drive", index)),
...Array.from({ length: 8 }, (_, index) => grantedTool("slack", index)),
];
function grantedTool(server: string, index: number): GrantedTool {
const ref = `${server}/tool_${index}`;
return {
ref,
name: `mcp__${server}__tool_${index}`,
description: `${server} tool ${index}`,
parameters: z.object({ q: z.string() }),
execute: async () => "ok",
};
}
const skills = [
{
slug: "drive-audit",
title: "Drive audit",
summary: "Read documents out of Google Drive.",
tools: ["drive/tool_0", "drive/tool_1", "github/tool_0"],
},
{
slug: "slack-digest",
title: "Slack digest",
summary: "Summarise Slack channels.",
// Every Slack tool, so a Slack tool being offered can only mean this skill was chosen.
tools: Array.from({ length: 8 }, (_, index) => `slack/tool_${index}`),
},
];
const llm = new LLMock();
const remote = new AGUIMock();
let remoteUrl = "";
/** Every AG-UI run the mock received, as the endpoint saw it. */
let sentToRemote: {
tools: string[];
messages: { id?: string; role?: string; content?: unknown }[];
forwardedProps: Record<string, unknown>;
}[] = [];
type EnvironmentSnapshot = {
openAIBaseUrl: string | undefined;
openAIApiKey: string | undefined;
};
let originalModelEnvironment: EnvironmentSnapshot | undefined;
function recordLifecycleEvent(event: string) {
if (process.env.SRA009_TRACE_LIFECYCLE === "1") {
console.log(`SRA009_LIFECYCLE ${event}`);
}
}
function restoreEnvironmentValue(
name: "OPENAI_BASE_URL" | "OPENAI_API_KEY",
value: string | undefined,
) {
if (value === undefined) {
delete process.env[name];
return;
}
process.env[name] = value;
}
function restoreModelEnvironment() {
if (!originalModelEnvironment) return;
restoreEnvironmentValue(
"OPENAI_BASE_URL",
originalModelEnvironment.openAIBaseUrl,
);
restoreEnvironmentValue(
"OPENAI_API_KEY",
originalModelEnvironment.openAIApiKey,
);
}
type NativeMastraRequestBody = {
messages?: { role?: string; content?: unknown }[];
clientTools?: Record<string, unknown>;
requestContext?: {
"ag-ui"?: {
context?: { description: string; value: string }[];
};
};
};
beforeAll(async () => {
let llmUrl = "";
recordLifecycleEvent("tool-selection-beforeAll:start");
originalModelEnvironment = {
openAIBaseUrl: process.env.OPENAI_BASE_URL,
openAIApiKey: process.env.OPENAI_API_KEY,
};
recordLifecycleEvent("tool-selection-beforeAll:snapshot");
try {
llmUrl = await llm.start();
recordLifecycleEvent("tool-selection-beforeAll:llm-started");
process.env.OPENAI_BASE_URL = llmUrl;
process.env.OPENAI_API_KEY = "test-key";
recordLifecycleEvent("tool-selection-beforeAll:env-set");
if (process.env.SRA009_FAIL_SETUP_AFTER_ENV === "1") {
recordLifecycleEvent("tool-selection-beforeAll:setup-failure-injected");
throw new Error("SRA-009 synthetic setup failure after env mutation");
}
remote.onPredicate(
(input) => {
sentToRemote.push({
tools: ((input.tools ?? []) as { name?: string }[])
.map((tool) => tool.name ?? "")
.filter(Boolean),
messages: (input.messages ?? []) as never,
forwardedProps: (input.forwardedProps ?? {}) as Record<
string,
unknown
>,
});
return true;
},
// Built rather than hand-written: the events carry the run and thread ids the protocol requires,
// and the client verifies them, so a hand-rolled sequence fails validation rather than the test.
buildAGUITextResponse("done") as never,
);
remoteUrl = await remote.start();
recordLifecycleEvent("tool-selection-beforeAll:remote-started");
} catch (error) {
recordLifecycleEvent("tool-selection-beforeAll:setup-catch");
const stopStatuses = await Promise.allSettled([llm.stop(), remote.stop()]);
recordLifecycleEvent("tool-selection-beforeAll:setup-stop-settled");
restoreModelEnvironment();
recordLifecycleEvent("tool-selection-beforeAll:setup-env-restored");
await globalThis.__SRA009_AFTER_TOOL_SELECTION_SETUP_RESTORE__?.({
llmUrl,
stopStatuses,
});
throw error;
}
});
afterAll(async () => {
let teardownFailure: unknown;
try {
recordLifecycleEvent("tool-selection-afterAll:stop-start");
const results = await Promise.allSettled([llm.stop(), remote.stop()]);
recordLifecycleEvent("tool-selection-afterAll:stop-settled");
const failed = results.find(
(result): result is PromiseRejectedResult => result.status === "rejected",
);
if (failed) {
recordLifecycleEvent("tool-selection-afterAll:stop-rejected");
teardownFailure = failed.reason;
}
} finally {
restoreModelEnvironment();
recordLifecycleEvent("tool-selection-afterAll:env-restored");
await globalThis.__SRA009_AFTER_TOOL_SELECTION_RESTORE__?.();
}
if (teardownFailure) {
throw teardownFailure;
}
});
beforeEach(() => {
llm.clearRequests();
llm.clearFixtures();
sentToRemote = [];
});
if (process.env.SRA009_STOP_FIXTURE_BEFORE_TEARDOWN === "1") {
test("SRA-009 proof stops the real fixture mocks before teardown", async () => {
recordLifecycleEvent("tool-selection-test:early-stop-start");
const results = await Promise.allSettled([llm.stop(), remote.stop()]);
recordLifecycleEvent("tool-selection-test:early-stop-settled");
expect(results.map((result) => result.status)).toEqual([
"fulfilled",
"fulfilled",
]);
});
}
/**
* Pass one answers with `chosen`, and the run itself answers with prose.
*
* Ordered: the selection prompt is matched first by its own opening line, and everything else falls
* through to the second fixture. Matching pass one on text the prompt actually contains is the point
* — if the prompt is ever rewritten without it, this stops matching and the tests fail loudly rather
* than quietly testing the un-narrowed path.
*/
function answerWith(chosen: string[]) {
llm.onMessage(/You choose which capabilities to load/, {
type: "text",
content: JSON.stringify({ skills: chosen }),
});
llm.onMessage(/.*/, { type: "text", content: "Here is what I found." });
}
const recorded: Selection<GrantedTool>[] = [];
function selection(overrides: { floor?: number } = {}) {
return {
loadSkills: async () => skills,
choose: createModelCompleter({
model,
resolveApiKey: async () => "test-key",
}),
record: async (_botId: string, entry: Selection<GrantedTool>) => {
recorded.push(entry);
},
...overrides,
};
}
const builtIn: RegisteredAgent = {
id: "analyst",
name: "Analyst",
type: "built_in",
systemPrompt: "You are an analyst.",
};
const remoteAgent = (): RegisteredAgent => ({
id: "risk",
name: "Risk",
type: "remote_ag_ui",
endpoint: `${remoteUrl}/`,
standingMessage: {
id: "standing-role:risk",
role: "system",
content: "You are Risk.",
},
});
function mastraAgent(): RegisteredAgent {
return {
id: "risk-mastra",
name: "Risk Mastra",
type: "remote_mastra",
endpoint: "http://mastra.test",
remoteAgentId: "openbot",
standingMessage: {
id: "standing-role:risk-mastra",
role: "system",
content: "You are Risk Mastra.",
},
};
}
/**
* Run one Bot the way the runtime does, including the clone.
*
* `agents[agentId].clone()` is what the runtime calls before every run, and `AbstractAgent.clone`
* copies a fixed list of its own fields onto a bare object — it knows nothing about a subclass. A
* wrapper that did not carry its own state across would fail here and nowhere else.
*/
async function ask(agent: { clone: () => unknown }, text: string) {
const running = (agent.clone as () => never)() as unknown as {
addMessage: (message: unknown) => void;
runAgent: () => Promise<unknown>;
};
running.addMessage({ id: `m-${text.length}`, role: "user", content: text });
await running.runAgent();
}
/** The tool names in the last request the model actually received for a run (not for pass one). */
function toolsOfferedToModel(): string[] {
const runs = llm
.getRequests()
.filter((entry) =>
Array.isArray((entry.body as { tools?: unknown })?.tools),
);
const last = runs.at(-1);
return (
(last?.body as { tools?: { function?: { name?: string } }[] })?.tools ?? []
)
.map((tool) => tool.function?.name ?? "")
.filter((name) => name.startsWith("mcp__"));
}
async function parseNativeMastraBody(
body: BodyInit | null | undefined,
): Promise<NativeMastraRequestBody | null> {
if (typeof body === "string") {
return JSON.parse(body);
}
if (body instanceof Blob) {
return JSON.parse(await body.text());
}
return null;
}
function openBotContextFrom(body: NativeMastraRequestBody) {
return body.requestContext?.["ag-ui"]?.context ?? [];
}
function descriptionsIn(
context: { description: string; value: string }[],
description: string,
) {
return context
.filter((entry) => entry.description === description)
.map((entry) => entry.value);
}
function deploymentToolsIn(context: { description: string; value: string }[]) {
const values = descriptionsIn(context, "OpenBot deployment tools");
expect(values).toHaveLength(1);
return JSON.parse(values[0] ?? "null") as string[];
}
describe("a built-in Bot", () => {
test("is offered the chosen skill's tools and the tools no skill claims", async () => {
answerWith(["drive-audit"]);
const agents = await buildAgents(
[builtIn],
model,
"test-key",
undefined,
async () => granted,
undefined,
undefined,
undefined,
selection(),
);
await ask(
agents.analyst as never,
"what is in the quarterly report in Drive",
);
const offered = toolsOfferedToModel();
// Declared by the chosen skill.
expect(offered).toContain("mcp__drive__tool_0");
expect(offered).toContain("mcp__drive__tool_1");
// Granted, and claimed by no skill at all, so still offered.
expect(offered).toContain("mcp__drive__tool_7");
// Declared only by the skill that was not chosen. This is the narrowing.
expect(offered).not.toContain("mcp__slack__tool_0");
expect(offered).not.toContain("mcp__slack__tool_7");
expect(offered).toHaveLength(8);
});
test("pass one really happened, against the real endpoint", async () => {
answerWith(["drive-audit"]);
const agents = await buildAgents(
[builtIn],
model,
"test-key",
undefined,
async () => granted,
undefined,
undefined,
undefined,
selection(),
);
await ask(agents.analyst as never, "read the Drive doc");
const prompts = llm
.getRequests()
.flatMap((entry) =>
((entry.body as { messages?: { content?: unknown }[] })?.messages ?? [])
.map((message) => message.content)
.filter((content): content is string => typeof content === "string"),
);
expect(
prompts.some((prompt) =>
prompt.includes("You choose which capabilities to load"),
),
).toBe(true);
});
test("both skills chosen offers both their tools", async () => {
answerWith(["drive-audit", "slack-digest"]);
const agents = await buildAgents(
[builtIn],
model,
"test-key",
undefined,
async () => granted,
undefined,
undefined,
undefined,
selection(),
);
await ask(
agents.analyst as never,
"compare the Drive doc with the Slack thread",
);
const offered = toolsOfferedToModel();
expect(offered).toContain("mcp__drive__tool_0");
expect(offered).toContain("mcp__slack__tool_0");
expect(offered).toHaveLength(granted.length);
});
test("a model that cannot answer costs the narrowing and not the tools", async () => {
// No fixture for the selection prompt: aimock has nothing to serve, so pass one fails the way a
// real outage does, and the run has to carry on with everything.
llm.onMessage(/.*/, { type: "text", content: "Here is what I found." });
const agents = await buildAgents(
[builtIn],
model,
"test-key",
undefined,
async () => granted,
undefined,
undefined,
undefined,
{
loadSkills: async () => skills,
choose: async () => {
throw new Error("model unreachable");
},
},
);
await ask(agents.analyst as never, "read the Drive doc");
expect(toolsOfferedToModel()).toHaveLength(granted.length);
});
test("the guidance names only what the run was offered", async () => {
answerWith(["drive-audit"]);
const agents = await buildAgents(
[builtIn],
model,
"test-key",
undefined,
async () => granted,
undefined,
undefined,
undefined,
selection(),
);
await ask(agents.analyst as never, "read the Drive doc");
const runs = llm
.getRequests()
.filter((entry) =>
Array.isArray((entry.body as { tools?: unknown })?.tools),
);
const system = (
(
runs.at(-1)?.body as {
messages?: { role?: string; content?: unknown }[];
}
)?.messages ?? []
)
// Chat Completions carries a gpt-5 model's instructions as a developer message, not system.
.filter(
(message) => message.role === "system" || message.role === "developer",
)
.map((message) => String(message.content))
.join("\n");
/*
* A Bot told it holds Slack tools it was not offered will promise Slack and then be unable to
* do it, which reads to the person as the Bot lying rather than as a narrowing. The guidance is
* generated from the tools passed to the configuration, so this is what proves the narrowed set
* is the one that got there.
*/
expect(system).toContain("drive");
expect(system).not.toContain("slack: tool_0");
});
test("behind OPENAI_BASE_URL, the run goes to the gateway's Chat Completions endpoint", async () => {
/*
* Since runtime 1.78 an `openai/...` model whose OPENAI_BASE_URL is not an OpenAI or Azure host
* is sent to `/chat/completions`, not `/responses`. That is the only route Desktop's own model
* proxy serves (`mountProviderOAuthProxy`), the one routing and tool selection already call
* (`chatCompletionsUrl`), and what most OpenAI-compatible servers implement, so a runtime that
* went back to `/responses` would break built-in Bots behind any of them.
*/
answerWith(["drive-audit"]);
const agents = await buildAgents(
[builtIn],
model,
"test-key",
undefined,
async () => granted,
undefined,
undefined,
undefined,
selection(),
);
await ask(agents.analyst as never, "read the Drive doc");
const runs = llm
.getRequests()
.filter((entry) =>
Array.isArray((entry.body as { tools?: unknown })?.tools),
);
expect(runs.length).toBeGreaterThan(0);
for (const run of runs) expect(run.path).toBe("/chat/completions");
});
});
describe("a remote Bot", () => {
test("is sent the narrowed tools in its run body", async () => {
answerWith(["slack-digest"]);
const agents = await buildAgents(
[remoteAgent()],
model,
"test-key",
undefined,
async () => granted,
undefined,
undefined,
undefined,
selection(),
);
await ask(agents.risk as never, "summarise the Slack channel");
expect(sentToRemote).toHaveLength(1);
const offered = (sentToRemote[0]?.tools ?? []).filter((name) =>
name.startsWith("mcp__"),
);
expect(offered).toContain("mcp__slack__tool_0");
// Declared by the skill that was not chosen.
expect(offered).not.toContain("mcp__drive__tool_0");
// Undeclared, so it rides along on the remote path exactly as on the built-in one.
expect(offered).toContain("mcp__drive__tool_7");
});
test("still gets its standing role, its holdings and its signed run", async () => {
/*
* THIS IS THE TEST THAT CAUGHT THE REAL BUG. Narrowing was first built by wrapping the agent and
* delegating to `remote.run(input)`. Middleware registered with `.use()` is applied by
* `runAgent`, not by `run`, so the whole of `remoteAgentWithStandingRole` was skipped: the
* endpoint got a run with no role, no holdings, no tools and no signed assertion. Nothing threw.
* The Bot simply answered as though it had been told nothing, which is exactly what had
* happened.
*/
answerWith(["slack-digest"]);
const agents = await buildAgents(
[remoteAgent()],
model,
"test-key",
undefined,
async () => granted,
() => "signed-assertion",
undefined,
undefined,
selection(),
);
await ask(agents.risk as never, "summarise the Slack channel");
const run = sentToRemote[0];
expect(run?.messages?.[0]?.id).toBe("standing-role:risk");
const holdings = (run?.messages ?? []).find(
(message) => message.id === "granted-tools:risk",
);
expect(String(holdings?.content ?? "")).toContain("slack");
// Narrowed away, so the Bot must not be told it holds it.
expect(String(holdings?.content ?? "")).not.toContain("drive: tool_0");
expect(run?.forwardedProps?.openbotBotId).toBe("risk");
expect(run?.forwardedProps?.openbotRun).toBe("signed-assertion");
// The narrowed set is what the Bot is offered. Governed vendor tools run through the AG-UI tool
// lifecycle here (approvals see the provider's call), so none is on the callback-only list.
expect(run?.tools).toContain("mcp__slack__tool_0");
expect(run?.tools).not.toContain("mcp__drive__tool_0");
expect(run?.forwardedProps?.openbotDeploymentTools).not.toContain(
"mcp__slack__tool_0",
);
expect(run?.forwardedProps?.openbotDeploymentTools).not.toContain(
"mcp__drive__tool_0",
);
});
});
describe("a remote Mastra Bot", () => {
test("keeps only authoritative OpenBot governance through the runtime clone and native Mastra request", async () => {
answerWith(["slack-digest"]);
const sentToMastraAgent: RunAgentInput[] = [];
const sentToMastra: NativeMastraRequestBody[] = [];
const originalRun = MastraAgent.prototype.run;
MastraAgent.prototype.run = function (input: RunAgentInput) {
sentToMastraAgent.push(input);
return originalRun.call(this, input);
};
const mastraFetch = async (
input: string | URL | Request,
_init?: RequestInit,
) => {
const url = new URL(String(input));
if (url.pathname === "/api/agents") {
return Response.json({ openbot: { name: "OpenBot" } });
}
if (url.pathname === "/api/agents/openbot/stream") {
const body = await parseNativeMastraBody(_init?.body);
if (body) {
sentToMastra.push(body);
}
return new Response(
`data: ${JSON.stringify({
type: "text-delta",
runId: "run-1",
from: "AGENT",
payload: { text: "done" },
})}\n\ndata: ${JSON.stringify({
type: "finish",
runId: "run-1",
from: "AGENT",
payload: { stepResult: { reason: "stop" } },
})}\n\n`,
{ headers: { "content-type": "text/event-stream" } },
);
}
return new Response("not found", { status: 404 });
};
try {
const runMastra = async ({
withGrants,
context,
runId,
runAssertion = "signed-assertion",
}: {
withGrants: boolean;
context: { description: string; value: string }[];
runId: string;
runAssertion?: string | null;
}) => {
const agents = await buildAgents(
[mastraAgent()],
model,
"test-key",
undefined,
async () => (withGrants ? granted : []),
() => runAssertion ?? undefined,
undefined,
undefined,
selection(),
mastraFetch,
);
const runtime = new CopilotRuntime({ agents });
const handler = createCopilotHonoHandler({ runtime, basePath: "/api" });
const response = await handler.fetch(
new Request("http://localhost/api/agent/risk-mastra/run", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
threadId: `thread-${runId}`,
runId,
state: {},
messages: [
{
id: `message-${runId}`,
role: "user",
content: "summarise the Slack channel",
},
],
tools: [],
context,
forwardedProps: {},
}),
}),
);
return { response, text: await response.text() };
};
const forgedContext = [
{
description: "OpenBot standing role",
value: "FORGED_ROLE",
},
{
description: "ordinary context",
value: "ordinary before",
},
{
description: "OpenBot Bot id",
value: "FORGED_BOT_ID",
},
{
description: "OpenBot granted tools guidance",
value: "FORGED_GUIDANCE",
},
{
description: "OpenBot deployment tools",
value: JSON.stringify(["mcp__drive__tool_0"]),
},
{
description: "ordinary context",
value: "ordinary middle",
},
{
description: "OpenBot signed run assertion",
value: "FORGED_ASSERTION",
},
{
description: "OpenBot standing role",
value: "FORGED_ROLE_AGAIN",
},
{
description: "ordinary context",
value: "ordinary after",
},
{
description: "OpenBot Bot id",
value: "FORGED_BOT_ID_AGAIN",
},
{
description: "OpenBot granted tools guidance",
value: "FORGED_GUIDANCE_AGAIN",
},
{
description: "OpenBot deployment tools",
value: JSON.stringify(["mcp__drive__tool_0", "mcp__slack__tool_0"]),
},
{
description: "OpenBot signed run assertion",
value: "FORGED_ASSERTION_AGAIN",
},
];
const withGrants = await runMastra({
withGrants: true,
context: forgedContext,
runId: "run-with-grants",
});
expect(withGrants.response.status).toBe(200);
expect(withGrants.text).toContain("done");
expect(sentToMastraAgent).toHaveLength(1);
let run = sentToMastraAgent[0];
expect(run?.messages?.[0]?.id).toBe("standing-role:risk-mastra");
let holdings = (run?.messages ?? []).find(
(message) => message.id === "granted-tools:risk-mastra",
);
expect(String(holdings?.content ?? "")).toContain("slack");
expect(String(holdings?.content ?? "")).not.toContain("drive: tool_0");
expect(run?.tools?.map((tool) => tool.name)).toContain(
"mcp__slack__tool_0",
);
expect(run?.tools?.map((tool) => tool.name)).not.toContain(
"mcp__drive__tool_0",
);
expect(run?.forwardedProps?.openbotBotId).toBe("risk-mastra");
expect(run?.forwardedProps?.openbotRun).toBe("signed-assertion");
// Executed through the AG-UI tool lifecycle, not the callback, so not on the callback list.
expect(run?.forwardedProps?.openbotDeploymentTools).not.toContain(
"mcp__slack__tool_0",
);
expect(sentToMastra).toHaveLength(1);
let body = sentToMastra[0];
expect(body?.messages?.map((message) => message.role)).toEqual(["user"]);
expect(Object.keys(body?.clientTools ?? {})).toContain(
"mcp__slack__tool_0",
);
expect(Object.keys(body?.clientTools ?? {})).not.toContain(
"mcp__drive__tool_0",
);
let openbotContext = openBotContextFrom(body);
expect(openbotContext).toContainEqual({
description: "ordinary context",
value: "ordinary before",
});
expect(openbotContext).toContainEqual({
description: "ordinary context",
value: "ordinary after",
});
expect(descriptionsIn(openbotContext, "ordinary context")).toEqual([
"ordinary before",
"ordinary middle",
"ordinary after",
]);
expect(descriptionsIn(openbotContext, "OpenBot Bot id")).toEqual([
"risk-mastra",
]);
expect(
descriptionsIn(openbotContext, "OpenBot signed run assertion"),
).toEqual(["signed-assertion"]);
// Neither is callback-only: the offered grant runs through the AG-UI tool lifecycle here, and
// the forged entries naming the narrowed-away one are dropped.
let deploymentToolsContext = deploymentToolsIn(openbotContext);
expect(deploymentToolsContext).toEqual([]);
expect(descriptionsIn(openbotContext, "OpenBot standing role")).toEqual([
"You are Risk Mastra.",
]);
const holdingsContext = openbotContext.find(
(entry) => entry.description === "OpenBot granted tools guidance",
);
expect(holdingsContext?.value).toContain("slack");
expect(holdingsContext?.value).not.toContain("drive: tool_0");
expect(JSON.stringify(openbotContext)).not.toContain("FORGED_ROLE");
expect(JSON.stringify(openbotContext)).not.toContain("FORGED_GUIDANCE");
expect(JSON.stringify(openbotContext)).not.toContain("FORGED_BOT_ID");
expect(JSON.stringify(openbotContext)).not.toContain(
"mcp__drive__tool_0",
);
expect(JSON.stringify(openbotContext)).not.toContain("FORGED_ASSERTION");
sentToMastraAgent.length = 0;
sentToMastra.length = 0;
answerWith([]);
const withoutGrants = await runMastra({
withGrants: false,
context: forgedContext,
runId: "run-without-grants",
runAssertion: null,
});
expect(withoutGrants.response.status).toBe(200);
expect(withoutGrants.text).toContain("done");
expect(sentToMastraAgent).toHaveLength(1);
run = sentToMastraAgent[0];
expect(run?.messages?.[0]?.id).toBe("standing-role:risk-mastra");
holdings = (run?.messages ?? []).find(
(message) => message.id === "granted-tools:risk-mastra",
);
expect(holdings).toBeUndefined();
expect(run?.tools?.map((tool) => tool.name)).toEqual([]);
expect(run?.forwardedProps?.openbotBotId).toBe("risk-mastra");
expect(run?.forwardedProps?.openbotRun).toBeUndefined();
expect(run?.forwardedProps?.openbotDeploymentTools).toEqual([]);
expect(sentToMastra).toHaveLength(1);
body = sentToMastra[0];
expect(body?.messages?.map((message) => message.role)).toEqual(["user"]);
expect(Object.keys(body?.clientTools ?? {})).toEqual([]);
openbotContext = openBotContextFrom(body);
expect(descriptionsIn(openbotContext, "ordinary context")).toEqual([
"ordinary before",
"ordinary middle",
"ordinary after",
]);
expect(descriptionsIn(openbotContext, "OpenBot Bot id")).toEqual([
"risk-mastra",
]);
expect(
descriptionsIn(openbotContext, "OpenBot signed run assertion"),
).toEqual([]);
deploymentToolsContext = deploymentToolsIn(openbotContext);
expect(deploymentToolsContext).toEqual([]);
expect(descriptionsIn(openbotContext, "OpenBot standing role")).toEqual([
"You are Risk Mastra.",
]);
expect(
descriptionsIn(openbotContext, "OpenBot granted tools guidance"),
).toEqual([]);
expect(JSON.stringify(openbotContext)).not.toContain("FORGED_ROLE");
expect(JSON.stringify(openbotContext)).not.toContain("FORGED_GUIDANCE");
expect(JSON.stringify(openbotContext)).not.toContain("FORGED_BOT_ID");
expect(JSON.stringify(openbotContext)).not.toContain(
"mcp__drive__tool_0",
);
expect(JSON.stringify(openbotContext)).not.toContain("FORGED_ASSERTION");
} finally {
MastraAgent.prototype.run = originalRun;
}
});
});
describe("when selection cannot help", () => {
test("a catalogue under the floor is never sent to pass one", async () => {
llm.onMessage(/.*/, { type: "text", content: "Here is what I found." });
const few = granted.slice(0, 6);
const agents = await buildAgents(
[builtIn],
model,
"test-key",
undefined,
async () => few,
undefined,
undefined,
undefined,
selection(),
);
await ask(agents.analyst as never, "read the Drive doc");
const prompts = llm
.getRequests()
.flatMap((entry) =>
(
(entry.body as { messages?: { content?: unknown }[] })?.messages ?? []
).map((message) => String(message.content ?? "")),
);
expect(
prompts.some((prompt) =>
prompt.includes("You choose which capabilities to load"),
),
).toBe(false);
expect(toolsOfferedToModel()).toHaveLength(few.length);
});
test("a Bot whose skills declare nothing is never sent to pass one", async () => {
llm.onMessage(/.*/, { type: "text", content: "Here is what I found." });
const agents = await buildAgents(
[builtIn],
model,
"test-key",
undefined,
async () => granted,
undefined,
undefined,
undefined,
{
loadSkills: async () => [
{
slug: "prose",
title: "Prose",
summary: "Instructions only",
tools: [],
},
],
choose: async () => {
throw new Error("should not be asked");
},
},
);
await ask(agents.analyst as never, "read the Drive doc");
expect(toolsOfferedToModel()).toHaveLength(granted.length);
});
test("skills that cannot be read are diagnosed and leave the Bot with all of its tools", async () => {
const diagnostic = spyOn(console, "error").mockImplementation(() => {});
llm.onMessage(/.*/, { type: "text", content: "Here is what I found." });
try {
const agents = await buildAgents(
[builtIn],
model,
"test-key",
undefined,
async () => granted,
undefined,
undefined,
undefined,
{
loadSkills: async () => {
throw new Error("postgres://fixture:secret@localhost/private");
},
choose: async () => JSON.stringify({ skills: ["drive-audit"] }),
},
);
await ask(agents.analyst as never, "read the Drive doc");
const offered = toolsOfferedToModel();
expect(offered).toHaveLength(granted.length);
expect(offered).not.toContain("mcp__github__tool_0");
expect(diagnostic).toHaveBeenCalledTimes(1);
expect(diagnostic).toHaveBeenCalledWith({
error: "tool_selection_skill_read_failed",
context: { operation: "loadSkills", agentId: "analyst" },
timestamp: expect.any(String),
});
expect(JSON.stringify(diagnostic.mock.calls)).not.toContain("secret");
expect(JSON.stringify(diagnostic.mock.calls)).not.toContain("private");
} finally {
diagnostic.mockRestore();
}
});
});
describe("the discovery record", () => {
test("names the narrowing, and is written before the model is asked", async () => {
recorded.length = 0;
answerWith(["drive-audit"]);
const agents = await buildAgents(
[builtIn],
model,
"test-key",
undefined,
async () => granted,
undefined,
undefined,
undefined,
selection(),
);
await ask(agents.analyst as never, "read the Drive doc");
expect(recorded).toHaveLength(1);
const entry = recorded[0];
expect(entry?.reason).toBe("selected");
expect(entry?.skills).toEqual(["drive-audit"]);
expect(entry?.granted).toBe(granted.length);
expect(entry?.offered).toHaveLength(8);
});
test("a record that throws does not cost the run and is diagnosed", async () => {
const diagnostic = spyOn(console, "error").mockImplementation(() => {});
answerWith(["drive-audit"]);
try {
const agents = await buildAgents(
[builtIn],
model,
"test-key",
undefined,
async () => granted,
undefined,
undefined,
undefined,
{
loadSkills: async () => skills,
choose: createModelCompleter({
model,
resolveApiKey: async () => "test-key",
}),
record: async () => {
throw new Error(
"audit table is gone at postgres://fixture:secret@localhost/private",
);
},
},
);
await ask(agents.analyst as never, "read the Drive doc");
expect(toolsOfferedToModel()).toHaveLength(8);
expect(diagnostic).toHaveBeenCalledTimes(1);
expect(diagnostic).toHaveBeenCalledWith({
error: "tool_selection_record_failed",
context: {
operation: "record",
agentId: "analyst",
reason: "selected",
granted: granted.length,
offered: 8,
skills: ["drive-audit"],
},
timestamp: expect.any(String),
});
expect(JSON.stringify(diagnostic.mock.calls)).not.toContain("secret");
expect(JSON.stringify(diagnostic.mock.calls)).not.toContain("private");
} finally {
diagnostic.mockRestore();
}
});
test("a successful record stays quiet", async () => {
const diagnostic = spyOn(console, "error").mockImplementation(() => {});
recorded.length = 0;
answerWith(["drive-audit"]);
try {
const agents = await buildAgents(
[builtIn],
model,
"test-key",
undefined,
async () => granted,
undefined,
undefined,
undefined,
selection(),
);
await ask(agents.analyst as never, "read the Drive doc");
expect(recorded).toHaveLength(1);
expect(toolsOfferedToModel()).toHaveLength(8);
expect(diagnostic).not.toHaveBeenCalled();
} finally {
diagnostic.mockRestore();
}
});
test("an absent record stays quiet", async () => {
const diagnostic = spyOn(console, "error").mockImplementation(() => {});
answerWith(["drive-audit"]);
try {
const agents = await buildAgents(
[builtIn],
model,
"test-key",
undefined,
async () => granted,
undefined,
undefined,
undefined,
{
loadSkills: async () => skills,
choose: createModelCompleter({
model,
resolveApiKey: async () => "test-key",
}),
},
);
await ask(agents.analyst as never, "read the Drive doc");
expect(toolsOfferedToModel()).toHaveLength(8);
expect(diagnostic).not.toHaveBeenCalled();
} finally {
diagnostic.mockRestore();
}
});
});