Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
Tell a Bot what it holds, not only hand it the tools (#143)
David McKay1213cfef780f
diff --git a/server/src/copilot.ts b/server/src/copilot.tsindex 766ce4f2..8582a85d 100644--- a/server/src/copilot.ts+++ b/server/src/copilot.ts@@ -8,6 +8,7 @@ import {import { createCopilotHonoHandler } from "@copilotkit/runtime/v2/hono";import { z } from "zod";import { COMPUTER_GUIDANCE } from "../../shared/bot-prompt";+import { grantedToolGuidance } from "./plugins/tools";import type { AgentActor } from "./agents/profile-types";import type { StallGuard } from "./channels/stall-guard";import type { DeploymentConfig } from "./config";@@ -200,9 +201,18 @@ export function builtInAgentConfiguration(return {model: `${model.provider}/${model.defaultModel}`,- prompt: computerGuidance- ? `${agent.systemPrompt}\n\n${computerGuidance}`- : agent.systemPrompt,+ /*+ * The package's role, then what this Bot actually holds, then the computer.+ *+ * The grants go BEFORE the computer prose on purpose. That prose is long and emphatic about the+ * browser and mentions connectors nowhere, so a Bot that read it last reached for the browser+ * even when it held a tool for the exact system being asked about.+ */+ prompt: [+ agent.systemPrompt,+ ...(grantedToolGuidance(tools) ? [grantedToolGuidance(tools)] : []),+ ...(computerGuidance ? [computerGuidance] : []),+ ].join("\n\n"),apiKey,/** A run stops after one step unless told otherwise, which for a Bot with tools means it calls@@ -327,13 +337,37 @@ function remoteAgentWithStandingRole(? { fetch: stallGuard.watch({ id: agent.id, name: agent.name }) }: {}),});+ /*+ * What this Bot holds, as a second standing message.+ *+ * Beside the role rather than inside it, because the role comes from the package and this comes+ * from the grants: they change for different reasons and at different times. Sent on every run for+ * the same reason the tools are, so switching a connector on reaches the next run.+ *+ * The remote path needs this more than the built-in one, not less. A framework Bot is handed the+ * tools as an offer and decides for itself what to call, with `COMPUTER_GUIDANCE` as its whole+ * prompt — a page about the browser that mentions connectors nowhere. That is the Bot that browsed+ * to drive.google.com holding four Drive tools.+ */+ const holdings = grantedToolGuidance(tools);+ const holdingsMessage = holdings+ ? {+ id: `granted-tools:${agent.id}`,+ role: "system" as const,+ content: holdings,+ }+ : null;+remote.use((input, next) =>next.run({...input,messages: [agent.standingMessage,+ ...(holdingsMessage ? [holdingsMessage] : []),...input.messages.filter(- (message) => message.id !== agent.standingMessage.id,+ (message) =>+ message.id !== agent.standingMessage.id &&+ message.id !== holdingsMessage?.id,),],/*diff --git a/server/src/plugins/tools.ts b/server/src/plugins/tools.tsindex 0bc35518..35aa4753 100644--- a/server/src/plugins/tools.ts+++ b/server/src/plugins/tools.ts@@ -48,6 +48,48 @@ export function parametersFor(inputSchema: Record<string, unknown>): z.ZodType {return z.object({}).catchall(z.unknown());}+/**+ * What this Bot holds, said in its instructions rather than left to be inferred from a tool list.+ *+ * A tool array tells a model a tool exists. It does not tell it that the tool is the right way to+ * reach that system, and it competes with a page of prose about the browser that every Bot is given+ * whether or not it has any connectors at all. The browser prose wins: it is emphatic, it is about+ * capability, and it says "never claim you cannot browse".+ *+ * So a Bot holding four Google Drive tools browsed to drive.google.com, met a sign-in page its+ * container could never satisfy, and asked its person to sign in to a vendor that person had already+ * connected. The tools were there the whole time.+ *+ * Generated from the grants rather than written down, because the point is that it tracks them. An+ * administrator switching a connector on, or granting one more of its tools, changes what the Bot is+ * told on its next run with nothing else to remember and nothing to keep in step.+ *+ * Empty when the Bot holds nothing, so a deployment with no connectors says nothing about them.+ */+export function grantedToolGuidance(tools: GrantedTool[]): string {+ if (tools.length === 0) return "";++ const bySystem = new Map<string, string[]>();+ for (const tool of tools) {+ // `mcp__server__tool`, which is the shape the model is offered.+ const parts = tool.name.replace(/^mcp__/, "").split("__");+ const system = parts.length > 1 ? (parts[0] as string) : "this deployment";+ const rest = parts.length > 1 ? parts.slice(1).join("__") : tool.name;+ bySystem.set(system, [...(bySystem.get(system) ?? []), rest]);+ }++ return [+ "You can reach these systems directly, as the person asking, with their own access:",+ ...[...bySystem.entries()].map(+ ([system, names]) => `- ${system}: ${names.join(", ")}`,+ ),+ "Use them for anything about those systems. Do NOT browse to one of their websites instead: your",+ "browser is signed in as nobody, so it sees less than these tools do and will meet a sign-in wall",+ "that connecting an account has already solved. If one of these systems is involved but no tool",+ "above covers the part you need, say which part is missing rather than going around it.",+ ].join("\n");+}+/*** Every MCP tool granted to one Bot, ready to hand to the runtime.*diff --git a/server/tests/copilot.test.ts b/server/tests/copilot.test.tsindex 5bd14d47..ee623640 100644--- a/server/tests/copilot.test.ts+++ b/server/tests/copilot.test.ts@@ -9,6 +9,7 @@ import {resolveRuntimeAgents,standingRoleMessage,} from "../src/copilot";+import { grantedToolGuidance } from "../src/plugins/tools";// Every agent row now joins its profile, so the row a coworker is built from always names it.const assistantRow = {@@ -502,3 +503,63 @@ function fakeAgUiEndpoint() {[Symbol.asyncDispose]: () => server.stop(true),};}++/**+ * A Bot is told what it holds, not only handed it.+ *+ * A tool array tells a model a tool exists. It does not say the tool is the right way to reach that+ * system, and it competes with `COMPUTER_GUIDANCE`: a page of emphatic prose about the browser that+ * every Bot gets whether or not it has a single connector, and that mentions connectors nowhere.+ *+ * The browser prose won. A Bot holding four Google Drive tools browsed to drive.google.com, met a+ * sign-in page its container could never satisfy, and asked its person to sign in to a vendor that+ * person had already connected. Asked a question with no tool for it, another went reading a+ * government website and looped on its 404 page.+ *+ * Both kinds are asserted because they are built by different functions, and the remote one is the+ * one that failed in the product.+ */+describe("what a Bot is told it holds", () => {+ const drive = [+ { name: "mcp__google-drive__search_files" },+ { name: "mcp__google-drive__read_file_content" },+ ] as never[];++ test("names the system and its tools", () => {+ const guidance = grantedToolGuidance(drive);+ expect(guidance).toContain("google-drive");+ expect(guidance).toContain("search_files");+ expect(guidance).toContain("read_file_content");+ });++ test("says not to browse to a vendor it has a tool for", () => {+ // The whole point. Without this line the tool list is inert beside the browser prose.+ expect(grantedToolGuidance(drive).toLowerCase()).toContain("do not browse");+ });++ test("says nothing at all when the Bot holds nothing", () => {+ // A deployment with no connectors must not be told about connectors it does not have.+ expect(grantedToolGuidance([])).toBe("");+ });++ test("a built-in Bot is told before it is told about the browser", () => {+ const prompt = builtInAgentConfiguration(+ {+ id: "risk-analyst",+ name: "Risk Analyst",+ type: "built_in",+ systemPrompt: "Investigate policies.",+ },+ { provider: "openai", defaultModel: "gpt-4.1" },+ "openai-secret",+ drive,+ "BROWSER GUIDANCE HERE",+ ).prompt as string;++ // Order is the fix, not merely presence: the grants have to land before the browser prose.+ expect(prompt.indexOf("google-drive")).toBeGreaterThan(-1);+ expect(prompt.indexOf("google-drive")).toBeLessThan(+ prompt.indexOf("BROWSER GUIDANCE HERE"),+ );+ });+});