Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
Make LangChain the default harness, with the tool loop governed (#33)
David McKayafde20f35270
diff --git a/.env.example b/.env.exampleindex 3205c401..5ad0165f 100644--- a/.env.example+++ b/.env.example@@ -172,7 +172,11 @@ AGENT_COMPUTER_ALLOW_PRIVATE_HOSTS=true# The managed coworker AG-UI endpoint. Required: use an HTTP(S) URL.-MANAGED_AGENT_AG_UI_URL=http://localhost:4200/ag-ui+#+# Defaults to agent-langgraph on 4201, which runs a real framework and its own tool loop. The+# proof-of-concept on 4200 hand-writes the protocol and leaves the loop to whatever is watching, so+# it is a reference rather than something to build a deployment on.+MANAGED_AGENT_AG_UI_URL=http://localhost:4201/ag-ui# The second Bot in the box runs on http://localhost:4201/ag-ui, on a framework rather than# proof of concept, and is reached the same way: point MANAGED_AGENT_AG_UI_URL at it, or add it as a@@ -192,3 +196,9 @@ SUPERVISOR_TOKEN=# Set to runsc to run every computer under gVisor, if the host has it. Unset, a computer is an# ordinary container and shares the host kernel, which is worth knowing when the Bot is not ours.COMPUTER_RUNTIME=++# The secret a framework Bot presents when it calls a tool back through this server. A Bot runs its+# own loop, in its own process, and still may not reach a vendor directly: it calls the deployment+# that granted the tool, which is where the grant, the policy and the audit row are. Absent, no Bot+# may call tools back and it is told so rather than being quietly allowed.+AGENT_TOOL_TOKEN=diff --git a/agent-langgraph/src/index.ts b/agent-langgraph/src/index.tsindex c8a00402..fd747f81 100644--- a/agent-langgraph/src/index.ts+++ b/agent-langgraph/src/index.ts@@ -55,7 +55,15 @@ const PORT = Number.parseInt(process.env.PORT ?? "4201", 10);* The default is unchanged so the two shipped Bots stay comparable out of the box.*/const PROVIDER = (process.env.BOT_PROVIDER ?? "openai").toLowerCase();-const MODEL = process.env.BOT_MODEL ?? defaultModelFor(PROVIDER);+/*+ * An unset model and an empty one are the same thing.+ *+ * `??` only catches undefined, and a compose file passing `BOT_MODEL: ${BOT_MODEL:-}` hands this an+ * empty string, which is a value. The agent then asked its provider for a model named "" and the+ * run died with "you must provide a model parameter", which reads as a broken Bot rather than as+ * missing configuration.+ */+const MODEL = process.env.BOT_MODEL?.trim() || defaultModelFor(PROVIDER);/** OpenAI only. Its newer models require the Responses API, which the integration handles. */const USE_RESPONSES_API = process.env.BOT_RESPONSES_API === "true";/**@@ -216,13 +224,62 @@ function buildModel() {}/**- * The graph.+ * Where this Bot runs a tool.*- * One node is enough while the tool loop lives on the client. The graph provides model orchestration- * without changing the AG-UI contract.+ * Not the vendor: this deployment. A Bot that called an MCP server directly would be a Bot that+ * walked around the grant, the policy and the audit row, and those are the product. So the loop runs+ * here, in this process, and every call it makes goes back through the deployment that granted it.+ */+const TOOL_URL =+ process.env.OPENBOT_TOOL_URL ?? "http://localhost:3001/api/agent-tools/call";+const TOOL_TOKEN = process.env.AGENT_TOOL_TOKEN ?? "";++async function callTool(+ botId: string,+ name: string,+ args: Record<string, unknown>,+): Promise<string> {+ if (!TOOL_TOKEN) {+ return "Refused. This Bot has no credential for calling tools back through its deployment.";+ }+ try {+ const response = await fetch(TOOL_URL, {+ method: "POST",+ headers: {+ "content-type": "application/json",+ "x-openbot-agent-token": TOOL_TOKEN,+ },+ body: JSON.stringify({ botId, name, args }),+ });+ const body = (await response.json()) as { text?: string };+ return body.text ?? "The tool returned nothing.";+ } catch (error) {+ // Reported to the model as a result rather than thrown: the run continues and says what broke.+ return `That tool could not be called: ${+ error instanceof Error ? error.message : "unknown error"+ }`;+ }+}++/** Which Bot is running, so the deployment can attribute the call it is about to be asked for. */+function botIdOf(input: RunAgentInput): string {+ const props = input.forwardedProps as { openbotBotId?: unknown } | undefined;+ return typeof props?.openbotBotId === "string" ? props.openbotBotId : "";+}++/**+ * The graph, with the tool loop where it belongs.+ *+ * The loop used to run in the browser: this emitted a call, ended the run, and waited for a surface+ * to execute it and start another. That made a watching browser a requirement for a Bot to do+ * anything, which rules out an embed, a schedule, and anything unattended.+ *+ * Now it answers, calls what it needs, reads the results and answers again, which is what a harness+ * is for. `recursionLimit` bounds a model that would otherwise call tools in a circle.*/function buildGraph(input: RunAgentInput) {const model = buildModel();+ const botId = botIdOf(input);const tools = toBoundTools(input);const bound = tools.length > 0 ? model.bindTools(tools) : model;@@ -231,8 +288,30 @@ function buildGraph(input: RunAgentInput) {.addNode("answer", async (state) => ({messages: [await bound.invoke(state.messages)],}))+ .addNode("tools", async (state) => {+ const last = state.messages.at(-1) as AIMessage;+ const results = await Promise.all(+ (last.tool_calls ?? []).map(async (call) => {+ const text = await callTool(+ botId,+ call.name,+ (call.args ?? {}) as Record<string, unknown>,+ );+ return new ToolMessage({+ content: text,+ tool_call_id: call.id ?? call.name,+ name: call.name,+ });+ }),+ );+ return { messages: results };+ }).addEdge(START, "answer")- .addEdge("answer", END)+ .addConditionalEdges("answer", (state) => {+ const last = state.messages.at(-1) as AIMessage | undefined;+ return (last?.tool_calls?.length ?? 0) > 0 ? "tools" : END;+ })+ .addEdge("tools", "answer").compile();}@@ -250,8 +329,23 @@ async function runAgent(input: RunAgentInput): Promise<Response> {runId: input.runId,} as BaseEvent);- const messageId = `msg_${input.runId}`;+ /*+ * One message id per stretch of prose.+ *+ * A run is several turns now: the Bot may speak, call a tool, read the result and speak+ * again. Reusing one id reopens a message the surface has already closed, and the second half+ * of the answer is dropped.+ */+ let messageIndex = 0;+ let messageId = `msg_${input.runId}_0`;let textOpen = false;+ const closeText = () => {+ if (!textOpen) return;+ send({ type: "TEXT_MESSAGE_END", messageId } as BaseEvent);+ textOpen = false;+ messageIndex += 1;+ messageId = `msg_${input.runId}_${messageIndex}`;+ };try {const graph = buildGraph(input);@@ -264,6 +358,11 @@ async function runAgent(input: RunAgentInput): Promise<Response> {// fragments and AG-UI wants one call. The framework hands back assembled `tool_calls` on the// final message, which is precisely the plumbing agent-bot does by hand.let finalMessage: AIMessage | null = null;+ /** Calls seen on the way past, so a result can be paired with the arguments it answered. */+ const pending = new Map<+ string,+ { name: string; args: Record<string, unknown> }+ >();for await (const event of events) {if (event.event === "on_chat_model_stream") {@@ -291,31 +390,56 @@ async function runAgent(input: RunAgentInput): Promise<Response> {if (event.event === "on_chat_model_end") {const output = event.data?.output as AIMessage | undefined;- if (output) finalMessage = output;+ if (output) {+ finalMessage = output;+ for (const call of output.tool_calls ?? []) {+ pending.set(call.id ?? call.name, {+ name: call.name,+ args: (call.args ?? {}) as Record<string, unknown>,+ });+ }+ }}- }- if (textOpen) {- send({ type: "TEXT_MESSAGE_END", messageId } as BaseEvent);+ /*+ * The tools node finished. Reported here, in order, rather than collected for the end: the+ * surface draws a conversation, and a call arriving after the answer it informed reads as+ * though the Bot spoke first and did the work afterwards.+ */+ if (event.event === "on_chain_end" && event.name === "tools") {+ const output = event.data?.output as+ | { messages?: { tool_call_id?: string; content?: unknown }[] }+ | undefined;+ // Prose and tool calls cannot interleave inside one message.+ closeText();+ for (const message of output?.messages ?? []) {+ const id = message.tool_call_id ?? "";+ const call = pending.get(id);+ if (!call) continue;+ send({+ type: "TOOL_CALL_START",+ toolCallId: id,+ toolCallName: call.name,+ } as BaseEvent);+ send({+ type: "TOOL_CALL_ARGS",+ toolCallId: id,+ delta: JSON.stringify(call.args),+ } as BaseEvent);+ send({ type: "TOOL_CALL_END", toolCallId: id } as BaseEvent);+ send({+ type: "TOOL_CALL_RESULT",+ messageId: `${id}-result`,+ toolCallId: id,+ content: String(message.content ?? ""),+ role: "tool",+ } as BaseEvent);+ pending.delete(id);+ }+ }}- for (const call of finalMessage?.tool_calls ?? []) {- send({- type: "TOOL_CALL_START",- toolCallId: call.id ?? `call_${input.runId}_${call.name}`,- toolCallName: call.name,- parentMessageId: messageId,- } as BaseEvent);- send({- type: "TOOL_CALL_ARGS",- toolCallId: call.id ?? `call_${input.runId}_${call.name}`,- delta: JSON.stringify(call.args ?? {}),- } as BaseEvent);- send({- type: "TOOL_CALL_END",- toolCallId: call.id ?? `call_${input.runId}_${call.name}`,- } as BaseEvent);- }+ closeText();send({type: "RUN_FINISHED",diff --git a/docker-compose.yml b/docker-compose.ymlindex 5baf2106..9f38be9c 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -190,8 +190,14 @@ services:ANTHROPIC_BASE_URL: ${ANTHROPIC_BASE_URL:-}GOOGLE_API_KEY: ${GOOGLE_API_KEY:-}GOOGLE_GENERATIVE_AI_BASE_URL: ${GOOGLE_GENERATIVE_AI_BASE_URL:-}- BOT_MODEL: ${BOT_MODEL:-}+ BOT_MODEL: ${BOT_MODEL:-gpt-5.5}BOT_RESPONSES_API: ${BOT_RESPONSES_API:-false}+ # Where this Bot runs a tool: back through the deployment that granted it, never at the vendor.+ # `host.docker.internal` because the API server runs on the host, not in this network.+ OPENBOT_TOOL_URL: ${OPENBOT_TOOL_URL:-http://host.docker.internal:3001/api/agent-tools/call}+ AGENT_TOOL_TOKEN: ${AGENT_TOOL_TOKEN:-}+ extra_hosts:+ - "host.docker.internal:host-gateway"healthcheck:test: ["CMD-SHELL", "bun -e \"await fetch('http://localhost:4201/health')\""]interval: 10sdiff --git a/examples/fintech/agents.yaml b/examples/fintech/agents.yamlindex 81161564..979454d2 100644--- a/examples/fintech/agents.yaml+++ b/examples/fintech/agents.yaml@@ -24,4 +24,4 @@ agents:role_description: Investigate policies, transaction monitoring, and control evidence.avatar_seed: risk-analysttype: remote-ag-ui- endpoint: ${MANAGED_AGENT_AG_UI_URL:-http://localhost:4200/ag-ui}+ endpoint: ${MANAGED_AGENT_AG_UI_URL:-http://localhost:4201/ag-ui}diff --git a/server/src/app.ts b/server/src/app.tsindex 7675d476..42489099 100644--- a/server/src/app.ts+++ b/server/src/app.ts@@ -27,6 +27,7 @@ import type { DeploymentConfig } from "./config";import type { ConnectorAdminService } from "./connectors";import type { CredentialAdminService, CredentialInput } from "./credentials";import { createPluginRoutes } from "./plugins/routes";+import { REFUSAL_MARKER } from "./plugins/tools";import type { PluginStore } from "./plugins/store";import type { PackageStatusReader } from "./tenant-package";@@ -344,6 +345,54 @@ export function createApp(app.route("/api/plugins", createPluginRoutes(pluginStore, requireUser));}+ /*+ * Where a framework Bot runs a tool.+ *+ * A Bot that runs its own loop, in its own process, is the honest shape: the run does not need a+ * browser and does not stop when one closes. What it must not have is a route to a vendor that+ * goes around this deployment, so it calls here and this calls the plugin store, which asks the+ * same two questions it asks of everything else and writes the same audit row.+ *+ * Authenticated by a shared secret rather than a session, because the caller is a service and has+ * no person behind it. Absent secret means the route does not exist: a deployment that has not+ * configured this refuses rather than accepting anybody who can reach the port.+ */+ if (pluginStore && config.agentToolToken) {+ const token = config.agentToolToken;+ app.post("/api/agent-tools/call", async (context) => {+ if (context.req.header("x-openbot-agent-token") !== token) {+ return context.json({ error: "Not authorised." }, 401);+ }+ const body = (await context.req.json().catch(() => null)) as {+ botId?: string;+ actorId?: string;+ name?: string;+ args?: Record<string, unknown>;+ } | null;+ if (!body?.botId || !body.name) {+ return context.json({ error: "A Bot and a tool are required." }, 400);+ }+ try {+ const result = await pluginStore.callTool({+ // The model is offered `mcp__server__tool`; the store speaks `server/tool`.+ ref: body.name.replace(/^mcp__/, "").replace("__", "/"),+ args: body.args ?? {},+ botId: body.botId,+ actorId: body.actorId ?? "agent",+ });+ return context.json({ text: result.text, isError: result.isError });+ } catch (error) {+ // A refusal is an answer, not a failure: the Bot says what was blocked and carries on. The+ // marker leads it for the same reason it does on the in-process path, so a transcript can+ // draw a refusal as one without reading the wording.+ return context.json({+ text: `${REFUSAL_MARKER} ${error instanceof Error ? error.message : "That tool could not be called."}`,+ isError: true,+ });+ }+ });+ }+if (sandboxedStore) {app.route("/api/sandboxed",diff --git a/server/src/config.ts b/server/src/config.tsindex 9e9c2c8c..a85e8f01 100644--- a/server/src/config.ts+++ b/server/src/config.ts@@ -83,6 +83,18 @@ export type DeploymentConfig = {*/policy?: ActionPolicy;};+ /**+ * The secret a Bot presents when it calls a tool back through this server.+ *+ * A framework Bot runs its own tool loop, in its own process, which is what makes it a real+ * harness rather than a shape the browser drives. It still may not reach a vendor directly: it+ * calls here, and here is where the grant, the policy and the audit row are. This is what tells+ * that call apart from anybody else on the network.+ *+ * Absent means no Bot may call tools back, and a deployment that wanted them gets a refusal rather+ * than an open door.+ */+ agentToolToken?: string;};type Environment = Record<string, string | undefined>;@@ -369,5 +381,8 @@ export function loadConfig(auth: authConfig(environment, google),devNoAuth: devAuthEnabled(environment),computer: computerConfig(environment),+ ...(optional(environment, "AGENT_TOOL_TOKEN")+ ? { agentToolToken: optional(environment, "AGENT_TOOL_TOKEN") as string }+ : {}),};}diff --git a/server/src/copilot.ts b/server/src/copilot.tsindex ac4d934e..cbdd98a5 100644--- a/server/src/copilot.ts+++ b/server/src/copilot.ts@@ -9,6 +9,7 @@ import { createCopilotHonoHandler } from "@copilotkit/runtime/v2/hono";import type { AgentActor } from "./agents/profile-types";import type { StallGuard } from "./channels/stall-guard";import type { DeploymentConfig } from "./config";+import { z } from "zod";import type { GrantedTool } from "./plugins/tools";/**@@ -257,7 +258,11 @@ async function buildAgent(if (agent.type === "unavailable") {return new UnavailableAgent(agent);}- return remoteAgentWithStandingRole(agent, stallGuard);+ return remoteAgentWithStandingRole(+ agent,+ stallGuard,+ await loadTools(agent.id),+ );}/**@@ -274,7 +279,15 @@ async function buildAgent(*/function remoteAgentWithStandingRole(agent: RegisteredRemoteAgent,- stallGuard?: StallGuard,+ stallGuard: StallGuard | undefined,+ /**+ * What this Bot was granted, described rather than executable.+ *+ * A framework Bot runs its own loop and calls these back through `/api/agent-tools/call`, so what+ * it needs from here is the offer: the name, what the tool is for, and the arguments it takes.+ * The executing half stays on this side, where the grant and the policy are.+ */+ tools: GrantedTool[] = [],) {const remote = new HttpAgent({url: agent.endpoint,@@ -295,7 +308,30 @@ function remoteAgentWithStandingRole((message) => message.id !== agent.standingMessage.id,),],- }),+ /*+ * The Bot's own grants, added to whatever the surface offered.+ *+ * Sent on every run rather than configured once on the endpoint, because a grant an+ * administrator adds or revokes has to apply to the next run and the endpoint is somebody+ * else's process.+ */+ tools: [+ ...(input.tools ?? []),+ ...tools.map((tool) => ({+ name: tool.name,+ description: tool.description,+ parameters: z.toJSONSchema(tool.parameters) as Record<+ string,+ unknown+ >,+ })),+ ],+ // Who the Bot is calling back as, so the audit row names it rather than "an agent".+ forwardedProps: {+ ...(input.forwardedProps ?? {}),+ openbotBotId: agent.id,+ },+ } as never),);return remote;}