Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

Make LangChain the default harness, with the tool loop governed (#33)

David McKayafde20f35270
diff --git a/.env.example b/.env.example
index 3205c401..5ad0165f 100644
--- a/.env.example
+++ b/.env.example
@@ -172,7 +172,11 @@ AGENT_COMPUTER_ALLOW_PRIVATE_HOSTS=true
# The managed coworker AG-UI endpoint. Required: use an HTTP(S) URL.
-MANAGED_AGENT_AG_UI_URL=http://localhost:4200/ag-ui
+#
+# Defaults to agent-langgraph on 4201, which runs a real framework and its own tool loop. The
+# proof-of-concept on 4200 hand-writes the protocol and leaves the loop to whatever is watching, so
+# it is a reference rather than something to build a deployment on.
+MANAGED_AGENT_AG_UI_URL=http://localhost:4201/ag-ui
# The second Bot in the box runs on http://localhost:4201/ag-ui, on a framework rather than
# proof of concept, and is reached the same way: point MANAGED_AGENT_AG_UI_URL at it, or add it as a
@@ -192,3 +196,9 @@ SUPERVISOR_TOKEN=
# Set to runsc to run every computer under gVisor, if the host has it. Unset, a computer is an
# ordinary container and shares the host kernel, which is worth knowing when the Bot is not ours.
COMPUTER_RUNTIME=
+
+# The secret a framework Bot presents when it calls a tool back through this server. A Bot runs its
+# own loop, in its own process, and still may not reach a vendor directly: it calls the deployment
+# that granted the tool, which is where the grant, the policy and the audit row are. Absent, no Bot
+# may call tools back and it is told so rather than being quietly allowed.
+AGENT_TOOL_TOKEN=
diff --git a/agent-langgraph/src/index.ts b/agent-langgraph/src/index.ts
index c8a00402..fd747f81 100644
--- a/agent-langgraph/src/index.ts
+++ b/agent-langgraph/src/index.ts
@@ -55,7 +55,15 @@ const PORT = Number.parseInt(process.env.PORT ?? "4201", 10);
* The default is unchanged so the two shipped Bots stay comparable out of the box.
*/
const PROVIDER = (process.env.BOT_PROVIDER ?? "openai").toLowerCase();
-const MODEL = process.env.BOT_MODEL ?? defaultModelFor(PROVIDER);
+/*
+ * An unset model and an empty one are the same thing.
+ *
+ * `??` only catches undefined, and a compose file passing `BOT_MODEL: ${BOT_MODEL:-}` hands this an
+ * empty string, which is a value. The agent then asked its provider for a model named "" and the
+ * run died with "you must provide a model parameter", which reads as a broken Bot rather than as
+ * missing configuration.
+ */
+const MODEL = process.env.BOT_MODEL?.trim() || defaultModelFor(PROVIDER);
/** OpenAI only. Its newer models require the Responses API, which the integration handles. */
const USE_RESPONSES_API = process.env.BOT_RESPONSES_API === "true";
/**
@@ -216,13 +224,62 @@ function buildModel() {
}
/**
- * The graph.
+ * Where this Bot runs a tool.
*
- * One node is enough while the tool loop lives on the client. The graph provides model orchestration
- * without changing the AG-UI contract.
+ * Not the vendor: this deployment. A Bot that called an MCP server directly would be a Bot that
+ * walked around the grant, the policy and the audit row, and those are the product. So the loop runs
+ * here, in this process, and every call it makes goes back through the deployment that granted it.
+ */
+const TOOL_URL =
+ process.env.OPENBOT_TOOL_URL ?? "http://localhost:3001/api/agent-tools/call";
+const TOOL_TOKEN = process.env.AGENT_TOOL_TOKEN ?? "";
+
+async function callTool(
+ botId: string,
+ name: string,
+ args: Record<string, unknown>,
+): Promise<string> {
+ if (!TOOL_TOKEN) {
+ return "Refused. This Bot has no credential for calling tools back through its deployment.";
+ }
+ try {
+ const response = await fetch(TOOL_URL, {
+ method: "POST",
+ headers: {
+ "content-type": "application/json",
+ "x-openbot-agent-token": TOOL_TOKEN,
+ },
+ body: JSON.stringify({ botId, name, args }),
+ });
+ const body = (await response.json()) as { text?: string };
+ return body.text ?? "The tool returned nothing.";
+ } catch (error) {
+ // Reported to the model as a result rather than thrown: the run continues and says what broke.
+ return `That tool could not be called: ${
+ error instanceof Error ? error.message : "unknown error"
+ }`;
+ }
+}
+
+/** Which Bot is running, so the deployment can attribute the call it is about to be asked for. */
+function botIdOf(input: RunAgentInput): string {
+ const props = input.forwardedProps as { openbotBotId?: unknown } | undefined;
+ return typeof props?.openbotBotId === "string" ? props.openbotBotId : "";
+}
+
+/**
+ * The graph, with the tool loop where it belongs.
+ *
+ * The loop used to run in the browser: this emitted a call, ended the run, and waited for a surface
+ * to execute it and start another. That made a watching browser a requirement for a Bot to do
+ * anything, which rules out an embed, a schedule, and anything unattended.
+ *
+ * Now it answers, calls what it needs, reads the results and answers again, which is what a harness
+ * is for. `recursionLimit` bounds a model that would otherwise call tools in a circle.
*/
function buildGraph(input: RunAgentInput) {
const model = buildModel();
+ const botId = botIdOf(input);
const tools = toBoundTools(input);
const bound = tools.length > 0 ? model.bindTools(tools) : model;
@@ -231,8 +288,30 @@ function buildGraph(input: RunAgentInput) {
.addNode("answer", async (state) => ({
messages: [await bound.invoke(state.messages)],
}))
+ .addNode("tools", async (state) => {
+ const last = state.messages.at(-1) as AIMessage;
+ const results = await Promise.all(
+ (last.tool_calls ?? []).map(async (call) => {
+ const text = await callTool(
+ botId,
+ call.name,
+ (call.args ?? {}) as Record<string, unknown>,
+ );
+ return new ToolMessage({
+ content: text,
+ tool_call_id: call.id ?? call.name,
+ name: call.name,
+ });
+ }),
+ );
+ return { messages: results };
+ })
.addEdge(START, "answer")
- .addEdge("answer", END)
+ .addConditionalEdges("answer", (state) => {
+ const last = state.messages.at(-1) as AIMessage | undefined;
+ return (last?.tool_calls?.length ?? 0) > 0 ? "tools" : END;
+ })
+ .addEdge("tools", "answer")
.compile();
}
@@ -250,8 +329,23 @@ async function runAgent(input: RunAgentInput): Promise<Response> {
runId: input.runId,
} as BaseEvent);
- const messageId = `msg_${input.runId}`;
+ /*
+ * One message id per stretch of prose.
+ *
+ * A run is several turns now: the Bot may speak, call a tool, read the result and speak
+ * again. Reusing one id reopens a message the surface has already closed, and the second half
+ * of the answer is dropped.
+ */
+ let messageIndex = 0;
+ let messageId = `msg_${input.runId}_0`;
let textOpen = false;
+ const closeText = () => {
+ if (!textOpen) return;
+ send({ type: "TEXT_MESSAGE_END", messageId } as BaseEvent);
+ textOpen = false;
+ messageIndex += 1;
+ messageId = `msg_${input.runId}_${messageIndex}`;
+ };
try {
const graph = buildGraph(input);
@@ -264,6 +358,11 @@ async function runAgent(input: RunAgentInput): Promise<Response> {
// fragments and AG-UI wants one call. The framework hands back assembled `tool_calls` on the
// final message, which is precisely the plumbing agent-bot does by hand.
let finalMessage: AIMessage | null = null;
+ /** Calls seen on the way past, so a result can be paired with the arguments it answered. */
+ const pending = new Map<
+ string,
+ { name: string; args: Record<string, unknown> }
+ >();
for await (const event of events) {
if (event.event === "on_chat_model_stream") {
@@ -291,31 +390,56 @@ async function runAgent(input: RunAgentInput): Promise<Response> {
if (event.event === "on_chat_model_end") {
const output = event.data?.output as AIMessage | undefined;
- if (output) finalMessage = output;
+ if (output) {
+ finalMessage = output;
+ for (const call of output.tool_calls ?? []) {
+ pending.set(call.id ?? call.name, {
+ name: call.name,
+ args: (call.args ?? {}) as Record<string, unknown>,
+ });
+ }
+ }
}
- }
- if (textOpen) {
- send({ type: "TEXT_MESSAGE_END", messageId } as BaseEvent);
+ /*
+ * The tools node finished. Reported here, in order, rather than collected for the end: the
+ * surface draws a conversation, and a call arriving after the answer it informed reads as
+ * though the Bot spoke first and did the work afterwards.
+ */
+ if (event.event === "on_chain_end" && event.name === "tools") {
+ const output = event.data?.output as
+ | { messages?: { tool_call_id?: string; content?: unknown }[] }
+ | undefined;
+ // Prose and tool calls cannot interleave inside one message.
+ closeText();
+ for (const message of output?.messages ?? []) {
+ const id = message.tool_call_id ?? "";
+ const call = pending.get(id);
+ if (!call) continue;
+ send({
+ type: "TOOL_CALL_START",
+ toolCallId: id,
+ toolCallName: call.name,
+ } as BaseEvent);
+ send({
+ type: "TOOL_CALL_ARGS",
+ toolCallId: id,
+ delta: JSON.stringify(call.args),
+ } as BaseEvent);
+ send({ type: "TOOL_CALL_END", toolCallId: id } as BaseEvent);
+ send({
+ type: "TOOL_CALL_RESULT",
+ messageId: `${id}-result`,
+ toolCallId: id,
+ content: String(message.content ?? ""),
+ role: "tool",
+ } as BaseEvent);
+ pending.delete(id);
+ }
+ }
}
- for (const call of finalMessage?.tool_calls ?? []) {
- send({
- type: "TOOL_CALL_START",
- toolCallId: call.id ?? `call_${input.runId}_${call.name}`,
- toolCallName: call.name,
- parentMessageId: messageId,
- } as BaseEvent);
- send({
- type: "TOOL_CALL_ARGS",
- toolCallId: call.id ?? `call_${input.runId}_${call.name}`,
- delta: JSON.stringify(call.args ?? {}),
- } as BaseEvent);
- send({
- type: "TOOL_CALL_END",
- toolCallId: call.id ?? `call_${input.runId}_${call.name}`,
- } as BaseEvent);
- }
+ closeText();
send({
type: "RUN_FINISHED",
diff --git a/docker-compose.yml b/docker-compose.yml
index 5baf2106..9f38be9c 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -190,8 +190,14 @@ services:
ANTHROPIC_BASE_URL: ${ANTHROPIC_BASE_URL:-}
GOOGLE_API_KEY: ${GOOGLE_API_KEY:-}
GOOGLE_GENERATIVE_AI_BASE_URL: ${GOOGLE_GENERATIVE_AI_BASE_URL:-}
- BOT_MODEL: ${BOT_MODEL:-}
+ BOT_MODEL: ${BOT_MODEL:-gpt-5.5}
BOT_RESPONSES_API: ${BOT_RESPONSES_API:-false}
+ # Where this Bot runs a tool: back through the deployment that granted it, never at the vendor.
+ # `host.docker.internal` because the API server runs on the host, not in this network.
+ OPENBOT_TOOL_URL: ${OPENBOT_TOOL_URL:-http://host.docker.internal:3001/api/agent-tools/call}
+ AGENT_TOOL_TOKEN: ${AGENT_TOOL_TOKEN:-}
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
healthcheck:
test: ["CMD-SHELL", "bun -e \"await fetch('http://localhost:4201/health')\""]
interval: 10s
diff --git a/examples/fintech/agents.yaml b/examples/fintech/agents.yaml
index 81161564..979454d2 100644
--- a/examples/fintech/agents.yaml
+++ b/examples/fintech/agents.yaml
@@ -24,4 +24,4 @@ agents:
role_description: Investigate policies, transaction monitoring, and control evidence.
avatar_seed: risk-analyst
type: remote-ag-ui
- endpoint: ${MANAGED_AGENT_AG_UI_URL:-http://localhost:4200/ag-ui}
+ endpoint: ${MANAGED_AGENT_AG_UI_URL:-http://localhost:4201/ag-ui}
diff --git a/server/src/app.ts b/server/src/app.ts
index 7675d476..42489099 100644
--- a/server/src/app.ts
+++ b/server/src/app.ts
@@ -27,6 +27,7 @@ import type { DeploymentConfig } from "./config";
import type { ConnectorAdminService } from "./connectors";
import type { CredentialAdminService, CredentialInput } from "./credentials";
import { createPluginRoutes } from "./plugins/routes";
+import { REFUSAL_MARKER } from "./plugins/tools";
import type { PluginStore } from "./plugins/store";
import type { PackageStatusReader } from "./tenant-package";
@@ -344,6 +345,54 @@ export function createApp(
app.route("/api/plugins", createPluginRoutes(pluginStore, requireUser));
}
+ /*
+ * Where a framework Bot runs a tool.
+ *
+ * A Bot that runs its own loop, in its own process, is the honest shape: the run does not need a
+ * browser and does not stop when one closes. What it must not have is a route to a vendor that
+ * goes around this deployment, so it calls here and this calls the plugin store, which asks the
+ * same two questions it asks of everything else and writes the same audit row.
+ *
+ * Authenticated by a shared secret rather than a session, because the caller is a service and has
+ * no person behind it. Absent secret means the route does not exist: a deployment that has not
+ * configured this refuses rather than accepting anybody who can reach the port.
+ */
+ if (pluginStore && config.agentToolToken) {
+ const token = config.agentToolToken;
+ app.post("/api/agent-tools/call", async (context) => {
+ if (context.req.header("x-openbot-agent-token") !== token) {
+ return context.json({ error: "Not authorised." }, 401);
+ }
+ const body = (await context.req.json().catch(() => null)) as {
+ botId?: string;
+ actorId?: string;
+ name?: string;
+ args?: Record<string, unknown>;
+ } | null;
+ if (!body?.botId || !body.name) {
+ return context.json({ error: "A Bot and a tool are required." }, 400);
+ }
+ try {
+ const result = await pluginStore.callTool({
+ // The model is offered `mcp__server__tool`; the store speaks `server/tool`.
+ ref: body.name.replace(/^mcp__/, "").replace("__", "/"),
+ args: body.args ?? {},
+ botId: body.botId,
+ actorId: body.actorId ?? "agent",
+ });
+ return context.json({ text: result.text, isError: result.isError });
+ } catch (error) {
+ // A refusal is an answer, not a failure: the Bot says what was blocked and carries on. The
+ // marker leads it for the same reason it does on the in-process path, so a transcript can
+ // draw a refusal as one without reading the wording.
+ return context.json({
+ text: `${REFUSAL_MARKER} ${error instanceof Error ? error.message : "That tool could not be called."}`,
+ isError: true,
+ });
+ }
+ });
+ }
+
if (sandboxedStore) {
app.route(
"/api/sandboxed",
diff --git a/server/src/config.ts b/server/src/config.ts
index 9e9c2c8c..a85e8f01 100644
--- a/server/src/config.ts
+++ b/server/src/config.ts
@@ -83,6 +83,18 @@ export type DeploymentConfig = {
*/
policy?: ActionPolicy;
};
+ /**
+ * The secret a Bot presents when it calls a tool back through this server.
+ *
+ * A framework Bot runs its own tool loop, in its own process, which is what makes it a real
+ * harness rather than a shape the browser drives. It still may not reach a vendor directly: it
+ * calls here, and here is where the grant, the policy and the audit row are. This is what tells
+ * that call apart from anybody else on the network.
+ *
+ * Absent means no Bot may call tools back, and a deployment that wanted them gets a refusal rather
+ * than an open door.
+ */
+ agentToolToken?: string;
};
type Environment = Record<string, string | undefined>;
@@ -369,5 +381,8 @@ export function loadConfig(
auth: authConfig(environment, google),
devNoAuth: devAuthEnabled(environment),
computer: computerConfig(environment),
+ ...(optional(environment, "AGENT_TOOL_TOKEN")
+ ? { agentToolToken: optional(environment, "AGENT_TOOL_TOKEN") as string }
+ : {}),
};
}
diff --git a/server/src/copilot.ts b/server/src/copilot.ts
index ac4d934e..cbdd98a5 100644
--- a/server/src/copilot.ts
+++ b/server/src/copilot.ts
@@ -9,6 +9,7 @@ import { createCopilotHonoHandler } from "@copilotkit/runtime/v2/hono";
import type { AgentActor } from "./agents/profile-types";
import type { StallGuard } from "./channels/stall-guard";
import type { DeploymentConfig } from "./config";
+import { z } from "zod";
import type { GrantedTool } from "./plugins/tools";
/**
@@ -257,7 +258,11 @@ async function buildAgent(
if (agent.type === "unavailable") {
return new UnavailableAgent(agent);
}
- return remoteAgentWithStandingRole(agent, stallGuard);
+ return remoteAgentWithStandingRole(
+ agent,
+ stallGuard,
+ await loadTools(agent.id),
+ );
}
/**
@@ -274,7 +279,15 @@ async function buildAgent(
*/
function remoteAgentWithStandingRole(
agent: RegisteredRemoteAgent,
- stallGuard?: StallGuard,
+ stallGuard: StallGuard | undefined,
+ /**
+ * What this Bot was granted, described rather than executable.
+ *
+ * A framework Bot runs its own loop and calls these back through `/api/agent-tools/call`, so what
+ * it needs from here is the offer: the name, what the tool is for, and the arguments it takes.
+ * The executing half stays on this side, where the grant and the policy are.
+ */
+ tools: GrantedTool[] = [],
) {
const remote = new HttpAgent({
url: agent.endpoint,
@@ -295,7 +308,30 @@ function remoteAgentWithStandingRole(
(message) => message.id !== agent.standingMessage.id,
),
],
- }),
+ /*
+ * The Bot's own grants, added to whatever the surface offered.
+ *
+ * Sent on every run rather than configured once on the endpoint, because a grant an
+ * administrator adds or revokes has to apply to the next run and the endpoint is somebody
+ * else's process.
+ */
+ tools: [
+ ...(input.tools ?? []),
+ ...tools.map((tool) => ({
+ name: tool.name,
+ description: tool.description,
+ parameters: z.toJSONSchema(tool.parameters) as Record<
+ string,
+ unknown
+ >,
+ })),
+ ],
+ // Who the Bot is calling back as, so the audit row names it rather than "an agent".
+ forwardedProps: {
+ ...(input.forwardedProps ?? {}),
+ openbotBotId: agent.id,
+ },
+ } as never),
);
return remote;
}