Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
Give the Bot in the box the fix the other Bot got (#165)
David McKay6507bb40cf23
diff --git a/agent-bot/src/history.ts b/agent-bot/src/history.tsnew file mode 100644index 00000000..5a3e4920--- /dev/null+++ b/agent-bot/src/history.ts@@ -0,0 +1,88 @@+/**+ * The conversation AG-UI carries, as the shape the model provider expects.+ *+ * Its own module so it can be tested without starting a server: `index.ts` calls `serve()` at module+ * scope, so importing it to reach one pure function binds a port. `agent-langgraph/src/history.ts`+ * and `agent-computer/src/control.ts` were split out for the same reason.+ */+import type { RunAgentInput } from "@ag-ui/core";+import type OpenAI from "openai";+import { COMPUTER_GUIDANCE, NO_ANSWER_CAME } from "../../shared/bot-prompt";++export { NO_ANSWER_CAME };++/** Translate the conversation AG-UI carries into the shape the model provider expects. */+export function toProviderMessages(+ input: RunAgentInput,+): OpenAI.Chat.ChatCompletionMessageParam[] {+ const messages: OpenAI.Chat.ChatCompletionMessageParam[] = [+ { role: "system", content: COMPUTER_GUIDANCE },+ ];++ /*+ * Which calls in this history were ever answered.+ *+ * Collected up front because an answer arrives as a later message than the call it answers. See+ * `NO_ANSWER_CAME` for what happens to a conversation carrying a call nothing ever answered.+ */+ const answered = new Set(+ input.messages+ .filter((message) => message.role === "tool")+ .map((message) => (message as { toolCallId?: string }).toolCallId)+ .filter((id): id is string => Boolean(id)),+ );++ for (const message of input.messages) {+ if (message.role === "user") {+ messages.push({ role: "user", content: String(message.content ?? "") });+ continue;+ }+ if (message.role === "system" || message.role === "developer") {+ messages.push({ role: "system", content: String(message.content ?? "") });+ continue;+ }+ if (message.role === "tool") {+ // Tool results are appended so the model can continue from the completed call.+ messages.push({+ role: "tool",+ tool_call_id: message.toolCallId,+ content: String(message.content ?? ""),+ });+ continue;+ }+ if (message.role === "assistant") {+ const toolCalls = message.toolCalls?.map((call) => ({+ id: call.id,+ type: "function" as const,+ function: {+ name: call.function.name,+ arguments: call.function.arguments,+ },+ }));+ messages.push({+ role: "assistant",+ content: message.content ?? null,+ ...(toolCalls?.length ? { tool_calls: toolCalls } : {}),+ });++ /*+ * Close any of its calls that nothing ever answered, immediately after it.+ *+ * Position is not cosmetic: a tool result has to follow the assistant message that made the+ * call, so these go here rather than being appended at the end. A call answered later in the+ * history is left alone and its real answer arrives in its own turn.+ */+ for (const call of message.toolCalls ?? []) {+ if (call.id && !answered.has(call.id)) {+ messages.push({+ role: "tool",+ tool_call_id: call.id,+ content: NO_ANSWER_CAME,+ });+ }+ }+ }+ }++ return messages;+}diff --git a/agent-bot/src/index.ts b/agent-bot/src/index.tsindex 962d8381..da4d18a0 100644--- a/agent-bot/src/index.ts+++ b/agent-bot/src/index.ts@@ -3,7 +3,7 @@ import { EventEncoder } from "@ag-ui/encoder";import { serve } from "bun";import OpenAI from "openai";import { hasManagedAgentToken } from "../../shared/agent-authorisation";-import { COMPUTER_GUIDANCE } from "../../shared/bot-prompt";+import { toProviderMessages } from "./history";/*** The built-in Bot is an AG-UI HTTP service registered the same way as any customer-provided Bot.@@ -67,50 +67,6 @@ const openai = new OpenAI({baseURL: BASE_URL,});-/** Translate the conversation AG-UI carries into the shape the model provider expects. */-function toProviderMessages(input: RunAgentInput) {- const messages: OpenAI.Chat.ChatCompletionMessageParam[] = [- { role: "system", content: COMPUTER_GUIDANCE },- ];-- for (const message of input.messages) {- if (message.role === "user") {- messages.push({ role: "user", content: String(message.content ?? "") });- continue;- }- if (message.role === "system" || message.role === "developer") {- messages.push({ role: "system", content: String(message.content ?? "") });- continue;- }- if (message.role === "tool") {- // Tool results are appended so the model can continue from the completed call.- messages.push({- role: "tool",- tool_call_id: message.toolCallId,- content: String(message.content ?? ""),- });- continue;- }- if (message.role === "assistant") {- const toolCalls = message.toolCalls?.map((call) => ({- id: call.id,- type: "function" as const,- function: {- name: call.function.name,- arguments: call.function.arguments,- },- }));- messages.push({- role: "assistant",- content: message.content ?? null,- ...(toolCalls?.length ? { tool_calls: toolCalls } : {}),- });- }- }-- return messages;-}-/** Every tool comes from the caller. This service publishes none of its own, on purpose. */function toProviderTools(input: RunAgentInput) {if (!input.tools?.length) return undefined;diff --git a/agent-bot/tests/history.test.ts b/agent-bot/tests/history.test.tsnew file mode 100644index 00000000..1318c638--- /dev/null+++ b/agent-bot/tests/history.test.ts@@ -0,0 +1,148 @@+import { describe, expect, test } from "bun:test";+import type { RunAgentInput } from "@ag-ui/core";+import { NO_ANSWER_CAME, toProviderMessages } from "../src/history";++/**+ * The Bot that ships in the box, and the conversation a declined handover used to end.+ *+ * `agent-langgraph` was fixed for this and `agent-bot` was not, so the Bot behind the Browser Bot+ * went on failing in exactly the same way. Found by driving it: take the wheel at a sign-in wall,+ * decline to finish, and the next turn answers+ *+ * 400 An assistant message with 'tool_calls' must be followed by tool messages responding to each+ * 'tool_call_id'+ *+ * on screen, in red, for every message after it. These are the same four cases the other Bot has,+ * against this one's provider shape.+ */++type Message = RunAgentInput["messages"][number];++function input(messages: Message[]): RunAgentInput {+ return { messages } as RunAgentInput;+}++function call(id: string, name = "computer_request_help") {+ return { id, type: "function" as const, function: { name, arguments: "{}" } };+}++/** The system prompt is always first and is not what any of this is about. */+function withoutGuidance(messages: ReturnType<typeof toProviderMessages>) {+ return messages.slice(1);+}++describe("a tool call nothing ever answered", () => {+ test("is answered, so the next turn is not refused outright", () => {+ const messages = withoutGuidance(+ toProviderMessages(+ input([+ {+ id: "1",+ role: "user",+ content: "Read my display name.",+ } as Message,+ {+ id: "2",+ role: "assistant",+ content: "",+ toolCalls: [call("c1")],+ } as unknown as Message,+ {+ id: "3",+ role: "user",+ content: "Never mind. What is 17 times 3?",+ } as Message,+ ]),+ ),+ );++ const answer = messages.find(+ (m) =>+ m.role === "tool" &&+ (m as { tool_call_id?: string }).tool_call_id === "c1",+ );+ expect(answer).toBeDefined();+ expect((answer as { content?: string }).content).toBe(NO_ANSWER_CAME);+ });++ test("says no result rather than inventing a successful one", () => {+ // A fake success would have the Bot report reading a page it never reached.+ expect(NO_ANSWER_CAME.toLowerCase()).toContain("no result");+ expect(NO_ANSWER_CAME.toLowerCase()).toContain(+ "do not assume it succeeded",+ );+ });++ test("lands directly after the assistant message that made it", () => {+ /*+ * Position is the requirement, not presence. A provider matches a tool result to the assistant+ * message it follows, so an answer appended at the end of the history fixes nothing.+ */+ const messages = withoutGuidance(+ toProviderMessages(+ input([+ { id: "1", role: "user", content: "Go." } as Message,+ {+ id: "2",+ role: "assistant",+ content: "",+ toolCalls: [call("c1")],+ } as unknown as Message,+ { id: "3", role: "user", content: "Stop." } as Message,+ ]),+ ),+ );++ const assistantAt = messages.findIndex((m) => m.role === "assistant");+ expect(messages[assistantAt + 1]?.role).toBe("tool");+ expect(+ (messages[assistantAt + 1] as { tool_call_id?: string }).tool_call_id,+ ).toBe("c1");+ });++ test("a call that was answered keeps its real answer and gains nothing", () => {+ const messages = withoutGuidance(+ toProviderMessages(+ input([+ {+ id: "1",+ role: "assistant",+ content: "",+ toolCalls: [call("c1", "computer_navigate")],+ } as unknown as Message,+ {+ id: "2",+ role: "tool",+ toolCallId: "c1",+ content: "Example Domain",+ } as unknown as Message,+ ]),+ ),+ );++ const answers = messages.filter((m) => m.role === "tool");+ expect(answers).toHaveLength(1);+ expect((answers[0] as { content?: string }).content).toBe("Example Domain");+ });++ test("several unanswered calls in one message each get their own answer", () => {+ // A provider names every unanswered id, not just the first, so closing one is not enough.+ const messages = withoutGuidance(+ toProviderMessages(+ input([+ {+ id: "1",+ role: "assistant",+ content: "",+ toolCalls: [call("c1"), call("c2", "computer_snapshot")],+ } as unknown as Message,+ ]),+ ),+ );++ const ids = messages+ .filter((m) => m.role === "tool")+ .map((m) => (m as { tool_call_id?: string }).tool_call_id);+ expect(ids).toEqual(["c1", "c2"]);+ });+});diff --git a/agent-langgraph/src/history.ts b/agent-langgraph/src/history.tsindex f81e34ed..e95f376a 100644--- a/agent-langgraph/src/history.ts+++ b/agent-langgraph/src/history.ts@@ -13,19 +13,13 @@ import {SystemMessage,ToolMessage,} from "@langchain/core/messages";-import { COMPUTER_GUIDANCE } from "../../shared/bot-prompt";+import { COMPUTER_GUIDANCE, NO_ANSWER_CAME } from "../../shared/bot-prompt";-/**- * What a tool call is given when its answer never came.- *- * Written for the model rather than for a log, because the model is the only reader: it has to- * understand that the call is over and not worth waiting for, and be able to say something useful- * about it. "Nothing happened" would leave it repeating the request.+/*+ * Re-exported so this module's own tests and callers keep reading it from here, while the wording+ * itself lives in `shared` where the other Bot can reach it. Both have to say the same thing.*/-export const NO_ANSWER_CAME =- "No result. The person did not answer this, and the run it belonged to has ended. " +- "Do not wait for it and do not assume it succeeded. Carry on without it, and say plainly what " +- "you could not do if it mattered.";+export { NO_ANSWER_CAME };/** Translate the conversation AG-UI carries into LangChain's message classes. */export function toLangChainMessages(input: RunAgentInput): BaseMessage[] {diff --git a/shared/bot-prompt.ts b/shared/bot-prompt.tsindex 49a5c306..7b487d40 100644--- a/shared/bot-prompt.ts+++ b/shared/bot-prompt.ts@@ -137,3 +137,29 @@ export const PROVENANCE_GUIDANCE = PROVENANCE_GUIDANCE_LINES.reduce<string[]>(},[""],).join("\n\n");++/**+ * What a tool call is given when its answer never came.+ *+ * A tool call the surface owns ends the run without a result on purpose: the surface draws it, or+ * puts it to a person, and starts the next run carrying the answer. When nobody answers — a Bot asks+ * for the wheel to get past a sign-in and the person decides they do not need it after all — no+ * answer is ever carried, and the call stays in the history with nothing following it.+ *+ * Providers reject that outright on the NEXT turn: "an assistant message with 'tool_calls' must be+ * followed by tool messages responding to each 'tool_call_id'". So the conversation is not merely+ * stuck on that one request, it is finished, and the only escape is starting a new one.+ *+ * Written for the model rather than for a log, because the model is the only reader: it has to+ * understand the call is over and not worth waiting for, and be able to say something useful about+ * it. "Nothing happened" would leave it repeating the request, and a fake success would have it+ * report work it never did.+ *+ * Shared because both Bots in this repo have to say the same thing. The first fix for this landed in+ * `agent-langgraph` alone, and `agent-bot` — the Bot that ships in the box, and the one behind the+ * Browser Bot — went on failing in exactly the same way until somebody drove it.+ */+export const NO_ANSWER_CAME =+ "No result. The person did not answer this, and the run it belonged to has ended. " ++ "Do not wait for it and do not assume it succeeded. Carry on without it, and say plainly what " ++ "you could not do if it mattered.";