Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
Read a message with a file attached as its words and the file, not as [object Object] (#506)
kevine8f6d05abe26
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 4d7d421f..0d8eeb23 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,13 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.## Unreleased+### The Bot in the box and the LangGraph Bot read a message that has a file attached++A message with a file attached reached both Bots as `[object Object],[object Object]`, in place of+what the person typed and the file both: they read a message as a string, and one carrying a file is+a list of parts. Each part now reaches the model as what it is — the words, the text of an attached+file, an attached image — and a part neither can read is named rather than dropped.+### A fractional or out-of-range computer setting takes the fallback instead of breaking the boot`numberFromEnv` accepted anything `Number` called finite and positive, so `PORT=80.5` bounddiff --git a/agent-bot/src/history.ts b/agent-bot/src/history.tsindex 881debfc..670cf1b9 100644--- a/agent-bot/src/history.ts+++ b/agent-bot/src/history.ts@@ -8,6 +8,7 @@import type { RunAgentInput } from "@ag-ui/core";import type OpenAI from "openai";import { COMPUTER_GUIDANCE, NO_ANSWER_CAME } from "../../shared/bot-prompt";+import { userContent } from "../../shared/user-content";export { NO_ANSWER_CAME };@@ -58,7 +59,7 @@ export function toProviderMessages(// Placed with the call they answer, below, rather than wherever they arrived.if (message.role === "tool") continue;if (message.role === "user") {- messages.push({ role: "user", content: String(message.content ?? "") });+ messages.push({ role: "user", content: userContent(message.content) });continue;}if (message.role === "system" || message.role === "developer") {diff --git a/agent-bot/tests/history.test.ts b/agent-bot/tests/history.test.tsindex f8c1854d..6fcb03a0 100644--- a/agent-bot/tests/history.test.ts+++ b/agent-bot/tests/history.test.ts@@ -279,3 +279,66 @@ describe("a tool call restored from the thread store", () => {expect(fn?.arguments).toBe('{"url":"https://news.ycombinator.com"}');});});++/**+ * A message somebody attached a file to.+ *+ * The composer sends it as a list of parts rather than a string: what the person typed, then the+ * file. `copilot.ts` resolves the file before the run leaves the server, so a text file arrives here+ * as a text part and an image as an `image` part carrying its bytes. `String()` of that list is+ * `[object Object],[object Object]`, and that is what the model was sent in place of the question and+ * the file both.+ */+describe("a message with a file attached", () => {+ const typed = { type: "text", text: "How many rows say failed?" };+ const csv = 'Attached file "runs.csv":\n\nid,status\n1,failed\n2,ok';++ function userContent(content: unknown) {+ const [user] = withoutGuidance(+ toProviderMessages(+ input([{ id: "1", role: "user", content } as unknown as Message]),+ ),+ );+ return user?.content;+ }++ test("keeps what the person typed and the text of the file", () => {+ expect(userContent([typed, { type: "text", text: csv }])).toEqual([+ { type: "text", text: "How many rows say failed?" },+ { type: "text", text: csv },+ ]);+ });++ test("puts an attached image in front of the model", () => {+ const image = {+ type: "image",+ source: { type: "data", value: "iVBORw0KGgo=", mimeType: "image/png" },+ metadata: { attachmentId: "a1", filename: "chart.png" },+ };+ expect(userContent([typed, image])).toEqual([+ { type: "text", text: "How many rows say failed?" },+ {+ type: "image_url",+ image_url: { url: "data:image/png;base64,iVBORw0KGgo=" },+ },+ ]);+ });++ test("names a part it cannot read rather than dropping it", () => {+ // A model told "[audio]" can say something was attached that it cannot hear. A model handed+ // nothing answers as though nothing was attached.+ const audio = {+ type: "audio",+ source: { type: "data", value: "UklGRg==", mimeType: "audio/wav" },+ };+ expect(userContent([typed, audio])).toEqual([+ { type: "text", text: "How many rows say failed?" },+ { type: "text", text: "[audio]" },+ ]);+ });++ test("sends a message that is only text exactly as it was typed", () => {+ // Nearly every message. Unchanged by this, and pinned so it stays that way.+ expect(userContent("What is 17 times 3?")).toBe("What is 17 times 3?");+ });+});diff --git a/agent-langgraph/src/history.ts b/agent-langgraph/src/history.tsindex 3b9db862..4e124df9 100644--- a/agent-langgraph/src/history.ts+++ b/agent-langgraph/src/history.ts@@ -14,6 +14,7 @@ import {ToolMessage,} from "@langchain/core/messages";import { COMPUTER_GUIDANCE, NO_ANSWER_CAME } from "../../shared/bot-prompt";+import { userContent } from "../../shared/user-content";/** Re-exported so this module's own tests and callers keep reading it from here, while the wording@@ -57,7 +58,9 @@ export function toLangChainMessages(input: RunAgentInput): BaseMessage[] {for (const message of input.messages) {if (message.role === "user") {- messages.push(new HumanMessage(String(message.content ?? "")));+ messages.push(+ new HumanMessage({ content: userContent(message.content) }),+ );continue;}if (message.role === "system" || message.role === "developer") {diff --git a/agent-langgraph/tests/history.test.ts b/agent-langgraph/tests/history.test.tsindex f078fb4c..0e8fe764 100644--- a/agent-langgraph/tests/history.test.ts+++ b/agent-langgraph/tests/history.test.ts@@ -139,3 +139,62 @@ describe("history with a tool call nobody answered", () => {expect(byId.get("orphan")).toBe(NO_ANSWER_CAME);});});++/**+ * A message somebody attached a file to.+ *+ * The composer sends it as a list of parts rather than a string: what the person typed, then the+ * file. `copilot.ts` resolves the file before the run leaves the server, so a text file arrives here+ * as a text part and an image as an `image` part carrying its bytes. `String()` of that list is+ * `[object Object],[object Object]`, and that is what the model was sent in place of the question and+ * the file both.+ */+describe("a message with a file attached", () => {+ const typed = { type: "text", text: "How many rows say failed?" };+ const csv = 'Attached file "runs.csv":\n\nid,status\n1,failed\n2,ok';++ function userContent(content: unknown) {+ return toLangChainMessages(input([{ role: "user", content }])).at(-1)+ ?.content;+ }++ test("keeps what the person typed and the text of the file", () => {+ expect(userContent([typed, { type: "text", text: csv }])).toEqual([+ { type: "text", text: "How many rows say failed?" },+ { type: "text", text: csv },+ ]);+ });++ test("puts an attached image in front of the model", () => {+ const image = {+ type: "image",+ source: { type: "data", value: "iVBORw0KGgo=", mimeType: "image/png" },+ metadata: { attachmentId: "a1", filename: "chart.png" },+ };+ expect(userContent([typed, image])).toEqual([+ { type: "text", text: "How many rows say failed?" },+ {+ type: "image_url",+ image_url: { url: "data:image/png;base64,iVBORw0KGgo=" },+ },+ ]);+ });++ test("names a part it cannot read rather than dropping it", () => {+ // A model told "[audio]" can say something was attached that it cannot hear. A model handed+ // nothing answers as though nothing was attached.+ const audio = {+ type: "audio",+ source: { type: "data", value: "UklGRg==", mimeType: "audio/wav" },+ };+ expect(userContent([typed, audio])).toEqual([+ { type: "text", text: "How many rows say failed?" },+ { type: "text", text: "[audio]" },+ ]);+ });++ test("sends a message that is only text exactly as it was typed", () => {+ // Nearly every message. Unchanged by this, and pinned so it stays that way.+ expect(userContent("What is 17 times 3?")).toBe("What is 17 times 3?");+ });+});diff --git a/shared/user-content.ts b/shared/user-content.tsnew file mode 100644index 00000000..0859ac9c--- /dev/null+++ b/shared/user-content.ts@@ -0,0 +1,51 @@+/**+ * What a person said, in the shape a chat model reads. Read by both Bots.+ *+ * A message is a string until somebody attaches a file. Then the composer sends a list of parts —+ * what the person typed, then the file — and `copilot.ts` resolves each file before the run leaves+ * the server: a text file becomes a text part carrying its contents, an image an `image` part+ * carrying its bytes. Both Bots read `content` with `String()`, which for that list is+ * `[object Object],[object Object]`, so that is what the model was sent in place of the question and+ * the file both.+ *+ * ONE DECLARATION FOR BOTH BOTS, for the reason `bot-prompt.ts` gives for `NO_ANSWER_CAME`. The two+ * shapes returned are the ones OpenAI's chat completions take, and LangChain's OpenAI, Anthropic and+ * Google converters read the same two blocks, so one answer serves `agent-bot` and `agent-langgraph`+ * alike and the two cannot drift.+ *+ * A part neither shape can carry is NAMED, not dropped, which is the rule `resultText` in+ * `server/src/plugins/mcp.ts` follows for a tool result: a model told "[audio]" can say something was+ * attached that it cannot hear, and a model handed nothing answers as though nothing was attached.+ */+export type UserContentPart =+ | { type: "text"; text: string }+ | { type: "image_url"; image_url: { url: string } };++export function userContent(content: unknown): string | UserContentPart[] {+ if (!Array.isArray(content)) return String(content ?? "");+ return content.map((part): UserContentPart => {+ const item = (part ?? {}) as {+ type?: unknown;+ text?: unknown;+ source?: { type?: unknown; value?: unknown; mimeType?: unknown } | null;+ };+ if (item.type === "text" && typeof item.text === "string") {+ return { type: "text", text: item.text };+ }+ const source = item.source;+ if (+ item.type === "image" &&+ source?.type === "data" &&+ typeof source.value === "string" &&+ typeof source.mimeType === "string"+ ) {+ return {+ type: "image_url",+ image_url: { url: `data:${source.mimeType};base64,${source.value}` },+ };+ }+ const name =+ typeof item.type === "string" && item.type ? item.type : "unknown";+ return { type: "text", text: `[${name}]` };+ });+}