Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

Read a message with a file attached as its words and the file, not as [object Object] (#506)

kevine8f6d05abe26
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4d7d421f..0d8eeb23 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,13 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
## Unreleased
+### The Bot in the box and the LangGraph Bot read a message that has a file attached
+
+A message with a file attached reached both Bots as `[object Object],[object Object]`, in place of
+what the person typed and the file both: they read a message as a string, and one carrying a file is
+a list of parts. Each part now reaches the model as what it is — the words, the text of an attached
+file, an attached image — and a part neither can read is named rather than dropped.
+
### A fractional or out-of-range computer setting takes the fallback instead of breaking the boot
`numberFromEnv` accepted anything `Number` called finite and positive, so `PORT=80.5` bound
diff --git a/agent-bot/src/history.ts b/agent-bot/src/history.ts
index 881debfc..670cf1b9 100644
--- a/agent-bot/src/history.ts
+++ b/agent-bot/src/history.ts
@@ -8,6 +8,7 @@
import type { RunAgentInput } from "@ag-ui/core";
import type OpenAI from "openai";
import { COMPUTER_GUIDANCE, NO_ANSWER_CAME } from "../../shared/bot-prompt";
+import { userContent } from "../../shared/user-content";
export { NO_ANSWER_CAME };
@@ -58,7 +59,7 @@ export function toProviderMessages(
// Placed with the call they answer, below, rather than wherever they arrived.
if (message.role === "tool") continue;
if (message.role === "user") {
- messages.push({ role: "user", content: String(message.content ?? "") });
+ messages.push({ role: "user", content: userContent(message.content) });
continue;
}
if (message.role === "system" || message.role === "developer") {
diff --git a/agent-bot/tests/history.test.ts b/agent-bot/tests/history.test.ts
index f8c1854d..6fcb03a0 100644
--- a/agent-bot/tests/history.test.ts
+++ b/agent-bot/tests/history.test.ts
@@ -279,3 +279,66 @@ describe("a tool call restored from the thread store", () => {
expect(fn?.arguments).toBe('{"url":"https://news.ycombinator.com"}');
});
});
+
+/**
+ * A message somebody attached a file to.
+ *
+ * The composer sends it as a list of parts rather than a string: what the person typed, then the
+ * file. `copilot.ts` resolves the file before the run leaves the server, so a text file arrives here
+ * as a text part and an image as an `image` part carrying its bytes. `String()` of that list is
+ * `[object Object],[object Object]`, and that is what the model was sent in place of the question and
+ * the file both.
+ */
+describe("a message with a file attached", () => {
+ const typed = { type: "text", text: "How many rows say failed?" };
+ const csv = 'Attached file "runs.csv":\n\nid,status\n1,failed\n2,ok';
+
+ function userContent(content: unknown) {
+ const [user] = withoutGuidance(
+ toProviderMessages(
+ input([{ id: "1", role: "user", content } as unknown as Message]),
+ ),
+ );
+ return user?.content;
+ }
+
+ test("keeps what the person typed and the text of the file", () => {
+ expect(userContent([typed, { type: "text", text: csv }])).toEqual([
+ { type: "text", text: "How many rows say failed?" },
+ { type: "text", text: csv },
+ ]);
+ });
+
+ test("puts an attached image in front of the model", () => {
+ const image = {
+ type: "image",
+ source: { type: "data", value: "iVBORw0KGgo=", mimeType: "image/png" },
+ metadata: { attachmentId: "a1", filename: "chart.png" },
+ };
+ expect(userContent([typed, image])).toEqual([
+ { type: "text", text: "How many rows say failed?" },
+ {
+ type: "image_url",
+ image_url: { url: "data:image/png;base64,iVBORw0KGgo=" },
+ },
+ ]);
+ });
+
+ test("names a part it cannot read rather than dropping it", () => {
+ // A model told "[audio]" can say something was attached that it cannot hear. A model handed
+ // nothing answers as though nothing was attached.
+ const audio = {
+ type: "audio",
+ source: { type: "data", value: "UklGRg==", mimeType: "audio/wav" },
+ };
+ expect(userContent([typed, audio])).toEqual([
+ { type: "text", text: "How many rows say failed?" },
+ { type: "text", text: "[audio]" },
+ ]);
+ });
+
+ test("sends a message that is only text exactly as it was typed", () => {
+ // Nearly every message. Unchanged by this, and pinned so it stays that way.
+ expect(userContent("What is 17 times 3?")).toBe("What is 17 times 3?");
+ });
+});
diff --git a/agent-langgraph/src/history.ts b/agent-langgraph/src/history.ts
index 3b9db862..4e124df9 100644
--- a/agent-langgraph/src/history.ts
+++ b/agent-langgraph/src/history.ts
@@ -14,6 +14,7 @@ import {
ToolMessage,
} from "@langchain/core/messages";
import { COMPUTER_GUIDANCE, NO_ANSWER_CAME } from "../../shared/bot-prompt";
+import { userContent } from "../../shared/user-content";
/*
* Re-exported so this module's own tests and callers keep reading it from here, while the wording
@@ -57,7 +58,9 @@ export function toLangChainMessages(input: RunAgentInput): BaseMessage[] {
for (const message of input.messages) {
if (message.role === "user") {
- messages.push(new HumanMessage(String(message.content ?? "")));
+ messages.push(
+ new HumanMessage({ content: userContent(message.content) }),
+ );
continue;
}
if (message.role === "system" || message.role === "developer") {
diff --git a/agent-langgraph/tests/history.test.ts b/agent-langgraph/tests/history.test.ts
index f078fb4c..0e8fe764 100644
--- a/agent-langgraph/tests/history.test.ts
+++ b/agent-langgraph/tests/history.test.ts
@@ -139,3 +139,62 @@ describe("history with a tool call nobody answered", () => {
expect(byId.get("orphan")).toBe(NO_ANSWER_CAME);
});
});
+
+/**
+ * A message somebody attached a file to.
+ *
+ * The composer sends it as a list of parts rather than a string: what the person typed, then the
+ * file. `copilot.ts` resolves the file before the run leaves the server, so a text file arrives here
+ * as a text part and an image as an `image` part carrying its bytes. `String()` of that list is
+ * `[object Object],[object Object]`, and that is what the model was sent in place of the question and
+ * the file both.
+ */
+describe("a message with a file attached", () => {
+ const typed = { type: "text", text: "How many rows say failed?" };
+ const csv = 'Attached file "runs.csv":\n\nid,status\n1,failed\n2,ok';
+
+ function userContent(content: unknown) {
+ return toLangChainMessages(input([{ role: "user", content }])).at(-1)
+ ?.content;
+ }
+
+ test("keeps what the person typed and the text of the file", () => {
+ expect(userContent([typed, { type: "text", text: csv }])).toEqual([
+ { type: "text", text: "How many rows say failed?" },
+ { type: "text", text: csv },
+ ]);
+ });
+
+ test("puts an attached image in front of the model", () => {
+ const image = {
+ type: "image",
+ source: { type: "data", value: "iVBORw0KGgo=", mimeType: "image/png" },
+ metadata: { attachmentId: "a1", filename: "chart.png" },
+ };
+ expect(userContent([typed, image])).toEqual([
+ { type: "text", text: "How many rows say failed?" },
+ {
+ type: "image_url",
+ image_url: { url: "data:image/png;base64,iVBORw0KGgo=" },
+ },
+ ]);
+ });
+
+ test("names a part it cannot read rather than dropping it", () => {
+ // A model told "[audio]" can say something was attached that it cannot hear. A model handed
+ // nothing answers as though nothing was attached.
+ const audio = {
+ type: "audio",
+ source: { type: "data", value: "UklGRg==", mimeType: "audio/wav" },
+ };
+ expect(userContent([typed, audio])).toEqual([
+ { type: "text", text: "How many rows say failed?" },
+ { type: "text", text: "[audio]" },
+ ]);
+ });
+
+ test("sends a message that is only text exactly as it was typed", () => {
+ // Nearly every message. Unchanged by this, and pinned so it stays that way.
+ expect(userContent("What is 17 times 3?")).toBe("What is 17 times 3?");
+ });
+});
diff --git a/shared/user-content.ts b/shared/user-content.ts
new file mode 100644
index 00000000..0859ac9c
--- /dev/null
+++ b/shared/user-content.ts
@@ -0,0 +1,51 @@
+/**
+ * What a person said, in the shape a chat model reads. Read by both Bots.
+ *
+ * A message is a string until somebody attaches a file. Then the composer sends a list of parts —
+ * what the person typed, then the file — and `copilot.ts` resolves each file before the run leaves
+ * the server: a text file becomes a text part carrying its contents, an image an `image` part
+ * carrying its bytes. Both Bots read `content` with `String()`, which for that list is
+ * `[object Object],[object Object]`, so that is what the model was sent in place of the question and
+ * the file both.
+ *
+ * ONE DECLARATION FOR BOTH BOTS, for the reason `bot-prompt.ts` gives for `NO_ANSWER_CAME`. The two
+ * shapes returned are the ones OpenAI's chat completions take, and LangChain's OpenAI, Anthropic and
+ * Google converters read the same two blocks, so one answer serves `agent-bot` and `agent-langgraph`
+ * alike and the two cannot drift.
+ *
+ * A part neither shape can carry is NAMED, not dropped, which is the rule `resultText` in
+ * `server/src/plugins/mcp.ts` follows for a tool result: a model told "[audio]" can say something was
+ * attached that it cannot hear, and a model handed nothing answers as though nothing was attached.
+ */
+export type UserContentPart =
+ | { type: "text"; text: string }
+ | { type: "image_url"; image_url: { url: string } };
+
+export function userContent(content: unknown): string | UserContentPart[] {
+ if (!Array.isArray(content)) return String(content ?? "");
+ return content.map((part): UserContentPart => {
+ const item = (part ?? {}) as {
+ type?: unknown;
+ text?: unknown;
+ source?: { type?: unknown; value?: unknown; mimeType?: unknown } | null;
+ };
+ if (item.type === "text" && typeof item.text === "string") {
+ return { type: "text", text: item.text };
+ }
+ const source = item.source;
+ if (
+ item.type === "image" &&
+ source?.type === "data" &&
+ typeof source.value === "string" &&
+ typeof source.mimeType === "string"
+ ) {
+ return {
+ type: "image_url",
+ image_url: { url: `data:${source.mimeType};base64,${source.value}` },
+ };
+ }
+ const name =
+ typeof item.type === "string" && item.type ? item.type : "unknown";
+ return { type: "text", text: `[${name}]` };
+ });
+}