The Open Framework for Building Agents
feat(eve): send images to the model via toModelOutput content parts (#1250)
Ruic8041412139c
diff --git a/.changeset/tool-model-output-content-parts.md b/.changeset/tool-model-output-content-parts.mdnew file mode 100644index 000000000..39d251eb1--- /dev/null+++ b/.changeset/tool-model-output-content-parts.md@@ -0,0 +1,5 @@+---+"eve": patch+---++`toModelOutput` can now return `{ type: "content", value }` with text and file parts, so a tool can send images (screenshots, rendered charts) to vision-capable models as actual pixels instead of descriptions. Build outputs with the new `toolOutput.text` / `toolOutput.json` / `toolOutput.content` helpers and parts with `toolOutputPart.text` / `toolOutputPart.file`, all from `eve/tools`; file payloads must be base64 strings.diff --git a/docs/tools/overview.mdx b/docs/tools/overview.mdxindex b0fa3fbea..20b8d8a58 100644--- a/docs/tools/overview.mdx+++ b/docs/tools/overview.mdx@@ -82,6 +82,33 @@ toModelOutput(output) {Tool outputs must be JSON-serializable. Return plain objects, arrays, strings, numbers, booleans, or `null`; convert values like `Date`, `Map`, `Set`, `NaN`, and cyclic objects before returning them from `execute` or from a `{ type: "json" }` `toModelOutput`.+### Send images to the model with content parts++A tool that produces an image — a screenshot, a rendered chart — can hand the pixels to a vision-capable model by returning a `content` output from `toModelOutput`. Build outputs with the `toolOutput` helpers and parts with the `toolOutputPart` helpers, both from `eve/tools`:++```ts+import { defineTool, toolOutput, toolOutputPart } from "eve/tools";++export default defineTool({+ description: "Capture a screenshot of the current page",+ inputSchema: z.object({ url: z.string() }),+ async execute(input) {+ const png = await captureScreenshot(input.url);+ return { path: png.path, screenshotBase64: png.base64 };+ },+ toModelOutput(output) {+ return toolOutput.content([+ toolOutputPart.text(`Screenshot of ${output.path}:`),+ toolOutputPart.file(output.screenshotBase64, { mediaType: "image/png" }),+ ]);+ },+});+```++The `toolOutput.text` and `toolOutput.json` builders construct the other two output shapes; hand-written literals remain valid everywhere.++File payloads must be base64 strings — raw bytes (`Uint8Array`, `Buffer`) are rejected because they do not survive eve's durable JSON boundary. Keep payloads small: a content-part image is persisted in session history and re-sent on every subsequent model call, and eve warns above 3 MiB. Sending image parts to a model without vision support fails with that provider's error, the same as image parts in user messages.+Do not return secrets, credentials, unnecessary personal data, or unbounded sensitive content from tools. Filter, minimize, and redact tool outputs before returning them.## What to read nextdiff --git a/e2e/fixtures/agent-tools/agent/tools/render-stripes.ts b/e2e/fixtures/agent-tools/agent/tools/render-stripes.tsnew file mode 100644index 000000000..7f564b9fc--- /dev/null+++ b/e2e/fixtures/agent-tools/agent/tools/render-stripes.ts@@ -0,0 +1,79 @@+import { crc32, deflateSync } from "node:zlib";+import { defineTool, toolOutput, toolOutputPart } from "eve/tools";+import { z } from "zod";++// Single-token names no model paraphrases (unlike cyan/teal or purple/violet),+// so the eval can match the reply against the rendered sequence verbatim.+const PALETTE = {+ black: [0, 0, 0],+ blue: [0, 0, 255],+ green: [0, 160, 0],+ orange: [255, 140, 0],+ red: [255, 0, 0],+ yellow: [255, 220, 0],+} as const;++type ColorName = keyof typeof PALETTE;++const WIDTH = 240;+const HEIGHT = 120;+const STRIPE_COUNT = 3;++function pngChunk(type: string, data: Buffer): Buffer {+ const length = Buffer.alloc(4);+ length.writeUInt32BE(data.length);+ const typeBuffer = Buffer.from(type, "latin1");+ const crc = Buffer.alloc(4);+ crc.writeUInt32BE(crc32(Buffer.concat([typeBuffer, data])) >>> 0);+ return Buffer.concat([length, typeBuffer, data, crc]);+}++function renderStripesPng(stripes: readonly ColorName[]): Buffer {+ const ihdr = Buffer.alloc(13);+ ihdr.writeUInt32BE(WIDTH, 0);+ ihdr.writeUInt32BE(HEIGHT, 4);+ ihdr[8] = 8; // bit depth+ ihdr[9] = 2; // truecolor++ const stripeWidth = WIDTH / stripes.length;+ const row = Buffer.alloc(1 + WIDTH * 3); // leading scanline filter byte 0+ for (let x = 0; x < WIDTH; x += 1) {+ const stripe = Math.min(Math.floor(x / stripeWidth), stripes.length - 1);+ const color = PALETTE[stripes[stripe]!];+ row.set(color, 1 + x * 3);+ }+ const idat = deflateSync(Buffer.concat(Array.from({ length: HEIGHT }, () => row)), { level: 9 });++ return Buffer.concat([+ Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),+ pngChunk("IHDR", ihdr),+ pngChunk("IDAT", idat),+ pngChunk("IEND", Buffer.alloc(0)),+ ]);+}++export default defineTool({+ description:+ "Smoke-test fixture: renders an image of colored vertical stripes chosen at random. " ++ "Only call when the user explicitly asks to use `render-stripes`. The colors appear " ++ "ONLY in the returned image — inspect it visually; do not guess.",+ inputSchema: z.object({}),+ async execute() {+ const names = Object.keys(PALETTE) as ColorName[];+ const colors = [...names].sort(() => Math.random() - 0.5).slice(0, STRIPE_COUNT);+ const png = renderStripesPng(colors);+ // `colors` is the eval's answer key. It reaches action.result (and the+ // eval's event stream) but never the model: the projection below sends+ // only the pixels.+ return { colors, imageBase64: png.toString("base64") };+ },+ toModelOutput(output) {+ return toolOutput.content([+ toolOutputPart.text("Rendered stripes:"),+ toolOutputPart.file(output.imageBase64, {+ filename: "stripes.png",+ mediaType: "image/png",+ }),+ ]);+ },+});diff --git a/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.tsnew file mode 100644index 000000000..cab2a09d0--- /dev/null+++ b/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts@@ -0,0 +1,82 @@+import type { HandleMessageStreamEvent } from "eve/client";+import { defineEval } from "eve/evals";++const TOOL_NAME = "render-stripes";++// The stripe colors are randomized per run, so a blind model cannot pass by+// guessing; the eval stays self-contained by validating the reply against+// the answer key the tool records on action.result. The pixels reach the+// model exclusively through `toModelOutput` content parts.+export default defineEval({+ description: "Static tools smoke: toModelOutput content parts deliver an image to the model.",+ async test(t) {+ await t.send(+ `Call \`${TOOL_NAME}\` exactly once, look at the rendered image, and reply with only ` ++ "the stripe colors left to right, comma-separated.",+ );++ t.succeeded();+ t.noFailedActions();+ t.calledTool(TOOL_NAME, { count: 1, output: isRenderStripesOutput });+ t.eventsSatisfy("a reply names the rendered colors in order", (events) => {+ const answer = assistantAnswers(events)[0];+ return answer !== undefined && namesColorsInOrder(events, answer);+ });++ // The content part is baked into persisted history, so a follow-up turn+ // must answer from replay without re-running the tool.+ await t.send(+ "Without calling any tool, repeat the stripe colors left to right, comma-separated.",+ );++ t.succeeded();+ t.calledTool(TOOL_NAME, { count: 1 });+ t.eventsSatisfy("the replayed image still answers the follow-up", (events) => {+ const answer = assistantAnswers(events).at(-1);+ return answer !== undefined && namesColorsInOrder(events, answer);+ });+ },+});++function isRenderStripesOutput(value: unknown): boolean {+ if (typeof value !== "object" || value === null) return false;+ const output = value as { readonly colors?: unknown; readonly imageBase64?: unknown };+ return (+ Array.isArray(output.colors) &&+ output.colors.length > 0 &&+ output.colors.every((color) => typeof color === "string") &&+ typeof output.imageBase64 === "string" &&+ output.imageBase64.startsWith("iVBOR") // PNG magic bytes, base64-encoded+ );+}++function renderedColors(events: readonly HandleMessageStreamEvent[]): readonly string[] {+ for (const event of events) {+ if (event.type !== "action.result" || event.data.result.kind !== "tool-result") continue;+ if (event.data.result.toolName !== TOOL_NAME) continue;+ const output = event.data.result.output as { readonly colors?: unknown };+ if (Array.isArray(output?.colors) && output.colors.every((c) => typeof c === "string")) {+ return output.colors as string[];+ }+ }+ return [];+}++/** Final (non-tool-call) assistant messages, in turn order. */+function assistantAnswers(events: readonly HandleMessageStreamEvent[]): readonly string[] {+ return events.flatMap((event) =>+ event.type === "message.completed" &&+ event.data.finishReason !== "tool-calls" &&+ event.data.message !== null &&+ event.data.message.trim().length > 0+ ? [event.data.message]+ : [],+ );+}++function namesColorsInOrder(events: readonly HandleMessageStreamEvent[], answer: string): boolean {+ const colors = renderedColors(events);+ if (colors.length === 0) return false;+ const pattern = new RegExp(colors.map((color) => `\\b${color}\\b`).join("[\\s\\S]*"), "iu");+ return pattern.test(answer);+}diff --git a/packages/eve/extension-contracts/compatibility/dynamicTool/v3.ts b/packages/eve/extension-contracts/compatibility/dynamicTool/v3.tsnew file mode 100644index 000000000..675d31a89--- /dev/null+++ b/packages/eve/extension-contracts/compatibility/dynamicTool/v3.ts@@ -0,0 +1,26 @@+import { defineDynamic, defineTool } from "#public/tools/index.js";++export default defineDynamic({+ events: {+ "session.started": (_event, ctx) => ({+ summarize_report: defineTool({+ description: "Summarize a report for the model",+ inputSchema: { type: "object", properties: {} },+ async execute(_input, toolContext) {+ return {+ internal: "details",+ resolverSessionId: ctx.session.id,+ sessionId: toolContext.session.id,+ summary: "Report generated",+ };+ },+ toModelOutput(output) {+ return {+ type: "text",+ value: (output as { summary: string }).summary,+ };+ },+ }),+ }),+ },+});diff --git a/packages/eve/extension-contracts/compatibility/tool/v2.ts b/packages/eve/extension-contracts/compatibility/tool/v2.tsnew file mode 100644index 000000000..801d59638--- /dev/null+++ b/packages/eve/extension-contracts/compatibility/tool/v2.ts@@ -0,0 +1,19 @@+import { defineTool } from "#public/tools/index.js";++export default defineTool({+ description: "Summarize a report for the model",+ inputSchema: { type: "object", properties: {} },+ async execute(_input, ctx) {+ return {+ internal: "details",+ sessionId: ctx.session.id,+ summary: "Report generated",+ };+ },+ toModelOutput(output) {+ return {+ type: "text",+ value: (output as { summary: string }).summary,+ };+ },+});diff --git a/packages/eve/extension-contracts/entrypoints/tool.ts b/packages/eve/extension-contracts/entrypoints/tool.tsindex b03622f7f..f28a73a37 100644--- a/packages/eve/extension-contracts/entrypoints/tool.ts+++ b/packages/eve/extension-contracts/entrypoints/tool.ts@@ -9,5 +9,7 @@ export {experimental_workflow,isDisabledToolSentinel,isExperimentalWorkflowToolDefinition,+ toolOutput,+ toolOutputPart,toolResultFrom,} from "../../src/public/tools/index.ts";diff --git a/packages/eve/extension-contracts/reports/dynamicTool/v4.json b/packages/eve/extension-contracts/reports/dynamicTool/v4.jsonnew file mode 100644index 000000000..dcaf3e5e8--- /dev/null+++ b/packages/eve/extension-contracts/reports/dynamicTool/v4.json@@ -0,0 +1,13 @@+{+ "kind": "eve-extension-capability-contract",+ "capability": "dynamicTool",+ "epoch": 4,+ "sha256": "d5ed3d2576c48abe76044aaf8a9aa0e8ab734d3fc6d4d3bc7be91c3a6648a198",+ "exports": [+ "DynamicToolEntry",+ "DynamicToolEvents",+ "DynamicToolResult",+ "DynamicToolSet",+ "defineDynamic"+ ]+}diff --git a/packages/eve/extension-contracts/reports/tool/v3.json b/packages/eve/extension-contracts/reports/tool/v3.jsonnew file mode 100644index 000000000..fb1b02c70--- /dev/null+++ b/packages/eve/extension-contracts/reports/tool/v3.json@@ -0,0 +1,21 @@+{+ "kind": "eve-extension-capability-contract",+ "capability": "tool",+ "epoch": 3,+ "sha256": "64e6354ca546aaa2bbd07a05b5fb66596d34d14633222a41e5c647898211e14f",+ "exports": [+ "defineBashTool",+ "defineGlobTool",+ "defineGrepTool",+ "defineReadFileTool",+ "defineTool",+ "defineWriteFileTool",+ "disableTool",+ "experimental_workflow",+ "isDisabledToolSentinel",+ "isExperimentalWorkflowToolDefinition",+ "toolOutput",+ "toolOutputPart",+ "toolResultFrom"+ ]+}diff --git a/packages/eve/src/compiler/extension-compatibility.ts b/packages/eve/src/compiler/extension-compatibility.tsindex 13a8d2d47..fd3151903 100644--- a/packages/eve/src/compiler/extension-compatibility.ts+++ b/packages/eve/src/compiler/extension-compatibility.ts@@ -21,8 +21,8 @@ interface ExtensionCapabilityContract {const EXTENSION_CAPABILITY_CONTRACTS = {extension: { current: 1, supported: [1], dropped: {} },- tool: { current: 2, supported: [1, 2], dropped: {} },- dynamicTool: { current: 3, supported: [1, 2, 3], dropped: {} },+ tool: { current: 3, supported: [1, 2, 3], dropped: {} },+ dynamicTool: { current: 4, supported: [1, 2, 3, 4], dropped: {} },connection: { current: 2, supported: [1, 2], dropped: {} },hook: { current: 2, supported: [1, 2], dropped: {} },skill: { current: 1, supported: [1], dropped: {} },diff --git a/packages/eve/src/discover/agent.integration.test.ts b/packages/eve/src/discover/agent.integration.test.tsindex ad08e405a..90cd2bb19 100644--- a/packages/eve/src/discover/agent.integration.test.ts+++ b/packages/eve/src/discover/agent.integration.test.ts@@ -3,6 +3,10 @@ import { resolve } from "node:path";import { describe, expect, it } from "vitest";import { buildMemoryAgentProject } from "#internal/testing/memory-agent-source.js";+import {+ EXTENSION_CAPABILITY_SUPPORT,+ EXTENSION_CAPABILITY_VERSIONS,+} from "#compiler/extension-compatibility.js";import { discoverAgent } from "#discover/discover-agent.js";import {DISCOVER_EXTENSION_CAPABILITY_INCOMPATIBLE,@@ -983,6 +987,9 @@ describe("discoverAgent (memory)", () => {});it("rejects a mounted extension that requires an unsupported capability version", async () => {+ // One past the current epoch is unsupported by construction, so the+ // fixture keeps rejecting after future capability bumps.+ const unsupportedToolVersion = EXTENSION_CAPABILITY_VERSIONS.tool + 1;const project = buildMemoryAgentProject({appFiles: {"node_modules/@acme/crm/package.json": JSON.stringify({@@ -994,7 +1001,7 @@ describe("discoverAgent (memory)", () => {kind: "eve-extension",formatVersion: 1,builtWithEve: "9.0.0",- requires: { extension: 1, tool: 3 },+ requires: { extension: 1, tool: unsupportedToolVersion },}),"node_modules/@acme/crm/extension/extension.ts": "export default {};\n","node_modules/@acme/crm/extension/tools/search.ts": "export default {};\n",@@ -1015,8 +1022,10 @@ describe("discoverAgent (memory)", () => {(diagnostic) => diagnostic.code === DISCOVER_EXTENSION_CAPABILITY_INCOMPATIBLE,);expect(incompatible).toBeDefined();- expect(incompatible?.message).toContain("tool contract v3");- expect(incompatible?.message).toContain("versions: v1, v2");+ expect(incompatible?.message).toContain(`tool contract v${unsupportedToolVersion}`);+ expect(incompatible?.message).toContain(+ `versions: ${EXTENSION_CAPABILITY_SUPPORT.tool.map((version) => `v${version}`).join(", ")}`,+ );expect(result.manifest.resolvedExtensions).toEqual([]);});diff --git a/packages/eve/src/harness/tool-model-output.test.ts b/packages/eve/src/harness/tool-model-output.test.tsnew file mode 100644index 000000000..e5d2fbe9a--- /dev/null+++ b/packages/eve/src/harness/tool-model-output.test.ts@@ -0,0 +1,132 @@+import { describe, expect, it } from "vitest";++import { toolOutput, toolOutputPart } from "#public/tools/output-builders.js";+import { normalizeToolJsonOutput, normalizeToolModelOutput } from "#harness/tool-model-output.js";++function normalize(output: unknown): unknown {+ return normalizeToolModelOutput({ output, toolCallId: "call_1", toolName: "screenshot" });+}++describe("normalizeToolJsonOutput", () => {+ it("normalizes top-level undefined to null", () => {+ expect(+ normalizeToolJsonOutput({ boundary: "execute", output: undefined, toolName: "echo" }),+ ).toBeNull();+ });++ it("rejects non-JSON-serializable values with the boundary in the error", () => {+ expect(() =>+ normalizeToolJsonOutput({+ boundary: "execute",+ output: { now: new Date("2026-01-02T03:04:05.000Z") },+ toolCallId: "call_timestamp",+ toolName: "timestamp",+ }),+ ).toThrow(+ 'Tool "timestamp" call "call_timestamp" returned a non-JSON-serializable result. ' ++ "Expected a JSON-serializable value.",+ );+ });+});++describe("normalizeToolModelOutput", () => {+ it("normalizes a content output of text and file parts into the AI SDK shape", () => {+ const pixel =+ "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PQQkAAAgAsetfWiP4FgYrsKZeS0BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEDgsqnc8OJg6Ln3AAAAAElFTkSuQmCC";++ expect(+ normalize({+ type: "content",+ value: [+ { type: "text", text: "Screenshot:" },+ {+ type: "file",+ data: { type: "data", data: pixel },+ mediaType: "image/png",+ filename: "pixel.png",+ },+ ],+ }),+ ).toEqual({+ type: "content",+ value: [+ { type: "text", text: "Screenshot:" },+ {+ type: "file",+ data: { type: "data", data: pixel },+ mediaType: "image/png",+ filename: "pixel.png",+ },+ ],+ });+ });++ it("accepts outputs built with the toolOutput builders", () => {+ expect(normalize(toolOutput.text("visible"))).toEqual({ type: "text", value: "visible" });+ expect(normalize(toolOutput.json({ summary: "ok" }))).toEqual({+ type: "json",+ value: { summary: "ok" },+ });+ expect(+ normalize(+ toolOutput.content([+ toolOutputPart.text("Screenshot:"),+ toolOutputPart.file("aGVsbG8=", { mediaType: "image/png" }),+ ]),+ ),+ ).toEqual({+ type: "content",+ value: [+ { type: "text", text: "Screenshot:" },+ { type: "file", data: { type: "data", data: "aGVsbG8=" }, mediaType: "image/png" },+ ],+ });+ });++ it.each([+ [+ "raw Uint8Array payload",+ { type: "data", data: new Uint8Array([1, 2, 3]) },+ /base64 string, received raw bytes/u,+ ],+ ["raw untagged bytes", new Uint8Array([1, 2, 3]), /base64 string, received raw bytes/u],+ [+ "url file-data tag",+ { type: "url", url: "https://example.com/a.png" },+ /"url" is not supported yet/u,+ ],+ [+ "reference file-data tag",+ { type: "reference", reference: "file_abc" },+ /"reference" is not supported yet/u,+ ],+ ["text file-data tag", { type: "text", text: "inline" }, /"text" is not supported yet/u],+ ["untagged string data", "aGVsbG8=", /expected object, received string at "value\[0\]\.data"/u],+ ] as const)("rejects content file parts with %s", (_label, data, expected) => {+ expect(() =>+ normalize({ type: "content", value: [{ type: "file", data, mediaType: "image/png" }] }),+ ).toThrow(expected);+ });++ it("rejects empty content part arrays with the toModelOutput error identity", () => {+ expect(() => normalize({ type: "content", value: [] })).toThrow(+ 'Tool "screenshot" call "call_1" returned a non-JSON-serializable model output. ' ++ 'Too small: expected array to have >=1 items at "value"',+ );+ });++ it("rejects unknown content part types", () => {+ expect(() =>+ normalize({+ type: "content",+ value: [{ type: "media", data: "aGVsbG8=", mediaType: "image/png" }],+ }),+ ).toThrow(/Invalid discriminator value. Expected 'text' \| 'file' at "value\[0\]\.type"/u);+ });++ it("rejects unknown output types", () => {+ expect(() => normalize({ type: "markdown", value: "# nope" })).toThrow(+ /Invalid discriminator value. Expected 'text' \| 'json' \| 'content' at "type"/u,+ );+ });+});diff --git a/packages/eve/src/harness/tool-model-output.ts b/packages/eve/src/harness/tool-model-output.tsnew file mode 100644index 000000000..9ad4bb66e--- /dev/null+++ b/packages/eve/src/harness/tool-model-output.ts@@ -0,0 +1,181 @@+import type { JSONValue } from "ai";++import { z } from "#compiled/zod/index.js";+import { createLogger } from "#internal/logging.js";+import { parseJsonValue, type JsonValue } from "#shared/json.js";+import type { ToolModelOutputPart } from "#shared/tool-definition.js";+import { formatValidationError } from "#runtime/validation.js";+import { withToolOutputSerializationError } from "#harness/tool-output-serialization.js";++/**+ * A validated {@link ToolModelOutput} in the AI SDK's expected shape:+ * `json` values are proven JSON-serializable and `content` arrays are+ * mutable so the value is assignable to the SDK's `ToolResultOutput`.+ */+export type ToolModelOutputValue =+ | { readonly type: "json"; readonly value: JSONValue }+ | { readonly type: "text"; readonly value: string }+ | { readonly type: "content"; readonly value: ToolModelOutputPart[] };++const log = createLogger("harness.tool-model-output");++/**+ * Content-part file payloads above this decoded size warn. Matches the+ * attachment pipeline's inline-image hydration cap: unlike sandbox-ref+ * attachments, a content part is baked into persisted history and+ * re-sent on every subsequent model call, so large payloads compound.+ */+const CONTENT_FILE_WARN_BYTES = 3 * 1024 * 1024;++/** File-data tags the AI SDK accepts but eve does not support yet. */+const UNSUPPORTED_FILE_DATA_TAGS = new Set(["url", "reference", "text"]);++const toolModelOutputPartSchema = z.discriminatedUnion("type", [+ z.object({ type: z.literal("text"), text: z.string() }),+ z.object({+ type: z.literal("file"),+ data: z.object({ type: z.literal("data"), data: z.string() }),+ mediaType: z.string().min(1),+ filename: z.string().optional(),+ }),+]);++const toolModelOutputSchema = z.discriminatedUnion("type", [+ z.object({ type: z.literal("text"), value: z.string() }),+ z.object({ type: z.literal("json"), value: z.unknown() }),+ z.object({ type: z.literal("content"), value: z.array(toolModelOutputPartSchema).min(1) }),+]);++/**+ * Validates a tool output as JSON at one of the serialization boundaries,+ * normalizing top-level `undefined` to `null`. Throws+ * `ToolOutputSerializationError` on non-JSON-serializable values.+ */+export function normalizeToolJsonOutput(input: {+ readonly boundary: "execute" | "toModelOutput";+ readonly output: unknown;+ readonly toolCallId?: string;+ readonly toolName: string;+}): JsonValue {+ const candidate = input.output === undefined ? null : input.output;++ return withToolOutputSerializationError(input, () => {+ parseJsonValue(candidate);+ return candidate as JsonValue;+ });+}++/**+ * Single funnel for authored `toModelOutput` results. Validates the+ * eve-owned {@link ToolModelOutput} union and returns it in the AI SDK's+ * expected shape; every rejection throws `ToolOutputSerializationError`+ * at the `toModelOutput` boundary.+ */+export function normalizeToolModelOutput(input: {+ readonly output: unknown;+ readonly toolCallId?: string;+ readonly toolName: string;+}): ToolModelOutputValue {+ return withToolOutputSerializationError(+ {+ boundary: "toModelOutput",+ toolCallId: input.toolCallId,+ toolName: input.toolName,+ },+ () => {+ assertSupportedFilePartData(input.output);++ const parsed = toolModelOutputSchema.safeParse(input.output);+ if (!parsed.success) {+ throw new TypeError(formatValidationError(parsed.error));+ }++ const output = parsed.data;+ if (output.type === "json") {+ return {+ type: "json",+ value: normalizeToolJsonOutput({+ boundary: "toModelOutput",+ output: output.value,+ toolCallId: input.toolCallId,+ toolName: input.toolName,+ }) as JSONValue,+ };+ }+ if (output.type === "content") {+ for (const part of output.value) {+ if (part.type === "file") {+ warnOversizedFilePayload(part, input.toolName);+ }+ }+ }+ return output;+ },+ );+}++/**+ * Asserts every file part's `data` is the supported form, covering the two+ * author mistakes the schema cannot explain well: raw bytes (which+ * `JSON.stringify` would corrupt silently at the durable boundary — the+ * #497 failure class), and AI SDK `FileData` tags eve does not support+ * yet. On a `Uint8Array` the schema would report a missing `data.type`+ * key; the real fix is to base64-encode. Everything structural is left to+ * the schema.+ */+function assertSupportedFilePartData(output: unknown): void {+ if (output === null || typeof output !== "object") return;+ const candidate = output as { readonly type?: unknown; readonly value?: unknown };+ if (candidate.type !== "content" || !Array.isArray(candidate.value)) return;++ for (const part of candidate.value) {+ if (part === null || typeof part !== "object") continue;+ const filePart = part as { readonly type?: unknown; readonly data?: unknown };+ if (filePart.type !== "file") continue;++ const data = filePart.data;+ const taggedPayload =+ data !== null && typeof data === "object"+ ? (data as { readonly data?: unknown }).data+ : undefined;+ if (isBinaryPayload(data) || isBinaryPayload(taggedPayload)) {+ throw new TypeError(+ "Expected file content part data to be a base64 string, received raw bytes. " ++ "Base64-encode binary payloads before returning them from toModelOutput.",+ );+ }++ const tag =+ data !== null && typeof data === "object"+ ? (data as { readonly type?: unknown }).type+ : undefined;+ if (typeof tag === "string" && UNSUPPORTED_FILE_DATA_TAGS.has(tag)) {+ throw new TypeError(+ `File content part data tag "${tag}" is not supported yet; ` ++ 'only { type: "data" } with a base64 string is accepted.',+ );+ }+ }+}++function warnOversizedFilePayload(+ part: { readonly data: { readonly data: string }; readonly mediaType: string },+ toolName: string,+): void {+ const estimatedBytes = Math.floor((part.data.data.length * 3) / 4);+ if (estimatedBytes <= CONTENT_FILE_WARN_BYTES) return;+ log.warn(+ "content-part file payload exceeds the inline size guideline — it is persisted in " ++ "history and re-sent on every subsequent model call",+ {+ estimatedBytes,+ mediaType: part.mediaType,+ toolName,+ warnBytes: CONTENT_FILE_WARN_BYTES,+ },+ );+}++function isBinaryPayload(value: unknown): value is Uint8Array | ArrayBuffer {+ return value instanceof Uint8Array || value instanceof ArrayBuffer;+}diff --git a/packages/eve/src/harness/tools.test.ts b/packages/eve/src/harness/tools.test.tsindex e2d665d42..0cba032bd 100644--- a/packages/eve/src/harness/tools.test.ts+++ b/packages/eve/src/harness/tools.test.ts@@ -5,6 +5,7 @@ import { ContextContainer, contextStorage } from "#context/container.js";import { SessionKey, type Session } from "#context/keys.js";import { SCHEDULE_APP_AUTH } from "#channel/schedule-auth.js";import { always, never, once } from "#public/tools/approval/approval-helpers.js";+import type { RuntimeModelReference } from "#runtime/agent/bootstrap.js";import {WEB_SEARCH_ANTHROPIC_OUTPUT_SCHEMA,@@ -635,6 +636,51 @@ describe("buildToolSet", () => {);});+ it("passes valid content toModelOutput values through in the AI SDK shape", async () => {+ const pixel =+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/q842iQAAAABJRU5ErkJggg==";+ const tools: HarnessToolMap = new Map<string, HarnessToolDefinition>([+ [+ "screenshot",+ {+ description: "Capture a screenshot.",+ execute: async () => ({ ok: true }),+ inputSchema: jsonSchema({}),+ name: "screenshot",+ toModelOutput: () => ({+ type: "content" as const,+ value: [+ { type: "text" as const, text: "Screenshot:" },+ {+ type: "file" as const,+ data: { type: "data" as const, data: pixel },+ mediaType: "image/png",+ filename: "pixel.png",+ },+ ],+ }),+ },+ ],+ ]);++ const result = buildToolSet({ tools });++ await expect(+ projectSdkToolOutput({ output: { ok: true }, tool: result.screenshot }),+ ).resolves.toEqual({+ type: "content",+ value: [+ { type: "text", text: "Screenshot:" },+ {+ type: "file",+ data: { type: "data", data: pixel },+ mediaType: "image/png",+ filename: "pixel.png",+ },+ ],+ });+ });+it("passes valid text toModelOutput values through", async () => {const tools: HarnessToolMap = new Map<string, HarnessToolDefinition>([[diff --git a/packages/eve/src/harness/tools.ts b/packages/eve/src/harness/tools.tsindex 4431365e9..46cbf6267 100644--- a/packages/eve/src/harness/tools.ts+++ b/packages/eve/src/harness/tools.ts@@ -1,17 +1,10 @@-import {- type JSONValue,- type ToolApprovalConfiguration,- type ToolApprovalStatus,- type ToolSet,- tool,-} from "ai";+import { type ToolApprovalConfiguration, type ToolApprovalStatus, type ToolSet, tool } from "ai";import type { SessionCapabilities } from "#channel/types.js";import type { RuntimeModelReference } from "#runtime/agent/bootstrap.js";import { ASK_QUESTION_TOOL_NAME } from "#runtime/framework-tools/ask-question.js";import { WEB_SEARCH_TOOL_DEFINITION } from "#runtime/framework-tools/web-search.js";import { isObject } from "#shared/guards.js";-import { parseJsonValue, type JsonValue } from "#shared/json.js";import type { HarnessToolDefinition } from "#harness/execute-tool.js";import type { ApprovalStatus } from "#public/definitions/approval.js";import { resolveWebSearchBackend, resolveWebSearchProviderTool } from "#harness/provider-tools.js";@@ -25,13 +18,9 @@ import {modelFacingAuthorizationOutput,} from "#harness/authorization.js";import { stashToolInterrupt } from "#harness/tool-interrupts.js";-import { withToolOutputSerializationError } from "#harness/tool-output-serialization.js";+import { normalizeToolJsonOutput, normalizeToolModelOutput } from "#harness/tool-model-output.js";import type { ToolExecuteOptions } from "#shared/tool-definition.js";-type ToolModelOutputValue =- | { readonly type: "json"; readonly value: JSONValue }- | { readonly type: "text"; readonly value: string };-type NativeApprovalStatus = Exclude<ApprovalStatus, boolean>;const toolApprovals = new WeakMap<@@ -192,63 +181,6 @@ export function wrapToolExecute(};}-function normalizeToolJsonOutput(input: {- readonly boundary: "execute" | "toModelOutput";- readonly output: unknown;- readonly toolCallId?: string;- readonly toolName: string;-}): JsonValue {- const candidate = input.output === undefined ? null : input.output;-- return withToolOutputSerializationError(input, () => {- parseJsonValue(candidate);- return candidate as JsonValue;- });-}--function normalizeToolModelOutput(input: {- readonly output: unknown;- readonly toolCallId?: string;- readonly toolName: string;-}): ToolModelOutputValue {- return withToolOutputSerializationError(- {- boundary: "toModelOutput",- toolCallId: input.toolCallId,- toolName: input.toolName,- },- () => {- if (input.output === null || typeof input.output !== "object") {- throw new TypeError("Expected a tool model output object.");- }-- const output = input.output as { readonly type?: unknown; readonly value?: unknown };-- if (output.type === "text") {- if (typeof output.value !== "string") {- throw new TypeError('Expected text model output to include a string "value".');- }-- return { type: "text", value: output.value };- }-- if (output.type === "json") {- return {- type: "json",- value: normalizeToolJsonOutput({- boundary: "toModelOutput",- output: output.value,- toolCallId: input.toolCallId,- toolName: input.toolName,- }) as JSONValue,- };- }-- throw new TypeError('Expected tool model output type to be "text" or "json".');- },- );-}-/*** Builds the AI SDK ToolSet for one harness step.*diff --git a/packages/eve/src/internal/nitro/host/build-extension.scenario.test.ts b/packages/eve/src/internal/nitro/host/build-extension.scenario.test.tsindex 1e150589e..847df3a66 100644--- a/packages/eve/src/internal/nitro/host/build-extension.scenario.test.ts+++ b/packages/eve/src/internal/nitro/host/build-extension.scenario.test.ts@@ -6,7 +6,10 @@ import { pathToFileURL } from "node:url";import { describe, expect, it } from "vitest";-import { parseExtensionCompatibilityManifest } from "#compiler/extension-compatibility.js";+import {+ EXTENSION_CAPABILITY_VERSIONS,+ parseExtensionCompatibilityManifest,+} from "#compiler/extension-compatibility.js";import {buildExtensionPackage,tryReadExtensionBuildConfig,@@ -223,13 +226,15 @@ describe("extension build output", () => {);expect(manifest.kind).toBe("eve-extension");expect(manifest.builtWithEve).toMatch(/^\d+\.\d+\.\d+/);+ // The set of stamped capabilities is the assertion; the versions track+ // the current contract table so epoch bumps do not break this test.expect(manifest.requires).toEqual({- extension: 1,- tool: 2,- dynamicTool: 3,- skill: 1,- config: 1,- state: 2,+ extension: EXTENSION_CAPABILITY_VERSIONS.extension,+ tool: EXTENSION_CAPABILITY_VERSIONS.tool,+ dynamicTool: EXTENSION_CAPABILITY_VERSIONS.dynamicTool,+ skill: EXTENSION_CAPABILITY_VERSIONS.skill,+ config: EXTENSION_CAPABILITY_VERSIONS.config,+ state: EXTENSION_CAPABILITY_VERSIONS.state,});const dynamicToolDeclaration = await readFile(join(outDir, "extension", "tools", "crm_search.d.ts"),diff --git a/packages/eve/src/public/definitions/tool.ts b/packages/eve/src/public/definitions/tool.tsindex d6f21ecab..4dd0f4cb1 100644--- a/packages/eve/src/public/definitions/tool.ts+++ b/packages/eve/src/public/definitions/tool.ts@@ -28,7 +28,7 @@ type DynamicEventMapResult<TEvents extends DynamicEvents> = Awaited<ReturnType<DynamicEventMapHandler<TEvents>>>;-export type { ToolModelOutput } from "#shared/tool-definition.js";+export type { ToolModelOutput, ToolModelOutputPart } from "#shared/tool-definition.js";/*** Authorization provider passed to {@link ToolContext.getToken} ordiff --git a/packages/eve/src/public/tools/index.ts b/packages/eve/src/public/tools/index.tsindex efc0ee235..35bd0f198 100644--- a/packages/eve/src/public/tools/index.ts+++ b/packages/eve/src/public/tools/index.ts@@ -17,7 +17,9 @@ export {type ToolDefinition,type ToolContext,type ToolModelOutput,+ type ToolModelOutputPart,} from "#public/definitions/tool.js";+export { toolOutput, toolOutputPart } from "#public/tools/output-builders.js";export type { Approval, ApprovalContext, ApprovalStatus } from "#public/definitions/approval.js";export type {DynamicToolEntry,diff --git a/packages/eve/src/public/tools/output-builders.ts b/packages/eve/src/public/tools/output-builders.tsnew file mode 100644index 000000000..c15124085--- /dev/null+++ b/packages/eve/src/public/tools/output-builders.ts@@ -0,0 +1,78 @@+import type { ToolModelOutput, ToolModelOutputPart } from "#shared/tool-definition.js";++/**+ * Builders for the model-facing {@link ToolModelOutput} returned by+ * `toModelOutput`. Pure sugar over the union — each returns the+ * corresponding literal, and hand-written literals remain valid.+ *+ * ```ts+ * toModelOutput(output) {+ * return toolOutput.content([+ * toolOutputPart.text(`Screenshot of ${output.path}:`),+ * toolOutputPart.file(output.screenshotBase64, { mediaType: "image/png" }),+ * ]);+ * }+ * ```+ */+export const toolOutput = {+ /** Builds a text output: the model sees `value` as the tool result. */+ text(value: string): ToolModelOutput {+ return { type: "text", value };+ },+ /** Builds a JSON output; `value` must be JSON-serializable. */+ json(value: unknown): ToolModelOutput {+ return { type: "json", value };+ },+ /** Builds a content output from ordered {@link ToolModelOutputPart} entries. */+ content(value: readonly ToolModelOutputPart[]): ToolModelOutput {+ return { type: "content", value };+ },+};++/**+ * Builders for `content` {@link ToolModelOutput} parts. Pure sugar over+ * {@link ToolModelOutputPart} — each returns the corresponding part+ * literal, and hand-written literals remain valid.+ *+ * ```ts+ * toModelOutput(output) {+ * return {+ * type: "content",+ * value: [+ * toolOutputPart.text(`Screenshot of ${output.path}:`),+ * toolOutputPart.file(output.screenshotBase64, { mediaType: "image/png" }),+ * ],+ * };+ * }+ * ```+ */+export const toolOutputPart = {+ /** Builds a text part. */+ text(text: string): ToolModelOutputPart {+ return { type: "text", text };+ },+ /**+ * Builds a file part from a base64 payload. Binary data must be+ * base64-encoded by the caller; raw bytes are rejected by the harness+ * because they do not survive the durable JSON boundary.+ */+ file(+ base64: string,+ options: { readonly mediaType: string; readonly filename?: string },+ ): ToolModelOutputPart {+ const part: {+ type: "file";+ data: { type: "data"; data: string };+ mediaType: string;+ filename?: string;+ } = {+ type: "file",+ data: { type: "data", data: base64 },+ mediaType: options.mediaType,+ };+ if (options.filename !== undefined) {+ part.filename = options.filename;+ }+ return part;+ },+};diff --git a/packages/eve/src/shared/tool-definition.ts b/packages/eve/src/shared/tool-definition.tsindex 468fb4ba9..239a905af 100644--- a/packages/eve/src/shared/tool-definition.ts+++ b/packages/eve/src/shared/tool-definition.ts@@ -72,7 +72,31 @@ export interface PublicToolDefinitionWithExecuteFn<* eve-owned shape for the model-facing tool result produced by* `toModelOutput`. Structurally compatible with the AI SDK's* `ToolResultOutput` so the harness can forward it without conversion.+ *+ * The `content` variant carries an ordered list of+ * {@link ToolModelOutputPart} entries, letting a tool hand the model+ * text alongside inline files (e.g. a screenshot as vision input).*/export type ToolModelOutput =| { readonly type: "text"; readonly value: string }- | { readonly type: "json"; readonly value: unknown };+ | { readonly type: "json"; readonly value: unknown }+ | { readonly type: "content"; readonly value: readonly ToolModelOutputPart[] };++/**+ * One part of a `content` {@link ToolModelOutput}. Mirrors the AI SDK's+ * `ToolResultOutput` content parts narrowed to the JSON-safe subset:+ * file data is the SDK's tagged `FileData` union restricted to+ * `{ type: "data" }` with a base64 string, so persisted tool results+ * survive the durable JSON boundary. Use the `toolOutputPart` builders+ * from `eve/tools` to construct parts without hand-writing the nesting.+ */+export type ToolModelOutputPart =+ | { readonly type: "text"; readonly text: string }+ | {+ readonly type: "file";+ /** Tagged file data; only JSON-safe base64 payloads are accepted. */+ readonly data: { readonly type: "data"; readonly data: string };+ /** IANA media type, e.g. `image/png`. */+ readonly mediaType: string;+ readonly filename?: string;+ };