The Open Framework for Building Agents
fix(eve): catch unserializable tool output values instead of sending them to the model (#244)
Felix Arntz25b1b140c167
diff --git a/.changeset/rare-mails-take.md b/.changeset/rare-mails-take.mdnew file mode 100644index 000000000..371688ed7--- /dev/null+++ b/.changeset/rare-mails-take.md@@ -0,0 +1,5 @@+---+"eve": patch+---++fix(eve): catch unserializable tool output values instead of sending them to the modeldiff --git a/docs/tools/overview.mdx b/docs/tools/overview.mdxindex c43637a02..3c51181f0 100644--- a/docs/tools/overview.mdx+++ b/docs/tools/overview.mdx@@ -77,6 +77,8 @@ toModelOutput(output) {`toModelOutput` receives the full, typed `execute` return and only affects the model. Channel event handlers and hooks still get the full output on `action.result`, so a channel can render rich platform output (Slack Block Kit, say) the model never sees. Return `{ type: "text", value }` for a summary, or `{ type: "json", value }` for a smaller object.+Tool outputs must be JSON-serializable. Return plain objects, arrays, strings, numbers, booleans, or `null`; convert values like `Date`, `Map`, `Set`, `NaN`, and cyclic objects before returning them from `execute` or from a `{ type: "json" }` `toModelOutput`.+Do not return secrets, credentials, unnecessary personal data, or unbounded sensitive content from tools. Filter, minimize, and redact tool outputs before returning them.## What to read nextdiff --git a/packages/eve/src/harness/action-result-helpers.test.ts b/packages/eve/src/harness/action-result-helpers.test.tsnew file mode 100644index 000000000..25507c5bd--- /dev/null+++ b/packages/eve/src/harness/action-result-helpers.test.ts@@ -0,0 +1,49 @@+import { describe, expect, it } from "vitest";++import { createRuntimeToolResultFromValue } from "#harness/action-result-helpers.js";++describe("createRuntimeToolResultFromValue", () => {+ it("rejects non-JSON-serializable successful action results", () => {+ expect(() =>+ createRuntimeToolResultFromValue({+ callId: "call_timestamp",+ output: { now: new Date("2026-01-02T03:04:05.000Z") },+ toolName: "timestamp",+ }),+ ).toThrow(+ 'Tool "timestamp" call "call_timestamp" produced a non-JSON-serializable action result. Expected a JSON-serializable value.',+ );+ });++ it("normalizes top-level undefined action results to null", () => {+ expect(+ createRuntimeToolResultFromValue({+ callId: "call_empty",+ output: undefined,+ toolName: "empty",+ }),+ ).toEqual({+ callId: "call_empty",+ kind: "tool-result",+ output: null,+ toolName: "empty",+ });+ });++ it("projects Error instances for failed action results to their message", () => {+ expect(+ createRuntimeToolResultFromValue({+ callId: "call_failed",+ isError: true,+ output: new Error("tool failed"),+ toolName: "broken",+ }),+ ).toEqual({+ callId: "call_failed",+ isError: true,+ kind: "tool-result",+ output: "tool failed",+ toolName: "broken",+ });+ });+});diff --git a/packages/eve/src/harness/action-result-helpers.ts b/packages/eve/src/harness/action-result-helpers.tsindex 04edca81a..b1aa78f0b 100644--- a/packages/eve/src/harness/action-result-helpers.ts+++ b/packages/eve/src/harness/action-result-helpers.ts@@ -7,19 +7,17 @@ import {isAuthorizationSignal,isAuthorizationPendingModelOutput,} from "#harness/authorization.js";+import { withToolOutputSerializationError } from "#harness/tool-output-serialization.js";type ToolResponsePart = Extract<ModelMessage, { role: "tool" }>["content"][number];type ToolResultPart = Extract<ToolResponsePart, { type: "tool-result" }>;+type ToolResultOutputCandidate =+ | ToolResultPart["output"]+ | { readonly type: "error-json" | "json"; readonly value: unknown };/**- * Coerces an arbitrary value to a JSON-safe {@link JsonValue} without- * premature stringification.- *- * - Strings, numbers, booleans, and `null` pass through as primitives.- * - `Error` instances surface only their message (no stack leak).- * - Plain objects and arrays pass through structurally.- * - Non-JSON-representable values (functions, symbols, BigInts) fall- * back to `String(value)`.+ * Coerces framework-owned sentinel values and validates the result payload as+ * JSON without attaching tool-specific context.*/function toJsonValue(value: unknown): JsonValue {if (isAuthorizationSignal(value)) {@@ -33,24 +31,7 @@ function toJsonValue(value: unknown): JsonValue {return parseJsonValue(authorizationPendingAsJsonObject(value));}- if (- value === null ||- typeof value === "string" ||- typeof value === "number" ||- typeof value === "boolean"- ) {- return value;- }-- if (value instanceof Error) {- return value.message;- }-- if (typeof value === "object") {- return value as JsonValue;- }-- return String(value);+ return parseJsonValue(value === undefined ? null : value);}/**@@ -60,9 +41,8 @@ function toJsonValue(value: unknown): JsonValue {* native tool execution (via {@link createRuntimeToolResultFromStepResult} /* {@link createRuntimeToolResultFromMessagePart}) and code-mode nested tool* calls funnel through here, so the raw-output-vs-`toModelOutput` decision —- * always raw — is decided once. The output is passed through structurally- * because it is already JSON-serialized (the AI SDK tool result, or the- * code-mode worker bridge).+ * always raw — is decided once. The output is validated as JSON here so bad+ * values never reach protocol events or persisted history.*/export function createRuntimeToolResultFromValue(input: {readonly callId: string;@@ -73,7 +53,17 @@ export function createRuntimeToolResultFromValue(input: {const result: RuntimeToolResultActionResult = {callId: input.callId,kind: "tool-result",- output: toJsonValue(input.output),+ output: toolResultOutputToJsonValue({+ output: {+ type: input.isError === true ? "error-json" : "json",+ value:+ input.isError === true && input.output instanceof Error+ ? input.output.message+ : input.output,+ },+ toolCallId: input.callId,+ toolName: input.toolName,+ }),toolName: input.toolName,};@@ -107,28 +97,45 @@ export function createRuntimeToolResultFromMessagePart(): RuntimeToolResultActionResult {return createRuntimeToolResultFromValue({callId: part.toolCallId,- output: toolResultOutputToJsonValue(part.output),+ output: toolResultOutputToJsonValue({+ output: part.output,+ toolCallId: part.toolCallId,+ toolName: part.toolName,+ }),toolName: part.toolName,isError: isToolResultError(part.output),});}-function toolResultOutputToJsonValue(output: ToolResultPart["output"]): JsonValue {- switch (output.type) {- case "text":- case "error-text":- return output.value;- case "json":- case "error-json":- return toJsonValue(output.value);- case "execution-denied":- return {- code: "TOOL_EXECUTION_DENIED",- message: output.reason ?? "Tool execution was denied.",- };- case "content":- return toJsonValue(output.value);- }+function toolResultOutputToJsonValue(input: {+ readonly output: ToolResultOutputCandidate;+ readonly toolCallId: string;+ readonly toolName: string;+}): JsonValue {+ return withToolOutputSerializationError(+ {+ boundary: "action.result",+ toolCallId: input.toolCallId,+ toolName: input.toolName,+ },+ () => {+ switch (input.output.type) {+ case "text":+ case "error-text":+ return input.output.value;+ case "json":+ case "error-json":+ return toJsonValue(input.output.value);+ case "execution-denied":+ return {+ code: "TOOL_EXECUTION_DENIED",+ message: input.output.reason ?? "Tool execution was denied.",+ };+ case "content":+ return toJsonValue(input.output.value);+ }+ },+ );}function isToolResultError(output: ToolResultPart["output"]): boolean {diff --git a/packages/eve/src/harness/tool-output-serialization.ts b/packages/eve/src/harness/tool-output-serialization.tsnew file mode 100644index 000000000..0b370dc98--- /dev/null+++ b/packages/eve/src/harness/tool-output-serialization.ts@@ -0,0 +1,66 @@+type ToolOutputSerializationBoundary = "action.result" | "execute" | "toModelOutput";++export class ToolOutputSerializationError extends TypeError {+ readonly toolCallId: string | undefined;+ readonly toolName: string;++ constructor(input: {+ readonly boundary: ToolOutputSerializationBoundary;+ readonly cause?: unknown;+ readonly toolCallId?: string;+ readonly toolName: string;+ }) {+ super(formatToolOutputSerializationMessage(input));+ this.name = "ToolOutputSerializationError";+ this.toolCallId = input.toolCallId;+ this.toolName = input.toolName;+ if (input.cause !== undefined) {+ (this as Error & { cause?: unknown }).cause = input.cause;+ }+ }+}++function formatToolOutputSerializationMessage(input: {+ readonly boundary: ToolOutputSerializationBoundary;+ readonly cause?: unknown;+ readonly toolCallId?: string;+ readonly toolName: string;+}): string {+ const subject =+ input.toolCallId === undefined+ ? `Tool "${input.toolName}"`+ : `Tool "${input.toolName}" call "${input.toolCallId}"`;+ const action =+ input.boundary === "execute"+ ? "returned a non-JSON-serializable result"+ : input.boundary === "toModelOutput"+ ? "returned a non-JSON-serializable model output"+ : "produced a non-JSON-serializable action result";+ const detail =+ input.cause instanceof Error && input.cause.message.length > 0 ? ` ${input.cause.message}` : "";++ return `${subject} ${action}.${detail}`;+}++export function withToolOutputSerializationError<T>(+ input: {+ readonly boundary: ToolOutputSerializationBoundary;+ readonly toolCallId?: string;+ readonly toolName: string;+ },+ fn: () => T,+): T {+ try {+ return fn();+ } catch (error) {+ if (error instanceof ToolOutputSerializationError) {+ throw error;+ }+ throw new ToolOutputSerializationError({+ boundary: input.boundary,+ cause: error,+ toolCallId: input.toolCallId,+ toolName: input.toolName,+ });+ }+}diff --git a/packages/eve/src/harness/tools.test.ts b/packages/eve/src/harness/tools.test.tsindex ee25b5d69..4970bae23 100644--- a/packages/eve/src/harness/tools.test.ts+++ b/packages/eve/src/harness/tools.test.ts@@ -22,6 +22,45 @@ function getOutputJsonSchema(tool: unknown): unknown {return (tool as { outputSchema: { jsonSchema: unknown } }).outputSchema.jsonSchema;}+async function executeSdkTool(input: {+ readonly tool: unknown;+ readonly toolCallId?: string;+ readonly toolInput?: unknown;+}): Promise<unknown> {+ const execute = (+ input.tool as {+ readonly execute?: (+ toolInput: unknown,+ options: { readonly toolCallId: string },+ ) => Promise<unknown> | unknown;+ }+ ).execute;+ expect(execute).toBeTypeOf("function");+ return await execute!(input.toolInput ?? {}, { toolCallId: input.toolCallId ?? "call_1" });+}++async function projectSdkToolOutput(input: {+ readonly output: unknown;+ readonly tool: unknown;+ readonly toolCallId?: string;+}): Promise<unknown> {+ const toModelOutput = (+ input.tool as {+ readonly toModelOutput?: (options: {+ readonly input: unknown;+ readonly output: unknown;+ readonly toolCallId: string;+ }) => Promise<unknown> | unknown;+ }+ ).toModelOutput;+ expect(toModelOutput).toBeTypeOf("function");+ return await toModelOutput!({+ input: {},+ output: input.output,+ toolCallId: input.toolCallId ?? "call_1",+ });+}+describe("buildToolSet", () => {it("passes through the input schema to the SDK tool", () => {const schema = {@@ -317,6 +356,125 @@ describe("buildToolSet", () => {expect(projected).toEqual({ type: "text", value: "summary" });});+ it("rejects non-JSON-serializable execute output at the tool boundary", async () => {+ const tools: HarnessToolMap = new Map<string, HarnessToolDefinition>([+ [+ "timestamp",+ {+ description: "Return a timestamp.",+ execute: async () => ({ now: new Date("2026-01-02T03:04:05.000Z") }),+ inputSchema: jsonSchema({}),+ name: "timestamp",+ },+ ],+ ]);++ const result = buildToolSet({ tools });++ await expect(+ executeSdkTool({+ tool: result.timestamp,+ toolCallId: "call_timestamp",+ }),+ ).rejects.toThrow(+ 'Tool "timestamp" call "call_timestamp" returned a non-JSON-serializable result. Expected a JSON-serializable value.',+ );+ });++ it("preserves valid execute output identity", async () => {+ const output = { summary: "ok" };+ const tools: HarnessToolMap = new Map<string, HarnessToolDefinition>([+ [+ "report",+ {+ description: "Return a report.",+ execute: async () => output,+ inputSchema: jsonSchema({}),+ name: "report",+ },+ ],+ ]);++ const result = buildToolSet({ tools });++ await expect(executeSdkTool({ tool: result.report })).resolves.toBe(output);+ });++ it("normalizes top-level undefined execute output to null", async () => {+ const tools: HarnessToolMap = new Map<string, HarnessToolDefinition>([+ [+ "maybe_empty",+ {+ description: "Return no value.",+ execute: async () => undefined,+ inputSchema: jsonSchema({}),+ name: "maybe_empty",+ },+ ],+ ]);++ const result = buildToolSet({ tools });++ await expect(executeSdkTool({ tool: result.maybe_empty })).resolves.toBeNull();+ });++ it("rejects non-JSON-serializable toModelOutput JSON values", async () => {+ const tools: HarnessToolMap = new Map<string, HarnessToolDefinition>([+ [+ "timestamp",+ {+ description: "Return a timestamp.",+ execute: async () => ({ ok: true }),+ inputSchema: jsonSchema({}),+ name: "timestamp",+ toModelOutput: () => ({+ type: "json" as const,+ value: { now: new Date("2026-01-02T03:04:05.000Z") },+ }),+ },+ ],+ ]);++ const result = buildToolSet({ tools });++ await expect(+ projectSdkToolOutput({+ output: { ok: true },+ tool: result.timestamp,+ toolCallId: "call_timestamp",+ }),+ ).rejects.toThrow(+ 'Tool "timestamp" call "call_timestamp" returned a non-JSON-serializable model output. Expected a JSON-serializable value.',+ );+ });++ it("passes valid text toModelOutput values through", async () => {+ const tools: HarnessToolMap = new Map<string, HarnessToolDefinition>([+ [+ "report",+ {+ description: "Return a report.",+ execute: async () => ({ ok: true }),+ inputSchema: jsonSchema({}),+ name: "report",+ toModelOutput: () => ({ type: "text" as const, value: "visible" }),+ },+ ],+ ]);++ const result = buildToolSet({ tools });++ await expect(+ projectSdkToolOutput({+ output: { ok: true },+ tool: result.report,+ }),+ ).resolves.toEqual({+ type: "text",+ value: "visible",+ });+ });+describe("tool-level needsApproval override", () => {type NeedsApprovalFn = (input: unknown, context: unknown) => Promise<boolean> | boolean;diff --git a/packages/eve/src/harness/tools.ts b/packages/eve/src/harness/tools.tsindex f7020540c..e4fca9603 100644--- a/packages/eve/src/harness/tools.ts+++ b/packages/eve/src/harness/tools.ts@@ -5,6 +5,7 @@ import type { RuntimeModelReference } from "#runtime/agent/bootstrap.js";import { ASK_QUESTION_TOOL_NAME } from "#runtime/framework-tools/ask-question.js";import { WEB_SEARCH_TOOL_DEFINITION } from "#runtime/framework-tools/web-search.js";import { isObject } from "#shared/guards.js";+import { parseJsonValue, type JsonValue } from "#shared/json.js";import type { HarnessToolDefinition } from "#harness/execute-tool.js";import { resolveWebSearchBackend, resolveWebSearchProviderTool } from "#harness/provider-tools.js";import type { HarnessToolMap } from "#harness/types.js";@@ -16,8 +17,13 @@ import {modelFacingAuthorizationOutput,} from "#harness/authorization.js";import { stashToolInterrupt } from "#harness/tool-interrupts.js";+import { withToolOutputSerializationError } from "#harness/tool-output-serialization.js";import { isCodeModeToolExecutionOptions } from "#runtime/framework-tools/code-mode-connection-auth.js";+type ToolModelOutputValue =+ | { readonly type: "json"; readonly value: JSONValue }+ | { readonly type: "text"; readonly value: string };+/*** Builds an AI SDK `ToolSet` from unified harness tool definitions.*@@ -62,7 +68,13 @@ export function buildToolSet(input: {outputSchema: definition.outputSchema,...(definition.execute !== undefined? {- toModelOutput: ({ output }: { output: unknown }) => {+ toModelOutput: async ({+ output,+ toolCallId,+ }: {+ readonly output: unknown;+ readonly toolCallId?: string;+ }) => {if (isAuthorizationPendingModelOutput(output)) {return {type: "text" as const,@@ -70,22 +82,36 @@ export function buildToolSet(input: {};}if (authorToModelOutput !== undefined) {- return authorToModelOutput(output) as- | { type: "text"; value: string }- | { type: "json"; value: JSONValue };+ return normalizeToolModelOutput({+ output: await authorToModelOutput(output),+ toolCallId,+ toolName: definition.name,+ });}if (typeof output === "string") {return { type: "text" as const, value: output };}- return { type: "json" as const, value: (output ?? null) as JSONValue };+ return normalizeToolModelOutput({+ output: { type: "json" as const, value: output ?? null },+ toolCallId,+ toolName: definition.name,+ });},}: authorToModelOutput !== undefined? {- toModelOutput: ({ output }: { output: unknown }) =>- authorToModelOutput(output) as- | { type: "text"; value: string }- | { type: "json"; value: JSONValue },+ toModelOutput: async ({+ output,+ toolCallId,+ }: {+ readonly output: unknown;+ readonly toolCallId?: string;+ }) =>+ normalizeToolModelOutput({+ output: await authorToModelOutput(output),+ toolCallId,+ toolName: definition.name,+ }),}: {}),});@@ -138,13 +164,77 @@ export function wrapToolExecute(if (execute === undefined) return undefined;return async (input, options) => {const output = await execute(input);- if (!isAuthorizationSignal(output)) return output;- if (isCodeModeToolExecutionOptions(options)) return output;- stashToolInterrupt(loadContext(), options.toolCallId, output);- return modelFacingAuthorizationOutput(output);+ if (isAuthorizationSignal(output)) {+ if (isCodeModeToolExecutionOptions(options)) return output;+ stashToolInterrupt(loadContext(), options.toolCallId, output);+ return modelFacingAuthorizationOutput(output);+ }+ return normalizeToolJsonOutput({+ boundary: "execute",+ output,+ toolCallId: options.toolCallId,+ toolName: definition.name,+ });};}+function normalizeToolJsonOutput(input: {+ readonly boundary: "execute" | "toModelOutput";+ readonly output: unknown;+ readonly toolCallId?: string;+ readonly toolName: string;+}): JsonValue {+ const candidate = input.output === undefined ? null : input.output;++ return withToolOutputSerializationError(input, () => {+ parseJsonValue(candidate);+ return candidate as JsonValue;+ });+}++function normalizeToolModelOutput(input: {+ readonly output: unknown;+ readonly toolCallId?: string;+ readonly toolName: string;+}): ToolModelOutputValue {+ return withToolOutputSerializationError(+ {+ boundary: "toModelOutput",+ toolCallId: input.toolCallId,+ toolName: input.toolName,+ },+ () => {+ if (input.output === null || typeof input.output !== "object") {+ throw new TypeError("Expected a tool model output object.");+ }++ const output = input.output as { readonly type?: unknown; readonly value?: unknown };++ if (output.type === "text") {+ if (typeof output.value !== "string") {+ throw new TypeError('Expected text model output to include a string "value".');+ }++ return { type: "text", value: output.value };+ }++ if (output.type === "json") {+ return {+ type: "json",+ value: normalizeToolJsonOutput({+ boundary: "toModelOutput",+ output: output.value,+ toolCallId: input.toolCallId,+ toolName: input.toolName,+ }) as JSONValue,+ };+ }++ throw new TypeError('Expected tool model output type to be "text" or "json".');+ },+ );+}+/*** Builds the AI SDK ToolSet for one harness step.*