The Open Framework for Building Agents
fix(eve): pass error messages when tool call input is invalid back to model instead of throwing so that it can try again (#576)
Felix Arntzbd287b1724e5
diff --git a/.changeset/swift-clouds-juggle.md b/.changeset/swift-clouds-juggle.mdnew file mode 100644index 000000000..8a40c374c--- /dev/null+++ b/.changeset/swift-clouds-juggle.md@@ -0,0 +1,5 @@+---+"eve": patch+---++fix(eve): pass error messages when tool call input is invalid back to model instead of throwing so that it can try againdiff --git a/packages/eve/src/harness/emission.test.ts b/packages/eve/src/harness/emission.test.tsindex a8afe5440..602c41573 100644--- a/packages/eve/src/harness/emission.test.ts+++ b/packages/eve/src/harness/emission.test.ts@@ -493,6 +493,68 @@ describe("emitStreamContent action requests", () => {]);expect(providerResult.trailingInlineToolResultParts).toEqual([]);});++ it("turns non-object tool call input into a failed tool result for the model", async () => {+ const tools = new Map<string, HarnessToolDefinition>([+ [+ "web_search",+ {+ description: "Search the web.",+ execute: async () => ({ results: [] }),+ inputSchema: jsonSchema({ type: "object" }),+ name: "web_search",+ },+ ],+ ]);+ const emit = createEmitStub();+ const message =+ 'Failed to parse tool-call arguments for "web_search" (call-bad): Expected a JSON-serializable object.';++ const result = await emitStreamContent(+ emit,+ EMISSION_STATE,+ streamOf([+ {+ input: "not an object",+ toolCallId: "call-bad",+ toolName: "web_search",+ type: "tool-call",+ },+ { finishReason: "tool-calls", type: "finish-step" },+ ] as TextStreamPart<ToolSet>[]),+ {+ excludedActionToolNames: new Set(),+ tools,+ },+ );++ const events = vi.mocked(emit).mock.calls.map(([event]) => event);+ expect(events).toEqual([+ expect.objectContaining({+ data: expect.objectContaining({+ error: { code: "ACTION_RESULT_FAILED", message },+ result: {+ callId: "call-bad",+ isError: true,+ kind: "tool-result",+ output: message,+ toolName: "web_search",+ },+ status: "failed",+ }),+ type: "action.result",+ }),+ ]);+ expect([...result.invalidInputToolCallIds]).toEqual(["call-bad"]);+ expect(result.trailingInlineToolResultParts).toEqual([+ {+ output: { type: "error-text", value: message },+ toolCallId: "call-bad",+ toolName: "web_search",+ type: "tool-result",+ },+ ]);+ });});describe("emitStreamContent error-part handling", () => {diff --git a/packages/eve/src/harness/emission.ts b/packages/eve/src/harness/emission.tsindex fa9b35f29..69593e05a 100644--- a/packages/eve/src/harness/emission.ts+++ b/packages/eve/src/harness/emission.ts@@ -43,6 +43,7 @@ import {createRuntimeActionRequestFromToolCall,resolveToolCallInputObject,} from "#harness/runtime-actions.js";+import { createInvalidToolCallInputError } from "#harness/tool-call-input-errors.js";import type {RuntimeActionRequest,RuntimeToolResultActionResult,@@ -332,6 +333,7 @@ export function normalizeAssistantStepFinishReason(interface EmittedStreamContent {readonly emittedActionCallIds: ReadonlySet<string>;readonly handledInlineToolResultCallIds: ReadonlySet<string>;+ readonly invalidInputToolCallIds: ReadonlySet<string>;readonly inlineAuthorizationResults: readonly TypedToolResult<ToolSet>[];readonly inlineToolResultParts: readonly InlineToolResultPart[];readonly trailingInlineToolResultParts: readonly InlineToolResultPart[];@@ -366,6 +368,7 @@ export async function emitStreamContent(const providerToolCallIdsSeen = new Set<string>();const providerActionBatch = createProviderStreamActionBatch({ emitFn, state });const handledInlineToolResultCallIds = new Set<string>();+ const invalidInputToolCallIds = new Set<string>();const inlineAuthorizationResults: TypedToolResult<ToolSet>[] = [];const inlineToolResultParts: InlineToolResultPart[] = [];const trailingInlineToolResultParts: InlineToolResultPart[] = [];@@ -467,10 +470,15 @@ export async function emitStreamContent(}),);} catch (error) {- // A malformed tool call can arrive before the SDK marks its final call- // invalid. Let the SDK's recovery path handle it instead of failing the- // whole step while projecting UI events.if (error instanceof TypeError) {+ const toolError = createInvalidToolCallInputError({ error, toolCall });+ invalidInputToolCallIds.add(toolCall.toolCallId);+ if (currentMessage.trim().length > 0) {+ await flushCurrentMessage();+ }+ await emitActionResult(createRuntimeToolResultFromToolError(toolError));+ handledInlineToolResultCallIds.add(toolCall.toolCallId);+ trailingInlineToolResultParts.push(createToolResultMessagePartFromToolError(toolError));return;}throw error;@@ -666,6 +674,7 @@ export async function emitStreamContent(return {emittedActionCallIds,handledInlineToolResultCallIds,+ invalidInputToolCallIds,inlineAuthorizationResults,inlineToolResultParts,trailingInlineToolResultParts,diff --git a/packages/eve/src/harness/input-extraction.test.ts b/packages/eve/src/harness/input-extraction.test.tsindex 21d092840..c818179de 100644--- a/packages/eve/src/harness/input-extraction.test.ts+++ b/packages/eve/src/harness/input-extraction.test.ts@@ -198,6 +198,27 @@ describe("extractToolApprovalInputRequests", () => {expect(result).toEqual([]);});+ it("skips approval requests for excluded tool calls before parsing input", () => {+ const result = extractToolApprovalInputRequests({+ content: [+ {+ input: [],+ toolCallId: "call-1",+ toolName: "bash",+ type: "tool-call",+ } as never,+ {+ approvalId: "approval-1",+ toolCallId: "call-1",+ type: "tool-approval-request",+ } as never,+ ],+ excludedCallIds: new Set(["call-1"]),+ });++ expect(result).toEqual([]);+ });+it("skips non-approval content parts", () => {const result = extractToolApprovalInputRequests({content: [diff --git a/packages/eve/src/harness/input-extraction.ts b/packages/eve/src/harness/input-extraction.tsindex a9dcf1a42..53c8d251b 100644--- a/packages/eve/src/harness/input-extraction.ts+++ b/packages/eve/src/harness/input-extraction.ts@@ -64,6 +64,7 @@ export function extractQuestionInputRequests(input: {*/export function extractToolApprovalInputRequests(input: {readonly content: readonly ContentPart<ToolSet>[];+ readonly excludedCallIds?: ReadonlySet<string>;}): InputRequest[] {const requests: InputRequest[] = [];const toolCallsById = new Map<string, TypedToolCall<ToolSet>>();@@ -97,6 +98,10 @@ export function extractToolApprovalInputRequests(input: {continue;}+ if (input.excludedCallIds?.has(toolCall.toolCallId)) {+ continue;+ }+const action = createRuntimeToolCallActionFromToolCall({toolCall,});diff --git a/packages/eve/src/harness/input-requests.test.ts b/packages/eve/src/harness/input-requests.test.tsindex db236320b..08485bd9a 100644--- a/packages/eve/src/harness/input-requests.test.ts+++ b/packages/eve/src/harness/input-requests.test.ts@@ -99,6 +99,21 @@ describe("createRuntimeToolCallActionFromToolCall", () => {path: "/workspace/foo.txt",});});++ it("includes the tool name when tool call input is not a JSON object", () => {+ expect(() =>+ createRuntimeToolCallActionFromToolCall({+ toolCall: {+ toolCallId: "call-123",+ toolName: "bash",+ input: [],+ type: "tool-call",+ } as never,+ }),+ ).toThrow(+ 'Failed to parse tool-call arguments for "bash" (call-123): Expected a JSON-serializable object.',+ );+ });});describe("resolvePendingInput", () => {diff --git a/packages/eve/src/harness/input-requests.ts b/packages/eve/src/harness/input-requests.tsindex 9d9f46bc7..631b372de 100644--- a/packages/eve/src/harness/input-requests.ts+++ b/packages/eve/src/harness/input-requests.ts@@ -6,7 +6,7 @@ import type {} from "#runtime/actions/types.js";import type { InputRequest, InputResponse } from "#runtime/input/types.js";import { resolveTextToResponses } from "#channel/resolve-text.js";-import { parseJsonObject } from "#shared/json.js";+import { parseJsonObject, type JsonObject } from "#shared/json.js";import { coalesceTurnInputs } from "#harness/messages.js";import {isSessionLimitContinuationRequest,@@ -648,16 +648,30 @@ export function createRuntimeToolCallActionFromToolCall(input: {}): RuntimeToolCallActionRequest {return {callId: input.toolCall.toolCallId,- input: resolveToolCallInputObject(input.toolCall.input),+ input: resolveToolCallInputObject(input.toolCall.input, {+ callId: input.toolCall.toolCallId,+ toolName: input.toolCall.toolName,+ }),kind: "tool-call",toolName: input.toolCall.toolName,};}-function resolveToolCallInputObject(value: unknown) {+function resolveToolCallInputObject(+ value: unknown,+ context: { readonly callId: string; readonly toolName: string },+): JsonObject {if (value === undefined || value === null) {return {};}- return parseJsonObject(value);+ try {+ return parseJsonObject(value);+ } catch (error) {+ const detail = error instanceof Error ? error.message : String(error);+ throw new TypeError(+ `Failed to parse tool-call arguments for "${context.toolName}" (${context.callId}): ${detail}`,+ { cause: error },+ );+ }}diff --git a/packages/eve/src/harness/step-hooks.ts b/packages/eve/src/harness/step-hooks.tsindex f751e454b..0f7c9da98 100644--- a/packages/eve/src/harness/step-hooks.ts+++ b/packages/eve/src/harness/step-hooks.ts@@ -57,7 +57,9 @@ export type HarnessStepResult = Pick<| "toolCalls"| "toolResults"| "usage"->;+> & {+ readonly invalidInputToolCallIds?: ReadonlySet<string>;+};// ---------------------------------------------------------------------------// Hook builder input / output@@ -209,6 +211,7 @@ export async function emitStepActions(step: HarnessStepResult,options: {readonly emittedActionCallIds?: ReadonlySet<string>;+ readonly excludedActionCallIds?: ReadonlySet<string>;readonly excludedActionToolNames: ReadonlySet<string>;readonly handledInlineToolResultCallIds?: ReadonlySet<string>;readonly tools: ToolLoopHarnessConfig["tools"];@@ -220,9 +223,11 @@ export async function emitStepActions(.map((toolCall) => toolCall.toolCallId),);const excludedCallIds = new Set<string>([+ ...(options.excludedActionCallIds ?? []),...providerExecutedCallIds,...extractToolApprovalInputRequests({content: (step.content ?? []) as ContentPart<ToolSet>[],+ excludedCallIds: options.excludedActionCallIds,}).map((request) => request.action.callId),...(step.toolCalls as TypedToolCall<ToolSet>[]).filter(isInvalidToolCall)diff --git a/packages/eve/src/harness/tool-call-input-errors.ts b/packages/eve/src/harness/tool-call-input-errors.tsnew file mode 100644index 000000000..33e6a432f--- /dev/null+++ b/packages/eve/src/harness/tool-call-input-errors.ts@@ -0,0 +1,66 @@+import type { ToolSet, TypedToolCall, TypedToolError } from "ai";++import { resolveToolCallInputObject } from "#harness/runtime-actions.js";++export function createInvalidToolCallInputError(input: {+ readonly error: unknown;+ readonly toolCall: TypedToolCall<ToolSet>;+}): TypedToolError<ToolSet> {+ const { toolCall } = input;++ const toolError: {+ dynamic?: true;+ error: unknown;+ input: unknown;+ providerExecuted?: true;+ providerMetadata?: TypedToolCall<ToolSet>["providerMetadata"];+ toolCallId: string;+ toolMetadata?: TypedToolCall<ToolSet>["toolMetadata"];+ toolName: string;+ type: "tool-error";+ } = {+ type: "tool-error",+ toolCallId: toolCall.toolCallId,+ toolName: toolCall.toolName,+ input: toolCall.input,+ error: input.error,+ };++ if (toolCall.dynamic === true) {+ toolError.dynamic = true;+ }+ if (toolCall.providerExecuted === true) {+ toolError.providerExecuted = true;+ }+ if (toolCall.providerMetadata !== undefined) {+ toolError.providerMetadata = toolCall.providerMetadata;+ }+ if (toolCall.toolMetadata !== undefined) {+ toolError.toolMetadata = toolCall.toolMetadata;+ }++ return toolError as TypedToolError<ToolSet>;+}++export function getInvalidToolCallInputError(input: {+ readonly toolCall: TypedToolCall<ToolSet>;+}): TypedToolError<ToolSet> | undefined {+ const { toolCall } = input;++ if (toolCall.invalid === true) {+ return undefined;+ }++ try {+ resolveToolCallInputObject(toolCall.input, {+ callId: toolCall.toolCallId,+ toolName: toolCall.toolName,+ });+ return undefined;+ } catch (error) {+ if (error instanceof TypeError) {+ return createInvalidToolCallInputError({ error, toolCall });+ }+ throw error;+ }+}diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex 696474c47..5e2265a6c 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -287,7 +287,7 @@ async function* createMockFullStream(yield { id: "text-1", text: part.text, type: "text-delta" };break;case "tool-call":- yield part as Record<string, unknown>;+ yield toolCallsById.get(String(part.toolCallId)) ?? (part as Record<string, unknown>);break;case "tool-approval-request": {const toolCall = toolCallsById.get(String(part.toolCallId));@@ -2283,6 +2283,101 @@ describe("createToolLoopHarness", () => {});});+ it("feeds non-object tool call input back to the model as a failed tool result", async () => {+ const invalidInput = "not an object";+ const errorMessage =+ 'Failed to parse tool-call arguments for "add" (call-bad): Expected a JSON-serializable object.';+ setupMockAgent({+ finishReason: "tool-calls",+ response: {+ messages: [+ {+ content: [+ {+ input: invalidInput,+ toolCallId: "call-bad",+ toolName: "add",+ type: "tool-call",+ },+ ],+ role: "assistant",+ },+ ],+ },+ text: "",+ toolCalls: [+ {+ input: invalidInput,+ toolCallId: "call-bad",+ toolName: "add",+ type: "tool-call",+ },+ ],+ toolResults: [],+ });++ const { emit } = createEventCollector();+ const runStep = createToolLoopHarness(createTestConfig("conversation", emit));+ const firstStep = await runStep(createTestSession(), { message: "Add these" });++ expect(firstStep.next).toBe(runStep);+ expect(firstStep.session.history).toEqual([+ { content: "Add these", role: "user" },+ {+ content: [+ {+ input: invalidInput,+ toolCallId: "call-bad",+ toolName: "add",+ type: "tool-call",+ },+ ],+ role: "assistant",+ },+ {+ content: [+ {+ output: { type: "error-text", value: errorMessage },+ toolCallId: "call-bad",+ toolName: "add",+ type: "tool-result",+ },+ ],+ role: "tool",+ },+ ]);++ setupMockAgent({+ finishReason: "stop",+ response: {+ messages: [{ content: "I'll use object arguments next time.", role: "assistant" }],+ },+ text: "I'll use object arguments next time.",+ toolCalls: [],+ toolResults: [],+ });+ await runStep(firstStep.session);++ const secondInstance = vi.mocked(ToolLoopAgent).mock.results[1]?.value as+ | { stream: ReturnType<typeof vi.fn> }+ | undefined;+ const secondCall = secondInstance?.stream.mock.calls[0]?.[0] as+ | { messages: ModelMessage[] }+ | undefined;+ expect(secondCall?.messages).toEqual(firstStep.session.history);+ expect(secondCall?.messages.at(-1)).toEqual({+ content: [+ {+ output: { type: "error-text", value: errorMessage },+ toolCallId: "call-bad",+ toolName: "add",+ type: "tool-result",+ },+ ],+ role: "tool",+ });+ });+it("only emits valid tool calls when a step mixes valid and invalid tool calls", async () => {setupMockAgent({finishReason: "tool-calls",diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.tsindex ed283865a..4f052297a 100644--- a/packages/eve/src/harness/tool-loop.ts+++ b/packages/eve/src/harness/tool-loop.ts@@ -14,6 +14,7 @@ import {ToolLoopAgent,type ToolSet,type TypedToolCall,+ type TypedToolError,type TypedToolResult,} from "ai";import { isScheduleAppAuth } from "#channel/schedule-auth.js";@@ -93,6 +94,7 @@ import {extractQuestionInputRequests,extractToolApprovalInputRequests,} from "#harness/input-extraction.js";+import { createToolResultMessagePartFromToolError } from "#harness/action-result-helpers.js";import { buildTelemetryRuntimeContext } from "#harness/instrumentation-runtime-context.js";import {consumeDeferredStepInput,@@ -144,6 +146,7 @@ import {resolvePendingRuntimeActions,setPendingRuntimeActionBatch,} from "#harness/runtime-actions.js";+import { getInvalidToolCallInputError } from "#harness/tool-call-input-errors.js";import {buildStepHooks,emitStepActions,@@ -774,6 +777,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {const {emittedActionCallIds,handledInlineToolResultCallIds,+ invalidInputToolCallIds,inlineAuthorizationResults,inlineToolResultParts,trailingInlineToolResultParts,@@ -793,6 +797,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {}await emitStepActions(emit, emissionState, stepResult, {emittedActionCallIds,+ excludedActionCallIds: invalidInputToolCallIds,excludedActionToolNames,handledInlineToolResultCallIds,tools: advertisedHarnessTools,@@ -837,6 +842,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {text: stepResult.text,toolCalls: stepResult.toolCalls,toolResults: [...toolResultsByCallId.values()],+ invalidInputToolCallIds,usage: stepResult.usage,};}@@ -1357,6 +1363,25 @@ function insertInlineToolResultMessages(input: {] satisfies StepResponseMessage[];}+function getInvalidToolCallInputErrors(input: {+ readonly toolCalls: readonly TypedToolCall<ToolSet>[];+}): readonly TypedToolError<ToolSet>[] {+ const errors: TypedToolError<ToolSet>[] = [];++ for (const toolCall of input.toolCalls) {+ if (toolCall.toolName === FINAL_OUTPUT_TOOL_NAME) {+ continue;+ }++ const toolError = getInvalidToolCallInputError({ toolCall });+ if (toolError !== undefined) {+ errors.push(toolError);+ }+ }++ return errors;+}+function extractToolResultCallIds(messages: readonly StepResponseMessage[]): ReadonlySet<string> {const callIds = new Set<string>();@@ -1572,7 +1597,22 @@ async function handleStepResult(input: {result.finishReason !== "tool-calls" &&result.toolCalls.length === 0 &&hasEmptyDeliverySentinel(resolvedStepOutput);- const rawResponseMessages = emptyDelivery ? [] : result.response.messages;+ const invalidInputToolErrors = getInvalidToolCallInputErrors({+ toolCalls: result.toolCalls as TypedToolCall<ToolSet>[],+ });+ const invalidInputToolCallIds = new Set([+ ...(result.invalidInputToolCallIds ?? []),+ ...invalidInputToolErrors.map((toolError) => toolError.toolCallId),+ ]);+ const rawResponseMessages = emptyDelivery+ ? []+ : insertInlineToolResultMessages({+ append: invalidInputToolErrors.map((toolError) =>+ createToolResultMessagePartFromToolError(toolError),+ ),+ prepend: [],+ responseMessages: result.response.messages,+ });const stepOutput = emptyDelivery ? null : resolvedStepOutput;const providerExecutedOutcomeIds = new Set<string>();@@ -1617,11 +1657,14 @@ async function handleStepResult(input: {}}- const approvalRequests = extractToolApprovalInputRequests({ content: result.content ?? [] });+ const approvalRequests = extractToolApprovalInputRequests({+ content: result.content ?? [],+ excludedCallIds: invalidInputToolCallIds,+ });const approvalRequestCallIds = new Set(approvalRequests.map((request) => request.action.callId));const questionRequests = extractQuestionInputRequests({toolCalls: result.toolCalls,- excludedCallIds: approvalRequestCallIds,+ excludedCallIds: new Set([...invalidInputToolCallIds, ...approvalRequestCallIds]),});const inputRequests: InputRequest[] = [...approvalRequests, ...questionRequests];const advertisedRuntimeActionTools = getAdvertisedTools({@@ -1630,6 +1673,7 @@ async function handleStepResult(input: {});const pendingRuntimeActions = ((result.toolCalls ?? []) as TypedToolCall<ToolSet>[]).filter((toolCall) => !isInvalidToolCall(toolCall))+ .filter((toolCall) => !invalidInputToolCallIds.has(toolCall.toolCallId)).filter((toolCall) => config.tools.get(toolCall.toolName)?.runtimeAction !== undefined).filter((toolCall) => {if (advertisedRuntimeActionTools.get(toolCall.toolName)?.runtimeAction !== undefined) {