Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): pass error messages when tool call input is invalid back to model instead of throwing so that it can try again (#576)

Felix Arntzbd287b1724e5
diff --git a/.changeset/swift-clouds-juggle.md b/.changeset/swift-clouds-juggle.md
new file mode 100644
index 000000000..8a40c374c
--- /dev/null
+++ b/.changeset/swift-clouds-juggle.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+fix(eve): pass error messages when tool call input is invalid back to model instead of throwing so that it can try again
diff --git a/packages/eve/src/harness/emission.test.ts b/packages/eve/src/harness/emission.test.ts
index a8afe5440..602c41573 100644
--- a/packages/eve/src/harness/emission.test.ts
+++ b/packages/eve/src/harness/emission.test.ts
@@ -493,6 +493,68 @@ describe("emitStreamContent action requests", () => {
]);
expect(providerResult.trailingInlineToolResultParts).toEqual([]);
});
+
+ it("turns non-object tool call input into a failed tool result for the model", async () => {
+ const tools = new Map<string, HarnessToolDefinition>([
+ [
+ "web_search",
+ {
+ description: "Search the web.",
+ execute: async () => ({ results: [] }),
+ inputSchema: jsonSchema({ type: "object" }),
+ name: "web_search",
+ },
+ ],
+ ]);
+ const emit = createEmitStub();
+ const message =
+ 'Failed to parse tool-call arguments for "web_search" (call-bad): Expected a JSON-serializable object.';
+
+ const result = await emitStreamContent(
+ emit,
+ EMISSION_STATE,
+ streamOf([
+ {
+ input: "not an object",
+ toolCallId: "call-bad",
+ toolName: "web_search",
+ type: "tool-call",
+ },
+ { finishReason: "tool-calls", type: "finish-step" },
+ ] as TextStreamPart<ToolSet>[]),
+ {
+ excludedActionToolNames: new Set(),
+ tools,
+ },
+ );
+
+ const events = vi.mocked(emit).mock.calls.map(([event]) => event);
+ expect(events).toEqual([
+ expect.objectContaining({
+ data: expect.objectContaining({
+ error: { code: "ACTION_RESULT_FAILED", message },
+ result: {
+ callId: "call-bad",
+ isError: true,
+ kind: "tool-result",
+ output: message,
+ toolName: "web_search",
+ },
+ status: "failed",
+ }),
+ type: "action.result",
+ }),
+ ]);
+ expect([...result.invalidInputToolCallIds]).toEqual(["call-bad"]);
+ expect(result.trailingInlineToolResultParts).toEqual([
+ {
+ output: { type: "error-text", value: message },
+ toolCallId: "call-bad",
+ toolName: "web_search",
+ type: "tool-result",
+ },
+ ]);
+ });
});
describe("emitStreamContent error-part handling", () => {
diff --git a/packages/eve/src/harness/emission.ts b/packages/eve/src/harness/emission.ts
index fa9b35f29..69593e05a 100644
--- a/packages/eve/src/harness/emission.ts
+++ b/packages/eve/src/harness/emission.ts
@@ -43,6 +43,7 @@ import {
createRuntimeActionRequestFromToolCall,
resolveToolCallInputObject,
} from "#harness/runtime-actions.js";
+import { createInvalidToolCallInputError } from "#harness/tool-call-input-errors.js";
import type {
RuntimeActionRequest,
RuntimeToolResultActionResult,
@@ -332,6 +333,7 @@ export function normalizeAssistantStepFinishReason(
interface EmittedStreamContent {
readonly emittedActionCallIds: ReadonlySet<string>;
readonly handledInlineToolResultCallIds: ReadonlySet<string>;
+ readonly invalidInputToolCallIds: ReadonlySet<string>;
readonly inlineAuthorizationResults: readonly TypedToolResult<ToolSet>[];
readonly inlineToolResultParts: readonly InlineToolResultPart[];
readonly trailingInlineToolResultParts: readonly InlineToolResultPart[];
@@ -366,6 +368,7 @@ export async function emitStreamContent(
const providerToolCallIdsSeen = new Set<string>();
const providerActionBatch = createProviderStreamActionBatch({ emitFn, state });
const handledInlineToolResultCallIds = new Set<string>();
+ const invalidInputToolCallIds = new Set<string>();
const inlineAuthorizationResults: TypedToolResult<ToolSet>[] = [];
const inlineToolResultParts: InlineToolResultPart[] = [];
const trailingInlineToolResultParts: InlineToolResultPart[] = [];
@@ -467,10 +470,15 @@ export async function emitStreamContent(
}),
);
} catch (error) {
- // A malformed tool call can arrive before the SDK marks its final call
- // invalid. Let the SDK's recovery path handle it instead of failing the
- // whole step while projecting UI events.
if (error instanceof TypeError) {
+ const toolError = createInvalidToolCallInputError({ error, toolCall });
+ invalidInputToolCallIds.add(toolCall.toolCallId);
+ if (currentMessage.trim().length > 0) {
+ await flushCurrentMessage();
+ }
+ await emitActionResult(createRuntimeToolResultFromToolError(toolError));
+ handledInlineToolResultCallIds.add(toolCall.toolCallId);
+ trailingInlineToolResultParts.push(createToolResultMessagePartFromToolError(toolError));
return;
}
throw error;
@@ -666,6 +674,7 @@ export async function emitStreamContent(
return {
emittedActionCallIds,
handledInlineToolResultCallIds,
+ invalidInputToolCallIds,
inlineAuthorizationResults,
inlineToolResultParts,
trailingInlineToolResultParts,
diff --git a/packages/eve/src/harness/input-extraction.test.ts b/packages/eve/src/harness/input-extraction.test.ts
index 21d092840..c818179de 100644
--- a/packages/eve/src/harness/input-extraction.test.ts
+++ b/packages/eve/src/harness/input-extraction.test.ts
@@ -198,6 +198,27 @@ describe("extractToolApprovalInputRequests", () => {
expect(result).toEqual([]);
});
+ it("skips approval requests for excluded tool calls before parsing input", () => {
+ const result = extractToolApprovalInputRequests({
+ content: [
+ {
+ input: [],
+ toolCallId: "call-1",
+ toolName: "bash",
+ type: "tool-call",
+ } as never,
+ {
+ approvalId: "approval-1",
+ toolCallId: "call-1",
+ type: "tool-approval-request",
+ } as never,
+ ],
+ excludedCallIds: new Set(["call-1"]),
+ });
+
+ expect(result).toEqual([]);
+ });
+
it("skips non-approval content parts", () => {
const result = extractToolApprovalInputRequests({
content: [
diff --git a/packages/eve/src/harness/input-extraction.ts b/packages/eve/src/harness/input-extraction.ts
index a9dcf1a42..53c8d251b 100644
--- a/packages/eve/src/harness/input-extraction.ts
+++ b/packages/eve/src/harness/input-extraction.ts
@@ -64,6 +64,7 @@ export function extractQuestionInputRequests(input: {
*/
export function extractToolApprovalInputRequests(input: {
readonly content: readonly ContentPart<ToolSet>[];
+ readonly excludedCallIds?: ReadonlySet<string>;
}): InputRequest[] {
const requests: InputRequest[] = [];
const toolCallsById = new Map<string, TypedToolCall<ToolSet>>();
@@ -97,6 +98,10 @@ export function extractToolApprovalInputRequests(input: {
continue;
}
+ if (input.excludedCallIds?.has(toolCall.toolCallId)) {
+ continue;
+ }
+
const action = createRuntimeToolCallActionFromToolCall({
toolCall,
});
diff --git a/packages/eve/src/harness/input-requests.test.ts b/packages/eve/src/harness/input-requests.test.ts
index db236320b..08485bd9a 100644
--- a/packages/eve/src/harness/input-requests.test.ts
+++ b/packages/eve/src/harness/input-requests.test.ts
@@ -99,6 +99,21 @@ describe("createRuntimeToolCallActionFromToolCall", () => {
path: "/workspace/foo.txt",
});
});
+
+ it("includes the tool name when tool call input is not a JSON object", () => {
+ expect(() =>
+ createRuntimeToolCallActionFromToolCall({
+ toolCall: {
+ toolCallId: "call-123",
+ toolName: "bash",
+ input: [],
+ type: "tool-call",
+ } as never,
+ }),
+ ).toThrow(
+ 'Failed to parse tool-call arguments for "bash" (call-123): Expected a JSON-serializable object.',
+ );
+ });
});
describe("resolvePendingInput", () => {
diff --git a/packages/eve/src/harness/input-requests.ts b/packages/eve/src/harness/input-requests.ts
index 9d9f46bc7..631b372de 100644
--- a/packages/eve/src/harness/input-requests.ts
+++ b/packages/eve/src/harness/input-requests.ts
@@ -6,7 +6,7 @@ import type {
} from "#runtime/actions/types.js";
import type { InputRequest, InputResponse } from "#runtime/input/types.js";
import { resolveTextToResponses } from "#channel/resolve-text.js";
-import { parseJsonObject } from "#shared/json.js";
+import { parseJsonObject, type JsonObject } from "#shared/json.js";
import { coalesceTurnInputs } from "#harness/messages.js";
import {
isSessionLimitContinuationRequest,
@@ -648,16 +648,30 @@ export function createRuntimeToolCallActionFromToolCall(input: {
}): RuntimeToolCallActionRequest {
return {
callId: input.toolCall.toolCallId,
- input: resolveToolCallInputObject(input.toolCall.input),
+ input: resolveToolCallInputObject(input.toolCall.input, {
+ callId: input.toolCall.toolCallId,
+ toolName: input.toolCall.toolName,
+ }),
kind: "tool-call",
toolName: input.toolCall.toolName,
};
}
-function resolveToolCallInputObject(value: unknown) {
+function resolveToolCallInputObject(
+ value: unknown,
+ context: { readonly callId: string; readonly toolName: string },
+): JsonObject {
if (value === undefined || value === null) {
return {};
}
- return parseJsonObject(value);
+ try {
+ return parseJsonObject(value);
+ } catch (error) {
+ const detail = error instanceof Error ? error.message : String(error);
+ throw new TypeError(
+ `Failed to parse tool-call arguments for "${context.toolName}" (${context.callId}): ${detail}`,
+ { cause: error },
+ );
+ }
}
diff --git a/packages/eve/src/harness/step-hooks.ts b/packages/eve/src/harness/step-hooks.ts
index f751e454b..0f7c9da98 100644
--- a/packages/eve/src/harness/step-hooks.ts
+++ b/packages/eve/src/harness/step-hooks.ts
@@ -57,7 +57,9 @@ export type HarnessStepResult = Pick<
| "toolCalls"
| "toolResults"
| "usage"
->;
+> & {
+ readonly invalidInputToolCallIds?: ReadonlySet<string>;
+};
// ---------------------------------------------------------------------------
// Hook builder input / output
@@ -209,6 +211,7 @@ export async function emitStepActions(
step: HarnessStepResult,
options: {
readonly emittedActionCallIds?: ReadonlySet<string>;
+ readonly excludedActionCallIds?: ReadonlySet<string>;
readonly excludedActionToolNames: ReadonlySet<string>;
readonly handledInlineToolResultCallIds?: ReadonlySet<string>;
readonly tools: ToolLoopHarnessConfig["tools"];
@@ -220,9 +223,11 @@ export async function emitStepActions(
.map((toolCall) => toolCall.toolCallId),
);
const excludedCallIds = new Set<string>([
+ ...(options.excludedActionCallIds ?? []),
...providerExecutedCallIds,
...extractToolApprovalInputRequests({
content: (step.content ?? []) as ContentPart<ToolSet>[],
+ excludedCallIds: options.excludedActionCallIds,
}).map((request) => request.action.callId),
...(step.toolCalls as TypedToolCall<ToolSet>[])
.filter(isInvalidToolCall)
diff --git a/packages/eve/src/harness/tool-call-input-errors.ts b/packages/eve/src/harness/tool-call-input-errors.ts
new file mode 100644
index 000000000..33e6a432f
--- /dev/null
+++ b/packages/eve/src/harness/tool-call-input-errors.ts
@@ -0,0 +1,66 @@
+import type { ToolSet, TypedToolCall, TypedToolError } from "ai";
+
+import { resolveToolCallInputObject } from "#harness/runtime-actions.js";
+
+export function createInvalidToolCallInputError(input: {
+ readonly error: unknown;
+ readonly toolCall: TypedToolCall<ToolSet>;
+}): TypedToolError<ToolSet> {
+ const { toolCall } = input;
+
+ const toolError: {
+ dynamic?: true;
+ error: unknown;
+ input: unknown;
+ providerExecuted?: true;
+ providerMetadata?: TypedToolCall<ToolSet>["providerMetadata"];
+ toolCallId: string;
+ toolMetadata?: TypedToolCall<ToolSet>["toolMetadata"];
+ toolName: string;
+ type: "tool-error";
+ } = {
+ type: "tool-error",
+ toolCallId: toolCall.toolCallId,
+ toolName: toolCall.toolName,
+ input: toolCall.input,
+ error: input.error,
+ };
+
+ if (toolCall.dynamic === true) {
+ toolError.dynamic = true;
+ }
+ if (toolCall.providerExecuted === true) {
+ toolError.providerExecuted = true;
+ }
+ if (toolCall.providerMetadata !== undefined) {
+ toolError.providerMetadata = toolCall.providerMetadata;
+ }
+ if (toolCall.toolMetadata !== undefined) {
+ toolError.toolMetadata = toolCall.toolMetadata;
+ }
+
+ return toolError as TypedToolError<ToolSet>;
+}
+
+export function getInvalidToolCallInputError(input: {
+ readonly toolCall: TypedToolCall<ToolSet>;
+}): TypedToolError<ToolSet> | undefined {
+ const { toolCall } = input;
+
+ if (toolCall.invalid === true) {
+ return undefined;
+ }
+
+ try {
+ resolveToolCallInputObject(toolCall.input, {
+ callId: toolCall.toolCallId,
+ toolName: toolCall.toolName,
+ });
+ return undefined;
+ } catch (error) {
+ if (error instanceof TypeError) {
+ return createInvalidToolCallInputError({ error, toolCall });
+ }
+ throw error;
+ }
+}
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index 696474c47..5e2265a6c 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -287,7 +287,7 @@ async function* createMockFullStream(
yield { id: "text-1", text: part.text, type: "text-delta" };
break;
case "tool-call":
- yield part as Record<string, unknown>;
+ yield toolCallsById.get(String(part.toolCallId)) ?? (part as Record<string, unknown>);
break;
case "tool-approval-request": {
const toolCall = toolCallsById.get(String(part.toolCallId));
@@ -2283,6 +2283,101 @@ describe("createToolLoopHarness", () => {
});
});
+ it("feeds non-object tool call input back to the model as a failed tool result", async () => {
+ const invalidInput = "not an object";
+ const errorMessage =
+ 'Failed to parse tool-call arguments for "add" (call-bad): Expected a JSON-serializable object.';
+ setupMockAgent({
+ finishReason: "tool-calls",
+ response: {
+ messages: [
+ {
+ content: [
+ {
+ input: invalidInput,
+ toolCallId: "call-bad",
+ toolName: "add",
+ type: "tool-call",
+ },
+ ],
+ role: "assistant",
+ },
+ ],
+ },
+ text: "",
+ toolCalls: [
+ {
+ input: invalidInput,
+ toolCallId: "call-bad",
+ toolName: "add",
+ type: "tool-call",
+ },
+ ],
+ toolResults: [],
+ });
+
+ const { emit } = createEventCollector();
+ const runStep = createToolLoopHarness(createTestConfig("conversation", emit));
+ const firstStep = await runStep(createTestSession(), { message: "Add these" });
+
+ expect(firstStep.next).toBe(runStep);
+ expect(firstStep.session.history).toEqual([
+ { content: "Add these", role: "user" },
+ {
+ content: [
+ {
+ input: invalidInput,
+ toolCallId: "call-bad",
+ toolName: "add",
+ type: "tool-call",
+ },
+ ],
+ role: "assistant",
+ },
+ {
+ content: [
+ {
+ output: { type: "error-text", value: errorMessage },
+ toolCallId: "call-bad",
+ toolName: "add",
+ type: "tool-result",
+ },
+ ],
+ role: "tool",
+ },
+ ]);
+
+ setupMockAgent({
+ finishReason: "stop",
+ response: {
+ messages: [{ content: "I'll use object arguments next time.", role: "assistant" }],
+ },
+ text: "I'll use object arguments next time.",
+ toolCalls: [],
+ toolResults: [],
+ });
+ await runStep(firstStep.session);
+
+ const secondInstance = vi.mocked(ToolLoopAgent).mock.results[1]?.value as
+ | { stream: ReturnType<typeof vi.fn> }
+ | undefined;
+ const secondCall = secondInstance?.stream.mock.calls[0]?.[0] as
+ | { messages: ModelMessage[] }
+ | undefined;
+ expect(secondCall?.messages).toEqual(firstStep.session.history);
+ expect(secondCall?.messages.at(-1)).toEqual({
+ content: [
+ {
+ output: { type: "error-text", value: errorMessage },
+ toolCallId: "call-bad",
+ toolName: "add",
+ type: "tool-result",
+ },
+ ],
+ role: "tool",
+ });
+ });
+
it("only emits valid tool calls when a step mixes valid and invalid tool calls", async () => {
setupMockAgent({
finishReason: "tool-calls",
diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.ts
index ed283865a..4f052297a 100644
--- a/packages/eve/src/harness/tool-loop.ts
+++ b/packages/eve/src/harness/tool-loop.ts
@@ -14,6 +14,7 @@ import {
ToolLoopAgent,
type ToolSet,
type TypedToolCall,
+ type TypedToolError,
type TypedToolResult,
} from "ai";
import { isScheduleAppAuth } from "#channel/schedule-auth.js";
@@ -93,6 +94,7 @@ import {
extractQuestionInputRequests,
extractToolApprovalInputRequests,
} from "#harness/input-extraction.js";
+import { createToolResultMessagePartFromToolError } from "#harness/action-result-helpers.js";
import { buildTelemetryRuntimeContext } from "#harness/instrumentation-runtime-context.js";
import {
consumeDeferredStepInput,
@@ -144,6 +146,7 @@ import {
resolvePendingRuntimeActions,
setPendingRuntimeActionBatch,
} from "#harness/runtime-actions.js";
+import { getInvalidToolCallInputError } from "#harness/tool-call-input-errors.js";
import {
buildStepHooks,
emitStepActions,
@@ -774,6 +777,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
const {
emittedActionCallIds,
handledInlineToolResultCallIds,
+ invalidInputToolCallIds,
inlineAuthorizationResults,
inlineToolResultParts,
trailingInlineToolResultParts,
@@ -793,6 +797,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
}
await emitStepActions(emit, emissionState, stepResult, {
emittedActionCallIds,
+ excludedActionCallIds: invalidInputToolCallIds,
excludedActionToolNames,
handledInlineToolResultCallIds,
tools: advertisedHarnessTools,
@@ -837,6 +842,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
text: stepResult.text,
toolCalls: stepResult.toolCalls,
toolResults: [...toolResultsByCallId.values()],
+ invalidInputToolCallIds,
usage: stepResult.usage,
};
}
@@ -1357,6 +1363,25 @@ function insertInlineToolResultMessages(input: {
] satisfies StepResponseMessage[];
}
+function getInvalidToolCallInputErrors(input: {
+ readonly toolCalls: readonly TypedToolCall<ToolSet>[];
+}): readonly TypedToolError<ToolSet>[] {
+ const errors: TypedToolError<ToolSet>[] = [];
+
+ for (const toolCall of input.toolCalls) {
+ if (toolCall.toolName === FINAL_OUTPUT_TOOL_NAME) {
+ continue;
+ }
+
+ const toolError = getInvalidToolCallInputError({ toolCall });
+ if (toolError !== undefined) {
+ errors.push(toolError);
+ }
+ }
+
+ return errors;
+}
+
function extractToolResultCallIds(messages: readonly StepResponseMessage[]): ReadonlySet<string> {
const callIds = new Set<string>();
@@ -1572,7 +1597,22 @@ async function handleStepResult(input: {
result.finishReason !== "tool-calls" &&
result.toolCalls.length === 0 &&
hasEmptyDeliverySentinel(resolvedStepOutput);
- const rawResponseMessages = emptyDelivery ? [] : result.response.messages;
+ const invalidInputToolErrors = getInvalidToolCallInputErrors({
+ toolCalls: result.toolCalls as TypedToolCall<ToolSet>[],
+ });
+ const invalidInputToolCallIds = new Set([
+ ...(result.invalidInputToolCallIds ?? []),
+ ...invalidInputToolErrors.map((toolError) => toolError.toolCallId),
+ ]);
+ const rawResponseMessages = emptyDelivery
+ ? []
+ : insertInlineToolResultMessages({
+ append: invalidInputToolErrors.map((toolError) =>
+ createToolResultMessagePartFromToolError(toolError),
+ ),
+ prepend: [],
+ responseMessages: result.response.messages,
+ });
const stepOutput = emptyDelivery ? null : resolvedStepOutput;
const providerExecutedOutcomeIds = new Set<string>();
@@ -1617,11 +1657,14 @@ async function handleStepResult(input: {
}
}
- const approvalRequests = extractToolApprovalInputRequests({ content: result.content ?? [] });
+ const approvalRequests = extractToolApprovalInputRequests({
+ content: result.content ?? [],
+ excludedCallIds: invalidInputToolCallIds,
+ });
const approvalRequestCallIds = new Set(approvalRequests.map((request) => request.action.callId));
const questionRequests = extractQuestionInputRequests({
toolCalls: result.toolCalls,
- excludedCallIds: approvalRequestCallIds,
+ excludedCallIds: new Set([...invalidInputToolCallIds, ...approvalRequestCallIds]),
});
const inputRequests: InputRequest[] = [...approvalRequests, ...questionRequests];
const advertisedRuntimeActionTools = getAdvertisedTools({
@@ -1630,6 +1673,7 @@ async function handleStepResult(input: {
});
const pendingRuntimeActions = ((result.toolCalls ?? []) as TypedToolCall<ToolSet>[])
.filter((toolCall) => !isInvalidToolCall(toolCall))
+ .filter((toolCall) => !invalidInputToolCallIds.has(toolCall.toolCallId))
.filter((toolCall) => config.tools.get(toolCall.toolName)?.runtimeAction !== undefined)
.filter((toolCall) => {
if (advertisedRuntimeActionTools.get(toolCall.toolName)?.runtimeAction !== undefined) {