The Open Framework for Building Agents
fix(eve): runtime reliability - preserve tool errors and enforce eval timeouts (#1431)
Andrew Barbaf3ecdcdaf7d3
diff --git a/.changeset/enforce-eval-timeouts.md b/.changeset/enforce-eval-timeouts.mdnew file mode 100644index 000000000..8333be9c9--- /dev/null+++ b/.changeset/enforce-eval-timeouts.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Eval config timeouts are now applied to every eval and enforced even when an eval task does not cooperate with its abort signal, preventing stalled eval runs.diff --git a/.changeset/preserve-malformed-tool-errors.md b/.changeset/preserve-malformed-tool-errors.mdnew file mode 100644index 000000000..6b07048c2--- /dev/null+++ b/.changeset/preserve-malformed-tool-errors.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Malformed raw tool arguments now preserve the original JSON syntax error and return it to the model as a failed tool result instead of reporting a misleading serialization error.diff --git a/packages/eve/src/evals/runner/execute-task.test.ts b/packages/eve/src/evals/runner/execute-task.test.tsindex 15b1dc3a9..3be1e69a4 100644--- a/packages/eve/src/evals/runner/execute-task.test.ts+++ b/packages/eve/src/evals/runner/execute-task.test.ts@@ -26,6 +26,23 @@ function createTestEval(test: (t: EveEvalContext) => unknown, id = "test-eval"):}describe("executeTask", () => {+ it("settles when an eval ignores its timeout signal", async () => {+ const outcome = await executeTask({+ client: new Client({ host: target.url }),+ target,+ evaluation: createTestEval(+ async () =>+ await new Promise<void>(() => {+ // Deliberately never settles.+ }),+ "timeout",+ ),+ timeoutMs: 1,+ });++ expect(outcome.error).toMatch(/timed out|timeout/i);+ });+it("exposes a sleep helper with a one-second default", async () => {vi.useFakeTimers();let settled = false;diff --git a/packages/eve/src/evals/runner/execute-task.ts b/packages/eve/src/evals/runner/execute-task.tsindex 6e154989d..e3ad73717 100644--- a/packages/eve/src/evals/runner/execute-task.ts+++ b/packages/eve/src/evals/runner/execute-task.ts@@ -71,7 +71,7 @@ export async function executeTask(options: ExecuteTaskOptions): Promise<ExecuteTlet error: string | undefined;let skipReason: string | undefined;try {- await evaluation.test(context);+ await runUntilAborted(evaluation.test(context), signal);} catch (err) {if (err instanceof EvalSkipped) {skipReason = err.reason;@@ -159,3 +159,21 @@ function sum<T>(entries: readonly T[], read: (entry: T) => number): number {function neverAbortSignal(): AbortSignal {return new AbortController().signal;}++async function runUntilAborted(task: void | Promise<void>, signal: AbortSignal): Promise<void> {+ signal.throwIfAborted();++ let onAbort: (() => void) | undefined;+ const aborted = new Promise<never>((_resolve, reject) => {+ onAbort = () => reject(signal.reason);+ signal.addEventListener("abort", onAbort, { once: true });+ });++ try {+ await Promise.race([task, aborted]);+ } finally {+ if (onAbort !== undefined) {+ signal.removeEventListener("abort", onAbort);+ }+ }+}diff --git a/packages/eve/src/evals/runner/run-evals.test.ts b/packages/eve/src/evals/runner/run-evals.test.tsindex 59b2e9592..b4c8bb7d5 100644--- a/packages/eve/src/evals/runner/run-evals.test.ts+++ b/packages/eve/src/evals/runner/run-evals.test.ts@@ -382,6 +382,56 @@ describe("runEvals", () => {expect(peak).toBe(2);});+ it("applies config timeouts while preserving eval and CLI overrides", async () => {+ mockArtifacts();+ mockedRunnerDependencies.executeTask.mockResolvedValue(createTaskResult("done"));++ await run({+ evaluations: [createEval("uses-config"), createEval("uses-eval", { timeoutMs: 2_000 })],+ config: {+ _tag: "EveEvalConfig",+ timeoutMs: 1_000,+ },+ target: localTarget,+ client: unusedClient,+ appRoot: "/tmp/app",+ reporters: [],+ maxConcurrency: 1,+ });++ expect(mockedRunnerDependencies.executeTask.mock.calls[0]?.[0]).toEqual(+ expect.objectContaining({+ evaluation: expect.objectContaining({ id: "uses-config", timeoutMs: 1_000 }),+ timeoutMs: 1_000,+ }),+ );+ expect(mockedRunnerDependencies.executeTask.mock.calls[1]?.[0]).toEqual(+ expect.objectContaining({+ evaluation: expect.objectContaining({ id: "uses-eval", timeoutMs: 2_000 }),+ timeoutMs: 2_000,+ }),+ );++ mockedRunnerDependencies.executeTask.mockClear();++ await run({+ evaluations: [createEval("uses-cli", { timeoutMs: 2_000 })],+ config: {+ _tag: "EveEvalConfig",+ timeoutMs: 1_000,+ },+ target: localTarget,+ client: unusedClient,+ appRoot: "/tmp/app",+ reporters: [],+ timeoutMs: 500,+ });++ expect(mockedRunnerDependencies.executeTask).toHaveBeenCalledWith(+ expect.objectContaining({ timeoutMs: 500 }),+ );+ });+it("drives config reporters across every eval and dedupes eval references", async () => {mockArtifacts();mockedRunnerDependencies.executeTask.mockResolvedValue(createTaskResult("done"));diff --git a/packages/eve/src/evals/runner/run-evals.ts b/packages/eve/src/evals/runner/run-evals.tsindex 522a32fee..c9787f22f 100644--- a/packages/eve/src/evals/runner/run-evals.ts+++ b/packages/eve/src/evals/runner/run-evals.ts@@ -185,17 +185,19 @@ function buildReporterBindings(return bindings;}-/**- * Fills an eval's judge model from the run config when the eval does not set- * its own. The judge model only ever drives `t.judge.*` assertions.- */+/** Fills per-eval defaults from the run config without replacing authored values. */function applyConfigDefaults(evaluation: EveEval, config: EveEvalConfig): EveEval {- if (evaluation.judge !== undefined || config.judge === undefined) {+ const judge = evaluation.judge ?? config.judge;+ const timeoutMs = evaluation.timeoutMs ?? config.timeoutMs;++ if (judge === evaluation.judge && timeoutMs === evaluation.timeoutMs) {return evaluation;}+return {...evaluation,- judge: config.judge,+ judge,+ timeoutMs,};}diff --git a/packages/eve/src/harness/emission.test.ts b/packages/eve/src/harness/emission.test.tsindex 4412593c0..ba6917204 100644--- a/packages/eve/src/harness/emission.test.ts+++ b/packages/eve/src/harness/emission.test.ts@@ -557,7 +557,7 @@ describe("emitStreamContent action requests", () => {EMISSION_STATE,streamOf([{- input: "not an object",+ input: '"not an object"',toolCallId: "call-bad",toolName: "web_search",type: "tool-call",@@ -600,8 +600,6 @@ describe("emitStreamContent action requests", () => {it("turns malformed provider-executed tool call input into a failed tool result", async () => {const emit = createEmitStub();- const message =- 'Failed to parse tool-call arguments for "web_search" (call-bad): Expected a JSON-serializable object.';const result = await emitStreamContent(emit,@@ -625,6 +623,13 @@ describe("emitStreamContent action requests", () => {);const events = vi.mocked(emit).mock.calls.map(([event]) => event);+ const actionResult = events.find((event) => event.type === "action.result");+ if (actionResult?.data.error === undefined) {+ throw new Error("Expected a failed action.result event.");+ }+ const { message } = actionResult.data.error;+ expect(message).toMatch(/Failed to parse tool-call arguments for "web_search" \(call-bad\):/u);+ expect(message).not.toContain("Expected a JSON-serializable object.");expect(events).toEqual([expect.objectContaining({data: expect.objectContaining({@@ -642,8 +647,14 @@ describe("emitStreamContent action requests", () => {}),]);expect([...result.invalidInputToolCallIds]).toEqual(["call-bad"]);- // Provider results live in the assistant response; no local tool message.- expect(result.trailingInlineToolResultParts).toEqual([]);+ expect(result.trailingInlineToolResultParts).toEqual([+ {+ output: { type: "error-text", value: message },+ toolCallId: "call-bad",+ toolName: "web_search",+ type: "tool-result",+ },+ ]);});});diff --git a/packages/eve/src/harness/emission.ts b/packages/eve/src/harness/emission.tsindex 0d8bb4b24..4e0fe2634 100644--- a/packages/eve/src/harness/emission.ts+++ b/packages/eve/src/harness/emission.ts@@ -458,6 +458,9 @@ async function consumeStreamContent(invalidInputToolCallIds.add(toolCall.toolCallId);await emitActionResult(createRuntimeToolResultFromToolError(resolved.toolError));handledInlineToolResultCallIds.add(toolCall.toolCallId);+ trailingInlineToolResultParts.push(+ createToolResultMessagePartFromToolError(resolved.toolError),+ );return;}diff --git a/packages/eve/src/harness/runtime-actions.test.ts b/packages/eve/src/harness/runtime-actions.test.tsindex ce74af0ea..f811bf3a6 100644--- a/packages/eve/src/harness/runtime-actions.test.ts+++ b/packages/eve/src/harness/runtime-actions.test.ts@@ -159,7 +159,17 @@ describe("resolveToolCallInputObject", () => {expect(() => resolveToolCallInputObject('"query"', context)).toThrow(/web_search.*call-1.*Expected a JSON-serializable object/su,);- expect(() => resolveToolCallInputObject("not json", context)).toThrow(/web_search.*call-1/su);++ try {+ resolveToolCallInputObject("not json", context);+ expect.unreachable("malformed JSON should throw");+ } catch (error) {+ expect(error).toMatchObject({+ cause: expect.objectContaining({ name: "SyntaxError" }),+ });+ expect((error as Error).message).toMatch(/web_search.*call-1/su);+ expect((error as Error).message).not.toContain("Expected a JSON-serializable object.");+ }});it("rejects non-object JSON values", () => {diff --git a/packages/eve/src/harness/runtime-actions.ts b/packages/eve/src/harness/runtime-actions.tsindex a16c0ff14..41d3a8cef 100644--- a/packages/eve/src/harness/runtime-actions.ts+++ b/packages/eve/src/harness/runtime-actions.ts@@ -452,13 +452,7 @@ export function resolveToolCallInputObject(}function parseJsonStringInput(value: string): unknown {- try {- return JSON.parse(value);- } catch {- // Not JSON at all — return the raw string so parseJsonObject rejects it- // with the canonical "Expected a JSON-serializable object." detail.- return value;- }+ return JSON.parse(value);}function toToolResultOutput(result: RuntimeActionResult): ToolResultPart["output"] {diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex 13a20284d..c451d5c3d 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -2609,7 +2609,7 @@ describe("createToolLoopHarness", () => {});it("feeds non-object tool call input back to the model as a failed tool result", async () => {- const invalidInput = "not an object";+ const invalidInput = '"not an object"';const errorMessage ='Failed to parse tool-call arguments for "add" (call-bad): Expected a JSON-serializable object.';setupMockAgent({@@ -3326,6 +3326,128 @@ describe("createToolLoopHarness", () => {}});+ it("feeds malformed provider web search input back to the model", async () => {+ const invalidInput = '{"query": 2025 NBA Finals champion}';+ setupMockAgent({+ content: [+ {+ input: invalidInput,+ providerExecuted: true,+ toolCallId: "search-malformed",+ toolName: "web_search",+ type: "tool-call",+ },+ ],+ finishReason: "tool-calls",+ fullStreamParts: [+ {+ input: invalidInput,+ providerExecuted: true,+ toolCallId: "search-malformed",+ toolName: "web_search",+ type: "tool-call",+ },+ { finishReason: "tool-calls", type: "finish-step" },+ ],+ response: {+ messages: [+ {+ content: [+ {+ input: invalidInput,+ providerExecuted: true,+ toolCallId: "search-malformed",+ toolName: "web_search",+ type: "tool-call",+ },+ ],+ role: "assistant",+ },+ ],+ },+ text: "",+ toolCalls: [+ {+ input: invalidInput,+ providerExecuted: true,+ toolCallId: "search-malformed",+ toolName: "web_search",+ type: "tool-call",+ },+ ],+ toolResults: [],+ });++ const { emit, events } = createEventCollector();+ const runStep = createToolLoopHarness(+ createTestConfig("conversation", emit, {+ tools: new Map(),+ }),+ );+ const session = createTestSession({+ agent: {+ modelReference: { id: "anthropic/claude-opus-5" },+ system: "You are a test assistant.",+ tools: [{ description: "Search the web", inputSchema: null, name: "web_search" }],+ },+ });+ const firstStep = await runStep(session, {+ message: "Who won the 2025 NBA Finals?",+ });++ expect(firstStep.next).toBe(runStep);+ expect(events.filter((event) => event.type === "action.result")).toEqual([+ expect.objectContaining({+ data: expect.objectContaining({+ error: expect.objectContaining({+ message: expect.stringMatching(+ /Failed to parse tool-call arguments for "web_search" \(search-malformed\):/u,+ ),+ }),+ status: "failed",+ }),+ type: "action.result",+ }),+ ]);+ const failedToolMessage = firstStep.session.history.at(-1);+ expect(failedToolMessage).toMatchObject({+ content: [+ {+ output: {+ type: "error-text",+ value: expect.stringMatching(+ /Failed to parse tool-call arguments for "web_search" \(search-malformed\):/u,+ ),+ },+ toolCallId: "search-malformed",+ toolName: "web_search",+ type: "tool-result",+ },+ ],+ role: "tool",+ });+ expect(JSON.stringify(failedToolMessage)).not.toContain("Expected a JSON-serializable object.");++ setupMockAgent({+ finishReason: "stop",+ response: {+ messages: [{ content: "I'll fix the search arguments.", role: "assistant" }],+ },+ text: "I'll fix the search arguments.",+ toolCalls: [],+ toolResults: [],+ });+ await runStep(firstStep.session);++ const secondInstance = vi.mocked(ToolLoopAgent).mock.results[1]?.value as+ | { stream: ReturnType<typeof vi.fn> }+ | undefined;+ const secondCall = secondInstance?.stream.mock.calls[0]?.[0] as+ | { messages: ModelMessage[] }+ | undefined;+ expect(secondCall?.messages).toEqual(firstStep.session.history);+ });+it("does not start a model call when the turn signal is already aborted", async () => {const abortController = new AbortController();const cancellation = new TurnCancelledError();