Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): runtime reliability - preserve tool errors and enforce eval timeouts (#1431)

Andrew Barbaf3ecdcdaf7d3
diff --git a/.changeset/enforce-eval-timeouts.md b/.changeset/enforce-eval-timeouts.md
new file mode 100644
index 000000000..8333be9c9
--- /dev/null
+++ b/.changeset/enforce-eval-timeouts.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Eval config timeouts are now applied to every eval and enforced even when an eval task does not cooperate with its abort signal, preventing stalled eval runs.
diff --git a/.changeset/preserve-malformed-tool-errors.md b/.changeset/preserve-malformed-tool-errors.md
new file mode 100644
index 000000000..6b07048c2
--- /dev/null
+++ b/.changeset/preserve-malformed-tool-errors.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Malformed raw tool arguments now preserve the original JSON syntax error and return it to the model as a failed tool result instead of reporting a misleading serialization error.
diff --git a/packages/eve/src/evals/runner/execute-task.test.ts b/packages/eve/src/evals/runner/execute-task.test.ts
index 15b1dc3a9..3be1e69a4 100644
--- a/packages/eve/src/evals/runner/execute-task.test.ts
+++ b/packages/eve/src/evals/runner/execute-task.test.ts
@@ -26,6 +26,23 @@ function createTestEval(test: (t: EveEvalContext) => unknown, id = "test-eval"):
}
describe("executeTask", () => {
+ it("settles when an eval ignores its timeout signal", async () => {
+ const outcome = await executeTask({
+ client: new Client({ host: target.url }),
+ target,
+ evaluation: createTestEval(
+ async () =>
+ await new Promise<void>(() => {
+ // Deliberately never settles.
+ }),
+ "timeout",
+ ),
+ timeoutMs: 1,
+ });
+
+ expect(outcome.error).toMatch(/timed out|timeout/i);
+ });
+
it("exposes a sleep helper with a one-second default", async () => {
vi.useFakeTimers();
let settled = false;
diff --git a/packages/eve/src/evals/runner/execute-task.ts b/packages/eve/src/evals/runner/execute-task.ts
index 6e154989d..e3ad73717 100644
--- a/packages/eve/src/evals/runner/execute-task.ts
+++ b/packages/eve/src/evals/runner/execute-task.ts
@@ -71,7 +71,7 @@ export async function executeTask(options: ExecuteTaskOptions): Promise<ExecuteT
let error: string | undefined;
let skipReason: string | undefined;
try {
- await evaluation.test(context);
+ await runUntilAborted(evaluation.test(context), signal);
} catch (err) {
if (err instanceof EvalSkipped) {
skipReason = err.reason;
@@ -159,3 +159,21 @@ function sum<T>(entries: readonly T[], read: (entry: T) => number): number {
function neverAbortSignal(): AbortSignal {
return new AbortController().signal;
}
+
+async function runUntilAborted(task: void | Promise<void>, signal: AbortSignal): Promise<void> {
+ signal.throwIfAborted();
+
+ let onAbort: (() => void) | undefined;
+ const aborted = new Promise<never>((_resolve, reject) => {
+ onAbort = () => reject(signal.reason);
+ signal.addEventListener("abort", onAbort, { once: true });
+ });
+
+ try {
+ await Promise.race([task, aborted]);
+ } finally {
+ if (onAbort !== undefined) {
+ signal.removeEventListener("abort", onAbort);
+ }
+ }
+}
diff --git a/packages/eve/src/evals/runner/run-evals.test.ts b/packages/eve/src/evals/runner/run-evals.test.ts
index 59b2e9592..b4c8bb7d5 100644
--- a/packages/eve/src/evals/runner/run-evals.test.ts
+++ b/packages/eve/src/evals/runner/run-evals.test.ts
@@ -382,6 +382,56 @@ describe("runEvals", () => {
expect(peak).toBe(2);
});
+ it("applies config timeouts while preserving eval and CLI overrides", async () => {
+ mockArtifacts();
+ mockedRunnerDependencies.executeTask.mockResolvedValue(createTaskResult("done"));
+
+ await run({
+ evaluations: [createEval("uses-config"), createEval("uses-eval", { timeoutMs: 2_000 })],
+ config: {
+ _tag: "EveEvalConfig",
+ timeoutMs: 1_000,
+ },
+ target: localTarget,
+ client: unusedClient,
+ appRoot: "/tmp/app",
+ reporters: [],
+ maxConcurrency: 1,
+ });
+
+ expect(mockedRunnerDependencies.executeTask.mock.calls[0]?.[0]).toEqual(
+ expect.objectContaining({
+ evaluation: expect.objectContaining({ id: "uses-config", timeoutMs: 1_000 }),
+ timeoutMs: 1_000,
+ }),
+ );
+ expect(mockedRunnerDependencies.executeTask.mock.calls[1]?.[0]).toEqual(
+ expect.objectContaining({
+ evaluation: expect.objectContaining({ id: "uses-eval", timeoutMs: 2_000 }),
+ timeoutMs: 2_000,
+ }),
+ );
+
+ mockedRunnerDependencies.executeTask.mockClear();
+
+ await run({
+ evaluations: [createEval("uses-cli", { timeoutMs: 2_000 })],
+ config: {
+ _tag: "EveEvalConfig",
+ timeoutMs: 1_000,
+ },
+ target: localTarget,
+ client: unusedClient,
+ appRoot: "/tmp/app",
+ reporters: [],
+ timeoutMs: 500,
+ });
+
+ expect(mockedRunnerDependencies.executeTask).toHaveBeenCalledWith(
+ expect.objectContaining({ timeoutMs: 500 }),
+ );
+ });
+
it("drives config reporters across every eval and dedupes eval references", async () => {
mockArtifacts();
mockedRunnerDependencies.executeTask.mockResolvedValue(createTaskResult("done"));
diff --git a/packages/eve/src/evals/runner/run-evals.ts b/packages/eve/src/evals/runner/run-evals.ts
index 522a32fee..c9787f22f 100644
--- a/packages/eve/src/evals/runner/run-evals.ts
+++ b/packages/eve/src/evals/runner/run-evals.ts
@@ -185,17 +185,19 @@ function buildReporterBindings(
return bindings;
}
-/**
- * Fills an eval's judge model from the run config when the eval does not set
- * its own. The judge model only ever drives `t.judge.*` assertions.
- */
+/** Fills per-eval defaults from the run config without replacing authored values. */
function applyConfigDefaults(evaluation: EveEval, config: EveEvalConfig): EveEval {
- if (evaluation.judge !== undefined || config.judge === undefined) {
+ const judge = evaluation.judge ?? config.judge;
+ const timeoutMs = evaluation.timeoutMs ?? config.timeoutMs;
+
+ if (judge === evaluation.judge && timeoutMs === evaluation.timeoutMs) {
return evaluation;
}
+
return {
...evaluation,
- judge: config.judge,
+ judge,
+ timeoutMs,
};
}
diff --git a/packages/eve/src/harness/emission.test.ts b/packages/eve/src/harness/emission.test.ts
index 4412593c0..ba6917204 100644
--- a/packages/eve/src/harness/emission.test.ts
+++ b/packages/eve/src/harness/emission.test.ts
@@ -557,7 +557,7 @@ describe("emitStreamContent action requests", () => {
EMISSION_STATE,
streamOf([
{
- input: "not an object",
+ input: '"not an object"',
toolCallId: "call-bad",
toolName: "web_search",
type: "tool-call",
@@ -600,8 +600,6 @@ describe("emitStreamContent action requests", () => {
it("turns malformed provider-executed tool call input into a failed tool result", async () => {
const emit = createEmitStub();
- const message =
- 'Failed to parse tool-call arguments for "web_search" (call-bad): Expected a JSON-serializable object.';
const result = await emitStreamContent(
emit,
@@ -625,6 +623,13 @@ describe("emitStreamContent action requests", () => {
);
const events = vi.mocked(emit).mock.calls.map(([event]) => event);
+ const actionResult = events.find((event) => event.type === "action.result");
+ if (actionResult?.data.error === undefined) {
+ throw new Error("Expected a failed action.result event.");
+ }
+ const { message } = actionResult.data.error;
+ expect(message).toMatch(/Failed to parse tool-call arguments for "web_search" \(call-bad\):/u);
+ expect(message).not.toContain("Expected a JSON-serializable object.");
expect(events).toEqual([
expect.objectContaining({
data: expect.objectContaining({
@@ -642,8 +647,14 @@ describe("emitStreamContent action requests", () => {
}),
]);
expect([...result.invalidInputToolCallIds]).toEqual(["call-bad"]);
- // Provider results live in the assistant response; no local tool message.
- expect(result.trailingInlineToolResultParts).toEqual([]);
+ expect(result.trailingInlineToolResultParts).toEqual([
+ {
+ output: { type: "error-text", value: message },
+ toolCallId: "call-bad",
+ toolName: "web_search",
+ type: "tool-result",
+ },
+ ]);
});
});
diff --git a/packages/eve/src/harness/emission.ts b/packages/eve/src/harness/emission.ts
index 0d8bb4b24..4e0fe2634 100644
--- a/packages/eve/src/harness/emission.ts
+++ b/packages/eve/src/harness/emission.ts
@@ -458,6 +458,9 @@ async function consumeStreamContent(
invalidInputToolCallIds.add(toolCall.toolCallId);
await emitActionResult(createRuntimeToolResultFromToolError(resolved.toolError));
handledInlineToolResultCallIds.add(toolCall.toolCallId);
+ trailingInlineToolResultParts.push(
+ createToolResultMessagePartFromToolError(resolved.toolError),
+ );
return;
}
diff --git a/packages/eve/src/harness/runtime-actions.test.ts b/packages/eve/src/harness/runtime-actions.test.ts
index ce74af0ea..f811bf3a6 100644
--- a/packages/eve/src/harness/runtime-actions.test.ts
+++ b/packages/eve/src/harness/runtime-actions.test.ts
@@ -159,7 +159,17 @@ describe("resolveToolCallInputObject", () => {
expect(() => resolveToolCallInputObject('"query"', context)).toThrow(
/web_search.*call-1.*Expected a JSON-serializable object/su,
);
- expect(() => resolveToolCallInputObject("not json", context)).toThrow(/web_search.*call-1/su);
+
+ try {
+ resolveToolCallInputObject("not json", context);
+ expect.unreachable("malformed JSON should throw");
+ } catch (error) {
+ expect(error).toMatchObject({
+ cause: expect.objectContaining({ name: "SyntaxError" }),
+ });
+ expect((error as Error).message).toMatch(/web_search.*call-1/su);
+ expect((error as Error).message).not.toContain("Expected a JSON-serializable object.");
+ }
});
it("rejects non-object JSON values", () => {
diff --git a/packages/eve/src/harness/runtime-actions.ts b/packages/eve/src/harness/runtime-actions.ts
index a16c0ff14..41d3a8cef 100644
--- a/packages/eve/src/harness/runtime-actions.ts
+++ b/packages/eve/src/harness/runtime-actions.ts
@@ -452,13 +452,7 @@ export function resolveToolCallInputObject(
}
function parseJsonStringInput(value: string): unknown {
- try {
- return JSON.parse(value);
- } catch {
- // Not JSON at all — return the raw string so parseJsonObject rejects it
- // with the canonical "Expected a JSON-serializable object." detail.
- return value;
- }
+ return JSON.parse(value);
}
function toToolResultOutput(result: RuntimeActionResult): ToolResultPart["output"] {
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index 13a20284d..c451d5c3d 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -2609,7 +2609,7 @@ describe("createToolLoopHarness", () => {
});
it("feeds non-object tool call input back to the model as a failed tool result", async () => {
- const invalidInput = "not an object";
+ const invalidInput = '"not an object"';
const errorMessage =
'Failed to parse tool-call arguments for "add" (call-bad): Expected a JSON-serializable object.';
setupMockAgent({
@@ -3326,6 +3326,128 @@ describe("createToolLoopHarness", () => {
}
});
+ it("feeds malformed provider web search input back to the model", async () => {
+ const invalidInput = '{"query": 2025 NBA Finals champion}';
+ setupMockAgent({
+ content: [
+ {
+ input: invalidInput,
+ providerExecuted: true,
+ toolCallId: "search-malformed",
+ toolName: "web_search",
+ type: "tool-call",
+ },
+ ],
+ finishReason: "tool-calls",
+ fullStreamParts: [
+ {
+ input: invalidInput,
+ providerExecuted: true,
+ toolCallId: "search-malformed",
+ toolName: "web_search",
+ type: "tool-call",
+ },
+ { finishReason: "tool-calls", type: "finish-step" },
+ ],
+ response: {
+ messages: [
+ {
+ content: [
+ {
+ input: invalidInput,
+ providerExecuted: true,
+ toolCallId: "search-malformed",
+ toolName: "web_search",
+ type: "tool-call",
+ },
+ ],
+ role: "assistant",
+ },
+ ],
+ },
+ text: "",
+ toolCalls: [
+ {
+ input: invalidInput,
+ providerExecuted: true,
+ toolCallId: "search-malformed",
+ toolName: "web_search",
+ type: "tool-call",
+ },
+ ],
+ toolResults: [],
+ });
+
+ const { emit, events } = createEventCollector();
+ const runStep = createToolLoopHarness(
+ createTestConfig("conversation", emit, {
+ tools: new Map(),
+ }),
+ );
+ const session = createTestSession({
+ agent: {
+ modelReference: { id: "anthropic/claude-opus-5" },
+ system: "You are a test assistant.",
+ tools: [{ description: "Search the web", inputSchema: null, name: "web_search" }],
+ },
+ });
+ const firstStep = await runStep(session, {
+ message: "Who won the 2025 NBA Finals?",
+ });
+
+ expect(firstStep.next).toBe(runStep);
+ expect(events.filter((event) => event.type === "action.result")).toEqual([
+ expect.objectContaining({
+ data: expect.objectContaining({
+ error: expect.objectContaining({
+ message: expect.stringMatching(
+ /Failed to parse tool-call arguments for "web_search" \(search-malformed\):/u,
+ ),
+ }),
+ status: "failed",
+ }),
+ type: "action.result",
+ }),
+ ]);
+ const failedToolMessage = firstStep.session.history.at(-1);
+ expect(failedToolMessage).toMatchObject({
+ content: [
+ {
+ output: {
+ type: "error-text",
+ value: expect.stringMatching(
+ /Failed to parse tool-call arguments for "web_search" \(search-malformed\):/u,
+ ),
+ },
+ toolCallId: "search-malformed",
+ toolName: "web_search",
+ type: "tool-result",
+ },
+ ],
+ role: "tool",
+ });
+ expect(JSON.stringify(failedToolMessage)).not.toContain("Expected a JSON-serializable object.");
+
+ setupMockAgent({
+ finishReason: "stop",
+ response: {
+ messages: [{ content: "I'll fix the search arguments.", role: "assistant" }],
+ },
+ text: "I'll fix the search arguments.",
+ toolCalls: [],
+ toolResults: [],
+ });
+ await runStep(firstStep.session);
+
+ const secondInstance = vi.mocked(ToolLoopAgent).mock.results[1]?.value as
+ | { stream: ReturnType<typeof vi.fn> }
+ | undefined;
+ const secondCall = secondInstance?.stream.mock.calls[0]?.[0] as
+ | { messages: ModelMessage[] }
+ | undefined;
+ expect(secondCall?.messages).toEqual(firstStep.session.history);
+ });
+
it("does not start a model call when the turn signal is already aborted", async () => {
const abortController = new AbortController();
const cancellation = new TurnCancelledError();