The Open Framework for Building Agents
fix(eve): repair orphaned tool calls before replay (#5169)
Ruif5b36b3da5e8
diff --git a/.changeset/reject-incomplete-model-streams.md b/.changeset/reject-incomplete-model-streams.mdnew file mode 100644index 000000000..1d6144afa--- /dev/null+++ b/.changeset/reject-incomplete-model-streams.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Repair orphaned local tool calls in saved history before the next model request with an explicit unknown-outcome error. The repair never reruns tools or moves recorded results, and leaves provider-executed calls, approval requests and duplicate call IDs alone.diff --git a/docs/concepts/sessions-runs-and-streaming.md b/docs/concepts/sessions-runs-and-streaming.mdindex fcfbd4bf0..88971b289 100644--- a/docs/concepts/sessions-runs-and-streaming.md+++ b/docs/concepts/sessions-runs-and-streaming.md@@ -134,6 +134,8 @@ A turn doesn't end while tasks are working. When the model ends its text early,`step.failed` and `turn.failed` carry `{ code, message, details? }` for the failed fragment or turn, and `session.failed` is the terminal session-level variant. `turn.failed` also carries `terminal`: `true` when the failure ends the session and `session.failed` follows, `false` when `session.waiting` follows and the next message can retry. Events from eve versions before `terminal` was added omit it. `turn.cancelled` is not a failure: the cancelled turn ends without any failure event, `session.waiting` follows, and the session accepts the next message normally. Whatever the turn streamed before cancellation stays on the stream. Durable history keeps the accepted user input and previously settled work, and discards incomplete assistant output. Tool calls the cancellation stopped stay in history, each answered as cancelled, so the model sees that the work started and stopped instead of a request left unanswered. Before `turn.cancelled`, the stream reports requests the cancel strands as `cancelled` in `input.resolved`. These include the turn's own questions, the session-limit prompt, and requests relayed from the work it stops. When a turn requested an output schema, the finalized payload lands on `result.completed` as `data.result` before the turn boundary. `authorization.required` carries the sign-in challenge (`data.authorization` may include `url`, `userCode`, `expiresAt`, `instructions`), and `authorization.completed` carries `data.outcome` (`"authorized" | "declined" | "failed" | "timed-out"`). Both carry `data.principalId` when a session principal started the sign-in. It holds the same value as `responderPrincipalId` on approval events, so a channel can deliver the challenge privately to that person even after someone else starts a later turn.+Before the next model request, eve repairs unambiguous orphaned local tool calls in saved history only when no result was recorded anywhere. An error result states that the execution outcome is unknown and external state must be checked before repeating side effects. This repair never executes the old call, moves recorded results, or settles work still owned by approvals, sign-in, workflows, or tasks. Provider-executed calls, SDK approvals, and duplicate call IDs are left unchanged.+A provider response ending with `content-filter` fails with `MODEL_CALL_FAILED`,`details.semanticErrorId: "model-response-content-filtered"`, and`details.finishReason: "content-filter"`. Details also include the Gatewaydiff --git a/packages/eve/src/harness/step/prompt.ts b/packages/eve/src/harness/step/prompt.tsindex f66e64e22..7202e8330 100644--- a/packages/eve/src/harness/step/prompt.ts+++ b/packages/eve/src/harness/step/prompt.ts@@ -7,6 +7,7 @@ import {validateHarnessModelMessages,} from "#harness/messages.js";import { hasUnansweredToolCall } from "#harness/model-call/response.js";+import { repairToolHistory } from "#harness/tool-history-repair.js";import {getTurnClientContextState,setTurnClientContextState,@@ -28,7 +29,20 @@ export interface Prompt {}export async function buildPrompt(step: Step, turn: TurnInput): Promise<Prompt> {- const messages = validateHarnessModelMessages([...step.session.history]);+ const repaired = repairToolHistory({+ messages: step.session.history,+ insertionIndex: turn.storedClientContext?.insertionIndex,+ });+ const messages = validateHarnessModelMessages(repaired.messages);+ step.session = { ...step.session, history: [...messages] };+ if (turn.storedClientContext !== undefined && repaired.insertionIndex !== undefined) {+ const storedClientContext = {+ ...turn.storedClientContext,+ insertionIndex: repaired.insertionIndex,+ };+ turn = { ...turn, storedClientContext };+ step.session = setTurnClientContextState(step.session, storedClientContext);+ }if (!hasUnansweredToolCall(messages)) {const taskContext = await appendTaskContext({messages,diff --git a/packages/eve/src/harness/tool-history-repair.integration.test.ts b/packages/eve/src/harness/tool-history-repair.integration.test.tsnew file mode 100644index 000000000..de186d8a8--- /dev/null+++ b/packages/eve/src/harness/tool-history-repair.integration.test.ts@@ -0,0 +1,155 @@+import { jsonSchema, type ModelMessage } from "ai";+import { convertArrayToReadableStream, MockLanguageModelV4 } from "ai/test";+import { describe, expect, it, vi } from "vitest";++import { ContextContainer, contextStorage } from "#context/container.js";+import { createUserMessage } from "#harness/messages.js";+import { enterSessionProjection } from "#harness/session-machine/current.js";+import { repairToolHistory } from "#harness/tool-history-repair.js";+import { createToolLoopHarness } from "#harness/tool-loop.js";+import { setTurnClientContextState } from "#harness/turn-client-context.js";+import type { HarnessSession } from "#harness/types.js";+import { foldingHandler, withOpenTurn } from "#internal/testing/session-machine.js";++const call = { type: "tool-call" as const, toolCallId: "old-call", toolName: "work", input: {} };+const calling: ModelMessage = { role: "assistant", content: [call] };+const user = createUserMessage("user", "What happened?");+const result = {+ type: "tool-result" as const,+ toolCallId: call.toolCallId,+ toolName: call.toolName,+ output: { type: "json" as const, value: { saved: true } },+};+const unknownResult = {+ ...result,+ output: { type: "error-text", value: expect.stringContaining("execution outcome is unknown") },+};+function results(messages: readonly ModelMessage[]) {+ return messages.flatMap((message) =>+ message.role === "tool" ? message.content.filter((part) => part.type === "tool-result") : [],+ );+}++describe("saved tool history", () => {+ it.each(["missing result", "partial results", "failed request"])(+ "repairs seeded history across followups: %s",+ async (kind) => {+ const execute = vi.fn(async () => ({ saved: true }));+ const doStream = vi.fn<MockLanguageModelV4["doStream"]>(async () => ({+ stream: convertArrayToReadableStream([+ { type: "text-start", id: "text" },+ { type: "text-delta", id: "text", delta: "Continuing safely." },+ { type: "text-end", id: "text" },+ {+ type: "finish",+ finishReason: { unified: "stop", raw: undefined },+ usage: {+ inputTokens: { cacheRead: 0, cacheWrite: 0, noCache: 1, total: 1 },+ outputTokens: { reasoning: 0, text: 1, total: 1 },+ },+ },+ ]),+ }));+ if (kind === "failed request") doStream.mockRejectedValueOnce(new Error("Model unavailable"));+ const sibling = { ...result, toolCallId: "sibling" };+ const partial = kind === "partial results";+ const history: HarnessSession["history"] = partial+ ? [+ { role: "assistant", content: [call, { ...call, toolCallId: sibling.toolCallId }] },+ { role: "tool", content: [sibling] },+ user,+ ]+ : [calling, user];+ const tool = {+ name: "work",+ description: "Save work",+ inputSchema: jsonSchema({ type: "object" }),+ execute,+ };+ const step = createToolLoopHarness({+ handleEvent: foldingHandler(),+ resolveModel: async () => new MockLanguageModelV4({ doStream }),+ tools: new Map([["work", tool]]),+ });+ const run = (session: HarnessSession) => {+ const ctx = new ContextContainer();+ enterSessionProjection(ctx, session.state);+ return contextStorage.run(ctx, () => step(session, { message: "Continue" }));+ };+ const initial = setTurnClientContextState(+ withOpenTurn(+ {+ agent: { modelReference: { id: "repair-test" }, system: "Test assistant", tools: [] },+ compaction: { recentWindowSize: 10, threshold: 100_000 },+ continuationToken: "http:repair-test",+ sessionId: "repair-test",+ history,+ },+ { sequence: 1, turnId: "retry-turn" },+ ),+ {+ turnId: "retry-turn",+ insertionIndex: partial ? 2 : 1,+ messages: ["Client context"],+ },+ );+ const first = await run(initial);+ expect(first.next).toBeNull();+ expect(results(first.session.history)).toEqual(+ partial ? [sibling, unknownResult] : [unknownResult],+ );+ const prompt = doStream.mock.calls[0]![0].prompt;+ const contextIndex = prompt.findIndex(+ (message) =>+ message.role === "user" &&+ message.content.some((part) => part.type === "text" && part.text === "Client context"),+ );+ expect(contextIndex).toBeGreaterThan(0);+ expect(prompt[contextIndex - 1]).toMatchObject({+ role: "tool",+ content: expect.arrayContaining([expect.objectContaining({ toolCallId: call.toolCallId })]),+ });+ expect(prompt[contextIndex + 1]).toMatchObject({+ role: "user",+ content: [{ type: "text", text: "What happened?" }],+ });+ const next = await run(first.session);+ expect(doStream).toHaveBeenCalledTimes(2);+ expect(results(next.session.history)).toEqual(results(first.session.history));+ expect(JSON.stringify(next.session.history)).toContain("Continuing safely.");+ expect(JSON.stringify(next.session.history)).not.toContain("Client context");+ expect(execute).not.toHaveBeenCalled();+ },+ );++ it.each([+ "provider",+ "inline-result",+ "sdk-approval",+ "duplicate-call",+ "late-result",+ "earlier-result",+ ])("leaves %s history untouched", (kind) => {+ let messages: ModelMessage[] = [calling, user];+ if (kind === "provider")+ messages = [{ role: "assistant", content: [{ ...call, providerExecuted: true }] }, user];+ if (kind === "inline-result") messages = [{ role: "assistant", content: [call, result] }, user];+ if (kind === "sdk-approval") {+ const request = {+ type: "tool-approval-request" as const,+ approvalId: "approval",+ toolCallId: call.toolCallId,+ };+ messages = [{ role: "assistant", content: [call, request] }, user];+ }+ if (kind === "duplicate-call") messages.push(calling);+ if (kind === "late-result") messages.push({ role: "tool", content: [result] });+ if (kind === "earlier-result") messages.unshift({ role: "tool", content: [result] });+ const original = structuredClone(messages);+ expect(repairToolHistory({ messages, insertionIndex: 1 })).toEqual({+ messages: original,+ insertionIndex: 1,+ });+ expect(messages).toEqual(original);+ });+});diff --git a/packages/eve/src/harness/tool-history-repair.ts b/packages/eve/src/harness/tool-history-repair.tsnew file mode 100644index 000000000..4bb178cd8--- /dev/null+++ b/packages/eve/src/harness/tool-history-repair.ts@@ -0,0 +1,51 @@+import type { ModelMessage } from "ai";++import { extractToolResultCallIds } from "#harness/model-call/response.js";+import { withResult } from "#harness/session-machine/transitions.js";++/** Answer orphaned historical calls without executing them. */+export function repairToolHistory(input: {+ readonly messages: readonly ModelMessage[];+ readonly insertionIndex?: number;+}): { messages: ModelMessage[]; insertionIndex: number | undefined } {+ let messages = [...input.messages];+ let insertionIndex = input.insertionIndex;+ // Any recorded result, even misplaced or inline, belongs to its original history.+ const protectedIds = new Set(extractToolResultCallIds(messages));+ const seen = new Set<string>();+ for (const message of messages) {+ if (message.role !== "assistant" || !Array.isArray(message.content)) continue;+ for (const part of message.content) {+ if (part.type === "tool-approval-request") protectedIds.add(part.toolCallId);+ if (part.type !== "tool-call") continue;+ if (seen.has(part.toolCallId) || part.providerExecuted === true)+ protectedIds.add(part.toolCallId);+ seen.add(part.toolCallId);+ }+ }+ for (const message of input.messages) {+ if (message.role !== "assistant" || !Array.isArray(message.content)) continue;+ for (const call of message.content) {+ if (call.type !== "tool-call" || protectedIds.has(call.toolCallId)) continue;+ const callIndex = messages.indexOf(message);+ const repaired = withResult(messages, {+ type: "tool-result",+ toolCallId: call.toolCallId,+ toolName: call.toolName,+ output: {+ type: "error-text",+ value:+ "The previous attempt ended without a recorded result; execution outcome is unknown; verify external state before repeating side effects.",+ },+ });+ if (+ insertionIndex !== undefined &&+ repaired.length > messages.length &&+ callIndex < insertionIndex+ )+ insertionIndex++;+ messages = repaired;+ }+ }+ return { messages, insertionIndex };+}diff --git a/packages/eve/src/harness/tool-input-validation.integration.test.ts b/packages/eve/src/harness/tool-input-validation.integration.test.tsindex 14c1bc5d3..9caf1a545 100644--- a/packages/eve/src/harness/tool-input-validation.integration.test.ts+++ b/packages/eve/src/harness/tool-input-validation.integration.test.ts@@ -43,7 +43,7 @@ function toolCall(toolCallId: string, toolName: string, input: string): StreamPareturn { input, toolCallId, toolName, type: "tool-call" };}-function finish(unified: "length" | "stop" | "tool-calls"): StreamPart {+function finish(unified: "length" | "other" | "stop" | "tool-calls"): StreamPart {return { finishReason: { raw: undefined, unified }, type: "finish", usage };}@@ -121,18 +121,21 @@ describe("framework tool input validation (real AI SDK)", () => {it.each([{- name: "beside a truncated call",+ name: "length beside a truncated call",+ reason: "length" as const,truncated: [{ input: '{"path": "notes/bob', toolCallId: "read-truncated" }],},- { name: "alone", truncated: [] },- ])("answers complete tool calls cut short at the output limit $name", async ({ truncated }) => {+ { name: "length alone", reason: "length" as const, truncated: [] },+ { name: "other", reason: "other" as const, truncated: [] },+ { name: "missing finish event", reason: undefined, truncated: [] },+ ])("answers complete tool calls ended early: $name", async ({ reason, truncated }) => {const skippedCallId = "read-skipped";const model = new MockLanguageModelV4({doStream: scriptedStreams([[toolCall(skippedCallId, "read_note", JSON.stringify({ path: "notes/alice.md" })),...truncated.map((call) => toolCall(call.toolCallId, "read_note", call.input)),- finish("length"),+ ...(reason === undefined ? [] : [finish(reason)]),],[{ id: "reply", type: "text-start" },@@ -179,14 +182,21 @@ describe("framework tool input validation (real AI SDK)", () => {if (typeof cutShortStep.next !== "function") {throw new TypeError("Expected the cut-short step to continue the tool loop.");}+ const expectedResult = {+ output: {+ type: "error-text",+ value: expect.stringContaining(`finish reason: ${reason ?? "other"}`),+ },+ };+ expect(findToolResult(cutShortStep.session.history, skippedCallId)).toMatchObject(+ expectedResult,+ );const nextStep = await cutShortStep.next(cutShortStep.session);expect(execute).not.toHaveBeenCalled();expect(model.doStreamCalls).toHaveLength(2);const retryPrompt = model.doStreamCalls[1]?.prompt ?? [];- expect(findToolResult(retryPrompt, skippedCallId)).toMatchObject({- output: { type: "error-text", value: expect.stringContaining("finish reason: length") },- });+ expect(findToolResult(retryPrompt, skippedCallId)).toMatchObject(expectedResult);for (const call of truncated) {expect(findToolResult(retryPrompt, call.toolCallId)).toBeDefined();}