The Open Framework for Building Agents
fix(eve): deliver stale HITL responses as follow-ups (#888)
Rui6a5a36afa3a8
diff --git a/.changeset/late-hitl-responses.md b/.changeset/late-hitl-responses.mdnew file mode 100644index 000000000..042cd87f3--- /dev/null+++ b/.changeset/late-hitl-responses.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Deliver stale HITL responses — answers to a question or approval that is no longer pending — as a new user message, letting the model decide whether the old selection still matters. A stale approval never authorizes the earlier tool call.diff --git a/docs/concepts/sessions-runs-and-streaming.md b/docs/concepts/sessions-runs-and-streaming.mdindex 0f92b2472..422f44cf9 100644--- a/docs/concepts/sessions-runs-and-streaming.md+++ b/docs/concepts/sessions-runs-and-streaming.md@@ -87,6 +87,12 @@ The follow-up reuses the same durable session: same history, same state.If the session is waiting on a human-in-the-loop approval, a matching text reply such as `approve` or `deny` answers the approval. Other follow-up text is held until the approval is answered, so an unrelated message does not implicitly deny the pending tool call.+If the session is waiting on `ask_question`, a follow-up message clears that pending request before the model continues. An exact option match or permitted freeform response answers the question; any other message marks the question unanswered and starts the follow-up turn.++A response is stale when its request is no longer pending: the question or approval was already answered, cleared by a follow-up message, or cancelled. eve delivers a stale response to the model as a new user message, and the model decides whether the old selection still matters. A stale approval never authorizes the earlier tool call; the model must request the action and approval again if they are still needed.++Responses match pending requests by request ID, so a response to an older request stays a plain user message even while a different question or approval is pending. Like any follow-up message, a stale response clears a pending question and is held while an approval is pending.+For deterministic ordering, send one follow-up at a time and wait for the next `session.waiting` event before sending another message to the same session. See [message delivery and queueing](./execution-model-and-durability#message-delivery-and-queueing) for the current runtime contract.## Cancel the in-flight turndiff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection-idle.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection-idle.eval.tsnew file mode 100644index 000000000..e91c43316--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection-idle.eval.ts@@ -0,0 +1,59 @@+import { defineEval } from "eve/evals";++/**+ * The plain stale path: a follow-up message resolves the ask_question+ * request and the turn completes with nothing pending. A later click on the+ * old control must start a new user turn instead of erroring or reviving+ * the request.+ */+export default defineEval({+ description:+ "HITL smoke: a stale ask-question selection becomes a new user turn when nothing is pending.",+ async test(t) {+ await t.send(+ [+ "Use the `ask_question` tool exactly once to ask me which context to use.",+ "Set prompt to: 'Which context should I use?'",+ "Set allowFreeform to true.",+ "Provide exactly two options:",+ '- id "current", label "Use current context"',+ '- id "candidate", label "Use STALE-CANDIDATE-7Q4M"',+ "Do not answer the question yourself, wait for my response.",+ ].join("\n"),+ );++ const request = t.requireInputRequest({+ optionIds: ["current", "candidate"],+ toolName: "ask_question",+ });++ const intervening = await t.send(+ "Use current context instead and reply with exactly INTERVENING-HITL-OK.",+ );+ intervening.expectOk();+ intervening.notEvent("input.requested");+ intervening.messageIncludes(/INTERVENING-HITL-OK/i);++ const staleSelection = await t.respond({+ requestId: request.requestId,+ optionId: "candidate",+ });+ staleSelection.expectOk();+ staleSelection.notEvent("input.requested");+ staleSelection.event("message.received", {+ count: 1,+ data: { message: "Use STALE-CANDIDATE-7Q4M" },+ });+ // The stale selection reaches the model as context it may act on or+ // disregard; judge that the reply engages with it instead of demanding+ // a literal echo the model can rightly decline.+ t.judge.autoevals+ .closedQA(+ "The reply acknowledges a late selection of the STALE-CANDIDATE-7Q4M option, either by applying it or by explaining that the question was already answered and the selection is no longer relevant.",+ { on: staleSelection.message },+ )+ .atLeast(0.5);++ t.succeeded();+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection.eval.tsnew file mode 100644index 000000000..452e4cd98--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection.eval.ts@@ -0,0 +1,79 @@+import { defineEval } from "eve/evals";++/**+ * A freeform answer can resolve an ask_question request before the user clicks+ * one of its rendered controls. A later click must become a new user turn,+ * even if another question is pending, so the model can decide whether that+ * old selection is still relevant.+ */+export default defineEval({+ description:+ "HITL smoke: a stale ask-question selection becomes a new user turn while another question is pending.",+ async test(t) {+ await t.send(+ [+ "Use the `ask_question` tool exactly once to ask me which context to use.",+ "Set prompt to: 'Which context should I use?'",+ "Set allowFreeform to true.",+ "Provide exactly two options:",+ '- id "current", label "Use current context"',+ '- id "candidate", label "Use STALE-CANDIDATE-7Q4M"',+ "Do not answer the question yourself, wait for my response.",+ ].join("\n"),+ );++ const request = t.requireInputRequest({+ optionIds: ["current", "candidate"],+ toolName: "ask_question",+ });++ const intervening = await t.send(+ "Use current context instead and reply with exactly INTERVENING-HITL-OK.",+ );+ intervening.expectOk();+ intervening.notEvent("input.requested");+ intervening.messageIncludes(/INTERVENING-HITL-OK/i);++ const nextQuestionTurn = await t.send(+ [+ "Use the `ask_question` tool exactly once to ask which new context to use.",+ "Set prompt to: 'Which new context should I use?'",+ "Set allowFreeform to false.",+ "Provide exactly two options:",+ '- id "alpha", label "Use alpha"',+ '- id "beta", label "Use beta"',+ "Do not answer the question yourself, wait for my response.",+ ].join("\n"),+ );+ nextQuestionTurn.expectOk();+ nextQuestionTurn.event("input.requested", { count: 1 });+ const nextRequest = t.requireInputRequest({+ optionIds: ["alpha", "beta"],+ toolName: "ask_question",+ });+ if (nextRequest.requestId === request.requestId) {+ throw new Error("The second ask_question call reused the stale request ID.");+ }++ const staleSelection = await t.respond({+ requestId: request.requestId,+ optionId: "candidate",+ });+ staleSelection.expectOk();+ staleSelection.event("message.received", {+ count: 1,+ data: { message: "Use STALE-CANDIDATE-7Q4M" },+ });+ // The stale selection reaches the model as context it may act on or+ // disregard; judge that the reply engages with it instead of demanding+ // a literal echo the model can rightly decline.+ t.judge.autoevals+ .closedQA(+ "The reply acknowledges a late selection of the STALE-CANDIDATE-7Q4M option, either by applying it or by explaining that the question was already answered and the selection is no longer relevant.",+ { on: staleSelection.message },+ )+ .atLeast(0.5);++ t.succeeded();+ },+});diff --git a/packages/eve/src/execution/turn-cancellation.integration.test.ts b/packages/eve/src/execution/turn-cancellation.integration.test.tsindex e87e73f8c..8f02a74a8 100644--- a/packages/eve/src/execution/turn-cancellation.integration.test.ts+++ b/packages/eve/src/execution/turn-cancellation.integration.test.ts@@ -637,13 +637,15 @@ describe("turn cancellation integration", () => {expect(followUpTurn.at(-1)?.type).toBe("session.waiting");expect(filterEventsByType(followUpTurn, "turn.cancelled")).toHaveLength(0);expect(filterEventsByType(followUpTurn, "turn.started")).toHaveLength(1);+ expect(filterEventsByType(followUpTurn, "step.completed")).toHaveLength(1);expect(filterEventsByType(followUpTurn, "session.waiting")).toHaveLength(1);expectNoFailureEvents(followUpTurn);expect(followUpTurn.some((event) =>- event.type === "message.completed" &&- event.data.message?.includes("answer after hitl cancel") === true,+ event.type === "message.received" &&+ typeof event.data.message === "string" &&+ event.data.message.includes("answer after hitl cancel"),),).toBe(true);diff --git a/packages/eve/src/harness/input-extraction.ts b/packages/eve/src/harness/input-extraction.tsindex 53c8d251b..eb1cedc0b 100644--- a/packages/eve/src/harness/input-extraction.ts+++ b/packages/eve/src/harness/input-extraction.ts@@ -1,9 +1,36 @@-import type { ContentPart, ToolApprovalRequestOutput, ToolSet, TypedToolCall } from "ai";+import type { ContentPart, ModelMessage, ToolSet, TypedToolCall } from "ai";+import { z } from "zod";import { ASK_QUESTION_TOOL_NAME } from "#runtime/framework-tools/ask-question.js";import type { InputRequest } from "#runtime/input/types.js";import { createRuntimeToolCallActionFromToolCall } from "#harness/input-requests.js";+// Persisted history parts lose AI SDK typing on the storage round trip. The+// schemas are the single source for the runtime narrowing and the static+// types, so the checks and the annotations cannot drift apart.+const ToolCallDescriptorSchema = z.object({+ input: z.unknown(),+ toolCallId: z.string(),+ toolName: z.string(),+});++type ToolCallDescriptor = z.infer<typeof ToolCallDescriptorSchema>;++const PersistedToolCallSchema = ToolCallDescriptorSchema.extend({+ type: z.literal("tool-call"),+});++// Malformed optional metadata degrades to `undefined` instead of dropping the+// whole approval request: a broken `toolCall` falls back to the sibling+// tool-call lookup and a broken `isAutomatic` counts as not automatic.+const ToolApprovalRequestSchema = z.object({+ approvalId: z.string(),+ isAutomatic: z.boolean().optional().catch(undefined),+ toolCall: ToolCallDescriptorSchema.optional().catch(undefined),+ toolCallId: z.string().optional().catch(undefined),+ type: z.literal("tool-approval-request"),+});+/*** Extracts question input requests from tool calls that target the* `ask_question` framework tool.@@ -11,6 +38,13 @@ import { createRuntimeToolCallActionFromToolCall } from "#harness/input-requestsexport function extractQuestionInputRequests(input: {readonly excludedCallIds: ReadonlySet<string>;readonly toolCalls: readonly TypedToolCall<ToolSet>[];+}): InputRequest[] {+ return extractQuestionRequests(input);+}++function extractQuestionRequests(input: {+ readonly excludedCallIds: ReadonlySet<string>;+ readonly toolCalls: readonly ToolCallDescriptor[];}): InputRequest[] {const requests: InputRequest[] = [];@@ -65,35 +99,51 @@ export function extractQuestionInputRequests(input: {export function extractToolApprovalInputRequests(input: {readonly content: readonly ContentPart<ToolSet>[];readonly excludedCallIds?: ReadonlySet<string>;+}): InputRequest[] {+ return extractApprovalRequests(input);+}++// Persisted history parts lose AI SDK typing, so this core narrows each part+// at runtime. The exported wrapper above keeps live call sites compile-checked+// against the AI SDK shapes.+function extractApprovalRequests(input: {+ readonly content: readonly unknown[];+ readonly excludedCallIds?: ReadonlySet<string>;+ readonly includedRequestIds?: ReadonlySet<string>;}): InputRequest[] {const requests: InputRequest[] = [];- const toolCallsById = new Map<string, TypedToolCall<ToolSet>>();+ const toolCallsById = new Map<string, ToolCallDescriptor>();for (const part of input.content) {- if (part.type === "tool-call") {- toolCallsById.set(part.toolCallId, part);+ const toolCall = PersistedToolCallSchema.safeParse(part);+ if (toolCall.success) {+ toolCallsById.set(toolCall.data.toolCallId, toolCall.data);}}for (const part of input.content) {- if (part.type !== "tool-approval-request") {+ const parsed = ToolApprovalRequestSchema.safeParse(part);+ if (!parsed.success) {+ continue;+ }+ const approval = parsed.data;++ if (+ input.includedRequestIds !== undefined &&+ !input.includedRequestIds.has(approval.approvalId)+ ) {continue;}- const approvalRequest = part as ToolApprovalRequestOutput<ToolSet> & {- readonly toolCall?: TypedToolCall<ToolSet>;- readonly toolCallId?: string;- };// AI SDK records automatic decisions as request/response pairs for history;// only unresolved requests should become eve input.- if (approvalRequest.isAutomatic === true) {+ if (approval.isAutomatic === true) {continue;}+const toolCall =- approvalRequest.toolCall ??- (approvalRequest.toolCallId === undefined- ? undefined- : toolCallsById.get(approvalRequest.toolCallId));+ approval.toolCall ??+ (approval.toolCallId === undefined ? undefined : toolCallsById.get(approval.toolCallId));if (toolCall === undefined) {continue;}@@ -102,12 +152,8 @@ export function extractToolApprovalInputRequests(input: {continue;}- const action = createRuntimeToolCallActionFromToolCall({- toolCall,- });-requests.push({- action,+ action: createRuntimeToolCallActionFromToolCall({ toolCall }),allowFreeform: false,display: "confirmation",options: [@@ -115,9 +161,56 @@ export function extractToolApprovalInputRequests(input: {{ id: "deny", label: "No" },],prompt: `Approve tool call: ${toolCall.toolName}`,- requestId: approvalRequest.approvalId,+ requestId: approval.approvalId,});}return requests;}++/**+ * Recovers request metadata for submitted input response IDs from model+ * history. The newest occurrence wins so compacted or repeated history does+ * not replace the request that is closest to the current turn.+ */+export function extractHistoricalInputRequests(input: {+ readonly history: readonly ModelMessage[];+ readonly requestIds: ReadonlySet<string>;+}): ReadonlyMap<string, InputRequest> {+ const requests = new Map<string, InputRequest>();++ for (let index = input.history.length - 1; index >= 0; index -= 1) {+ const message = input.history[index];+ if (message?.role !== "assistant" || !Array.isArray(message.content)) {+ continue;+ }++ const toolCalls = message.content.flatMap((part: unknown) => {+ const toolCall = PersistedToolCallSchema.safeParse(part);+ return toolCall.success && input.requestIds.has(toolCall.data.toolCallId)+ ? [toolCall.data]+ : [];+ });+ const candidates = [+ ...extractQuestionRequests({ excludedCallIds: new Set(), toolCalls }),+ ...extractApprovalRequests({+ content: message.content,+ includedRequestIds: input.requestIds,+ }),+ ];++ for (const request of candidates) {+ if (!input.requestIds.has(request.requestId) || requests.has(request.requestId)) {+ continue;+ }++ requests.set(request.requestId, request);+ }++ if (requests.size === input.requestIds.size) {+ break;+ }+ }++ return requests;+}diff --git a/packages/eve/src/harness/input-requests.ts b/packages/eve/src/harness/input-requests.tsindex 0eeb30d37..af55b521f 100644--- a/packages/eve/src/harness/input-requests.ts+++ b/packages/eve/src/harness/input-requests.ts@@ -1,4 +1,4 @@-import type { ModelMessage, ToolSet, TypedToolCall } from "ai";+import type { ModelMessage } from "ai";import type {RuntimeToolCallActionRequest,@@ -340,6 +340,13 @@ export function hasPendingInputBatch(state: SessionStateMap | undefined): booleareturn getPendingInputBatch(state) !== undefined;}+/**+ * Returns the request IDs in the currently pending HITL batch.+ */+export function getPendingInputRequestIds(state: SessionStateMap | undefined): ReadonlySet<string> {+ return new Set(getPendingInputBatch(state)?.requests.map((request) => request.requestId));+}+function getPendingInputBatch(state: SessionStateMap | undefined): PendingInputBatch | undefined {const value = state?.[PENDING_INPUT_BATCH_KEY];@@ -636,7 +643,8 @@ function buildToolResponsePartsForRequest(];}-function isApprovalRequest(request: InputRequest): boolean {+/** Shared approval predicate: a request whose options are exactly `approve` / `deny`. */+export function isApprovalRequest(request: InputRequest): boolean {return (request.options?.length === 2 &&request.options[0]?.id === "approve" &&@@ -652,7 +660,11 @@ function isApprovalRequest(request: InputRequest): boolean {* Creates a runtime tool-call action shape from an AI SDK tool call.*/export function createRuntimeToolCallActionFromToolCall(input: {- readonly toolCall: TypedToolCall<ToolSet>;+ readonly toolCall: {+ readonly input: unknown;+ readonly toolCallId: string;+ readonly toolName: string;+ };}): RuntimeToolCallActionRequest {return {callId: input.toolCall.toolCallId,diff --git a/packages/eve/src/harness/messages.ts b/packages/eve/src/harness/messages.tsindex 4dff2b5a9..6928a594c 100644--- a/packages/eve/src/harness/messages.ts+++ b/packages/eve/src/harness/messages.ts@@ -153,11 +153,24 @@ function coalesceMessage(input: {return input.a;}- if (typeof input.a === "string" && typeof input.b === "string") {- return `${input.a}\n\n${input.b}`;- }+ return appendUserContent({ appended: input.b, existing: input.a });+}- const merged: UserContentArray = [...toUserContentArray(input.a), ...toUserContentArray(input.b)];+/**+ * Appends user content while preserving structured attachment parts.+ */+export function appendUserContent(input: {+ readonly appended: string | UserContent;+ readonly existing: string | UserContent;+}): string | UserContent {+ if (typeof input.existing === "string" && typeof input.appended === "string") {+ return `${input.existing}\n\n${input.appended}`;+ }++ const merged: UserContentArray = [+ ...toUserContentArray(input.existing),+ ...toUserContentArray(input.appended),+ ];return merged;}diff --git a/packages/eve/src/harness/stale-input-responses.test.ts b/packages/eve/src/harness/stale-input-responses.test.tsnew file mode 100644index 000000000..637dcf6df--- /dev/null+++ b/packages/eve/src/harness/stale-input-responses.test.ts@@ -0,0 +1,147 @@+import { expect, it } from "vitest";++import type { ModelMessage } from "ai";++import { convertStaleResponsesToUserMessage } from "#harness/stale-input-responses.js";++const approvalHistory: ModelMessage[] = [+ {+ content: [+ {+ input: { command: "deploy --force" },+ toolCallId: "call-1",+ toolName: "bash",+ type: "tool-call",+ },+ {+ approvalId: "approval-1",+ toolCallId: "call-1",+ type: "tool-approval-request",+ },+ ],+ role: "assistant",+ },+];++const questionHistory: ModelMessage[] = [+ {+ content: [+ {+ input: {+ allowFreeform: true,+ options: [+ { id: "current", label: "Use current context" },+ { id: "candidate", label: "Use the candidate" },+ ],+ prompt: "Which context should I use?",+ },+ toolCallId: "question-1",+ toolName: "ask_question",+ type: "tool-call",+ },+ ],+ role: "assistant",+ },+];++it("converts a stale approval into a non-authorizing user message", () => {+ const result = convertStaleResponsesToUserMessage({+ history: approvalHistory,+ pendingRequestIds: new Set(),+ stepInput: {+ inputResponses: [{ optionId: "approve", requestId: "approval-1" }],+ },+ });++ expect(result.kind).toBe("converted");+ if (result.kind !== "converted") {+ throw new Error("Expected the stale response to be converted.");+ }++ expect(result.displayMessage).toBe("Yes");+ expect(result.stepInput.inputResponses).toBeUndefined();+ expect(result.stepInput.message).toEqual(expect.stringContaining("Approve tool call: bash"));+ expect(result.stepInput.message).toEqual(expect.stringContaining('"label": "Yes"'));+ expect(result.stepInput.message).toEqual(+ expect.stringContaining("This does not authorize an earlier action"),+ );+});++it("converts a stale question selection using its option label", () => {+ const result = convertStaleResponsesToUserMessage({+ history: questionHistory,+ pendingRequestIds: new Set(),+ stepInput: {+ inputResponses: [{ optionId: "candidate", requestId: "question-1" }],+ },+ });++ expect(result.kind).toBe("converted");+ if (result.kind !== "converted") {+ throw new Error("Expected the stale response to be converted.");+ }++ expect(result.displayMessage).toBe("Use the candidate");+ expect(result.stepInput.message).toEqual(expect.stringContaining("Which context should I use?"));+ expect(result.stepInput.message).toEqual(expect.stringContaining('"requestType": "question"'));+ expect(result.stepInput.message).not.toEqual(+ expect.stringContaining("This does not authorize an earlier action"),+ );+});++it("keeps responses for pending requests structured while converting stale ones", () => {+ const result = convertStaleResponsesToUserMessage({+ history: questionHistory,+ pendingRequestIds: new Set(["question-2"]),+ stepInput: {+ inputResponses: [+ { optionId: "alpha", requestId: "question-2" },+ { optionId: "candidate", requestId: "question-1" },+ ],+ },+ });++ expect(result.kind).toBe("converted");+ if (result.kind !== "converted") {+ throw new Error("Expected the stale response to be converted.");+ }++ expect(result.stepInput.inputResponses).toEqual([{ optionId: "alpha", requestId: "question-2" }]);+ expect(result.stepInput.message).toEqual(expect.stringContaining("question-1"));+ expect(result.stepInput.message).not.toEqual(expect.stringContaining("question-2"));+});++it("keeps the non-authorization notice when request metadata is missing", () => {+ const result = convertStaleResponsesToUserMessage({+ history: [],+ pendingRequestIds: new Set(),+ stepInput: {+ inputResponses: [{ optionId: "approve", requestId: "approval-gone" }],+ },+ });++ expect(result.kind).toBe("converted");+ if (result.kind !== "converted") {+ throw new Error("Expected the stale response to be converted.");+ }++ expect(result.displayMessage).toBe("approve");+ expect(result.stepInput.message).toEqual(expect.stringContaining("approval-gone"));+ expect(result.stepInput.message).toEqual(+ expect.stringContaining("This does not authorize an earlier action"),+ );+});++it("returns unchanged when every response matches the pending batch", () => {+ const stepInput = {+ inputResponses: [{ optionId: "approve", requestId: "approval-1" }],+ };+ const result = convertStaleResponsesToUserMessage({+ history: approvalHistory,+ pendingRequestIds: new Set(["approval-1"]),+ stepInput,+ });++ expect(result.kind).toBe("unchanged");+ expect(result.stepInput).toBe(stepInput);+});diff --git a/packages/eve/src/harness/stale-input-responses.ts b/packages/eve/src/harness/stale-input-responses.tsnew file mode 100644index 000000000..985a42009--- /dev/null+++ b/packages/eve/src/harness/stale-input-responses.ts@@ -0,0 +1,166 @@+import type { ModelMessage, UserContent } from "ai";++import { extractHistoricalInputRequests } from "#harness/input-extraction.js";+import { isApprovalRequest } from "#harness/input-requests.js";+import { appendUserContent } from "#harness/messages.js";+import type { StepInput } from "#harness/types.js";+import type { InputRequest, InputResponse } from "#runtime/input/types.js";++type StaleResponseConversion =+ | {+ readonly kind: "unchanged";+ readonly stepInput?: StepInput;+ }+ | {+ readonly displayMessage: string | UserContent;+ readonly kind: "converted";+ readonly stepInput: StepInput;+ };++/**+ * A response is stale when its request ID is not in the currently pending+ * HITL batch: the request was already answered, cleared by a follow-up+ * message, or cancelled.+ *+ * Responses for pending requests stay structured; stale responses become+ * plain user-message text. A stale response never reaches structured HITL+ * processing, so a stale approval cannot authorize an earlier tool call.+ * Request details recovered from history are best-effort model context.+ */+export function convertStaleResponsesToUserMessage(input: {+ readonly history: readonly ModelMessage[];+ readonly pendingRequestIds: ReadonlySet<string>;+ readonly stepInput?: StepInput;+}): StaleResponseConversion {+ const responses = input.stepInput?.inputResponses;+ if (input.stepInput === undefined || responses === undefined || responses.length === 0) {+ return { kind: "unchanged", stepInput: input.stepInput };+ }++ const currentResponses: InputResponse[] = [];+ const staleResponses: InputResponse[] = [];+ for (const response of responses) {+ if (input.pendingRequestIds.has(response.requestId)) {+ currentResponses.push(response);+ } else {+ staleResponses.push(response);+ }+ }++ if (staleResponses.length === 0) {+ return { kind: "unchanged", stepInput: input.stepInput };+ }++ const requests = extractHistoricalInputRequests({+ history: input.history,+ requestIds: new Set(staleResponses.map((response) => response.requestId)),+ });+ const modelMessage = appendOptionalUserContent(+ input.stepInput.message,+ formatModelMessage(staleResponses, requests),+ );+ const displayMessage = appendOptionalUserContent(+ input.stepInput.message,+ formatDisplayMessage(staleResponses, requests),+ );+ const { inputResponses: _responses, ...remainingInput } = input.stepInput;+ const stepInput: { -readonly [K in keyof StepInput]: StepInput[K] } = {+ ...remainingInput,+ message: modelMessage,+ };+ if (currentResponses.length > 0) {+ stepInput.inputResponses = currentResponses;+ }++ return { displayMessage, kind: "converted", stepInput };+}++function formatModelMessage(+ responses: readonly InputResponse[],+ requests: ReadonlyMap<string, InputRequest>,+): string {+ const resolvedResponses = responses.map((response) => {+ const request = requests.get(response.requestId);+ const option = request?.options?.find((candidate) => candidate.id === response.optionId);++ const responseDetails: {+ optionId?: string;+ selectedOption?: { description?: string; id: string; label: string };+ text?: string;+ } = {};+ if (response.optionId !== undefined) {+ responseDetails.optionId = response.optionId;+ }+ if (option !== undefined) {+ const selectedOption: { description?: string; id: string; label: string } = {+ id: option.id,+ label: option.label,+ };+ if (option.description !== undefined) {+ selectedOption.description = option.description;+ }+ responseDetails.selectedOption = selectedOption;+ }+ if (response.text !== undefined) {+ responseDetails.text = response.text;+ }++ const resolved: {+ prompt?: string;+ requestId: string;+ requestType?: "approval" | "question";+ response: typeof responseDetails;+ } = { requestId: response.requestId, response: responseDetails };+ if (request !== undefined) {+ resolved.prompt = request.prompt;+ resolved.requestType = isApprovalRequest(request) ? "approval" : "question";+ }++ return resolved;+ });+ // Request metadata can be missing (compacted history, subagent-proxied+ // request), so a response without it may still be an approval: default to+ // including the notice.+ const mayIncludeApproval = responses.some((response) => {+ const request = requests.get(response.requestId);+ return request === undefined || isApprovalRequest(request);+ });+ const approvalNotice = mayIncludeApproval+ ? " This does not authorize an earlier action; request approval again if that action is still needed."+ : "";++ return [+ "The user submitted the following response to an earlier interactive prompt.",+ `Treat it as new input at the current point in the conversation and decide whether it is still relevant.${approvalNotice}`,+ JSON.stringify(resolvedResponses, null, 2),+ ].join("\n");+}++function formatDisplayMessage(+ responses: readonly InputResponse[],+ requests: ReadonlyMap<string, InputRequest>,+): string {+ return responses+ .map((response) => {+ if (response.text !== undefined && response.text.length > 0) {+ return response.text;+ }++ const option = requests+ .get(response.requestId)+ ?.options?.find((candidate) => candidate.id === response.optionId);+ return option?.label ?? response.optionId ?? "Response to an earlier interactive prompt";+ })+ .join("\n");+}++function appendOptionalUserContent(+ existing: string | UserContent | undefined,+ appended: string,+): string | UserContent {+ if (existing === undefined) {+ return appended;+ }++ return appendUserContent({ appended, existing });+}diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex d1f24a33d..db3d624ff 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -36,7 +36,11 @@ import {modelFacingAuthorizationOutput,requestAuthorization,} from "#harness/authorization.js";-import { hasDeferredStepInput, setPendingInputBatch } from "#harness/input-requests.js";+import {+ hasDeferredStepInput,+ hasPendingInputBatch,+ setPendingInputBatch,+} from "#harness/input-requests.js";import { getPendingRuntimeActionBatch } from "#harness/runtime-actions.js";import { stashToolInterrupt } from "#harness/tool-interrupts.js";import { createToolLoopHarness } from "#harness/tool-loop.js";@@ -7052,6 +7056,232 @@ describe("createToolLoopHarness", () => {});});+ it("delivers a stale ask_question selection as a new user turn while another question is pending", async () => {+ const nextQuestionInput = {+ allowFreeform: false,+ options: [+ { id: "alpha", label: "Use alpha" },+ { id: "beta", label: "Use beta" },+ ],+ prompt: "Which new context should I use?",+ };+ setupMockAgent({+ content: [],+ finishReason: "tool-calls",+ response: {+ messages: [+ {+ content: [+ {+ input: nextQuestionInput,+ toolCallId: "question-2",+ toolName: "ask_question",+ type: "tool-call",+ },+ ],+ role: "assistant",+ },+ ],+ },+ text: "",+ toolCalls: [+ {+ input: nextQuestionInput,+ toolCallId: "question-2",+ toolName: "ask_question",+ type: "tool-call",+ },+ ],+ toolResults: [],+ });+ const nextQuestionImplementation = vi.mocked(ToolLoopAgent).getMockImplementation();+ setupMockAgent({+ finishReason: "stop",+ response: { messages: [{ content: "I will use the candidate.", role: "assistant" }] },+ text: "I will use the candidate.",+ toolCalls: [],+ toolResults: [],+ });+ vi.mocked(ToolLoopAgent).mockImplementationOnce(nextQuestionImplementation!);++ const { emit, events } = createEventCollector();+ const runStep = createToolLoopHarness(createTestConfig("conversation", emit));+ const questionInput = {+ allowFreeform: true,+ options: [+ {+ description: "Use the current conversation context.",+ id: "current",+ label: "Use current context",+ },+ {+ description: "Use the candidate from the earlier question.",+ id: "candidate",+ label: "Use STALE-CANDIDATE-7Q4M",+ },+ ],+ prompt: "Which context should I use?",+ };+ const session = setPendingInputBatch({+ requests: [+ {+ action: {+ callId: "question-1",+ input: questionInput,+ kind: "tool-call",+ toolName: "ask_question",+ },+ allowFreeform: true,+ display: "select",+ options: questionInput.options,+ prompt: questionInput.prompt,+ requestId: "question-1",+ },+ ],+ responseMessages: [+ {+ content: [+ {+ input: questionInput,+ toolCallId: "question-1",+ toolName: "ask_question",+ type: "tool-call",+ },+ ],+ role: "assistant",+ },+ ],+ session: createTestSession({+ history: [{ content: "Help me choose.", role: "user" }],+ }),+ });++ const followupResult = await runStep(session, {+ message: "Use current context instead.",+ });++ expect(followupResult.next).toBeNull();+ expect(hasPendingInputBatch(followupResult.session.state)).toBe(true);++ const secondTurnEventIndex = events.length;+ const result = await runStep(followupResult.session, {+ inputResponses: [{ requestId: "question-1", optionId: "candidate" }],+ });++ const agent = vi.mocked(ToolLoopAgent).mock.results.at(-1)?.value;+ if (agent === undefined) {+ throw new Error("ToolLoopAgent mock did not return an instance.");+ }+ const modelMessages = vi.mocked(agent.stream).mock.calls[0]?.[0].messages;++ expect(result.next).toBeNull();+ expect(hasPendingInputBatch(result.session.state)).toBe(false);+ expect(modelMessages?.at(-1)).toEqual({+ content: expect.stringContaining("STALE-CANDIDATE-7Q4M"),+ role: "user",+ });+ expect(+ events.slice(secondTurnEventIndex).find((event) => event.type === "message.received"),+ ).toMatchObject({+ data: {+ message: "Use STALE-CANDIDATE-7Q4M",+ },+ });+ });++ it("delivers a stale ask_question selection as a new user turn when nothing is pending", async () => {+ setupMockAgent({+ finishReason: "stop",+ response: { messages: [{ content: "Understood.", role: "assistant" }] },+ text: "Understood.",+ toolCalls: [],+ toolResults: [],+ });++ const { emit, events } = createEventCollector();+ const runStep = createToolLoopHarness(createTestConfig("conversation", emit));+ const questionInput = {+ allowFreeform: true,+ options: [+ { id: "current", label: "Use current context" },+ { id: "candidate", label: "Use STALE-CANDIDATE-7Q4M" },+ ],+ prompt: "Which context should I use?",+ };+ const session = setPendingInputBatch({+ requests: [+ {+ action: {+ callId: "question-1",+ input: questionInput,+ kind: "tool-call",+ toolName: "ask_question",+ },+ allowFreeform: true,+ display: "select",+ options: questionInput.options,+ prompt: questionInput.prompt,+ requestId: "question-1",+ },+ ],+ responseMessages: [+ {+ content: [+ {+ input: questionInput,+ toolCallId: "question-1",+ toolName: "ask_question",+ type: "tool-call",+ },+ ],+ role: "assistant",+ },+ ],+ session: createTestSession({+ history: [{ content: "Help me choose.", role: "user" }],+ }),+ });++ // The follow-up resolves question-1 as freeform; the turn completes with+ // nothing pending.+ const followupResult = await runStep(session, {+ message: "Use current context instead.",+ });++ expect(followupResult.next).toBeNull();+ expect(hasPendingInputBatch(followupResult.session.state)).toBe(false);++ const secondTurnEventIndex = events.length;+ const result = await runStep(followupResult.session, {+ inputResponses: [{ requestId: "question-1", optionId: "candidate" }],+ });++ const agent = vi.mocked(ToolLoopAgent).mock.results.at(-1)?.value;+ if (agent === undefined) {+ throw new Error("ToolLoopAgent mock did not return an instance.");+ }+ const modelMessages = vi.mocked(agent.stream).mock.calls[0]?.[0].messages;++ expect(result.next).toBeNull();+ expect(hasPendingInputBatch(result.session.state)).toBe(false);+ expect(modelMessages?.at(-1)).toEqual({+ content: expect.stringContaining("STALE-CANDIDATE-7Q4M"),+ role: "user",+ });+ // The stale selection must not append a second tool result for+ // question-1: only the freeform answer from the follow-up turn exists.+ expect(modelMessages?.filter((message: ModelMessage) => message.role === "tool")).toHaveLength(+ 1,+ );+ expect(+ events.slice(secondTurnEventIndex).find((event) => event.type === "message.received"),+ ).toMatchObject({+ data: {+ message: "Use STALE-CANDIDATE-7Q4M",+ },+ });+ });+it("emits compaction.requested and compaction.completed when compaction triggers", async () => {vi.mocked(shouldCompact).mockReturnValue(true);vi.mocked(compactMessages).mockResolvedValue([diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.tsindex 2838eea0a..c1ef01486 100644--- a/packages/eve/src/harness/tool-loop.ts+++ b/packages/eve/src/harness/tool-loop.ts@@ -101,11 +101,13 @@ import { buildTelemetryRuntimeContext } from "#harness/instrumentation-runtime-cimport {consumeDeferredStepInput,getApprovedTools,+ getPendingInputRequestIds,hasDeferredStepInput,hasStepInput,resolvePendingInput,setPendingInputBatch,} from "#harness/input-requests.js";+import { convertStaleResponsesToUserMessage } from "#harness/stale-input-responses.js";import { getInstrumentationConfig } from "#harness/instrumentation-config.js";import { resolveAssistantStepText } from "#harness/messages.js";import { normalizeProviderToolHistory } from "#harness/provider-tool-history.js";@@ -540,17 +542,28 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {}session = resolvedRuntimeActions.session;+ const staleConversion = convertStaleResponsesToUserMessage({+ history: resolvedRuntimeActions.messages,+ pendingRequestIds: getPendingInputRequestIds(session.state),+ stepInput: stepInput.input,+ });+ const effectiveStepInput = staleConversion.stepInput;+ const preambleStepInput =+ staleConversion.kind === "converted"+ ? { ...effectiveStepInput, message: staleConversion.displayMessage }+ : effectiveStepInput;+const pending = resolvePendingInput({history: resolvedRuntimeActions.messages,resolveApprovalKey: resolveApprovalKeyFromTools(config.tools),session,- stepInput: stepInput.input,+ stepInput: effectiveStepInput,});if (pending.outcome === "unresolved") {if (emit && pending.deferredMessage === true && hasStepInput(input)) {emissionState = await emitTurnPreamble(emit,- input ?? {},+ preambleStepInput ?? {},emissionState,config.runtimeIdentity,);@@ -592,7 +605,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {if (emit && hasStepInput(input)) {emissionState = await emitTurnPreamble(emit,- input ?? {},+ preambleStepInput ?? {},emissionState,config.runtimeIdentity,);@@ -620,14 +633,14 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {}session = continuation.session;- if (stepInput.input?.context !== undefined && pending.deferredContext !== true) {- for (const entry of stepInput.input.context) {+ if (effectiveStepInput?.context !== undefined && pending.deferredContext !== true) {+ for (const entry of effectiveStepInput.context) {messages.push({ content: entry, role: "user" });}}if (- stepInput.input?.message !== undefined &&+ effectiveStepInput?.message !== undefined &&!pending.deferredMessage &&!pending.consumedMessage) {@@ -636,7 +649,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {// `messages` array — and everything that flows into// `session.history` from it — therefore never carries raw// attachment bytes across step boundaries.- const content = await stageAttachmentsToSandbox(stepInput.input.message);+ const content = await stageAttachmentsToSandbox(effectiveStepInput.message);messages.push({ content, role: "user" });}@@ -1008,7 +1021,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {// Workflow continuations replay the sandbox after step.started so nested// action lifecycle events keep the active turn's emission coordinates.const pendingWorkflowInterrupt = await continuePendingWorkflowInterrupt({- childResults: stepInput.input?.runtimeActionResults,+ childResults: effectiveStepInput?.runtimeActionResults,config,emit,emissionState,