The Open Framework for Building Agents
fix(eve): include message after cancelled approval (#4396)
Casey Gowrie77c2d3ecd0e3
diff --git a/.changeset/include-message-after-cancelled-approval.md b/.changeset/include-message-after-cancelled-approval.mdnew file mode 100644index 000000000..3c46a4255--- /dev/null+++ b/.changeset/include-message-after-cancelled-approval.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Include a new message in the first model step after cancelling a turn that was waiting for tool approval, instead of first replying to the cancelled call alone.diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/cancelled-approval-followup.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/cancelled-approval-followup.eval.tsnew file mode 100644index 000000000..15ed1d45c--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/cancelled-approval-followup.eval.ts@@ -0,0 +1,42 @@+import { defineEval } from "eve/evals";+import { satisfies } from "eve/evals/expect";++const TOOL_NAME = "gate";+const FOLLOWUP = "CANCELLED-APPROVAL-FOLLOW-UP-OK";++export default defineEval({+ description: "A message after cancelling an approval turn runs in the next turn's first step.",+ async test(t) {+ const held = await t.send(+ `Call the ${TOOL_NAME} tool exactly once with marker "cancelled-approval".`,+ );+ const session = held.session;+ held.calledTool(TOOL_NAME, { count: 1, status: "pending" });+ session.requireInputRequest({ display: "confirmation", toolName: TOOL_NAME });++ const cancellation = t.target.watchTurn(session.sessionId, {+ startIndex: session.state.streamIndex,+ });+ const accepted = await session.cancel();+ await t.require(+ accepted,+ satisfies(+ (value: typeof accepted) => value.status === "accepted",+ "the approval turn cancellation is accepted",+ ),+ );+ const cancelled = await cancellation.result();+ cancelled.event("turn.cancelled", { count: 1 });+ cancelled.eventOrder([{ type: "turn.cancelled" }, { type: "session.waiting" }]);+ cancelled.notEvent("turn.failed");++ const followup = await session.send(`Reply with exactly ${FOLLOWUP}.`);++ followup.expectOk();+ followup.messageIncludes(FOLLOWUP);+ followup.event("step.started", { count: 1 });+ followup.event("message.completed", { count: 1 });+ followup.notEvent("turn.failed");+ t.succeeded();+ },+});diff --git a/packages/eve/src/harness/input-requests.test.ts b/packages/eve/src/harness/input-requests.test.tsindex df74bfffb..693569807 100644--- a/packages/eve/src/harness/input-requests.test.ts+++ b/packages/eve/src/harness/input-requests.test.ts@@ -17,6 +17,7 @@ import {resolvePendingInput,appendPendingInputBatch,} from "#harness/input-requests.js";+import { cancelApprovalInputBatches } from "#harness/hitl/approval-input-requests.js";import { getDeferredStepInput } from "#harness/pending-input-batches.js";import { createSessionLimitContinuationRequest } from "#harness/session-limit-continuation.js";import { createRuntimeToolCallActionFromToolCall } from "#harness/tool-call-action.js";@@ -189,6 +190,84 @@ describe("resolvePendingInput", () => {expect(getDeferredStepInput(deferred.session)).toBeUndefined();});+ it("includes a new message immediately after a cancelled approval turn", () => {+ const pending = appendPendingInputBatch({+ requests: [+ {+ action: {+ callId: "approval-call",+ input: { command: "rm -rf /tmp/demo" },+ kind: "tool-call",+ toolName: "bash",+ },+ allowFreeform: false,+ display: "confirmation",+ kind: "tool-approval",+ options: [+ { id: "approve", label: "Yes" },+ { id: "cancel", label: "No" },+ ],+ prompt: "Approve tool call: bash",+ requestId: "approval-1",+ } satisfies InputRequest,+ ],+ responseMessages: [+ {+ content: [+ {+ input: { command: "rm -rf /tmp/demo" },+ toolCallId: "approval-call",+ toolName: "bash",+ type: "tool-call",+ },+ {+ approvalId: "approval-1",+ toolCallId: "approval-call",+ type: "tool-approval-request",+ },+ ],+ role: "assistant",+ } satisfies ModelMessage,+ ],+ session: createHarnessSession(),+ });+ const session = cancelApprovalInputBatches(pending);++ const result = resolvePendingInput({+ session,+ stepInput: { message: "Answer this in the first step." },+ });++ expect(result.outcome).toBe("continue");+ expect(result.deferredMessage).toBeUndefined();+ expect(getDeferredStepInput(result.session)).toBeUndefined();+ });++ it("still defers a new message after an approved response", () => {+ const session: HarnessSession = {+ ...createHarnessSession(),+ history: [+ ...createHarnessSession().history,+ {+ content: [+ {+ approvalId: "approval-1",+ approved: true,+ type: "tool-approval-response",+ },+ ],+ role: "tool",+ },+ ],+ };++ const result = resolvePendingInput({ session, stepInput: { message: "Wait for the tool." } });++ expect(result.outcome).toBe("resolved");+ expect(result.deferredMessage).toBe(true);+ expect(getDeferredStepInput(result.session)).toBeDefined();+ });+it("defers channel context until after tool approvals are resolved", () => {const session = appendPendingInputBatch({requests: [diff --git a/packages/eve/src/harness/input-requests.ts b/packages/eve/src/harness/input-requests.tsindex be95bb3e6..785dcfedb 100644--- a/packages/eve/src/harness/input-requests.ts+++ b/packages/eve/src/harness/input-requests.ts@@ -116,7 +116,11 @@ export function resolvePendingInput(input: {const route = routePendingInput(batches);// Finish already-approved work before another batch or user message can hide// the approval response from the SDK. Session-limit prompts still take priority.- if (route.kind === "approvals" && hasTailApprovalResponse(baseHistory)) {+ if (+ route.kind === "approvals" &&+ hasTailApprovalResponse(baseHistory) &&+ (batches.length > 0 || hasApprovedTailResponse(baseHistory))+ ) {return finishResolvedInput({deferTurnInput: true,leftoverResponses: input.stepInput?.inputResponses ?? [],@@ -199,6 +203,14 @@ function routePendingInput(batches: readonly PendingInputBatch[]): PendingInputRreturn { kind: "approvals" };}+function hasApprovedTailResponse(messages: readonly ModelMessage[]): boolean {+ const tail = messages.at(-1);+ return (+ tail?.role === "tool" &&+ tail.content.some((part) => part.type === "tool-approval-response" && part.approved)+ );+}+function canonicalizeInputResponses(responses: readonly InputResponse[]): readonly InputResponse[] {const byRequestId = new Map<string, InputResponse>();for (const response of responses) byRequestId.set(response.requestId, response);