Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): include message after cancelled approval (#4396)

Casey Gowrie77c2d3ecd0e3
diff --git a/.changeset/include-message-after-cancelled-approval.md b/.changeset/include-message-after-cancelled-approval.md
new file mode 100644
index 000000000..3c46a4255
--- /dev/null
+++ b/.changeset/include-message-after-cancelled-approval.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Include a new message in the first model step after cancelling a turn that was waiting for tool approval, instead of first replying to the cancelled call alone.
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/cancelled-approval-followup.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/cancelled-approval-followup.eval.ts
new file mode 100644
index 000000000..15ed1d45c
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/cancelled-approval-followup.eval.ts
@@ -0,0 +1,42 @@
+import { defineEval } from "eve/evals";
+import { satisfies } from "eve/evals/expect";
+
+const TOOL_NAME = "gate";
+const FOLLOWUP = "CANCELLED-APPROVAL-FOLLOW-UP-OK";
+
+export default defineEval({
+ description: "A message after cancelling an approval turn runs in the next turn's first step.",
+ async test(t) {
+ const held = await t.send(
+ `Call the ${TOOL_NAME} tool exactly once with marker "cancelled-approval".`,
+ );
+ const session = held.session;
+ held.calledTool(TOOL_NAME, { count: 1, status: "pending" });
+ session.requireInputRequest({ display: "confirmation", toolName: TOOL_NAME });
+
+ const cancellation = t.target.watchTurn(session.sessionId, {
+ startIndex: session.state.streamIndex,
+ });
+ const accepted = await session.cancel();
+ await t.require(
+ accepted,
+ satisfies(
+ (value: typeof accepted) => value.status === "accepted",
+ "the approval turn cancellation is accepted",
+ ),
+ );
+ const cancelled = await cancellation.result();
+ cancelled.event("turn.cancelled", { count: 1 });
+ cancelled.eventOrder([{ type: "turn.cancelled" }, { type: "session.waiting" }]);
+ cancelled.notEvent("turn.failed");
+
+ const followup = await session.send(`Reply with exactly ${FOLLOWUP}.`);
+
+ followup.expectOk();
+ followup.messageIncludes(FOLLOWUP);
+ followup.event("step.started", { count: 1 });
+ followup.event("message.completed", { count: 1 });
+ followup.notEvent("turn.failed");
+ t.succeeded();
+ },
+});
diff --git a/packages/eve/src/harness/input-requests.test.ts b/packages/eve/src/harness/input-requests.test.ts
index df74bfffb..693569807 100644
--- a/packages/eve/src/harness/input-requests.test.ts
+++ b/packages/eve/src/harness/input-requests.test.ts
@@ -17,6 +17,7 @@ import {
resolvePendingInput,
appendPendingInputBatch,
} from "#harness/input-requests.js";
+import { cancelApprovalInputBatches } from "#harness/hitl/approval-input-requests.js";
import { getDeferredStepInput } from "#harness/pending-input-batches.js";
import { createSessionLimitContinuationRequest } from "#harness/session-limit-continuation.js";
import { createRuntimeToolCallActionFromToolCall } from "#harness/tool-call-action.js";
@@ -189,6 +190,84 @@ describe("resolvePendingInput", () => {
expect(getDeferredStepInput(deferred.session)).toBeUndefined();
});
+ it("includes a new message immediately after a cancelled approval turn", () => {
+ const pending = appendPendingInputBatch({
+ requests: [
+ {
+ action: {
+ callId: "approval-call",
+ input: { command: "rm -rf /tmp/demo" },
+ kind: "tool-call",
+ toolName: "bash",
+ },
+ allowFreeform: false,
+ display: "confirmation",
+ kind: "tool-approval",
+ options: [
+ { id: "approve", label: "Yes" },
+ { id: "cancel", label: "No" },
+ ],
+ prompt: "Approve tool call: bash",
+ requestId: "approval-1",
+ } satisfies InputRequest,
+ ],
+ responseMessages: [
+ {
+ content: [
+ {
+ input: { command: "rm -rf /tmp/demo" },
+ toolCallId: "approval-call",
+ toolName: "bash",
+ type: "tool-call",
+ },
+ {
+ approvalId: "approval-1",
+ toolCallId: "approval-call",
+ type: "tool-approval-request",
+ },
+ ],
+ role: "assistant",
+ } satisfies ModelMessage,
+ ],
+ session: createHarnessSession(),
+ });
+ const session = cancelApprovalInputBatches(pending);
+
+ const result = resolvePendingInput({
+ session,
+ stepInput: { message: "Answer this in the first step." },
+ });
+
+ expect(result.outcome).toBe("continue");
+ expect(result.deferredMessage).toBeUndefined();
+ expect(getDeferredStepInput(result.session)).toBeUndefined();
+ });
+
+ it("still defers a new message after an approved response", () => {
+ const session: HarnessSession = {
+ ...createHarnessSession(),
+ history: [
+ ...createHarnessSession().history,
+ {
+ content: [
+ {
+ approvalId: "approval-1",
+ approved: true,
+ type: "tool-approval-response",
+ },
+ ],
+ role: "tool",
+ },
+ ],
+ };
+
+ const result = resolvePendingInput({ session, stepInput: { message: "Wait for the tool." } });
+
+ expect(result.outcome).toBe("resolved");
+ expect(result.deferredMessage).toBe(true);
+ expect(getDeferredStepInput(result.session)).toBeDefined();
+ });
+
it("defers channel context until after tool approvals are resolved", () => {
const session = appendPendingInputBatch({
requests: [
diff --git a/packages/eve/src/harness/input-requests.ts b/packages/eve/src/harness/input-requests.ts
index be95bb3e6..785dcfedb 100644
--- a/packages/eve/src/harness/input-requests.ts
+++ b/packages/eve/src/harness/input-requests.ts
@@ -116,7 +116,11 @@ export function resolvePendingInput(input: {
const route = routePendingInput(batches);
// Finish already-approved work before another batch or user message can hide
// the approval response from the SDK. Session-limit prompts still take priority.
- if (route.kind === "approvals" && hasTailApprovalResponse(baseHistory)) {
+ if (
+ route.kind === "approvals" &&
+ hasTailApprovalResponse(baseHistory) &&
+ (batches.length > 0 || hasApprovedTailResponse(baseHistory))
+ ) {
return finishResolvedInput({
deferTurnInput: true,
leftoverResponses: input.stepInput?.inputResponses ?? [],
@@ -199,6 +203,14 @@ function routePendingInput(batches: readonly PendingInputBatch[]): PendingInputR
return { kind: "approvals" };
}
+function hasApprovedTailResponse(messages: readonly ModelMessage[]): boolean {
+ const tail = messages.at(-1);
+ return (
+ tail?.role === "tool" &&
+ tail.content.some((part) => part.type === "tool-approval-response" && part.approved)
+ );
+}
+
function canonicalizeInputResponses(responses: readonly InputResponse[]): readonly InputResponse[] {
const byRequestId = new Map<string, InputResponse>();
for (const response of responses) byRequestId.set(response.requestId, response);