Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): HITL approvals - repair resume contract (#373)

Andrew Barba8713a71a4ec6
diff --git a/.changeset/hitl-approval-resume-contract.md b/.changeset/hitl-approval-resume-contract.md
new file mode 100644
index 000000000..5b1a47cb6
--- /dev/null
+++ b/.changeset/hitl-approval-resume-contract.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Fix human-in-the-loop approval resume behavior so text replies like `approve` resolve pending tool approvals and unrelated follow-up messages no longer synthesize a denial. Rejected approval results now include explicit approval and not-run metadata for clients.
diff --git a/docs/concepts/sessions-runs-and-streaming.md b/docs/concepts/sessions-runs-and-streaming.md
index 3ec193195..5689628ce 100644
--- a/docs/concepts/sessions-runs-and-streaming.md
+++ b/docs/concepts/sessions-runs-and-streaming.md
@@ -84,6 +84,8 @@ curl -X POST http://127.0.0.1:3000/eve/v1/session/<sessionId> \
The follow-up reuses the same durable session: same history, same state.
+If the session is waiting on a human-in-the-loop approval, a matching text reply such as `approve` or `deny` answers the approval. Other follow-up text is held until the approval is answered, so an unrelated message does not implicitly deny the pending tool call.
+
For deterministic ordering, send one follow-up at a time and wait for the next `session.waiting` event before sending another message to the same session. See [message delivery and queueing](./execution-model-and-durability#message-delivery-and-queueing) for the current runtime contract.
## Reconnect and rewind
diff --git a/docs/tools/human-in-the-loop.md b/docs/tools/human-in-the-loop.md
index 534fdb6ed..250c45a8a 100644
--- a/docs/tools/human-in-the-loop.md
+++ b/docs/tools/human-in-the-loop.md
@@ -100,10 +100,12 @@ Approvals and questions share one protocol:
1. The model requests input (an approval, or an `ask_question`).
2. eve emits an `input.requested` stream event carrying the pending requests.
3. The turn parks at `session.waiting`, durably, for as long as it takes.
-4. The client answers with `inputResponses` (structured, keyed by `requestId`) or a normal follow-up `message`. A follow-up whose text matches an option label (case-insensitive) resolves automatically.
+4. The client answers with `inputResponses` (structured, keyed by `requestId`) or a normal follow-up `message`. A follow-up whose text matches an option ID, option label, or numeric option index resolves automatically, including approval options such as `approve` and `deny`.
The run picks back up exactly where it parked. Because the pause is durable, nothing is held in memory while it waits — the process can restart and the parked turn survives.
+For approval requests, unrelated follow-up text does not deny the tool call. eve keeps the approval pending and holds that text until the approval is answered, then replays it as the next message in the session.
+
See [Sessions, runs & streaming](/docs/concepts/sessions-runs-and-streaming) for the full event and resume contract that this builds on.
## Answering from a client or channel
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/deny-then-regate.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/deny-then-regate.eval.ts
index 49a345bfb..0a01180d5 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/deny-then-regate.eval.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/deny-then-regate.eval.ts
@@ -9,12 +9,23 @@ export default defineEval({
description: "HITL smoke: a denied once() call does not execute and re-gates the next call.",
async test(t) {
await t.send('Call the guarded-echo tool with note "denied-call".');
- t.requireInputRequest({ toolName: "guarded-echo" });
+ const request = t.requireInputRequest({ toolName: "guarded-echo" });
const denied = await t.respondAll("deny");
denied.expectOk();
denied.event("action.result", {
- data: { result: { kind: "tool-result", toolName: "guarded-echo" }, status: "rejected" },
+ data: {
+ result: {
+ kind: "tool-result",
+ output: {
+ approval: { requestId: request.requestId, status: "denied" },
+ code: "TOOL_EXECUTION_DENIED",
+ tool: { result: "not_run" },
+ },
+ toolName: "guarded-echo",
+ },
+ status: "rejected",
+ },
count: 1,
});
// The denial returns to the model as context; real models paraphrase it,
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/text-approve.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/text-approve.eval.ts
new file mode 100644
index 000000000..960650866
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/text-approve.eval.ts
@@ -0,0 +1,37 @@
+import { defineEval } from "eve/evals";
+
+import { GUARDED_ECHO_TOKEN } from "./shared.js";
+
+/**
+ * HITL flow: a plain follow-up message whose text matches an approval option
+ * resolves the pending approval the same way as structured inputResponses.
+ */
+export default defineEval({
+ description: "HITL smoke: text approve resolves a pending tool approval.",
+ async test(t) {
+ const parked = await t.send('Call the guarded-echo tool with note "text-approve".');
+ parked.calledTool("guarded-echo", { status: "pending", count: 1 });
+ t.requireInputRequest({ display: "confirmation", toolName: "guarded-echo" });
+
+ const approved = await t.send("approve");
+ approved.expectOk();
+ approved.event("action.result", {
+ data: {
+ result: {
+ kind: "tool-result",
+ output: new RegExp(GUARDED_ECHO_TOKEN),
+ toolName: "guarded-echo",
+ },
+ status: "completed",
+ },
+ count: 1,
+ });
+
+ t.succeeded();
+ t.calledTool("guarded-echo", {
+ output: new RegExp(GUARDED_ECHO_TOKEN),
+ status: "completed",
+ count: 1,
+ });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/unrelated-message-queued.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/unrelated-message-queued.eval.ts
new file mode 100644
index 000000000..62787885e
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/unrelated-message-queued.eval.ts
@@ -0,0 +1,49 @@
+import { defineEval } from "eve/evals";
+
+import { GUARDED_ECHO_TOKEN } from "./shared.js";
+
+/**
+ * HITL flow: unrelated text sent while an approval is pending must not deny
+ * the approval or disappear. The message is replayed after the approval
+ * receives an explicit terminal answer.
+ */
+export default defineEval({
+ description: "HITL smoke: unrelated message during approval is queued.",
+ async test(t) {
+ const parked = await t.send('Call the guarded-echo tool with note "queued-approval".');
+ parked.calledTool("guarded-echo", { status: "pending", count: 1 });
+ const request = t.requireInputRequest({
+ display: "confirmation",
+ toolName: "guarded-echo",
+ });
+
+ const queued = await t.send(
+ "After the pending approval is resolved, reply with exactly QUEUED-HITL-OK.",
+ );
+ queued.expectOk();
+ queued.notEvent("action.result", {
+ data: { result: { toolName: "guarded-echo" }, status: "rejected" },
+ });
+ queued.event("session.waiting", { count: 1 });
+
+ const approved = await t.respond({
+ requestId: request.requestId,
+ optionId: "approve",
+ });
+ approved.expectOk();
+ approved.event("action.result", {
+ data: {
+ result: {
+ kind: "tool-result",
+ output: new RegExp(GUARDED_ECHO_TOKEN),
+ toolName: "guarded-echo",
+ },
+ status: "completed",
+ },
+ count: 1,
+ });
+ approved.messageIncludes(/QUEUED-HITL-OK/i);
+
+ t.succeeded();
+ },
+});
diff --git a/packages/eve/src/harness/input-requests.test.ts b/packages/eve/src/harness/input-requests.test.ts
index e364c8b97..06d78ea51 100644
--- a/packages/eve/src/harness/input-requests.test.ts
+++ b/packages/eve/src/harness/input-requests.test.ts
@@ -101,7 +101,7 @@ describe("createRuntimeToolCallActionFromToolCall", () => {
});
describe("resolvePendingInput", () => {
- it("resolves pending question input with responses", () => {
+ it("keeps approvals pending when another request is answered first", () => {
const session = setPendingInputBatch({
requests: [
{
@@ -161,52 +161,22 @@ describe("resolvePendingInput", () => {
session,
});
- const pendingResponseMessage = (
- session.state?.["eve.runtime.pendingInputBatch"] as
- | { responseMessages?: readonly ModelMessage[] }
- | undefined
- )?.responseMessages?.[0];
+ expect(result.outcome).toBe("unresolved");
+ expect(result.messages).toEqual([{ content: "previous", role: "user" }]);
+ expect(hasDeferredStepInput(result.session)).toBe(true);
- expect(result.outcome).toBe("resolved");
- expect(result.messages).toEqual([
- { content: "previous", role: "user" },
- pendingResponseMessage,
- {
- content: [
- {
- output: {
- type: "json",
- value: {
- optionId: "yes",
- status: "answered",
- },
- },
- toolCallId: "question-call",
- toolName: "ask_question",
- type: "tool-result",
- },
- {
- approvalId: "approval-1",
- approved: false,
- reason: "Ignored because the user continued without responding.",
- type: "tool-approval-response",
- },
- {
- output: {
- type: "execution-denied",
- reason: "Ignored because the user continued without responding.",
- },
- toolCallId: "approval-call",
- toolName: "bash",
- type: "tool-result",
- },
- ],
- role: "tool",
- },
- ]);
+ const deferred = consumeDeferredStepInput({ session: result.session });
+ expect(deferred.input).toEqual({
+ inputResponses: [
+ {
+ requestId: "question-call",
+ optionId: "yes",
+ },
+ ],
+ });
});
- it("synthesizes ignored responses before a follow-up message", () => {
+ it("resolves freeform question input from a follow-up message", () => {
const session = setPendingInputBatch({
requests: [
{
@@ -238,8 +208,6 @@ describe("resolvePendingInput", () => {
session: createHarnessSession(),
});
- // A message-only delivery with no inputResponses — the pending batch
- // is auto-ignored so the model can continue.
const result = resolvePendingInput({
stepInput: {
message: "Ignore that and continue.",
@@ -254,7 +222,9 @@ describe("resolvePendingInput", () => {
output: {
type: "json",
value: {
- status: "ignored",
+ optionId: undefined,
+ text: "Ignore that and continue.",
+ status: "answered",
},
},
toolCallId: "question-call",
@@ -333,6 +303,70 @@ describe("resolvePendingInput", () => {
expect(hasDeferredStepInput(deferred.session)).toBe(false);
});
+ it("resolves approval when follow-up text matches an option", () => {
+ const session = setPendingInputBatch({
+ requests: [
+ {
+ action: {
+ callId: "approval-call",
+ input: { command: "pwd" },
+ kind: "tool-call",
+ toolName: "bash",
+ },
+ allowFreeform: false,
+ display: "confirmation",
+ options: [
+ { id: "approve", label: "Yes" },
+ { id: "deny", label: "No" },
+ ],
+ prompt: "Approve tool call: bash",
+ requestId: "approval-1",
+ } satisfies InputRequest,
+ ],
+ responseMessages: [
+ {
+ content: [
+ {
+ input: { command: "pwd" },
+ toolCallId: "approval-call",
+ toolName: "bash",
+ type: "tool-call",
+ },
+ {
+ approvalId: "approval-1",
+ toolCallId: "approval-call",
+ type: "tool-approval-request",
+ },
+ ],
+ role: "assistant",
+ } satisfies ModelMessage,
+ ],
+ session: createHarnessSession(),
+ });
+
+ const result = resolvePendingInput({
+ stepInput: { message: "approve" },
+ session,
+ });
+
+ expect(result.outcome).toBe("resolved");
+ expect(result.deferredMessage).toBeUndefined();
+ expect(result.consumedMessage).toBe(true);
+ expect(result.messages.at(-1)).toEqual({
+ content: [
+ {
+ approvalId: "approval-1",
+ approved: true,
+ reason: undefined,
+ type: "tool-approval-response",
+ },
+ ],
+ role: "tool",
+ });
+ expect(getApprovedTools(result.session).has("bash")).toBe(true);
+ expect(hasDeferredStepInput(result.session)).toBe(false);
+ });
+
it("records compound approval key when resolveApprovalKey is provided", () => {
const session = setPendingInputBatch({
requests: [
@@ -453,11 +487,11 @@ describe("resolvePendingInput", () => {
{
approvalId: "approval-1",
approved: false,
- reason: undefined,
+ reason: "Tool execution was denied.",
type: "tool-approval-response",
},
{
- output: { type: "execution-denied", reason: undefined },
+ output: { type: "execution-denied", reason: "Tool execution was denied." },
toolCallId: "approval-call",
toolName: "bash",
type: "tool-result",
@@ -508,8 +542,15 @@ describe("resolvePendingInput", () => {
isError: true,
kind: "tool-result",
output: {
+ approval: {
+ requestId: "approval-1",
+ status: "denied",
+ },
code: "TOOL_EXECUTION_DENIED",
message: "Tool execution was denied.",
+ tool: {
+ result: "not_run",
+ },
},
toolName: "bash",
},
@@ -553,7 +594,7 @@ describe("resolvePendingInput", () => {
expect(result.rejectedActions).toBeUndefined();
});
- it("returns a rejected action when a pending approval is auto-denied by a follow-up message", () => {
+ it("keeps a pending approval and queues an unrelated follow-up message", () => {
const session = setPendingInputBatch({
event: { sequence: 7, stepIndex: 2, turnId: "turn_1" },
requests: [
@@ -583,20 +624,15 @@ describe("resolvePendingInput", () => {
session,
});
- expect(result.outcome).toBe("resolved");
- expect(result.rejectedActions?.event).toEqual({ sequence: 7, stepIndex: 2, turnId: "turn_1" });
- expect(result.rejectedActions?.results).toEqual([
- {
- callId: "approval-call",
- isError: true,
- kind: "tool-result",
- output: {
- code: "TOOL_EXECUTION_DENIED",
- message: "Ignored because the user continued without responding.",
- },
- toolName: "bash",
- },
- ]);
+ expect(result.outcome).toBe("unresolved");
+ expect(result.rejectedActions).toBeUndefined();
+ expect(result.messages).toEqual([{ content: "previous", role: "user" }]);
+ expect(hasDeferredStepInput(result.session)).toBe(true);
+
+ const deferred = consumeDeferredStepInput({ session: result.session });
+ expect(deferred.input).toEqual({
+ message: "Never mind, do something else.",
+ });
});
it("falls back to tool name when no approvalKey is provided", () => {
diff --git a/packages/eve/src/harness/input-requests.ts b/packages/eve/src/harness/input-requests.ts
index dc8bff59f..c905d98a4 100644
--- a/packages/eve/src/harness/input-requests.ts
+++ b/packages/eve/src/harness/input-requests.ts
@@ -5,6 +5,7 @@ import type {
RuntimeToolResultActionResult,
} from "#runtime/actions/types.js";
import type { InputRequest, InputResponse } from "#runtime/input/types.js";
+import { resolveTextToResponses } from "#channel/resolve-text.js";
import { parseJsonObject } from "#shared/json.js";
import { coalesceTurnInputs } from "#harness/messages.js";
import type { HarnessSession, SessionStateMap, StepInput } from "#harness/types.js";
@@ -17,6 +18,7 @@ const IGNORED_INPUT_REASON = "Ignored because the user continued without respond
const TOOL_EXECUTION_DENIED_CODE = "TOOL_EXECUTION_DENIED";
const TOOL_EXECUTION_DENIED_MESSAGE = "Tool execution was denied.";
+const TOOL_EXECUTION_INVALID_APPROVAL_MESSAGE = "Invalid approval response.";
type ToolResponsePart = Extract<ModelMessage, { role: "tool" }>["content"][number];
@@ -48,6 +50,8 @@ export interface RejectedActionBatch {
readonly results: readonly RuntimeToolResultActionResult[];
}
+type ApprovalTerminalStatus = "approved" | "denied" | "ignored" | "invalid";
+
/**
* Returns true when the step input carries user-facing turn input.
*/
@@ -139,16 +143,25 @@ export function resolvePendingInput(input: {
}
// Pending batch exists -- only resolve if we have actual responses.
- const responses = stepInput?.inputResponses ?? [];
+ const resolvedStepInput = resolveTextMessageInput(pendingBatch, stepInput);
+ const responses = resolvedStepInput?.inputResponses ?? [];
- if (responses.length === 0 && stepInput?.message === undefined) {
+ if (responses.length === 0 && resolvedStepInput?.message === undefined) {
return { outcome: "unresolved", messages: baseHistory, session };
}
- if (responses.length === 0 && stepInput?.message !== undefined) {
- // A follow-up message arrived but no explicit responses. Auto-deny
- // all pending requests so the model can continue, and either defer
- // the message (for approval batches) or pass it through.
+ if (
+ pendingBatch.requests.some((request) => isApprovalRequest(request)) &&
+ hasUnansweredApproval({ pendingBatch, responses })
+ ) {
+ session = queueDeferredStepInput(session, compactStepInput(resolvedStepInput));
+ return { deferredMessage: true, outcome: "unresolved", messages: baseHistory, session };
+ }
+
+ if (responses.length === 0 && resolvedStepInput?.message !== undefined) {
+ // A follow-up message arrived for question-only input with no explicit
+ // responses. Keep the existing question semantics: mark unanswered
+ // question requests ignored so the model can continue with the message.
const toolParts = buildToolResponseParts(pendingBatch, []);
const messages: ModelMessage[] = [...baseHistory, ...pendingBatch.responseMessages];
if (toolParts.length > 0) {
@@ -158,14 +171,13 @@ export function resolvePendingInput(input: {
const rejectedActions = buildRejectedActionBatch(pendingBatch, []);
session = clearPendingInputBatch(session);
- if (pendingBatch.requests.some((request) => isApprovalRequest(request))) {
- session = queueDeferredStepInput(session, {
- message: stepInput.message,
- });
- return { deferredMessage: true, outcome: "resolved", messages, rejectedActions, session };
- }
-
- return { outcome: "resolved", messages, rejectedActions, session };
+ return {
+ consumedMessage: resolvedStepInput?.messageConsumed,
+ outcome: "resolved",
+ messages,
+ rejectedActions,
+ session,
+ };
}
// Record approved tools before clearing the batch.
@@ -191,20 +203,99 @@ export function resolvePendingInput(input: {
// in the same request. Defer the message so the approval is resolved in
// isolation; `consumeDeferredStepInput` replays it on the next step.
if (
- stepInput?.message !== undefined &&
+ resolvedStepInput?.message !== undefined &&
pendingBatch.requests.some((request) => isApprovalRequest(request))
) {
session = queueDeferredStepInput(session, {
- message: stepInput.message,
+ message: resolvedStepInput.message,
});
- return { deferredMessage: true, outcome: "resolved", messages, rejectedActions, session };
+ return {
+ consumedMessage: resolvedStepInput?.messageConsumed,
+ deferredMessage: true,
+ outcome: "resolved",
+ messages,
+ rejectedActions,
+ session,
+ };
+ }
+
+ return {
+ consumedMessage: resolvedStepInput?.messageConsumed,
+ outcome: "resolved",
+ messages,
+ rejectedActions,
+ session,
+ };
+}
+
+function resolveTextMessageInput(
+ pendingBatch: PendingInputBatch,
+ stepInput: StepInput | undefined,
+): (StepInput & { readonly messageConsumed?: boolean }) | undefined {
+ if (typeof stepInput?.message !== "string" || (stepInput.inputResponses?.length ?? 0) > 0) {
+ return stepInput;
+ }
+
+ const responses = resolveTextToResponses(stepInput.message, pendingBatch.requests);
+ if (responses.length === 0) {
+ return stepInput;
+ }
+
+ return compactStepInput({
+ ...stepInput,
+ inputResponses: responses,
+ messageConsumed: true,
+ message: undefined,
+ });
+}
+
+function compactStepInput(
+ input: (StepInput & { readonly messageConsumed?: boolean }) | undefined,
+): StepInput & { readonly messageConsumed?: boolean } {
+ if (input === undefined) {
+ return {};
+ }
+
+ const result: {
+ context?: StepInput["context"];
+ inputResponses?: StepInput["inputResponses"];
+ message?: StepInput["message"];
+ messageConsumed?: boolean;
+ outputSchema?: StepInput["outputSchema"];
+ } = {};
+
+ if ((input.context?.length ?? 0) > 0) {
+ result.context = input.context;
+ }
+ if ((input.inputResponses?.length ?? 0) > 0) {
+ result.inputResponses = input.inputResponses;
+ }
+ if (input.message !== undefined) {
+ result.message = input.message;
+ }
+ if (input.messageConsumed === true) {
+ result.messageConsumed = true;
+ }
+ if (input.outputSchema !== undefined) {
+ result.outputSchema = input.outputSchema;
}
- return { outcome: "resolved", messages, rejectedActions, session };
+ return result;
+}
+
+function hasUnansweredApproval(input: {
+ readonly pendingBatch: PendingInputBatch;
+ readonly responses: readonly InputResponse[];
+}): boolean {
+ const responseIds = new Set(input.responses.map((response) => response.requestId));
+ return input.pendingBatch.requests.some(
+ (request) => isApprovalRequest(request) && !responseIds.has(request.requestId),
+ );
}
type ResolvePendingInputResult = {
+ readonly consumedMessage?: boolean;
readonly deferredMessage?: boolean;
readonly outcome: "resolved" | "continue" | "unresolved";
readonly messages: ModelMessage[];
@@ -364,10 +455,36 @@ function recordApprovedTools(input: {
function resolveApprovalOutcome(response: InputResponse | undefined): {
readonly approved: boolean;
readonly reason: string | undefined;
+ readonly status: ApprovalTerminalStatus;
} {
+ if (response === undefined) {
+ return {
+ approved: false,
+ reason: IGNORED_INPUT_REASON,
+ status: "ignored",
+ };
+ }
+
+ if (response.optionId === "approve") {
+ return {
+ approved: true,
+ reason: undefined,
+ status: "approved",
+ };
+ }
+
+ if (response.optionId === "deny") {
+ return {
+ approved: false,
+ reason: TOOL_EXECUTION_DENIED_MESSAGE,
+ status: "denied",
+ };
+ }
+
return {
- approved: response?.optionId === "approve",
- reason: response === undefined ? IGNORED_INPUT_REASON : undefined,
+ approved: false,
+ reason: TOOL_EXECUTION_INVALID_APPROVAL_MESSAGE,
+ status: "invalid",
};
}
@@ -390,7 +507,7 @@ function buildRejectedActionBatch(
continue;
}
- const { approved, reason } = resolveApprovalOutcome(responseMap.get(request.requestId));
+ const { approved, reason, status } = resolveApprovalOutcome(responseMap.get(request.requestId));
if (approved) {
continue;
}
@@ -400,8 +517,15 @@ function buildRejectedActionBatch(
isError: true,
kind: "tool-result",
output: {
+ approval: {
+ requestId: request.requestId,
+ status,
+ },
code: TOOL_EXECUTION_DENIED_CODE,
message: reason ?? TOOL_EXECUTION_DENIED_MESSAGE,
+ tool: {
+ result: "not_run",
+ },
},
toolName: request.action.toolName,
});
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index b6ed71b24..34e9fb671 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -33,7 +33,7 @@ import {
modelFacingAuthorizationOutput,
requestAuthorization,
} from "#harness/authorization.js";
-import { setPendingInputBatch } from "#harness/input-requests.js";
+import { hasDeferredStepInput, setPendingInputBatch } from "#harness/input-requests.js";
import { stashToolInterrupt } from "#harness/tool-interrupts.js";
import { createToolLoopHarness } from "#harness/tool-loop.js";
import type { HarnessEmitFn, HarnessSession, ToolLoopHarnessConfig } from "#harness/types.js";
@@ -305,6 +305,7 @@ type MockAgentConstructor =
ConstructorParameters<typeof ToolLoopAgent> extends [infer S]
? (settings: S) => ToolLoopAgent
: never;
+type MockAgentInstance = ToolLoopAgent & Record<string, unknown>;
function setupMockAgent(result: Record<string, unknown>): void {
vi.mocked(ToolLoopAgent).mockImplementation(function (
@@ -5283,7 +5284,7 @@ describe("createToolLoopHarness", () => {
expect(events.filter((event) => event.type === "action.result")).toHaveLength(1);
});
- it("continues with a follow-up user message after resolving an ignored tool approval", async () => {
+ it("queues a follow-up user message until the pending tool approval resolves", async () => {
const generateCalls: unknown[] = [];
const agentResults = [
{
@@ -5306,7 +5307,7 @@ describe("createToolLoopHarness", () => {
let instanceIndex = 0;
vi.mocked(ToolLoopAgent).mockImplementation(function (
- this: Record<string, unknown>,
+ this: MockAgentInstance,
settings: MockAgentSettings,
) {
const result = agentResults[instanceIndex];
@@ -5326,10 +5327,8 @@ describe("createToolLoopHarness", () => {
if (onStepFinish) await onStepFinish(result);
return result;
});
- return this as unknown as ToolLoopAgent;
- } as unknown as ConstructorParameters<typeof ToolLoopAgent> extends [infer S]
- ? (settings: S) => ToolLoopAgent
- : never);
+ return this;
+ } as MockAgentConstructor);
const session = setPendingInputBatch({
requests: [
@@ -5396,7 +5395,15 @@ describe("createToolLoopHarness", () => {
message: "Hi instead.",
});
- expect(typeof firstResult.next).toBe("function");
+ expect(firstResult.next).toBeNull();
+ expect(generateCalls).toEqual([]);
+ expect(hasDeferredStepInput(firstResult.session)).toBe(true);
+
+ const deniedResult = await createToolLoopHarness(config)(firstResult.session, {
+ inputResponses: [{ requestId: "approval-1", optionId: "deny" }],
+ });
+
+ expect(typeof deniedResult.next).toBe("function");
expect(generateCalls[0]).toEqual([
{
content: [
@@ -5419,13 +5426,13 @@ describe("createToolLoopHarness", () => {
{
approvalId: "approval-1",
approved: false,
- reason: "Ignored because the user continued without responding.",
+ reason: "Tool execution was denied.",
type: "tool-approval-response",
},
{
output: {
type: "execution-denied",
- reason: "Ignored because the user continued without responding.",
+ reason: "Tool execution was denied.",
},
toolCallId: "call-1",
toolName: "bash",
@@ -5436,7 +5443,7 @@ describe("createToolLoopHarness", () => {
},
]);
- const secondResult = await createToolLoopHarness(config)(firstResult.session);
+ const secondResult = await createToolLoopHarness(config)(deniedResult.session);
expect(secondResult.next).toBeNull();
expect((generateCalls[1] as { role: string; content: unknown }[]).at(-1)).toEqual({
@@ -5453,10 +5460,143 @@ describe("createToolLoopHarness", () => {
});
});
- it("deferred message lands as last non-system message after approval auto-deny", async () => {
- // Step 1: pending approval + user sends a follow-up message.
- // The approval is auto-denied and the message is deferred.
- // Step 2: the deferred message is consumed and appears as the
+ it("consumes text approval shortcuts without appending them as user messages", async () => {
+ const generateCalls: unknown[] = [];
+
+ vi.mocked(ToolLoopAgent).mockImplementation(function (
+ this: Record<string, unknown>,
+ settings: MockAgentSettings,
+ ) {
+ const { onStepFinish, prepareStep } = settings;
+ this.generate = vi.fn().mockImplementation(async (input: { messages: unknown[] }) => {
+ if (prepareStep) {
+ await prepareStep({
+ messages: input.messages,
+ steps: [],
+ stepNumber: 0,
+ model: {},
+ context: undefined,
+ });
+ }
+ generateCalls.push(input.messages);
+ const result = {
+ finishReason: "stop",
+ response: { messages: [{ content: "Approved.", role: "assistant" }] },
+ text: "Approved.",
+ toolCalls: [],
+ toolResults: [],
+ };
+ if (onStepFinish) await onStepFinish(result);
+ return result;
+ });
+ return this as unknown as ToolLoopAgent;
+ } as unknown as ConstructorParameters<typeof ToolLoopAgent> extends [infer S]
+ ? (settings: S) => ToolLoopAgent
+ : never);
+
+ const session = setPendingInputBatch({
+ requests: [
+ {
+ action: {
+ callId: "call-1",
+ input: { note: "text approval" },
+ kind: "tool-call",
+ toolName: "guarded_echo",
+ },
+ allowFreeform: false,
+ display: "confirmation",
+ options: [
+ { id: "approve", label: "Yes" },
+ { id: "deny", label: "No" },
+ ],
+ prompt: "Approve tool call: guarded_echo",
+ requestId: "approval-1",
+ },
+ ],
+ responseMessages: [
+ {
+ content: [
+ {
+ input: { note: "text approval" },
+ toolCallId: "call-1",
+ toolName: "guarded_echo",
+ type: "tool-call",
+ },
+ {
+ approvalId: "approval-1",
+ toolCallId: "call-1",
+ type: "tool-approval-request",
+ },
+ ],
+ role: "assistant",
+ },
+ ],
+ session: createTestSession({
+ agent: {
+ modelReference: { id: "test-model" },
+ system: "You are a test assistant.",
+ tools: [
+ {
+ description: "Echo a note",
+ name: "guarded_echo",
+ inputSchema: { type: "object" },
+ },
+ ],
+ },
+ }),
+ });
+
+ const config = createTestConfig("conversation", undefined, {
+ tools: new Map([
+ [
+ "guarded_echo",
+ {
+ description: "Echo a note",
+ execute: vi.fn().mockResolvedValue("ok"),
+ inputSchema: jsonSchema({ type: "object" }),
+ name: "guarded_echo",
+ },
+ ],
+ ]),
+ });
+
+ await createToolLoopHarness(config)(session, { message: "approve" });
+
+ expect(generateCalls[0]).toEqual([
+ {
+ content: [
+ {
+ input: { note: "text approval" },
+ toolCallId: "call-1",
+ toolName: "guarded_echo",
+ type: "tool-call",
+ },
+ {
+ approvalId: "approval-1",
+ toolCallId: "call-1",
+ type: "tool-approval-request",
+ },
+ ],
+ role: "assistant",
+ },
+ {
+ content: [
+ {
+ approvalId: "approval-1",
+ approved: true,
+ reason: undefined,
+ type: "tool-approval-response",
+ },
+ ],
+ role: "tool",
+ },
+ ]);
+ });
+
+ it("deferred message lands as last non-system message after explicit approval denial", async () => {
+ // Step 1: pending approval + user sends a follow-up message. The approval
+ // remains pending and the message is deferred. Step 2: the user denies the
+ // approval. Step 3: the deferred message is consumed and appears as the
// last message the model sees.
const generateCalls: Array<Array<{ role: string; content: unknown }>> = [];
const agentResults = [
@@ -5568,24 +5708,28 @@ describe("createToolLoopHarness", () => {
});
// Step 1: user sends "Do something else" while approval is pending.
- // Approval is auto-denied; message is deferred.
+ // Approval remains pending; message is deferred.
const firstResult = await createToolLoopHarness(config)(session, {
message: "Do something else",
});
- expect(typeof firstResult.next).toBe("function");
+ expect(firstResult.next).toBeNull();
+ expect(generateCalls).toEqual([]);
- // Step 1 messages: model sees [assistant(tool-call+approval), tool(denied)]
- // — the deferred message is NOT in this call.
- const step1Last = generateCalls[0]?.at(-1);
- expect(step1Last?.role).toBe("tool");
+ // Step 2: user denies the approval; the deferred message is NOT in this call.
+ const deniedResult = await createToolLoopHarness(config)(firstResult.session, {
+ inputResponses: [{ requestId: "approval-1", optionId: "deny" }],
+ });
+ expect(typeof deniedResult.next).toBe("function");
+ const step2Last = generateCalls[0]?.at(-1);
+ expect(step2Last?.role).toBe("tool");
- // Step 2: harness consumes the deferred message.
- const secondResult = await createToolLoopHarness(config)(firstResult.session);
+ // Step 3: harness consumes the deferred message.
+ const secondResult = await createToolLoopHarness(config)(deniedResult.session);
expect(secondResult.next).toBeNull();
// The deferred user message is the last message the model sees.
- const step2Last = generateCalls[1]?.at(-1);
- expect(step2Last).toEqual({ content: "Do something else", role: "user" });
+ const step3Last = generateCalls[1]?.at(-1);
+ expect(step3Last).toEqual({ content: "Do something else", role: "user" });
// History reflects the full conversation.
expect(secondResult.session.history.at(-1)).toEqual({
diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.ts
index cddb5844e..a93310076 100644
--- a/packages/eve/src/harness/tool-loop.ts
+++ b/packages/eve/src/harness/tool-loop.ts
@@ -427,6 +427,20 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
stepInput: stepInput.input,
});
if (pending.outcome === "unresolved") {
+ if (emit && pending.deferredMessage === true && hasStepInput(input)) {
+ emissionState = await emitTurnPreamble(
+ emit,
+ input ?? {},
+ emissionState,
+ config.runtimeIdentity,
+ );
+ emissionState = await emitTurnEpilogue(emit, emissionState, config.mode);
+ return {
+ next: null,
+ session: setHarnessEmissionState(pending.session, emissionState),
+ };
+ }
+
return { next: null, session: pending.session };
}
@@ -476,7 +490,11 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
}
}
- if (stepInput.input?.message !== undefined && !pending.deferredMessage) {
+ if (
+ stepInput.input?.message !== undefined &&
+ !pending.deferredMessage &&
+ !pending.consumedMessage
+ ) {
// Staging writes FilePart bytes into the sandbox and replaces
// each part's `data` with a compact `eve-sandbox:` URL. The
// `messages` array — and everything that flows into