Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): continue past historical hitl requests (#3564)

Ruie110eb2af4e4
diff --git a/.changeset/continue-past-historical-hitl.md b/.changeset/continue-past-historical-hitl.md
new file mode 100644
index 000000000..e892cb2d2
--- /dev/null
+++ b/.changeset/continue-past-historical-hitl.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Allow an active turn to finish its tool-result continuation when an earlier turn's HITL request remains unanswered, while preserving current-turn HITL parking. Partial approval responses remain saved until their batch can resolve; they do not cause an extra model call after an unrelated answer.
diff --git a/docs/concepts/sessions-runs-and-streaming.md b/docs/concepts/sessions-runs-and-streaming.md
index e99b6d06a..2dfd4273e 100644
--- a/docs/concepts/sessions-runs-and-streaming.md
+++ b/docs/concepts/sessions-runs-and-streaming.md
@@ -214,7 +214,7 @@ With one question-only batch, an exact option match or permitted freeform respon
A structured response matches any currently pending request by ID, not only the newest batch. It becomes stale only after that request was answered, cleared, or cancelled. eve delivers a stale response to the model as a new user message, and the model decides whether the old selection still matters. A stale approval never authorizes the earlier tool call; the model must request the action and approval again if they are still needed.
-One delivery can answer requests from several batches. eve resumes approval-bearing batches in durable order and carries later answers forward until each batch can resume.
+One delivery can answer requests from several batches. eve resumes approval-bearing batches in durable order and carries later answers forward until each batch can resume. If you answer only some approvals in a batch, eve saves those responses until the remaining approvals are answered. Meanwhile, unrelated messages can run tools and receive a completed reply. The saved partial responses neither block that reply nor trigger another model call after it.
Multiple steering messages retain their durable arrival order and may be folded into one input at the next boundary. A message accepted after turn settlement starts the next turn. See [message delivery and steering](./execution-model-and-durability#message-delivery-and-steering).
diff --git a/e2e/fixtures/agent-tools-hitl/agent/agent.ts b/e2e/fixtures/agent-tools-hitl/agent/agent.ts
index 43b8a887d..a7ea0250f 100644
--- a/e2e/fixtures/agent-tools-hitl/agent/agent.ts
+++ b/e2e/fixtures/agent-tools-hitl/agent/agent.ts
@@ -1,11 +1,15 @@
import { e2eAgentConfig } from "@eve-e2e/config";
-import { defineAgent } from "eve";
+import { defineAgent, defineDynamic } from "eve";
import type { MockModelRequest, MockModelResponse } from "eve/evals";
+import { continuationModel } from "./lib/continuation/model.ts";
+
const AUTH_PROBE_DIRECTIVE = /call the auth-probe tool exactly once with marker "([^"]+)"/iu;
const SCOPED_APPROVAL_DIRECTIVE =
/call the dynamic_scoped_approval tool exactly once with scope "([^"]+)"/iu;
const REPLY_DIRECTIVE = /reply with exactly ([A-Z0-9-]+)/iu;
+const APPROVAL_FOLLOWUP_DIRECTIVE =
+ /call the (gate|read-status) tool exactly once with marker "([^"]+)"/iu;
/**
* Scripted mock for the world suites: untagged evals in this fixture phrase
@@ -16,6 +20,20 @@ const REPLY_DIRECTIVE = /reply with exactly ([A-Z0-9-]+)/iu;
function respond(request: MockModelRequest): MockModelResponse | string {
const message = request.lastUserMessage ?? "";
+ const approvalFollowup = APPROVAL_FOLLOWUP_DIRECTIVE.exec(message);
+ if (approvalFollowup?.[1] !== undefined && approvalFollowup[2] !== undefined) {
+ const roles = request.messages.map((entry) => entry.role);
+ if (roles.lastIndexOf("tool") < roles.lastIndexOf("user")) {
+ return {
+ toolCalls: [{ name: approvalFollowup[1], input: { marker: approvalFollowup[2] } }],
+ };
+ }
+ const output = [...request.toolResults]
+ .reverse()
+ .find((result) => result.name === approvalFollowup[1])?.output;
+ return JSON.stringify(output ?? "Missing tool result");
+ }
+
const reply = REPLY_DIRECTIVE.exec(message);
if (reply?.[1] !== undefined) {
return reply[1];
@@ -48,7 +66,23 @@ function respond(request: MockModelRequest): MockModelResponse | string {
return `Mock reply: ${message}`;
}
+const base = e2eAgentConfig({ mock: respond });
+
export default defineAgent({
- ...e2eAgentConfig({ mock: respond }),
+ experimental: base.experimental,
reasoning: "high",
+ // Budget evals exhaust this with synthetic usage; ordinary HITL sessions do not.
+ limits: { maxOutputTokensPerSession: 1_000_000 },
+ model: defineDynamic({
+ events: {
+ "session.started": () => ({
+ model: typeof base.model === "string" ? base.model : "openai/gpt-5.6-sol",
+ modelContextWindowTokens: base.modelContextWindowTokens,
+ }),
+ "step.started": (_event, ctx) =>
+ ctx.session.auth.initiator?.attributes?.model === "continuation"
+ ? { model: continuationModel(), modelContextWindowTokens: 1_000_000 }
+ : { model: base.model, modelContextWindowTokens: base.modelContextWindowTokens },
+ },
+ }),
});
diff --git a/e2e/fixtures/agent-tools-hitl/agent/channels/eve.ts b/e2e/fixtures/agent-tools-hitl/agent/channels/eve.ts
index dd6784991..a95240c49 100644
--- a/e2e/fixtures/agent-tools-hitl/agent/channels/eve.ts
+++ b/e2e/fixtures/agent-tools-hitl/agent/channels/eve.ts
@@ -5,7 +5,10 @@ export default eveChannel({
auth: (request) => {
const principalId = request.headers.get("x-eve-fixture-user") ?? "e2e-approval-responder";
return {
- attributes: { fixture: "authorized-response" },
+ attributes: {
+ fixture: "authorized-response",
+ model: request.headers.get("x-eve-fixture-model") ?? "default",
+ },
authenticator: "e2e-fixture",
issuer: "e2e",
principalId,
diff --git a/e2e/fixtures/agent-tools-hitl/agent/lib/continuation/model.ts b/e2e/fixtures/agent-tools-hitl/agent/lib/continuation/model.ts
new file mode 100644
index 000000000..1cae9d182
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/lib/continuation/model.ts
@@ -0,0 +1,54 @@
+import { mockModel } from "eve/evals";
+import { respond } from "./respond.ts";
+
+export function continuationModel() {
+ const model = mockModel({ modelId: "hitl-continuation", respond });
+ if (typeof model === "string" || model.specificationVersion !== "v3") {
+ throw new Error("This fixture expects mockModel's v3 provider boundary.");
+ }
+ const generate = model.doGenerate.bind(model);
+ model.doGenerate = async (options) => {
+ const response = await generate(options);
+ const content: typeof response.content = [];
+ for (const part of response.content) {
+ if (part.type === "tool-call" && part.toolCallId === "provider-lookup") {
+ content.push({ ...part, providerExecuted: true });
+ content.push({
+ type: "tool-result",
+ toolCallId: part.toolCallId,
+ toolName: part.toolName,
+ result: { draftId: "draft-3494", status: "provider-ready" },
+ });
+ } else content.push(part);
+ }
+ return { ...response, content };
+ };
+ const stream = model.doStream.bind(model);
+ type Part =
+ Awaited<ReturnType<typeof stream>>["stream"] extends ReadableStream<infer T> ? T : never;
+
+ // Model the provider's own result on the provider stream, not as a local tool
+ // return. This leaves an assistant message at the end of the model history.
+ model.doStream = async (options) => {
+ const response = await stream(options);
+ return {
+ ...response,
+ stream: response.stream.pipeThrough(
+ new TransformStream<Part, Part>({
+ transform(part, controller) {
+ if (part.type === "tool-call" && part.toolCallId === "provider-lookup") {
+ controller.enqueue({ ...part, providerExecuted: true });
+ controller.enqueue({
+ type: "tool-result",
+ toolCallId: part.toolCallId,
+ toolName: part.toolName,
+ result: { draftId: "draft-3494", status: "provider-ready" },
+ });
+ } else controller.enqueue(part);
+ },
+ }),
+ ),
+ };
+ };
+ return model;
+}
diff --git a/e2e/fixtures/agent-tools-hitl/agent/lib/continuation/respond.ts b/e2e/fixtures/agent-tools-hitl/agent/lib/continuation/respond.ts
new file mode 100644
index 000000000..8baf7ab1f
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/lib/continuation/respond.ts
@@ -0,0 +1,225 @@
+import type { MockModelRequest, MockModelResponse, MockModelToolCall } from "eve/evals";
+
+const read = (id: string): MockModelToolCall => ({
+ id,
+ name: "read-draft",
+ input: { draftId: "draft-3494" },
+});
+const call = (id: string, name: string): MockModelToolCall => ({ id, name, input: {} });
+
+// Each sentence is a user instruction in an eval. Results, rather than a mutable
+// call counter, select the next response so durable replay uses the same script.
+export function respond(request: MockModelRequest): MockModelResponse {
+ const isRuntimeContext = (text: string) =>
+ text.startsWith("[Task state]") || text.startsWith("Background task ");
+ const instructions = request.userMessages.filter((text) => !isRuntimeContext(text));
+ let message = instructions.at(-1) ?? "";
+ const roles = request.messages
+ .filter((entry) => entry.role !== "user" || !isRuntimeContext(entry.text))
+ .map((entry) => entry.role);
+ const lastResult = request.toolResults.at(-1);
+ if (lastResult && roles.lastIndexOf("tool") > roles.lastIndexOf("user")) {
+ // Responding to an older request resumes its instruction. A newer user
+ // message in the history does not transfer ownership of that tool result.
+ const owner = instructions.find((instruction) => {
+ switch (lastResult.id) {
+ case "change-a":
+ return (
+ instruction.startsWith("Prepare change A") ||
+ instruction === "Prepare changes A and B together."
+ );
+ case "change-b-read":
+ case "change-b":
+ return (
+ instruction.startsWith("Prepare change B") ||
+ instruction === "Prepare changes A and B together."
+ );
+ case "authorized-read":
+ case "authorized":
+ return instruction.startsWith("Prepare an authorized change");
+ case "color-read":
+ case "color":
+ return instruction.startsWith("Ask which color");
+ case "size":
+ return instruction.startsWith("Ask which size");
+ default:
+ return false;
+ }
+ });
+ if (owner !== undefined) message = owner;
+ }
+ const result = (id: string) => request.toolResults.find((entry) => entry.id === id);
+ const run = (calls: MockModelToolCall[], answer: () => string): MockModelResponse => {
+ const missing = calls.filter((tool) => result(tool.id!) === undefined);
+ return missing.length > 0 ? { toolCalls: missing } : { text: answer() };
+ };
+ const status = (id: string): string => {
+ const output = result(id)?.output;
+ if (output === null || typeof output !== "object" || !("status" in output)) {
+ throw new Error(`Missing status in tool result ${id}: ${JSON.stringify(output)}`);
+ }
+ return String(output.status);
+ };
+ const readStatus = (id: string) => run([read(id)], () => `Draft status: ${status(id)}.`);
+ const changeThenRead = (id: string, name: string) => {
+ if (result(id) === undefined) return { toolCalls: [call(id, name)] };
+ return readStatus(`${id}-read`);
+ };
+ let response: MockModelResponse;
+ switch (message) {
+ case "Prepare change A.":
+ case "Prepare change A using the remaining budget.":
+ response = run([call("change-a", "change-a")], () => "Change A resolved.");
+ break;
+ case "Prepare change B, then read the draft status.":
+ response = changeThenRead("change-b", "change-b");
+ break;
+ case "Prepare change B, then acknowledge my decision.":
+ response = run([call("change-b", "change-b")], () => "Change B resolved.");
+ break;
+ case "Prepare an authorized change.":
+ response = run(
+ [call("authorized", "authorized-change")],
+ () => "Authorized change resolved.",
+ );
+ break;
+ case "Prepare an authorized change, then read the draft status.":
+ response = changeThenRead("authorized", "authorized-change");
+ break;
+ case "Prepare changes A and B together.":
+ response = run(
+ [call("change-a", "change-a"), call("change-b", "change-b")],
+ () => "Both changes resolved.",
+ );
+ break;
+ case "Read the draft status.":
+ case "Read the draft status using the remaining budget.":
+ response = readStatus("read");
+ break;
+ case "Save the draft and report how many times it was written.":
+ response = run(
+ [call("save", "save-draft")],
+ () => `Draft saved: ${JSON.stringify(result("save")!.output)}.`,
+ );
+ break;
+ case "Read and save the draft in parallel.":
+ response = run(
+ [read("read"), call("save", "save-draft")],
+ () =>
+ `Draft status: ${status("read")}. Draft saved: ${JSON.stringify(result("save")!.output)}.`,
+ );
+ break;
+ case "Try the unavailable draft store and explain the error.":
+ response = run([call("failure", "unavailable-draft")], () => {
+ if (!result("failure")!.isError) throw new Error("Expected the draft tool to fail.");
+ return `Could not read the draft: ${JSON.stringify(result("failure")!.output)}`;
+ });
+ break;
+ case "Try a numeric draft ID, then correct it and read the status.":
+ if (result("invalid") === undefined) {
+ response = { toolCalls: [{ id: "invalid", name: "read-draft", input: { draftId: 3494 } }] };
+ } else {
+ if (!result("invalid")!.isError)
+ throw new Error("The numeric draft ID must fail validation.");
+ response = run(
+ [read("corrected")],
+ () =>
+ `Draft status: ${status("corrected")}. Validation error: ${JSON.stringify(result("invalid")!.output)}`,
+ );
+ }
+ break;
+ case "Read the draft through a workflow.":
+ response = run(
+ [call("workflow", "workflow-draft")],
+ () => `Workflow draft status: ${status("workflow")}.`,
+ );
+ break;
+ case "Prepare change A and read the workflow draft together.":
+ response = run(
+ [call("change-a", "change-a"), call("workflow", "workflow-draft")],
+ () => `Workflow draft status: ${status("workflow")}. Change A resolved.`,
+ );
+ break;
+ case "Start the background draft and acknowledge its receipt.":
+ response = run(
+ [call("background", "background-draft")],
+ () => `Background receipt: ${JSON.stringify(result("background")!.output)}`,
+ );
+ break;
+ case "Cancel the background draft task and confirm cancellation.": {
+ const receipt = result("background")?.output;
+ if (
+ receipt === null ||
+ typeof receipt !== "object" ||
+ !("taskId" in receipt) ||
+ typeof receipt.taskId !== "string"
+ ) {
+ throw new Error("The background tool must return a real task ID before cancellation.");
+ }
+ response = run(
+ [{ id: "cancel", name: "task_cancel", input: { taskIds: [receipt.taskId] } }],
+ () => `Cancellation result: ${JSON.stringify(result("cancel")!.output)}`,
+ );
+ break;
+ }
+ case "Look up the draft with the provider and report its status.":
+ response = run(
+ [read("provider-lookup")],
+ () => `Provider draft status: ${status("provider-lookup")}.`,
+ );
+ break;
+ case "Ask which color to use, then read the draft status.":
+ case "Ask which size to use.": {
+ const color = message.includes("color");
+ const id = color ? "color" : "size";
+ response =
+ result(id) === undefined
+ ? {
+ toolCalls: [
+ {
+ id,
+ name: "ask_question",
+ input: {
+ prompt: color ? "Which color?" : "Which size?",
+ options: color
+ ? [
+ { id: "red", label: "Red" },
+ { id: "blue", label: "Blue" },
+ ]
+ : [
+ { id: "small", label: "Small" },
+ { id: "large", label: "Large" },
+ ],
+ },
+ },
+ ],
+ }
+ : color
+ ? readStatus("color-read")
+ : { text: "Size resolved." };
+ break;
+ }
+ case "Explain what is waiting, without calling any tools.":
+ response = { text: "Your changes are waiting for approval." };
+ break;
+ case "Use the remaining budget to say hello.":
+ response = { text: "Hello." };
+ break;
+ default:
+ // The runtime converts a response for an already resolved request into a
+ // user message. Only that explicitly recognizable input gets this script.
+ if (
+ message.startsWith(
+ "The user submitted the following response to an earlier interactive prompt.",
+ )
+ ) {
+ response = readStatus("stale-response-read");
+ break;
+ }
+ throw new Error(`Unexpected fixture message: ${message}`);
+ }
+ return {
+ ...response,
+ usage: { inputTokens: 1, outputTokens: message.includes("remaining budget") ? 1_000_000 : 1 },
+ };
+}
diff --git a/e2e/fixtures/agent-tools-hitl/agent/tools/authorized-change.ts b/e2e/fixtures/agent-tools-hitl/agent/tools/authorized-change.ts
new file mode 100644
index 000000000..2de3b3cbe
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/tools/authorized-change.ts
@@ -0,0 +1,22 @@
+import { defineState } from "eve/context";
+import { defineTool } from "eve/tools";
+import { always } from "eve/tools/approval";
+import { z } from "zod";
+
+const executions = defineState("authorized-change.executions", () => 0);
+
+export default defineTool({
+ description: "Apply a change only after the fixture's authorized user approves it.",
+ inputSchema: z.object({}),
+ approval: {
+ request: always(),
+ response: ({ responder }) =>
+ responder.principalId === "e2e-approval-responder"
+ ? { status: "allowed" }
+ : { status: "rejected", reason: "Wrong responder." },
+ },
+ async execute() {
+ executions.update((count) => count + 1);
+ return { change: "authorized", executions: executions.get() };
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/agent/tools/background-draft.ts b/e2e/fixtures/agent-tools-hitl/agent/tools/background-draft.ts
new file mode 100644
index 000000000..c5aeb32d0
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/tools/background-draft.ts
@@ -0,0 +1,14 @@
+import { defineWorkflowTool } from "eve/tools";
+import { sleep } from "workflow";
+import { z } from "zod";
+
+export default defineWorkflowTool({
+ description: "Hold a background draft task open until the eval cancels it.",
+ execution: "background",
+ inputSchema: z.object({}),
+ async execute() {
+ "use workflow";
+ await sleep("10m");
+ return { status: "ready" };
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/agent/tools/change-a.ts b/e2e/fixtures/agent-tools-hitl/agent/tools/change-a.ts
new file mode 100644
index 000000000..95e3c74ab
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/tools/change-a.ts
@@ -0,0 +1,16 @@
+import { defineTool } from "eve/tools";
+import { always } from "eve/tools/approval";
+import { defineState } from "eve/context";
+import { z } from "zod";
+
+const executions = defineState("change-a.executions", () => 0);
+
+export default defineTool({
+ description: "Apply fixture change A after approval.",
+ inputSchema: z.object({}),
+ approval: always(),
+ async execute() {
+ executions.update((count) => count + 1);
+ return { change: "A", executions: executions.get() };
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/agent/tools/change-b.ts b/e2e/fixtures/agent-tools-hitl/agent/tools/change-b.ts
new file mode 100644
index 000000000..a8aae6a4e
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/tools/change-b.ts
@@ -0,0 +1,16 @@
+import { defineTool } from "eve/tools";
+import { always } from "eve/tools/approval";
+import { defineState } from "eve/context";
+import { z } from "zod";
+
+const executions = defineState("change-b.executions", () => 0);
+
+export default defineTool({
+ description: "Apply fixture change B after approval.",
+ inputSchema: z.object({}),
+ approval: always(),
+ async execute() {
+ executions.update((count) => count + 1);
+ return { change: "B", executions: executions.get() };
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/agent/tools/read-draft.ts b/e2e/fixtures/agent-tools-hitl/agent/tools/read-draft.ts
new file mode 100644
index 000000000..0a5e36441
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/tools/read-draft.ts
@@ -0,0 +1,10 @@
+import { defineTool } from "eve/tools";
+import { z } from "zod";
+
+export default defineTool({
+ description: "Read the fixture draft's status.",
+ inputSchema: z.object({ draftId: z.string() }),
+ async execute({ draftId }) {
+ return { draftId, status: "ready" };
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/agent/tools/read-status.ts b/e2e/fixtures/agent-tools-hitl/agent/tools/read-status.ts
new file mode 100644
index 000000000..2369030b6
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/tools/read-status.ts
@@ -0,0 +1,10 @@
+import { defineTool } from "eve/tools";
+import { z } from "zod";
+
+export default defineTool({
+ description: "Read the current draft status without requiring approval.",
+ inputSchema: z.object({ marker: z.string() }),
+ async execute({ marker }) {
+ return { marker, status: "ready" };
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/agent/tools/save-draft.ts b/e2e/fixtures/agent-tools-hitl/agent/tools/save-draft.ts
new file mode 100644
index 000000000..66e86eec4
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/tools/save-draft.ts
@@ -0,0 +1,14 @@
+import { defineState } from "eve/context";
+import { defineTool } from "eve/tools";
+import { z } from "zod";
+
+const writes = defineState("draft.writes", () => 0);
+
+export default defineTool({
+ description: "Save the fixture draft and return the number of writes in this session.",
+ inputSchema: z.object({}),
+ async execute() {
+ writes.update((count) => count + 1);
+ return { writes: writes.get() };
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/agent/tools/unavailable-draft.ts b/e2e/fixtures/agent-tools-hitl/agent/tools/unavailable-draft.ts
new file mode 100644
index 000000000..ae12b4200
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/tools/unavailable-draft.ts
@@ -0,0 +1,10 @@
+import { defineTool } from "eve/tools";
+import { z } from "zod";
+
+export default defineTool({
+ description: "Fail with a known, recoverable fixture error.",
+ inputSchema: z.object({}),
+ async execute(): Promise<never> {
+ throw new Error("The draft store is unavailable.");
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/agent/tools/workflow-draft.ts b/e2e/fixtures/agent-tools-hitl/agent/tools/workflow-draft.ts
new file mode 100644
index 000000000..ed6096d2b
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/tools/workflow-draft.ts
@@ -0,0 +1,11 @@
+import { defineWorkflowTool } from "eve/tools";
+import { z } from "zod";
+
+export default defineWorkflowTool({
+ description: "Read a draft through a durable workflow tool.",
+ inputSchema: z.object({}),
+ async execute() {
+ "use workflow";
+ return { status: "ready" };
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts b/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts
index 7a61a5b55..576b6419b 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts
@@ -2,5 +2,6 @@ import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
export default defineEvalConfig({
+ maxConcurrency: 4,
judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/README.md b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/README.md
new file mode 100644
index 000000000..aab16cda8
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/README.md
@@ -0,0 +1,78 @@
+# User-message continuation
+
+Every accepted user message must reach its own answer, required input request, or explicit failure/cancellation. An older unanswered request cannot silently stop that message's work.
+
+This directory contains **37 scripted evals: 23 regressions and 14 controls**. They exercise the HTTP session, approval, tool, workflow, and event-stream paths. A deterministic model chooses calls and constructs answers from the results it receives. There is no model judge and no manually seeded pending state.
+
+These sessions select the scripted model through a fixture-only header in **every** CI world, including Local. The sibling [`pending-approval-tool-followup.eval.ts`](../pending-approval-tool-followup.eval.ts) adds two cases using the CI-selected model: real models in Local, the fixture mock in Postgres and Vercel. Passing scripted cases does not establish live-model coverage of the whole matrix.
+
+Start with [`read.eval.ts`](./read.eval.ts): leave a change's approval pending, ask for a draft status, require its answer, then approve the saved change. Other cases vary the pending requests, delivery order, or result type.
+
+## Scenario syntax and classification
+
+Each body uses **Given / When / Then** comments to identify the initial state, accepted input, and observable outcome. `defineEval.description` names the scenario. Native `tags` classify role (`regression` or `control`), triggering input (`user-message` or `input-response`), and behavior (`tool-result`, `tool-error`, `validation`, `workflow`, `provider-result`, `background-task`, `runtime-control`, `approval`, `authorization`, `question`, `partial-approval`, `stale-response`, `budget`, or `text-reply`). Every case also carries `hitl` and `continuation`.
+
+The tags are filters, not expected verdicts: every case must pass. `eve eval --tag regression`, `--tag control`, or `--tag partial-approval` selects the corresponding conversations. This is ordinary executable `defineEval` code with native assertions; Given / When / Then is not a separate executable spec language.
+
+## What makes a passing answer
+
+[`expectReply`](./helpers.ts) requires exactly one matching `message.completed`, followed by exactly one `turn.completed`, both attributed to the same turn. A tool result, unrelated reply, or completion without an answer cannot pass. New messages use their own `message.received` turn ID. Approval and question responses require successful resolution of the saved request before its matching reply and completion in the delivery's resumed turn. Multiple responses accepted together can resolve across several steps of that one turn; a later resolution need not emit another `turn.started`.
+
+Each approval ID is saved when emitted. The driver's latest-turn request list is not treated as durable pending state. Tests check that unrelated replies do not execute the old change, then generally approve the saved request and require its execution and completed reply. Runtime-control coverage stops after its answer to avoid mixing later background notifications into that assertion. Budget cases require the next budget request instead of an answer beyond the granted limit.
+
+## Regressions
+
+Each row maps to exactly one eval. “Response authorization” here means the fixture checks the authenticated responder's principal; it does not exercise an OAuth callback.
+
+| Conversation | Required outcome | Eval |
+| ------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------- | -------------------------------------------------------------------- |
+| Read while A waits | Report returned status; A remains answerable | [read](./read.eval.ts) |
+| Write while A waits | Confirm exactly one write; A remains answerable | [write](./write.eval.ts) |
+| Parallel read and write while A waits | Report both results | [parallel tools](./parallel-tools.eval.ts) |
+| Tool throws while A waits | Explain the actual error | [tool error](./tool-error.eval.ts) |
+| Invalid tool input while A waits | Correct input and report the result | [invalid input](./invalid-input.eval.ts) |
+| Response-authorized approval waits | Finish an unrelated read | [authorized pending](./authorized-pending.eval.ts) |
+| Approve newer B while A waits | Execute B, read, and reply | [approve sibling](./approve-sibling.eval.ts) |
+| Cancel newer B while A waits | Leave B unexecuted, read, and reply | [cancel sibling](./cancel-sibling.eval.ts) |
+| Approve newer response-authorized request while A waits | Settle authorization, read, and reply | [authorized sibling](./authorized-sibling.eval.ts) |
+| Approve older response-authorized request while newer A waits | Read and reply; A remains answerable | [authorized older sibling](./authorized-older-sibling.eval.ts) |
+| Approve older ordinary A while a newer response-authorized request waits | Reply for A; newer request remains answerable | [ordinary older sibling](./ordinary-older-sibling.eval.ts) |
+| Approve A and answer a later question in one delivery while B waits | Resolve both responses and deliver both replies in one completed turn | [approval and question](./approval-and-question.eval.ts) |
+| Answer a question while A waits | Read and reply for the answered question | [answer question](./answer-question.eval.ts) |
+| Workflow completes while A waits | Interpret its result | [workflow result](./workflow-result.eval.ts) |
+| Cancel a background task while A waits | Interpret the runtime control result | [runtime control](./runtime-control.eval.ts) |
+| Start a background task while A waits | Acknowledge its working receipt | [background receipt](./background-receipt.eval.ts) |
+| Provider supplies a tool result while A waits | Report the distinct provider result | [provider result](./provider-result.eval.ts) |
+| Only two questions remain unanswered | Finish a new read; both questions remain answerable | [multiple questions](./multiple-questions.eval.ts) |
+| Submit A from a same-batch A+B pair, then request a read | Complete the read; later B executes both changes once and completes a reply | [partial approval, tool](./partial-approval-tool.eval.ts) |
+| Submit A from a same-batch A+B pair, then request text only | Complete a reply without tools; later B executes both changes once and completes a reply | [partial approval, text](./partial-approval-text.eval.ts) |
+| Repeat a resolved response while A waits | Process the new input without authorizing A | [stale response](./stale-response.eval.ts) |
+| Grant another budget window while A waits | Run one tool, then request the next required grant | [budget grant](./budget-grant.eval.ts) |
+| Submit A and B from one batch in separate deliveries | Accumulate both responses, execute both once, and reply | [separate approval responses](./separate-approval-responses.eval.ts) |
+
+## Controls
+
+| Conversation | Required outcome | Eval |
+| ------------------------------------------------------- | -------------------------------------------------------------------------------- | ---------------------------------------------------------------- |
+| Read without older input | Report returned status | [read](./read.control.eval.ts) |
+| Write without older input | Confirm exactly one write | [write](./write.control.eval.ts) |
+| Parallel tools without older input | Report both results | [parallel tools](./parallel-tools.control.eval.ts) |
+| Tool throws without older input | Explain the error | [tool error](./tool-error.control.eval.ts) |
+| Invalid input without older input | Correct input and report result | [invalid input](./invalid-input.control.eval.ts) |
+| Workflow without older input | Interpret its result | [workflow result](./workflow-result.control.eval.ts) |
+| Runtime control without older input | Interpret cancellation result | [runtime control](./runtime-control.control.eval.ts) |
+| Background admission without older input | Acknowledge working receipt | [background receipt](./background-receipt.control.eval.ts) |
+| Provider result without older input | Report distinct provider result | [provider result](./provider-result.control.eval.ts) |
+| Budget renewal without older approval | Run one tool, then request next grant | [budget grant](./budget-grant.control.eval.ts) |
+| Text-only message while A waits | Reply without tools; A remains answerable | [text only](./text-only.control.eval.ts) |
+| Resolve the only approval | Execute, read, and reply | [resolve only approval](./resolve-only-approval.control.eval.ts) |
+| Approve both calls together | Execute each once and reply | [approve both](./approve-both.control.eval.ts) |
+| Workflow finishes beside an approval from the same turn | Observe workflow completion before approval; require approval before final reply | [same-turn workflow](./same-turn-workflow.control.eval.ts) |
+
+## Evidence boundaries
+
+The [adversarial integration probes](../../../../../../packages/eve/src/harness/issue-3494-adversarial.integration.test.ts) also cover sole-question dismissal, `final_output` with an older approval, multiple independent approvals plus an internally deferred message, a new question beside a partial approval, and a complete independent batch beside a partially answered batch. Those exact scenarios are **integration-only**, not extra E2E cases. Integration workflow/control results and background receipts are injected at runtime boundaries; the E2E cases execute fixture tools through the durable runtime.
+
+Partial approvals in E2E use an accepted HTTP response followed by a separate user message; the API rejects combined message/response payloads. The provider cases supply a provider-executed result at the scripted model stream boundary; they do not contact a provider that performs the tool. Background coverage establishes admission and acknowledgement, not completion or wake correctness. Runtime control cancels an actual task using its returned ID. Budget scripts report synthetic token usage against the fixture's one-million-output-token limit.
+
+These tests serialize inputs. They do not establish restart/replay safety, concurrent-delivery correctness, child-agent settlement, OAuth recovery, or every provider/channel combination. Run E2E only in CI. A timeout identifies a runtime regression only when the captured events establish that the intended setup and tool path ran; a fixture error is not such evidence.
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/answer-question.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/answer-question.eval.ts
new file mode 100644
index 000000000..d8c5fc61b
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/answer-question.eval.ts
@@ -0,0 +1,34 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectResponseReply,
+ expectToolResult,
+ requestFrom,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "answer-question: resolving the current request must finish its work while change A still waits.",
+ tags: ["hitl", "continuation", "regression", "input-response", "question"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given A awaits approval and a later color question must be followed by a read.
+ const first = await t.send("Prepare change A.", scriptedSession);
+ const approvalA = requestFrom(first, "change-a");
+ const session = first.session;
+ const second = await session.send("Ask which color to use, then read the draft status.");
+ const current = requestFrom(second, "ask_question");
+
+ // When the user answers the color question.
+ const live = await session.startRespond([{ requestId: current.requestId, optionId: "red" }]);
+
+ // Then the read gets a completed reply; A stays unexecuted and answerable.
+ await expectToolResult(t, live, "read-draft");
+ const reply = await expectResponseReply(t, live, "Draft status: ready.", current.requestId);
+ reply.calledTool("read-draft", { status: "completed", count: 1 });
+ expectChangeStillUnexecuted(session);
+ await approveSavedChange(t, session, approvalA);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/approval-and-question.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/approval-and-question.eval.ts
new file mode 100644
index 000000000..2026d4d65
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/approval-and-question.eval.ts
@@ -0,0 +1,50 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectResponseReply,
+ requestFrom,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "Answering an approval and a later question in one delivery must finish both while another approval waits.",
+ tags: ["hitl", "continuation", "regression", "input-response", "approval", "question"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given independent approvals A and B, followed by a color question.
+ const first = await t.send("Prepare change A.", scriptedSession);
+ const approvalA = requestFrom(first, "change-a");
+ const session = first.session;
+ const second = await session.send("Prepare change B, then acknowledge my decision.");
+ const approvalB = requestFrom(second, "change-b");
+ const third = await session.send("Ask which color to use, then read the draft status.");
+ const question = requestFrom(third, "ask_question");
+
+ // When one delivery approves A and answers the question, leaving B unanswered.
+ const live = await session.startRespond([
+ { requestId: approvalA.requestId, optionId: "approve" },
+ { requestId: question.requestId, optionId: "red" },
+ ]);
+
+ // Then both requests resolve and both replies complete within this delivery's resumed turn.
+ await expectResponseReply(t, live, "Change A resolved.", approvalA.requestId);
+ const reply = await expectResponseReply(t, live, "Draft status: ready.", question.requestId);
+ reply.event("turn.started", { count: 1 });
+ reply.event("turn.completed", { count: 1 });
+ reply.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });
+ reply.calledTool("read-draft", { status: "completed", count: 1 });
+ reply.eventsSatisfy("Both saved requests resolve", (events) =>
+ [approvalA.requestId, question.requestId].every((requestId) =>
+ events.some(
+ (event) =>
+ event.type === "input.resolved" &&
+ event.data.resolutions.some((resolution) => resolution.requestId === requestId),
+ ),
+ ),
+ );
+ expectChangeStillUnexecuted(session, "change-b");
+ await approveSavedChange(t, session, approvalB);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/approve-both.control.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/approve-both.control.eval.ts
new file mode 100644
index 000000000..387ec58e0
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/approve-both.control.eval.ts
@@ -0,0 +1,34 @@
+import { defineEval } from "eve/evals";
+import { scriptedSession, expectResponseReply, requestFrom } from "./helpers.ts";
+
+export default defineEval({
+ description: "Control: approving both calls together executes each once and completes the reply.",
+ tags: ["hitl", "continuation", "control", "input-response", "partial-approval"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given A and B were requested together and both await approval.
+ const parked = await t.send("Prepare changes A and B together.", scriptedSession);
+ const approvalA = requestFrom(parked, "change-a");
+ const approvalB = requestFrom(parked, "change-b");
+
+ // When the user approves both in one HTTP request.
+ const live = await parked.session.startRespond([
+ { requestId: approvalA.requestId, optionId: "approve" },
+ { requestId: approvalB.requestId, optionId: "approve" },
+ ]);
+
+ // Then both changes execute once and the resumed turn replies that both resolved.
+ const reply = await expectResponseReply(t, live, "Both changes resolved.", approvalB.requestId);
+ reply.event("input.resolved", {
+ data: {
+ resolutions: (items) =>
+ [approvalA.requestId, approvalB.requestId].every((id) =>
+ items.some((item) => item.requestId === id && item.outcome === "approved"),
+ ),
+ },
+ count: 1,
+ });
+ reply.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });
+ reply.calledTool("change-b", { status: "completed", output: { executions: 1 }, count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/approve-sibling.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/approve-sibling.eval.ts
new file mode 100644
index 000000000..51cf21ec3
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/approve-sibling.eval.ts
@@ -0,0 +1,36 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectResponseReply,
+ expectToolResult,
+ requestFrom,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "approve-sibling: resolving the current request must finish its work while change A still waits.",
+ tags: ["hitl", "continuation", "regression", "input-response", "approval"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given A and B await separate approvals, and B must be followed by a read.
+ const first = await t.send("Prepare change A.", scriptedSession);
+ const approvalA = requestFrom(first, "change-a");
+ const session = first.session;
+ const second = await session.send("Prepare change B, then read the draft status.");
+ const current = requestFrom(second, "change-b");
+
+ // When the user approves B only.
+ const live = await session.startRespond([
+ { requestId: current.requestId, optionId: "approve" },
+ ]);
+
+ // Then B executes once and the read gets a completed reply; A stays unexecuted and answerable.
+ await expectToolResult(t, live, "read-draft");
+ const reply = await expectResponseReply(t, live, "Draft status: ready.", current.requestId);
+ reply.calledTool("change-b", { status: "completed", output: { executions: 1 }, count: 1 });
+ expectChangeStillUnexecuted(session);
+ await approveSavedChange(t, session, approvalA);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/authorized-older-sibling.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/authorized-older-sibling.eval.ts
new file mode 100644
index 000000000..a91e91fe6
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/authorized-older-sibling.eval.ts
@@ -0,0 +1,47 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectResponseReply,
+ expectToolResult,
+ requestFrom,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "Approving the older response-authorized request completes its read and reply while newer approval A stays answerable.",
+ tags: ["hitl", "continuation", "regression", "input-response", "authorization"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given a response-authorized change is pending before an ordinary approval A.
+ const first = await t.send(
+ "Prepare an authorized change, then read the draft status.",
+ scriptedSession,
+ );
+ const current = requestFrom(first, "authorized-change");
+ const session = first.session;
+ const second = await session.send("Prepare change A.");
+ const approvalA = requestFrom(second, "change-a");
+
+ // When the authorized responder approves the older change.
+ const live = await session.startRespond([
+ { requestId: current.requestId, optionId: "approve" },
+ ]);
+
+ // Then authorization settles, the change executes once, and the read gets a reply; A stays answerable.
+ await expectToolResult(t, live, "read-draft");
+ const reply = await expectResponseReply(t, live, "Draft status: ready.", current.requestId);
+ reply.calledTool("authorized-change", {
+ status: "completed",
+ output: { executions: 1 },
+ count: 1,
+ });
+ reply.event("approval.settled", {
+ data: { requestId: current.requestId, outcome: "approved" },
+ count: 1,
+ });
+ expectChangeStillUnexecuted(session);
+ await approveSavedChange(t, session, approvalA);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/authorized-pending.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/authorized-pending.eval.ts
new file mode 100644
index 000000000..f200d2334
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/authorized-pending.eval.ts
@@ -0,0 +1,31 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectReply,
+ expectToolResult,
+ requestFrom,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "An approval requiring an authenticated responder cannot silence an unrelated tool reply.",
+ tags: ["hitl", "continuation", "regression", "user-message", "authorization"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given a change is waiting for approval from an authorized responder.
+ const parked = await t.send("Prepare an authorized change.", scriptedSession);
+ const approval = requestFrom(parked, "authorized-change");
+ const session = parked.session;
+
+ // When the user asks to read the draft.
+ const live = await session.start("Read the draft status.");
+
+ // Then the reply reports ready and completes; the authorized change stays answerable.
+ await expectToolResult(t, live, "read-draft");
+ await expectReply(t, live, "Draft status: ready.");
+ expectChangeStillUnexecuted(session, "authorized-change");
+ await approveSavedChange(t, session, approval);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/authorized-sibling.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/authorized-sibling.eval.ts
new file mode 100644
index 000000000..e0b6e0dfc
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/authorized-sibling.eval.ts
@@ -0,0 +1,44 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectResponseReply,
+ expectToolResult,
+ requestFrom,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "authorized-sibling: resolving the current request must finish its work while change A still waits.",
+ tags: ["hitl", "continuation", "regression", "input-response", "authorization"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given A awaits approval and a separate authorized change must be followed by a read.
+ const first = await t.send("Prepare change A.", scriptedSession);
+ const approvalA = requestFrom(first, "change-a");
+ const session = first.session;
+ const second = await session.send("Prepare an authorized change, then read the draft status.");
+ const current = requestFrom(second, "authorized-change");
+
+ // When the authorized responder approves the separate change.
+ const live = await session.startRespond([
+ { requestId: current.requestId, optionId: "approve" },
+ ]);
+
+ // Then authorization settles, the change executes once, and the read gets a reply; A stays answerable.
+ await expectToolResult(t, live, "read-draft");
+ const reply = await expectResponseReply(t, live, "Draft status: ready.", current.requestId);
+ reply.calledTool("authorized-change", {
+ status: "completed",
+ output: { executions: 1 },
+ count: 1,
+ });
+ reply.event("approval.settled", {
+ data: { requestId: current.requestId, outcome: "approved" },
+ count: 1,
+ });
+ expectChangeStillUnexecuted(session);
+ await approveSavedChange(t, session, approvalA);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/background-receipt.control.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/background-receipt.control.eval.ts
new file mode 100644
index 000000000..11d12aedf
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/background-receipt.control.eval.ts
@@ -0,0 +1,27 @@
+import { defineEval } from "eve/evals";
+import { scriptedSession, expectReply } from "./helpers.ts";
+
+export default defineEval({
+ description: "Control: background-receipt finishes without an older approval.",
+ tags: ["hitl", "continuation", "control", "user-message", "background-task"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given a fresh session has no pending approval.
+ const session = await t.session();
+
+ // When the user asks to start a background draft.
+ const live = await session.start(
+ "Start the background draft and acknowledge its receipt.",
+ scriptedSession,
+ );
+
+ // Then the reply acknowledges its working receipt; a later cancellation also gets a reply.
+ const turn = await expectReply(t, live, /^Background receipt: .*"status":"working"/);
+ turn.calledTool("background-draft", { count: 1 });
+ await expectReply(
+ t,
+ await session.start("Cancel the background draft task and confirm cancellation."),
+ /Cancellation result: .*"status":"cancelled"/,
+ );
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/background-receipt.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/background-receipt.eval.ts
new file mode 100644
index 000000000..49992fb9f
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/background-receipt.eval.ts
@@ -0,0 +1,37 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectReply,
+ requestFrom,
+ expectToolResult,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "Starting a background task must produce an acknowledgement while an older approval waits.",
+ tags: ["hitl", "continuation", "regression", "user-message", "background-task"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given change A is waiting for approval.
+ const parked = await t.send("Prepare change A.", scriptedSession);
+ const approval = requestFrom(parked, "change-a");
+ const session = parked.session;
+
+ // When the user asks to start a background draft.
+ const live = await session.start("Start the background draft and acknowledge its receipt.");
+
+ // Then the reply acknowledges its working receipt; A stays unexecuted and answerable.
+ await expectToolResult(t, live, "background-draft");
+ const turn = await expectReply(t, live, /^Background receipt: .*"status":"working"/);
+ turn.calledTool("background-draft", { count: 1 });
+ expectChangeStillUnexecuted(session);
+ await approveSavedChange(t, session, approval);
+ await expectReply(
+ t,
+ await session.start("Cancel the background draft task and confirm cancellation."),
+ /Cancellation result: .*"status":"cancelled"/,
+ );
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/budget-grant.control.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/budget-grant.control.eval.ts
new file mode 100644
index 000000000..2f75e5050
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/budget-grant.control.eval.ts
@@ -0,0 +1,30 @@
+import { defineEval } from "eve/evals";
+import { equals } from "eve/evals/expect";
+import { scriptedSession, expectToolResult, requestFrom } from "./helpers.ts";
+
+export default defineEval({
+ description: "Control: a renewed budget runs one tool, then asks for the next grant.",
+ tags: ["hitl", "continuation", "control", "input-response", "budget"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given a requested read is blocked on the exhausted session budget with no older approval.
+ const first = await t.send("Use the remaining budget to say hello.", scriptedSession);
+ const session = first.session;
+ const limited = await session.send("Read the draft status using the remaining budget.");
+ const budget = requestFrom(limited, "session_limit_continuation");
+
+ // When the user grants another budget window.
+ const live = await session.startRespond([
+ { requestId: budget.requestId, optionId: "continue" },
+ ]);
+
+ // Then the read executes and a new budget request appears before any final reply.
+ await expectToolResult(t, live, "read-draft");
+ const next = (await live.result()).expectOk();
+ const nextBudget = requestFrom(next, "session_limit_continuation");
+ await t.check(nextBudget.requestId === budget.requestId, equals(false));
+ next.calledTool("read-draft", { status: "completed", count: 1 });
+ next.notEvent("message.completed");
+ await session.respond([{ requestId: nextBudget.requestId, optionId: "stop" }]);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/budget-grant.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/budget-grant.eval.ts
new file mode 100644
index 000000000..0e55238fb
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/budget-grant.eval.ts
@@ -0,0 +1,37 @@
+import { defineEval } from "eve/evals";
+import { equals } from "eve/evals/expect";
+import {
+ scriptedSession,
+ expectChangeStillUnexecuted,
+ expectToolResult,
+ requestFrom,
+} from "./helpers.ts";
+
+export default defineEval({
+ description: "After a budget grant, an older approval cannot swallow the next budget request.",
+ tags: ["hitl", "continuation", "regression", "input-response", "budget"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given A awaits approval and a requested read is blocked on the exhausted session budget.
+ const first = await t.send("Prepare change A using the remaining budget.", scriptedSession);
+ requestFrom(first, "change-a");
+ const session = first.session;
+ const limited = await session.send("Read the draft status using the remaining budget.");
+ const budget = requestFrom(limited, "session_limit_continuation");
+
+ // When the user grants another budget window.
+ const live = await session.startRespond([
+ { requestId: budget.requestId, optionId: "continue" },
+ ]);
+
+ // Then the read executes and a new budget request appears; A stays unexecuted.
+ await expectToolResult(t, live, "read-draft");
+ const next = (await live.result()).expectOk();
+ const nextBudget = requestFrom(next, "session_limit_continuation");
+ await t.check(nextBudget.requestId === budget.requestId, equals(false));
+ next.calledTool("read-draft", { status: "completed", count: 1 });
+ next.notEvent("message.completed");
+ expectChangeStillUnexecuted(session);
+ await session.respond([{ requestId: nextBudget.requestId, optionId: "stop" }]);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/cancel-sibling.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/cancel-sibling.eval.ts
new file mode 100644
index 000000000..a63a22432
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/cancel-sibling.eval.ts
@@ -0,0 +1,36 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectResponseReply,
+ expectToolResult,
+ requestFrom,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "cancel-sibling: resolving the current request must finish its work while change A still waits.",
+ tags: ["hitl", "continuation", "regression", "input-response", "approval"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given A and B await separate approvals, and B must be followed by a read.
+ const first = await t.send("Prepare change A.", scriptedSession);
+ const approvalA = requestFrom(first, "change-a");
+ const session = first.session;
+ const second = await session.send("Prepare change B, then read the draft status.");
+ const current = requestFrom(second, "change-b");
+
+ // When the user cancels B only.
+ const live = await session.startRespond([{ requestId: current.requestId, optionId: "cancel" }]);
+
+ // Then B does not execute and the read gets a completed reply; A stays unexecuted and answerable.
+ await expectToolResult(t, live, "read-draft");
+ const reply = await expectResponseReply(t, live, "Draft status: ready.", current.requestId);
+ reply.notEvent("action.result", {
+ data: { status: "completed", result: { toolName: "change-b" } },
+ });
+ expectChangeStillUnexecuted(session);
+ await approveSavedChange(t, session, approvalA);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/helpers.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/helpers.ts
new file mode 100644
index 000000000..05fb398b2
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/helpers.ts
@@ -0,0 +1,156 @@
+import type {
+ EveEvalContext,
+ EveEvalLiveTurn,
+ EveEvalSession,
+ EveEvalTurn,
+ InputRequest,
+} from "eve/evals";
+import { equals, satisfies } from "eve/evals/expect";
+
+export const scriptedSession = { headers: { "x-eve-fixture-model": "continuation" } };
+
+export function requestFrom(turn: EveEvalTurn, toolName: string): InputRequest {
+ turn.expectOk();
+ const matches = turn.inputRequests.filter((request) => request.action.toolName === toolName);
+ if (matches.length !== 1)
+ throw new Error(`Expected one ${toolName} request; found ${matches.length}.`);
+ return matches[0]!;
+}
+
+export async function expectReply(
+ t: EveEvalContext,
+ live: EveEvalLiveTurn,
+ expected: string | RegExp,
+ owner?: string,
+): Promise<EveEvalTurn> {
+ t.log(`Accepted input in ${live.sessionId}; awaiting the reply and its turn completion.`);
+ const turnId = owner ?? (await live.waitForEvent("message.received")).data.turnId;
+ const turn = (await live.result()).expectOk();
+ const replies = turn.events.filter(
+ (event) =>
+ event.type === "message.completed" &&
+ event.data.turnId === turnId &&
+ typeof event.data.message === "string" &&
+ (typeof expected === "string"
+ ? event.data.message === expected
+ : expected.test(event.data.message)),
+ );
+ await t.require(
+ replies.length,
+ satisfies<number>(
+ (count) => count === 1,
+ `Exactly one reply matching ${String(expected)} in ${turnId}`,
+ ),
+ );
+ const completions = turn.events.filter(
+ (event) => event.type === "turn.completed" && event.data.turnId === turnId,
+ );
+ await t.require(
+ completions.length,
+ satisfies<number>((count) => count === 1, `Exactly one completion for ${turnId}`),
+ );
+ turn.eventOrder([
+ { type: "message.completed", data: { turnId, message: expected }, count: 1 },
+ { type: "turn.completed", data: { turnId }, count: 1 },
+ ]);
+ turn.notEvent("input.requested", { data: { turnId } });
+ t.log(`Checking answer and completion for ${turnId}.`);
+ return turn;
+}
+
+export async function expectResponseReply(
+ t: EveEvalContext,
+ live: EveEvalLiveTurn,
+ expected: string | RegExp,
+ requestId: string,
+): Promise<EveEvalTurn> {
+ t.log(`Accepted response for ${requestId}; awaiting resolution and its resumed turn.`);
+ await live.waitForEvent("input.resolved", {
+ data: { resolutions: (items) => items.some((item) => item.requestId === requestId) },
+ });
+ // Several responses in one delivery can resolve during the same resumed turn.
+ const resumed = await live.waitForEvent("turn.started");
+ const turn = await expectReply(t, live, expected, resumed.data.turnId);
+ turn.eventOrder([
+ {
+ type: "input.resolved",
+ data: {
+ resolutions: (items) =>
+ items.some(
+ (item) =>
+ item.requestId === requestId &&
+ item.outcome !== "ignored" &&
+ item.outcome !== "invalid",
+ ),
+ },
+ count: 1,
+ },
+ {
+ type: "message.completed",
+ data: { turnId: resumed.data.turnId, message: expected },
+ count: 1,
+ },
+ { type: "turn.completed", data: { turnId: resumed.data.turnId }, count: 1 },
+ ]);
+ turn.eventOrder([
+ { type: "turn.started", data: { turnId: resumed.data.turnId }, count: 1 },
+ {
+ type: "message.completed",
+ data: { turnId: resumed.data.turnId, message: expected },
+ count: 1,
+ },
+ ]);
+ return turn;
+}
+
+export async function expectToolResult(t: EveEvalContext, live: EveEvalLiveTurn, toolName: string) {
+ t.log(`Accepted input in ${live.sessionId}; awaiting ${toolName}.`);
+ const event = await live.waitForEvent("action.result", { data: { result: { toolName } } });
+ t.log(`${toolName} returned before the reply: ${JSON.stringify(event.data)}`);
+ return event;
+}
+
+export function expectChangeStillUnexecuted(session: EveEvalSession, toolName = "change-a") {
+ session.notEvent("action.result", { data: { result: { toolName } } });
+}
+
+// A partial approval has no turn boundary to await. Await the real HTTP
+// acceptance, then send the next message on that same session's ordered inbox.
+export async function submitPartialApproval(
+ t: EveEvalContext,
+ session: EveEvalSession,
+ request: InputRequest,
+) {
+ const response = await t.target.fetch(
+ `/eve/v1/session/${encodeURIComponent(session.sessionId)}`,
+ {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ signal: t.signal,
+ body: JSON.stringify({
+ inputResponses: [{ requestId: request.requestId, optionId: "approve" }],
+ }),
+ },
+ );
+ await t.require(response.status, equals(202));
+ const accepted = await response.json();
+ t.log(`Partial approval accepted: ${JSON.stringify(accepted)}`);
+}
+
+export async function approveSavedChange(
+ t: EveEvalContext,
+ session: EveEvalSession,
+ request: InputRequest,
+) {
+ const live = await session.startRespond([{ requestId: request.requestId, optionId: "approve" }]);
+ const approved = await expectResponseReply(t, live, /\S/, request.requestId);
+ approved.calledTool(request.action.toolName, {
+ status: "completed",
+ output: { executions: 1 },
+ count: 1,
+ });
+ session.event("action.result", {
+ data: { result: { toolName: request.action.toolName } },
+ count: 1,
+ });
+}
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/invalid-input.control.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/invalid-input.control.eval.ts
new file mode 100644
index 000000000..c10abe374
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/invalid-input.control.eval.ts
@@ -0,0 +1,26 @@
+import { defineEval } from "eve/evals";
+import { scriptedSession, expectReply } from "./helpers.ts";
+
+export default defineEval({
+ description: "Control: invalid-input finishes without an older approval.",
+ tags: ["hitl", "continuation", "control", "user-message", "validation"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given a fresh session has no pending approval.
+ const session = await t.session();
+
+ // When the user asks for a read that first uses an invalid draft ID.
+ const live = await session.start(
+ "Try a numeric draft ID, then correct it and read the status.",
+ scriptedSession,
+ );
+
+ // Then the model corrects the ID and replies with ready plus the validation error.
+ const turn = await expectReply(t, live, /Draft status: ready\. Validation error: .*draftId/);
+ turn.calledTool("read-draft", {
+ status: "completed",
+ input: { draftId: "draft-3494" },
+ count: 1,
+ });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/invalid-input.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/invalid-input.eval.ts
new file mode 100644
index 000000000..580c3fa5c
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/invalid-input.eval.ts
@@ -0,0 +1,40 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectReply,
+ requestFrom,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "The model must be able to correct invalid tool input while an older approval waits.",
+ tags: ["hitl", "continuation", "regression", "user-message", "validation"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given change A is waiting for approval.
+ const parked = await t.send("Prepare change A.", scriptedSession);
+ const approval = requestFrom(parked, "change-a");
+ const session = parked.session;
+
+ // When the user asks for a read that first uses an invalid draft ID.
+ const live = await session.start(
+ "Try a numeric draft ID, then correct it and read the status.",
+ );
+
+ // Then the model corrects the input and reports ready plus the validation error; A stays answerable.
+ const rejectedStep = await live.waitForEvent("step.completed", { data: { stepIndex: 0 } });
+ t.log(
+ `Initial tool-call step completed; awaiting correction: ${JSON.stringify(rejectedStep.data)}`,
+ );
+ const turn = await expectReply(t, live, /Draft status: ready\. Validation error: .*draftId/);
+ turn.calledTool("read-draft", {
+ status: "completed",
+ input: { draftId: "draft-3494" },
+ count: 1,
+ });
+ expectChangeStillUnexecuted(session);
+ await approveSavedChange(t, session, approval);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/multiple-questions.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/multiple-questions.eval.ts
new file mode 100644
index 000000000..8bfa2010d
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/multiple-questions.eval.ts
@@ -0,0 +1,64 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ expectReply,
+ expectResponseReply,
+ expectToolResult,
+ requestFrom,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "Two unanswered questions cannot swallow a tool reply even when no approval remains.",
+ tags: ["hitl", "continuation", "regression", "user-message", "question"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given two questions remain unanswered after the only approval is cancelled.
+ const first = await t.send("Prepare change A.", scriptedSession);
+ const approval = requestFrom(first, "change-a");
+ const session = first.session;
+ const color = await session.send("Ask which color to use, then read the draft status.");
+ const colorRequest = requestFrom(color, "ask_question");
+ const size = await session.send("Ask which size to use.");
+ const sizeRequest = requestFrom(size, "ask_question");
+ await expectResponseReply(
+ t,
+ await session.startRespond([{ requestId: approval.requestId, optionId: "cancel" }]),
+ "Change A resolved.",
+ approval.requestId,
+ );
+ session.notEvent("action.result", {
+ data: { status: "completed", result: { toolName: "change-a" } },
+ });
+
+ // When the user asks to read the draft.
+ const live = await session.start("Read the draft status.");
+
+ // Then the read gets a completed reply and neither question is silently answered.
+ await expectToolResult(t, live, "read-draft");
+ await expectReply(t, live, "Draft status: ready.");
+ session.eventsSatisfy("Neither unanswered question was silently resolved", (events) =>
+ events.every(
+ (event) =>
+ event.type !== "input.resolved" ||
+ event.data.resolutions.every(
+ (resolution) =>
+ resolution.requestId !== colorRequest.requestId &&
+ resolution.requestId !== sizeRequest.requestId,
+ ),
+ ),
+ );
+ await expectResponseReply(
+ t,
+ await session.startRespond([{ requestId: colorRequest.requestId, optionId: "red" }]),
+ "Draft status: ready.",
+ colorRequest.requestId,
+ );
+ await expectResponseReply(
+ t,
+ await session.startRespond([{ requestId: sizeRequest.requestId, optionId: "small" }]),
+ "Size resolved.",
+ sizeRequest.requestId,
+ );
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/ordinary-older-sibling.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/ordinary-older-sibling.eval.ts
new file mode 100644
index 000000000..ada67dd15
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/ordinary-older-sibling.eval.ts
@@ -0,0 +1,42 @@
+import { defineEval } from "eve/evals";
+import {
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectResponseReply,
+ requestFrom,
+ scriptedSession,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "An older ordinary approval completes while a newer response-authorized approval stays answerable.",
+ tags: ["hitl", "continuation", "regression", "input-response", "approval", "authorization"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given A awaits approval and a newer authorized change also awaits approval.
+ const first = await t.send("Prepare change A.", scriptedSession);
+ const approvalA = requestFrom(first, "change-a");
+ const session = first.session;
+ const second = await session.send("Prepare an authorized change, then read the draft status.");
+ const newer = requestFrom(second, "authorized-change");
+
+ // When the user resolves the older request first.
+ const live = await session.startRespond([
+ { requestId: approvalA.requestId, optionId: "approve" },
+ ]);
+ const resolved = await expectResponseReply(t, live, "Change A resolved.", approvalA.requestId);
+
+ // Then A executes once, and the newer request remains answerable.
+ resolved.calledTool("change-a", {
+ status: "completed",
+ output: { executions: 1 },
+ count: 1,
+ });
+ resolved.event("input.resolved", {
+ data: { resolutions: [{ requestId: approvalA.requestId }] },
+ count: 1,
+ });
+ expectChangeStillUnexecuted(session, "authorized-change");
+ await approveSavedChange(t, session, newer);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/parallel-tools.control.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/parallel-tools.control.eval.ts
new file mode 100644
index 000000000..e082f2d5b
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/parallel-tools.control.eval.ts
@@ -0,0 +1,20 @@
+import { defineEval } from "eve/evals";
+import { scriptedSession, expectReply } from "./helpers.ts";
+
+export default defineEval({
+ description: "Control: parallel-tools finishes without an older approval.",
+ tags: ["hitl", "continuation", "control", "user-message", "tool-result"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given a fresh session has no pending approval.
+ const session = await t.session();
+
+ // When the user asks to read and save in parallel.
+ const live = await session.start("Read and save the draft in parallel.", scriptedSession);
+
+ // Then both tools execute once and their results appear in a completed reply.
+ const turn = await expectReply(t, live, 'Draft status: ready. Draft saved: {"writes":1}.');
+ turn.calledTool("read-draft", { status: "completed", count: 1 });
+ turn.calledTool("save-draft", { status: "completed", count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/parallel-tools.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/parallel-tools.eval.ts
new file mode 100644
index 000000000..3c9e91a3f
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/parallel-tools.eval.ts
@@ -0,0 +1,32 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectReply,
+ requestFrom,
+ expectToolResult,
+} from "./helpers.ts";
+
+export default defineEval({
+ description: "An older approval cannot silence the reply after parallel reads and writes.",
+ tags: ["hitl", "continuation", "regression", "user-message", "tool-result"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given change A is waiting for approval.
+ const parked = await t.send("Prepare change A.", scriptedSession);
+ const approval = requestFrom(parked, "change-a");
+ const session = parked.session;
+
+ // When the user asks to read and save in parallel.
+ const live = await session.start("Read and save the draft in parallel.");
+
+ // Then the reply reports both results and completes; A stays unexecuted and answerable.
+ await expectToolResult(t, live, "save-draft");
+ const turn = await expectReply(t, live, 'Draft status: ready. Draft saved: {"writes":1}.');
+ turn.calledTool("read-draft", { status: "completed", count: 1 });
+ turn.calledTool("save-draft", { status: "completed", count: 1 });
+ expectChangeStillUnexecuted(session);
+ await approveSavedChange(t, session, approval);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/partial-approval-text.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/partial-approval-text.eval.ts
new file mode 100644
index 000000000..0b2254ab4
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/partial-approval-text.eval.ts
@@ -0,0 +1,51 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ expectChangeStillUnexecuted,
+ expectReply,
+ expectResponseReply,
+ requestFrom,
+ submitPartialApproval,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "A partial approval must not prevent a new text-only message from receiving a completed reply.",
+ tags: ["hitl", "continuation", "regression", "user-message", "partial-approval"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given A and B were requested together and only A has an accepted approval response.
+ const parked = await t.send("Prepare changes A and B together.", scriptedSession);
+ const approvalA = requestFrom(parked, "change-a");
+ const approvalB = requestFrom(parked, "change-b");
+ const session = parked.session;
+ await submitPartialApproval(t, session, approvalA);
+
+ // When the user asks what is waiting without requesting tools.
+ const live = await session.start("Explain what is waiting, without calling any tools.");
+
+ // Then the text reply completes; the batch stays unexecuted until B is approved.
+ const reply = await expectReply(t, live, "Your changes are waiting for approval.");
+ reply.usedNoTools();
+ expectChangeStillUnexecuted(session, "change-a");
+ expectChangeStillUnexecuted(session, "change-b");
+
+ const approved = await expectResponseReply(
+ t,
+ await session.startRespond([{ requestId: approvalB.requestId, optionId: "approve" }]),
+ "Both changes resolved.",
+ approvalB.requestId,
+ );
+ approved.event("input.resolved", {
+ data: {
+ resolutions: (items) =>
+ [approvalA.requestId, approvalB.requestId].every((id) =>
+ items.some((item) => item.requestId === id && item.outcome === "approved"),
+ ),
+ },
+ count: 1,
+ });
+ approved.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });
+ approved.calledTool("change-b", { status: "completed", output: { executions: 1 }, count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/partial-approval-tool.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/partial-approval-tool.eval.ts
new file mode 100644
index 000000000..5d8097d38
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/partial-approval-tool.eval.ts
@@ -0,0 +1,52 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ expectChangeStillUnexecuted,
+ expectReply,
+ expectResponseReply,
+ expectToolResult,
+ requestFrom,
+ submitPartialApproval,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "A partial approval must not prevent a new tool message from receiving a completed reply.",
+ tags: ["hitl", "continuation", "regression", "user-message", "partial-approval"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given A and B were requested together and only A has an accepted approval response.
+ const parked = await t.send("Prepare changes A and B together.", scriptedSession);
+ const approvalA = requestFrom(parked, "change-a");
+ const approvalB = requestFrom(parked, "change-b");
+ const session = parked.session;
+ await submitPartialApproval(t, session, approvalA);
+
+ // When the user asks to read the draft.
+ const live = await session.start("Read the draft status.");
+
+ // Then the read gets a completed reply; the batch stays unexecuted until B is approved.
+ await expectToolResult(t, live, "read-draft");
+ await expectReply(t, live, "Draft status: ready.");
+ expectChangeStillUnexecuted(session, "change-a");
+ expectChangeStillUnexecuted(session, "change-b");
+
+ const approved = await expectResponseReply(
+ t,
+ await session.startRespond([{ requestId: approvalB.requestId, optionId: "approve" }]),
+ "Both changes resolved.",
+ approvalB.requestId,
+ );
+ approved.event("input.resolved", {
+ data: {
+ resolutions: (items) =>
+ [approvalA.requestId, approvalB.requestId].every((id) =>
+ items.some((item) => item.requestId === id && item.outcome === "approved"),
+ ),
+ },
+ count: 1,
+ });
+ approved.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });
+ approved.calledTool("change-b", { status: "completed", output: { executions: 1 }, count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/provider-result.control.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/provider-result.control.eval.ts
new file mode 100644
index 000000000..22cf621be
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/provider-result.control.eval.ts
@@ -0,0 +1,26 @@
+import { defineEval } from "eve/evals";
+import { scriptedSession, expectReply } from "./helpers.ts";
+
+export default defineEval({
+ description: "Control: provider-result finishes without an older approval.",
+ tags: ["hitl", "continuation", "control", "user-message", "provider-result"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given a fresh session has no pending approval.
+ const session = await t.session();
+
+ // When the user asks the provider to look up the draft.
+ const live = await session.start(
+ "Look up the draft with the provider and report its status.",
+ scriptedSession,
+ );
+
+ // Then the reply reports provider-ready and completes its turn.
+ const reply = await expectReply(t, live, "Provider draft status: provider-ready.");
+ reply.calledTool("read-draft", {
+ status: "completed",
+ output: { status: "provider-ready" },
+ count: 1,
+ });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/provider-result.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/provider-result.eval.ts
new file mode 100644
index 000000000..13e685a88
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/provider-result.eval.ts
@@ -0,0 +1,36 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectReply,
+ expectToolResult,
+ requestFrom,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "A provider-executed tool result must get a follow-up model call while an older approval waits.",
+ tags: ["hitl", "continuation", "regression", "user-message", "provider-result"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given change A is waiting for approval.
+ const parked = await t.send("Prepare change A.", scriptedSession);
+ const approval = requestFrom(parked, "change-a");
+ const session = parked.session;
+
+ // When the user asks the provider to look up the draft.
+ const live = await session.start("Look up the draft with the provider and report its status.");
+
+ // Then the reply reports provider-ready and completes; A stays unexecuted and answerable.
+ await expectToolResult(t, live, "read-draft");
+ const reply = await expectReply(t, live, "Provider draft status: provider-ready.");
+ reply.calledTool("read-draft", {
+ status: "completed",
+ output: { status: "provider-ready" },
+ count: 1,
+ });
+ expectChangeStillUnexecuted(session);
+ await approveSavedChange(t, session, approval);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/read.control.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/read.control.eval.ts
new file mode 100644
index 000000000..09c5f16c9
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/read.control.eval.ts
@@ -0,0 +1,19 @@
+import { defineEval } from "eve/evals";
+import { scriptedSession, expectReply } from "./helpers.ts";
+
+export default defineEval({
+ description: "Control: read finishes without an older approval.",
+ tags: ["hitl", "continuation", "control", "user-message", "tool-result"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given a fresh session has no pending approval.
+ const session = await t.session();
+
+ // When the user asks to read the draft.
+ const live = await session.start("Read the draft status.", scriptedSession);
+
+ // Then the reply reports ready and completes its turn.
+ const turn = await expectReply(t, live, "Draft status: ready.");
+ turn.calledTool("read-draft", { status: "completed", count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/read.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/read.eval.ts
new file mode 100644
index 000000000..48f262ef9
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/read.eval.ts
@@ -0,0 +1,31 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectReply,
+ requestFrom,
+ expectToolResult,
+} from "./helpers.ts";
+
+export default defineEval({
+ description: "An older approval cannot silence the answer after a read.",
+ tags: ["hitl", "continuation", "regression", "user-message", "tool-result"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given change A is waiting for approval.
+ const parked = await t.send("Prepare change A.", scriptedSession);
+ const approval = requestFrom(parked, "change-a");
+ const session = parked.session;
+
+ // When the user asks to read the draft.
+ const live = await session.start("Read the draft status.");
+
+ // Then the reply reports ready and completes; A stays unexecuted and answerable.
+ await expectToolResult(t, live, "read-draft");
+ const turn = await expectReply(t, live, "Draft status: ready.");
+ turn.calledTool("read-draft", { status: "completed", count: 1 });
+ expectChangeStillUnexecuted(session);
+ await approveSavedChange(t, session, approval);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/resolve-only-approval.control.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/resolve-only-approval.control.eval.ts
new file mode 100644
index 000000000..131d502f8
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/resolve-only-approval.control.eval.ts
@@ -0,0 +1,23 @@
+import { defineEval } from "eve/evals";
+import { scriptedSession, expectResponseReply, requestFrom } from "./helpers.ts";
+
+export default defineEval({
+ description: "Control: resolving the only approval runs its tool, reads the status, and replies.",
+ tags: ["hitl", "continuation", "control", "input-response", "approval"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given only B awaits approval and must be followed by a read.
+ const parked = await t.send("Prepare change B, then read the draft status.", scriptedSession);
+ const approval = requestFrom(parked, "change-b");
+
+ // When the user approves B.
+ const live = await parked.session.startRespond([
+ { requestId: approval.requestId, optionId: "approve" },
+ ]);
+
+ // Then B executes once, the draft is read, and the reply completes.
+ const reply = await expectResponseReply(t, live, "Draft status: ready.", approval.requestId);
+ reply.calledTool("change-b", { status: "completed", output: { executions: 1 }, count: 1 });
+ reply.calledTool("read-draft", { status: "completed", count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/runtime-control.control.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/runtime-control.control.eval.ts
new file mode 100644
index 000000000..7d11a7291
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/runtime-control.control.eval.ts
@@ -0,0 +1,27 @@
+import { defineEval } from "eve/evals";
+import { scriptedSession, expectReply } from "./helpers.ts";
+
+export default defineEval({
+ description: "Control: runtime-control finishes without an older approval.",
+ tags: ["hitl", "continuation", "control", "user-message", "runtime-control", "background-task"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given a background draft is running and its receipt has been acknowledged.
+ const session = await t.session();
+ await expectReply(
+ t,
+ await session.start(
+ "Start the background draft and acknowledge its receipt.",
+ scriptedSession,
+ ),
+ /^Background receipt: .*"status":"working"/,
+ );
+
+ // When the user asks to cancel that task.
+ const live = await session.start("Cancel the background draft task and confirm cancellation.");
+
+ // Then task_cancel completes once and its cancellation result gets a completed reply.
+ const turn = await expectReply(t, live, /Cancellation result: .*"status":"cancelled"/);
+ turn.calledTool("task_cancel", { status: "completed", count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/runtime-control.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/runtime-control.eval.ts
new file mode 100644
index 000000000..a712f2a2c
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/runtime-control.eval.ts
@@ -0,0 +1,44 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ expectChangeStillUnexecuted,
+ expectReply,
+ requestFrom,
+ expectToolResult,
+} from "./helpers.ts";
+
+export default defineEval({
+ description: "A task-control result must be interpreted while an older approval waits.",
+ tags: [
+ "hitl",
+ "continuation",
+ "regression",
+ "user-message",
+ "runtime-control",
+ "background-task",
+ ],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given a background draft is running and change A is waiting for approval.
+ const session = await t.session();
+ await expectReply(
+ t,
+ await session.start(
+ "Start the background draft and acknowledge its receipt.",
+ scriptedSession,
+ ),
+ /^Background receipt: .*"status":"working"/,
+ );
+ const parked = await session.send("Prepare change A.");
+ requestFrom(parked, "change-a");
+
+ // When the user asks to cancel that task.
+ const live = await session.start("Cancel the background draft task and confirm cancellation.");
+
+ // Then the reply confirms cancellation and completes; A stays unexecuted.
+ await expectToolResult(t, live, "task_cancel");
+ const turn = await expectReply(t, live, /Cancellation result: .*"status":"cancelled"/);
+ turn.calledTool("task_cancel", { status: "completed", count: 1 });
+ expectChangeStillUnexecuted(session);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/same-turn-workflow.control.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/same-turn-workflow.control.eval.ts
new file mode 100644
index 000000000..db0dbab6b
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/same-turn-workflow.control.eval.ts
@@ -0,0 +1,43 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ expectChangeStillUnexecuted,
+ expectResponseReply,
+ requestFrom,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "Control: a workflow finishing beside an approval cannot bypass that turn's approval.",
+ tags: ["hitl", "continuation", "control", "user-message", "input-response", "workflow"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given a fresh session with no pending input.
+ // When the user requests change A and a workflow read together.
+ const parked = await t.send(
+ "Prepare change A and read the workflow draft together.",
+ scriptedSession,
+ );
+ // Then A still requires approval and the workflow cannot produce a final reply alone.
+ const approval = requestFrom(parked, "change-a");
+ parked.calledTool("workflow-draft", { status: "completed", count: 1 });
+ parked.notEvent("message.completed");
+ const session = parked.session;
+ expectChangeStillUnexecuted(session);
+
+ // When the user approves A.
+ const live = await session.startRespond([
+ { requestId: approval.requestId, optionId: "approve" },
+ ]);
+
+ // Then both tools execute once and their combined reply completes the resumed turn.
+ await expectResponseReply(
+ t,
+ live,
+ "Workflow draft status: ready. Change A resolved.",
+ approval.requestId,
+ );
+ session.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });
+ session.calledTool("workflow-draft", { status: "completed", count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/separate-approval-responses.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/separate-approval-responses.eval.ts
new file mode 100644
index 000000000..3052a3cb6
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/separate-approval-responses.eval.ts
@@ -0,0 +1,41 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ expectResponseReply,
+ requestFrom,
+ submitPartialApproval,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "Separate authenticated responses must accumulate until both approvals from one batch are answered.",
+ tags: ["hitl", "continuation", "regression", "input-response", "partial-approval"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given A and B were requested together and only A has an accepted approval response.
+ const parked = await t.send("Prepare changes A and B together.", scriptedSession);
+ const approvalA = requestFrom(parked, "change-a");
+ const approvalB = requestFrom(parked, "change-b");
+ const session = parked.session;
+ await submitPartialApproval(t, session, approvalA);
+
+ // When the user approves B in a separate HTTP request.
+ const live = await session.startRespond([
+ { requestId: approvalB.requestId, optionId: "approve" },
+ ]);
+
+ // Then both changes execute once and the resumed turn replies that both resolved.
+ const reply = await expectResponseReply(t, live, "Both changes resolved.", approvalB.requestId);
+ reply.event("input.resolved", {
+ data: {
+ resolutions: (items) =>
+ [approvalA.requestId, approvalB.requestId].every((id) =>
+ items.some((item) => item.requestId === id && item.outcome === "approved"),
+ ),
+ },
+ count: 1,
+ });
+ reply.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });
+ reply.calledTool("change-b", { status: "completed", output: { executions: 1 }, count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/stale-response.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/stale-response.eval.ts
new file mode 100644
index 000000000..5d08675d7
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/stale-response.eval.ts
@@ -0,0 +1,44 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectReply,
+ expectResponseReply,
+ expectToolResult,
+ requestFrom,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "A repeated answer becomes new input and must get a reply without authorizing an old change.",
+ tags: ["hitl", "continuation", "regression", "input-response", "stale-response"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given A awaits approval and B has already been cancelled and acknowledged.
+ const first = await t.send("Prepare change A.", scriptedSession);
+ const approvalA = requestFrom(first, "change-a");
+ const session = first.session;
+ const second = await session.send("Prepare change B, then acknowledge my decision.");
+ const approvalB = requestFrom(second, "change-b");
+ const cancelB = [{ requestId: approvalB.requestId, optionId: "cancel" }];
+ await expectResponseReply(
+ t,
+ await session.startRespond(cancelB),
+ "Change B resolved.",
+ approvalB.requestId,
+ );
+
+ // When the user repeats the saved cancellation response for B.
+ const repeated = await session.startRespond(cancelB);
+
+ // Then the response becomes new input and gets a read reply; neither change executes.
+ await expectToolResult(t, repeated, "read-draft");
+ await expectReply(t, repeated, "Draft status: ready.");
+ expectChangeStillUnexecuted(session, "change-a");
+ session.notEvent("action.result", {
+ data: { status: "completed", result: { toolName: "change-b" } },
+ });
+ await approveSavedChange(t, session, approvalA);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/text-only.control.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/text-only.control.eval.ts
new file mode 100644
index 000000000..0679a7605
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/text-only.control.eval.ts
@@ -0,0 +1,30 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectReply,
+ requestFrom,
+} from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "Control: a text-only follow-up completes while the original approval stays answerable.",
+ tags: ["hitl", "continuation", "control", "user-message", "text-reply"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given change A is waiting for approval.
+ const parked = await t.send("Prepare change A.", scriptedSession);
+ const approval = requestFrom(parked, "change-a");
+ const session = parked.session;
+
+ // When the user asks what is waiting without requesting tools.
+ const live = await session.start("Explain what is waiting, without calling any tools.");
+
+ // Then the text reply completes; A stays unexecuted and answerable.
+ const reply = await expectReply(t, live, "Your changes are waiting for approval.");
+ reply.usedNoTools();
+ expectChangeStillUnexecuted(session);
+ await approveSavedChange(t, session, approval);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/tool-error.control.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/tool-error.control.eval.ts
new file mode 100644
index 000000000..936dc38d6
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/tool-error.control.eval.ts
@@ -0,0 +1,26 @@
+import { defineEval } from "eve/evals";
+import { scriptedSession, expectReply } from "./helpers.ts";
+
+export default defineEval({
+ description: "Control: tool-error finishes without an older approval.",
+ tags: ["hitl", "continuation", "control", "user-message", "tool-error"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given a fresh session has no pending approval.
+ const session = await t.session();
+
+ // When the user asks to read from a failing tool.
+ const live = await session.start(
+ "Try the unavailable draft store and explain the error.",
+ scriptedSession,
+ );
+
+ // Then the reply explains the actual tool error and completes its turn.
+ const turn = await expectReply(
+ t,
+ live,
+ /Could not read the draft: .*The draft store is unavailable/,
+ );
+ turn.calledTool("unavailable-draft", { status: "failed", count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/tool-error.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/tool-error.eval.ts
new file mode 100644
index 000000000..8c6c0bf58
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/tool-error.eval.ts
@@ -0,0 +1,35 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectReply,
+ requestFrom,
+ expectToolResult,
+} from "./helpers.ts";
+
+export default defineEval({
+ description: "A tool error must reach the user while an older approval remains open.",
+ tags: ["hitl", "continuation", "regression", "user-message", "tool-error"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given change A is waiting for approval.
+ const parked = await t.send("Prepare change A.", scriptedSession);
+ const approval = requestFrom(parked, "change-a");
+ const session = parked.session;
+
+ // When the user asks to read from a failing tool.
+ const live = await session.start("Try the unavailable draft store and explain the error.");
+
+ // Then the reply explains the actual error and completes; A stays unexecuted and answerable.
+ await expectToolResult(t, live, "unavailable-draft");
+ const turn = await expectReply(
+ t,
+ live,
+ /Could not read the draft: .*The draft store is unavailable/,
+ );
+ turn.calledTool("unavailable-draft", { status: "failed", count: 1 });
+ expectChangeStillUnexecuted(session);
+ await approveSavedChange(t, session, approval);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/workflow-result.control.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/workflow-result.control.eval.ts
new file mode 100644
index 000000000..b0678ebee
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/workflow-result.control.eval.ts
@@ -0,0 +1,19 @@
+import { defineEval } from "eve/evals";
+import { scriptedSession, expectReply } from "./helpers.ts";
+
+export default defineEval({
+ description: "Control: workflow-result finishes without an older approval.",
+ tags: ["hitl", "continuation", "control", "user-message", "workflow"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given a fresh session has no pending approval.
+ const session = await t.session();
+
+ // When the user asks a workflow to read the draft.
+ const live = await session.start("Read the draft through a workflow.", scriptedSession);
+
+ // Then the reply reports the workflow result and completes its turn.
+ const turn = await expectReply(t, live, "Workflow draft status: ready.");
+ turn.calledTool("workflow-draft", { status: "completed", count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/workflow-result.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/workflow-result.eval.ts
new file mode 100644
index 000000000..81a161c03
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/workflow-result.eval.ts
@@ -0,0 +1,31 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectReply,
+ requestFrom,
+ expectToolResult,
+} from "./helpers.ts";
+
+export default defineEval({
+ description: "A completed workflow result must be interpreted for the current user message.",
+ tags: ["hitl", "continuation", "regression", "user-message", "workflow"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given change A is waiting for approval.
+ const parked = await t.send("Prepare change A.", scriptedSession);
+ const approval = requestFrom(parked, "change-a");
+ const session = parked.session;
+
+ // When the user asks a workflow to read the draft.
+ const live = await session.start("Read the draft through a workflow.");
+
+ // Then the reply reports the workflow result and completes; A stays unexecuted and answerable.
+ await expectToolResult(t, live, "workflow-draft");
+ const turn = await expectReply(t, live, "Workflow draft status: ready.");
+ turn.calledTool("workflow-draft", { status: "completed", count: 1 });
+ expectChangeStillUnexecuted(session);
+ await approveSavedChange(t, session, approval);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/write.control.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/write.control.eval.ts
new file mode 100644
index 000000000..33f329915
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/write.control.eval.ts
@@ -0,0 +1,22 @@
+import { defineEval } from "eve/evals";
+import { scriptedSession, expectReply } from "./helpers.ts";
+
+export default defineEval({
+ description: "Control: write finishes without an older approval.",
+ tags: ["hitl", "continuation", "control", "user-message", "tool-result"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given a fresh session has no pending approval.
+ const session = await t.session();
+
+ // When the user asks to save the draft.
+ const live = await session.start(
+ "Save the draft and report how many times it was written.",
+ scriptedSession,
+ );
+
+ // Then the reply confirms one write and completes its turn.
+ const turn = await expectReply(t, live, 'Draft saved: {"writes":1}.');
+ turn.calledTool("save-draft", { status: "completed", count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/write.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/write.eval.ts
new file mode 100644
index 000000000..d6769f0ce
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/write.eval.ts
@@ -0,0 +1,31 @@
+import { defineEval } from "eve/evals";
+import {
+ scriptedSession,
+ approveSavedChange,
+ expectChangeStillUnexecuted,
+ expectReply,
+ requestFrom,
+ expectToolResult,
+} from "./helpers.ts";
+
+export default defineEval({
+ description: "An older approval cannot silence confirmation of a completed write.",
+ tags: ["hitl", "continuation", "regression", "user-message", "tool-result"],
+ timeoutMs: 60_000,
+ async test(t) {
+ // Given change A is waiting for approval.
+ const parked = await t.send("Prepare change A.", scriptedSession);
+ const approval = requestFrom(parked, "change-a");
+ const session = parked.session;
+
+ // When the user asks to save the draft.
+ const live = await session.start("Save the draft and report how many times it was written.");
+
+ // Then the reply confirms one write and completes; A stays unexecuted and answerable.
+ await expectToolResult(t, live, "save-draft");
+ const turn = await expectReply(t, live, 'Draft saved: {"writes":1}.');
+ turn.calledTool("save-draft", { status: "completed", count: 1 });
+ expectChangeStillUnexecuted(session);
+ await approveSavedChange(t, session, approval);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/pending-approval-tool-followup.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/pending-approval-tool-followup.eval.ts
new file mode 100644
index 000000000..74036095d
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/pending-approval-tool-followup.eval.ts
@@ -0,0 +1,125 @@
+import { defineEval } from "eve/evals";
+import { equals } from "eve/evals/expect";
+import { requestFrom } from "./continuation/helpers.ts";
+
+const MARKER = "draft-status-3494";
+const READ_STATUS =
+ `Alice is checking her draft. Call the read-status tool exactly once with marker "${MARKER}". ` +
+ "After the tool returns, tell Alice the status and marker from its result.";
+
+export default [
+ defineEval({
+ description:
+ "An ungated status tool completes and produces a reply without a pending approval.",
+ tags: ["hitl", "continuation", "control", "user-message", "tool-result"],
+ timeoutMs: 120_000,
+ async test(t) {
+ // Given a fresh session with no pending approval.
+ // When the user asks to read the draft status.
+ const session = await t.session();
+ const live = await session.start(READ_STATUS);
+ const received = await live.waitForEvent("message.received");
+ const turn = await live.result();
+
+ // Then the tool executes once and the completed reply includes its status and marker.
+ turn.expectOk();
+ turn.calledTool("read-status", { status: "completed", count: 1 });
+ turn.event("turn.completed", { count: 1 });
+ turn.messageIncludes(MARKER);
+ turn.messageIncludes("ready");
+ turn.eventOrder([
+ { type: "message.received", data: { turnId: received.data.turnId }, count: 1 },
+ {
+ type: "action.result",
+ data: {
+ turnId: received.data.turnId,
+ status: "completed",
+ result: { toolName: "read-status" },
+ },
+ count: 1,
+ },
+ {
+ type: "message.completed",
+ data: {
+ turnId: received.data.turnId,
+ message: (text) =>
+ typeof text === "string" && text.includes(MARKER) && text.includes("ready"),
+ },
+ count: 1,
+ },
+ { type: "turn.completed", data: { turnId: received.data.turnId }, count: 1 },
+ ]);
+ },
+ }),
+ defineEval({
+ description:
+ "An ungated tool follow-up completes while an older approval remains answerable (#3494).",
+ tags: ["hitl", "continuation", "regression", "user-message", "tool-result"],
+ timeoutMs: 120_000,
+ async test(t) {
+ // Given an account change is waiting for approval.
+ const parked = await t.send(
+ 'Alice is preparing an account change. Call the gate tool exactly once with marker "account-change-3494".',
+ );
+ const session = parked.session;
+ parked.calledTool("gate", { status: "pending", count: 1 });
+ parked.notEvent("action.result", { data: { result: { toolName: "gate" } } });
+ const approval = requestFrom(parked, "gate");
+ t.log(`Original gate approval is pending: ${approval.requestId}`);
+
+ // When the user leaves that approval pending and asks to read the draft status.
+ const live = await session.start(
+ `Alice will review the account change later. Leave its approval pending. ${READ_STATUS}`,
+ );
+ // Then the tool result reaches a completed reply without executing the account change.
+ const result = await live.waitForEvent("action.result", {
+ data: { status: "completed", result: { toolName: "read-status" } },
+ });
+ t.log(`Follow-up tool completed before waiting for its reply: ${JSON.stringify(result)}`);
+ const received = await live.waitForEvent("message.received");
+ await live.waitForEvent("turn.completed", { data: { turnId: received.data.turnId } });
+ const followup = await live.result();
+ followup.expectOk();
+ followup.calledTool("read-status", { status: "completed", count: 1 });
+ followup.messageIncludes(MARKER);
+ followup.messageIncludes("ready");
+ followup.eventOrder([
+ { type: "message.received", data: { turnId: received.data.turnId }, count: 1 },
+ {
+ type: "action.result",
+ data: {
+ turnId: received.data.turnId,
+ status: "completed",
+ result: { toolName: "read-status" },
+ },
+ count: 1,
+ },
+ {
+ type: "message.completed",
+ data: {
+ turnId: received.data.turnId,
+ message: (text) =>
+ typeof text === "string" && text.includes(MARKER) && text.includes("ready"),
+ },
+ count: 1,
+ },
+ { type: "turn.completed", data: { turnId: received.data.turnId }, count: 1 },
+ ]);
+ followup.notEvent("action.result", { data: { result: { toolName: "gate" } } });
+ followup.notEvent("input.requested");
+
+ // When the user later approves the original account change.
+ const approved = await session.respond([
+ { requestId: approval.requestId, optionId: "approve" },
+ ]);
+ // Then that saved approval executes exactly once.
+ approved.expectOk();
+ approved.calledTool("gate", { status: "completed", count: 1 });
+ session.event("action.result", {
+ count: 1,
+ data: { status: "completed", result: { toolName: "gate" } },
+ });
+ t.check(session.pendingInputRequests.length, equals(0));
+ },
+ }),
+];
diff --git a/e2e/fixtures/agent-tools-hitl/tsconfig.json b/e2e/fixtures/agent-tools-hitl/tsconfig.json
index 85e3eceb8..f825104a4 100644
--- a/e2e/fixtures/agent-tools-hitl/tsconfig.json
+++ b/e2e/fixtures/agent-tools-hitl/tsconfig.json
@@ -9,9 +9,10 @@
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
- "declaration": true,
+ "declaration": false,
"noEmit": true,
- "types": ["node"]
+ "types": ["node", "eve/workflow-modules"],
+ "allowImportingTsExtensions": true
},
"include": ["agent/**/*.ts", "evals/**/*.ts"]
}
diff --git a/packages/eve/src/execution/session/entry.integration.test.ts b/packages/eve/src/execution/session/entry.integration.test.ts
index ee3839f4c..fc1f860bc 100644
--- a/packages/eve/src/execution/session/entry.integration.test.ts
+++ b/packages/eve/src/execution/session/entry.integration.test.ts
@@ -1574,6 +1574,7 @@ describe("workflowEntry integration", () => {
});
try {
await stream.nextTurn();
+ await waitForParkedTurnStep(anchor.runId);
await workflowRuntime.dispatchSession({
command: followUp(
"dpl_b",
diff --git a/packages/eve/src/harness/approval-delivery-coordinator.test.ts b/packages/eve/src/harness/approval-delivery-coordinator.test.ts
index d81c7745d..690dff48c 100644
--- a/packages/eve/src/harness/approval-delivery-coordinator.test.ts
+++ b/packages/eve/src/harness/approval-delivery-coordinator.test.ts
@@ -86,6 +86,48 @@ describe("coordinateApprovalDelivery", () => {
]);
});
+ it("keeps an earlier settled response when the remaining batch response arrives", async () => {
+ const secondRequest: InputRequest = {
+ ...request,
+ action: { ...request.action, callId: "call-2", toolName: "gate-2" },
+ prompt: "Approve tool call: gate-2",
+ requestId: "approval-2",
+ };
+ const parked = appendPendingInputBatch({
+ requests: [request, secondRequest],
+ responseMessages: [],
+ session: {
+ agent: { modelReference: { id: "test" }, system: "", tools: [] },
+ compaction: { recentWindowSize: 10, threshold: 0.8 },
+ continuationToken: "test",
+ history: [],
+ sessionId: "session-1",
+ },
+ });
+ const settled = settleDirectApprovalResponse({
+ actor: responder,
+ outcome: "allowed",
+ requestId: request.requestId,
+ settledAt: 100,
+ state: parked.state,
+ });
+ const result = await coordinateApprovalDelivery({
+ now: 101,
+ session: { ...parked, state: settled.state },
+ stepInput: {
+ inputResponses: [
+ { optionId: "approve", requestId: request.requestId },
+ { optionId: "approve", requestId: secondRequest.requestId },
+ ],
+ },
+ tools: new Map(),
+ });
+ expect(result.stepInput?.inputResponses).toEqual([
+ { optionId: "approve", requestId: request.requestId },
+ { optionId: "approve", requestId: secondRequest.requestId },
+ ]);
+ });
+
it("forwards an unrelated message while a response-authorized approval remains pending", async () => {
const messageAuth: SessionAuthContext = { ...responder, principalId: "user-2" };
const result = await coordinateApprovalDelivery({
diff --git a/packages/eve/src/harness/approval-delivery-coordinator.ts b/packages/eve/src/harness/approval-delivery-coordinator.ts
index 6afc6ceb1..151c0251d 100644
--- a/packages/eve/src/harness/approval-delivery-coordinator.ts
+++ b/packages/eve/src/harness/approval-delivery-coordinator.ts
@@ -165,9 +165,16 @@ export async function coordinateApprovalDelivery(input: {
pendingRequestIds.has(settlement.requestId),
);
const settledRequestIds = new Set(audit.settlements.map((settlement) => settlement.requestId));
- const discardedDuplicate = hasResponseForRequest(input.stepInput, settledRequestIds);
+ // A direct response settles its individual request before the whole batch
+ // resolves. Keep that response while its request is still pending so a
+ // later response can complete the batch; only discard responses for requests
+ // that have left pending input entirely.
+ const deduplicableSettledRequestIds = new Set(
+ [...settledRequestIds].filter((requestId) => !pendingRequestIds.has(requestId)),
+ );
+ const discardedDuplicate = hasResponseForRequest(input.stepInput, deduplicableSettledRequestIds);
const deduplicatedInput = discardedDuplicate
- ? removeConsumedResponses(input.stepInput, settledRequestIds)
+ ? removeConsumedResponses(input.stepInput, deduplicableSettledRequestIds)
: input.stepInput;
if (
discardedDuplicate &&
@@ -506,11 +513,19 @@ function appendSettledResponses(
settlements: readonly ApprovalSettlementAuditRecord[],
): StepInput | undefined {
if (settlements.length === 0) return stepInput;
+ const existingRequestIds = new Set([
+ ...(stepInput?.inputResponses ?? []).map((response) => response.requestId),
+ ...(stepInput?.attributedInputResponses ?? []).map(({ response }) => response.requestId),
+ ]);
+ const missingSettlements = settlements.filter(
+ (settlement) => !existingRequestIds.has(settlement.requestId),
+ );
+ if (missingSettlements.length === 0) return stepInput;
return {
...stepInput,
inputResponses: [
...(stepInput?.inputResponses ?? []),
- ...settlements.map((settlement) => ({
+ ...missingSettlements.map((settlement) => ({
optionId: settlement.outcome === "allowed" ? "approve" : "cancel",
requestId: settlement.requestId,
})),
diff --git a/packages/eve/src/harness/hitl/approval-input-requests.ts b/packages/eve/src/harness/hitl/approval-input-requests.ts
index ccfa397ef..6f599fa18 100644
--- a/packages/eve/src/harness/hitl/approval-input-requests.ts
+++ b/packages/eve/src/harness/hitl/approval-input-requests.ts
@@ -40,12 +40,12 @@ type ToolApprovalInputRequest = InputRequest & { readonly kind: "tool-approval"
export type RejectedActionBatch = ResolvedInputActionBatch;
-export function hasAnsweredApprovalBatch(
+export function findAnsweredApprovalBatches(
batches: readonly PendingInputBatch[],
responses: readonly InputResponse[],
-): boolean {
+): PendingInputBatch[] {
const responseIds = new Set(responses.map((response) => response.requestId));
- return batches.some((batch) =>
+ return batches.filter((batch) =>
batch.requests.every(
(request) => !isApprovalRequest(request) || responseIds.has(request.requestId),
),
@@ -59,13 +59,8 @@ export function resolveApprovalInputBatches(
readonly resolveApprovalKey?: (request: InputRequest) => string | undefined;
},
): ResolvePendingInputResult {
- const responseIds = new Set(input.responses.map((response) => response.requestId));
const answeredApprovalBatches = new Set(
- input.approvalBatches.filter((batch) =>
- batch.requests.every(
- (request) => !isApprovalRequest(request) || responseIds.has(request.requestId),
- ),
- ),
+ findAnsweredApprovalBatches(input.approvalBatches, input.responses),
);
const answeredQuestionBatches = new Set(
findAnsweredQuestionBatches(input.questionBatches, input.responses),
diff --git a/packages/eve/src/harness/input-requests.test.ts b/packages/eve/src/harness/input-requests.test.ts
index 32d6881d4..46cf635fb 100644
--- a/packages/eve/src/harness/input-requests.test.ts
+++ b/packages/eve/src/harness/input-requests.test.ts
@@ -1269,6 +1269,34 @@ describe("pending input batch collection", () => {
new Set(["approval-1", "question-1"]),
);
});
+
+ it("continues an internal step past HITL emitted by an older turn", () => {
+ const session = appendPendingInputBatch({
+ event: { sequence: 5, stepIndex: 1, turnId: "turn_0" },
+ requests: [approvalRequest("approval-1", "call-1")],
+ responseMessages: [batchOutput("call-1", "bash")],
+ session: createHarnessSession(),
+ });
+
+ const result = resolvePendingInput({ activeTurnId: "turn_1", internalStep: true, session });
+
+ expect(result.outcome).toBe("continue");
+ expect(getPendingInputRequestIds(result.session.state)).toEqual(new Set(["approval-1"]));
+ });
+
+ it("parks when the current turn emitted the pending HITL", () => {
+ const session = appendPendingInputBatch({
+ event: { sequence: 5, stepIndex: 1, turnId: "turn_1" },
+ requests: [approvalRequest("approval-1", "call-1")],
+ responseMessages: [batchOutput("call-1", "bash")],
+ session: createHarnessSession(),
+ });
+
+ const result = resolvePendingInput({ activeTurnId: "turn_1", internalStep: true, session });
+
+ expect(result.outcome).toBe("unresolved");
+ expect(getPendingInputRequestIds(result.session.state)).toEqual(new Set(["approval-1"]));
+ });
});
describe("resolvePendingInput with a session-limit continuation batch", () => {
diff --git a/packages/eve/src/harness/input-requests.ts b/packages/eve/src/harness/input-requests.ts
index 4a1d8d327..9176f6062 100644
--- a/packages/eve/src/harness/input-requests.ts
+++ b/packages/eve/src/harness/input-requests.ts
@@ -6,19 +6,26 @@ import { resolveTextToResponses } from "#channel/resolve-text.js";
import { hasTailApprovalResponse } from "#harness/current-messages.js";
import {
getApprovedTools,
- hasAnsweredApprovalBatch,
+ findAnsweredApprovalBatches,
resolveApprovalInputBatches,
} from "#harness/hitl/approval-input-requests.js";
import type { RejectedActionBatch } from "#harness/hitl/approval-input-requests.js";
import { isApprovalRequest } from "#harness/input-request-class.js";
import type { PendingInputBatch } from "#harness/pending-input-batches.js";
-import { getPendingInputBatches, queueDeferredStepInput } from "#harness/pending-input-batches.js";
+import {
+ getDeferredStepInput,
+ getPendingInputBatches,
+ queueDeferredStepInput,
+} from "#harness/pending-input-batches.js";
import { compactStepInput } from "#harness/hitl/pending-input-resolution.js";
import type {
ResolvePendingInputResult,
ResolvedStepInput,
} from "#harness/hitl/pending-input-resolution.js";
-import { resolveQuestionOnlyInputBatches } from "#harness/hitl/question-input-requests.js";
+import {
+ findAnsweredQuestionBatches,
+ resolveQuestionOnlyInputBatches,
+} from "#harness/hitl/question-input-requests.js";
import { resolveToolCallInputObject } from "#harness/coordination.js";
import {
clearPendingSessionLimitPrompt,
@@ -45,6 +52,42 @@ export function hasStepInput(input?: StepInput): boolean {
return input.message !== undefined || (input.inputResponses?.length ?? 0) > 0;
}
+/** Stored partial answers are not runnable work until they can resolve a batch. */
+export function hasRunnableDeferredStepInput(session: HarnessSession): boolean {
+ const deferred = getDeferredStepInput(session);
+ if (deferred === undefined) return false;
+ if (
+ deferred.message !== undefined ||
+ (deferred.context?.length ?? 0) > 0 ||
+ readClientContext(deferred) !== undefined ||
+ deferred.outputSchema !== undefined ||
+ (deferred.runtimeActionResults?.length ?? 0) > 0
+ )
+ return true;
+
+ const responses = [
+ ...(deferred.inputResponses ?? []),
+ ...(deferred.attributedInputResponses ?? []).map(({ response }) => response),
+ ];
+ if (responses.length === 0) return false;
+ const batches = getPendingInputBatches(session.state);
+ const route = routePendingInput(batches);
+ switch (route.kind) {
+ case "session-limit":
+ return route.batch.requests.every((request) =>
+ responses.some((response) => response.requestId === request.requestId),
+ );
+ case "with-approvals":
+ // An unanswered approval must not prevent an answered question from running.
+ return (
+ findAnsweredApprovalBatches(route.approvalBatches, responses).length > 0 ||
+ findAnsweredQuestionBatches(route.questionBatches, responses).length > 0
+ );
+ case "questions-only":
+ return findAnsweredQuestionBatches(batches, responses).length > 0;
+ }
+}
+
/** Returns true when any pending batch still contains a tool approval. */
export function hasPendingApprovalBatch(session: HarnessSession): boolean {
return getPendingInputBatches(session.state).some((batch) =>
@@ -60,6 +103,10 @@ export function hasPendingApprovalBatch(session: HarnessSession): boolean {
* requirement; question-only batches retain dismiss-and-continue behavior.
*/
export function resolvePendingInput(input: {
+ /** The turn currently advancing through the harness tool loop. */
+ readonly activeTurnId?: string;
+ /** True while the harness has an open turn to continue. */
+ readonly internalStep?: boolean;
readonly deferMessagesWhileApprovalsPending?: boolean;
readonly history?: readonly ModelMessage[];
readonly resolveApprovalKey?: (request: InputRequest) => string | undefined;
@@ -83,10 +130,30 @@ export function resolvePendingInput(input: {
const responses = canonicalizeInputResponses(resolvedStepInput?.inputResponses ?? []);
if (
- route.kind === "approval" &&
+ input.internalStep === true &&
+ canContinuePastHistoricalInput({
+ activeTurnId: input.activeTurnId,
+ batches,
+ responses,
+ route,
+ }) &&
+ resolvedStepInput?.message === undefined
+ ) {
+ return {
+ outcome: "continue",
+ messages: baseHistory,
+ session:
+ resolvedStepInput === undefined
+ ? input.session
+ : queueDeferredStepInput(input.session, compactStepInput(resolvedStepInput)),
+ };
+ }
+
+ if (
+ route.kind === "with-approvals" &&
input.deferMessagesWhileApprovalsPending === true &&
resolvedStepInput?.message !== undefined &&
- !hasAnsweredApprovalBatch(route.approvalBatches, responses)
+ findAnsweredApprovalBatches(route.approvalBatches, responses).length === 0
) {
return {
deferredMessage: true,
@@ -118,26 +185,51 @@ export function resolvePendingInput(input: {
switch (route.kind) {
case "session-limit":
return resolveSessionLimitInput({ ...resolverInput, pendingBatch: route.batch });
- case "approval":
+ case "with-approvals":
return resolveApprovalInputBatches({
...resolverInput,
approvalBatches: route.approvalBatches,
questionBatches: route.questionBatches,
resolveApprovalKey: input.resolveApprovalKey,
});
- case "question":
+ case "questions-only":
return resolveQuestionOnlyInputBatches(resolverInput);
}
}
+/**
+ * An internal tool-loop step must not be parked by input emitted by an older
+ * turn. The current turn can still park on its own HITL request; session-limit
+ * prompts remain a harness gate regardless of the turn that created them.
+ */
+function canContinuePastHistoricalInput(input: {
+ readonly activeTurnId?: string;
+ readonly batches: readonly PendingInputBatch[];
+ readonly responses: readonly InputResponse[];
+ readonly route: PendingInputRoute;
+}): boolean {
+ if (input.activeTurnId === undefined || input.route.kind === "session-limit") return false;
+ if (
+ input.responses.length > 0 &&
+ (input.route.kind !== "with-approvals" ||
+ findAnsweredApprovalBatches(input.route.approvalBatches, input.responses).length > 0 ||
+ findAnsweredQuestionBatches(input.route.questionBatches, input.responses).length > 0)
+ ) {
+ return false;
+ }
+ return input.batches.every(
+ (batch) => batch.event !== undefined && batch.event.turnId !== input.activeTurnId,
+ );
+}
+
type PendingInputRoute =
| { readonly batch: PendingInputBatch; readonly kind: "session-limit" }
| {
readonly approvalBatches: readonly PendingInputBatch[];
- readonly kind: "approval";
+ readonly kind: "with-approvals";
readonly questionBatches: readonly PendingInputBatch[];
}
- | { readonly kind: "question" };
+ | { readonly kind: "questions-only" };
type PendingInputBatchDomain = "approval" | "question" | "session-limit";
@@ -153,12 +245,12 @@ function routePendingInput(batches: readonly PendingInputBatch[]): PendingInputR
const approvalSet = new Set(approvalBatches);
return {
approvalBatches,
- kind: "approval",
+ kind: "with-approvals",
questionBatches: batches.filter((batch) => !approvalSet.has(batch)),
};
}
- return { kind: "question" };
+ return { kind: "questions-only" };
}
function classifyPendingInputBatch(batch: PendingInputBatch): PendingInputBatchDomain {
diff --git a/packages/eve/src/harness/issue-3494-adversarial.integration.test.ts b/packages/eve/src/harness/issue-3494-adversarial.integration.test.ts
new file mode 100644
index 000000000..6f7f351a1
--- /dev/null
+++ b/packages/eve/src/harness/issue-3494-adversarial.integration.test.ts
@@ -0,0 +1,670 @@
+import { jsonSchema, simulateReadableStream } from "ai";
+import { MockLanguageModelV4 } from "ai/test";
+import { afterAll, expect, it } from "vitest";
+import { z } from "zod";
+import { ContextContainer, contextStorage } from "#context/container.js";
+import { SessionKey } from "#context/keys.js";
+import { BackgroundToolExecutorKey } from "#harness/background-tools.js";
+import { getHarnessEmissionState } from "#harness/emission.js";
+import { getPendingCoordinationBatch } from "#harness/coordination.js";
+import { getPendingInputBatches } from "#harness/pending-input-batches.js";
+import { createToolLoopHarness } from "#harness/tool-loop.js";
+import type { HarnessToolDefinition } from "#harness/execute-tool.js";
+import type { HarnessSession, StepInput, StepResult } from "#harness/types.js";
+import type { UnstampedMessageStreamEvent } from "#protocol/message.js";
+import { always } from "#tools/approval/policies.js";
+
+// Diagnostic probes assert desired outcomes. Pending state is created by the
+// real harness and SDK; only provider output and runtime result delivery are scripted.
+type Reply = string | Array<{ toolName: string; input?: unknown; providerExecuted?: true }>;
+const calls = (...names: string[]): Reply => names.map((toolName) => ({ toolName }));
+const reports: unknown[] = [];
+afterAll(() => {
+ if (process.env.EVE_3494_REPORT === "1")
+ console.log("EVE_3494_REPORT=" + JSON.stringify(reports));
+});
+
+function fixture(
+ name: string,
+ responseAuthorized = false,
+ outputLimit?: number,
+ outputSchema?: HarnessSession["outputSchema"],
+) {
+ const script: Reply[] = [];
+ const events: UnstampedMessageStreamEvent[] = [];
+ const executions: string[] = [];
+ const steps: unknown[] = [];
+ const prompts: unknown[] = [];
+ let modelCalls = 0;
+ let session: HarnessSession = {
+ agent: { system: "Test", tools: [], modelReference: { id: "test/model" } },
+ compaction: { recentWindowSize: 10, threshold: 100_000 },
+ continuationToken: name,
+ sessionId: name,
+ history: [],
+ limits: outputLimit === undefined ? undefined : { maxOutputTokensPerSession: outputLimit },
+ outputSchema,
+ };
+ const model = new MockLanguageModelV4({
+ doStream: async (options) => {
+ const call = modelCalls++;
+ prompts.push(options.prompt);
+ const reply = script.shift();
+ if (reply === undefined) throw new Error("Provider script exhausted");
+ return {
+ stream: simulateReadableStream({
+ initialDelayInMs: null,
+ chunkDelayInMs: null,
+ chunks: [
+ { type: "stream-start", warnings: [] },
+ ...(typeof reply === "string"
+ ? [
+ { type: "text-start" as const, id: "answer" },
+ { type: "text-delta" as const, id: "answer", delta: reply },
+ { type: "text-end" as const, id: "answer" },
+ ]
+ : reply.flatMap((tool, index) => [
+ {
+ type: "tool-call" as const,
+ toolCallId: `call-${call}-${index}`,
+ toolName: tool.toolName,
+ input: JSON.stringify(
+ tool.input ??
+ (tool.toolName === "question"
+ ? {
+ prompt: "Choose a color",
+ options: [
+ { id: "red", label: "Red" },
+ { id: "blue", label: "Blue" },
+ ],
+ }
+ : { n: 1 }),
+ ),
+ providerExecuted: tool.providerExecuted,
+ },
+ ...(tool.providerExecuted
+ ? [
+ {
+ type: "tool-result" as const,
+ toolCallId: `call-${call}-${index}`,
+ toolName: tool.toolName,
+ result: { marker: "provider-RESULT" },
+ providerExecuted: true as const,
+ },
+ ]
+ : []),
+ ])),
+ {
+ type: "finish",
+ finishReason: {
+ unified: typeof reply === "string" ? "stop" : "tool-calls",
+ raw: "stop",
+ },
+ usage: {
+ inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 },
+ outputTokens: { total: 1, text: 1, reasoning: 0 },
+ },
+ },
+ ],
+ }),
+ };
+ },
+ });
+ const tools = new Map<string, HarnessToolDefinition>();
+ for (const toolName of ["gateA", "gateB", "read", "write", "fail"]) {
+ tools.set(toolName, {
+ name: toolName,
+ description: toolName,
+ inputSchema: z.object({ n: z.number() }),
+ ...(toolName.startsWith("gate")
+ ? {
+ approval: responseAuthorized
+ ? { request: always(), response: async () => ({ status: "allowed" as const }) }
+ : always(),
+ }
+ : {}),
+ execute: async () => {
+ executions.push(toolName);
+ if (toolName === "fail") throw new Error("Recoverable tool failure");
+ return { marker: `${toolName}-RESULT` };
+ },
+ });
+ }
+ tools.set("question", {
+ name: "question",
+ description: "Ask a question",
+ inputSchema: jsonSchema({ type: "object" }),
+ behavior: {
+ availability: ["requires-request-input"],
+ handling: { kind: "request-input", request: "question" },
+ },
+ });
+ tools.set("workflow", {
+ name: "workflow",
+ description: "Deferred workflow",
+ inputSchema: jsonSchema({ type: "object" }),
+ workflowId: "diagnostic-workflow",
+ });
+ tools.set("control", {
+ name: "control",
+ description: "Deferred runtime control",
+ inputSchema: jsonSchema({ type: "object" }),
+ runtimeAction: { kind: "task-control" },
+ });
+ tools.set("background", {
+ name: "background",
+ description: "Background workflow",
+ inputSchema: jsonSchema({ type: "object" }),
+ workflowId: "diagnostic-background",
+ execution: "background",
+ execute: async () => {
+ throw new Error("Must dispatch through executor");
+ },
+ });
+ const harness = createToolLoopHarness({
+ mode: "conversation",
+ capabilities: { requestInput: true },
+ tools,
+ resolveModel: async () => model,
+ handleEvent: async (event) => {
+ events.push(event);
+ },
+ });
+ async function step(input?: StepInput): Promise<StepResult> {
+ const before = events.length;
+ const beforeCalls = modelCalls;
+ const emission = getHarnessEmissionState(session.state);
+ const ctx = new ContextContainer();
+ ctx.set(SessionKey, {
+ sessionId: session.sessionId,
+ auth: { current: null, initiator: null },
+ turn: { id: emission.turnId || `turn_${emission.sequence}`, sequence: emission.sequence },
+ });
+ ctx.set(BackgroundToolExecutorKey, {
+ async execute({ batch, options }) {
+ expect(batch.calls.some((call) => call.callId === options.toolCallId)).toBe(true);
+ executions.push("background-admitted");
+ return { status: "working", taskId: "diagnostic-background-task" };
+ },
+ });
+ const result = await contextStorage.run(ctx, () => harness(session, input));
+ session = result.session;
+ steps.push({
+ input,
+ modelCalls: modelCalls - beforeCalls,
+ events: events.slice(before),
+ next: typeof result.next === "function" ? "continue" : result.next,
+ settledTurn: result.settledTurn,
+ historyLastRole: session.history.at(-1)?.role,
+ emission: getHarnessEmissionState(session.state),
+ deferred: session.state?.["eve.runtime.deferredStepInput"],
+ pending: getPendingInputBatches(session.state).map((batch) => ({
+ owner: batch.event,
+ requests: batch.requests.map((r) => ({
+ id: r.requestId,
+ tool: r.action.toolName,
+ kind: r.kind,
+ })),
+ })),
+ });
+ return result;
+ }
+ async function drive(input?: StepInput) {
+ let result = await step(input);
+ for (let count = 0; typeof result.next === "function"; count++) {
+ if (count >= 10) throw new Error("Continuation did not settle within 10 steps");
+ result = await step();
+ }
+ return result;
+ }
+ const report = { name, steps, executions, prompts };
+ reports.push(report);
+ return {
+ script,
+ executions,
+ events,
+ step,
+ drive,
+ get session() {
+ return session;
+ },
+ pending: () => getPendingInputBatches(session.state).flatMap((b) => b.requests),
+ async gate(...names: string[]) {
+ script.push(calls(...names));
+ await drive({ message: `Prepare ${names.join(" and ")}.` });
+ expect(this.pending().filter((r) => r.kind === "tool-approval")).toHaveLength(names.length);
+ expect(executions).toHaveLength(0);
+ },
+ respond(tool: string, optionId = "approve"): StepInput {
+ const request = this.pending().find((r) => r.action.toolName === tool);
+ if (!request) throw new Error(`Missing request for ${tool}`);
+ return { inputResponses: [{ requestId: request.requestId, optionId }] };
+ },
+ async finishRuntime() {
+ const batch = getPendingCoordinationBatch(session.state);
+ if (!batch) throw new Error("Expected actual pending coordination batch");
+ return drive({
+ runtimeActionResults: [...batch.tasks, ...batch.runtimeActions].map((r) => ({
+ kind: "tool-result" as const,
+ callId: r.callId,
+ toolName: r.toolName,
+ output: { marker: "runtime-RESULT" },
+ })),
+ });
+ },
+ };
+}
+
+for (const variant of [
+ "read",
+ "write",
+ "fail",
+ "parallel",
+ "invalid",
+ "response-authorized",
+] as const) {
+ it(`finishes unrelated ${variant} tool turn while an approval stays open`, async () => {
+ const f = fixture(variant, variant === "response-authorized");
+ await f.gate("gateA");
+ f.script.push(
+ variant === "parallel"
+ ? calls("read", "write")
+ : variant === "invalid"
+ ? [{ toolName: "read", input: { n: "invalid" } }]
+ : calls(variant === "response-authorized" ? "read" : variant),
+ "FINAL",
+ );
+ const result = await f.drive({ message: "Do the unrelated work." });
+ expect(f.pending()).toHaveLength(1);
+ expect(f.executions).not.toContain("gateA");
+ const expectedExecutions = {
+ read: ["read"],
+ write: ["write"],
+ fail: ["fail"],
+ parallel: ["read", "write"],
+ invalid: [],
+ "response-authorized": ["read"],
+ }[variant];
+ expect([...f.executions].sort()).toEqual(expectedExecutions.sort());
+ expect(result.settledTurn?.output).toBe("FINAL");
+ });
+}
+
+for (const variant of ["approve", "cancel"] as const) {
+ it(`continues after ${variant} of one independent approval while another remains`, async () => {
+ const f = fixture(`sibling-${variant}`);
+ await f.gate("gateA");
+ f.script.push(calls("gateB"));
+ await f.drive({ message: "Also prepare B." });
+ expect(f.pending()).toHaveLength(2);
+ f.script.push(calls("read"), "FINAL");
+ const result = await f.drive(f.respond("gateB", variant));
+ expect(f.pending().map((r) => r.action.toolName)).toEqual(["gateA"]);
+ expect(f.executions.filter((x) => x === "gateB")).toHaveLength(variant === "approve" ? 1 : 0);
+ expect(result.settledTurn?.output).toBe("FINAL");
+ });
+}
+
+it("finishes tool work after answering a question while an earlier approval remains", async () => {
+ const f = fixture("question-answer");
+ await f.gate("gateA");
+ f.script.push(calls("question"));
+ await f.drive({ message: "Ask me for a color." });
+ expect(f.pending()).toHaveLength(2);
+ f.script.push(calls("read"), "FINAL");
+ const result = await f.drive(f.respond("question", "red"));
+ expect(f.pending().map((r) => r.action.toolName)).toEqual(["gateA"]);
+ expect(result.settledTurn?.output).toBe("FINAL");
+});
+
+for (const tool of ["workflow", "control"]) {
+ it(`interprets a completed ${tool} result while an earlier approval remains`, async () => {
+ const f = fixture(tool);
+ await f.gate("gateA");
+ f.script.push(calls(tool), "FINAL");
+ await f.drive({ message: `Run unrelated ${tool}.` });
+ const result = await f.finishRuntime();
+ expect(JSON.stringify(result.session.history)).toContain("runtime-RESULT");
+ expect(f.pending()).toHaveLength(1);
+ expect(result.settledTurn?.output).toBe("FINAL");
+ });
+}
+
+it("allows a tool turn while only multiple question batches remain", async () => {
+ const f = fixture("multiple-questions");
+ await f.gate("gateA");
+ for (let i = 0; i < 2; i++) {
+ f.script.push(calls("question"));
+ await f.drive({ message: `Ask question ${i}.` });
+ }
+ f.script.push("Cancelled.");
+ await f.drive(f.respond("gateA", "cancel"));
+ expect(f.pending().map((r) => r.kind)).toEqual(["question", "question"]);
+ f.script.push(calls("read"), "FINAL");
+ const result = await f.drive({ message: "Read the status first." });
+ expect(result.settledTurn?.output).toBe("FINAL");
+});
+
+it("completes a plain tool turn without any pending input [control]", async () => {
+ const f = fixture("control-no-pending");
+ f.script.push(calls("read"), "FINAL");
+ expect((await f.drive({ message: "Read the status." })).settledTurn?.output).toBe("FINAL");
+});
+
+it("completes a text-only follow-up with a pending approval [control]", async () => {
+ const f = fixture("control-text");
+ await f.gate("gateA");
+ f.script.push("FINAL");
+ expect((await f.drive({ message: "Explain the change." })).settledTurn?.output).toBe("FINAL");
+ expect(f.pending()).toHaveLength(1);
+});
+
+it("finishes after resolving the only approval [control]", async () => {
+ const f = fixture("control-resolve-all");
+ await f.gate("gateA");
+ f.script.push(calls("read"), "FINAL");
+ expect((await f.drive(f.respond("gateA"))).settledTurn?.output).toBe("FINAL");
+ expect(f.executions.filter((x) => x === "gateA")).toHaveLength(1);
+ expect(f.pending()).toHaveLength(0);
+});
+
+it("dismisses a sole question and completes the follow-up tool turn [control]", async () => {
+ const f = fixture("control-sole-question");
+ f.script.push(calls("question"));
+ await f.drive({ message: "Ask a question." });
+ f.script.push(calls("read"), "FINAL");
+ expect((await f.drive({ message: "Never mind. Read the status." })).settledTurn?.output).toBe(
+ "FINAL",
+ );
+ expect(f.pending()).toHaveLength(0);
+});
+
+it("finishes a deferred workflow without pending input [control]", async () => {
+ const f = fixture("control-workflow");
+ f.script.push(calls("workflow"), "FINAL");
+ await f.drive({ message: "Run the workflow." });
+ expect((await f.finishRuntime()).settledTurn?.output).toBe("FINAL");
+});
+
+it("accumulates approvals from one batch across separate deliveries [control]", async () => {
+ const f = fixture("control-partial-batch");
+ await f.gate("gateA", "gateB");
+ await f.drive(f.respond("gateA"));
+ expect(f.executions).toHaveLength(0);
+ f.script.push("FINAL");
+ expect((await f.drive(f.respond("gateB"))).settledTurn?.output).toBe("FINAL");
+ expect(f.executions.filter((x) => x.startsWith("gate"))).toHaveLength(2);
+});
+
+it("finishes an unrelated tool turn carrying a partial approval response", async () => {
+ const f = fixture("partial-response-with-followup");
+ await f.gate("gateA", "gateB");
+ f.script.push(calls("read"), "FINAL");
+ const result = await f.drive({
+ ...f.respond("gateA"),
+ message: "While B waits, read the status.",
+ });
+ expect(f.executions).toEqual(["read"]);
+ expect(f.pending()).toHaveLength(2);
+ expect(result.settledTurn?.output).toBe("FINAL");
+});
+
+it("finishes after a stale approval response becomes a follow-up message", async () => {
+ const f = fixture("stale-response");
+ await f.gate("gateA");
+ f.script.push(calls("gateB"));
+ await f.drive({ message: "Also prepare B." });
+ const old = f.respond("gateB", "cancel");
+ f.script.push("Cancelled.");
+ await f.drive(old);
+ f.script.push(calls("read"), "FINAL");
+ const result = await f.drive(old);
+ expect(f.executions).toEqual(["read"]);
+ expect(result.settledTurn?.output).toBe("FINAL");
+});
+
+it("reaches the next budget prompt after a grant while an earlier approval remains", async () => {
+ const f = fixture("session-limit-grant", false, 1);
+ await f.gate("gateA");
+ await f.drive({ message: "Read the status." });
+ expect(f.pending().map((r) => r.kind)).toContain("session-limit");
+ f.script.push(calls("read"));
+ const result = await f.drive(f.respond("session_limit_continuation", "continue"));
+ expect(f.executions).toEqual(["read"]);
+ expect(result.next).toBeNull();
+ expect(f.pending().map((r) => r.kind)).toEqual(["tool-approval", "session-limit"]);
+});
+
+for (const variant of ["fail", "invalid"]) {
+ it(`recovers from ${variant} without pending input [control]`, async () => {
+ const f = fixture(`control-${variant}`);
+ f.script.push(
+ variant === "invalid" ? [{ toolName: "read", input: { n: "invalid" } }] : calls("fail"),
+ "FINAL",
+ );
+ expect((await f.drive({ message: "Try the tool." })).settledTurn?.output).toBe("FINAL");
+ if (variant === "invalid") expect(f.executions).toHaveLength(0);
+ });
+}
+
+it("settles both independent approval batches and a deferred follow-up [control]", async () => {
+ const f = fixture("control-resolve-multiple");
+ await f.gate("gateA");
+ f.script.push(calls("gateB"));
+ await f.drive({ message: "Also prepare B." });
+ f.script.push("First approved.", "Second approved.", calls("read"), "FINAL");
+ const result = await f.drive({
+ inputResponses: [...f.respond("gateA").inputResponses!, ...f.respond("gateB").inputResponses!],
+ message: "Then read the status.",
+ });
+ expect(f.executions.filter((x) => x.startsWith("gate"))).toHaveLength(2);
+ expect(f.pending()).toHaveLength(0);
+ expect(result.settledTurn?.output).toBe("FINAL");
+});
+
+it("keeps same-turn approval blocked when its parallel workflow finishes [control]", async () => {
+ const f = fixture("control-same-turn-workflow-and-approval");
+ f.script.push(calls("gateA", "workflow"));
+ await f.drive({ message: "Prepare change and run workflow together." });
+ const result = await f.finishRuntime();
+ expect(result.next).toBeNull();
+ expect(f.pending()).toHaveLength(1);
+ expect(f.executions).not.toContain("gateA");
+ // Resolving the owning request must still recover the held tool transcript.
+ f.script.push("FINAL");
+ expect((await f.drive(f.respond("gateA"))).settledTurn?.output).toBe("FINAL");
+ expect(f.executions).toEqual(["gateA"]);
+});
+
+it("settles a text-only follow-up carrying a partial approval response", async () => {
+ const f = fixture("partial-response-with-text");
+ await f.gate("gateA", "gateB");
+ f.script.push("FINAL");
+ const result = await f.drive({
+ ...f.respond("gateA"),
+ message: "Explain the change while B waits.",
+ });
+ expect(f.executions).toHaveLength(0);
+ expect(f.events.some((e) => e.type === "message.completed")).toBe(true);
+ expect(result.settledTurn?.output).toBe("FINAL");
+});
+
+it("reaches the next budget prompt after a grant without an older approval [control]", async () => {
+ const f = fixture("control-session-limit", false, 1);
+ f.script.push("Initial text.");
+ await f.drive({ message: "Say hello." });
+ await f.drive({ message: "Read the status." });
+ expect(f.pending().map((r) => r.kind)).toEqual(["session-limit"]);
+ f.script.push(calls("read"));
+ await f.drive(f.respond("session_limit_continuation", "continue"));
+ expect(f.executions).toEqual(["read"]);
+ expect(f.pending().map((r) => r.kind)).toEqual(["session-limit"]);
+ expect(f.pending()[0]?.requestId).toContain(":output:2");
+});
+
+for (const pending of [true, false]) {
+ it(`continues after a background admission receipt ${pending ? "with pending approval" : "[control]"}`, async () => {
+ const f = fixture(`background-${pending}`);
+ if (pending) await f.gate("gateA");
+ f.script.push(calls("background"), "FINAL");
+ const result = await f.drive({ message: "Start background work and acknowledge admission." });
+ expect(f.executions).toEqual(["background-admitted"]);
+ expect(result.settledTurn?.output).toBe("FINAL");
+ });
+ it(`continues after a provider-executed result ${pending ? "with pending approval" : "[control]"}`, async () => {
+ const f = fixture(`provider-${pending}`);
+ if (pending) await f.gate("gateA");
+ f.script.push([{ toolName: "read", providerExecuted: true }], "FINAL");
+ const first = await f.step({ message: "Look it up." });
+ expect(first.session.history.at(-1)?.role).toBe("assistant");
+ expect(typeof first.next).toBe("function");
+ const result = await f.drive();
+ expect(f.executions).toHaveLength(0);
+ expect(JSON.stringify(result.session.history)).toContain("provider-RESULT");
+ expect(result.settledTurn?.output).toBe("FINAL");
+ });
+}
+
+it("settles final_output alongside an ordinary tool with older approval [control]", async () => {
+ const f = fixture("control-final-output", false, undefined, {
+ type: "object",
+ properties: { status: { type: "string" } },
+ required: ["status"],
+ additionalProperties: false,
+ });
+ await f.gate("gateA");
+ f.script.push([{ toolName: "read" }, { toolName: "final_output", input: { status: "ready" } }]);
+ const result = await f.drive({
+ message: "Read the status and return it.",
+ });
+ expect(result.settledTurn?.output).toEqual({ status: "ready" });
+ expect(f.executions).toEqual(["read"]);
+ expect(f.pending()).toHaveLength(1);
+});
+
+it("continues after responder-authorized approval with an older approval still open", async () => {
+ const f = fixture("authorized-sibling", true);
+ await f.gate("gateA");
+ f.script.push(calls("gateB"));
+ await f.drive({ message: "Also prepare B." });
+ f.script.push(calls("read"), "FINAL");
+ const result = await f.drive({
+ attributedInputResponses: f.respond("gateB").inputResponses!.map((response) => ({
+ response,
+ auth: {
+ attributes: {},
+ authenticator: "test",
+ issuer: "test",
+ principalId: "user-1",
+ principalType: "user" as const,
+ },
+ })),
+ });
+ expect(f.executions).toEqual(["gateB", "read"]);
+ expect(f.pending().map((r) => r.action.toolName)).toEqual(["gateA"]);
+ expect(result.settledTurn?.output).toBe("FINAL");
+});
+
+it("continues after responder-authorized approval of the older batch", async () => {
+ const f = fixture("authorized-older-sibling", true);
+ await f.gate("gateA");
+ f.script.push(calls("gateB"));
+ await f.drive({ message: "Also prepare B." });
+ expect(f.pending().map((request) => request.action.toolName)).toEqual(["gateA", "gateB"]);
+ f.script.push(calls("read"), "FINAL");
+ const response = f.respond("gateA").inputResponses![0]!;
+ const result = await f.drive({
+ attributedInputResponses: [
+ {
+ response,
+ auth: {
+ attributes: {},
+ authenticator: "test",
+ issuer: "test",
+ principalId: "user-1",
+ principalType: "user",
+ },
+ },
+ ],
+ });
+ expect(f.executions).toEqual(["gateA", "read"]);
+ expect(f.pending().map((request) => request.action.toolName)).toEqual(["gateB"]);
+ expect(result.settledTurn?.output).toBe("FINAL");
+});
+
+it("parks on a new question while retaining a partial approval [control]", async () => {
+ const f = fixture("partial-approval-new-question");
+ // Given A and B need approval together, and only A has been answered.
+ await f.gate("gateA", "gateB");
+ await f.drive(f.respond("gateA"));
+
+ // When the next user message causes the model to ask a question.
+ f.script.push(calls("question"));
+ const result = await f.step({ message: "Ask which color to use." });
+
+ // Then the new question parks immediately, and A's answer is still remembered.
+ expect(result.next).toBeNull();
+ expect(f.pending().map((request) => request.kind)).toEqual([
+ "tool-approval",
+ "tool-approval",
+ "question",
+ ]);
+ expect(f.executions).toHaveLength(0);
+ f.script.push("Approved.");
+ await f.drive(f.respond("gateB"));
+ expect(f.executions).toEqual(["gateA", "gateB"]);
+ expect(f.pending().map((request) => request.kind)).toEqual(["question"]);
+});
+
+it("resumes a complete batch while another batch has only a partial approval [control]", async () => {
+ const f = fixture("complete-and-partial-approvals");
+ // Given A and B need approval together, and a second independent B is pending.
+ await f.gate("gateA", "gateB");
+ const approvalA = f.respond("gateA");
+ const approvalB = f.respond("gateB");
+ f.script.push(calls("gateB"));
+ await f.drive({ message: "Prepare another independent B." });
+ const independentB = f.pending().at(-1)!;
+
+ // When one delivery answers A and the independent B.
+ f.script.push("Independent B approved.");
+ const result = await f.drive({
+ inputResponses: [
+ ...approvalA.inputResponses!,
+ { requestId: independentB.requestId, optionId: "approve" },
+ ],
+ });
+
+ // Then only the complete batch executes and replies, without another model call.
+ expect(result.settledTurn?.output).toBe("Independent B approved.");
+ expect(f.executions).toEqual(["gateB"]);
+ expect(f.pending()).toHaveLength(2);
+ f.script.push("Both approved.");
+ await f.drive(approvalB);
+ expect(f.executions).toEqual(["gateB", "gateA", "gateB"]);
+ expect(f.pending()).toHaveLength(0);
+});
+
+it("answers a queued question after an approval while another approval stays pending", async () => {
+ const f = fixture("approval-then-question-with-sibling");
+ // Given independent approvals A and B, followed by a question.
+ await f.gate("gateA");
+ f.script.push(calls("gateB"));
+ await f.drive({ message: "Prepare B." });
+ f.script.push(calls("question"));
+ await f.drive({ message: "Ask which color." });
+
+ // When one delivery answers A and the question, leaving B unanswered.
+ f.script.push("A approved.", "Color answered.");
+ const result = await f.drive({
+ inputResponses: [
+ ...f.respond("gateA").inputResponses!,
+ ...f.respond("question", "red").inputResponses!,
+ ],
+ });
+
+ // Then both answers are processed and B alone stays pending.
+ expect(result.settledTurn?.output).toBe("Color answered.");
+ expect(f.executions).toEqual(["gateA"]);
+ expect(f.pending().map((request) => request.action.toolName)).toEqual(["gateB"]);
+});
diff --git a/packages/eve/src/harness/issue-3494-diagnosis.integration.test.ts b/packages/eve/src/harness/issue-3494-diagnosis.integration.test.ts
new file mode 100644
index 000000000..71a810ae9
--- /dev/null
+++ b/packages/eve/src/harness/issue-3494-diagnosis.integration.test.ts
@@ -0,0 +1,184 @@
+import { writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { jsonSchema, simulateReadableStream } from "ai";
+import { MockLanguageModelV4 } from "ai/test";
+import { expect, it, vi } from "vitest";
+import { ContextContainer, contextStorage } from "#context/container.js";
+import { SessionKey } from "#context/keys.js";
+import { getHarnessEmissionState } from "#harness/emission.js";
+import type { HarnessToolDefinition } from "#harness/execute-tool.js";
+import { resolvePendingInput } from "#harness/input-requests.js";
+import {
+ getPendingInputBatches,
+ removePendingInputBatches,
+} from "#harness/pending-input-batches.js";
+import { createToolLoopHarness } from "#harness/tool-loop.js";
+import type { HarnessSession, StepInput, ToolLoopHarnessConfig } from "#harness/types.js";
+import type { UnstampedMessageStreamEvent } from "#protocol/message.js";
+import { always } from "#tools/approval/policies.js";
+
+it("continues the second step past an older approval batch", async () => {
+ const events: UnstampedMessageStreamEvent[] = [];
+ const gate = vi.fn(async () => ({ changed: true }));
+ const read = vi.fn(async () => ({ status: "ready" }));
+ let modelCalls = 0;
+ const model = new MockLanguageModelV4({
+ doStream: async () => {
+ const call = modelCalls++;
+ return {
+ stream: simulateReadableStream({
+ initialDelayInMs: null,
+ chunkDelayInMs: null,
+ chunks: [
+ { type: "stream-start", warnings: [] },
+ ...(call < 2
+ ? [
+ {
+ type: "tool-call" as const,
+ toolCallId: `call-${call}`,
+ toolName: call === 0 ? "gate" : "read",
+ input: "{}",
+ },
+ ]
+ : [
+ { type: "text-start" as const, id: "reply" },
+ { type: "text-delta" as const, id: "reply", delta: "Your draft is ready." },
+ { type: "text-end" as const, id: "reply" },
+ ]),
+ {
+ type: "finish",
+ finishReason: { unified: call < 2 ? "tool-calls" : "stop", raw: "stop" },
+ usage: {
+ inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 },
+ outputTokens: { total: 1, text: 1, reasoning: 0 },
+ },
+ },
+ ],
+ }),
+ };
+ },
+ });
+ const config: ToolLoopHarnessConfig = {
+ mode: "conversation",
+ tools: new Map<string, HarnessToolDefinition>([
+ [
+ "gate",
+ {
+ name: "gate",
+ description: "Change account",
+ inputSchema: jsonSchema({ type: "object" }),
+ approval: always(),
+ execute: gate,
+ },
+ ],
+ [
+ "read",
+ {
+ name: "read",
+ description: "Read draft status",
+ inputSchema: jsonSchema({ type: "object" }),
+ execute: read,
+ },
+ ],
+ ]),
+ resolveModel: async () => model,
+ handleEvent: async (event) => {
+ events.push(event);
+ },
+ };
+ const initial: HarnessSession = {
+ agent: { system: "Test", tools: [], modelReference: { id: "test/model" } },
+ compaction: { recentWindowSize: 10, threshold: 100_000 },
+ continuationToken: "diagnosis",
+ sessionId: "diagnosis-3494",
+ history: [],
+ };
+ const run = async (session: HarnessSession, input?: StepInput) => {
+ const state = getHarnessEmissionState(session.state);
+ const ctx = new ContextContainer();
+ ctx.set(SessionKey, {
+ sessionId: session.sessionId,
+ auth: { current: null, initiator: null },
+ turn: { id: state.turnId || `turn_${state.sequence}`, sequence: state.sequence },
+ });
+ return contextStorage.run(ctx, () => createToolLoopHarness(config)(session, input));
+ };
+
+ const parked = await run(initial, { message: "Prepare the account change." });
+ expect(parked.next).toBeNull();
+ expect(gate).not.toHaveBeenCalled();
+ const [approval] = getPendingInputBatches(parked.session.state);
+ expect(approval?.event?.turnId).toBe("turn_0");
+ expect(approval?.requests[0]?.kind).toBe("tool-approval");
+
+ const first = await run(parked.session, { message: "What is the draft status?" });
+ expect(typeof first.next).toBe("function");
+ expect(read).toHaveBeenCalledTimes(1);
+ expect(modelCalls).toBe(2);
+ expect(getHarnessEmissionState(first.session.state)).toMatchObject({
+ turnId: "turn_1",
+ stepIndex: 1,
+ });
+ expect(first.session.history.at(-1)?.role).toBe("tool");
+ const before = events.length;
+
+ const pendingDecision = resolvePendingInput({ session: first.session });
+ expect(pendingDecision.outcome).toBe("unresolved");
+ const continued = await run(first.session);
+ expect(continued.settledTurn?.output).toBe("Your draft is ready.");
+ expect(continued.next).toBeNull();
+ expect(modelCalls).toBe(3);
+ expect(events.slice(before).some((event) => event.type === "turn.completed")).toBe(true);
+ expect(getPendingInputBatches(continued.session.state)).toEqual([approval]);
+ const continuedObservation = {
+ phase: "continued",
+ modelCalls,
+ emission: getHarnessEmissionState(continued.session.state),
+ pendingOwner: approval?.event,
+ emittedBySecondStep: events.slice(before).map((event) => event.type),
+ firstTurnEvents: events
+ .filter(
+ (event) => "data" in event && "turnId" in event.data && event.data.turnId === "turn_1",
+ )
+ .map((event) => event.type),
+ };
+
+ // Remove the batch only from a cloned state to isolate causality; this is not a fix.
+ const control = removePendingInputBatches(first.session, [approval!]);
+ expect(control.history).toBe(first.session.history);
+ expect(getHarnessEmissionState(control.state)).toEqual(
+ getHarnessEmissionState(first.session.state),
+ );
+ expect(resolvePendingInput({ session: control }).outcome).toBe("continue");
+ const beforeControl = events.length;
+ await run(control);
+ expect(modelCalls).toBe(4);
+ expect(events.slice(beforeControl).some((event) => event.type === "turn.completed")).toBe(true);
+ expect(gate).not.toHaveBeenCalled();
+ expect(getPendingInputBatches(first.session.state)).toEqual([approval]);
+ const controlObservation = {
+ phase: "clone-without-batch",
+ modelCalls,
+ events: events.slice(beforeControl).map((event) => event.type),
+ originalStillPending: getPendingInputBatches(first.session.state).length,
+ gateExecutions: gate.mock.calls.length,
+ };
+
+ const beforeApproval = events.length;
+ const approved = await run(continued.session, {
+ inputResponses: [{ requestId: approval!.requests[0]!.requestId, optionId: "approve" }],
+ });
+ expect(gate).toHaveBeenCalledTimes(1);
+ expect(getPendingInputBatches(approved.session.state)).toHaveLength(0);
+ const approvalObservation = {
+ phase: "approve-original-session",
+ gateExecutions: gate.mock.calls.length,
+ pending: getPendingInputBatches(approved.session.state).length,
+ events: events.slice(beforeApproval).map((event) => event.type),
+ };
+ writeFileSync(
+ join(tmpdir(), "eve-3494-diagnosis.json"),
+ JSON.stringify({ continuedObservation, controlObservation, approvalObservation }, null, 2),
+ );
+});
diff --git a/packages/eve/src/harness/pending-input-batches.ts b/packages/eve/src/harness/pending-input-batches.ts
index eb9ed4877..37e461f2e 100644
--- a/packages/eve/src/harness/pending-input-batches.ts
+++ b/packages/eve/src/harness/pending-input-batches.ts
@@ -246,7 +246,7 @@ export function hasDeferredStepInput(session: HarnessSession): boolean {
return getDeferredStepInput(session) !== undefined;
}
-function getDeferredStepInput(session: HarnessSession): StepInput | undefined {
+export function getDeferredStepInput(session: HarnessSession): StepInput | undefined {
return session.state?.[DEFERRED_STEP_INPUT_KEY] as StepInput | undefined;
}
diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.ts
index 0da3ba8d8..848470c64 100644
--- a/packages/eve/src/harness/tool-loop.ts
+++ b/packages/eve/src/harness/tool-loop.ts
@@ -107,6 +107,7 @@ import {
emitTurnEpilogue,
emitTurnPreamble,
getHarnessEmissionState,
+ isHarnessBetweenTurns,
setHarnessEmissionState,
} from "#harness/emission.js";
import {
@@ -139,7 +140,7 @@ import {
consumeDeferredStepInput,
getApprovedTools,
getPendingInputRequestIds,
- hasDeferredStepInput,
+ hasRunnableDeferredStepInput,
hasPendingApprovalBatch,
hasStepInput,
resolvePendingInput,
@@ -205,6 +206,7 @@ import {
import { resolveFrameworkToolFromUpstreamType } from "#harness/provider-tools.js";
import {
createCoordinationRequestFromToolCall,
+ getPendingCoordinationBatch,
resolvePendingCoordination,
setPendingCoordinationBatch,
} from "#harness/coordination.js";
@@ -660,6 +662,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
});
session = stepInput.session;
+ const pendingCoordination = getPendingCoordinationBatch(session.state);
const resolvedCoordination = await resolvePendingCoordination({
emit,
session,
@@ -844,8 +847,10 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
}
const pending = resolvePendingInput({
+ activeTurnId: pendingCoordination?.event.turnId ?? activeTurnId(emissionState),
deferMessagesWhileApprovalsPending: config.mode !== "conversation",
history: resolvedCoordination.messages,
+ internalStep: !isHarnessBetweenTurns(session),
resolveApprovalKey: resolveApprovalKeyFromTools(responseAuthorizationTools),
session,
stepInput: coordinated.stepInput,
@@ -1020,7 +1025,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
let instructionMessages: UserModelMessage[] = [];
let memoryCommit: ReturnType<typeof drainMemoryCommit> = undefined;
- if (emit && hasStepInput(input)) {
+ if (emit && (hasStepInput(effectiveStepInput) || hasStepInput(coordinated.stepInput))) {
if (store !== undefined) {
prepareDynamicInstructionPreamble(
store,
@@ -2792,7 +2797,7 @@ async function handleStepResult(input: {
}
return {
- next: hasDeferredStepInput(parkedSession) ? runStep : null,
+ next: hasRunnableDeferredStepInput(parkedSession) ? runStep : null,
session: parkedSession,
};
}
@@ -2884,7 +2889,7 @@ async function handleStepResult(input: {
!calledFinalOutput &&
(continuationMessages.at(-1)?.role === "tool" ||
normalizedProviderHistory.outcomeEndsResponse ||
- hasDeferredStepInput(nextSession));
+ hasRunnableDeferredStepInput(nextSession));
if (continueLoop) {
if (emit) {
emissionState = advanceStep(emissionState);