The Open Framework for Building Agents
test(e2e): cover the human-input rules main already meets (#4492)
Rui88e5b57cc88d
diff --git a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts b/e2e/fixtures/agent-subagents-hitl/agent/agent.tsindex fa81db70f..f7f7c4728 100644--- a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts+++ b/e2e/fixtures/agent-subagents-hitl/agent/agent.ts@@ -35,6 +35,14 @@ function respond(request: MockModelRequest): MockModelResponse | string {],};}+ // RELAY-<child>: delegate to that mock child once, wait for it, and report its result.+ const relay = /RELAY-(question-child|budget-child)/u.exec(message)?.[1];+ if (relay !== undefined) {+ const result = taskResultOf(request, relay);+ if (result !== undefined) return `RELAY-RESULT ${result}`;+ if (hasReceipt(request, relay)) return waitForTasks();+ return { toolCalls: [{ input: { message: "Handle Alice's release request." }, name: relay }] };+ }if (message.includes("Call the stock-price subagent exactly once")) {const quote = taskResultOf(request, "stock-price");if (quote !== undefined) return `The stock-price subagent returned: ${quote}`;diff --git a/e2e/fixtures/agent-subagents-hitl/agent/subagents/budget-child/agent.ts b/e2e/fixtures/agent-subagents-hitl/agent/subagents/budget-child/agent.tsnew file mode 100644index 000000000..490bc1754--- /dev/null+++ b/e2e/fixtures/agent-subagents-hitl/agent/subagents/budget-child/agent.ts@@ -0,0 +1,17 @@+import { defineAgent } from "eve";+import { mockModel } from "eve/evals";++/**+ * Records one release step on a one-token input budget, so its second model+ * call stops at its budget question, which its parent relays.+ */+export default defineAgent({+ description: "Record the release step, then confirm it.",+ limits: { maxInputTokensPerSession: 1 },+ model: mockModel(({ toolResults }) =>+ toolResults.some((entry) => entry.name === "record_step")+ ? "BUDGET-CHILD-RESULT recorded"+ : { toolCalls: [{ input: {}, name: "record_step" }] },+ ),+ modelContextWindowTokens: 1_000_000,+});diff --git a/e2e/fixtures/agent-subagents-hitl/agent/subagents/budget-child/tools/record_step.ts b/e2e/fixtures/agent-subagents-hitl/agent/subagents/budget-child/tools/record_step.tsnew file mode 100644index 000000000..1a20622c5--- /dev/null+++ b/e2e/fixtures/agent-subagents-hitl/agent/subagents/budget-child/tools/record_step.ts@@ -0,0 +1,8 @@+import { defineTool } from "eve/tools";+import { z } from "zod";++export default defineTool({+ description: "Record the release step.",+ inputSchema: z.strictObject({}),+ execute: () => ({ recorded: true }),+});diff --git a/e2e/fixtures/agent-subagents-hitl/agent/subagents/question-child/agent.ts b/e2e/fixtures/agent-subagents-hitl/agent/subagents/question-child/agent.tsnew file mode 100644index 000000000..270938b77--- /dev/null+++ b/e2e/fixtures/agent-subagents-hitl/agent/subagents/question-child/agent.ts@@ -0,0 +1,26 @@+import { defineAgent } from "eve";+import { mockModel } from "eve/evals";++/** Asks which environment a release goes to, then reports the answer it got. */+export default defineAgent({+ description: "Ask which environment the release should go to, then report the choice.",+ model: mockModel(({ toolResults }) => {+ const answer = toolResults.find((entry) => entry.name === "ask_question");+ if (answer !== undefined) return `QUESTION-CHILD-RESULT ${JSON.stringify(answer.output)}`;+ return {+ toolCalls: [+ {+ input: {+ options: [+ { description: "Release to staging first.", label: "Staging" },+ { description: "Release straight to production.", label: "Production" },+ ],+ question: "Which environment should the release go to?",+ },+ name: "ask_question",+ },+ ],+ };+ }),+ modelContextWindowTokens: 1_000_000,+});diff --git a/e2e/fixtures/agent-subagents-hitl/agent/subagents/question-child/tools/ask_question.ts b/e2e/fixtures/agent-subagents-hitl/agent/subagents/question-child/tools/ask_question.tsnew file mode 100644index 000000000..533cc8526--- /dev/null+++ b/e2e/fixtures/agent-subagents-hitl/agent/subagents/question-child/tools/ask_question.ts@@ -0,0 +1,3 @@+import { askQuestion } from "eve/tools/ask_question";++export default askQuestion();diff --git a/e2e/fixtures/agent-subagents-hitl/evals/hitl/cancel-withdraws-relayed-question.eval.ts b/e2e/fixtures/agent-subagents-hitl/evals/hitl/cancel-withdraws-relayed-question.eval.tsnew file mode 100644index 000000000..6365a3062--- /dev/null+++ b/e2e/fixtures/agent-subagents-hitl/evals/hitl/cancel-withdraws-relayed-question.eval.ts@@ -0,0 +1,27 @@+import { defineEval } from "eve/evals";++import { next, waitForRelayed } from "./helpers";++/**+ * Alice cancels her turn while the question child waits on her answer. The+ * cancel stops the child, so nobody can answer its question: the parent+ * withdraws it as cancelled.+ */+export default defineEval({+ description: "Cancelling the parent's turn withdraws a child's relayed question.",+ tags: ["hitl", "relayed", "cancel"],+ timeoutMs: 90_000,+ async test(t) {+ const started = await t.send("RELAY-question-child Alice asks where the release should go.");+ const { request, session } = await waitForRelayed(t, started.session, "ask_question");++ const following = next(t, session);+ await session.cancel();+ const cancelled = await following;+ cancelled.event("input.resolved", {+ count: 1,+ data: { resolutions: [{ outcome: "cancelled", requestId: request.requestId }] },+ });+ cancelled.event("turn.cancelled", { count: 1 });+ },+});diff --git a/e2e/fixtures/agent-subagents-hitl/evals/hitl/helpers.ts b/e2e/fixtures/agent-subagents-hitl/evals/hitl/helpers.tsnew file mode 100644index 000000000..ca4a4f87c--- /dev/null+++ b/e2e/fixtures/agent-subagents-hitl/evals/hitl/helpers.ts@@ -0,0 +1,42 @@+import type { EveEvalContext, EveEvalSession, EveEvalTurn, InputRequest } from "eve/evals";++/**+ * Follows the parent session, turn by turn, until a child's request matching+ * `toolName` is pending on it. A child asks after its own model call, so the+ * request may land a turn or two after the delegation.+ */+export async function waitForRelayed(+ t: EveEvalContext,+ initial: EveEvalSession,+ toolName: string,+): Promise<{ readonly request: InputRequest; readonly session: EveEvalSession }> {+ let session = initial;+ for (let attempt = 0; attempt < 5; attempt += 1) {+ const request = session.pendingInputRequests.find(+ (pending) => pending.action.toolName === toolName,+ );+ if (request !== undefined) return { request, session };+ session = (await next(t, session)).session;+ }+ throw new Error(`No relayed ${toolName} request reached the parent after five turns.`);+}++/** Follows the parent until a turn's reply contains `marker`. */+export async function waitForReply(+ t: EveEvalContext,+ initial: EveEvalSession,+ marker: string,+): Promise<EveEvalTurn> {+ let session = initial;+ for (let attempt = 0; attempt < 5; attempt += 1) {+ const turn = await next(t, session);+ if (turn.message?.includes(marker) === true) return turn;+ session = turn.session;+ }+ throw new Error(`No parent reply contained ${marker} after five turns.`);+}++/** The parent's events from its current cursor to the next turn boundary. */+export function next(t: EveEvalContext, session: EveEvalSession): Promise<EveEvalTurn> {+ return t.target.watchTurn(session.sessionId, { startIndex: session.state.streamIndex }).result();+}diff --git a/e2e/fixtures/agent-subagents-hitl/evals/hitl/relayed-budget-stop-cancels-parent.eval.ts b/e2e/fixtures/agent-subagents-hitl/evals/hitl/relayed-budget-stop-cancels-parent.eval.tsnew file mode 100644index 000000000..f7a73a635--- /dev/null+++ b/e2e/fixtures/agent-subagents-hitl/evals/hitl/relayed-budget-stop-cancels-parent.eval.ts@@ -0,0 +1,38 @@+import { defineEval } from "eve/evals";++import { waitForRelayed } from "./helpers";++/**+ * The budget child runs out of budget mid-task and its question reaches+ * Alice through her session. She chooses Stop: the answer goes to the child,+ * and because Stop means stop, her own turn is cancelled too.+ */+export default defineEval({+ description: "Stop on a child's relayed budget question also cancels the parent's turn.",+ tags: ["hitl", "relayed", "budget"],+ timeoutMs: 90_000,+ async test(t) {+ const started = await t.send("RELAY-budget-child Alice records the release step.");+ const { request, session } = await waitForRelayed(+ t,+ started.session,+ "session_limit_continuation",+ );+ if (request.requestId.startsWith(`${session.sessionId}:`)) {+ throw new Error("Expected the child's budget question, not the parent's.");+ }++ const stopped = await session.respond([{ optionId: "stop", requestId: request.requestId }]);+ stopped.event("input.resolved", {+ count: 1,+ data: {+ resolutions: [+ { outcome: "answered", requestId: request.requestId, response: { optionId: "stop" } },+ ],+ },+ });+ stopped.event("turn.cancelled", { count: 1 });+ stopped.notEvent("turn.failed");+ stopped.notEvent("message.completed", { data: { message: /RELAY-RESULT/u } });+ },+});diff --git a/e2e/fixtures/agent-subagents-hitl/evals/hitl/relayed-question-answered.eval.ts b/e2e/fixtures/agent-subagents-hitl/evals/hitl/relayed-question-answered.eval.tsnew file mode 100644index 000000000..9887fa86c--- /dev/null+++ b/e2e/fixtures/agent-subagents-hitl/evals/hitl/relayed-question-answered.eval.ts@@ -0,0 +1,41 @@+import { defineEval } from "eve/evals";++import { waitForRelayed, waitForReply } from "./helpers";++/**+ * The question child asks Alice, through her session, where the release+ * goes. Her answer on the parent is forwarded to the child that asked, which+ * resolves its question with her choice and reports back.+ */+export default defineEval({+ description: "A child's question is relayed through the parent and answered there.",+ tags: ["hitl", "relayed"],+ timeoutMs: 90_000,+ async test(t) {+ const started = await t.send("RELAY-question-child Alice asks where the release should go.");+ const { request, session } = await waitForRelayed(t, started.session, "ask_question");+ const production = request.options?.find((option) => option.label === "Production");+ if (production === undefined) throw new Error("Expected a Production option.");++ const answered = await session.respond([+ { optionId: production.id, requestId: request.requestId },+ ]);+ answered.event("input.resolved", {+ count: 1,+ data: {+ resolutions: [+ {+ outcome: "answered",+ requestId: request.requestId,+ response: { optionId: production.id },+ },+ ],+ },+ });+ const reply = answered.message?.includes("RELAY-RESULT")+ ? answered+ : await waitForReply(t, answered.session, "RELAY-RESULT");+ reply.messageIncludes(/RELAY-RESULT QUESTION-CHILD-RESULT .*Production/u);+ t.calledSubagent("question-child", { status: "completed", count: 1 });+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/agent/agent.ts b/e2e/fixtures/agent-tools-hitl/agent/agent.tsindex 3bcbe6a8f..22effa3fe 100644--- a/e2e/fixtures/agent-tools-hitl/agent/agent.ts+++ b/e2e/fixtures/agent-tools-hitl/agent/agent.ts@@ -3,6 +3,7 @@ import { defineAgent, defineDynamic } from "eve";import type { MockModelRequest, MockModelResponse } from "eve/evals";import { continuationModel } from "./lib/continuation/model.ts";+import { humanInputModel } from "./lib/hitl/model.ts";const AUTH_PROBE_DIRECTIVE = /call the auth-probe tool exactly once with marker "([^"]+)"/iu;const SCOPED_APPROVAL_DIRECTIVE =@@ -105,10 +106,16 @@ export default defineAgent({model: typeof base.model === "string" ? base.model : "openai/gpt-6.1-sol",modelContextWindowTokens: base.modelContextWindowTokens,}),- "step.started": (_event, ctx) =>- ctx.session.auth.initiator?.attributes?.model === "continuation"- ? { model: continuationModel(), modelContextWindowTokens: 1_000_000 }- : { model: base.model, modelContextWindowTokens: base.modelContextWindowTokens },+ "step.started": (_event, ctx) => {+ switch (ctx.session.auth.initiator?.attributes?.model) {+ case "continuation":+ return { model: continuationModel(), modelContextWindowTokens: 1_000_000 };+ case "hitl":+ return { model: humanInputModel(), modelContextWindowTokens: 1_000_000 };+ default:+ return { model: base.model, modelContextWindowTokens: base.modelContextWindowTokens };+ }+ },},}),});diff --git a/e2e/fixtures/agent-tools-hitl/agent/channels/eve.ts b/e2e/fixtures/agent-tools-hitl/agent/channels/eve.tsindex a95240c49..3846ff159 100644--- a/e2e/fixtures/agent-tools-hitl/agent/channels/eve.ts+++ b/e2e/fixtures/agent-tools-hitl/agent/channels/eve.ts@@ -7,6 +7,8 @@ export default eveChannel({return {attributes: {fixture: "authorized-response",+ // Lets a hitl eval change what the current caller's policies and tools see.+ flag: request.headers.get("x-eve-fixture-flag") ?? "none",model: request.headers.get("x-eve-fixture-model") ?? "default",},authenticator: "e2e-fixture",diff --git a/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/model.ts b/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/model.tsnew file mode 100644index 000000000..8e50052fb--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/model.ts@@ -0,0 +1,38 @@+import { mockModel } from "eve/evals";++import { respond } from "./respond.ts";++type Prompt = Parameters<Exclude<ReturnType<typeof mockModel>, string>["doGenerate"]>[0]["prompt"];++/**+ * The hitl suite's scripted model. eve runs approved calls itself, so+ * no prompt may carry an AI SDK approval part; one fails the turn.+ */+export function humanInputModel() {+ const model = mockModel({ modelId: "hitl", respond });+ if (typeof model === "string" || model.specificationVersion !== "v4") {+ throw new Error("This fixture expects mockModel's v4 provider boundary.");+ }+ const generate = model.doGenerate.bind(model);+ const stream = model.doStream.bind(model);+ model.doGenerate = async (options) => {+ rejectApprovalParts(options.prompt);+ return generate(options);+ };+ model.doStream = async (options) => {+ rejectApprovalParts(options.prompt);+ return stream(options);+ };+ return model;+}++function rejectApprovalParts(prompt: Prompt) {+ for (const message of prompt) {+ if (typeof message.content === "string") continue;+ for (const part of message.content) {+ if (part.type.includes("approval")) {+ throw new Error(`Model history carries an AI SDK approval part: ${part.type}.`);+ }+ }+ }+}diff --git a/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/respond.ts b/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/respond.tsnew file mode 100644index 000000000..6baca07fc--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/respond.ts@@ -0,0 +1,124 @@+import type { MockModelRequest, MockModelResponse, MockModelToolCall } from "eve/evals";++/** What people say in the hitl evals. Each sentence selects one script below. */+export const SAY = {+ changeA: "Alice asks for change A.",+ changesAB: "Alice asks for changes A and B together.",+ changeAAndAuthorized: "Alice asks for change A and an authorized change together.",+ authorized: "Alice asks for an authorized change.",+ oauthChecked: "Alice asks for an OAuth-checked change.",+ firstEcho: "Alice asks for a guarded echo of the first note.",+ secondEcho: "Alice asks for a guarded echo of the second note.",+ retiring: "Alice asks for the retiring change.",+ checkAccess: "Alice asks to check her Fixture Auth access.",+ publish: "Alice asks to publish the draft.",+ hello: "Alice changes her mind and asks for a short hello instead.",+ bobStatus: "Bob asks for a status update.",+ spendBudget: "Alice spends the remaining budget on a summary.",+ statusNote: "Alice asks for a status note.",+ deadline: "Alice adds that the note should mention the deadline.",+} as const;++export const REPLY = {+ hello: "Hello, Alice.",+ helloAfterAuthorization: "Hello, Alice. I dropped the authorization you no longer need.",+ bobStatus: "Bob's status update is ready.",+ summary: "Summary written.",+ statusNote: "Status note ready.",+ statusNoteWithDeadline: "Status note ready. It mentions the deadline.",+ staleAnswer: "Noted your earlier answer; it authorized nothing.",+} as const;++const STALE_ANSWER_PREFIX =+ "The user submitted the following response to an earlier interactive prompt.";+const SIGN_IN_DROPPED = "was cancelled because the user sent a new message instead";++const call = (id: string, name: string, input: object = {}): MockModelToolCall => ({+ id,+ input,+ name,+});++/**+ * Scripted model for `evals/hitl`. The newest message with a script+ * drives the step; tool results, not a counter, select the next response, so+ * durable replay follows the same script.+ */+export function respond(request: MockModelRequest): MockModelResponse {+ const result = (id: string) => request.toolResults.find((entry) => entry.id === id);+ const outcome = (id: string): string => {+ const found = result(id);+ if (found === undefined) return "missing";+ return found.isError ? "not run" : `done ${JSON.stringify(found.output)}`;+ };+ const run = (calls: MockModelToolCall[], answer: () => string): MockModelResponse => {+ const missing = calls.filter((tool) => result(tool.id!) === undefined);+ return missing.length > 0 ? { toolCalls: missing } : { text: answer() };+ };+ const allText = request.messages.map((message) => message.text).join("\n");++ const scripts: Record<string, () => MockModelResponse> = {+ [SAY.changeA]: () => run([call("a", "change-a")], () => `Change A: ${outcome("a")}.`),+ [SAY.changesAB]: () =>+ run(+ [call("a", "change-a"), call("b", "change-b")],+ () => `Change A: ${outcome("a")}. Change B: ${outcome("b")}.`,+ ),+ [SAY.changeAAndAuthorized]: () =>+ run(+ [call("a", "change-a"), call("auth", "authorized-change")],+ () => `Change A: ${outcome("a")}. Authorized change: ${outcome("auth")}.`,+ ),+ [SAY.authorized]: () =>+ run([call("auth", "authorized-change")], () => `Authorized change: ${outcome("auth")}.`),+ [SAY.oauthChecked]: () =>+ run(+ [call("oauth", "oauth-authorized-gate", { marker: "hitl" })],+ () => `OAuth-checked change: ${outcome("oauth")}.`,+ ),+ [SAY.firstEcho]: () =>+ run(+ [call("echo-first", "guarded-echo", { note: "first" })],+ () => `First echo: ${outcome("echo-first")}.`,+ ),+ [SAY.secondEcho]: () =>+ run(+ [call("echo-second", "guarded-echo", { note: "second" })],+ () => `Second echo: ${outcome("echo-second")}.`,+ ),+ [SAY.retiring]: () =>+ run([call("retiring", "retiring-change")], () => `Retiring change: ${outcome("retiring")}.`),+ [SAY.checkAccess]: () =>+ run(+ [call("probe", "auth-probe", { marker: "hitl" })],+ () => `Access check: ${outcome("probe")}.`,+ ),+ [SAY.publish]: () =>+ run([call("publish", "publish-draft")], () => `Publish: ${outcome("publish")}.`),+ [SAY.hello]: () => ({+ text: allText.includes(SIGN_IN_DROPPED) ? REPLY.helloAfterAuthorization : REPLY.hello,+ }),+ [SAY.bobStatus]: () => ({ text: REPLY.bobStatus }),+ [SAY.spendBudget]: () => ({+ text: REPLY.summary,+ usage: { inputTokens: 1, outputTokens: 1_000_000 },+ }),+ [SAY.statusNote]: () => ({+ text: request.userMessages.includes(SAY.deadline)+ ? REPLY.statusNoteWithDeadline+ : REPLY.statusNote,+ }),+ };+ scripts[SAY.deadline] = scripts[SAY.statusNote]!;++ for (const message of [...request.userMessages].reverse()) {+ if (message.startsWith(STALE_ANSWER_PREFIX)) return reply({ text: REPLY.staleAnswer });+ const script = scripts[message];+ if (script !== undefined) return reply(script());+ }+ throw new Error(`No hitl script for: ${JSON.stringify(request.userMessages)}`);+}++function reply(response: MockModelResponse): MockModelResponse {+ return { usage: { inputTokens: 1, outputTokens: 1 }, ...response };+}diff --git a/e2e/fixtures/agent-tools-hitl/agent/tools/publish-draft.ts b/e2e/fixtures/agent-tools-hitl/agent/tools/publish-draft.tsnew file mode 100644index 000000000..5db494ff4--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/agent/tools/publish-draft.ts@@ -0,0 +1,47 @@+import {+ ConnectionAuthorizationRequiredError,+ defineInteractiveAuthorization,+} from "eve/connections";+import { defineState } from "eve/context";+import { defineTool } from "eve/tools";+import { once } from "eve/tools/approval";+import { z } from "zod";++const publications = defineState("publish-draft.publications", () => 0);++// Shares auth-probe's authorization (same `authKey`), so a publish asked while an+// auth-probe authorization is open supersedes that attempt.+const fixtureAuth = defineInteractiveAuthorization<{ marker: string }>({+ async getToken() {+ throw new ConnectionAuthorizationRequiredError("auth-probe");+ },+ async startAuthorization({ callbackUrl }) {+ const url = new URL(callbackUrl);+ url.searchParams.set("code", "fixture-ok");+ return {+ challenge: { displayName: "Fixture Auth", url: url.href },+ resume: { marker: "fixture-resume" },+ };+ },+ async completeAuthorization({ callback, resume }) {+ if (callback.params.code !== "fixture-ok" || resume?.marker !== "fixture-resume") {+ throw new Error("Fixture authorization state mismatch.");+ }+ return { token: "publish-token-Q2W7" };+ },+});++/** Needs approval once, then an authorization each time it runs. */+export default defineTool({+ description: "Publish the fixture draft. Requires approval and a Fixture Auth sign-in.",+ inputSchema: z.object({}),+ approval: once(),+ async execute(_input, ctx) {+ await ctx.getToken(fixtureAuth, { authKey: "auth-probe", displayName: "Fixture Auth" });+ publications.update((count) => count + 1);+ return {+ actor: ctx.session.auth.current?.principalId ?? "none",+ publications: publications.get(),+ };+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/agent/tools/retiring-change.ts b/e2e/fixtures/agent-tools-hitl/agent/tools/retiring-change.tsnew file mode 100644index 000000000..e524570f8--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/agent/tools/retiring-change.ts@@ -0,0 +1,22 @@+import { defineDynamic, defineTool } from "eve/tools";+import { always } from "eve/tools/approval";+import { z } from "zod";++/**+ * A step-scoped tool offered only while the caller's fixture flag is not+ * "retire", for proving an approved call keeps the tools of the step that+ * asked even when a later step would not offer this one.+ */+export default defineDynamic({+ events: {+ "step.started": (_event, ctx) =>+ ctx.session.auth.current?.attributes?.flag === "retire"+ ? null+ : defineTool({+ description: "Apply the retiring fixture change after approval.",+ inputSchema: z.object({}),+ approval: always(),+ execute: () => ({ change: "retiring" }),+ }),+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md b/e2e/fixtures/agent-tools-hitl/evals/hitl/README.mdnew file mode 100644index 000000000..b88de5ea9--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md@@ -0,0 +1,74 @@+# Human input++One eval per rule of eve's human input (`packages/eve/src/harness/hitl/`):+tool approvals, response policies, the budget question, authorizations, and requests+relayed from child sessions and workflow runs. Each eval drives a real session+over HTTP and asserts on its event stream.++Every eval here is deterministic in every world. Sessions select the scripted+`hitl` model with the `x-eve-fixture-model` header+([`agent/lib/hitl`](../../agent/lib/hitl)), so they need no+`real-model` tag. That model also fails the turn if its history ever carries an+AI SDK approval part. eve runs approved calls itself, so a passing run also shows+that history has no such part.++People are named by the fixture's `x-eve-fixture-user` header: Alice asks, Bob+is someone else in the session, and `e2e-approval-responder` is the release+manager whom the fixture's response policies allow. `x-eve-fixture-flag` lets a+caller retire a step tool (`retiring-change`).++Relayed rules live with the fixtures that own children and workflow tools:+[`agent-subagents-hitl/evals/hitl`](../../../agent-subagents-hitl/evals/hitl)+and [`agent-workflow-tools/evals/hitl`](../../../agent-workflow-tools/evals/hitl).++## Coverage++| Rule | Eval |+| ------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |+| Approve runs the call in eve, result before the next step | [approve-runs-the-call](./approvals/approve-runs-the-call.eval.ts) |+| Deny resolves `denied` and records a not-run result | [deny-records-not-run](./approvals/deny-records-not-run.eval.ts) |+| A partial answer keeps the turn held, with no model call | [partial-answer-keeps-turn-held](./approvals/partial-answer-keeps-turn-held.eval.ts) |+| A typed reply naming an option answers | [typed-approve-answers](./approvals/typed-approve-answers.eval.ts) |+| A typed reply answers only what it matches | [typed-reply-answers-only-what-matches](./approvals/typed-reply-answers-only-what-matches.eval.ts) |+| Steering ignores unanswered approvals and keeps given answers | [steer-ignores-unanswered](./approvals/steer-ignores-unanswered.eval.ts) |+| Someone else's message waits for the held turn | [other-person-message-waits](./approvals/other-person-message-waits.eval.ts) |+| An answer to a request that is not open authorizes nothing | [stale-answer-authorizes-nothing](./approvals/stale-answer-authorizes-nothing.eval.ts) |+| A `once()` grant is reused | [once-grant-is-reused](./approvals/once-grant-is-reused.eval.ts) |+| An approved call runs with the asking step's tools | [approved-call-uses-asking-step-tools](./approvals/approved-call-uses-asking-step-tools.eval.ts) |+| A rejected responder leaves the approval open | [rejected-responder-leaves-approval-open](./response-policies/rejected-responder-leaves-approval-open.eval.ts) |+| An allowed responder settles the approval | [allowed-responder-settles](./response-policies/allowed-responder-settles.eval.ts) |+| A responder's authorization, then settlement | [responder-authorization-then-settles](./response-policies/responder-authorization-then-settles.eval.ts) |+| Stop cancels the turn, resolving once | [stop-cancels-once](./budget/stop-cancels-once.eval.ts) |+| No step starts behind the budget question | [no-step-behind-budget](./budget/no-step-behind-budget.eval.ts) |+| A late budget answer is dropped | [stale-budget-answer-dropped](./budget/stale-budget-answer-dropped.eval.ts) |+| An authorization holds the turn; the callback resumes it as the requester | [authorization-holds-then-resumes-as-requester](./authorizations/authorization-holds-then-resumes-as-requester.eval.ts) |+| Steering declines an authorization and tells the model | [steer-declines-authorization](./authorizations/steer-declines-authorization.eval.ts) |+| Cancel declines an authorization | [cancel-declines-authorization](./authorizations/cancel-declines-authorization.eval.ts) |+| An approved call that needs an authorization | [approved-call-needs-authorization](./authorizations/approved-call-needs-authorization.eval.ts) |++The rule that the model never runs while a request is open is checked inside+the evals above by `expectNoModelCallWhileOpen` and+`expectNoModelCallDuringAuthorization` ([helpers](./helpers.ts)).++## Not covered here++- **Candidate expiry.** A candidate times out after a 10-minute wall-clock+ TTL, and e2e cannot advance the clock. Covered by+ `harness/hitl/candidates.test.ts` ("expires only candidates whose deadline+ has passed") and `harness/hitl/coordinator.test.ts` ("completes an expired+ candidate without executing policy").+- **A `once()` grant hidden while an approval for its key waits.** Policies+ only run at a step's start or right before an approved call, and neither+ happens while that approval waits, so HTTP cannot observe the hiding.+ Covered by `harness/session-machine/machine.test.ts` ("grants a once()+ approval's key, except to a call still asking for it").+- **An approved tool removed before the answer.** Step tools persist with+ their step, so only a redeploy between the ask and the answer removes one.+ Local e2e cannot redeploy.+- **A fresh relayed batch replacing an older one.** This needs a child or+ remote agent to ask again from the same source while its earlier batch is+ open. Each workflow `ctx.ask()` is its own source. Covered by+ `harness/proxy-input-requests.test.ts` ("replaces prior entries for the same+ child continuation token").+- **Handoff blocked while a request is open.** This needs a deployment+ handoff, which only the Vercel redeploy suite performs.diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/approve-runs-the-call.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/approve-runs-the-call.eval.tsnew file mode 100644index 000000000..6650ac1a9--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/approve-runs-the-call.eval.ts@@ -0,0 +1,42 @@+import { defineEval } from "eve/evals";++import {+ SAY,+ aliceSession,+ answers,+ approvalFor,+ asAlice,+ expectHeld,+ expectNoModelCallWhileOpen,+ expectResolved,+} from "../helpers.ts";++/**+ * Alice asks for change A and approves it. eve runs the call itself, so its+ * result lands before the model's next step, and the model's history never+ * carries an AI SDK approval part (the scripted model fails the turn if it does).+ */+export default defineEval({+ description: "Approving a call runs it in eve, before the model's next step.",+ tags: ["hitl", "approval"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await aliceSession(t);+ const held = await session.send(SAY.changeA, asAlice);+ expectHeld(held);+ const request = approvalFor(held, "change-a");++ const approved = (await session.respond(answers("approve", request), asAlice)).expectOk();+ expectResolved(approved, request, "approved");+ approved.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });+ approved.notEvent("turn.started");+ approved.eventOrder([+ { type: "input.resolved" },+ { type: "step.started" },+ { type: "action.result", data: { status: "completed", result: { toolName: "change-a" } } },+ { type: "message.completed", data: { message: /^Change A: done / } },+ { type: "turn.completed" },+ ]);+ expectNoModelCallWhileOpen(session, request.requestId);+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/approved-call-uses-asking-step-tools.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/approved-call-uses-asking-step-tools.eval.tsnew file mode 100644index 000000000..a0580b912--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/approved-call-uses-asking-step-tools.eval.ts@@ -0,0 +1,34 @@+import { defineEval } from "eve/evals";++import { ALICE, SAY, aliceSession, answers, approvalFor, as, asAlice } from "../helpers.ts";++/**+ * The retiring change is a step-scoped dynamic tool that the fixture stops+ * offering for callers with the "retire" flag. Alice approves it with that+ * flag on: the approved call still runs, because eve runs it with the tools+ * of the step that asked, as its approval saw them.+ *+ * The opposite case — the tool is gone from the deployment by the time the+ * answer arrives, which fails with "The approved tool is no longer+ * available" — needs a redeploy between the ask and the answer, so local+ * e2e cannot reach it.+ */+export default defineEval({+ description: "An approved call runs with the tools of the step that asked for it.",+ tags: ["hitl", "approval", "dynamic-tools"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await aliceSession(t);+ const request = approvalFor(await session.send(SAY.retiring, asAlice), "retiring-change");++ const approved = (+ await session.respond(answers("approve", request), as(ALICE, "retire"))+ ).expectOk();+ approved.calledTool("retiring-change", {+ status: "completed",+ output: { change: "retiring" },+ count: 1,+ });+ approved.event("message.completed", { data: { message: /^Retiring change: done / } });+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/deny-records-not-run.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/deny-records-not-run.eval.tsnew file mode 100644index 000000000..6d4224cad--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/deny-records-not-run.eval.ts@@ -0,0 +1,38 @@+import { defineEval } from "eve/evals";++import {+ SAY,+ aliceSession,+ answers,+ approvalFor,+ asAlice,+ expectNotRun,+ expectResolved,+} from "../helpers.ts";++/** Alice asks for change A, then declines it: the call never runs and the model hears so. */+export default defineEval({+ description: "Denying a call resolves it as denied and records a not-run result.",+ tags: ["hitl", "approval"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await aliceSession(t);+ const request = approvalFor(await session.send(SAY.changeA, asAlice), "change-a");++ const denied = (await session.respond(answers("cancel", request), asAlice)).expectOk();+ expectResolved(denied, request, "denied");+ expectNotRun(denied, "change-a");+ denied.event("action.result", {+ count: 1,+ data: {+ status: "rejected",+ result: {+ output: { approval: { status: "denied" }, code: "TOOL_EXECUTION_DENIED" },+ toolName: "change-a",+ },+ },+ });+ denied.event("message.completed", { data: { message: "Change A: not run." } });+ denied.event("turn.completed", { count: 1 });+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/once-grant-is-reused.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/once-grant-is-reused.eval.tsnew file mode 100644index 000000000..5719c590b--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/once-grant-is-reused.eval.ts@@ -0,0 +1,29 @@+import { defineEval } from "eve/evals";++import { SAY, aliceSession, answers, approvalFor, asAlice } from "../helpers.ts";++/**+ * `guarded-echo` uses `once()`. Alice approves the first echo; the grant lasts+ * the session, so her second echo runs without asking again.+ */+export default defineEval({+ description: "A once() approval grants its key for the rest of the session.",+ tags: ["hitl", "approval", "once"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await aliceSession(t);+ const first = approvalFor(await session.send(SAY.firstEcho, asAlice), "guarded-echo");+ (await session.respond(answers("approve", first), asAlice))+ .expectOk()+ .calledTool("guarded-echo", { status: "completed", count: 1 });++ const second = (await session.send(SAY.secondEcho, asAlice)).expectOk();+ second.notEvent("input.requested");+ second.calledTool("guarded-echo", {+ status: "completed",+ output: { echoed: "second" },+ count: 1,+ });+ second.event("message.completed", { data: { message: /^Second echo: done / } });+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/other-person-message-waits.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/other-person-message-waits.eval.tsnew file mode 100644index 000000000..52d1c1915--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/other-person-message-waits.eval.ts@@ -0,0 +1,46 @@+import { defineEval } from "eve/evals";++import {+ REPLY,+ SAY,+ aliceSession,+ answers,+ approvalFor,+ asAlice,+ asBob,+ expectResolved,+} from "../helpers.ts";++/**+ * Alice's change waits for her approval when Bob asks for a status update.+ * Only Alice's own message steers her turn: Bob's waits for the turn to end,+ * so Alice's approval still runs the change, and Bob's message runs next.+ */+export default defineEval({+ description: "A message from someone else waits for the held turn to end instead of steering it.",+ tags: ["hitl", "approval", "steer"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await aliceSession(t);+ const request = approvalFor(await session.send(SAY.changeA, asAlice), "change-a");++ const bob = await session.start(SAY.bobStatus, asBob);+ const approved = (await session.respond(answers("approve", request), asAlice)).expectOk();+ expectResolved(approved, request, "approved");+ approved.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });+ approved.notEvent("message.received", { data: { message: SAY.bobStatus } });++ approved.event("message.completed", { data: { message: /^Change A: done / } });+ approved.event("turn.completed", { count: 1 });++ // Bob's message starts its own turn once Alice's has ended.+ const bobTurn = (await bob.result()).expectOk();+ bobTurn.eventOrder([+ { type: "turn.started" },+ { type: "message.received", data: { message: SAY.bobStatus } },+ { type: "message.completed", data: { message: REPLY.bobStatus } },+ { type: "turn.completed" },+ ]);+ bobTurn.notEvent("input.resolved");+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/partial-answer-keeps-turn-held.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/partial-answer-keeps-turn-held.eval.tsnew file mode 100644index 000000000..e2ebf20d5--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/partial-answer-keeps-turn-held.eval.ts@@ -0,0 +1,53 @@+import { defineEval } from "eve/evals";++import {+ SAY,+ aliceSession,+ answers,+ approvalFor,+ asAlice,+ expectHeld,+ expectNoModelCallWhileOpen,+ expectNotRun,+} from "../helpers.ts";++/**+ * One step asks Alice to approve changes A and B. Her answer to A alone waits+ * in the turn: nothing resolves, nothing runs, and the model is not called.+ * Declining B then resolves both together, each with its own answer.+ */+export default defineEval({+ description: "A partial answer keeps the turn held until every approval of the step is answered.",+ tags: ["hitl", "approval", "partial-approval"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await aliceSession(t);+ const asked = await session.send(SAY.changesAB, asAlice);+ const a = approvalFor(asked, "change-a");+ const b = approvalFor(asked, "change-b");++ const partial = await session.respond(answers("approve", a), asAlice);+ expectHeld(partial);+ partial.notEvent("input.resolved");+ partial.notEvent("step.started");+ partial.notEvent("action.result");++ const finished = (await session.respond(answers("cancel", b), asAlice)).expectOk();+ finished.event("input.resolved", {+ count: 1,+ data: {+ resolutions: (items) =>+ items.length === 2 &&+ items.some((item) => item.requestId === a.requestId && item.outcome === "approved") &&+ items.some((item) => item.requestId === b.requestId && item.outcome === "denied"),+ },+ });+ finished.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });+ expectNotRun(finished, "change-b");+ finished.event("message.completed", {+ data: { message: /^Change A: done .*\. Change B: not run\.$/u },+ });+ expectNoModelCallWhileOpen(session, a.requestId);+ expectNoModelCallWhileOpen(session, b.requestId);+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/stale-answer-authorizes-nothing.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/stale-answer-authorizes-nothing.eval.tsnew file mode 100644index 000000000..d20704c41--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/stale-answer-authorizes-nothing.eval.ts@@ -0,0 +1,35 @@+import { defineEval } from "eve/evals";++import {+ REPLY,+ SAY,+ aliceSession,+ approvalFor,+ asAlice,+ expectNotRun,+ expectResolved,+} from "../helpers.ts";++/**+ * While change A waits, Alice's client submits "approve" for a request that+ * is not open (an old prompt). That answer becomes a message the model reads:+ * it steers the turn like any message, so it authorizes nothing, A included.+ */+export default defineEval({+ description: "An answer to a request that is not open becomes a message and authorizes nothing.",+ tags: ["hitl", "approval", "stale-response"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await aliceSession(t);+ const request = approvalFor(await session.send(SAY.changeA, asAlice), "change-a");++ const stale = (+ await session.respond([{ optionId: "approve", requestId: "an-earlier-prompt" }], asAlice)+ ).expectOk();+ expectResolved(stale, request, "ignored");+ expectNotRun(stale, "change-a");+ stale.event("message.received", { count: 1, data: { message: "approve" } });+ stale.event("message.completed", { data: { message: REPLY.staleAnswer } });+ stale.event("turn.completed", { count: 1 });+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/steer-ignores-unanswered.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/steer-ignores-unanswered.eval.tsnew file mode 100644index 000000000..53dc62cee--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/steer-ignores-unanswered.eval.ts@@ -0,0 +1,61 @@+import { defineEval } from "eve/evals";++import {+ REPLY,+ SAY,+ aliceSession,+ answers,+ approvalFor,+ asAlice,+ expectNotRun,+} from "../helpers.ts";++/**+ * Alice approves change A from a pair, then changes her mind and asks for a+ * hello instead. Her message steers the held turn: B, which nobody answered,+ * is ignored and never runs; A keeps her approval and runs; the model reads+ * her message in the same turn.+ */+export default defineEval({+ description: "Steering past approvals ignores unanswered ones and keeps the answers given.",+ tags: ["hitl", "approval", "partial-approval", "steer"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await aliceSession(t);+ const asked = await session.send(SAY.changesAB, asAlice);+ const a = approvalFor(asked, "change-a");+ const b = approvalFor(asked, "change-b");+ await session.respond(answers("approve", a), asAlice);++ const steered = (await session.send(SAY.hello, asAlice)).expectOk();+ steered.event("input.resolved", {+ count: 1,+ data: {+ resolutions: (items) =>+ items.some((item) => item.requestId === a.requestId && item.outcome === "approved") &&+ items.some((item) => item.requestId === b.requestId && item.outcome === "ignored"),+ },+ });+ steered.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });+ expectNotRun(steered, "change-b");+ steered.event("action.result", {+ data: {+ result: { output: { approval: { status: "ignored" } }, toolName: "change-b" },+ status: "rejected",+ },+ });+ steered.notEvent("turn.started");+ steered.event("message.received", { count: 1, data: { message: SAY.hello } });+ // The answers resolve first. A runs in the result-reading step; Alice's+ // queued message is received by the next step of the same turn.+ steered.eventOrder([+ { type: "input.resolved" },+ { type: "step.started", data: { stepIndex: 1 } },+ { type: "action.result", data: { status: "completed", result: { toolName: "change-a" } } },+ { type: "message.received", data: { message: SAY.hello } },+ { type: "step.started", data: { stepIndex: 2 } },+ { type: "message.completed", data: { message: REPLY.hello } },+ { type: "turn.completed" },+ ]);+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/typed-approve-answers.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/typed-approve-answers.eval.tsnew file mode 100644index 000000000..74b78a91e--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/typed-approve-answers.eval.ts@@ -0,0 +1,25 @@+import { defineEval } from "eve/evals";++import { SAY, aliceSession, approvalFor, asAlice, expectResolved } from "../helpers.ts";++/**+ * Alice answers her waiting change by typing "approve" instead of pressing the+ * button. The reply names the approval's option, so it answers it: the change+ * runs in the same turn, and the model never reads "approve" as a new request.+ */+export default defineEval({+ description: "A typed reply naming an approval's option answers it.",+ tags: ["hitl", "approval", "text-reply"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await aliceSession(t);+ const request = approvalFor(await session.send(SAY.changeA, asAlice), "change-a");++ const typed = (await session.send("approve", asAlice)).expectOk();+ expectResolved(typed, request, "approved");+ typed.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });+ typed.notEvent("turn.started");+ typed.event("message.completed", { data: { message: /^Change A: done / } });+ typed.event("turn.completed", { count: 1 });+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/typed-reply-answers-only-what-matches.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/typed-reply-answers-only-what-matches.eval.tsnew file mode 100644index 000000000..d8b4c9f7e--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/typed-reply-answers-only-what-matches.eval.ts@@ -0,0 +1,54 @@+import { defineEval } from "eve/evals";++import {+ SAY,+ aliceSession,+ answers,+ approvalFor,+ asAlice,+ asReleaseManager,+ expectHeld,+} from "../helpers.ts";++/**+ * One step asks for change A and an authorized change. Alice types "approve":+ * that answers A, but never the authorized change, whose response policy must+ * know who answered. The turn stays held until the release manager approves+ * it; then both resolve together and run.+ */+export default defineEval({+ description: "A typed reply answers the approvals it matches and leaves the rest open.",+ tags: ["hitl", "approval", "text-reply", "authorization"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await aliceSession(t);+ const asked = await session.send(SAY.changeAAndAuthorized, asAlice);+ const a = approvalFor(asked, "change-a");+ const authorized = approvalFor(asked, "authorized-change");++ const typed = await session.send("approve", asAlice);+ expectHeld(typed);+ typed.notEvent("input.resolved");+ typed.notEvent("action.result");+ typed.notEvent("step.started");++ const settled = (+ await session.respond(answers("approve", authorized), asReleaseManager)+ ).expectOk();+ settled.event("approval.settled", {+ count: 1,+ data: { outcome: "approved", requestId: authorized.requestId },+ });+ settled.event("input.resolved", {+ count: 1,+ data: {+ resolutions: (items) =>+ [a.requestId, authorized.requestId].every((id) =>+ items.some((item) => item.requestId === id && item.outcome === "approved"),+ ),+ },+ });+ settled.calledTool("change-a", { status: "completed", count: 1 });+ settled.calledTool("authorized-change", { status: "completed", count: 1 });+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/approved-call-needs-authorization.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/approved-call-needs-authorization.eval.tsnew file mode 100644index 000000000..8b001060f--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/approved-call-needs-authorization.eval.ts@@ -0,0 +1,53 @@+import { defineEval } from "eve/evals";++import {+ ALICE,+ SAY,+ aliceSession,+ answers,+ approvalFor,+ asAlice,+ completeAuthorization,+ expectHeld,+ expectResolved,+ follow,+ authorizationFrom,+} from "../helpers.ts";++/**+ * Publishing the draft needs approval (once) and an authorization. Alice approves;+ * the approved call then asks her to authorize, which holds the turn. After the+ * callback the model calls publish again, the once() grant covers it, and it+ * runs as Alice.+ */+export default defineEval({+ description: "An approved call that needs an authorization holds the turn and runs after it.",+ tags: ["hitl", "approval", "authorization"],+ timeoutMs: 90_000,+ async test(t) {+ const session = await aliceSession(t);+ const request = approvalFor(await session.send(SAY.publish, asAlice), "publish-draft");++ const signing = await session.respond(answers("approve", request), asAlice);+ expectResolved(signing, request, "approved");+ expectHeld(signing);+ signing.event("authorization.required", { count: 1, data: { principalId: ALICE } });+ signing.notEvent("action.result", { data: { status: "completed" } });+ const authorization = authorizationFrom(signing);++ const startIndex = session.state.streamIndex;+ await completeAuthorization(authorization.url);+ const resumed = (await follow(t, session, startIndex)).expectOk();+ resumed.event("authorization.completed", {+ count: 1,+ data: { attemptId: authorization.attemptId, outcome: "authorized" },+ });+ resumed.notEvent("input.requested");+ resumed.calledTool("publish-draft", {+ status: "completed",+ output: { actor: ALICE, publications: 1 },+ count: 1,+ });+ resumed.event("message.completed", { data: { message: /^Publish: done / } });+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/authorization-holds-then-resumes-as-requester.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/authorization-holds-then-resumes-as-requester.eval.tsnew file mode 100644index 000000000..efba7b644--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/authorization-holds-then-resumes-as-requester.eval.ts@@ -0,0 +1,49 @@+import { defineEval } from "eve/evals";++import {+ ALICE,+ SAY,+ aliceSession,+ asAlice,+ completeAuthorization,+ expectHeld,+ expectNoModelCallDuringAuthorization,+ follow,+ authorizationFrom,+} from "../helpers.ts";++/**+ * Alice's access check needs a Fixture Auth sign-in, which holds her turn.+ * The callback is an anonymous browser request, yet the turn resumes as+ * Alice: the tool runs again with her identity, in the same turn.+ */+export default defineEval({+ description:+ "A tool's authorization holds the turn, and its callback resumes it as the requester.",+ tags: ["hitl", "authorization"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await aliceSession(t);+ const held = await session.send(SAY.checkAccess, asAlice);+ expectHeld(held);+ held.event("authorization.required", { count: 1, data: { principalId: ALICE } });+ held.notEvent("authorization.completed");+ const authorization = authorizationFrom(held);++ const startIndex = session.state.streamIndex;+ await completeAuthorization(authorization.url);+ const resumed = (await follow(t, session, startIndex)).expectOk();+ resumed.notEvent("turn.started");+ resumed.event("authorization.completed", {+ count: 1,+ data: { attemptId: authorization.attemptId, outcome: "authorized", principalId: ALICE },+ });+ resumed.calledTool("auth-probe", {+ status: "completed",+ output: { actor: ALICE },+ count: 1,+ });+ resumed.event("message.completed", { data: { message: /^Access check: done / } });+ expectNoModelCallDuringAuthorization(resumed, authorization.attemptId, held.events);+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/cancel-declines-authorization.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/cancel-declines-authorization.eval.tsnew file mode 100644index 000000000..f4065e6a7--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/cancel-declines-authorization.eval.ts@@ -0,0 +1,25 @@+import { defineEval } from "eve/evals";++import { SAY, aliceSession, asAlice, follow, authorizationFrom } from "../helpers.ts";++/** Alice cancels her turn while it waits for her authorization: the authorization is declined. */+export default defineEval({+ description: "Cancelling a turn held on an authorization declines the authorization.",+ tags: ["hitl", "authorization", "cancel"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await aliceSession(t);+ const authorization = authorizationFrom(await session.send(SAY.checkAccess, asAlice));+ const startIndex = session.state.streamIndex;++ await session.cancel();+ const cancelled = await follow(t, session, startIndex);+ cancelled.event("authorization.completed", {+ count: 1,+ data: { attemptId: authorization.attemptId, outcome: "declined", reason: "Cancelled." },+ });+ cancelled.event("turn.cancelled", { count: 1 });+ cancelled.notEvent("step.started");+ cancelled.notEvent("action.result", { data: { status: "completed" } });+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/steer-declines-authorization.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/steer-declines-authorization.eval.tsnew file mode 100644index 000000000..cde806c87--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/steer-declines-authorization.eval.ts@@ -0,0 +1,35 @@+import { defineEval } from "eve/evals";++import { REPLY, SAY, aliceSession, asAlice, expectHeld, authorizationFrom } from "../helpers.ts";++/**+ * Alice's access check waits for her authorization when she asks for a hello+ * instead. Her message steers the turn: the authorization is declined, and the+ * model is told it ended so it asks again only if it still needs it.+ */+export default defineEval({+ description:+ "Steering past an authorization declines it and tells the model which authorization ended.",+ tags: ["hitl", "authorization", "steer"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await aliceSession(t);+ const held = await session.send(SAY.checkAccess, asAlice);+ expectHeld(held);+ const authorization = authorizationFrom(held);++ const steered = (await session.send(SAY.hello, asAlice)).expectOk();+ steered.notEvent("turn.started");+ steered.event("authorization.completed", {+ count: 1,+ data: {+ attemptId: authorization.attemptId,+ outcome: "declined",+ reason: "Cancelled because a new message arrived.",+ },+ });+ // The scripted model answers this way only when the authorization note reached it.+ steered.event("message.completed", { data: { message: REPLY.helloAfterAuthorization } });+ steered.notEvent("action.result", { data: { status: "completed" } });+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/no-step-behind-budget.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/no-step-behind-budget.eval.tsnew file mode 100644index 000000000..4f7843812--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/no-step-behind-budget.eval.ts@@ -0,0 +1,21 @@+import { defineEval } from "eve/evals";++import { SAY, asAlice, budgetQuestion, expectNoModelCallWhileOpen } from "../helpers.ts";++/**+ * While the budget question waits, Alice adds a message. The turn must stay+ * held without starting a model step: nothing may run while the question is+ * open.+ */+export default defineEval({+ description: "No model step starts while the budget question is open.",+ tags: ["hitl", "budget"],+ timeoutMs: 60_000,+ async test(t) {+ const { request, session } = await budgetQuestion(t);++ const added = await session.send(SAY.deadline, asAlice);+ added.notEvent("step.started");+ expectNoModelCallWhileOpen(session, request.requestId);+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/stale-budget-answer-dropped.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/stale-budget-answer-dropped.eval.tsnew file mode 100644index 000000000..9a02021f5--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/stale-budget-answer-dropped.eval.ts@@ -0,0 +1,26 @@+import { defineEval } from "eve/evals";++import { REPLY, SAY, answers, asAlice, budgetQuestion } from "../helpers.ts";++/**+ * Alice's client sends her Continue a second time after the question closed.+ * A late budget answer must be dropped, not read as a message: read as text,+ * a late Stop would seem to stop something. Her next message then runs as+ * usual, with nothing stale before it.+ */+export default defineEval({+ description: "An answer to a budget question that already closed is dropped.",+ tags: ["hitl", "budget", "stale-response"],+ timeoutMs: 60_000,+ async test(t) {+ const { request, session } = await budgetQuestion(t);+ (await session.respond(answers("continue", request), asAlice)).expectOk();++ await session.startRespond(answers("continue", request), asAlice);+ const next = (await session.send(SAY.bobStatus, asAlice)).expectOk();+ next.event("message.completed", { data: { message: REPLY.bobStatus } });+ session.notEvent("message.received", { data: { message: "continue" } });+ session.notEvent("message.completed", { data: { message: REPLY.staleAnswer } });+ session.event("input.resolved", { count: 1 });+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/stop-cancels-once.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/stop-cancels-once.eval.tsnew file mode 100644index 000000000..6a4cf0389--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/stop-cancels-once.eval.ts@@ -0,0 +1,25 @@+import { defineEval } from "eve/evals";++import { answers, asAlice, budgetQuestion, expectResolved } from "../helpers.ts";++/** Alice chooses Stop: the question resolves once, and the turn is cancelled without a model call. */+export default defineEval({+ description: "Stop resolves the budget question once and cancels the turn.",+ tags: ["hitl", "budget", "cancel"],+ timeoutMs: 60_000,+ async test(t) {+ const { request, session } = await budgetQuestion(t);++ const stopped = await session.respond(answers("stop", request), asAlice);+ expectResolved(stopped, request, "answered");+ stopped.event("input.resolved", { count: 1 });+ stopped.event("turn.cancelled", { count: 1 });+ stopped.notEvent("step.started");+ stopped.notEvent("message.completed");+ stopped.notEvent("turn.failed");+ session.event("input.resolved", {+ count: 1,+ data: { resolutions: (items) => items.some((item) => item.requestId === request.requestId) },+ });+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/helpers.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/helpers.tsnew file mode 100644index 000000000..8db4495b3--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/helpers.ts@@ -0,0 +1,170 @@+import type {+ EveEvalAssertions,+ EveEvalContext,+ EveEvalSession,+ EveEvalTurn,+ InputRequest,+} from "eve/evals";++import { SAY } from "../../agent/lib/hitl/respond.ts";++export { REPLY, SAY } from "../../agent/lib/hitl/respond.ts";++export const ALICE = "alice";+export const BOB = "bob";+/** The fixture's authorized approver for `authorized-change` and `responder-gate`. */+export const RELEASE_MANAGER = "e2e-approval-responder";++/** Headers for one person on the scripted hitl model; `flag` feeds fixture policies. */+export function as(principalId: string, flag?: "retire") {+ return {+ headers: {+ "x-eve-fixture-model": "hitl",+ "x-eve-fixture-user": principalId,+ ...(flag !== undefined && { "x-eve-fixture-flag": flag }),+ },+ };+}++/**+ * A session Alice starts, so the scripted model drives every turn of it.+ * Headers apply per request: pass `as(...)` to every send and respond too.+ */+export function aliceSession(t: EveEvalContext): Promise<EveEvalSession> {+ return t.session(as(ALICE));+}++export const asAlice = as(ALICE);+export const asBob = as(BOB);+export const asReleaseManager = as(RELEASE_MANAGER);++/** The one approval request `turn` raised for `toolName`. */+export function approvalFor(turn: EveEvalTurn, toolName: string): InputRequest {+ const matches = turn.inputRequests.filter((request) => request.action.toolName === toolName);+ if (matches.length !== 1) {+ throw new Error(`Expected one ${toolName} request; found ${matches.length}.`);+ }+ return matches[0]!;+}++/** `respond` answers for the given requests, all with one option. */+export function answers(optionId: string, ...requests: readonly InputRequest[]) {+ return requests.map(({ requestId }) => ({ optionId, requestId }));+}++/**+ * The model never runs while a request of the turn's own is open: no+ * `step.started` falls between the request's `input.requested` and its+ * `input.resolved` (or the end of the stream, while it is still open).+ */+export function expectNoModelCallWhileOpen(on: EveEvalAssertions, requestId: string) {+ on.eventsSatisfy(`no model step starts while ${requestId} is open`, (events) => {+ const asked = events.findIndex(+ (event) =>+ event.type === "input.requested" &&+ event.data.requests.some((request) => request.requestId === requestId),+ );+ if (asked < 0) return false;+ const resolved = events.findIndex(+ (event, index) =>+ index > asked &&+ event.type === "input.resolved" &&+ event.data.resolutions.some((resolution) => resolution.requestId === requestId),+ );+ const open = events.slice(asked + 1, resolved < 0 ? events.length : resolved);+ return !open.some((event) => event.type === "step.started");+ });+}++/**+ * No `step.started` falls between the authorization `attemptId` opening and closing.+ * `before` carries the events streamed before `on` started, such as the ask.+ */+export function expectNoModelCallDuringAuthorization(+ on: EveEvalAssertions,+ attemptId: string,+ before: EveEvalTurn["events"] = [],+) {+ on.eventsSatisfy(`no model step starts while authorization ${attemptId} is open`, (observed) => {+ const events = [...before, ...observed];+ const asked = events.findIndex(+ (event) => event.type === "authorization.required" && event.data.attemptId === attemptId,+ );+ if (asked < 0) return false;+ const closed = events.findIndex(+ (event, index) =>+ index > asked &&+ event.type === "authorization.completed" &&+ event.data.attemptId === attemptId,+ );+ const open = events.slice(asked + 1, closed < 0 ? events.length : closed);+ return !open.some((event) => event.type === "step.started");+ });+}++/** The latest authorization `turn` asked for: its attempt and its fixture callback URL. */+export function authorizationFrom(turn: Pick<EveEvalTurn, "events">): {+ attemptId: string;+ url: URL;+} {+ const required = [...turn.events]+ .reverse()+ .find((event) => event.type === "authorization.required");+ if (+ required?.type !== "authorization.required" ||+ required.data.attemptId === undefined ||+ required.data.authorization?.url === undefined+ ) {+ throw new Error("Expected an authorization with an attempt id and a callback URL.");+ }+ return { attemptId: required.data.attemptId, url: new URL(required.data.authorization.url) };+}++/** Completes a fixture authorization the way a browser would: an unauthenticated GET. */+export async function completeAuthorization(url: URL) {+ const callback = await fetch(url);+ if (!callback.ok) throw new Error(`Fixture authorization callback failed (${callback.status}).`);+}++/** Everything the session streams from `startIndex` to the next turn boundary. */+export function follow(t: EveEvalContext, session: EveEvalSession, startIndex: number) {+ return t.target.watchTurn(session.sessionId, { startIndex }).result();+}++/**+ * Alice spends the session's output budget on a summary, then asks for a+ * status note: the turn stops before its model call and asks whether to go on.+ */+export async function budgetQuestion(t: EveEvalContext) {+ const session = await aliceSession(t);+ (await session.send(SAY.spendBudget, asAlice)).expectOk();+ const held = await session.send(SAY.statusNote, asAlice);+ const request = approvalFor(held, "session_limit_continuation");+ return { held, request, session };+}++/** The turn is held on input: it says so, and neither completes nor fails. */+export function expectHeld(turn: EveEvalTurn) {+ turn.event("turn.waiting", { data: { on: "input" } });+ turn.notEvent("turn.completed");+ turn.notEvent("turn.failed");+}++/** `request` resolved exactly once in `on`, with `outcome`. */+export function expectResolved(on: EveEvalAssertions, request: InputRequest, outcome: string) {+ on.event("input.resolved", {+ count: 1,+ data: {+ resolutions: (items) =>+ items.some((item) => item.requestId === request.requestId && item.outcome === outcome),+ },+ });+}++/** `toolName`'s call never ran: it has a rejected not-run result and no completed one. */+export function expectNotRun(on: EveEvalAssertions, toolName: string) {+ on.event("action.result", {+ data: { status: "rejected", result: { toolName } },+ });+ on.notEvent("action.result", { data: { status: "completed", result: { toolName } } });+}diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/allowed-responder-settles.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/allowed-responder-settles.eval.tsnew file mode 100644index 000000000..f96e140d5--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/allowed-responder-settles.eval.ts@@ -0,0 +1,54 @@+import { defineEval } from "eve/evals";++import {+ RELEASE_MANAGER,+ SAY,+ aliceSession,+ answers,+ approvalFor,+ asAlice,+ asReleaseManager,+ expectResolved,+} from "../helpers.ts";++/**+ * The release manager approves Alice's authorized change. The policy allows+ * the candidate, which settles the approval as the release manager's; the+ * change then runs once.+ */+export default defineEval({+ description: "An answer the response policy allows settles the approval and runs the call.",+ tags: ["hitl", "authorization"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await aliceSession(t);+ const request = approvalFor(await session.send(SAY.authorized, asAlice), "authorized-change");++ const settled = (+ await session.respond(answers("approve", request), asReleaseManager)+ ).expectOk();+ settled.eventOrder([+ { type: "approval.candidate", data: { outcome: "pending", requestId: request.requestId } },+ {+ type: "approval.settled",+ data: {+ outcome: "approved",+ requestId: request.requestId,+ responderPrincipalId: RELEASE_MANAGER,+ },+ },+ { type: "input.resolved" },+ {+ type: "action.result",+ data: { status: "completed", result: { toolName: "authorized-change" } },+ },+ { type: "turn.completed" },+ ]);+ expectResolved(settled, request, "approved");+ settled.calledTool("authorized-change", {+ status: "completed",+ output: { executions: 1 },+ count: 1,+ });+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/rejected-responder-leaves-approval-open.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/rejected-responder-leaves-approval-open.eval.tsnew file mode 100644index 000000000..fd36bea15--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/rejected-responder-leaves-approval-open.eval.ts@@ -0,0 +1,42 @@+import { defineEval } from "eve/evals";++import {+ SAY,+ aliceSession,+ answers,+ approvalFor,+ asAlice,+ asBob,+ expectHeld,+ expectNoModelCallWhileOpen,+} from "../helpers.ts";++/**+ * Only the release manager may approve the authorized change. Bob approves+ * it anyway: his answer becomes a candidate the policy rejects, and the+ * approval stays open with the turn held.+ */+export default defineEval({+ description: "A response policy rejects an unauthorized answer and the approval stays open.",+ tags: ["hitl", "authorization"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await aliceSession(t);+ const request = approvalFor(await session.send(SAY.authorized, asAlice), "authorized-change");++ const refused = await session.respond(answers("approve", request), asBob);+ expectHeld(refused);+ refused.eventOrder([+ { type: "approval.candidate", data: { outcome: "pending", responderPrincipalId: "bob" } },+ {+ type: "approval.candidate",+ data: { outcome: "rejected", reason: "Wrong responder.", responderPrincipalId: "bob" },+ },+ { type: "turn.waiting", data: { on: "input" } },+ ]);+ refused.notEvent("approval.settled");+ refused.notEvent("input.resolved");+ refused.notEvent("action.result");+ expectNoModelCallWhileOpen(session, request.requestId);+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/responder-authorization-then-settles.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/responder-authorization-then-settles.eval.tsnew file mode 100644index 000000000..597b6a9f7--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/responder-authorization-then-settles.eval.ts@@ -0,0 +1,70 @@+import { defineEval } from "eve/evals";++import {+ SAY,+ aliceSession,+ answers,+ approvalFor,+ as,+ asAlice,+ completeAuthorization,+ expectHeld,+ expectNoModelCallDuringAuthorization,+ expectResolved,+ follow,+ authorizationFrom,+} from "../helpers.ts";++// A responder of its own: the fixture's authorizations outlive an eval, and other+// evals sign the default release manager in.+const RESPONDER = "hitl-authorization-responder";++/**+ * A responder approves the OAuth-checked change. Its policy needs the+ * approver authorized first, so the turn holds on the responder's authorization;+ * the callback runs the policy again, which settles the approval as the+ * responder and runs the change.+ */+export default defineEval({+ description:+ "A response policy that needs the responder to authorize settles after the authorization.",+ tags: ["hitl", "authorization", "authorization"],+ timeoutMs: 90_000,+ async test(t) {+ const session = await aliceSession(t);+ const request = approvalFor(+ await session.send(SAY.oauthChecked, asAlice),+ "oauth-authorized-gate",+ );++ const held = await session.respond(answers("approve", request), as(RESPONDER));+ expectHeld(held);+ held.event("approval.candidate", { count: 1, data: { outcome: "pending" } });+ held.event("authorization.required", { count: 1, data: { principalId: RESPONDER } });+ held.notEvent("approval.settled");+ const authorization = authorizationFrom(held);++ const startIndex = session.state.streamIndex;+ await completeAuthorization(authorization.url);+ const resumed = (await follow(t, session, startIndex)).expectOk();+ resumed.notEvent("turn.started");+ resumed.eventOrder([+ {+ type: "authorization.completed",+ data: { attemptId: authorization.attemptId, outcome: "authorized" },+ },+ {+ type: "approval.settled",+ data: { outcome: "approved", responderPrincipalId: RESPONDER },+ },+ {+ type: "action.result",+ data: { status: "completed", result: { toolName: "oauth-authorized-gate" } },+ },+ { type: "message.completed", data: { message: /^OAuth-checked change: done / } },+ { type: "turn.completed" },+ ]);+ expectResolved(resumed, request, "approved");+ expectNoModelCallDuringAuthorization(resumed, authorization.attemptId, held.events);+ },+});diff --git a/e2e/fixtures/agent-workflow-tools/agent/agent.ts b/e2e/fixtures/agent-workflow-tools/agent/agent.tsindex 19604b735..6bc98220b 100644--- a/e2e/fixtures/agent-workflow-tools/agent/agent.ts+++ b/e2e/fixtures/agent-workflow-tools/agent/agent.ts@@ -147,6 +147,7 @@ async function respond(request: MockModelRequest): Promise<MockModelResponse | s["WORKFLOW-FANOUT-START", "fanout_deploy"],["WORKFLOW-WEBHOOK-START", "webhook_deploy"],["WORKFLOW-AGENT-FANOUT-START", "fanout_agents"],+ ["WORKFLOW-LAPSE-START", "lapsing_offer"],] as const) {if (!message.includes(directive)) continue;const result = [...request.toolResults].reverse().find((entry) => entry.name === tool);diff --git a/e2e/fixtures/agent-workflow-tools/agent/tools/lapsing_offer.ts b/e2e/fixtures/agent-workflow-tools/agent/tools/lapsing_offer.tsnew file mode 100644index 000000000..760fe2e0e--- /dev/null+++ b/e2e/fixtures/agent-workflow-tools/agent/tools/lapsing_offer.ts@@ -0,0 +1,31 @@+import { defineWorkflowTool } from "eve/tools";+import { sleep } from "workflow";+import { z } from "zod";++import { describePlan } from "../lib/plan.ts";++/**+ * Offers a deploy for a few seconds, then returns without withdrawing its+ * question. The run ends with the question still open, so the session must+ * withdraw it: nobody can answer a run that has ended.+ */+export default defineWorkflowTool({+ description: "Offer to deploy a service for a few seconds, then let the offer lapse.",+ inputSchema: z.strictObject({ service: z.string() }),+ async execute({ service }, ctx) {+ "use workflow";++ const answer = await Promise.race([+ ctx.ask({+ display: "confirmation",+ options: [+ { id: "approve", label: "Deploy now", style: "primary" },+ { id: "cancel", label: "Not now" },+ ],+ prompt: `Apply ${describePlan(service)} in the next few seconds?`,+ }),+ sleep("3s"),+ ]);+ return { offer: answer === undefined ? "lapsed" : "answered", service };+ },+});diff --git a/e2e/fixtures/agent-workflow-tools/evals/hitl/cancel-withdraws-relayed.eval.ts b/e2e/fixtures/agent-workflow-tools/evals/hitl/cancel-withdraws-relayed.eval.tsnew file mode 100644index 000000000..74e299805--- /dev/null+++ b/e2e/fixtures/agent-workflow-tools/evals/hitl/cancel-withdraws-relayed.eval.ts@@ -0,0 +1,27 @@+import { defineEval } from "eve/evals";++/**+ * Alice cancels her turn while the deploy's question waits for her. The+ * cancel stops the run, so nobody can answer what it relayed: the session+ * withdraws the question as cancelled.+ */+export default defineEval({+ description: "Cancelling the turn withdraws the questions relayed through it.",+ tags: ["hitl", "relayed", "cancel"],+ async test(t) {+ const parked = await t.send("WORKFLOW-CONFIRM-START");+ const request = parked.session.requireInputRequest({ toolName: "confirm_deploy" });+ const startIndex = parked.session.state.streamIndex;++ await parked.session.cancel();+ const cancelled = await t.target.watchTurn(parked.sessionId, { startIndex }).result();+ cancelled.event("input.resolved", {+ count: 1,+ data: { resolutions: [{ outcome: "cancelled", requestId: request.requestId }] },+ });+ cancelled.event("turn.cancelled", { count: 1 });+ cancelled.notEvent("action.result", {+ data: { status: "completed", result: { toolName: "confirm_deploy" } },+ });+ },+});diff --git a/e2e/fixtures/agent-workflow-tools/evals/hitl/run-end-withdraws-relayed.eval.ts b/e2e/fixtures/agent-workflow-tools/evals/hitl/run-end-withdraws-relayed.eval.tsnew file mode 100644index 000000000..33eff20b2--- /dev/null+++ b/e2e/fixtures/agent-workflow-tools/evals/hitl/run-end-withdraws-relayed.eval.ts@@ -0,0 +1,29 @@+import { defineEval } from "eve/evals";++/**+ * Alice's quick-deploy offer asks a question through her session and lapses+ * after a few seconds without withdrawing it. Once the run ends nobody can+ * answer it, so the session withdraws it as cancelled before the call's result.+ */+export default defineEval({+ description: "A workflow run that ends withdraws the question it relayed.",+ tags: ["hitl", "relayed"],+ async test(t) {+ const parked = await t.send("WORKFLOW-LAPSE-START Alice offers a quick deploy of the api.");+ const request = parked.session.requireInputRequest({ toolName: "lapsing_offer" });+ const startIndex = parked.session.state.streamIndex;++ const lapsed = (await t.target.watchTurn(parked.sessionId, { startIndex }).result()).expectOk();+ lapsed.event("input.resolved", {+ count: 1,+ data: { resolutions: [{ outcome: "cancelled", requestId: request.requestId }] },+ });+ lapsed.eventOrder([+ { type: "input.resolved" },+ { type: "action.result", data: { result: { toolName: "lapsing_offer" } } },+ { type: "turn.completed" },+ ]);+ lapsed.calledTool("lapsing_offer", { output: { offer: "lapsed" }, count: 1 });+ lapsed.messageIncludes('WORKFLOW-LAPSE-RESULT {"offer":"lapsed","service":"api"}');+ },+});