Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

test(e2e): cover the human-input rules main already meets (#4492)

Rui88e5b57cc88d
diff --git a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts b/e2e/fixtures/agent-subagents-hitl/agent/agent.ts
index fa81db70f..f7f7c4728 100644
--- a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts
+++ b/e2e/fixtures/agent-subagents-hitl/agent/agent.ts
@@ -35,6 +35,14 @@ function respond(request: MockModelRequest): MockModelResponse | string {
],
};
}
+ // RELAY-<child>: delegate to that mock child once, wait for it, and report its result.
+ const relay = /RELAY-(question-child|budget-child)/u.exec(message)?.[1];
+ if (relay !== undefined) {
+ const result = taskResultOf(request, relay);
+ if (result !== undefined) return `RELAY-RESULT ${result}`;
+ if (hasReceipt(request, relay)) return waitForTasks();
+ return { toolCalls: [{ input: { message: "Handle Alice's release request." }, name: relay }] };
+ }
if (message.includes("Call the stock-price subagent exactly once")) {
const quote = taskResultOf(request, "stock-price");
if (quote !== undefined) return `The stock-price subagent returned: ${quote}`;
diff --git a/e2e/fixtures/agent-subagents-hitl/agent/subagents/budget-child/agent.ts b/e2e/fixtures/agent-subagents-hitl/agent/subagents/budget-child/agent.ts
new file mode 100644
index 000000000..490bc1754
--- /dev/null
+++ b/e2e/fixtures/agent-subagents-hitl/agent/subagents/budget-child/agent.ts
@@ -0,0 +1,17 @@
+import { defineAgent } from "eve";
+import { mockModel } from "eve/evals";
+
+/**
+ * Records one release step on a one-token input budget, so its second model
+ * call stops at its budget question, which its parent relays.
+ */
+export default defineAgent({
+ description: "Record the release step, then confirm it.",
+ limits: { maxInputTokensPerSession: 1 },
+ model: mockModel(({ toolResults }) =>
+ toolResults.some((entry) => entry.name === "record_step")
+ ? "BUDGET-CHILD-RESULT recorded"
+ : { toolCalls: [{ input: {}, name: "record_step" }] },
+ ),
+ modelContextWindowTokens: 1_000_000,
+});
diff --git a/e2e/fixtures/agent-subagents-hitl/agent/subagents/budget-child/tools/record_step.ts b/e2e/fixtures/agent-subagents-hitl/agent/subagents/budget-child/tools/record_step.ts
new file mode 100644
index 000000000..1a20622c5
--- /dev/null
+++ b/e2e/fixtures/agent-subagents-hitl/agent/subagents/budget-child/tools/record_step.ts
@@ -0,0 +1,8 @@
+import { defineTool } from "eve/tools";
+import { z } from "zod";
+
+export default defineTool({
+ description: "Record the release step.",
+ inputSchema: z.strictObject({}),
+ execute: () => ({ recorded: true }),
+});
diff --git a/e2e/fixtures/agent-subagents-hitl/agent/subagents/question-child/agent.ts b/e2e/fixtures/agent-subagents-hitl/agent/subagents/question-child/agent.ts
new file mode 100644
index 000000000..270938b77
--- /dev/null
+++ b/e2e/fixtures/agent-subagents-hitl/agent/subagents/question-child/agent.ts
@@ -0,0 +1,26 @@
+import { defineAgent } from "eve";
+import { mockModel } from "eve/evals";
+
+/** Asks which environment a release goes to, then reports the answer it got. */
+export default defineAgent({
+ description: "Ask which environment the release should go to, then report the choice.",
+ model: mockModel(({ toolResults }) => {
+ const answer = toolResults.find((entry) => entry.name === "ask_question");
+ if (answer !== undefined) return `QUESTION-CHILD-RESULT ${JSON.stringify(answer.output)}`;
+ return {
+ toolCalls: [
+ {
+ input: {
+ options: [
+ { description: "Release to staging first.", label: "Staging" },
+ { description: "Release straight to production.", label: "Production" },
+ ],
+ question: "Which environment should the release go to?",
+ },
+ name: "ask_question",
+ },
+ ],
+ };
+ }),
+ modelContextWindowTokens: 1_000_000,
+});
diff --git a/e2e/fixtures/agent-subagents-hitl/agent/subagents/question-child/tools/ask_question.ts b/e2e/fixtures/agent-subagents-hitl/agent/subagents/question-child/tools/ask_question.ts
new file mode 100644
index 000000000..533cc8526
--- /dev/null
+++ b/e2e/fixtures/agent-subagents-hitl/agent/subagents/question-child/tools/ask_question.ts
@@ -0,0 +1,3 @@
+import { askQuestion } from "eve/tools/ask_question";
+
+export default askQuestion();
diff --git a/e2e/fixtures/agent-subagents-hitl/evals/hitl/cancel-withdraws-relayed-question.eval.ts b/e2e/fixtures/agent-subagents-hitl/evals/hitl/cancel-withdraws-relayed-question.eval.ts
new file mode 100644
index 000000000..6365a3062
--- /dev/null
+++ b/e2e/fixtures/agent-subagents-hitl/evals/hitl/cancel-withdraws-relayed-question.eval.ts
@@ -0,0 +1,27 @@
+import { defineEval } from "eve/evals";
+
+import { next, waitForRelayed } from "./helpers";
+
+/**
+ * Alice cancels her turn while the question child waits on her answer. The
+ * cancel stops the child, so nobody can answer its question: the parent
+ * withdraws it as cancelled.
+ */
+export default defineEval({
+ description: "Cancelling the parent's turn withdraws a child's relayed question.",
+ tags: ["hitl", "relayed", "cancel"],
+ timeoutMs: 90_000,
+ async test(t) {
+ const started = await t.send("RELAY-question-child Alice asks where the release should go.");
+ const { request, session } = await waitForRelayed(t, started.session, "ask_question");
+
+ const following = next(t, session);
+ await session.cancel();
+ const cancelled = await following;
+ cancelled.event("input.resolved", {
+ count: 1,
+ data: { resolutions: [{ outcome: "cancelled", requestId: request.requestId }] },
+ });
+ cancelled.event("turn.cancelled", { count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-subagents-hitl/evals/hitl/helpers.ts b/e2e/fixtures/agent-subagents-hitl/evals/hitl/helpers.ts
new file mode 100644
index 000000000..ca4a4f87c
--- /dev/null
+++ b/e2e/fixtures/agent-subagents-hitl/evals/hitl/helpers.ts
@@ -0,0 +1,42 @@
+import type { EveEvalContext, EveEvalSession, EveEvalTurn, InputRequest } from "eve/evals";
+
+/**
+ * Follows the parent session, turn by turn, until a child's request matching
+ * `toolName` is pending on it. A child asks after its own model call, so the
+ * request may land a turn or two after the delegation.
+ */
+export async function waitForRelayed(
+ t: EveEvalContext,
+ initial: EveEvalSession,
+ toolName: string,
+): Promise<{ readonly request: InputRequest; readonly session: EveEvalSession }> {
+ let session = initial;
+ for (let attempt = 0; attempt < 5; attempt += 1) {
+ const request = session.pendingInputRequests.find(
+ (pending) => pending.action.toolName === toolName,
+ );
+ if (request !== undefined) return { request, session };
+ session = (await next(t, session)).session;
+ }
+ throw new Error(`No relayed ${toolName} request reached the parent after five turns.`);
+}
+
+/** Follows the parent until a turn's reply contains `marker`. */
+export async function waitForReply(
+ t: EveEvalContext,
+ initial: EveEvalSession,
+ marker: string,
+): Promise<EveEvalTurn> {
+ let session = initial;
+ for (let attempt = 0; attempt < 5; attempt += 1) {
+ const turn = await next(t, session);
+ if (turn.message?.includes(marker) === true) return turn;
+ session = turn.session;
+ }
+ throw new Error(`No parent reply contained ${marker} after five turns.`);
+}
+
+/** The parent's events from its current cursor to the next turn boundary. */
+export function next(t: EveEvalContext, session: EveEvalSession): Promise<EveEvalTurn> {
+ return t.target.watchTurn(session.sessionId, { startIndex: session.state.streamIndex }).result();
+}
diff --git a/e2e/fixtures/agent-subagents-hitl/evals/hitl/relayed-budget-stop-cancels-parent.eval.ts b/e2e/fixtures/agent-subagents-hitl/evals/hitl/relayed-budget-stop-cancels-parent.eval.ts
new file mode 100644
index 000000000..f7a73a635
--- /dev/null
+++ b/e2e/fixtures/agent-subagents-hitl/evals/hitl/relayed-budget-stop-cancels-parent.eval.ts
@@ -0,0 +1,38 @@
+import { defineEval } from "eve/evals";
+
+import { waitForRelayed } from "./helpers";
+
+/**
+ * The budget child runs out of budget mid-task and its question reaches
+ * Alice through her session. She chooses Stop: the answer goes to the child,
+ * and because Stop means stop, her own turn is cancelled too.
+ */
+export default defineEval({
+ description: "Stop on a child's relayed budget question also cancels the parent's turn.",
+ tags: ["hitl", "relayed", "budget"],
+ timeoutMs: 90_000,
+ async test(t) {
+ const started = await t.send("RELAY-budget-child Alice records the release step.");
+ const { request, session } = await waitForRelayed(
+ t,
+ started.session,
+ "session_limit_continuation",
+ );
+ if (request.requestId.startsWith(`${session.sessionId}:`)) {
+ throw new Error("Expected the child's budget question, not the parent's.");
+ }
+
+ const stopped = await session.respond([{ optionId: "stop", requestId: request.requestId }]);
+ stopped.event("input.resolved", {
+ count: 1,
+ data: {
+ resolutions: [
+ { outcome: "answered", requestId: request.requestId, response: { optionId: "stop" } },
+ ],
+ },
+ });
+ stopped.event("turn.cancelled", { count: 1 });
+ stopped.notEvent("turn.failed");
+ stopped.notEvent("message.completed", { data: { message: /RELAY-RESULT/u } });
+ },
+});
diff --git a/e2e/fixtures/agent-subagents-hitl/evals/hitl/relayed-question-answered.eval.ts b/e2e/fixtures/agent-subagents-hitl/evals/hitl/relayed-question-answered.eval.ts
new file mode 100644
index 000000000..9887fa86c
--- /dev/null
+++ b/e2e/fixtures/agent-subagents-hitl/evals/hitl/relayed-question-answered.eval.ts
@@ -0,0 +1,41 @@
+import { defineEval } from "eve/evals";
+
+import { waitForRelayed, waitForReply } from "./helpers";
+
+/**
+ * The question child asks Alice, through her session, where the release
+ * goes. Her answer on the parent is forwarded to the child that asked, which
+ * resolves its question with her choice and reports back.
+ */
+export default defineEval({
+ description: "A child's question is relayed through the parent and answered there.",
+ tags: ["hitl", "relayed"],
+ timeoutMs: 90_000,
+ async test(t) {
+ const started = await t.send("RELAY-question-child Alice asks where the release should go.");
+ const { request, session } = await waitForRelayed(t, started.session, "ask_question");
+ const production = request.options?.find((option) => option.label === "Production");
+ if (production === undefined) throw new Error("Expected a Production option.");
+
+ const answered = await session.respond([
+ { optionId: production.id, requestId: request.requestId },
+ ]);
+ answered.event("input.resolved", {
+ count: 1,
+ data: {
+ resolutions: [
+ {
+ outcome: "answered",
+ requestId: request.requestId,
+ response: { optionId: production.id },
+ },
+ ],
+ },
+ });
+ const reply = answered.message?.includes("RELAY-RESULT")
+ ? answered
+ : await waitForReply(t, answered.session, "RELAY-RESULT");
+ reply.messageIncludes(/RELAY-RESULT QUESTION-CHILD-RESULT .*Production/u);
+ t.calledSubagent("question-child", { status: "completed", count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/agent/agent.ts b/e2e/fixtures/agent-tools-hitl/agent/agent.ts
index 3bcbe6a8f..22effa3fe 100644
--- a/e2e/fixtures/agent-tools-hitl/agent/agent.ts
+++ b/e2e/fixtures/agent-tools-hitl/agent/agent.ts
@@ -3,6 +3,7 @@ import { defineAgent, defineDynamic } from "eve";
import type { MockModelRequest, MockModelResponse } from "eve/evals";
import { continuationModel } from "./lib/continuation/model.ts";
+import { humanInputModel } from "./lib/hitl/model.ts";
const AUTH_PROBE_DIRECTIVE = /call the auth-probe tool exactly once with marker "([^"]+)"/iu;
const SCOPED_APPROVAL_DIRECTIVE =
@@ -105,10 +106,16 @@ export default defineAgent({
model: typeof base.model === "string" ? base.model : "openai/gpt-6.1-sol",
modelContextWindowTokens: base.modelContextWindowTokens,
}),
- "step.started": (_event, ctx) =>
- ctx.session.auth.initiator?.attributes?.model === "continuation"
- ? { model: continuationModel(), modelContextWindowTokens: 1_000_000 }
- : { model: base.model, modelContextWindowTokens: base.modelContextWindowTokens },
+ "step.started": (_event, ctx) => {
+ switch (ctx.session.auth.initiator?.attributes?.model) {
+ case "continuation":
+ return { model: continuationModel(), modelContextWindowTokens: 1_000_000 };
+ case "hitl":
+ return { model: humanInputModel(), modelContextWindowTokens: 1_000_000 };
+ default:
+ return { model: base.model, modelContextWindowTokens: base.modelContextWindowTokens };
+ }
+ },
},
}),
});
diff --git a/e2e/fixtures/agent-tools-hitl/agent/channels/eve.ts b/e2e/fixtures/agent-tools-hitl/agent/channels/eve.ts
index a95240c49..3846ff159 100644
--- a/e2e/fixtures/agent-tools-hitl/agent/channels/eve.ts
+++ b/e2e/fixtures/agent-tools-hitl/agent/channels/eve.ts
@@ -7,6 +7,8 @@ export default eveChannel({
return {
attributes: {
fixture: "authorized-response",
+ // Lets a hitl eval change what the current caller's policies and tools see.
+ flag: request.headers.get("x-eve-fixture-flag") ?? "none",
model: request.headers.get("x-eve-fixture-model") ?? "default",
},
authenticator: "e2e-fixture",
diff --git a/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/model.ts b/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/model.ts
new file mode 100644
index 000000000..8e50052fb
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/model.ts
@@ -0,0 +1,38 @@
+import { mockModel } from "eve/evals";
+
+import { respond } from "./respond.ts";
+
+type Prompt = Parameters<Exclude<ReturnType<typeof mockModel>, string>["doGenerate"]>[0]["prompt"];
+
+/**
+ * The hitl suite's scripted model. eve runs approved calls itself, so
+ * no prompt may carry an AI SDK approval part; one fails the turn.
+ */
+export function humanInputModel() {
+ const model = mockModel({ modelId: "hitl", respond });
+ if (typeof model === "string" || model.specificationVersion !== "v4") {
+ throw new Error("This fixture expects mockModel's v4 provider boundary.");
+ }
+ const generate = model.doGenerate.bind(model);
+ const stream = model.doStream.bind(model);
+ model.doGenerate = async (options) => {
+ rejectApprovalParts(options.prompt);
+ return generate(options);
+ };
+ model.doStream = async (options) => {
+ rejectApprovalParts(options.prompt);
+ return stream(options);
+ };
+ return model;
+}
+
+function rejectApprovalParts(prompt: Prompt) {
+ for (const message of prompt) {
+ if (typeof message.content === "string") continue;
+ for (const part of message.content) {
+ if (part.type.includes("approval")) {
+ throw new Error(`Model history carries an AI SDK approval part: ${part.type}.`);
+ }
+ }
+ }
+}
diff --git a/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/respond.ts b/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/respond.ts
new file mode 100644
index 000000000..6baca07fc
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/respond.ts
@@ -0,0 +1,124 @@
+import type { MockModelRequest, MockModelResponse, MockModelToolCall } from "eve/evals";
+
+/** What people say in the hitl evals. Each sentence selects one script below. */
+export const SAY = {
+ changeA: "Alice asks for change A.",
+ changesAB: "Alice asks for changes A and B together.",
+ changeAAndAuthorized: "Alice asks for change A and an authorized change together.",
+ authorized: "Alice asks for an authorized change.",
+ oauthChecked: "Alice asks for an OAuth-checked change.",
+ firstEcho: "Alice asks for a guarded echo of the first note.",
+ secondEcho: "Alice asks for a guarded echo of the second note.",
+ retiring: "Alice asks for the retiring change.",
+ checkAccess: "Alice asks to check her Fixture Auth access.",
+ publish: "Alice asks to publish the draft.",
+ hello: "Alice changes her mind and asks for a short hello instead.",
+ bobStatus: "Bob asks for a status update.",
+ spendBudget: "Alice spends the remaining budget on a summary.",
+ statusNote: "Alice asks for a status note.",
+ deadline: "Alice adds that the note should mention the deadline.",
+} as const;
+
+export const REPLY = {
+ hello: "Hello, Alice.",
+ helloAfterAuthorization: "Hello, Alice. I dropped the authorization you no longer need.",
+ bobStatus: "Bob's status update is ready.",
+ summary: "Summary written.",
+ statusNote: "Status note ready.",
+ statusNoteWithDeadline: "Status note ready. It mentions the deadline.",
+ staleAnswer: "Noted your earlier answer; it authorized nothing.",
+} as const;
+
+const STALE_ANSWER_PREFIX =
+ "The user submitted the following response to an earlier interactive prompt.";
+const SIGN_IN_DROPPED = "was cancelled because the user sent a new message instead";
+
+const call = (id: string, name: string, input: object = {}): MockModelToolCall => ({
+ id,
+ input,
+ name,
+});
+
+/**
+ * Scripted model for `evals/hitl`. The newest message with a script
+ * drives the step; tool results, not a counter, select the next response, so
+ * durable replay follows the same script.
+ */
+export function respond(request: MockModelRequest): MockModelResponse {
+ const result = (id: string) => request.toolResults.find((entry) => entry.id === id);
+ const outcome = (id: string): string => {
+ const found = result(id);
+ if (found === undefined) return "missing";
+ return found.isError ? "not run" : `done ${JSON.stringify(found.output)}`;
+ };
+ const run = (calls: MockModelToolCall[], answer: () => string): MockModelResponse => {
+ const missing = calls.filter((tool) => result(tool.id!) === undefined);
+ return missing.length > 0 ? { toolCalls: missing } : { text: answer() };
+ };
+ const allText = request.messages.map((message) => message.text).join("\n");
+
+ const scripts: Record<string, () => MockModelResponse> = {
+ [SAY.changeA]: () => run([call("a", "change-a")], () => `Change A: ${outcome("a")}.`),
+ [SAY.changesAB]: () =>
+ run(
+ [call("a", "change-a"), call("b", "change-b")],
+ () => `Change A: ${outcome("a")}. Change B: ${outcome("b")}.`,
+ ),
+ [SAY.changeAAndAuthorized]: () =>
+ run(
+ [call("a", "change-a"), call("auth", "authorized-change")],
+ () => `Change A: ${outcome("a")}. Authorized change: ${outcome("auth")}.`,
+ ),
+ [SAY.authorized]: () =>
+ run([call("auth", "authorized-change")], () => `Authorized change: ${outcome("auth")}.`),
+ [SAY.oauthChecked]: () =>
+ run(
+ [call("oauth", "oauth-authorized-gate", { marker: "hitl" })],
+ () => `OAuth-checked change: ${outcome("oauth")}.`,
+ ),
+ [SAY.firstEcho]: () =>
+ run(
+ [call("echo-first", "guarded-echo", { note: "first" })],
+ () => `First echo: ${outcome("echo-first")}.`,
+ ),
+ [SAY.secondEcho]: () =>
+ run(
+ [call("echo-second", "guarded-echo", { note: "second" })],
+ () => `Second echo: ${outcome("echo-second")}.`,
+ ),
+ [SAY.retiring]: () =>
+ run([call("retiring", "retiring-change")], () => `Retiring change: ${outcome("retiring")}.`),
+ [SAY.checkAccess]: () =>
+ run(
+ [call("probe", "auth-probe", { marker: "hitl" })],
+ () => `Access check: ${outcome("probe")}.`,
+ ),
+ [SAY.publish]: () =>
+ run([call("publish", "publish-draft")], () => `Publish: ${outcome("publish")}.`),
+ [SAY.hello]: () => ({
+ text: allText.includes(SIGN_IN_DROPPED) ? REPLY.helloAfterAuthorization : REPLY.hello,
+ }),
+ [SAY.bobStatus]: () => ({ text: REPLY.bobStatus }),
+ [SAY.spendBudget]: () => ({
+ text: REPLY.summary,
+ usage: { inputTokens: 1, outputTokens: 1_000_000 },
+ }),
+ [SAY.statusNote]: () => ({
+ text: request.userMessages.includes(SAY.deadline)
+ ? REPLY.statusNoteWithDeadline
+ : REPLY.statusNote,
+ }),
+ };
+ scripts[SAY.deadline] = scripts[SAY.statusNote]!;
+
+ for (const message of [...request.userMessages].reverse()) {
+ if (message.startsWith(STALE_ANSWER_PREFIX)) return reply({ text: REPLY.staleAnswer });
+ const script = scripts[message];
+ if (script !== undefined) return reply(script());
+ }
+ throw new Error(`No hitl script for: ${JSON.stringify(request.userMessages)}`);
+}
+
+function reply(response: MockModelResponse): MockModelResponse {
+ return { usage: { inputTokens: 1, outputTokens: 1 }, ...response };
+}
diff --git a/e2e/fixtures/agent-tools-hitl/agent/tools/publish-draft.ts b/e2e/fixtures/agent-tools-hitl/agent/tools/publish-draft.ts
new file mode 100644
index 000000000..5db494ff4
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/tools/publish-draft.ts
@@ -0,0 +1,47 @@
+import {
+ ConnectionAuthorizationRequiredError,
+ defineInteractiveAuthorization,
+} from "eve/connections";
+import { defineState } from "eve/context";
+import { defineTool } from "eve/tools";
+import { once } from "eve/tools/approval";
+import { z } from "zod";
+
+const publications = defineState("publish-draft.publications", () => 0);
+
+// Shares auth-probe's authorization (same `authKey`), so a publish asked while an
+// auth-probe authorization is open supersedes that attempt.
+const fixtureAuth = defineInteractiveAuthorization<{ marker: string }>({
+ async getToken() {
+ throw new ConnectionAuthorizationRequiredError("auth-probe");
+ },
+ async startAuthorization({ callbackUrl }) {
+ const url = new URL(callbackUrl);
+ url.searchParams.set("code", "fixture-ok");
+ return {
+ challenge: { displayName: "Fixture Auth", url: url.href },
+ resume: { marker: "fixture-resume" },
+ };
+ },
+ async completeAuthorization({ callback, resume }) {
+ if (callback.params.code !== "fixture-ok" || resume?.marker !== "fixture-resume") {
+ throw new Error("Fixture authorization state mismatch.");
+ }
+ return { token: "publish-token-Q2W7" };
+ },
+});
+
+/** Needs approval once, then an authorization each time it runs. */
+export default defineTool({
+ description: "Publish the fixture draft. Requires approval and a Fixture Auth sign-in.",
+ inputSchema: z.object({}),
+ approval: once(),
+ async execute(_input, ctx) {
+ await ctx.getToken(fixtureAuth, { authKey: "auth-probe", displayName: "Fixture Auth" });
+ publications.update((count) => count + 1);
+ return {
+ actor: ctx.session.auth.current?.principalId ?? "none",
+ publications: publications.get(),
+ };
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/agent/tools/retiring-change.ts b/e2e/fixtures/agent-tools-hitl/agent/tools/retiring-change.ts
new file mode 100644
index 000000000..e524570f8
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/tools/retiring-change.ts
@@ -0,0 +1,22 @@
+import { defineDynamic, defineTool } from "eve/tools";
+import { always } from "eve/tools/approval";
+import { z } from "zod";
+
+/**
+ * A step-scoped tool offered only while the caller's fixture flag is not
+ * "retire", for proving an approved call keeps the tools of the step that
+ * asked even when a later step would not offer this one.
+ */
+export default defineDynamic({
+ events: {
+ "step.started": (_event, ctx) =>
+ ctx.session.auth.current?.attributes?.flag === "retire"
+ ? null
+ : defineTool({
+ description: "Apply the retiring fixture change after approval.",
+ inputSchema: z.object({}),
+ approval: always(),
+ execute: () => ({ change: "retiring" }),
+ }),
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md b/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md
new file mode 100644
index 000000000..b88de5ea9
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md
@@ -0,0 +1,74 @@
+# Human input
+
+One eval per rule of eve's human input (`packages/eve/src/harness/hitl/`):
+tool approvals, response policies, the budget question, authorizations, and requests
+relayed from child sessions and workflow runs. Each eval drives a real session
+over HTTP and asserts on its event stream.
+
+Every eval here is deterministic in every world. Sessions select the scripted
+`hitl` model with the `x-eve-fixture-model` header
+([`agent/lib/hitl`](../../agent/lib/hitl)), so they need no
+`real-model` tag. That model also fails the turn if its history ever carries an
+AI SDK approval part. eve runs approved calls itself, so a passing run also shows
+that history has no such part.
+
+People are named by the fixture's `x-eve-fixture-user` header: Alice asks, Bob
+is someone else in the session, and `e2e-approval-responder` is the release
+manager whom the fixture's response policies allow. `x-eve-fixture-flag` lets a
+caller retire a step tool (`retiring-change`).
+
+Relayed rules live with the fixtures that own children and workflow tools:
+[`agent-subagents-hitl/evals/hitl`](../../../agent-subagents-hitl/evals/hitl)
+and [`agent-workflow-tools/evals/hitl`](../../../agent-workflow-tools/evals/hitl).
+
+## Coverage
+
+| Rule | Eval |
+| ------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
+| Approve runs the call in eve, result before the next step | [approve-runs-the-call](./approvals/approve-runs-the-call.eval.ts) |
+| Deny resolves `denied` and records a not-run result | [deny-records-not-run](./approvals/deny-records-not-run.eval.ts) |
+| A partial answer keeps the turn held, with no model call | [partial-answer-keeps-turn-held](./approvals/partial-answer-keeps-turn-held.eval.ts) |
+| A typed reply naming an option answers | [typed-approve-answers](./approvals/typed-approve-answers.eval.ts) |
+| A typed reply answers only what it matches | [typed-reply-answers-only-what-matches](./approvals/typed-reply-answers-only-what-matches.eval.ts) |
+| Steering ignores unanswered approvals and keeps given answers | [steer-ignores-unanswered](./approvals/steer-ignores-unanswered.eval.ts) |
+| Someone else's message waits for the held turn | [other-person-message-waits](./approvals/other-person-message-waits.eval.ts) |
+| An answer to a request that is not open authorizes nothing | [stale-answer-authorizes-nothing](./approvals/stale-answer-authorizes-nothing.eval.ts) |
+| A `once()` grant is reused | [once-grant-is-reused](./approvals/once-grant-is-reused.eval.ts) |
+| An approved call runs with the asking step's tools | [approved-call-uses-asking-step-tools](./approvals/approved-call-uses-asking-step-tools.eval.ts) |
+| A rejected responder leaves the approval open | [rejected-responder-leaves-approval-open](./response-policies/rejected-responder-leaves-approval-open.eval.ts) |
+| An allowed responder settles the approval | [allowed-responder-settles](./response-policies/allowed-responder-settles.eval.ts) |
+| A responder's authorization, then settlement | [responder-authorization-then-settles](./response-policies/responder-authorization-then-settles.eval.ts) |
+| Stop cancels the turn, resolving once | [stop-cancels-once](./budget/stop-cancels-once.eval.ts) |
+| No step starts behind the budget question | [no-step-behind-budget](./budget/no-step-behind-budget.eval.ts) |
+| A late budget answer is dropped | [stale-budget-answer-dropped](./budget/stale-budget-answer-dropped.eval.ts) |
+| An authorization holds the turn; the callback resumes it as the requester | [authorization-holds-then-resumes-as-requester](./authorizations/authorization-holds-then-resumes-as-requester.eval.ts) |
+| Steering declines an authorization and tells the model | [steer-declines-authorization](./authorizations/steer-declines-authorization.eval.ts) |
+| Cancel declines an authorization | [cancel-declines-authorization](./authorizations/cancel-declines-authorization.eval.ts) |
+| An approved call that needs an authorization | [approved-call-needs-authorization](./authorizations/approved-call-needs-authorization.eval.ts) |
+
+The rule that the model never runs while a request is open is checked inside
+the evals above by `expectNoModelCallWhileOpen` and
+`expectNoModelCallDuringAuthorization` ([helpers](./helpers.ts)).
+
+## Not covered here
+
+- **Candidate expiry.** A candidate times out after a 10-minute wall-clock
+ TTL, and e2e cannot advance the clock. Covered by
+ `harness/hitl/candidates.test.ts` ("expires only candidates whose deadline
+ has passed") and `harness/hitl/coordinator.test.ts` ("completes an expired
+ candidate without executing policy").
+- **A `once()` grant hidden while an approval for its key waits.** Policies
+ only run at a step's start or right before an approved call, and neither
+ happens while that approval waits, so HTTP cannot observe the hiding.
+ Covered by `harness/session-machine/machine.test.ts` ("grants a once()
+ approval's key, except to a call still asking for it").
+- **An approved tool removed before the answer.** Step tools persist with
+ their step, so only a redeploy between the ask and the answer removes one.
+ Local e2e cannot redeploy.
+- **A fresh relayed batch replacing an older one.** This needs a child or
+ remote agent to ask again from the same source while its earlier batch is
+ open. Each workflow `ctx.ask()` is its own source. Covered by
+ `harness/proxy-input-requests.test.ts` ("replaces prior entries for the same
+ child continuation token").
+- **Handoff blocked while a request is open.** This needs a deployment
+ handoff, which only the Vercel redeploy suite performs.
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/approve-runs-the-call.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/approve-runs-the-call.eval.ts
new file mode 100644
index 000000000..6650ac1a9
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/approve-runs-the-call.eval.ts
@@ -0,0 +1,42 @@
+import { defineEval } from "eve/evals";
+
+import {
+ SAY,
+ aliceSession,
+ answers,
+ approvalFor,
+ asAlice,
+ expectHeld,
+ expectNoModelCallWhileOpen,
+ expectResolved,
+} from "../helpers.ts";
+
+/**
+ * Alice asks for change A and approves it. eve runs the call itself, so its
+ * result lands before the model's next step, and the model's history never
+ * carries an AI SDK approval part (the scripted model fails the turn if it does).
+ */
+export default defineEval({
+ description: "Approving a call runs it in eve, before the model's next step.",
+ tags: ["hitl", "approval"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const held = await session.send(SAY.changeA, asAlice);
+ expectHeld(held);
+ const request = approvalFor(held, "change-a");
+
+ const approved = (await session.respond(answers("approve", request), asAlice)).expectOk();
+ expectResolved(approved, request, "approved");
+ approved.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });
+ approved.notEvent("turn.started");
+ approved.eventOrder([
+ { type: "input.resolved" },
+ { type: "step.started" },
+ { type: "action.result", data: { status: "completed", result: { toolName: "change-a" } } },
+ { type: "message.completed", data: { message: /^Change A: done / } },
+ { type: "turn.completed" },
+ ]);
+ expectNoModelCallWhileOpen(session, request.requestId);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/approved-call-uses-asking-step-tools.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/approved-call-uses-asking-step-tools.eval.ts
new file mode 100644
index 000000000..a0580b912
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/approved-call-uses-asking-step-tools.eval.ts
@@ -0,0 +1,34 @@
+import { defineEval } from "eve/evals";
+
+import { ALICE, SAY, aliceSession, answers, approvalFor, as, asAlice } from "../helpers.ts";
+
+/**
+ * The retiring change is a step-scoped dynamic tool that the fixture stops
+ * offering for callers with the "retire" flag. Alice approves it with that
+ * flag on: the approved call still runs, because eve runs it with the tools
+ * of the step that asked, as its approval saw them.
+ *
+ * The opposite case — the tool is gone from the deployment by the time the
+ * answer arrives, which fails with "The approved tool is no longer
+ * available" — needs a redeploy between the ask and the answer, so local
+ * e2e cannot reach it.
+ */
+export default defineEval({
+ description: "An approved call runs with the tools of the step that asked for it.",
+ tags: ["hitl", "approval", "dynamic-tools"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const request = approvalFor(await session.send(SAY.retiring, asAlice), "retiring-change");
+
+ const approved = (
+ await session.respond(answers("approve", request), as(ALICE, "retire"))
+ ).expectOk();
+ approved.calledTool("retiring-change", {
+ status: "completed",
+ output: { change: "retiring" },
+ count: 1,
+ });
+ approved.event("message.completed", { data: { message: /^Retiring change: done / } });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/deny-records-not-run.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/deny-records-not-run.eval.ts
new file mode 100644
index 000000000..6d4224cad
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/deny-records-not-run.eval.ts
@@ -0,0 +1,38 @@
+import { defineEval } from "eve/evals";
+
+import {
+ SAY,
+ aliceSession,
+ answers,
+ approvalFor,
+ asAlice,
+ expectNotRun,
+ expectResolved,
+} from "../helpers.ts";
+
+/** Alice asks for change A, then declines it: the call never runs and the model hears so. */
+export default defineEval({
+ description: "Denying a call resolves it as denied and records a not-run result.",
+ tags: ["hitl", "approval"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const request = approvalFor(await session.send(SAY.changeA, asAlice), "change-a");
+
+ const denied = (await session.respond(answers("cancel", request), asAlice)).expectOk();
+ expectResolved(denied, request, "denied");
+ expectNotRun(denied, "change-a");
+ denied.event("action.result", {
+ count: 1,
+ data: {
+ status: "rejected",
+ result: {
+ output: { approval: { status: "denied" }, code: "TOOL_EXECUTION_DENIED" },
+ toolName: "change-a",
+ },
+ },
+ });
+ denied.event("message.completed", { data: { message: "Change A: not run." } });
+ denied.event("turn.completed", { count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/once-grant-is-reused.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/once-grant-is-reused.eval.ts
new file mode 100644
index 000000000..5719c590b
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/once-grant-is-reused.eval.ts
@@ -0,0 +1,29 @@
+import { defineEval } from "eve/evals";
+
+import { SAY, aliceSession, answers, approvalFor, asAlice } from "../helpers.ts";
+
+/**
+ * `guarded-echo` uses `once()`. Alice approves the first echo; the grant lasts
+ * the session, so her second echo runs without asking again.
+ */
+export default defineEval({
+ description: "A once() approval grants its key for the rest of the session.",
+ tags: ["hitl", "approval", "once"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const first = approvalFor(await session.send(SAY.firstEcho, asAlice), "guarded-echo");
+ (await session.respond(answers("approve", first), asAlice))
+ .expectOk()
+ .calledTool("guarded-echo", { status: "completed", count: 1 });
+
+ const second = (await session.send(SAY.secondEcho, asAlice)).expectOk();
+ second.notEvent("input.requested");
+ second.calledTool("guarded-echo", {
+ status: "completed",
+ output: { echoed: "second" },
+ count: 1,
+ });
+ second.event("message.completed", { data: { message: /^Second echo: done / } });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/other-person-message-waits.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/other-person-message-waits.eval.ts
new file mode 100644
index 000000000..52d1c1915
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/other-person-message-waits.eval.ts
@@ -0,0 +1,46 @@
+import { defineEval } from "eve/evals";
+
+import {
+ REPLY,
+ SAY,
+ aliceSession,
+ answers,
+ approvalFor,
+ asAlice,
+ asBob,
+ expectResolved,
+} from "../helpers.ts";
+
+/**
+ * Alice's change waits for her approval when Bob asks for a status update.
+ * Only Alice's own message steers her turn: Bob's waits for the turn to end,
+ * so Alice's approval still runs the change, and Bob's message runs next.
+ */
+export default defineEval({
+ description: "A message from someone else waits for the held turn to end instead of steering it.",
+ tags: ["hitl", "approval", "steer"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const request = approvalFor(await session.send(SAY.changeA, asAlice), "change-a");
+
+ const bob = await session.start(SAY.bobStatus, asBob);
+ const approved = (await session.respond(answers("approve", request), asAlice)).expectOk();
+ expectResolved(approved, request, "approved");
+ approved.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });
+ approved.notEvent("message.received", { data: { message: SAY.bobStatus } });
+
+ approved.event("message.completed", { data: { message: /^Change A: done / } });
+ approved.event("turn.completed", { count: 1 });
+
+ // Bob's message starts its own turn once Alice's has ended.
+ const bobTurn = (await bob.result()).expectOk();
+ bobTurn.eventOrder([
+ { type: "turn.started" },
+ { type: "message.received", data: { message: SAY.bobStatus } },
+ { type: "message.completed", data: { message: REPLY.bobStatus } },
+ { type: "turn.completed" },
+ ]);
+ bobTurn.notEvent("input.resolved");
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/partial-answer-keeps-turn-held.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/partial-answer-keeps-turn-held.eval.ts
new file mode 100644
index 000000000..e2ebf20d5
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/partial-answer-keeps-turn-held.eval.ts
@@ -0,0 +1,53 @@
+import { defineEval } from "eve/evals";
+
+import {
+ SAY,
+ aliceSession,
+ answers,
+ approvalFor,
+ asAlice,
+ expectHeld,
+ expectNoModelCallWhileOpen,
+ expectNotRun,
+} from "../helpers.ts";
+
+/**
+ * One step asks Alice to approve changes A and B. Her answer to A alone waits
+ * in the turn: nothing resolves, nothing runs, and the model is not called.
+ * Declining B then resolves both together, each with its own answer.
+ */
+export default defineEval({
+ description: "A partial answer keeps the turn held until every approval of the step is answered.",
+ tags: ["hitl", "approval", "partial-approval"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const asked = await session.send(SAY.changesAB, asAlice);
+ const a = approvalFor(asked, "change-a");
+ const b = approvalFor(asked, "change-b");
+
+ const partial = await session.respond(answers("approve", a), asAlice);
+ expectHeld(partial);
+ partial.notEvent("input.resolved");
+ partial.notEvent("step.started");
+ partial.notEvent("action.result");
+
+ const finished = (await session.respond(answers("cancel", b), asAlice)).expectOk();
+ finished.event("input.resolved", {
+ count: 1,
+ data: {
+ resolutions: (items) =>
+ items.length === 2 &&
+ items.some((item) => item.requestId === a.requestId && item.outcome === "approved") &&
+ items.some((item) => item.requestId === b.requestId && item.outcome === "denied"),
+ },
+ });
+ finished.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });
+ expectNotRun(finished, "change-b");
+ finished.event("message.completed", {
+ data: { message: /^Change A: done .*\. Change B: not run\.$/u },
+ });
+ expectNoModelCallWhileOpen(session, a.requestId);
+ expectNoModelCallWhileOpen(session, b.requestId);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/stale-answer-authorizes-nothing.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/stale-answer-authorizes-nothing.eval.ts
new file mode 100644
index 000000000..d20704c41
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/stale-answer-authorizes-nothing.eval.ts
@@ -0,0 +1,35 @@
+import { defineEval } from "eve/evals";
+
+import {
+ REPLY,
+ SAY,
+ aliceSession,
+ approvalFor,
+ asAlice,
+ expectNotRun,
+ expectResolved,
+} from "../helpers.ts";
+
+/**
+ * While change A waits, Alice's client submits "approve" for a request that
+ * is not open (an old prompt). That answer becomes a message the model reads:
+ * it steers the turn like any message, so it authorizes nothing, A included.
+ */
+export default defineEval({
+ description: "An answer to a request that is not open becomes a message and authorizes nothing.",
+ tags: ["hitl", "approval", "stale-response"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const request = approvalFor(await session.send(SAY.changeA, asAlice), "change-a");
+
+ const stale = (
+ await session.respond([{ optionId: "approve", requestId: "an-earlier-prompt" }], asAlice)
+ ).expectOk();
+ expectResolved(stale, request, "ignored");
+ expectNotRun(stale, "change-a");
+ stale.event("message.received", { count: 1, data: { message: "approve" } });
+ stale.event("message.completed", { data: { message: REPLY.staleAnswer } });
+ stale.event("turn.completed", { count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/steer-ignores-unanswered.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/steer-ignores-unanswered.eval.ts
new file mode 100644
index 000000000..53dc62cee
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/steer-ignores-unanswered.eval.ts
@@ -0,0 +1,61 @@
+import { defineEval } from "eve/evals";
+
+import {
+ REPLY,
+ SAY,
+ aliceSession,
+ answers,
+ approvalFor,
+ asAlice,
+ expectNotRun,
+} from "../helpers.ts";
+
+/**
+ * Alice approves change A from a pair, then changes her mind and asks for a
+ * hello instead. Her message steers the held turn: B, which nobody answered,
+ * is ignored and never runs; A keeps her approval and runs; the model reads
+ * her message in the same turn.
+ */
+export default defineEval({
+ description: "Steering past approvals ignores unanswered ones and keeps the answers given.",
+ tags: ["hitl", "approval", "partial-approval", "steer"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const asked = await session.send(SAY.changesAB, asAlice);
+ const a = approvalFor(asked, "change-a");
+ const b = approvalFor(asked, "change-b");
+ await session.respond(answers("approve", a), asAlice);
+
+ const steered = (await session.send(SAY.hello, asAlice)).expectOk();
+ steered.event("input.resolved", {
+ count: 1,
+ data: {
+ resolutions: (items) =>
+ items.some((item) => item.requestId === a.requestId && item.outcome === "approved") &&
+ items.some((item) => item.requestId === b.requestId && item.outcome === "ignored"),
+ },
+ });
+ steered.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });
+ expectNotRun(steered, "change-b");
+ steered.event("action.result", {
+ data: {
+ result: { output: { approval: { status: "ignored" } }, toolName: "change-b" },
+ status: "rejected",
+ },
+ });
+ steered.notEvent("turn.started");
+ steered.event("message.received", { count: 1, data: { message: SAY.hello } });
+ // The answers resolve first. A runs in the result-reading step; Alice's
+ // queued message is received by the next step of the same turn.
+ steered.eventOrder([
+ { type: "input.resolved" },
+ { type: "step.started", data: { stepIndex: 1 } },
+ { type: "action.result", data: { status: "completed", result: { toolName: "change-a" } } },
+ { type: "message.received", data: { message: SAY.hello } },
+ { type: "step.started", data: { stepIndex: 2 } },
+ { type: "message.completed", data: { message: REPLY.hello } },
+ { type: "turn.completed" },
+ ]);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/typed-approve-answers.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/typed-approve-answers.eval.ts
new file mode 100644
index 000000000..74b78a91e
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/typed-approve-answers.eval.ts
@@ -0,0 +1,25 @@
+import { defineEval } from "eve/evals";
+
+import { SAY, aliceSession, approvalFor, asAlice, expectResolved } from "../helpers.ts";
+
+/**
+ * Alice answers her waiting change by typing "approve" instead of pressing the
+ * button. The reply names the approval's option, so it answers it: the change
+ * runs in the same turn, and the model never reads "approve" as a new request.
+ */
+export default defineEval({
+ description: "A typed reply naming an approval's option answers it.",
+ tags: ["hitl", "approval", "text-reply"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const request = approvalFor(await session.send(SAY.changeA, asAlice), "change-a");
+
+ const typed = (await session.send("approve", asAlice)).expectOk();
+ expectResolved(typed, request, "approved");
+ typed.calledTool("change-a", { status: "completed", output: { executions: 1 }, count: 1 });
+ typed.notEvent("turn.started");
+ typed.event("message.completed", { data: { message: /^Change A: done / } });
+ typed.event("turn.completed", { count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/typed-reply-answers-only-what-matches.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/typed-reply-answers-only-what-matches.eval.ts
new file mode 100644
index 000000000..d8b4c9f7e
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approvals/typed-reply-answers-only-what-matches.eval.ts
@@ -0,0 +1,54 @@
+import { defineEval } from "eve/evals";
+
+import {
+ SAY,
+ aliceSession,
+ answers,
+ approvalFor,
+ asAlice,
+ asReleaseManager,
+ expectHeld,
+} from "../helpers.ts";
+
+/**
+ * One step asks for change A and an authorized change. Alice types "approve":
+ * that answers A, but never the authorized change, whose response policy must
+ * know who answered. The turn stays held until the release manager approves
+ * it; then both resolve together and run.
+ */
+export default defineEval({
+ description: "A typed reply answers the approvals it matches and leaves the rest open.",
+ tags: ["hitl", "approval", "text-reply", "authorization"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const asked = await session.send(SAY.changeAAndAuthorized, asAlice);
+ const a = approvalFor(asked, "change-a");
+ const authorized = approvalFor(asked, "authorized-change");
+
+ const typed = await session.send("approve", asAlice);
+ expectHeld(typed);
+ typed.notEvent("input.resolved");
+ typed.notEvent("action.result");
+ typed.notEvent("step.started");
+
+ const settled = (
+ await session.respond(answers("approve", authorized), asReleaseManager)
+ ).expectOk();
+ settled.event("approval.settled", {
+ count: 1,
+ data: { outcome: "approved", requestId: authorized.requestId },
+ });
+ settled.event("input.resolved", {
+ count: 1,
+ data: {
+ resolutions: (items) =>
+ [a.requestId, authorized.requestId].every((id) =>
+ items.some((item) => item.requestId === id && item.outcome === "approved"),
+ ),
+ },
+ });
+ settled.calledTool("change-a", { status: "completed", count: 1 });
+ settled.calledTool("authorized-change", { status: "completed", count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/approved-call-needs-authorization.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/approved-call-needs-authorization.eval.ts
new file mode 100644
index 000000000..8b001060f
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/approved-call-needs-authorization.eval.ts
@@ -0,0 +1,53 @@
+import { defineEval } from "eve/evals";
+
+import {
+ ALICE,
+ SAY,
+ aliceSession,
+ answers,
+ approvalFor,
+ asAlice,
+ completeAuthorization,
+ expectHeld,
+ expectResolved,
+ follow,
+ authorizationFrom,
+} from "../helpers.ts";
+
+/**
+ * Publishing the draft needs approval (once) and an authorization. Alice approves;
+ * the approved call then asks her to authorize, which holds the turn. After the
+ * callback the model calls publish again, the once() grant covers it, and it
+ * runs as Alice.
+ */
+export default defineEval({
+ description: "An approved call that needs an authorization holds the turn and runs after it.",
+ tags: ["hitl", "approval", "authorization"],
+ timeoutMs: 90_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const request = approvalFor(await session.send(SAY.publish, asAlice), "publish-draft");
+
+ const signing = await session.respond(answers("approve", request), asAlice);
+ expectResolved(signing, request, "approved");
+ expectHeld(signing);
+ signing.event("authorization.required", { count: 1, data: { principalId: ALICE } });
+ signing.notEvent("action.result", { data: { status: "completed" } });
+ const authorization = authorizationFrom(signing);
+
+ const startIndex = session.state.streamIndex;
+ await completeAuthorization(authorization.url);
+ const resumed = (await follow(t, session, startIndex)).expectOk();
+ resumed.event("authorization.completed", {
+ count: 1,
+ data: { attemptId: authorization.attemptId, outcome: "authorized" },
+ });
+ resumed.notEvent("input.requested");
+ resumed.calledTool("publish-draft", {
+ status: "completed",
+ output: { actor: ALICE, publications: 1 },
+ count: 1,
+ });
+ resumed.event("message.completed", { data: { message: /^Publish: done / } });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/authorization-holds-then-resumes-as-requester.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/authorization-holds-then-resumes-as-requester.eval.ts
new file mode 100644
index 000000000..efba7b644
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/authorization-holds-then-resumes-as-requester.eval.ts
@@ -0,0 +1,49 @@
+import { defineEval } from "eve/evals";
+
+import {
+ ALICE,
+ SAY,
+ aliceSession,
+ asAlice,
+ completeAuthorization,
+ expectHeld,
+ expectNoModelCallDuringAuthorization,
+ follow,
+ authorizationFrom,
+} from "../helpers.ts";
+
+/**
+ * Alice's access check needs a Fixture Auth sign-in, which holds her turn.
+ * The callback is an anonymous browser request, yet the turn resumes as
+ * Alice: the tool runs again with her identity, in the same turn.
+ */
+export default defineEval({
+ description:
+ "A tool's authorization holds the turn, and its callback resumes it as the requester.",
+ tags: ["hitl", "authorization"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const held = await session.send(SAY.checkAccess, asAlice);
+ expectHeld(held);
+ held.event("authorization.required", { count: 1, data: { principalId: ALICE } });
+ held.notEvent("authorization.completed");
+ const authorization = authorizationFrom(held);
+
+ const startIndex = session.state.streamIndex;
+ await completeAuthorization(authorization.url);
+ const resumed = (await follow(t, session, startIndex)).expectOk();
+ resumed.notEvent("turn.started");
+ resumed.event("authorization.completed", {
+ count: 1,
+ data: { attemptId: authorization.attemptId, outcome: "authorized", principalId: ALICE },
+ });
+ resumed.calledTool("auth-probe", {
+ status: "completed",
+ output: { actor: ALICE },
+ count: 1,
+ });
+ resumed.event("message.completed", { data: { message: /^Access check: done / } });
+ expectNoModelCallDuringAuthorization(resumed, authorization.attemptId, held.events);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/cancel-declines-authorization.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/cancel-declines-authorization.eval.ts
new file mode 100644
index 000000000..f4065e6a7
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/cancel-declines-authorization.eval.ts
@@ -0,0 +1,25 @@
+import { defineEval } from "eve/evals";
+
+import { SAY, aliceSession, asAlice, follow, authorizationFrom } from "../helpers.ts";
+
+/** Alice cancels her turn while it waits for her authorization: the authorization is declined. */
+export default defineEval({
+ description: "Cancelling a turn held on an authorization declines the authorization.",
+ tags: ["hitl", "authorization", "cancel"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const authorization = authorizationFrom(await session.send(SAY.checkAccess, asAlice));
+ const startIndex = session.state.streamIndex;
+
+ await session.cancel();
+ const cancelled = await follow(t, session, startIndex);
+ cancelled.event("authorization.completed", {
+ count: 1,
+ data: { attemptId: authorization.attemptId, outcome: "declined", reason: "Cancelled." },
+ });
+ cancelled.event("turn.cancelled", { count: 1 });
+ cancelled.notEvent("step.started");
+ cancelled.notEvent("action.result", { data: { status: "completed" } });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/steer-declines-authorization.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/steer-declines-authorization.eval.ts
new file mode 100644
index 000000000..cde806c87
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/steer-declines-authorization.eval.ts
@@ -0,0 +1,35 @@
+import { defineEval } from "eve/evals";
+
+import { REPLY, SAY, aliceSession, asAlice, expectHeld, authorizationFrom } from "../helpers.ts";
+
+/**
+ * Alice's access check waits for her authorization when she asks for a hello
+ * instead. Her message steers the turn: the authorization is declined, and the
+ * model is told it ended so it asks again only if it still needs it.
+ */
+export default defineEval({
+ description:
+ "Steering past an authorization declines it and tells the model which authorization ended.",
+ tags: ["hitl", "authorization", "steer"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const held = await session.send(SAY.checkAccess, asAlice);
+ expectHeld(held);
+ const authorization = authorizationFrom(held);
+
+ const steered = (await session.send(SAY.hello, asAlice)).expectOk();
+ steered.notEvent("turn.started");
+ steered.event("authorization.completed", {
+ count: 1,
+ data: {
+ attemptId: authorization.attemptId,
+ outcome: "declined",
+ reason: "Cancelled because a new message arrived.",
+ },
+ });
+ // The scripted model answers this way only when the authorization note reached it.
+ steered.event("message.completed", { data: { message: REPLY.helloAfterAuthorization } });
+ steered.notEvent("action.result", { data: { status: "completed" } });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/no-step-behind-budget.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/no-step-behind-budget.eval.ts
new file mode 100644
index 000000000..4f7843812
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/no-step-behind-budget.eval.ts
@@ -0,0 +1,21 @@
+import { defineEval } from "eve/evals";
+
+import { SAY, asAlice, budgetQuestion, expectNoModelCallWhileOpen } from "../helpers.ts";
+
+/**
+ * While the budget question waits, Alice adds a message. The turn must stay
+ * held without starting a model step: nothing may run while the question is
+ * open.
+ */
+export default defineEval({
+ description: "No model step starts while the budget question is open.",
+ tags: ["hitl", "budget"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const { request, session } = await budgetQuestion(t);
+
+ const added = await session.send(SAY.deadline, asAlice);
+ added.notEvent("step.started");
+ expectNoModelCallWhileOpen(session, request.requestId);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/stale-budget-answer-dropped.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/stale-budget-answer-dropped.eval.ts
new file mode 100644
index 000000000..9a02021f5
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/stale-budget-answer-dropped.eval.ts
@@ -0,0 +1,26 @@
+import { defineEval } from "eve/evals";
+
+import { REPLY, SAY, answers, asAlice, budgetQuestion } from "../helpers.ts";
+
+/**
+ * Alice's client sends her Continue a second time after the question closed.
+ * A late budget answer must be dropped, not read as a message: read as text,
+ * a late Stop would seem to stop something. Her next message then runs as
+ * usual, with nothing stale before it.
+ */
+export default defineEval({
+ description: "An answer to a budget question that already closed is dropped.",
+ tags: ["hitl", "budget", "stale-response"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const { request, session } = await budgetQuestion(t);
+ (await session.respond(answers("continue", request), asAlice)).expectOk();
+
+ await session.startRespond(answers("continue", request), asAlice);
+ const next = (await session.send(SAY.bobStatus, asAlice)).expectOk();
+ next.event("message.completed", { data: { message: REPLY.bobStatus } });
+ session.notEvent("message.received", { data: { message: "continue" } });
+ session.notEvent("message.completed", { data: { message: REPLY.staleAnswer } });
+ session.event("input.resolved", { count: 1 });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/stop-cancels-once.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/stop-cancels-once.eval.ts
new file mode 100644
index 000000000..6a4cf0389
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/budget/stop-cancels-once.eval.ts
@@ -0,0 +1,25 @@
+import { defineEval } from "eve/evals";
+
+import { answers, asAlice, budgetQuestion, expectResolved } from "../helpers.ts";
+
+/** Alice chooses Stop: the question resolves once, and the turn is cancelled without a model call. */
+export default defineEval({
+ description: "Stop resolves the budget question once and cancels the turn.",
+ tags: ["hitl", "budget", "cancel"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const { request, session } = await budgetQuestion(t);
+
+ const stopped = await session.respond(answers("stop", request), asAlice);
+ expectResolved(stopped, request, "answered");
+ stopped.event("input.resolved", { count: 1 });
+ stopped.event("turn.cancelled", { count: 1 });
+ stopped.notEvent("step.started");
+ stopped.notEvent("message.completed");
+ stopped.notEvent("turn.failed");
+ session.event("input.resolved", {
+ count: 1,
+ data: { resolutions: (items) => items.some((item) => item.requestId === request.requestId) },
+ });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/helpers.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/helpers.ts
new file mode 100644
index 000000000..8db4495b3
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/helpers.ts
@@ -0,0 +1,170 @@
+import type {
+ EveEvalAssertions,
+ EveEvalContext,
+ EveEvalSession,
+ EveEvalTurn,
+ InputRequest,
+} from "eve/evals";
+
+import { SAY } from "../../agent/lib/hitl/respond.ts";
+
+export { REPLY, SAY } from "../../agent/lib/hitl/respond.ts";
+
+export const ALICE = "alice";
+export const BOB = "bob";
+/** The fixture's authorized approver for `authorized-change` and `responder-gate`. */
+export const RELEASE_MANAGER = "e2e-approval-responder";
+
+/** Headers for one person on the scripted hitl model; `flag` feeds fixture policies. */
+export function as(principalId: string, flag?: "retire") {
+ return {
+ headers: {
+ "x-eve-fixture-model": "hitl",
+ "x-eve-fixture-user": principalId,
+ ...(flag !== undefined && { "x-eve-fixture-flag": flag }),
+ },
+ };
+}
+
+/**
+ * A session Alice starts, so the scripted model drives every turn of it.
+ * Headers apply per request: pass `as(...)` to every send and respond too.
+ */
+export function aliceSession(t: EveEvalContext): Promise<EveEvalSession> {
+ return t.session(as(ALICE));
+}
+
+export const asAlice = as(ALICE);
+export const asBob = as(BOB);
+export const asReleaseManager = as(RELEASE_MANAGER);
+
+/** The one approval request `turn` raised for `toolName`. */
+export function approvalFor(turn: EveEvalTurn, toolName: string): InputRequest {
+ const matches = turn.inputRequests.filter((request) => request.action.toolName === toolName);
+ if (matches.length !== 1) {
+ throw new Error(`Expected one ${toolName} request; found ${matches.length}.`);
+ }
+ return matches[0]!;
+}
+
+/** `respond` answers for the given requests, all with one option. */
+export function answers(optionId: string, ...requests: readonly InputRequest[]) {
+ return requests.map(({ requestId }) => ({ optionId, requestId }));
+}
+
+/**
+ * The model never runs while a request of the turn's own is open: no
+ * `step.started` falls between the request's `input.requested` and its
+ * `input.resolved` (or the end of the stream, while it is still open).
+ */
+export function expectNoModelCallWhileOpen(on: EveEvalAssertions, requestId: string) {
+ on.eventsSatisfy(`no model step starts while ${requestId} is open`, (events) => {
+ const asked = events.findIndex(
+ (event) =>
+ event.type === "input.requested" &&
+ event.data.requests.some((request) => request.requestId === requestId),
+ );
+ if (asked < 0) return false;
+ const resolved = events.findIndex(
+ (event, index) =>
+ index > asked &&
+ event.type === "input.resolved" &&
+ event.data.resolutions.some((resolution) => resolution.requestId === requestId),
+ );
+ const open = events.slice(asked + 1, resolved < 0 ? events.length : resolved);
+ return !open.some((event) => event.type === "step.started");
+ });
+}
+
+/**
+ * No `step.started` falls between the authorization `attemptId` opening and closing.
+ * `before` carries the events streamed before `on` started, such as the ask.
+ */
+export function expectNoModelCallDuringAuthorization(
+ on: EveEvalAssertions,
+ attemptId: string,
+ before: EveEvalTurn["events"] = [],
+) {
+ on.eventsSatisfy(`no model step starts while authorization ${attemptId} is open`, (observed) => {
+ const events = [...before, ...observed];
+ const asked = events.findIndex(
+ (event) => event.type === "authorization.required" && event.data.attemptId === attemptId,
+ );
+ if (asked < 0) return false;
+ const closed = events.findIndex(
+ (event, index) =>
+ index > asked &&
+ event.type === "authorization.completed" &&
+ event.data.attemptId === attemptId,
+ );
+ const open = events.slice(asked + 1, closed < 0 ? events.length : closed);
+ return !open.some((event) => event.type === "step.started");
+ });
+}
+
+/** The latest authorization `turn` asked for: its attempt and its fixture callback URL. */
+export function authorizationFrom(turn: Pick<EveEvalTurn, "events">): {
+ attemptId: string;
+ url: URL;
+} {
+ const required = [...turn.events]
+ .reverse()
+ .find((event) => event.type === "authorization.required");
+ if (
+ required?.type !== "authorization.required" ||
+ required.data.attemptId === undefined ||
+ required.data.authorization?.url === undefined
+ ) {
+ throw new Error("Expected an authorization with an attempt id and a callback URL.");
+ }
+ return { attemptId: required.data.attemptId, url: new URL(required.data.authorization.url) };
+}
+
+/** Completes a fixture authorization the way a browser would: an unauthenticated GET. */
+export async function completeAuthorization(url: URL) {
+ const callback = await fetch(url);
+ if (!callback.ok) throw new Error(`Fixture authorization callback failed (${callback.status}).`);
+}
+
+/** Everything the session streams from `startIndex` to the next turn boundary. */
+export function follow(t: EveEvalContext, session: EveEvalSession, startIndex: number) {
+ return t.target.watchTurn(session.sessionId, { startIndex }).result();
+}
+
+/**
+ * Alice spends the session's output budget on a summary, then asks for a
+ * status note: the turn stops before its model call and asks whether to go on.
+ */
+export async function budgetQuestion(t: EveEvalContext) {
+ const session = await aliceSession(t);
+ (await session.send(SAY.spendBudget, asAlice)).expectOk();
+ const held = await session.send(SAY.statusNote, asAlice);
+ const request = approvalFor(held, "session_limit_continuation");
+ return { held, request, session };
+}
+
+/** The turn is held on input: it says so, and neither completes nor fails. */
+export function expectHeld(turn: EveEvalTurn) {
+ turn.event("turn.waiting", { data: { on: "input" } });
+ turn.notEvent("turn.completed");
+ turn.notEvent("turn.failed");
+}
+
+/** `request` resolved exactly once in `on`, with `outcome`. */
+export function expectResolved(on: EveEvalAssertions, request: InputRequest, outcome: string) {
+ on.event("input.resolved", {
+ count: 1,
+ data: {
+ resolutions: (items) =>
+ items.some((item) => item.requestId === request.requestId && item.outcome === outcome),
+ },
+ });
+}
+
+/** `toolName`'s call never ran: it has a rejected not-run result and no completed one. */
+export function expectNotRun(on: EveEvalAssertions, toolName: string) {
+ on.event("action.result", {
+ data: { status: "rejected", result: { toolName } },
+ });
+ on.notEvent("action.result", { data: { status: "completed", result: { toolName } } });
+}
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/allowed-responder-settles.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/allowed-responder-settles.eval.ts
new file mode 100644
index 000000000..f96e140d5
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/allowed-responder-settles.eval.ts
@@ -0,0 +1,54 @@
+import { defineEval } from "eve/evals";
+
+import {
+ RELEASE_MANAGER,
+ SAY,
+ aliceSession,
+ answers,
+ approvalFor,
+ asAlice,
+ asReleaseManager,
+ expectResolved,
+} from "../helpers.ts";
+
+/**
+ * The release manager approves Alice's authorized change. The policy allows
+ * the candidate, which settles the approval as the release manager's; the
+ * change then runs once.
+ */
+export default defineEval({
+ description: "An answer the response policy allows settles the approval and runs the call.",
+ tags: ["hitl", "authorization"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const request = approvalFor(await session.send(SAY.authorized, asAlice), "authorized-change");
+
+ const settled = (
+ await session.respond(answers("approve", request), asReleaseManager)
+ ).expectOk();
+ settled.eventOrder([
+ { type: "approval.candidate", data: { outcome: "pending", requestId: request.requestId } },
+ {
+ type: "approval.settled",
+ data: {
+ outcome: "approved",
+ requestId: request.requestId,
+ responderPrincipalId: RELEASE_MANAGER,
+ },
+ },
+ { type: "input.resolved" },
+ {
+ type: "action.result",
+ data: { status: "completed", result: { toolName: "authorized-change" } },
+ },
+ { type: "turn.completed" },
+ ]);
+ expectResolved(settled, request, "approved");
+ settled.calledTool("authorized-change", {
+ status: "completed",
+ output: { executions: 1 },
+ count: 1,
+ });
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/rejected-responder-leaves-approval-open.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/rejected-responder-leaves-approval-open.eval.ts
new file mode 100644
index 000000000..fd36bea15
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/rejected-responder-leaves-approval-open.eval.ts
@@ -0,0 +1,42 @@
+import { defineEval } from "eve/evals";
+
+import {
+ SAY,
+ aliceSession,
+ answers,
+ approvalFor,
+ asAlice,
+ asBob,
+ expectHeld,
+ expectNoModelCallWhileOpen,
+} from "../helpers.ts";
+
+/**
+ * Only the release manager may approve the authorized change. Bob approves
+ * it anyway: his answer becomes a candidate the policy rejects, and the
+ * approval stays open with the turn held.
+ */
+export default defineEval({
+ description: "A response policy rejects an unauthorized answer and the approval stays open.",
+ tags: ["hitl", "authorization"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const request = approvalFor(await session.send(SAY.authorized, asAlice), "authorized-change");
+
+ const refused = await session.respond(answers("approve", request), asBob);
+ expectHeld(refused);
+ refused.eventOrder([
+ { type: "approval.candidate", data: { outcome: "pending", responderPrincipalId: "bob" } },
+ {
+ type: "approval.candidate",
+ data: { outcome: "rejected", reason: "Wrong responder.", responderPrincipalId: "bob" },
+ },
+ { type: "turn.waiting", data: { on: "input" } },
+ ]);
+ refused.notEvent("approval.settled");
+ refused.notEvent("input.resolved");
+ refused.notEvent("action.result");
+ expectNoModelCallWhileOpen(session, request.requestId);
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/responder-authorization-then-settles.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/responder-authorization-then-settles.eval.ts
new file mode 100644
index 000000000..597b6a9f7
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/response-policies/responder-authorization-then-settles.eval.ts
@@ -0,0 +1,70 @@
+import { defineEval } from "eve/evals";
+
+import {
+ SAY,
+ aliceSession,
+ answers,
+ approvalFor,
+ as,
+ asAlice,
+ completeAuthorization,
+ expectHeld,
+ expectNoModelCallDuringAuthorization,
+ expectResolved,
+ follow,
+ authorizationFrom,
+} from "../helpers.ts";
+
+// A responder of its own: the fixture's authorizations outlive an eval, and other
+// evals sign the default release manager in.
+const RESPONDER = "hitl-authorization-responder";
+
+/**
+ * A responder approves the OAuth-checked change. Its policy needs the
+ * approver authorized first, so the turn holds on the responder's authorization;
+ * the callback runs the policy again, which settles the approval as the
+ * responder and runs the change.
+ */
+export default defineEval({
+ description:
+ "A response policy that needs the responder to authorize settles after the authorization.",
+ tags: ["hitl", "authorization", "authorization"],
+ timeoutMs: 90_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const request = approvalFor(
+ await session.send(SAY.oauthChecked, asAlice),
+ "oauth-authorized-gate",
+ );
+
+ const held = await session.respond(answers("approve", request), as(RESPONDER));
+ expectHeld(held);
+ held.event("approval.candidate", { count: 1, data: { outcome: "pending" } });
+ held.event("authorization.required", { count: 1, data: { principalId: RESPONDER } });
+ held.notEvent("approval.settled");
+ const authorization = authorizationFrom(held);
+
+ const startIndex = session.state.streamIndex;
+ await completeAuthorization(authorization.url);
+ const resumed = (await follow(t, session, startIndex)).expectOk();
+ resumed.notEvent("turn.started");
+ resumed.eventOrder([
+ {
+ type: "authorization.completed",
+ data: { attemptId: authorization.attemptId, outcome: "authorized" },
+ },
+ {
+ type: "approval.settled",
+ data: { outcome: "approved", responderPrincipalId: RESPONDER },
+ },
+ {
+ type: "action.result",
+ data: { status: "completed", result: { toolName: "oauth-authorized-gate" } },
+ },
+ { type: "message.completed", data: { message: /^OAuth-checked change: done / } },
+ { type: "turn.completed" },
+ ]);
+ expectResolved(resumed, request, "approved");
+ expectNoModelCallDuringAuthorization(resumed, authorization.attemptId, held.events);
+ },
+});
diff --git a/e2e/fixtures/agent-workflow-tools/agent/agent.ts b/e2e/fixtures/agent-workflow-tools/agent/agent.ts
index 19604b735..6bc98220b 100644
--- a/e2e/fixtures/agent-workflow-tools/agent/agent.ts
+++ b/e2e/fixtures/agent-workflow-tools/agent/agent.ts
@@ -147,6 +147,7 @@ async function respond(request: MockModelRequest): Promise<MockModelResponse | s
["WORKFLOW-FANOUT-START", "fanout_deploy"],
["WORKFLOW-WEBHOOK-START", "webhook_deploy"],
["WORKFLOW-AGENT-FANOUT-START", "fanout_agents"],
+ ["WORKFLOW-LAPSE-START", "lapsing_offer"],
] as const) {
if (!message.includes(directive)) continue;
const result = [...request.toolResults].reverse().find((entry) => entry.name === tool);
diff --git a/e2e/fixtures/agent-workflow-tools/agent/tools/lapsing_offer.ts b/e2e/fixtures/agent-workflow-tools/agent/tools/lapsing_offer.ts
new file mode 100644
index 000000000..760fe2e0e
--- /dev/null
+++ b/e2e/fixtures/agent-workflow-tools/agent/tools/lapsing_offer.ts
@@ -0,0 +1,31 @@
+import { defineWorkflowTool } from "eve/tools";
+import { sleep } from "workflow";
+import { z } from "zod";
+
+import { describePlan } from "../lib/plan.ts";
+
+/**
+ * Offers a deploy for a few seconds, then returns without withdrawing its
+ * question. The run ends with the question still open, so the session must
+ * withdraw it: nobody can answer a run that has ended.
+ */
+export default defineWorkflowTool({
+ description: "Offer to deploy a service for a few seconds, then let the offer lapse.",
+ inputSchema: z.strictObject({ service: z.string() }),
+ async execute({ service }, ctx) {
+ "use workflow";
+
+ const answer = await Promise.race([
+ ctx.ask({
+ display: "confirmation",
+ options: [
+ { id: "approve", label: "Deploy now", style: "primary" },
+ { id: "cancel", label: "Not now" },
+ ],
+ prompt: `Apply ${describePlan(service)} in the next few seconds?`,
+ }),
+ sleep("3s"),
+ ]);
+ return { offer: answer === undefined ? "lapsed" : "answered", service };
+ },
+});
diff --git a/e2e/fixtures/agent-workflow-tools/evals/hitl/cancel-withdraws-relayed.eval.ts b/e2e/fixtures/agent-workflow-tools/evals/hitl/cancel-withdraws-relayed.eval.ts
new file mode 100644
index 000000000..74e299805
--- /dev/null
+++ b/e2e/fixtures/agent-workflow-tools/evals/hitl/cancel-withdraws-relayed.eval.ts
@@ -0,0 +1,27 @@
+import { defineEval } from "eve/evals";
+
+/**
+ * Alice cancels her turn while the deploy's question waits for her. The
+ * cancel stops the run, so nobody can answer what it relayed: the session
+ * withdraws the question as cancelled.
+ */
+export default defineEval({
+ description: "Cancelling the turn withdraws the questions relayed through it.",
+ tags: ["hitl", "relayed", "cancel"],
+ async test(t) {
+ const parked = await t.send("WORKFLOW-CONFIRM-START");
+ const request = parked.session.requireInputRequest({ toolName: "confirm_deploy" });
+ const startIndex = parked.session.state.streamIndex;
+
+ await parked.session.cancel();
+ const cancelled = await t.target.watchTurn(parked.sessionId, { startIndex }).result();
+ cancelled.event("input.resolved", {
+ count: 1,
+ data: { resolutions: [{ outcome: "cancelled", requestId: request.requestId }] },
+ });
+ cancelled.event("turn.cancelled", { count: 1 });
+ cancelled.notEvent("action.result", {
+ data: { status: "completed", result: { toolName: "confirm_deploy" } },
+ });
+ },
+});
diff --git a/e2e/fixtures/agent-workflow-tools/evals/hitl/run-end-withdraws-relayed.eval.ts b/e2e/fixtures/agent-workflow-tools/evals/hitl/run-end-withdraws-relayed.eval.ts
new file mode 100644
index 000000000..33eff20b2
--- /dev/null
+++ b/e2e/fixtures/agent-workflow-tools/evals/hitl/run-end-withdraws-relayed.eval.ts
@@ -0,0 +1,29 @@
+import { defineEval } from "eve/evals";
+
+/**
+ * Alice's quick-deploy offer asks a question through her session and lapses
+ * after a few seconds without withdrawing it. Once the run ends nobody can
+ * answer it, so the session withdraws it as cancelled before the call's result.
+ */
+export default defineEval({
+ description: "A workflow run that ends withdraws the question it relayed.",
+ tags: ["hitl", "relayed"],
+ async test(t) {
+ const parked = await t.send("WORKFLOW-LAPSE-START Alice offers a quick deploy of the api.");
+ const request = parked.session.requireInputRequest({ toolName: "lapsing_offer" });
+ const startIndex = parked.session.state.streamIndex;
+
+ const lapsed = (await t.target.watchTurn(parked.sessionId, { startIndex }).result()).expectOk();
+ lapsed.event("input.resolved", {
+ count: 1,
+ data: { resolutions: [{ outcome: "cancelled", requestId: request.requestId }] },
+ });
+ lapsed.eventOrder([
+ { type: "input.resolved" },
+ { type: "action.result", data: { result: { toolName: "lapsing_offer" } } },
+ { type: "turn.completed" },
+ ]);
+ lapsed.calledTool("lapsing_offer", { output: { offer: "lapsed" }, count: 1 });
+ lapsed.messageIncludes('WORKFLOW-LAPSE-RESULT {"offer":"lapsed","service":"api"}');
+ },
+});