The Open Framework for Building Agents
test(e2e): gate the final background task contract (#1807)
Rui4cf32bca6e5b
diff --git a/e2e/fixtures/fixture-tasks/.gitignore b/e2e/fixtures/fixture-tasks/.gitignorenew file mode 100644index 000000000..9a7c2abb6--- /dev/null+++ b/e2e/fixtures/fixture-tasks/.gitignore@@ -0,0 +1,11 @@+node_modules+.env*+.eve+.vercel+.next+.output+.nitro+dist+.DS_Store+*.tsbuildinfo+.env*.localdiff --git a/e2e/fixtures/fixture-tasks/.vercelignore b/e2e/fixtures/fixture-tasks/.vercelignorenew file mode 100644index 000000000..357664320--- /dev/null+++ b/e2e/fixtures/fixture-tasks/.vercelignore@@ -0,0 +1,6 @@+node_modules+.eve+.next+.output+.nitro+distdiff --git a/e2e/fixtures/fixture-tasks/agent/agent.ts b/e2e/fixtures/fixture-tasks/agent/agent.tsnew file mode 100644index 000000000..1c4d6c35c--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/agent.ts@@ -0,0 +1,484 @@+import { e2eAgentConfig } from "@eve-e2e/config";+import { defineAgent } from "eve";+import {+ mockModel,+ type MockModelRequest,+ type MockModelResponse,+ type MockModelToolResult,+} from "eve/evals";++const TASK_ID_PATTERN = /task_[a-z0-9]+/iu;++function respond(request: MockModelRequest): MockModelResponse | string {+ // Framework agent-list notes are model context, not scenario turns.+ const message = [...request.userMessages].reverse().find(isScenarioMessage) ?? "";+ if (message.includes("TASK-FANOUT-INTERACTIVE-CHECK")) return "TASK-FANOUT-INTERACTIVE-OK";+ if (message.includes("TASK-CANCEL-NOW")) return cancelWorkerTask(request);+ if (message.includes("CHILD-TASK-EXCLUSIVITY-RACE")) return raceBusyWorker(request);+ if (message.startsWith("CHILD-TASK-EXCLUSIVITY-LATER ")) {+ return laterBusyWorker(request, message);+ }+ if (message.startsWith("TASK-A2-CHILD-FAILURE-VERIFY ")) {+ return peekTask(request, "task-a2-child-failure-verify", "TASK-A2-FAILED", message);+ }+ if (message.startsWith("TASK-A3-UNKNOWN-VERIFY ")) {+ return peekTask(request, "task-a3-unknown-verify", "TASK-A3-UNKNOWN", message);+ }+ if (message.startsWith("TASK-D6-PARTIAL-FANOUT-VERIFY ")) {+ return peekTask(request, "task-d6-partial-fanout-verify", "TASK-D6-STATUS", message);+ }+ if (message.startsWith("TASK-D6-PARTIAL-FANOUT-UNKNOWN ")) {+ return peekTask(request, "task-d6-partial-fanout-unknown", "TASK-D6-UNKNOWN", message);+ }+ if (message.startsWith("TASK-C7-AUTHORIZATION-VERIFY ")) {+ return peekTask(request, "task-c7-authorization-verify", "TASK-C7-STATUS", message);+ }+ if (message.startsWith("TASK-C8-REMOTE-VERIFY ")) {+ return peekTask(request, "task-c8-remote-verify", "TASK-C8-STATUS", message);+ }+ if (message.includes("TASK-C8-REMOTE-CHILD")) return runRemoteGate(request);+ if (message.startsWith("Background task ")) {+ // Scenarios that act on wake notifications route to their script;+ // every other scenario acknowledges them without running tools.+ // The exclusivity race delegates so a completion notification that+ // coalesces into the RACE turn still runs the same-batch sends.+ if (request.userMessages.includes("TASK-FAN-IN")) return fanInNotification(request);+ if (request.userMessages.includes("CHILD-TASK-EXCLUSIVITY-RACE")) {+ return raceBusyWorker(request);+ }+ return "TASK-NOTIFICATION-ACK";+ }++ if (message === "TASK-FANOUT-PARENT-UPDATES") return fanoutTasks(request, 10);+ if (message === "TASK-PARENT-WAKE-UPDATES") return fanoutTasks(request, 3);+ if (message === "TASK-FAN-IN") return fanInTasks(request);+ if (message === "TASK-CANCEL-SETUP") return setupCancelWorker(request);+ if (message.startsWith("TASK-CANCEL-VERIFY ")) {+ return peekTask(request, "task-cancel-verify", "TASK-CANCEL-STATUS", message);+ }++ if (message.startsWith("TASK-HITL-VERIFY ")) {+ return peekTask(request, "task-hitl-verify", "TASK-HITL-STATUS", message);+ }+ if (message.startsWith("TASK-INPUT-BATCH-VERIFY ")) {+ return peekTask(request, "task-input-batch-verify", "TASK-INPUT-BATCH-STATUS", message);+ }+ if (message.startsWith("CHILD-TASK-EXCLUSIVITY-VERIFY ")) {+ return peekTask(+ request,+ "child-task-exclusivity-verify",+ "CHILD-TASK-EXCLUSIVITY-STATUS",+ message,+ );+ }+ if (message === "TASK-HITL-ROUTING") {+ return startApprovalWorker(request, "task-hitl-worker", "TASK-HITL-STARTED");+ }+ if (message === "TASK-C7-AUTHORIZATION") {+ return startApprovalWorker(request, "task-c7-authorization-worker", "TASK-C7-STARTED");+ }+ if (message === "TASK-C8-REMOTE-HITL") return startRemoteWorker(request);+ if (message === "TASK-INPUT-BATCH-ORDERING") {+ return startApprovalWorker(request, "task-input-batch-worker", "TASK-INPUT-BATCH-STARTED");+ }+ if (message === "TASK-CONTINUATION-HITL-SETUP") {+ return startApprovalWorker(+ request,+ "task-continuation-hitl-worker",+ "TASK-CONTINUATION-HITL-READY",+ );+ }+ if (message.startsWith("TASK-CONTINUATION-HITL-SEND ")) {+ return continueApprovalWorker(request, message);+ }+ if (message === "CHILD-TASK-EXCLUSIVITY-SETUP") return setupBusyWorker(request);+ if (message === "TASK-A2-CHILD-FAILURE") return startFailingBusyWorker(request);+ if (message === "TASK-A3-DISPATCH-START-FAILURE") return startUnstartableWorker(request);+ if (message === "TASK-D6-PARTIAL-FANOUT-FAILURE") return partialFailureFanout(request);++ return `Mock reply: ${message}`;+}++function fanoutTasks(request: MockModelRequest, size: number): MockModelResponse | string {+ const pending = Array.from({ length: size }, (_, index) => index + 1).filter(+ (index) => resultById(request, `task-fanout-${index}`) === undefined,+ );+ if (pending.length > 0) {+ return {+ toolCalls: pending.map((index) => ({+ id: `task-fanout-${index}`,+ input: { message: `FANOUT-WORKER-${index}` },+ name: "fanout-worker",+ })),+ };+ }+ return "TASK-FANOUT-STARTED";+}++const FAN_IN_CALL_IDS = ["task-fan-in-1", "task-fan-in-2"] as const;++function fanInTasks(request: MockModelRequest): MockModelResponse | string {+ const pending = FAN_IN_CALL_IDS.filter((id) => resultById(request, id) === undefined);+ if (pending.length > 0) {+ return {+ toolCalls: pending.map((id) => ({+ id,+ input: { message: `Run the release gate, then return ${id.toUpperCase()}.` },+ name: "fanout-worker",+ })),+ };+ }+ return "TASK-FAN-IN-STARTED";+}++/**+ * The deterministic join predicate: on every wake, peek every fan-in task+ * and answer only when all of them are completed. This is the model-side+ * join contract — the framework delivers one wake+ * per ready transition and the model decides whether the state suffices.+ */+function fanInNotification(request: MockModelRequest): MockModelResponse | string {+ const callId = `task-fan-in-check-${scenarioUserMessageCount(request)}`;+ const checked = resultById(request, callId);+ const taskIds = FAN_IN_CALL_IDS.map((id) => findTaskId(resultById(request, id)?.output)).filter(+ (taskId): taskId is string => taskId !== undefined,+ );+ if (taskIds.length !== FAN_IN_CALL_IDS.length) {+ throw new Error("Fan-in notification arrived before both task receipts.");+ }+ if (checked === undefined) {+ return { toolCalls: [{ id: callId, input: { taskIds }, name: "task_peek" }] };+ }+ return allTasksCompleted(checked.output, taskIds)+ ? "TASK-FAN-IN-COMPLETE"+ : "TASK-FAN-IN-WAITING";+}++function allTasksCompleted(output: unknown, expectedTaskIds: readonly string[]): boolean {+ if (output === null || typeof output !== "object") return false;+ const tasks = Reflect.get(output, "tasks");+ if (!Array.isArray(tasks) || tasks.length !== expectedTaskIds.length) return false;+ const byId = new Map(+ tasks+ .filter((task) => task !== null && typeof task === "object")+ .map((task) => [Reflect.get(task, "taskId"), task] as const),+ );+ return expectedTaskIds.every(+ (taskId) => Reflect.get(byId.get(taskId) ?? {}, "status") === "completed",+ );+}++function setupCancelWorker(request: MockModelRequest): MockModelResponse | string {+ if (resultById(request, "task-cancel-worker") === undefined) {+ return {+ toolCalls: [+ {+ id: "task-cancel-worker",+ input: { message: "Run the release gate, then return CANCEL-WORKER-DONE." },+ name: "fanout-worker",+ },+ ],+ };+ }+ return "TASK-CANCEL-READY";+}++function cancelWorkerTask(request: MockModelRequest): MockModelResponse | string {+ const callId = `task-cancel-call-${scenarioUserMessageCount(request)}`;+ if (resultById(request, callId) === undefined) {+ const taskId = findTaskId(resultById(request, "task-cancel-worker")?.output);+ if (taskId === undefined) throw new Error("Cancel scenario has no initial task id.");+ return { toolCalls: [{ id: callId, input: { taskIds: [taskId] }, name: "task_cancel" }] };+ }+ return "TASK-CANCEL-DONE";+}++function startApprovalWorker(+ request: MockModelRequest,+ callId: string,+ completedText: string,+): MockModelResponse | string {+ if (resultById(request, callId) === undefined) {+ return {+ toolCalls: [+ {+ id: callId,+ input: {+ message:+ callId === "task-c7-authorization-worker"+ ? "Run the C7 authorization mode, then return C7-AUTHORIZATION-COMPLETE."+ : callId === "task-hitl-worker"+ ? "Run three approval gates in order, then return CHILD-GATES-COMPLETE."+ : "Run both approval gates in order, then return CHILD-GATES-COMPLETE.",+ },+ name: "approval-worker",+ },+ ],+ };+ }+ return completedText;+}++function startRemoteWorker(request: MockModelRequest): MockModelResponse | string {+ const callId = "task-c8-remote-worker";+ if (resultById(request, callId) === undefined) {+ return {+ toolCalls: [+ {+ id: callId,+ input: {+ message:+ "TASK-C8-REMOTE-CHILD Run the remote gate, then return its principal marker verbatim.",+ },+ name: "remote-loopback",+ },+ ],+ };+ }+ return "TASK-C8-STARTED";+}++function runRemoteGate(request: MockModelRequest): MockModelResponse | string {+ const callId = "task-c8-remote-gate";+ const result = resultById(request, callId);+ if (result === undefined) {+ return {+ toolCalls: [+ {+ id: callId,+ input: { marker: "C8" },+ name: "remote_gate",+ },+ ],+ };+ }+ const marker = findString(result.output, "C8-REMOTE-PRINCIPAL:");+ if (marker === undefined) throw new Error("Remote gate returned no principal marker.");+ return `C8-REMOTE-COMPLETE ${marker}`;+}++function peekTask(+ request: MockModelRequest,+ callIdPrefix: string,+ completedText: string,+ message: string,+): MockModelResponse | string {+ const callId = `${callIdPrefix}-${scenarioUserMessageCount(request)}`;+ if (resultById(request, callId) === undefined) {+ const taskId = TASK_ID_PATTERN.exec(message)?.[0];+ if (taskId === undefined) throw new Error(`Verification message has no task id: ${message}`);+ return { toolCalls: [{ id: callId, input: { taskIds: [taskId] }, name: "task_peek" }] };+ }+ return completedText;+}++function isScenarioMessage(message: string): boolean {+ return !message.startsWith("[Agents]");+}++function scenarioUserMessageCount(request: MockModelRequest): number {+ return request.userMessages.filter(isScenarioMessage).length;+}++function setupBusyWorker(request: MockModelRequest): MockModelResponse | string {+ const delegated = resultById(request, "child-task-exclusivity-initial-worker");+ if (delegated === undefined) {+ return {+ toolCalls: [+ {+ id: "child-task-exclusivity-initial-worker",+ input: { message: "Return BUSY-WORKER-INITIAL." },+ name: "busy-worker",+ },+ ],+ };+ }++ return "CHILD-TASK-EXCLUSIVITY-READY";+}++function startFailingBusyWorker(request: MockModelRequest): MockModelResponse | string {+ const callId = "task-a2-failing-busy-worker";+ if (resultById(request, callId) === undefined) {+ return {+ toolCalls: [+ {+ id: callId,+ input: { message: "TASK-A2-BUSY-WORKER-FAILURE" },+ name: "busy-worker",+ },+ ],+ };+ }+ return "TASK-A2-CHILD-FAILURE-STARTED";+}++function startUnstartableWorker(request: MockModelRequest): MockModelResponse | string {+ const callId = "task-a3-unstartable-worker";+ if (resultById(request, callId) === undefined) {+ return {+ toolCalls: [+ {+ id: callId,+ input: { message: "TASK-A3-INTENTIONAL-START-FAILURE" },+ name: "unstartable-worker",+ },+ ],+ };+ }+ return "TASK-A3-PARENT-SURVIVED";+}++const PARTIAL_FANOUT_CALLS = [+ {+ id: "task-d6-success-first",+ message: "TASK-D6-FIRST-SUCCESS",+ name: "busy-worker",+ },+ {+ id: "task-d6-failure-middle",+ message: "TASK-D6-INTENTIONAL-START-FAILURE",+ name: "unstartable-worker",+ },+ {+ id: "task-d6-success-third",+ message: "TASK-D6-THIRD-SUCCESS",+ name: "busy-worker",+ },+] as const;++function partialFailureFanout(request: MockModelRequest): MockModelResponse | string {+ const pending = PARTIAL_FANOUT_CALLS.filter(({ id }) => resultById(request, id) === undefined);+ if (pending.length > 0) {+ return {+ toolCalls: pending.map(({ id, message, name }) => ({+ id,+ input: { message },+ name,+ })),+ };+ }+ return "TASK-D6-PARTIAL-FANOUT-STARTED";+}++function raceBusyWorker(request: MockModelRequest): MockModelResponse | string {+ const first = resultById(request, "child-task-exclusivity-send-a");+ const second = resultById(request, "child-task-exclusivity-send-b");+ if (first === undefined && second === undefined) {+ const initial = resultById(request, "child-task-exclusivity-initial-worker");+ const taskId = findTaskId(initial?.output);+ if (taskId === undefined) throw new Error("Busy-worker race has no initial task id.");+ return {+ toolCalls: [+ {+ id: "child-task-exclusivity-send-a",+ input: { message: "Return BUSY-WORKER-A.", taskId },+ name: "task_send",+ },+ {+ id: "child-task-exclusivity-send-b",+ input: { message: "Return BUSY-WORKER-B.", taskId },+ name: "task_send",+ },+ ],+ };+ }+ return "CHILD-TASK-EXCLUSIVITY-RACE-DONE";+}++function laterBusyWorker(request: MockModelRequest, message: string): MockModelResponse | string {+ const result = resultById(request, "child-task-exclusivity-later");+ if (result === undefined) {+ const taskId = findTaskId(message);+ if (taskId === undefined) throw new Error("Later exclusivity send has no task id.");+ return {+ toolCalls: [+ {+ id: "child-task-exclusivity-later",+ input: { message: "Return BUSY-WORKER-LATER.", taskId },+ name: "task_send",+ },+ ],+ };+ }+ return "CHILD-TASK-EXCLUSIVITY-LATER-DONE";+}++function continueApprovalWorker(+ request: MockModelRequest,+ message: string,+): MockModelResponse | string {+ const callId = "task-continuation-hitl-send";+ const result = resultById(request, callId);+ if (result === undefined) {+ const taskId = findTaskId(message);+ if (taskId === undefined) throw new Error("Continuation HITL send has no task id.");+ return {+ toolCalls: [+ {+ id: callId,+ input: {+ message:+ "Run four continuation approval gates in order, then return CHILD-GATES-COMPLETE.",+ taskId,+ },+ name: "task_send",+ },+ ],+ };+ }+ return "TASK-CONTINUATION-HITL-STARTED";+}++function resultById(request: MockModelRequest, id: string): MockModelToolResult | undefined {+ return request.toolResults.find((result) => result.id === id);+}++function findTaskId(value: unknown): string | undefined {+ if (typeof value === "string") return TASK_ID_PATTERN.exec(value)?.[0];+ if (Array.isArray(value)) {+ for (const entry of value) {+ const taskId = findTaskId(entry);+ if (taskId !== undefined) return taskId;+ }+ return undefined;+ }+ if (value !== null && typeof value === "object") {+ const taskId = Reflect.get(value, "taskId");+ if (typeof taskId === "string") return taskId;+ for (const entry of Object.values(value)) {+ const nested = findTaskId(entry);+ if (nested !== undefined) return nested;+ }+ }+ return undefined;+}++function findString(value: unknown, prefix: string): string | undefined {+ if (typeof value === "string") return value.startsWith(prefix) ? value : undefined;+ if (Array.isArray(value)) {+ for (const entry of value) {+ const match = findString(entry, prefix);+ if (match !== undefined) return match;+ }+ return undefined;+ }+ if (value !== null && typeof value === "object") {+ for (const entry of Object.values(value)) {+ const match = findString(entry, prefix);+ if (match !== undefined) return match;+ }+ }+ return undefined;+}++const base = e2eAgentConfig();++export default defineAgent({+ ...base,+ experimental: { ...base.experimental, tasks: true },+ // These evals target orchestration, not model planning. Keep every suite+ // deterministic while retaining the workflow-world override from `base`.+ model: mockModel(respond),+ modelContextWindowTokens: 1_000_000,+});diff --git a/e2e/fixtures/fixture-tasks/agent/channels/eve.ts b/e2e/fixtures/fixture-tasks/agent/channels/eve.tsnew file mode 100644index 000000000..3ccf45418--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/channels/eve.ts@@ -0,0 +1,33 @@+import type { AuthFn } from "eve/channels/auth";+import { eveChannel } from "eve/channels/eve";+import type { SessionAuthContext } from "eve/context";++const PRINCIPAL_A = "Bearer e2e-task-operation-a";+const PRINCIPAL_B = "Bearer e2e-task-operation-b";+const REMOTE_CHILD = "Bearer e2e-task-remote-loopback";++function principal(principalId: string): SessionAuthContext {+ return {+ attributes: {},+ authenticator: "e2e-task-fixture",+ issuer: "e2e",+ principalId,+ principalType: "user",+ subject: principalId,+ };+}++const authenticateA: AuthFn<Request> = (request) =>+ request.headers.get("authorization") === PRINCIPAL_A ? principal("operation-user-a") : null;++const authenticateB: AuthFn<Request> = (request) =>+ request.headers.get("authorization") === PRINCIPAL_B ? principal("operation-user-b") : null;++const authenticateRemoteChild: AuthFn<Request> = (request) =>+ request.headers.get("authorization") === REMOTE_CHILD ? principal("remote-http-child") : null;++const authenticateEvalDriver: AuthFn<Request> = () => principal("eval-driver");++export default eveChannel({+ auth: [authenticateA, authenticateB, authenticateRemoteChild, authenticateEvalDriver],+});diff --git a/e2e/fixtures/fixture-tasks/agent/instructions.md b/e2e/fixtures/fixture-tasks/agent/instructions.mdnew file mode 100644index 000000000..f3275e7f0--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/instructions.md@@ -0,0 +1 @@+Follow the scripted model's task-control calls exactly. This fixture validates durable background-task orchestration, not model planning.diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/agent.ts b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/agent.tsnew file mode 100644index 000000000..6bec5b5ea--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/agent.ts@@ -0,0 +1,64 @@+import { defineAgent } from "eve";+import { mockModel, type MockModelRequest, type MockModelResponse } from "eve/evals";++function respond(request: MockModelRequest): MockModelResponse | string {+ const authorization = request.userMessages.some((message) =>+ message.includes("C7 authorization mode"),+ );+ if (authorization) {+ if (request.toolResults.find((result) => result.id === "c7-authorization") === undefined) {+ return {+ toolCalls: [+ { id: "c7-authorization", input: { marker: "C7" }, name: "authorization_gate" },+ ],+ };+ }+ return "C7-AUTHORIZATION-COMPLETE";+ }++ const continuation = request.userMessages.some((message) =>+ message.includes("continuation approval gates"),+ );+ const threeGates =+ continuation ||+ request.userMessages.some((message) => message.includes("three approval gates"));+ const id = (gate: string) => (continuation ? `continuation-${gate}` : `approval-${gate}`);+ const first = request.toolResults.find((result) => result.id === id("first"));+ if (first === undefined) {+ return {+ toolCalls: [{ id: id("first"), input: { marker: "FIRST" }, name: "first_gate" }],+ };+ }++ const second = request.toolResults.find((result) => result.id === id("second"));+ if (second === undefined) {+ return {+ toolCalls: [{ id: id("second"), input: { marker: "SECOND" }, name: "second_gate" }],+ };+ }++ if (threeGates) {+ const third = request.toolResults.find((result) => result.id === id("third"));+ if (third === undefined) {+ return {+ toolCalls: [{ id: id("third"), input: { marker: "THIRD" }, name: "third_gate" }],+ };+ }+ if (continuation) {+ const fourth = request.toolResults.find((result) => result.id === id("fourth"));+ if (fourth === undefined) {+ return {+ toolCalls: [{ id: id("fourth"), input: { marker: "FOURTH" }, name: "fourth_gate" }],+ };+ }+ }+ }++ return "CHILD-GATES-COMPLETE";+}++export default defineAgent({+ description: "Run deterministic approval gates in order.",+ model: mockModel(respond),+ modelContextWindowTokens: 1_000_000,+});diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/authorization_gate.ts b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/authorization_gate.tsnew file mode 100644index 000000000..00304a576--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/authorization_gate.ts@@ -0,0 +1,45 @@+import {+ ConnectionAuthorizationRequiredError,+ defineInteractiveAuthorization,+} from "eve/connections";+import { defineTool } from "eve/tools";+import { z } from "zod";++const AUTHORIZATION_NAME = "c7-task-authorization";+const AUTHORIZATION_CODE = "c7-deterministic-code";+const AUTHORIZATION_TOKEN = "c7-deterministic-token";++const authorization = defineInteractiveAuthorization<{ marker: "C7" }>({+ async getToken() {+ throw new ConnectionAuthorizationRequiredError(AUTHORIZATION_NAME);+ },+ async startAuthorization() {+ return {+ challenge: {+ displayName: "C7 deterministic authorization",+ userCode: AUTHORIZATION_CODE,+ },+ resume: { marker: "C7" },+ };+ },+ async completeAuthorization({ callback, resume }) {+ if (callback.params.code !== AUTHORIZATION_CODE || resume?.marker !== "C7") {+ throw new Error("C7 authorization callback did not preserve its deterministic state.");+ }+ return { token: AUTHORIZATION_TOKEN };+ },+});++export default defineTool({+ description: "Exercises deterministic interactive authorization for a task-owned child.",+ inputSchema: z.object({ marker: z.literal("C7") }),+ async execute({ marker }, ctx) {+ const result = await ctx.getToken(authorization, {+ authKey: AUTHORIZATION_NAME,+ displayName: "C7 deterministic authorization",+ });+ if (result.token !== AUTHORIZATION_TOKEN)+ throw new Error("C7 authorization returned bad token.");+ return { marker: `${marker}-AUTHORIZED` };+ },+});diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/first_gate.ts b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/first_gate.tsnew file mode 100644index 000000000..8c2afeb0f--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/first_gate.ts@@ -0,0 +1,10 @@+import { defineTool } from "eve/tools";+import { once } from "eve/tools/approval";+import { z } from "zod";++export default defineTool({+ description: "First deterministic approval gate.",+ inputSchema: z.object({ marker: z.literal("FIRST") }),+ approval: once(),+ execute: async ({ marker }) => ({ marker, passed: true }),+});diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/fourth_gate.ts b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/fourth_gate.tsnew file mode 100644index 000000000..ec285ecf6--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/fourth_gate.ts@@ -0,0 +1,10 @@+import { defineTool } from "eve/tools";+import { once } from "eve/tools/approval";+import { z } from "zod";++export default defineTool({+ description: "Fourth deterministic approval gate for continued-task HITL routing.",+ inputSchema: z.object({ marker: z.literal("FOURTH") }),+ approval: once(),+ execute: async ({ marker }) => ({ marker, passed: true }),+});diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/second_gate.ts b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/second_gate.tsnew file mode 100644index 000000000..a3ec99c56--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/second_gate.ts@@ -0,0 +1,10 @@+import { defineTool } from "eve/tools";+import { once } from "eve/tools/approval";+import { z } from "zod";++export default defineTool({+ description: "Second deterministic approval gate.",+ inputSchema: z.object({ marker: z.literal("SECOND") }),+ approval: once(),+ execute: async ({ marker }) => ({ marker, passed: true }),+});diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/third_gate.ts b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/third_gate.tsnew file mode 100644index 000000000..6977b743a--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/subagents/approval-worker/tools/third_gate.ts@@ -0,0 +1,10 @@+import { defineTool } from "eve/tools";+import { once } from "eve/tools/approval";+import { z } from "zod";++export default defineTool({+ description: "Third deterministic approval gate for repeated HITL routing.",+ inputSchema: z.object({ marker: z.literal("THIRD") }),+ approval: once(),+ execute: async ({ marker }) => ({ marker, passed: true }),+});diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/busy-worker/agent.ts b/e2e/fixtures/fixture-tasks/agent/subagents/busy-worker/agent.tsnew file mode 100644index 000000000..bc7e21199--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/subagents/busy-worker/agent.ts@@ -0,0 +1,21 @@+import { defineAgent } from "eve";+import { mockModel } from "eve/evals";++export default defineAgent({+ description: "Return one deterministic marker for each message.",+ model: mockModel((request) => {+ const message = request.lastUserMessage ?? "";+ if (message.includes("TASK-A2-BUSY-WORKER-FAILURE")) {+ throw Object.assign(new Error("TASK-A2-BUSY-WORKER-FAILURE"), { statusCode: 400 });+ }+ if (message.includes("BUSY-WORKER-A") || message.includes("BUSY-WORKER-B")) {+ if (!request.toolResults.some((result) => result.id === "exclusivity-hold")) {+ return {+ toolCalls: [{ id: "exclusivity-hold", input: { marker: "HOLD" }, name: "hold" }],+ };+ }+ }+ return `BUSY-WORKER:${message}`;+ }),+ modelContextWindowTokens: 1_000_000,+});diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/busy-worker/tools/hold.ts b/e2e/fixtures/fixture-tasks/agent/subagents/busy-worker/tools/hold.tsnew file mode 100644index 000000000..d843ccc01--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/subagents/busy-worker/tools/hold.ts@@ -0,0 +1,11 @@+import { defineTool } from "eve/tools";+import { z } from "zod";++export default defineTool({+ description: "Keep an admitted continuation nonterminal long enough for a later-turn check.",+ inputSchema: z.object({ marker: z.literal("HOLD") }),+ execute: async ({ marker }) => {+ await new Promise((resolve) => setTimeout(resolve, 5_000));+ return { marker, released: true };+ },+});diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/fanout-worker/agent.ts b/e2e/fixtures/fixture-tasks/agent/subagents/fanout-worker/agent.tsnew file mode 100644index 000000000..b5dccf3dd--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/subagents/fanout-worker/agent.ts@@ -0,0 +1,20 @@+import { defineAgent } from "eve";+import { mockModel, type MockModelRequest, type MockModelResponse } from "eve/evals";++const FAN_IN_MARKER_PATTERN = /TASK-FAN-IN-[12]/u;++function respond(request: MockModelRequest): MockModelResponse | string {+ const fanInMarker = FAN_IN_MARKER_PATTERN.exec(request.lastUserMessage ?? "")?.[0];+ const released = request.toolResults.find((result) => result.name === "release");+ if (released === undefined) {+ return { toolCalls: [{ input: { marker: fanInMarker ?? "RELEASE" }, name: "release" }] };+ }+ if (fanInMarker !== undefined) return `FANOUT-COMPLETE:${fanInMarker}`;+ return `FANOUT-COMPLETE:${request.lastUserMessage ?? ""}`;+}++export default defineAgent({+ description: "Complete one fanout task with its deterministic marker.",+ model: mockModel(respond),+ modelContextWindowTokens: 1_000_000,+});diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/fanout-worker/tools/release.ts b/e2e/fixtures/fixture-tasks/agent/subagents/fanout-worker/tools/release.tsnew file mode 100644index 000000000..a15076a81--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/subagents/fanout-worker/tools/release.ts@@ -0,0 +1,10 @@+import { defineTool } from "eve/tools";+import { once } from "eve/tools/approval";+import { z } from "zod";++export default defineTool({+ description: "Release one fanout task after its parent remains interactive.",+ inputSchema: z.object({ marker: z.enum(["RELEASE", "TASK-FAN-IN-1", "TASK-FAN-IN-2"]) }),+ approval: once(),+ execute: async ({ marker }) => ({ marker, released: true }),+});diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/remote-loopback.ts b/e2e/fixtures/fixture-tasks/agent/subagents/remote-loopback.tsnew file mode 100644index 000000000..b388fe053--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/subagents/remote-loopback.ts@@ -0,0 +1,24 @@+import { defineDynamic, defineRemoteAgent } from "eve";++export default defineDynamic({+ events: {+ "session.started": () =>+ defineRemoteAgent({+ description: "Loopback remote child for deterministic task HITL coverage.",+ url: () =>+ process.env.VERCEL_URL !== undefined && process.env.VERCEL_URL !== ""+ ? `https://${process.env.VERCEL_URL}`+ : (process.env.WORKFLOW_LOCAL_BASE_URL ?? "http://127.0.0.1:3000"),+ headers: () => {+ const headers: Record<string, string> = {+ authorization: "Bearer e2e-task-remote-loopback",+ };+ const bypass = process.env.VERCEL_AUTOMATION_BYPASS_SECRET;+ if (bypass !== undefined && bypass !== "") {+ headers["x-vercel-protection-bypass"] = bypass;+ }+ return headers;+ },+ }),+ },+});diff --git a/e2e/fixtures/fixture-tasks/agent/subagents/unstartable-worker.ts b/e2e/fixtures/fixture-tasks/agent/subagents/unstartable-worker.tsnew file mode 100644index 000000000..24007899c--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/subagents/unstartable-worker.ts@@ -0,0 +1,6 @@+import { defineRemoteAgent } from "eve";++export default defineRemoteAgent({+ description: "Intentionally fail remote-agent fetch for deterministic task evals.",+ url: "http://127.0.0.1:1",+});diff --git a/e2e/fixtures/fixture-tasks/agent/tools/remote_gate.ts b/e2e/fixtures/fixture-tasks/agent/tools/remote_gate.tsnew file mode 100644index 000000000..aba06ee2d--- /dev/null+++ b/e2e/fixtures/fixture-tasks/agent/tools/remote_gate.ts@@ -0,0 +1,14 @@+import { defineTool } from "eve/tools";+import { once } from "eve/tools/approval";+import { z } from "zod";++export default defineTool({+ description: "Remote-child approval gate that reports the resumed HTTP principal.",+ inputSchema: z.object({ marker: z.literal("C8") }),+ approval: once(),+ async execute(_input, ctx) {+ const current = ctx.session.auth.current;+ const principal = current === null ? "none" : `${current.principalType}:${current.principalId}`;+ return { marker: `C8-REMOTE-PRINCIPAL:${principal}` };+ },+});diff --git a/e2e/fixtures/fixture-tasks/evals/README.md b/e2e/fixtures/fixture-tasks/evals/README.mdnew file mode 100644index 000000000..5b92a4349--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/README.md@@ -0,0 +1,38 @@+# Task eval transitions++These evals are executable evidence for the background-task contract. They do+not define that contract.++When sources disagree, use this precedence:++1. `research/tools-as-tasks.md` for settled externally observable intent.+2. Public tool and protocol contracts.+3. `packages/eve/src/tasks/types.ts` and `transitions.ts` for executable+ lifecycle semantics.+4. Evals and lower-level implementation as evidence.++Every `*.eval.ts` case uses `defineTaskEval` and declares exactly one primary+transition. The canonical specification in `task-transition.ts` supplies its+pre-state, semantic input, guards, outcome, post-state, events, and side+effects. `setup` records prerequisite transitions but does not claim coverage+for them.++Transition anchors follow:++```text+<machine>.<entity>.<input>.<outcome>[-<guard>]+```++Anchors describe semantics, never document order. Scenario variants such as+local and remote transport share an anchor and use a filename suffix plus the+`dimensions` declaration. Presentation labels such as A2 or C7 are not stable+identity.++State is factored across lifecycle, outstanding input, executor binding,+dispatch admission, agent occupancy, parent phase, transport, ownership, and+usage. A scenario declares only dimensions relevant to its primary transition;+it must not enumerate their Cartesian product.++`pnpm validate:transitions` enforces declaration and filename identity. The+fixture `typecheck` command runs that validation before building and checking+TypeScript.diff --git a/e2e/fixtures/fixture-tasks/evals/evals.config.ts b/e2e/fixtures/fixture-tasks/evals/evals.config.tsnew file mode 100644index 000000000..d551c179a--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/evals.config.ts@@ -0,0 +1,9 @@+import { defineEvalConfig } from "eve/evals";++export default defineEvalConfig({+ // Task evals dispatch durable child workflows. Serialize evals so they do+ // not starve each other's children past the per-eval timeout (same+ // rationale as agent-subagents).+ maxConcurrency: 1,+ timeoutMs: 120_000,+});diff --git a/e2e/fixtures/fixture-tasks/evals/shared.ts b/e2e/fixtures/fixture-tasks/evals/shared.tsnew file mode 100644index 000000000..5c99fc192--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/shared.ts@@ -0,0 +1,215 @@+import type { EveEvalContext, EveEvalSession, EveEvalTurn, InputRequest } from "eve/evals";+import { satisfies } from "eve/evals/expect";++export type TaskEvalSessionDriver = Pick<+ EveEvalSession,+ "pendingInputRequests" | "respond" | "send" | "sessionId" | "state"+>;++export interface PendingTaskInput {+ readonly observedTurns: readonly EveEvalTurn[];+ readonly request: InputRequest;+ readonly session: TaskEvalSessionDriver;+}++export interface FollowedQueuedTurn {+ readonly observedTurns: readonly EveEvalTurn[];+ readonly session: TaskEvalSessionDriver;+ readonly turn: EveEvalTurn;+}++interface FollowQueuedTurnOptions {+ readonly allowFailedActions?: boolean;+}++export function requireSessionStreamIndex(+ session: TaskEvalSessionDriver,+ operation: string,+): number {+ const state = session.state;+ if (state === undefined) throw new Error(`${operation} has no session state.`);+ return state.streamIndex;+}++/** Waits across server-initiated parent turns for one task-owned input request. */+export async function waitForTaskInput(+ t: EveEvalContext,+ initialSession: TaskEvalSessionDriver,+ toolName: string,+): Promise<PendingTaskInput> {+ let session = initialSession;+ const observedTurns: EveEvalTurn[] = [];+ for (let attempt = 0; attempt < 5; attempt += 1) {+ const matching = session.pendingInputRequests.filter(+ (request) => request.action.toolName === toolName,+ );+ if (matching.length > 1) {+ throw new Error(`Task surfaced ${matching.length} pending requests for tool "${toolName}".`);+ }+ if (matching[0] !== undefined) {+ if (session.pendingInputRequests.length !== 1) {+ throw new Error(+ `Task surfaced unexpected pending input alongside tool "${toolName}": ${session.pendingInputRequests+ .map((request) => request.action.toolName)+ .join(", ")}.`,+ );+ }+ return { observedTurns, request: matching[0], session };+ }+ if (session.pendingInputRequests.length > 0) {+ throw new Error(+ `Expected task input for tool "${toolName}"; received ${session.pendingInputRequests+ .map((request) => request.action.toolName)+ .join(", ")}.`,+ );+ }++ const sessionId = session.sessionId;+ if (sessionId === undefined) throw new Error("Task input wait has no parent session id.");+ const live = t.target.watchTurn(sessionId, {+ startIndex: requireSessionStreamIndex(session, "Task input wait"),+ });+ const turn = await live.result();+ turn.noFailedActions().label(`task input wait ${attempt + 1} has no failed actions`);+ observedTurns.push(turn);+ session = live.session;+ }+ throw new Error(`Task did not surface input for tool "${toolName}" after five turns.`);+}++/** Reads the task receipt attached to a background `subagent.completed` event. */+export function requireBackgroundTaskId(turn: EveEvalTurn): string {+ for (const event of turn.events) {+ if (event.type === "subagent.completed" && event.data.backgroundTask !== undefined) {+ return event.data.backgroundTask.taskId;+ }+ }+ throw new Error("Turn completed without a background task receipt.");+}++/** Returns the exact model-visible task view for one owned task id. */+export function requireTaskView(output: unknown, taskId: string): Record<string, unknown> {+ if (output === null || typeof output !== "object") {+ throw new Error("task_peek did not return an object.");+ }+ const tasks = Reflect.get(output, "tasks");+ if (!Array.isArray(tasks)) throw new Error("task_peek did not return a tasks array.");+ const matches = tasks.filter(+ (candidate) =>+ candidate !== null &&+ typeof candidate === "object" &&+ Reflect.get(candidate, "taskId") === taskId,+ );+ if (matches.length !== 1) {+ throw new Error(`task_peek returned ${matches.length} views for ${taskId}.`);+ }+ return matches[0] as Record<string, unknown>;+}++/** Follows queued server turns until the requested user message owns a turn. */+export async function sendAndFollowQueuedTurn(+ t: EveEvalContext,+ message: string,+ initialSession: TaskEvalSessionDriver = t,+ options: FollowQueuedTurnOptions = {},+): Promise<FollowedQueuedTurn> {+ let session = initialSession;+ let turn = await session.send(message);+ const observedTurns = [turn];+ for (let attempt = 0; attempt < 20; attempt += 1) {+ if (options.allowFailedActions !== true) {+ turn.noFailedActions().label(`queued turn ${attempt + 1} has no failed actions`);+ }+ const received = turn.events.some(+ (event) =>+ event.type === "message.received" && messageText(event.data.message).includes(message),+ );+ if (received) return { observedTurns, session, turn };++ const sessionId = session.sessionId;+ if (sessionId === undefined) throw new Error("Queued turn follow-up has no session id.");+ const live = t.target.watchTurn(sessionId, {+ startIndex: requireSessionStreamIndex(session, "Queued turn follow-up"),+ });+ turn = await live.result();+ observedTurns.push(turn);+ session = live.session;+ }+ throw new Error(`Queued message "${message}" was not received after 20 turns.`);+}++/** Polls the non-blocking task view until the expected task is completed. */+export async function waitForCompletedTask(+ t: EveEvalContext,+ session: TaskEvalSessionDriver,+ verificationMessage: string,+ taskId: string,+): Promise<EveEvalTurn> {+ return await waitForTaskStatus(t, session, verificationMessage, taskId, "completed");+}++/** Polls the non-blocking task view until the expected task reaches `status`. */+export async function waitForTaskStatus(+ t: EveEvalContext,+ session: TaskEvalSessionDriver,+ verificationMessage: string,+ taskId: string,+ status: string,+): Promise<EveEvalTurn> {+ let currentSession = session;+ for (let attempt = 0; attempt < 20; attempt += 1) {+ const followed = await sendAndFollowQueuedTurn(+ t,+ `${verificationMessage} ${taskId}`,+ currentSession,+ );+ for (const [turnIndex, observed] of followed.observedTurns.entries()) {+ observed+ .noFailedActions()+ .label(`task status attempt ${attempt + 1}, turn ${turnIndex + 1} has no failed actions`);+ }+ const turn = followed.turn;+ currentSession = followed.session;+ const peeked = turn.toolCalls.find((call) => call.name === "task_peek");+ if (taskStatus(peeked?.output, taskId) === status) {+ await t.require(+ peeked?.output,+ satisfies(+ (output) => taskStatus(output, taskId) === status,+ `task_peek returns ${status} task ${taskId}`,+ ),+ );+ return turn;+ }+ await t.sleep(100);+ }+ throw new Error(`Task ${taskId} did not reach "${status}" after 20 task_peek attempts.`);+}++function messageText(message: unknown): string {+ if (typeof message === "string") return message;+ if (!Array.isArray(message)) return "";+ return message+ .flatMap((part) =>+ part !== null &&+ typeof part === "object" &&+ Reflect.get(part, "type") === "text" &&+ typeof Reflect.get(part, "text") === "string"+ ? [Reflect.get(part, "text") as string]+ : [],+ )+ .join("\n");+}++function taskStatus(output: unknown, taskId: string): unknown {+ if (output === null || typeof output !== "object") return undefined;+ const tasks = Reflect.get(output, "tasks");+ if (!Array.isArray(tasks)) return undefined;+ const task = tasks.find(+ (candidate) =>+ candidate !== null &&+ typeof candidate === "object" &&+ Reflect.get(candidate, "taskId") === taskId,+ );+ return task === undefined ? undefined : Reflect.get(task, "status");+}diff --git a/e2e/fixtures/fixture-tasks/evals/task-transition.ts b/e2e/fixtures/fixture-tasks/evals/task-transition.tsnew file mode 100644index 000000000..f8a7e4708--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task-transition.ts@@ -0,0 +1,419 @@+import { defineEval, type EveEvalDefinition, type EveEvalInput } from "eve/evals";++type Lifecycle = "absent" | "working" | "input_required" | "completed" | "failed" | "cancelled";+type TransitionOutcome = "accepted" | "noop" | "rejected" | "emitted" | "observed";++interface StatePattern {+ readonly lifecycle?: Lifecycle | readonly Lifecycle[];+ readonly outstandingInput?: "none" | "batch" | "partial-batch" | "authorization";+ readonly executor?: "unbound" | "active" | "parked" | "terminal";+ readonly dispatch?: "absent" | "prepared" | "acknowledged" | "rejected";+ readonly agent?: "available" | "busy";+ readonly parent?: "active" | "parked" | "finalizing";+ readonly ownership?: "owned" | "unowned";+ readonly usage?: "absent" | "retained";+}++type SemanticInput =+ | "delegate"+ | "dispatch-start"+ | "dispatch-batch"+ | "complete"+ | "fail"+ | "cancel"+ | "settle-executor"+ | "require-input"+ | "answer-input"+ | "authorization-callback"+ | "start-turn"+ | "task-send"+ | "task-peek"+ | "task-notification"+ | "parent-message";++type SemanticEvent =+ | "background-receipt"+ | "input-requested"+ | "authorization-required"+ | "authorization-completed"+ | "task-ready-notification";++type SemanticSideEffect =+ | "task-view-append"+ | "task-index-write"+ | "child-dispatch"+ | "child-delivery"+ | "child-abort"+ | "parent-model-step"+ | "parent-wake"+ | "task-view-read"+ | "authorization-request";++interface TransitionSpec {+ readonly preState: StatePattern;+ readonly input: SemanticInput;+ readonly guards: readonly string[];+ readonly expected: {+ readonly outcome: TransitionOutcome;+ readonly postState: StatePattern;+ readonly events: {+ readonly emitted?: readonly SemanticEvent[];+ readonly suppressed?: readonly SemanticEvent[];+ };+ readonly sideEffects: {+ readonly executed?: readonly SemanticSideEffect[];+ readonly suppressed?: readonly SemanticSideEffect[];+ };+ };+}++function transition(spec: TransitionSpec): TransitionSpec {+ return spec;+}++/**+ * Stable semantic identities for the task acceptance contract. Presentation+ * order and transport variants never participate in anchor identity.+ */+export const TASK_TRANSITIONS = {+ "task.lifecycle.complete.accepted-nonterminal": transition({+ preState: { lifecycle: ["working", "input_required"] },+ input: "complete",+ guards: ["task-is-nonterminal"],+ expected: {+ outcome: "accepted",+ postState: { lifecycle: "completed", outstandingInput: "none" },+ events: { emitted: ["task-ready-notification"] },+ sideEffects: { executed: ["task-view-append", "parent-wake"] },+ },+ }),+ "task.lifecycle.fail.accepted-nonterminal": transition({+ preState: { lifecycle: ["working", "input_required"] },+ input: "fail",+ guards: ["task-is-nonterminal"],+ expected: {+ outcome: "accepted",+ postState: { lifecycle: "failed", outstandingInput: "none" },+ events: { emitted: ["task-ready-notification"] },+ sideEffects: { executed: ["task-view-append", "parent-wake"] },+ },+ }),+ "task.lifecycle.cancel.accepted-nonterminal": transition({+ preState: { lifecycle: ["working", "input_required"] },+ input: "cancel",+ guards: ["task-is-nonterminal"],+ expected: {+ outcome: "accepted",+ postState: { lifecycle: "cancelled", outstandingInput: "none" },+ events: { emitted: ["task-ready-notification"] },+ sideEffects: { executed: ["task-view-append", "parent-wake", "child-abort"] },+ },+ }),+ "task.lifecycle.cancel.noop-already-cancelled": transition({+ preState: { lifecycle: "cancelled" },+ input: "cancel",+ guards: ["task-is-already-cancelled"],+ expected: {+ outcome: "noop",+ postState: { lifecycle: "cancelled" },+ events: { suppressed: ["task-ready-notification"] },+ sideEffects: { suppressed: ["task-view-append", "parent-wake", "child-abort"] },+ },+ }),+ "task.lifecycle.result.rejected-already-terminal": transition({+ preState: { lifecycle: ["completed", "failed", "cancelled"] },+ input: "complete",+ guards: ["task-is-terminal"],+ expected: {+ outcome: "rejected",+ postState: { lifecycle: ["completed", "failed", "cancelled"] },+ events: { suppressed: ["task-ready-notification"] },+ sideEffects: { suppressed: ["task-view-append", "parent-wake"] },+ },+ }),+ "task.executor.settle.accepted-terminal": transition({+ preState: { lifecycle: ["completed", "failed", "cancelled"], executor: "parked" },+ input: "settle-executor",+ guards: ["task-is-terminal", "executor-is-not-terminal"],+ expected: {+ outcome: "accepted",+ postState: { lifecycle: ["completed", "failed", "cancelled"], executor: "terminal" },+ events: {},+ sideEffects: { executed: ["task-view-append"] },+ },+ }),+ "task.executor.settle.rejected-nonterminal": transition({+ preState: { lifecycle: ["working", "input_required"] },+ input: "settle-executor",+ guards: ["task-is-nonterminal"],+ expected: {+ outcome: "rejected",+ postState: { lifecycle: ["working", "input_required"] },+ events: {},+ sideEffects: { suppressed: ["task-view-append"] },+ },+ }),+ "task.input.require.accepted-valid-batch": transition({+ preState: { lifecycle: ["working", "input_required"] },+ input: "require-input",+ guards: ["batch-is-nonempty", "request-ids-are-unique"],+ expected: {+ outcome: "accepted",+ postState: { lifecycle: "input_required", outstandingInput: "batch" },+ events: { emitted: ["input-requested", "task-ready-notification"] },+ sideEffects: { executed: ["task-view-append", "parent-wake"] },+ },+ }),+ "task.input.require.rejected-invalid-batch": transition({+ preState: { lifecycle: ["working", "input_required"] },+ input: "require-input",+ guards: ["batch-is-empty-or-request-ids-are-invalid"],+ expected: {+ outcome: "rejected",+ postState: { lifecycle: ["working", "input_required"] },+ events: { suppressed: ["input-requested", "task-ready-notification"] },+ sideEffects: { suppressed: ["task-view-append", "parent-wake"] },+ },+ }),+ "task.input.answer.accepted-partial": transition({+ preState: { lifecycle: "input_required", outstandingInput: "batch" },+ input: "answer-input",+ guards: ["some-request-ids-match", "some-requests-remain"],+ expected: {+ outcome: "accepted",+ postState: { lifecycle: "input_required", outstandingInput: "partial-batch" },+ events: {},+ sideEffects: { executed: ["child-delivery", "task-view-append"] },+ },+ }),+ "task.input.answer.accepted-complete": transition({+ preState: { lifecycle: "input_required", outstandingInput: "batch" },+ input: "answer-input",+ guards: ["all-outstanding-request-ids-match"],+ expected: {+ outcome: "accepted",+ postState: { lifecycle: "working", outstandingInput: "none" },+ events: {},+ sideEffects: {+ executed: ["child-delivery", "task-view-append"],+ suppressed: ["parent-model-step"],+ },+ },+ }),+ "task.input.answer.noop-stale": transition({+ preState: { lifecycle: "input_required", outstandingInput: "batch" },+ input: "answer-input",+ guards: ["no-request-id-matches"],+ expected: {+ outcome: "noop",+ postState: { lifecycle: "input_required", outstandingInput: "batch" },+ events: {},+ sideEffects: { suppressed: ["child-delivery", "task-view-append", "parent-model-step"] },+ },+ }),+ "task.input.route.observed-stale-unrouted": transition({+ preState: { lifecycle: "input_required", outstandingInput: "batch" },+ input: "answer-input",+ guards: ["request-route-is-no-longer-registered"],+ expected: {+ outcome: "observed",+ postState: { lifecycle: "input_required", outstandingInput: "batch" },+ events: {},+ sideEffects: {+ executed: ["parent-model-step"],+ suppressed: ["child-delivery", "task-view-append"],+ },+ },+ }),+ "task.executor.start-turn.accepted-unbound": transition({+ preState: { lifecycle: ["working", "input_required"], executor: "unbound" },+ input: "start-turn",+ guards: ["task-id-matches", "child-session-is-unbound"],+ expected: {+ outcome: "accepted",+ postState: { lifecycle: ["working", "input_required"], executor: "active" },+ events: {},+ sideEffects: { executed: ["task-view-append"] },+ },+ }),+ "task.executor.start-turn.noop-same-turn": transition({+ preState: { lifecycle: ["working", "input_required"], executor: "active" },+ input: "start-turn",+ guards: ["task-id-matches", "child-session-and-turn-match"],+ expected: {+ outcome: "noop",+ postState: { lifecycle: ["working", "input_required"], executor: "active" },+ events: {},+ sideEffects: { suppressed: ["task-view-append"] },+ },+ }),+ "task.executor.start-turn.rejected-session-mismatch": transition({+ preState: { lifecycle: ["working", "input_required"], executor: "active" },+ input: "start-turn",+ guards: ["task-id-matches", "child-session-differs"],+ expected: {+ outcome: "rejected",+ postState: { lifecycle: ["working", "input_required"], executor: "active" },+ events: {},+ sideEffects: { suppressed: ["task-view-append"] },+ },+ }),+ "task.dispatch.start.accepted-acknowledged": transition({+ preState: { lifecycle: "absent", dispatch: "absent", ownership: "unowned" },+ input: "dispatch-start",+ guards: ["child-acknowledges-private-address"],+ expected: {+ outcome: "accepted",+ postState: { lifecycle: "working", dispatch: "acknowledged", ownership: "owned" },+ events: { emitted: ["background-receipt"] },+ sideEffects: { executed: ["child-dispatch", "task-index-write"] },+ },+ }),+ "task.dispatch.start.rejected-unreachable": transition({+ preState: { lifecycle: "absent", dispatch: "absent", ownership: "unowned" },+ input: "dispatch-start",+ guards: ["child-start-is-unreachable"],+ expected: {+ outcome: "rejected",+ postState: { lifecycle: "absent", dispatch: "rejected", ownership: "unowned" },+ events: { suppressed: ["background-receipt"] },+ sideEffects: { executed: ["child-dispatch"], suppressed: ["task-index-write"] },+ },+ }),+ "task.dispatch-batch.start.accepted-partial-failure": transition({+ preState: { dispatch: "absent" },+ input: "dispatch-batch",+ guards: ["one-member-is-unreachable", "other-members-are-reachable"],+ expected: {+ outcome: "accepted",+ postState: { dispatch: "acknowledged", ownership: "owned" },+ events: { emitted: ["background-receipt"] },+ sideEffects: { executed: ["child-dispatch", "task-index-write"] },+ },+ }),+ "task.agent.send.accepted-terminal-available": transition({+ preState: { lifecycle: ["completed", "failed", "cancelled"], agent: "available" },+ input: "task-send",+ guards: ["source-task-is-terminal", "agent-has-no-nonterminal-task"],+ expected: {+ outcome: "accepted",+ postState: { lifecycle: "working", agent: "busy", ownership: "owned" },+ events: { emitted: ["background-receipt"] },+ sideEffects: { executed: ["task-index-write", "child-delivery"] },+ },+ }),+ "task.agent.send.rejected-agent-busy": transition({+ preState: { agent: "busy" },+ input: "task-send",+ guards: ["agent-has-a-nonterminal-task"],+ expected: {+ outcome: "rejected",+ postState: { agent: "busy" },+ events: { suppressed: ["background-receipt"] },+ sideEffects: { suppressed: ["task-index-write", "child-delivery"] },+ },+ }),+ "task.authorization.callback.accepted-current-attempt": transition({+ preState: { lifecycle: "input_required", outstandingInput: "authorization" },+ input: "authorization-callback",+ guards: ["callback-matches-current-attempt"],+ expected: {+ outcome: "accepted",+ postState: { lifecycle: "working", outstandingInput: "none" },+ events: { emitted: ["authorization-completed"] },+ sideEffects: { executed: ["authorization-request", "task-view-append"] },+ },+ }),+ "task.control.peek.observed-owned": transition({+ preState: { ownership: "owned" },+ input: "task-peek",+ guards: ["parent-owns-every-requested-task"],+ expected: {+ outcome: "observed",+ postState: { ownership: "owned" },+ events: {},+ sideEffects: { executed: ["task-view-read"], suppressed: ["task-view-append"] },+ },+ }),+ "task.parent.wake.emitted-ready": transition({+ preState: { lifecycle: ["input_required", "completed", "failed", "cancelled"] },+ input: "task-notification",+ guards: ["task-entered-a-ready-status"],+ expected: {+ outcome: "emitted",+ postState: { lifecycle: ["input_required", "completed", "failed", "cancelled"] },+ events: { emitted: ["task-ready-notification"] },+ sideEffects: { executed: ["parent-wake"] },+ },+ }),+ "task.parent-interaction.send.accepted-live-children": transition({+ preState: { agent: "busy", parent: "active" },+ input: "parent-message",+ guards: ["background-children-remain-nonterminal"],+ expected: {+ outcome: "accepted",+ postState: { agent: "busy", parent: "active" },+ events: {},+ sideEffects: { executed: ["parent-model-step"] },+ },+ }),+ "task.join.evaluate.observed-partial": transition({+ preState: { lifecycle: ["completed", "input_required"] },+ input: "task-peek",+ guards: ["at-least-one-task-is-nonterminal"],+ expected: {+ outcome: "observed",+ postState: { lifecycle: ["completed", "input_required"] },+ events: {},+ sideEffects: { executed: ["task-view-read", "parent-model-step"] },+ },+ }),+ "task.join.evaluate.observed-all-terminal": transition({+ preState: { lifecycle: "completed" },+ input: "task-peek",+ guards: ["all-joined-tasks-are-terminal"],+ expected: {+ outcome: "observed",+ postState: { lifecycle: "completed" },+ events: {},+ sideEffects: { executed: ["task-view-read", "parent-model-step"] },+ },+ }),+} as const satisfies Record<string, TransitionSpec>;++export type TaskTransitionAnchor = keyof typeof TASK_TRANSITIONS;++export interface TaskTransitionDeclaration {+ readonly primary: TaskTransitionAnchor;+ readonly setup?: readonly TaskTransitionAnchor[];+ readonly dimensions: {+ readonly transport: "local" | "remote" | "mixed" | "not-applicable";+ readonly parentPhase?: "active" | "parked" | "finalizing";+ };+}++type TaskEvalInput = Omit<EveEvalInput, "metadata"> & {+ readonly metadata?: Readonly<Record<string, unknown>>;+ readonly transition: TaskTransitionDeclaration;+};++/** Defines one task eval and expands its transition into executable-run metadata. */+export function defineTaskEval(input: TaskEvalInput): EveEvalDefinition {+ const { metadata, transition: declaration, ...evalInput } = input;+ const spec = TASK_TRANSITIONS[declaration.primary];+ return defineEval({+ ...evalInput,+ metadata: {+ ...metadata,+ transition: {+ anchor: declaration.primary,+ dimensions: declaration.dimensions,+ expected: spec.expected,+ guards: spec.guards,+ input: spec.input,+ preState: spec.preState,+ setup: declaration.setup ?? [],+ },+ },+ });+}diff --git a/e2e/fixtures/fixture-tasks/evals/task.agent.send.accepted-terminal-available.eval.ts b/e2e/fixtures/fixture-tasks/evals/task.agent.send.accepted-terminal-available.eval.tsnew file mode 100644index 000000000..ce79a5c9f--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task.agent.send.accepted-terminal-available.eval.ts@@ -0,0 +1,112 @@+import { satisfies } from "eve/evals/expect";++import {+ requireBackgroundTaskId,+ requireTaskView,+ sendAndFollowQueuedTurn,+ waitForCompletedTask,+ waitForTaskInput,+} from "./shared.js";+import { defineTaskEval } from "./task-transition.js";++/** A reused child must route every new-turn event through the new owning task. */+export default defineTaskEval({+ description: "task_send rebinds a reused child's HITL events to the new task lifecycle.",+ transition: {+ primary: "task.agent.send.accepted-terminal-available",+ setup: ["task.input.answer.accepted-complete"],+ dimensions: { transport: "local" },+ },+ async test(t) {+ t.log("starting first task");+ const setup = await t.send("TASK-CONTINUATION-HITL-SETUP");+ setup.expectOk();+ const firstTaskId = requireBackgroundTaskId(setup);++ const first = await waitForTaskInput(t, t, "first_gate");+ t.log("answering first task gate one");+ const firstAnswered = await first.session.respond([+ {+ optionId: "approve",+ requestId: first.request.requestId,+ },+ ]);+ firstAnswered.notEvent("step.started");+ const second = await waitForTaskInput(t, first.session, "second_gate");+ t.log("answering first task gate two");+ await second.session.respond([{ optionId: "approve", requestId: second.request.requestId }]);+ const firstTerminal = await waitForCompletedTask(+ t,+ second.session,+ "TASK-HITL-VERIFY",+ firstTaskId,+ );+ const firstView = requireTaskView(+ firstTerminal.requireToolCall("task_peek").output,+ firstTaskId,+ );+ t.log("first task completed; sending continuation");++ const continued = await sendAndFollowQueuedTurn(+ t,+ `TASK-CONTINUATION-HITL-SEND ${firstTaskId}`,+ second.session,+ );+ continued.turn.expectOk();+ const send = continued.turn.toolCalls.find((call) => call.name === "task_send");+ const secondTaskId =+ send?.output !== null && typeof send?.output === "object"+ ? Reflect.get(send.output, "taskId")+ : undefined;+ if (typeof secondTaskId !== "string") throw new Error("task_send returned no new task id.");+ await t.require(+ secondTaskId,+ satisfies((taskId) => taskId !== firstTaskId, "task_send creates a new task identity"),+ );+ t.log("continuation admitted; waiting for rebound HITL");+ const continuedThird = await waitForTaskInput(t, continued.session, "third_gate");+ const continuedThirdAnswer = await continuedThird.session.respond([+ {+ optionId: "approve",+ requestId: continuedThird.request.requestId,+ },+ ]);+ continuedThirdAnswer.notEvent("step.started");+ const continuedFourth = await waitForTaskInput(t, continuedThird.session, "fourth_gate");+ await continuedFourth.session.respond([+ {+ optionId: "approve",+ requestId: continuedFourth.request.requestId,+ },+ ]);+ const secondTerminal = await waitForCompletedTask(+ t,+ continuedFourth.session,+ "TASK-HITL-VERIFY",+ secondTaskId,+ );+ const secondView = requireTaskView(+ secondTerminal.requireToolCall("task_peek").output,+ secondTaskId,+ );+ const firstAgentId = requireAgentId(firstView);+ await t.require(+ secondView,+ satisfies(+ (view: Record<string, unknown>) => requireAgentId(view) === firstAgentId,+ "the new task retains the reused child's agent identity",+ ),+ );+ t.noFailedActions();+ },+});++function requireAgentId(view: Record<string, unknown>): string {+ const metadata = Reflect.get(view, "metadata");+ const agentId =+ metadata !== null && typeof metadata === "object"+ ? Reflect.get(metadata, "agentId")+ : undefined;+ if (typeof agentId !== "string") throw new Error("Task view has no agent id.");+ return agentId;+}diff --git a/e2e/fixtures/fixture-tasks/evals/task.agent.send.rejected-agent-busy.eval.ts b/e2e/fixtures/fixture-tasks/evals/task.agent.send.rejected-agent-busy.eval.tsnew file mode 100644index 000000000..f1d874f42--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task.agent.send.rejected-agent-busy.eval.ts@@ -0,0 +1,110 @@+import { type EveEvalToolCall } from "eve/evals";+import { satisfies } from "eve/evals/expect";++import { defineTaskEval } from "./task-transition.js";+import {+ requireBackgroundTaskId,+ sendAndFollowQueuedTurn,+ waitForCompletedTask,+} from "./shared.js";++/** A persistent child with a nonterminal task rejects every competing send. */+export default defineTaskEval({+ description:+ "One competing task_send is admitted; same-batch and later sends are rejected as AGENT_BUSY without task receipts.",+ transition: {+ primary: "task.agent.send.rejected-agent-busy",+ setup: [+ "task.dispatch.start.accepted-acknowledged",+ "task.lifecycle.complete.accepted-nonterminal",+ "task.agent.send.accepted-terminal-available",+ ],+ dimensions: { transport: "local", parentPhase: "active" },+ },+ async test(t) {+ const setup = await t.send("CHILD-TASK-EXCLUSIVITY-SETUP");+ setup.expectOk();+ setup.messageIncludes("CHILD-TASK-EXCLUSIVITY-READY");+ const initialTaskId = requireBackgroundTaskId(setup);+ await waitForCompletedTask(t, t, "CHILD-TASK-EXCLUSIVITY-VERIFY", initialTaskId);++ const race = await sendAndFollowQueuedTurn(t, "CHILD-TASK-EXCLUSIVITY-RACE", t, {+ allowFailedActions: true,+ });+ const raced = race.turn;+ raced.expectOk();+ raced.messageIncludes("CHILD-TASK-EXCLUSIVITY-RACE-DONE");++ const sends = raced.toolCalls.filter((call) => call.name === "task_send");+ const admitted = sends.filter(hasTaskReceipt);+ await t.require(+ sends,+ satisfies(+ (calls: readonly EveEvalToolCall[]) =>+ calls.length === 2 &&+ calls.filter(hasTaskReceipt).length === 1 &&+ calls.filter((call) => isBusyRejection(call.output)).length === 1,+ "one competing send is admitted and one is rejected without a task receipt",+ ),+ );++ const admittedTaskId = taskIdFromReceipt(admitted[0]);+ const rejected = sends.find((call) => isBusyRejection(call.output));+ await t.require(+ rejected?.output,+ satisfies(+ (output) => isBusyRejection(output, admittedTaskId),+ "the same-batch rejection identifies the admitted task and creates no task receipt",+ ),+ );++ const later = await sendAndFollowQueuedTurn(+ t,+ `CHILD-TASK-EXCLUSIVITY-LATER ${initialTaskId}`,+ race.session,+ { allowFailedActions: true },+ );+ const laterSends = later.turn.toolCalls.filter((call) => call.name === "task_send");+ await t.require(+ laterSends,+ satisfies(+ (calls: readonly EveEvalToolCall[]) =>+ calls.length === 1 && isBusyRejection(calls[0]?.output, admittedTaskId),+ "the later rejection creates no task receipt while the admitted task is nonterminal",+ ),+ );++ await waitForCompletedTask(t, later.session, "CHILD-TASK-EXCLUSIVITY-VERIFY", admittedTaskId);+ },+});++function hasTaskReceipt(call: EveEvalToolCall): boolean {+ return (+ call.output !== null &&+ typeof call.output === "object" &&+ typeof Reflect.get(call.output, "agentId") === "string" &&+ typeof Reflect.get(call.output, "taskId") === "string" &&+ Reflect.get(call.output, "status") === "working"+ );+}++function taskIdFromReceipt(call: EveEvalToolCall | undefined): string {+ const output = call?.output;+ const taskId =+ output !== null && typeof output === "object" ? Reflect.get(output, "taskId") : null;+ if (typeof taskId !== "string") throw new Error("No admitted continuation task id.");+ return taskId;+}++function isBusyRejection(output: unknown, activeTaskId?: string): boolean {+ if (output === null || typeof output !== "object") return false;+ const keys = Object.keys(output);+ const message = Reflect.get(output, "message");+ return (+ keys.length === 1 &&+ keys[0] === "message" &&+ typeof message === "string" &&+ message.startsWith("AGENT_BUSY") &&+ (activeTaskId === undefined || message.includes(activeTaskId))+ );+}diff --git a/e2e/fixtures/fixture-tasks/evals/task.authorization.callback.accepted-current-attempt.eval.ts b/e2e/fixtures/fixture-tasks/evals/task.authorization.callback.accepted-current-attempt.eval.tsnew file mode 100644index 000000000..7a1b2dc63--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task.authorization.callback.accepted-current-attempt.eval.ts@@ -0,0 +1,169 @@+import type { EveEvalContext, EveEvalTurn } from "eve/evals";+import { equals, satisfies } from "eve/evals/expect";++import {+ requireBackgroundTaskId,+ requireSessionStreamIndex,+ requireTaskView,+ sendAndFollowQueuedTurn,+ type TaskEvalSessionDriver,+ waitForCompletedTask,+} from "./shared.js";+import { defineTaskEval } from "./task-transition.js";++const AUTHORIZATION_CODE = "c7-deterministic-code";+const AUTHORIZATION_NAME = "c7-task-authorization";++type AuthorizationEvent = Extract<+ EveEvalTurn["events"][number],+ { readonly type: "authorization.completed" | "authorization.required" }+>;++/** A task-owned interactive authorization keeps its distinct lifecycle and task blocker. */+export default defineTaskEval({+ description:+ "A background child surfaces interactive authorization, resumes through its webhook, and completes without masquerading as ordinary input.",+ transition: {+ primary: "task.authorization.callback.accepted-current-attempt",+ dimensions: { transport: "local" },+ },+ async test(t) {+ const started = await t.send("TASK-C7-AUTHORIZATION");+ started.expectOk();+ started.messageIncludes("TASK-C7-STARTED");+ started.event("subagent.completed", {+ count: 1,+ data: { backgroundTask: { status: "working" }, subagentName: "approval-worker" },+ });+ const taskId = requireBackgroundTaskId(started);++ const required = await waitForAuthorizationEvent(t, t, started, "authorization.required");+ required.turn.event("authorization.required", {+ count: 1,+ data: { authorization: { userCode: AUTHORIZATION_CODE }, name: AUTHORIZATION_NAME },+ });+ required.turn.notEvent("input.requested");++ const peeked = await sendAndFollowQueuedTurn(+ t,+ `TASK-C7-AUTHORIZATION-VERIFY ${taskId}`,+ required.session,+ );+ const taskCall = peeked.turn.requireToolCall("task_peek", { input: { taskIds: [taskId] } });+ const blockedView = requireTaskView(taskCall.output, taskId);+ const authorizationRequestId = `task:authorization:${required.event.data.attemptId ?? AUTHORIZATION_NAME}`;+ await t.require(+ blockedView,+ satisfies(+ (view: Record<string, unknown>) =>+ Reflect.get(view, "status") === "input_required" &&+ hasMetadata(view, "approval-worker", "local") &&+ hasAuthorizationBlocker(view, authorizationRequestId),+ "task view exposes only the reserved authorization blocker",+ ),+ );++ const webhookUrl = required.event.data.webhookUrl;+ if (webhookUrl === undefined) throw new Error("C7 authorization.required had no webhook URL.");+ const callback = new URL(webhookUrl);+ callback.searchParams.set("code", AUTHORIZATION_CODE);+ const callbackResponse = await fetch(callback, {+ method: "GET",+ });+ await t.require(callbackResponse.status, equals(200));++ const completed = await waitForAuthorizationEvent(+ t,+ peeked.session,+ undefined,+ "authorization.completed",+ );+ completed.turn.event("authorization.completed", {+ count: 1,+ data: { name: AUTHORIZATION_NAME, outcome: "authorized" },+ });++ const terminal = await waitForCompletedTask(+ t,+ completed.session,+ "TASK-C7-AUTHORIZATION-VERIFY",+ taskId,+ );+ const terminalView = requireTaskView(terminal.requireToolCall("task_peek").output, taskId);+ await t.require(+ terminalView,+ satisfies(+ (view: Record<string, unknown>) =>+ Reflect.get(view, "status") === "completed" &&+ Reflect.get(Reflect.get(view, "lastOutput") ?? {}, "type") === "result" &&+ Reflect.get(Reflect.get(view, "lastOutput") ?? {}, "data") ===+ "C7-AUTHORIZATION-COMPLETE",+ "authorized child reaches its deterministic terminal output",+ ),+ );+ t.event("authorization.required", {+ count: 1,+ data: { authorization: { userCode: AUTHORIZATION_CODE }, name: AUTHORIZATION_NAME },+ });+ t.event("authorization.completed", {+ count: 1,+ data: { name: AUTHORIZATION_NAME, outcome: "authorized" },+ });+ t.eventOrder([{ type: "authorization.required" }, { type: "authorization.completed" }]);+ t.notEvent("input.requested");+ t.noFailedActions();+ },+});++function hasAuthorizationBlocker(+ view: Record<string, unknown>,+ authorizationRequestId: string,+): boolean {+ const requests = Reflect.get(view, "inputRequests");+ return (+ Array.isArray(requests) &&+ requests.length === 1 &&+ Reflect.get(requests[0] ?? {}, "requestId") === authorizationRequestId &&+ Reflect.get(requests[0] ?? {}, "blockedOn") === "authorization"+ );+}++function hasMetadata(view: Record<string, unknown>, name: string, mode: string): boolean {+ const metadata = Reflect.get(view, "metadata");+ return (+ metadata !== null &&+ typeof metadata === "object" &&+ Reflect.get(metadata, "name") === name &&+ Reflect.get(metadata, "mode") === mode+ );+}++async function waitForAuthorizationEvent<TType extends AuthorizationEvent["type"]>(+ t: EveEvalContext,+ initialSession: TaskEvalSessionDriver,+ initialTurn: EveEvalTurn | undefined,+ type: TType,+): Promise<{+ readonly event: Extract<AuthorizationEvent, { readonly type: TType }>;+ readonly session: TaskEvalSessionDriver;+ readonly turn: EveEvalTurn;+}> {+ let session = initialSession;+ let turn = initialTurn;+ for (let attempt = 0; attempt < 10; attempt += 1) {+ const event = turn?.events.find(+ (candidate): candidate is Extract<AuthorizationEvent, { readonly type: TType }> =>+ candidate.type === type,+ );+ if (event !== undefined) return { event, session, turn: turn! };++ const sessionId = session.sessionId;+ if (sessionId === undefined) throw new Error("Authorization event wait has no session id.");+ const live = t.target.watchTurn(sessionId, {+ startIndex: requireSessionStreamIndex(session, "Authorization event wait"),+ });+ turn = await live.result();+ session = live.session;+ }+ throw new Error(`Task did not surface ${type} after ten turns.`);+}diff --git a/e2e/fixtures/fixture-tasks/evals/task.dispatch-batch.start.accepted-partial-failure.eval.ts b/e2e/fixtures/fixture-tasks/evals/task.dispatch-batch.start.accepted-partial-failure.eval.tsnew file mode 100644index 000000000..da9aebac9--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task.dispatch-batch.start.accepted-partial-failure.eval.ts@@ -0,0 +1,143 @@+import { type EveEvalTurn } from "eve/evals";+import { satisfies } from "eve/evals/expect";++import { requireTaskView, waitForCompletedTask } from "./shared.js";+import { defineTaskEval } from "./task-transition.js";++const FIRST_CALL_ID = "task-d6-success-first";+const FAILED_CALL_ID = "task-d6-failure-middle";+const THIRD_CALL_ID = "task-d6-success-third";++/** A failed middle dispatch does not stop later siblings or enter the task index. */+export default defineTaskEval({+ description:+ "An ordered success/failure/success fanout admits both siblings and excludes the failed start.",+ transition: {+ primary: "task.dispatch-batch.start.accepted-partial-failure",+ dimensions: { transport: "mixed", parentPhase: "active" },+ },+ async test(t) {+ const started = await t.send("TASK-D6-PARTIAL-FANOUT-FAILURE");+ started.expectOk();+ started.messageIncludes("TASK-D6-PARTIAL-FANOUT-STARTED");+ started.eventsSatisfy("dispatch results preserve success/failure/success order", (events) => {+ const callIds = events.flatMap((event) =>+ event.type === "action.result" && event.data.result.kind === "subagent-result"+ ? [event.data.result.callId]+ : [],+ );+ return callIds.join(",") === [FIRST_CALL_ID, FAILED_CALL_ID, THIRD_CALL_ID].join(",");+ });++ const receipts = backgroundReceipts(started);+ await t.require(+ receipts,+ satisfies(+ (values: readonly BackgroundReceipt[]) =>+ values.length === 2 &&+ values[0]?.callId === FIRST_CALL_ID &&+ values[1]?.callId === THIRD_CALL_ID &&+ values[0].taskId !== values[1].taskId,+ "first and third entries return distinct working task receipts",+ ),+ );+ const firstTaskId = requireReceiptTaskId(receipts, FIRST_CALL_ID);+ const thirdTaskId = requireReceiptTaskId(receipts, THIRD_CALL_ID);+ started.eventsSatisfy(+ "failed middle start has no task id, receipt, or index admission",+ (events) => {+ const failed = events.filter(+ (event) =>+ event.type === "action.result" &&+ event.data.result.kind === "subagent-result" &&+ event.data.result.callId === FAILED_CALL_ID,+ );+ const event = failed[0];+ if (event?.type !== "action.result") return false;+ const result = event.data.result;+ if (result.kind !== "subagent-result") return false;+ const output = result.output;+ return (+ failed.length === 1 &&+ event.data.status === "failed" &&+ result.origin === "dispatch" &&+ output !== null &&+ typeof output === "object" &&+ Reflect.get(output, "code") === "REMOTE_AGENT_START_FAILED" &&+ !Object.hasOwn(output, "taskId") &&+ !events.some(+ (event) => event.type === "subagent.called" && event.data.callId === FAILED_CALL_ID,+ ) &&+ !events.some(+ (event) =>+ event.type === "subagent.completed" &&+ event.data.callId === FAILED_CALL_ID &&+ event.data.backgroundTask !== undefined,+ )+ );+ },+ );+ started.event("subagent.completed", {+ count: 1,+ data: {+ backgroundTask: { status: "working", taskId: thirdTaskId },+ callId: THIRD_CALL_ID,+ subagentName: "busy-worker",+ },+ });++ const firstCompleted = await waitForCompletedTask(+ t,+ t,+ "TASK-D6-PARTIAL-FANOUT-VERIFY",+ firstTaskId,+ );+ assertCompletedBusyWorker(firstCompleted, firstTaskId, "TASK-D6-FIRST-SUCCESS");+ const thirdCompleted = await waitForCompletedTask(+ t,+ t,+ "TASK-D6-PARTIAL-FANOUT-VERIFY",+ thirdTaskId,+ );+ assertCompletedBusyWorker(thirdCompleted, thirdTaskId, "TASK-D6-THIRD-SUCCESS");+ },+});++interface BackgroundReceipt {+ readonly callId: string;+ readonly taskId: string;+}++function backgroundReceipts(turn: EveEvalTurn): readonly BackgroundReceipt[] {+ return turn.events.flatMap((event) =>+ event.type === "subagent.completed" && event.data.backgroundTask !== undefined+ ? [{ callId: event.data.callId, taskId: event.data.backgroundTask.taskId }]+ : [],+ );+}++function requireReceiptTaskId(receipts: readonly BackgroundReceipt[], callId: string): string {+ const matching = receipts.filter((receipt) => receipt.callId === callId);+ if (matching.length !== 1) throw new Error(`Expected one background receipt for ${callId}.`);+ return matching[0]!.taskId;+}++function assertCompletedBusyWorker(turn: EveEvalTurn, taskId: string, marker: string): void {+ turn.expectOk();+ turn.messageIncludes("TASK-D6-STATUS");+ const view = requireTaskView(turn.requireToolCall("task_peek").output, taskId);+ const metadata = Reflect.get(view, "metadata");+ const lastOutput = Reflect.get(view, "lastOutput");+ if (+ Reflect.get(view, "status") !== "completed" ||+ metadata === null ||+ typeof metadata !== "object" ||+ Reflect.get(metadata, "name") !== "busy-worker" ||+ lastOutput === null ||+ typeof lastOutput !== "object" ||+ Reflect.get(lastOutput, "type") !== "result" ||+ !String(Reflect.get(lastOutput, "data")).includes(marker)+ ) {+ throw new Error(`Task ${taskId} did not complete with marker ${marker}.`);+ }+}diff --git a/e2e/fixtures/fixture-tasks/evals/task.dispatch.start.rejected-unreachable.eval.ts b/e2e/fixtures/fixture-tasks/evals/task.dispatch.start.rejected-unreachable.eval.tsnew file mode 100644index 000000000..ba9e43ecc--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task.dispatch.start.rejected-unreachable.eval.ts@@ -0,0 +1,61 @@+import { defineTaskEval } from "./task-transition.js";++const CALL_ID = "task-a3-unstartable-worker";++/** A failed remote start never admits its prepared task or child session. */+export default defineTaskEval({+ description:+ "An unreachable remote URL fails dispatch without exposing a receipt, task id, or child session.",+ transition: {+ primary: "task.dispatch.start.rejected-unreachable",+ dimensions: { transport: "remote", parentPhase: "active" },+ },+ async test(t) {+ const started = await t.send("TASK-A3-DISPATCH-START-FAILURE");+ started.expectOk();+ started.messageIncludes("TASK-A3-PARENT-SURVIVED");+ started.calledSubagent("unstartable-worker", {+ callId: CALL_ID,+ childSessionId: undefined,+ count: 1,+ output: { code: "REMOTE_AGENT_START_FAILED" },+ status: "failed",+ });+ started.eventsSatisfy(+ "failed start exposes no task id, receipt, child session, or index admission",+ (events) => {+ const failed = events.flatMap((event) =>+ event.type === "action.result" &&+ event.data.result.kind === "subagent-result" &&+ event.data.result.callId === CALL_ID+ ? [event]+ : [],+ );+ const result = failed[0]?.data.result;+ return (+ failed.length === 1 &&+ failed[0]?.data.status === "failed" &&+ result?.kind === "subagent-result" &&+ result.origin === "dispatch" &&+ isStartFailureWithoutTaskId(result.output) &&+ !events.some(+ (event) =>+ event.type === "subagent.completed" &&+ event.data.callId === CALL_ID &&+ event.data.backgroundTask !== undefined,+ ) &&+ !events.some((event) => event.type === "subagent.called" && event.data.callId === CALL_ID)+ );+ },+ );+ },+});++function isStartFailureWithoutTaskId(output: unknown): boolean {+ return (+ output !== null &&+ typeof output === "object" &&+ Reflect.get(output, "code") === "REMOTE_AGENT_START_FAILED" &&+ !Object.hasOwn(output, "taskId")+ );+}diff --git a/e2e/fixtures/fixture-tasks/evals/task.input.answer.accepted-complete.local.eval.ts b/e2e/fixtures/fixture-tasks/evals/task.input.answer.accepted-complete.local.eval.tsnew file mode 100644index 000000000..c02a5880c--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task.input.answer.accepted-complete.local.eval.ts@@ -0,0 +1,62 @@+import { requireBackgroundTaskId, waitForCompletedTask, waitForTaskInput } from "./shared.js";+import { defineTaskEval } from "./task-transition.js";++/**+ * Task-owned child HITL must surface on the parent session, and answering it+ * must route to the child without running the parent model.+ */+export default defineTaskEval({+ description:+ "A background child's approval surfaces on the parent and routes back without a parent model step.",+ transition: {+ primary: "task.input.answer.accepted-complete",+ dimensions: { transport: "local" },+ },+ async test(t) {+ const started = await t.send("TASK-HITL-ROUTING");+ started.expectOk();+ started.event("subagent.completed", {+ count: 1,+ data: { backgroundTask: { status: "working" }, subagentName: "approval-worker" },+ });+ const taskId = requireBackgroundTaskId(started);++ const first = await waitForTaskInput(t, t, "first_gate");+ const answered = await first.session.respond([+ {+ optionId: "approve",+ requestId: first.request.requestId,+ },+ ]);+ answered.expectOk();+ answered.notEvent("step.started");+ answered.noFailedActions();++ // The next child request proves direct delivery of the first answer.+ const second = await waitForTaskInput(t, first.session, "second_gate");+ const answeredSecond = await second.session.respond([+ {+ optionId: "approve",+ requestId: second.request.requestId,+ },+ ]);+ answeredSecond.expectOk();++ // Remaining gates only let the fixture reach a terminal cleanup state.+ const third = await waitForTaskInput(t, second.session, "third_gate");+ const finished = await third.session.respond([+ {+ optionId: "approve",+ requestId: third.request.requestId,+ },+ ]);+ finished.expectOk();++ const verified = await waitForCompletedTask(t, third.session, "TASK-HITL-VERIFY", taskId);+ verified.expectOk();+ verified.messageIncludes("TASK-HITL-STATUS");++ t.event("input.requested", { data: { requests: [{ action: { toolName: "first_gate" } }] } });+ t.noFailedActions();+ },+});diff --git a/e2e/fixtures/fixture-tasks/evals/task.input.answer.accepted-complete.remote.eval.ts b/e2e/fixtures/fixture-tasks/evals/task.input.answer.accepted-complete.remote.eval.tsnew file mode 100644index 000000000..42fa59592--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task.input.answer.accepted-complete.remote.eval.ts@@ -0,0 +1,114 @@+import { satisfies } from "eve/evals/expect";++import {+ requireBackgroundTaskId,+ requireTaskView,+ waitForCompletedTask,+ waitForTaskInput,+} from "./shared.js";+import { defineTaskEval } from "./task-transition.js";++const REMOTE_PRINCIPAL_MARKER = "C8-REMOTE-PRINCIPAL:user:remote-http-child";++/** A remote task's HITL answer must use its persisted HTTP child route. */+export default defineTaskEval({+ description:+ "A loopback remote child surfaces HITL and resumes over its remote response route with the remote transport principal intact.",+ transition: {+ primary: "task.input.answer.accepted-complete",+ dimensions: { transport: "remote" },+ },+ async test(t) {+ const started = await t.send("TASK-C8-REMOTE-HITL");+ started.expectOk();+ started.messageIncludes("TASK-C8-STARTED");+ started.event("subagent.completed", {+ count: 1,+ data: {+ backgroundTask: { status: "working" },+ callId: "task-c8-remote-worker",+ subagentName: "remote-loopback",+ },+ });+ const taskId = requireBackgroundTaskId(started);++ const gate = await waitForTaskInput(t, t, "remote_gate");+ const answered = await gate.session.respond([+ {+ optionId: "approve",+ requestId: gate.request.requestId,+ },+ ]);+ answered.expectOk();+ answered.eventsSatisfy(+ "a following parent model step belongs only to a task-ready notification",+ (events) =>+ !events.some((event) => event.type === "step.started") ||+ events.some(+ (event) =>+ event.type === "message.received" &&+ messageText(event.data.message).includes(`Background task ${taskId}`),+ ),+ );+ answered.noFailedActions();++ const terminal = await waitForCompletedTask(t, gate.session, "TASK-C8-REMOTE-VERIFY", taskId);+ terminal.expectOk();+ const view = requireTaskView(terminal.requireToolCall("task_peek").output, taskId);+ await t.require(+ view,+ satisfies(+ (task: Record<string, unknown>) =>+ Reflect.get(task, "status") === "completed" &&+ hasRemoteMetadata(task) &&+ hasRemotePrincipalOutput(task),+ "completed remote task retains remote mode and the HTTP child principal",+ ),+ );++ t.event("input.requested", {+ count: 1,+ data: { requests: [{ action: { toolName: "remote_gate" } }] },+ });+ t.notEvent("authorization.required");+ t.noFailedActions();+ },+});++function hasRemoteMetadata(task: Record<string, unknown>): boolean {+ const metadata = Reflect.get(task, "metadata");+ return (+ metadata !== null &&+ typeof metadata === "object" &&+ Reflect.get(metadata, "kind") === "subagent" &&+ Reflect.get(metadata, "mode") === "remote" &&+ Reflect.get(metadata, "name") === "remote-loopback" &&+ typeof Reflect.get(metadata, "agentId") === "string"+ );+}++function hasRemotePrincipalOutput(task: Record<string, unknown>): boolean {+ const output = Reflect.get(task, "lastOutput");+ return (+ output !== null &&+ typeof output === "object" &&+ Reflect.get(output, "type") === "result" &&+ typeof Reflect.get(output, "data") === "string" &&+ Reflect.get(output, "data").includes(REMOTE_PRINCIPAL_MARKER)+ );+}++function messageText(message: unknown): string {+ if (typeof message === "string") return message;+ if (!Array.isArray(message)) return "";+ return message+ .flatMap((part) =>+ part !== null &&+ typeof part === "object" &&+ Reflect.get(part, "type") === "text" &&+ typeof Reflect.get(part, "text") === "string"+ ? [Reflect.get(part, "text") as string]+ : [],+ )+ .join("\n");+}diff --git a/e2e/fixtures/fixture-tasks/evals/task.input.route.observed-stale-unrouted.eval.ts b/e2e/fixtures/fixture-tasks/evals/task.input.route.observed-stale-unrouted.eval.tsnew file mode 100644index 000000000..08c883bfd--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task.input.route.observed-stale-unrouted.eval.ts@@ -0,0 +1,89 @@+import { satisfies } from "eve/evals/expect";++import {+ requireBackgroundTaskId,+ requireTaskView,+ sendAndFollowQueuedTurn,+ waitForCompletedTask,+ waitForTaskInput,+} from "./shared.js";+import { defineTaskEval } from "./task-transition.js";++/**+ * Once Q1's proxy route is retired, replaying its answer becomes new parent+ * input and cannot reach the child or clear Q2 from the task snapshot.+ */+export default defineTaskEval({+ description:+ "An answer for a retired proxy route becomes parent input without unblocking or erasing the child's newer request.",+ transition: {+ primary: "task.input.route.observed-stale-unrouted",+ dimensions: { transport: "local" },+ },+ async test(t) {+ const started = await t.send("TASK-INPUT-BATCH-ORDERING");+ started.expectOk();+ const taskId = requireBackgroundTaskId(started);++ const first = await waitForTaskInput(t, t, "first_gate");+ const firstAnswer = await first.session.respond([+ {+ optionId: "approve",+ requestId: first.request.requestId,+ },+ ]);+ firstAnswer.expectOk();++ const second = await waitForTaskInput(t, first.session, "second_gate");+ const secondChildRequestId = second.request.requestId.replace(`${taskId}:`, "");+ const stale = await second.session.respond([+ {+ optionId: "approve",+ requestId: first.request.requestId,+ },+ ]);+ stale.expectOk();+ stale.event("step.started", { count: 1 });+ stale.noFailedActions();++ const afterStale = await sendAndFollowQueuedTurn(+ t,+ `TASK-INPUT-BATCH-VERIFY ${taskId}`,+ second.session,+ );+ const peeked = afterStale.turn.toolCalls.find((call) => call.name === "task_peek");+ await t.require(+ peeked?.output,+ satisfies((output) => {+ const view = requireTaskView(output, taskId);+ const requests = Reflect.get(view, "inputRequests");+ return (+ Reflect.get(view, "status") === "input_required" &&+ Array.isArray(requests) &&+ requests.length === 1 &&+ Reflect.get(requests[0], "requestId") === secondChildRequestId+ );+ }, "the stale Q1 answer leaves exactly Q2 outstanding"),+ );++ // If the stale Q1 answer cleared Q2, this exact Q2 response cannot resume+ // the child and the task never reaches `completed`.+ const secondAnswer = await afterStale.session.respond([+ {+ optionId: "approve",+ requestId: second.request.requestId,+ },+ ]);+ secondAnswer.expectOk();++ const verified = await waitForCompletedTask(+ t,+ afterStale.session,+ "TASK-INPUT-BATCH-VERIFY",+ taskId,+ );+ verified.expectOk();+ verified.messageIncludes("TASK-INPUT-BATCH-STATUS");+ t.noFailedActions();+ },+});diff --git a/e2e/fixtures/fixture-tasks/evals/task.join.evaluate.observed-all-terminal.eval.ts b/e2e/fixtures/fixture-tasks/evals/task.join.evaluate.observed-all-terminal.eval.tsnew file mode 100644index 000000000..a022eb6c7--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task.join.evaluate.observed-all-terminal.eval.ts@@ -0,0 +1,185 @@+import { type EveEvalContext, type EveEvalTurn, type InputRequest } from "eve/evals";+import { satisfies } from "eve/evals/expect";++import { defineTaskEval } from "./task-transition.js";+import { requireSessionStreamIndex, type TaskEvalSessionDriver } from "./shared.js";++const FAN_IN_SIZE = 2;+const MAX_WAKE_TURNS = 8;+const FAN_IN_CALLS = [+ { callId: "task-fan-in-1", marker: "TASK-FAN-IN-1" },+ { callId: "task-fan-in-2", marker: "TASK-FAN-IN-2" },+] as const;++type FanInMarker = (typeof FAN_IN_CALLS)[number]["marker"];++/** A join completes only from a peek proving its exact task set is terminal. */+export default defineTaskEval({+ description: "A join observes both exact tasks completed before answering COMPLETE.",+ transition: {+ primary: "task.join.evaluate.observed-all-terminal",+ setup: [+ "task.dispatch.start.accepted-acknowledged",+ "task.input.require.accepted-valid-batch",+ "task.input.answer.accepted-complete",+ "task.lifecycle.complete.accepted-nonterminal",+ "task.parent.wake.emitted-ready",+ ],+ dimensions: { transport: "local", parentPhase: "active" },+ },+ async test(t) {+ const started = await t.send("TASK-FAN-IN");+ started.expectOk();+ started.messageIncludes("TASK-FAN-IN-STARTED");+ started.calledSubagent("fanout-worker", { count: FAN_IN_SIZE });++ const tasksByMarker = backgroundTasksByMarker(started);+ const taskIds = [...tasksByMarker.values()];+ await t.require(+ taskIds,+ satisfies(+ (ids: readonly string[]) => ids.length === FAN_IN_SIZE && new Set(ids).size === FAN_IN_SIZE,+ `${FAN_IN_SIZE} distinct background task receipts`,+ ),+ );+ const blocked = await waitForReleaseRequests(t, t, started);++ const released = await blocked.session.respond(+ FAN_IN_CALLS.map(({ marker }) => ({+ optionId: "approve",+ requestId: requireMappedValue(blocked.requestsByMarker, marker, "release request")+ .requestId,+ })),+ );+ released.expectOk();++ const complete = await waitForTurnMessage(t, blocked.session, "TASK-FAN-IN-COMPLETE", released);+ const completePeek = complete.toolCalls.find((call) => call.name === "task_peek");+ await requireExactCompletedPeek(t, completePeek, taskIds, tasksByMarker);+ t.noFailedActions();+ },+});++async function requireExactCompletedPeek(+ t: EveEvalContext,+ call: EveEvalTurn["toolCalls"][number] | undefined,+ taskIds: readonly string[],+ tasksByMarker: ReadonlyMap<FanInMarker, string>,+): Promise<void> {+ await t.require(+ { input: call?.input, output: call?.output },+ satisfies((subject: { readonly input: unknown; readonly output: unknown }) => {+ const inputIds = Reflect.get(subject.input ?? {}, "taskIds");+ const tasks = Reflect.get(subject.output ?? {}, "tasks");+ if (!Array.isArray(inputIds) || !Array.isArray(tasks)) return false;+ const expectedIds = [...taskIds].sort();+ const outputIds = tasks.map((task) => Reflect.get(task, "taskId")).sort();+ return (+ JSON.stringify([...inputIds].sort()) === JSON.stringify(expectedIds) &&+ JSON.stringify(outputIds) === JSON.stringify(expectedIds) &&+ FAN_IN_CALLS.every(({ marker }) => {+ const taskId = tasksByMarker.get(marker);+ const task = tasks.find((candidate) => Reflect.get(candidate ?? {}, "taskId") === taskId);+ const lastOutput = Reflect.get(task ?? {}, "lastOutput");+ return (+ Reflect.get(task ?? {}, "status") === "completed" &&+ Reflect.get(lastOutput ?? {}, "type") === "result" &&+ Reflect.get(lastOutput ?? {}, "data") === `FANOUT-COMPLETE:${marker}`+ );+ })+ );+ }, "the COMPLETE turn peeks both exact tasks with their completed outputs"),+ );+}++interface BlockedFanIn {+ readonly requestsByMarker: ReadonlyMap<FanInMarker, InputRequest>;+ readonly session: TaskEvalSessionDriver;+}++async function waitForReleaseRequests(+ t: EveEvalContext,+ initialSession: TaskEvalSessionDriver,+ initialTurn: EveEvalTurn,+): Promise<BlockedFanIn> {+ const requestsByMarker = new Map<FanInMarker, InputRequest>();+ let session = initialSession;+ collectReleaseRequests(initialTurn.inputRequests, requestsByMarker);+ for (+ let attempt = 0;+ attempt < MAX_WAKE_TURNS && requestsByMarker.size < FAN_IN_SIZE;+ attempt += 1+ ) {+ const sessionId = session.sessionId;+ if (sessionId === undefined) throw new Error("Fan-in has no parent session id.");+ const live = t.target.watchTurn(sessionId, {+ startIndex: requireSessionStreamIndex(session, "Fan-in release wait"),+ });+ const turn = await live.result();+ collectReleaseRequests(turn.inputRequests, requestsByMarker);+ session = live.session;+ }+ if (requestsByMarker.size !== FAN_IN_SIZE) {+ throw new Error(+ `Expected ${FAN_IN_SIZE} marked release requests; received ${requestsByMarker.size}.`,+ );+ }+ return { requestsByMarker, session };+}++function collectReleaseRequests(+ inputRequests: readonly InputRequest[],+ requestsByMarker: Map<FanInMarker, InputRequest>,+): void {+ for (const request of inputRequests) {+ if (request.action.toolName !== "release") continue;+ const marker = request.action.input.marker;+ if (isFanInMarker(marker)) requestsByMarker.set(marker, request);+ }+}++function isFanInMarker(value: unknown): value is FanInMarker {+ return FAN_IN_CALLS.some(({ marker }) => marker === value);+}++async function waitForTurnMessage(+ t: EveEvalContext,+ initialSession: TaskEvalSessionDriver,+ token: string,+ initialTurn: EveEvalTurn,+): Promise<EveEvalTurn> {+ if ((initialTurn.message ?? "").includes(token)) return initialTurn;+ let session = initialSession;+ for (let attempt = 0; attempt < MAX_WAKE_TURNS; attempt += 1) {+ const sessionId = session.sessionId;+ if (sessionId === undefined) throw new Error("Fan-in has no parent session id.");+ const live = t.target.watchTurn(sessionId, {+ startIndex: requireSessionStreamIndex(session, "Fan-in message wait"),+ });+ const turn = await live.result();+ session = live.session;+ if ((turn.message ?? "").includes(token)) return turn;+ }+ throw new Error(`No turn carried "${token}" after ${MAX_WAKE_TURNS} turns.`);+}++function backgroundTasksByMarker(turn: EveEvalTurn): ReadonlyMap<FanInMarker, string> {+ const tasksByMarker = new Map<FanInMarker, string>();+ for (const event of turn.events) {+ if (event.type !== "subagent.completed" || event.data.backgroundTask === undefined) continue;+ const fanInCall = FAN_IN_CALLS.find(({ callId }) => callId === event.data.callId);+ if (fanInCall !== undefined)+ tasksByMarker.set(fanInCall.marker, event.data.backgroundTask.taskId);+ }+ return tasksByMarker;+}++function requireMappedValue<T>(+ values: ReadonlyMap<FanInMarker, T>,+ marker: FanInMarker,+ description: string,+): T {+ const value = values.get(marker);+ if (value === undefined) throw new Error(`Fan-in has no ${description} for ${marker}.`);+ return value;+}diff --git a/e2e/fixtures/fixture-tasks/evals/task.join.evaluate.observed-partial.eval.ts b/e2e/fixtures/fixture-tasks/evals/task.join.evaluate.observed-partial.eval.tsnew file mode 100644index 000000000..eef76b817--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task.join.evaluate.observed-partial.eval.ts@@ -0,0 +1,243 @@+import { type EveEvalContext, type EveEvalTurn, type InputRequest } from "eve/evals";+import { satisfies } from "eve/evals/expect";++import { defineTaskEval } from "./task-transition.js";+import { requireSessionStreamIndex, type TaskEvalSessionDriver } from "./shared.js";++const FAN_IN_SIZE = 2;+const MAX_WAKE_TURNS = 8;+const FAN_IN_CALLS = [+ { callId: "task-fan-in-1", marker: "TASK-FAN-IN-1" },+ { callId: "task-fan-in-2", marker: "TASK-FAN-IN-2" },+] as const;++type FanInMarker = (typeof FAN_IN_CALLS)[number]["marker"];++/** A join remains waiting when its exact task set contains one nonterminal task. */+export default defineTaskEval({+ description:+ "A join observes one completed and one input-required task, answers WAITING, and does not answer COMPLETE.",+ transition: {+ primary: "task.join.evaluate.observed-partial",+ setup: [+ "task.dispatch.start.accepted-acknowledged",+ "task.input.require.accepted-valid-batch",+ "task.input.answer.accepted-complete",+ "task.lifecycle.complete.accepted-nonterminal",+ "task.parent.wake.emitted-ready",+ ],+ dimensions: { transport: "local", parentPhase: "active" },+ },+ async test(t) {+ const started = await t.send("TASK-FAN-IN");+ started.expectOk();+ started.messageIncludes("TASK-FAN-IN-STARTED");+ started.calledSubagent("fanout-worker", { count: FAN_IN_SIZE });++ const tasksByMarker = backgroundTasksByMarker(started);+ const taskIds = [...tasksByMarker.values()];+ await requireDistinctTasks(t, taskIds);+ const blocked = await waitForReleaseRequests(t, t, started);++ const marker1 = "TASK-FAN-IN-1";+ const marker2 = "TASK-FAN-IN-2";+ const marker1TaskId = requireMappedValue(tasksByMarker, marker1, "background task");+ const marker2TaskId = requireMappedValue(tasksByMarker, marker2, "background task");+ const marker2Request = requireMappedValue(blocked.requestsByMarker, marker2, "release request");++ const firstReleased = await blocked.session.respond([+ { optionId: "approve", requestId: marker2Request.requestId },+ ]);+ firstReleased.expectOk();++ const waiting = await waitForTurnMessage(+ t,+ blocked.session,+ "TASK-FAN-IN-WAITING",+ firstReleased,+ );+ const waitingPeek = waiting.turn.toolCalls.find((call) => call.name === "task_peek");+ await requireExactPeek(+ t,+ waitingPeek,+ taskIds,+ [+ { marker: marker1, status: "input_required", taskId: marker1TaskId },+ { marker: marker2, status: "completed", taskId: marker2TaskId },+ ],+ "the join reads one completed and one input-required exact task",+ );+ for (const turn of waiting.observedTurns) {+ await t.require(+ turn.message ?? "",+ satisfies(+ (message) => !String(message).includes("TASK-FAN-IN-COMPLETE"),+ "no COMPLETE answer while one joined task is nonterminal",+ ),+ );+ }+ t.noFailedActions();+ },+});++interface ExpectedTask {+ readonly marker: FanInMarker;+ readonly status: "completed" | "input_required";+ readonly taskId: string;+}++async function requireDistinctTasks(t: EveEvalContext, taskIds: readonly string[]): Promise<void> {+ await t.require(+ taskIds,+ satisfies(+ (ids: readonly string[]) => ids.length === FAN_IN_SIZE && new Set(ids).size === FAN_IN_SIZE,+ `${FAN_IN_SIZE} distinct background task receipts`,+ ),+ );+}++async function requireExactPeek(+ t: EveEvalContext,+ call: EveEvalTurn["toolCalls"][number] | undefined,+ taskIds: readonly string[],+ expectedTasks: readonly ExpectedTask[],+ description: string,+): Promise<void> {+ await t.require(+ { input: call?.input, output: call?.output },+ satisfies((subject: { readonly input: unknown; readonly output: unknown }) => {+ const inputIds = Reflect.get(subject.input ?? {}, "taskIds");+ const tasks = Reflect.get(subject.output ?? {}, "tasks");+ if (!Array.isArray(inputIds) || !Array.isArray(tasks)) return false;+ const expectedIds = [...taskIds].sort();+ const outputIds = tasks.map((task) => Reflect.get(task, "taskId")).sort();+ return (+ JSON.stringify([...inputIds].sort()) === JSON.stringify(expectedIds) &&+ JSON.stringify(outputIds) === JSON.stringify(expectedIds) &&+ expectedTasks.every((expectedTask) => {+ const task = tasks.find(+ (candidate) => Reflect.get(candidate ?? {}, "taskId") === expectedTask.taskId,+ );+ return taskMatchesExpected(task, expectedTask);+ })+ );+ }, description),+ );+}++function taskMatchesExpected(task: unknown, expected: ExpectedTask): boolean {+ if (Reflect.get(task ?? {}, "status") !== expected.status) return false;+ if (expected.status === "completed") {+ const lastOutput = Reflect.get(task ?? {}, "lastOutput");+ return (+ Reflect.get(lastOutput ?? {}, "type") === "result" &&+ Reflect.get(lastOutput ?? {}, "data") === `FANOUT-COMPLETE:${expected.marker}`+ );+ }+ const inputRequests = Reflect.get(task ?? {}, "inputRequests");+ if (!Array.isArray(inputRequests)) return false;+ const release = inputRequests.find(+ (request) => Reflect.get(Reflect.get(request ?? {}, "action") ?? {}, "toolName") === "release",+ );+ const action = Reflect.get(release ?? {}, "action");+ return Reflect.get(Reflect.get(action ?? {}, "input") ?? {}, "marker") === expected.marker;+}++interface BlockedFanIn {+ readonly requestsByMarker: ReadonlyMap<FanInMarker, InputRequest>;+ readonly session: TaskEvalSessionDriver;+}++async function waitForReleaseRequests(+ t: EveEvalContext,+ initialSession: TaskEvalSessionDriver,+ initialTurn: EveEvalTurn,+): Promise<BlockedFanIn> {+ const requestsByMarker = new Map<FanInMarker, InputRequest>();+ let session = initialSession;+ collectReleaseRequests(initialTurn.inputRequests, requestsByMarker);+ for (+ let attempt = 0;+ attempt < MAX_WAKE_TURNS && requestsByMarker.size < FAN_IN_SIZE;+ attempt += 1+ ) {+ const sessionId = session.sessionId;+ if (sessionId === undefined) throw new Error("Fan-in has no parent session id.");+ const live = t.target.watchTurn(sessionId, {+ startIndex: requireSessionStreamIndex(session, "Fan-in release wait"),+ });+ const turn = await live.result();+ collectReleaseRequests(turn.inputRequests, requestsByMarker);+ session = live.session;+ }+ if (requestsByMarker.size !== FAN_IN_SIZE) {+ throw new Error(+ `Expected ${FAN_IN_SIZE} marked release requests; received ${requestsByMarker.size}.`,+ );+ }+ return { requestsByMarker, session };+}++function collectReleaseRequests(+ inputRequests: readonly InputRequest[],+ requestsByMarker: Map<FanInMarker, InputRequest>,+): void {+ for (const request of inputRequests) {+ if (request.action.toolName !== "release") continue;+ const marker = request.action.input.marker;+ if (isFanInMarker(marker)) requestsByMarker.set(marker, request);+ }+}++function isFanInMarker(value: unknown): value is FanInMarker {+ return FAN_IN_CALLS.some(({ marker }) => marker === value);+}++interface ObservedTurnMessage {+ readonly observedTurns: readonly EveEvalTurn[];+ readonly turn: EveEvalTurn;+}++async function waitForTurnMessage(+ t: EveEvalContext,+ initialSession: TaskEvalSessionDriver,+ token: string,+ initialTurn: EveEvalTurn,+): Promise<ObservedTurnMessage> {+ const observedTurns: EveEvalTurn[] = [initialTurn];+ let session = initialSession;+ if ((initialTurn.message ?? "").includes(token)) return { observedTurns, turn: initialTurn };+ for (let attempt = 0; attempt < MAX_WAKE_TURNS; attempt += 1) {+ const sessionId = session.sessionId;+ if (sessionId === undefined) throw new Error("Fan-in has no parent session id.");+ const live = t.target.watchTurn(sessionId, {+ startIndex: requireSessionStreamIndex(session, "Fan-in message wait"),+ });+ const turn = await live.result();+ observedTurns.push(turn);+ session = live.session;+ if ((turn.message ?? "").includes(token)) return { observedTurns, turn };+ }+ throw new Error(`No turn carried "${token}" after ${MAX_WAKE_TURNS} turns.`);+}++function backgroundTasksByMarker(turn: EveEvalTurn): ReadonlyMap<FanInMarker, string> {+ const tasksByMarker = new Map<FanInMarker, string>();+ for (const event of turn.events) {+ if (event.type !== "subagent.completed" || event.data.backgroundTask === undefined) continue;+ const fanInCall = FAN_IN_CALLS.find(({ callId }) => callId === event.data.callId);+ if (fanInCall !== undefined)+ tasksByMarker.set(fanInCall.marker, event.data.backgroundTask.taskId);+ }+ return tasksByMarker;+}++function requireMappedValue<T>(+ values: ReadonlyMap<FanInMarker, T>,+ marker: FanInMarker,+ description: string,+): T {+ const value = values.get(marker);+ if (value === undefined) throw new Error(`Fan-in has no ${description} for ${marker}.`);+ return value;+}diff --git a/e2e/fixtures/fixture-tasks/evals/task.lifecycle.cancel.accepted-nonterminal.eval.ts b/e2e/fixtures/fixture-tasks/evals/task.lifecycle.cancel.accepted-nonterminal.eval.tsnew file mode 100644index 000000000..9b4746587--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task.lifecycle.cancel.accepted-nonterminal.eval.ts@@ -0,0 +1,92 @@+import { satisfies } from "eve/evals/expect";++import {+ requireBackgroundTaskId,+ requireTaskView,+ sendAndFollowQueuedTurn,+ waitForTaskInput,+ waitForTaskStatus,+} from "./shared.js";+import { defineTaskEval } from "./task-transition.js";++/** Cancellation commits `cancelled` while the child is waiting for input. */+export default defineTaskEval({+ description: "task_cancel commits a final cancelled state on a nonterminal task.",+ transition: {+ primary: "task.lifecycle.cancel.accepted-nonterminal",+ setup: ["task.dispatch.start.accepted-acknowledged", "task.input.require.accepted-valid-batch"],+ dimensions: { transport: "local", parentPhase: "active" },+ },+ async test(t) {+ const started = await t.send("TASK-CANCEL-SETUP");+ started.expectOk();+ started.messageIncludes("TASK-CANCEL-READY");+ started.event("subagent.completed", {+ count: 1,+ data: { backgroundTask: { status: "working" }, subagentName: "fanout-worker" },+ });+ const taskId = requireBackgroundTaskId(started);++ const blocked = await waitForTaskInput(t, t, "release");+ const cancelled = await sendAndFollowQueuedTurn(t, "TASK-CANCEL-NOW", blocked.session);+ cancelled.turn.expectOk();+ cancelled.turn.messageIncludes("TASK-CANCEL-DONE");+ cancelled.turn.calledTool("task_cancel", { input: { taskIds: [taskId] } });+ const cancelledCall = cancelled.turn.toolCalls.find((call) => call.name === "task_cancel");+ const cancelledView = requireTaskView(cancelledCall?.output, taskId);+ await t.require(+ cancelledView,+ satisfies(+ (view: Record<string, unknown>) => isCancelledTaskView(view, taskId),+ "the first cancellation returns the cancelled task without output or pending input",+ ),+ );++ const verified = await waitForTaskStatus(+ t,+ cancelled.session,+ "TASK-CANCEL-VERIFY",+ taskId,+ "cancelled",+ );+ verified.expectOk();+ verified.messageIncludes("TASK-CANCEL-STATUS");+ await t.require(+ requireTaskView(verified.requireToolCall("task_peek").output, taskId),+ satisfies(+ (view: Record<string, unknown>) => hasSameCancelledFields(view, cancelledView),+ "task_peek preserves the cancelled task's semantic fields",+ ),+ );+ },+});++function isCancelledTaskView(view: Record<string, unknown>, taskId: string): boolean {+ return (+ Reflect.get(view, "taskId") === taskId &&+ Reflect.get(view, "status") === "cancelled" &&+ !Object.hasOwn(view, "inputRequests") &&+ !Object.hasOwn(view, "lastOutput")+ );+}++function hasSameCancelledFields(+ actual: Record<string, unknown>,+ expected: Record<string, unknown>,+): boolean {+ const actualMetadata = Reflect.get(actual, "metadata");+ const expectedMetadata = Reflect.get(expected, "metadata");+ const expectedTaskId = Reflect.get(expected, "taskId");+ return (+ typeof expectedTaskId === "string" &&+ isCancelledTaskView(actual, expectedTaskId) &&+ actualMetadata !== null &&+ typeof actualMetadata === "object" &&+ expectedMetadata !== null &&+ typeof expectedMetadata === "object" &&+ Reflect.get(actualMetadata, "agentId") === Reflect.get(expectedMetadata, "agentId") &&+ Reflect.get(actualMetadata, "kind") === Reflect.get(expectedMetadata, "kind") &&+ Reflect.get(actualMetadata, "mode") === Reflect.get(expectedMetadata, "mode") &&+ Reflect.get(actualMetadata, "name") === Reflect.get(expectedMetadata, "name")+ );+}diff --git a/e2e/fixtures/fixture-tasks/evals/task.lifecycle.cancel.noop-already-cancelled.eval.ts b/e2e/fixtures/fixture-tasks/evals/task.lifecycle.cancel.noop-already-cancelled.eval.tsnew file mode 100644index 000000000..309b4bad1--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task.lifecycle.cancel.noop-already-cancelled.eval.ts@@ -0,0 +1,95 @@+import { satisfies } from "eve/evals/expect";++import {+ requireBackgroundTaskId,+ requireTaskView,+ sendAndFollowQueuedTurn,+ waitForTaskInput,+ waitForTaskStatus,+} from "./shared.js";+import { defineTaskEval } from "./task-transition.js";++/** Repeating cancellation preserves the existing cancelled task view. */+export default defineTaskEval({+ description: "task_cancel is a no-op when the task is already cancelled.",+ transition: {+ primary: "task.lifecycle.cancel.noop-already-cancelled",+ setup: [+ "task.dispatch.start.accepted-acknowledged",+ "task.input.require.accepted-valid-batch",+ "task.lifecycle.cancel.accepted-nonterminal",+ ],+ dimensions: { transport: "local", parentPhase: "active" },+ },+ async test(t) {+ const started = await t.send("TASK-CANCEL-SETUP");+ started.expectOk();+ started.messageIncludes("TASK-CANCEL-READY");+ started.event("subagent.completed", {+ count: 1,+ data: { backgroundTask: { status: "working" }, subagentName: "fanout-worker" },+ });+ const taskId = requireBackgroundTaskId(started);++ const blocked = await waitForTaskInput(t, t, "release");+ const cancelled = await sendAndFollowQueuedTurn(t, "TASK-CANCEL-NOW", blocked.session);+ cancelled.turn.expectOk();+ cancelled.turn.calledTool("task_cancel", { input: { taskIds: [taskId] } });+ const cancelledCall = cancelled.turn.toolCalls.find((call) => call.name === "task_cancel");+ const cancelledView = requireTaskView(cancelledCall?.output, taskId);++ const repeated = await sendAndFollowQueuedTurn(t, "TASK-CANCEL-NOW", cancelled.session);+ repeated.turn.expectOk();+ repeated.turn.messageIncludes("TASK-CANCEL-DONE");+ repeated.turn.calledTool("task_cancel", { input: { taskIds: [taskId] } });+ const repeatedCall = repeated.turn.toolCalls.find((call) => call.name === "task_cancel");+ await t.require(+ requireTaskView(repeatedCall?.output, taskId),+ satisfies(+ (view: Record<string, unknown>) => hasSameCancelledFields(view, cancelledView),+ "repeated cancellation preserves every model-visible cancelled-task field",+ ),+ );++ const still = await waitForTaskStatus(+ t,+ repeated.session,+ "TASK-CANCEL-VERIFY",+ taskId,+ "cancelled",+ );+ still.expectOk();+ await t.require(+ requireTaskView(still.requireToolCall("task_peek").output, taskId),+ satisfies(+ (view: Record<string, unknown>) => hasSameCancelledFields(view, cancelledView),+ "task_peek observes the same semantic fields after repeated cancellation",+ ),+ );+ },+});++function hasSameCancelledFields(+ actual: Record<string, unknown>,+ expected: Record<string, unknown>,+): boolean {+ const actualMetadata = Reflect.get(actual, "metadata");+ const expectedMetadata = Reflect.get(expected, "metadata");+ return (+ Reflect.get(actual, "taskId") === Reflect.get(expected, "taskId") &&+ Reflect.get(actual, "status") === "cancelled" &&+ Reflect.get(expected, "status") === "cancelled" &&+ !Object.hasOwn(actual, "inputRequests") &&+ !Object.hasOwn(actual, "lastOutput") &&+ !Object.hasOwn(expected, "inputRequests") &&+ !Object.hasOwn(expected, "lastOutput") &&+ actualMetadata !== null &&+ typeof actualMetadata === "object" &&+ expectedMetadata !== null &&+ typeof expectedMetadata === "object" &&+ Reflect.get(actualMetadata, "agentId") === Reflect.get(expectedMetadata, "agentId") &&+ Reflect.get(actualMetadata, "kind") === Reflect.get(expectedMetadata, "kind") &&+ Reflect.get(actualMetadata, "mode") === Reflect.get(expectedMetadata, "mode") &&+ Reflect.get(actualMetadata, "name") === Reflect.get(expectedMetadata, "name")+ );+}diff --git a/e2e/fixtures/fixture-tasks/evals/task.lifecycle.fail.accepted-nonterminal.eval.ts b/e2e/fixtures/fixture-tasks/evals/task.lifecycle.fail.accepted-nonterminal.eval.tsnew file mode 100644index 000000000..b8ecafe64--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task.lifecycle.fail.accepted-nonterminal.eval.ts@@ -0,0 +1,73 @@+import { satisfies } from "eve/evals/expect";++import { requireBackgroundTaskId, requireTaskView, waitForTaskStatus } from "./shared.js";+import { defineTaskEval } from "./task-transition.js";++const CALL_ID = "task-a2-failing-busy-worker";++/** A child execution failure remains observable through its durable task view. */+export default defineTaskEval({+ description:+ "A background child that fails terminally leaves a failed task with a stable error projection.",+ transition: {+ primary: "task.lifecycle.fail.accepted-nonterminal",+ setup: ["task.dispatch.start.accepted-acknowledged"],+ dimensions: { transport: "local" },+ },+ async test(t) {+ const started = await t.send("TASK-A2-CHILD-FAILURE");+ started.expectOk();+ started.messageIncludes("TASK-A2-CHILD-FAILURE-STARTED");+ started.event("subagent.completed", {+ count: 1,+ data: {+ backgroundTask: { status: "working" },+ callId: CALL_ID,+ subagentName: "busy-worker",+ },+ });+ const taskId = requireBackgroundTaskId(started);++ const failed = await waitForTaskStatus(t, t, "TASK-A2-CHILD-FAILURE-VERIFY", taskId, "failed");+ failed.expectOk();+ failed.messageIncludes("TASK-A2-FAILED");+ const peeked = failed.requireToolCall("task_peek", { input: { taskIds: [taskId] } });+ const view = requireTaskView(peeked.output, taskId);+ await t.require(+ view,+ satisfies(+ (task: Record<string, unknown>) =>+ Reflect.get(task, "taskId") === taskId &&+ Reflect.get(task, "status") === "failed" &&+ hasTaskMetadata(task, "busy-worker", "local") &&+ hasFailureOutput(task, "SUBAGENT_EXECUTION_FAILED"),+ "failed task keeps its id, busy-worker metadata, and stable error code",+ ),+ );+ },+});++function hasTaskMetadata(task: Record<string, unknown>, name: string, mode: string): boolean {+ const metadata = Reflect.get(task, "metadata");+ return (+ metadata !== null &&+ typeof metadata === "object" &&+ Reflect.get(metadata, "kind") === "subagent" &&+ Reflect.get(metadata, "mode") === mode &&+ Reflect.get(metadata, "name") === name &&+ typeof Reflect.get(metadata, "agentId") === "string"+ );+}++function hasFailureOutput(task: Record<string, unknown>, code: string): boolean {+ const lastOutput = Reflect.get(task, "lastOutput");+ if (+ lastOutput === null ||+ typeof lastOutput !== "object" ||+ Reflect.get(lastOutput, "type") !== "error"+ ) {+ return false;+ }+ const data = Reflect.get(lastOutput, "data");+ return data !== null && typeof data === "object" && Reflect.get(data, "code") === code;+}diff --git a/e2e/fixtures/fixture-tasks/evals/task.parent-interaction.send.accepted-live-children.eval.ts b/e2e/fixtures/fixture-tasks/evals/task.parent-interaction.send.accepted-live-children.eval.tsnew file mode 100644index 000000000..1f92c1a95--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task.parent-interaction.send.accepted-live-children.eval.ts@@ -0,0 +1,85 @@+import { type EveEvalTurn, type InputRequest } from "eve/evals";+import { satisfies } from "eve/evals/expect";++import { defineTaskEval } from "./task-transition.js";+import {+ requireSessionStreamIndex,+ sendAndFollowQueuedTurn,+ type TaskEvalSessionDriver,+} from "./shared.js";++const FANOUT_SIZE = 10;++/** The parent remains interactive while its background children await input. */+export default defineTaskEval({+ description: "A parent accepts a new tool-free turn while ten background children remain live.",+ transition: {+ primary: "task.parent-interaction.send.accepted-live-children",+ setup: ["task.dispatch.start.accepted-acknowledged", "task.input.require.accepted-valid-batch"],+ dimensions: { transport: "local", parentPhase: "active" },+ },+ async test(t) {+ const started = await t.send("TASK-FANOUT-PARENT-UPDATES");+ started.expectOk();+ started.messageIncludes("TASK-FANOUT-STARTED");+ started.calledSubagent("fanout-worker", { count: FANOUT_SIZE });++ const taskIds = backgroundTaskIds(started);+ await t.require(+ taskIds,+ satisfies(+ (ids: readonly string[]) => ids.length === FANOUT_SIZE && new Set(ids).size === FANOUT_SIZE,+ `${FANOUT_SIZE} distinct background task receipts`,+ ),+ );++ const blockedSession = await waitForReleaseRequests(t, t, started);+ const interactive = await sendAndFollowQueuedTurn(+ t,+ "TASK-FANOUT-INTERACTIVE-CHECK",+ blockedSession,+ );+ interactive.turn.expectOk();+ interactive.turn.messageIncludes("TASK-FANOUT-INTERACTIVE-OK");+ interactive.turn.usedNoTools();+ t.noFailedActions();+ },+});++async function waitForReleaseRequests(+ t: Parameters<typeof sendAndFollowQueuedTurn>[0],+ initialSession: TaskEvalSessionDriver,+ initialTurn: EveEvalTurn,+): Promise<TaskEvalSessionDriver> {+ const requests = new Map<string, InputRequest>();+ let session = initialSession;+ collectReleaseRequests(initialTurn, requests);+ for (let attempt = 0; attempt < FANOUT_SIZE && requests.size < FANOUT_SIZE; attempt += 1) {+ const sessionId = session.sessionId;+ if (sessionId === undefined) throw new Error("Task fanout has no parent session id.");+ const live = t.target.watchTurn(sessionId, {+ startIndex: requireSessionStreamIndex(session, "Task fanout release wait"),+ });+ const turn = await live.result();+ collectReleaseRequests(turn, requests);+ session = live.session;+ }+ if (requests.size !== FANOUT_SIZE) {+ throw new Error(`Expected ${FANOUT_SIZE} release requests; received ${requests.size}.`);+ }+ return session;+}++function collectReleaseRequests(turn: EveEvalTurn, requests: Map<string, InputRequest>): void {+ for (const request of turn.inputRequests) {+ if (request.action.toolName === "release") requests.set(request.requestId, request);+ }+}++function backgroundTaskIds(turn: EveEvalTurn): readonly string[] {+ return turn.events.flatMap((event) =>+ event.type === "subagent.completed" && event.data.backgroundTask !== undefined+ ? [event.data.backgroundTask.taskId]+ : [],+ );+}diff --git a/e2e/fixtures/fixture-tasks/evals/task.parent.wake.emitted-ready.eval.ts b/e2e/fixtures/fixture-tasks/evals/task.parent.wake.emitted-ready.eval.tsnew file mode 100644index 000000000..8a23cbedf--- /dev/null+++ b/e2e/fixtures/fixture-tasks/evals/task.parent.wake.emitted-ready.eval.ts@@ -0,0 +1,156 @@+import { type EveEvalContext, type EveEvalTurn, type InputRequest } from "eve/evals";+import { satisfies } from "eve/evals/expect";++import { defineTaskEval } from "./task-transition.js";+import { requireSessionStreamIndex, type TaskEvalSessionDriver } from "./shared.js";++const FANOUT_SIZE = 3;+const COMPLETED_NOTIFICATION = /Background task (task_[a-z0-9]+) \([^)]+\) is completed\./giu;++/** Every completed child emits one task-addressed ready notification to its parent. */+export default defineTaskEval({+ description:+ "Three completed children emit exactly one parent notification each, with no duplicate or unknown task ids.",+ transition: {+ primary: "task.parent.wake.emitted-ready",+ setup: [+ "task.dispatch.start.accepted-acknowledged",+ "task.input.require.accepted-valid-batch",+ "task.input.answer.accepted-complete",+ "task.lifecycle.complete.accepted-nonterminal",+ ],+ dimensions: { transport: "local", parentPhase: "active" },+ },+ async test(t) {+ const started = await t.send("TASK-PARENT-WAKE-UPDATES");+ started.expectOk();+ started.messageIncludes("TASK-FANOUT-STARTED");+ started.calledSubagent("fanout-worker", { count: FANOUT_SIZE });++ const taskIds = backgroundTaskIds(started);+ await t.require(+ taskIds,+ satisfies(+ (ids: readonly string[]) => ids.length === FANOUT_SIZE && new Set(ids).size === FANOUT_SIZE,+ `${FANOUT_SIZE} distinct background task receipts`,+ ),+ );++ const blocked = await waitForReleaseRequests(t, t, started);+ const released = await blocked.session.respond(+ blocked.requests.map((request) => ({+ optionId: "approve",+ requestId: request.requestId,+ })),+ );+ released.expectOk();++ const notifiedTaskIds = [+ ...completedNotificationTaskIds(started),+ ...blocked.observedTurns.flatMap(completedNotificationTaskIds),+ ...completedNotificationTaskIds(released),+ ];+ const observedKnownTaskIds = new Set(+ notifiedTaskIds.filter((taskId) => taskIds.includes(taskId)),+ );+ let session = blocked.session;+ for (+ let attempt = 0;+ attempt < FANOUT_SIZE && observedKnownTaskIds.size < FANOUT_SIZE;+ attempt += 1+ ) {+ const sessionId = session.sessionId;+ if (sessionId === undefined) throw new Error("Task fanout has no parent session id.");+ const live = t.target.watchTurn(sessionId, {+ startIndex: requireSessionStreamIndex(session, "Task fanout notification wait"),+ });+ const turn = await live.result();+ for (const taskId of completedNotificationTaskIds(turn)) {+ notifiedTaskIds.push(taskId);+ if (taskIds.includes(taskId)) observedKnownTaskIds.add(taskId);+ }+ session = live.session;+ }++ await t.require(+ notifiedTaskIds,+ satisfies(+ (ids: readonly string[]) =>+ ids.length === FANOUT_SIZE &&+ JSON.stringify([...ids].sort()) === JSON.stringify([...taskIds].sort()),+ "exactly one completed notification for every known task and none for unknown task ids",+ ),+ );+ t.noFailedActions();+ },+});++interface BlockedFanout {+ readonly observedTurns: readonly EveEvalTurn[];+ readonly requests: readonly InputRequest[];+ readonly session: TaskEvalSessionDriver;+}++async function waitForReleaseRequests(+ t: EveEvalContext,+ initialSession: TaskEvalSessionDriver,+ initialTurn: EveEvalTurn,+): Promise<BlockedFanout> {+ const requests = new Map<string, InputRequest>();+ const observedTurns = [initialTurn];+ let session = initialSession;+ collectReleaseRequests(initialTurn, requests);+ for (let attempt = 0; attempt < FANOUT_SIZE && requests.size < FANOUT_SIZE; attempt += 1) {+ const sessionId = session.sessionId;+ if (sessionId === undefined) throw new Error("Task fanout has no parent session id.");+ const live = t.target.watchTurn(sessionId, {+ startIndex: requireSessionStreamIndex(session, "Task fanout release wait"),+ });+ const turn = await live.result();+ observedTurns.push(turn);+ collectReleaseRequests(turn, requests);+ session = live.session;+ }+ if (requests.size !== FANOUT_SIZE) {+ throw new Error(`Expected ${FANOUT_SIZE} release requests; received ${requests.size}.`);+ }+ return { observedTurns, requests: [...requests.values()], session };+}++function collectReleaseRequests(turn: EveEvalTurn, requests: Map<string, InputRequest>): void {+ for (const request of turn.inputRequests) {+ if (request.action.toolName === "release") requests.set(request.requestId, request);+ }+}++function backgroundTaskIds(turn: EveEvalTurn): readonly string[] {+ return turn.events.flatMap((event) =>+ event.type === "subagent.completed" && event.data.backgroundTask !== undefined+ ? [event.data.backgroundTask.taskId]+ : [],+ );+}++function completedNotificationTaskIds(turn: EveEvalTurn): readonly string[] {+ return turn.events.flatMap((event) => {+ if (event.type !== "message.received") return [];+ return [...messageText(event.data.message).matchAll(COMPLETED_NOTIFICATION)].map(+ (match) => match[1] as string,+ );+ });+}++function messageText(message: unknown): string {+ if (typeof message === "string") return message;+ if (!Array.isArray(message)) return "";+ return message+ .flatMap((part) =>+ part !== null &&+ typeof part === "object" &&+ Reflect.get(part, "type") === "text" &&+ typeof Reflect.get(part, "text") === "string"+ ? [Reflect.get(part, "text") as string]+ : [],+ )+ .join("\n");+}diff --git a/e2e/fixtures/fixture-tasks/package.json b/e2e/fixtures/fixture-tasks/package.jsonnew file mode 100644index 000000000..c7c9ae730--- /dev/null+++ b/e2e/fixtures/fixture-tasks/package.json@@ -0,0 +1,24 @@+{+ "name": "fixture-tasks",+ "version": "0.0.0",+ "private": true,+ "type": "module",+ "scripts": {+ "build": "eve build",+ "dev": "eve dev",+ "start": "eve start",+ "typecheck": "pnpm validate:transitions && eve build && tsc",+ "validate:transitions": "node scripts/validate-transitions.mjs",+ "test:e2e": "eve eval --strict"+ },+ "dependencies": {+ "@eve-e2e/config": "workspace:*",+ "@workflow/world-postgres": "catalog:",+ "eve": "workspace:*",+ "zod": "catalog:"+ },+ "devDependencies": {+ "@types/node": "catalog:",+ "typescript": "catalog:"+ }+}diff --git a/e2e/fixtures/fixture-tasks/scripts/validate-transitions.mjs b/e2e/fixtures/fixture-tasks/scripts/validate-transitions.mjsnew file mode 100644index 000000000..907a2f64c--- /dev/null+++ b/e2e/fixtures/fixture-tasks/scripts/validate-transitions.mjs@@ -0,0 +1,53 @@+import { readdir, readFile } from "node:fs/promises";+import { dirname, join } from "node:path";+import { fileURLToPath } from "node:url";++const fixtureRoot = dirname(dirname(fileURLToPath(import.meta.url)));+const evalsRoot = join(fixtureRoot, "evals");+const files = await collectEvalFiles(evalsRoot);+const issues = [];++for (const file of files) {+ const source = await readFile(join(evalsRoot, file), "utf8");+ const anchors = [...source.matchAll(/primary:\s*"([^"]+)"/gu)].map((match) => match[1]);+ if (!source.includes("defineTaskEval(")) {+ issues.push(`${file}: task evals must use defineTaskEval().`);+ }+ if (source.includes("defineEval(")) {+ issues.push(`${file}: direct defineEval() bypasses transition validation.`);+ }+ if (anchors.length !== 1) {+ issues.push(+ `${file}: expected exactly one primary transition anchor; found ${anchors.length}.`,+ );+ continue;+ }+ const anchor = anchors[0];+ const fileName = file.split("/").at(-1);+ if (fileName !== `${anchor}.eval.ts` && !fileName?.startsWith(`${anchor}.`)) {+ issues.push(`${file}: filename must derive from primary anchor ${anchor}.`);+ }+ if (!source.includes("dimensions:")) {+ issues.push(`${file}: transition declaration must factor scenario dimensions.`);+ }+}++if (issues.length > 0) {+ process.stderr.write(`${issues.join("\n")}\n`);+ process.exitCode = 1;+} else {+ process.stdout.write(`Validated ${files.length} task eval transition declarations.\n`);+}++async function collectEvalFiles(directory, prefix = "") {+ const files = [];+ for (const entry of await readdir(directory, { withFileTypes: true })) {+ const relativePath = prefix === "" ? entry.name : `${prefix}/${entry.name}`;+ if (entry.isDirectory()) {+ files.push(...(await collectEvalFiles(join(directory, entry.name), relativePath)));+ } else if (entry.name.endsWith(".eval.ts")) {+ files.push(relativePath);+ }+ }+ return files.sort();+}diff --git a/e2e/fixtures/fixture-tasks/tsconfig.json b/e2e/fixtures/fixture-tasks/tsconfig.jsonnew file mode 100644index 000000000..85e3eceb8--- /dev/null+++ b/e2e/fixtures/fixture-tasks/tsconfig.json@@ -0,0 +1,17 @@+{+ "compilerOptions": {+ "target": "ES2022",+ "module": "esnext",+ "moduleResolution": "bundler",+ "outDir": "dist",+ "rootDir": ".",+ "strict": true,+ "esModuleInterop": true,+ "skipLibCheck": true,+ "forceConsistentCasingInFileNames": true,+ "declaration": true,+ "noEmit": true,+ "types": ["node"]+ },+ "include": ["agent/**/*.ts", "evals/**/*.ts"]+}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 24e125948..c9f88bc5f 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1035,6 +1035,28 @@ importers:specifier: 'catalog:'version: 7.0.2+ e2e/fixtures/fixture-tasks:+ dependencies:+ '@eve-e2e/config':+ specifier: workspace:*+ version: link:../e2e-config+ '@workflow/world-postgres':+ specifier: 'catalog:'+ version: 5.0.0-beta.33(@opentelemetry/[email protected])(@types/[email protected])(@upstash/[email protected])([email protected])([email protected])([email protected])+ eve:+ specifier: workspace:*+ version: link:../../../packages/eve+ zod:+ specifier: 'catalog:'+ version: 4.4.3+ devDependencies:+ '@types/node':+ specifier: 'catalog:'+ version: 25.9.1+ typescript:+ specifier: 'catalog:'+ version: 7.0.2+e2e/fixtures/gadget-extension:dependencies:slugify: