The Open Framework for Building Agents
fix(eve): report turns waiting on runtime calls (#4498)
Rui3e7bbb2b2ccc
diff --git a/.changeset/runtime-turn-waiting.md b/.changeset/runtime-turn-waiting.mdnew file mode 100644index 000000000..832fdbc54--- /dev/null+++ b/.changeset/runtime-turn-waiting.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Emit `turn.waiting` with `on: "tasks"` when a model step parks on blocking workflow execute calls, including calls dispatched after approval. Task and agent receipt dispatches do not emit a wait; their actual task waits report the park.diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md b/e2e/fixtures/agent-tools-hitl/evals/hitl/README.mdindex 5b4e4578a..0b5606206 100644--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md@@ -26,6 +26,8 @@ and [`agent-workflow-tools/evals/hitl`](../../../agent-workflow-tools/evals/hitl| Rule | Eval || ---------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- || Approve runs the call in eve, result before the next step | [approve-runs-the-call](./approvals/approve-runs-the-call.eval.ts) |+| Approval beside runtime work reports tasks, then input | [approval-runtime-park-emits-waiting](./continuation/approval-runtime-park-emits-waiting.eval.ts) |+| Runtime workflow parks report waiting on tasks | [runtime-park-emits-waiting](./continuation/runtime-park-emits-waiting.eval.ts) || Deny resolves `denied` and records a not-run result | [deny-records-not-run](./approvals/deny-records-not-run.eval.ts) || A partial answer keeps the turn held, with no model call | [partial-answer-keeps-turn-held](./approvals/partial-answer-keeps-turn-held.eval.ts) || A typed reply naming an option answers | [typed-approve-answers](./approvals/typed-approve-answers.eval.ts) |diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/approval-runtime-park-emits-waiting.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/approval-runtime-park-emits-waiting.eval.tsnew file mode 100644index 000000000..349fb806f--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/approval-runtime-park-emits-waiting.eval.ts@@ -0,0 +1,24 @@+import { defineEval } from "eve/evals";+import { scriptedSession, requestFrom } from "./helpers.ts";++export default defineEval({+ description:+ "An approval beside a running workflow reports tasks, then input when the workflow settles.",+ tags: ["hitl", "continuation", "workflow", "input-response"],+ timeoutMs: 60_000,+ async test(t) {+ const turn = await t.send(+ "Prepare change A and read the workflow draft together.",+ scriptedSession,+ );+ requestFrom(turn, "change-a");+ turn.eventOrder([+ { type: "turn.started", count: 1 },+ { type: "turn.waiting", data: { on: "tasks" }, count: 1 },+ { type: "action.result", data: { result: { toolName: "workflow-draft" } }, count: 1 },+ { type: "input.requested", count: 1 },+ { type: "turn.waiting", data: { on: "input" }, count: 1 },+ ]);+ turn.notEvent("turn.completed");+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/runtime-park-emits-waiting.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/runtime-park-emits-waiting.eval.tsnew file mode 100644index 000000000..cc07d7424--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/runtime-park-emits-waiting.eval.ts@@ -0,0 +1,19 @@+import { defineEval } from "eve/evals";+import { scriptedSession, expectReply } from "./helpers.ts";++export default defineEval({+ description: "A turn parked on a runtime workflow reports waiting on tasks before its result.",+ tags: ["hitl", "continuation", "workflow"],+ timeoutMs: 60_000,+ async test(t) {+ const session = await t.session();+ const live = await session.start("Read the draft through a workflow.", scriptedSession);+ const turn = await expectReply(t, live, "Workflow draft status: ready.");+ turn.eventOrder([+ { type: "turn.started", count: 1 },+ { type: "turn.waiting", data: { on: "tasks" }, count: 1 },+ { type: "action.result", data: { result: { toolName: "workflow-draft" } }, count: 1 },+ { type: "turn.completed", count: 1 },+ ]);+ },+});diff --git a/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization-close.integration.test.ts b/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization-close.integration.test.tsindex 963b32f6a..8f07aaeef 100644--- a/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization-close.integration.test.ts+++ b/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization-close.integration.test.ts@@ -48,7 +48,9 @@ describe("workflow step authorization failures", () => {const stream = captureTurnEvents(run);try {// The sign-in parks the open turn; the run keeps waiting for its callback.- const events = await stream.nextUntil((event) => event.type === "turn.waiting");+ const events = await stream.nextUntil(+ (event) => event.type === "turn.waiting" && event.data.on === "input",+ );expect(events.at(-2)?.type).toBe("authorization.required");expect(filterEventsByType(events, "turn.completed")).toHaveLength(0);const required = filterEventsByType(events, "authorization.required")[0]!;diff --git a/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization.integration.test.ts b/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization.integration.test.tsindex 5da3d89f7..acf60d4aa 100644--- a/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization.integration.test.ts+++ b/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization.integration.test.ts@@ -87,7 +87,9 @@ describe("workflow step authorization", () => {const stream = captureTurnEvents(run);try {// The sign-in parks the open turn; the run keeps waiting for its callback.- const events = await stream.nextUntil((event) => event.type === "turn.waiting");+ const events = await stream.nextUntil(+ (event) => event.type === "turn.waiting" && event.data.on === "input",+ );expect(events.at(-2)?.type).toBe("authorization.required");expect(filterEventsByType(events, "turn.completed")).toHaveLength(0);const required = filterEventsByType(events, "authorization.required")[0]!;diff --git a/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts b/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.tsindex ed2e45d47..3d77130e7 100644--- a/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts+++ b/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts@@ -202,7 +202,9 @@ describe("workflow tools", () => {});expect(request.options?.map((option) => option.id)).toEqual(["approve", "cancel"]);// The call is still running, so the question parks the open turn.- const [parked] = filterEventsByType(asked, "turn.waiting");+ const parked = filterEventsByType(asked, "turn.waiting").find(+ (event) => event.data.on === "input",+ );expect(asked.at(-1)).toBe(parked);expect(filterEventsByType(asked, "turn.completed")).toHaveLength(0);diff --git a/packages/eve/src/harness/hitl/index.ts b/packages/eve/src/harness/hitl/index.tsindex 522aef459..9dfa2165e 100644--- a/packages/eve/src/harness/hitl/index.ts+++ b/packages/eve/src/harness/hitl/index.ts@@ -5,7 +5,7 @@ import { AuthKey, SessionKey } from "#context/keys.js";import { clearPendingAuthorization } from "#harness/authorization.js";import type { resolveInlineAuthorizationInterrupt } from "#harness/inline-tool-authorization.js";import { validateHarnessModelMessages } from "#harness/messages.js";-import { fail } from "#harness/session-machine/transitions.js";+import { fail, hold } from "#harness/session-machine/transitions.js";import { readTurnState, writeTurnState } from "#harness/session-machine/state.js";import type { StepCoordinates } from "#harness/session-machine/view.js";import type { Step } from "#harness/step/context.js";@@ -62,8 +62,8 @@ export async function parkOnApprovals(readonly tasks: readonly RuntimeWorkflowTaskRequest[];/** The entries the step's calls ran, whose approvals the requests ask for. */readonly tools: HarnessToolLookup;- /** The step made calls the runtime runs, so the turn waits on them instead. */- readonly waitsOnRuntime: boolean;+ /** Runtime calls run first, or the caller will report a sibling sign-in hold. */+ readonly waitsOn: "runtime" | "sign-in" | undefined;},): Promise<StepResult> {const transition = parkOnApprovalsTransition(step.view(), {@@ -75,7 +75,14 @@ export async function parkOnApprovals(responseAuthRequiredRequestIds: responsePolicyRequestIds(input.tools, input.requests),});await step.apply(transition, [...step.session.history, ...(transition.commit ?? [])]);- if (input.waitsOnRuntime) return { next: null, session: step.session };+ if (input.waitsOn !== undefined) {+ if (+ input.waitsOn === "runtime" &&+ input.tasks.some((task) => task.entry.entryPoint === "execute")+ )+ await step.apply(hold(step.view(), { on: "tasks" }));+ return { next: null, session: step.session };+ }if (hasRunnableQueue(step.view())) return { next: step.runStep, session: step.session };return holdForInput(step);}diff --git a/packages/eve/src/harness/hitl/intake.ts b/packages/eve/src/harness/hitl/intake.tsindex 59555a56c..1edd6dbbd 100644--- a/packages/eve/src/harness/hitl/intake.ts+++ b/packages/eve/src/harness/hitl/intake.ts@@ -282,6 +282,9 @@ export async function dispatchApprovedWorkflows(step: Step, work: ApprovedWork):await step.apply(dispatch(step.view(), { approvers: approversOf(approved, step.session.state), tasks }),);+ if (tasks.some((task) => task.entry.entryPoint === "execute")) {+ await step.apply(hold(step.view(), { on: "tasks" }));+ }return true;}diff --git a/packages/eve/src/harness/step/after-model.ts b/packages/eve/src/harness/step/after-model.tsindex 0cd91470f..69ce18296 100644--- a/packages/eve/src/harness/step/after-model.ts+++ b/packages/eve/src/harness/step/after-model.ts@@ -167,7 +167,11 @@ export async function handleStepResult(step: Step, input: ModelResponse): Promis...parked,requests: approvalRequests,tools: input.catalog,- waitsOnRuntime: workflowToolCalls.length > 0 || authorizationInterrupt !== undefined,+ waitsOn: authorizationInterrupt+ ? "sign-in"+ : workflowToolCalls.length > 0+ ? "runtime"+ : undefined,});// Approval only precedes sign-in for the same call, not an executing sibling.if (authorizationInterrupt) {@@ -185,6 +189,9 @@ export async function handleStepResult(step: Step, input: ModelResponse): Promishistory: step.session.history,});}+ if (dispatched.workflowRequests.some((task) => task.entry.entryPoint === "execute")) {+ await step.apply(hold(step.view(), { on: "tasks" }));+ }return { next: null, session: step.session };}diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex 651906695..dfe8c1671 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -1686,6 +1686,82 @@ describe("createToolLoopHarness", () => {]);});+ it("approval, execute, and sign-in siblings emit one input wait for the park", async () => {+ const signal = requestAuthorization([+ {+ attemptId: "sibling-attempt",+ name: "probe",+ challenge: {+ url: "https://idp.example/auth",+ instructions: "Sign in",+ userCode: "ABC",+ },+ hookUrl: "https://app.example/callback",+ principal: { type: "app" },+ resume: { nonce: "n1" },+ },+ ]);+ const output = modelFacingAuthorizationOutput(signal);+ const gate = { type: "tool-call" as const, toolCallId: "gate-1", toolName: "add", input: {} };+ const work = {+ type: "tool-call" as const,+ toolCallId: "delegate-1",+ toolName: "delegate",+ input: { message: "probe" },+ };+ const auth = {+ type: "tool-call" as const,+ toolCallId: "probe-1",+ toolName: "probe",+ input: {},+ };+ const approval = {+ approvalId: "approval-gate",+ toolCallId: "gate-1",+ type: "tool-approval-request" as const,+ };+ const authResult = {+ type: "tool-result" as const,+ toolCallId: "probe-1",+ toolName: "probe",+ output,+ };+ setupMockAgent({+ content: [gate, approval, work, auth],+ finishReason: "tool-calls",+ response: {+ messages: [+ { role: "assistant", content: [gate, approval, work, auth] },+ { role: "tool", content: [authResult] },+ ],+ },+ text: "",+ toolCalls: [gate, work, auth],+ toolResults: [authResult],+ });+ const tools = new Map(withBash(createDelegationToolMap()));+ tools.set("probe", {+ name: "probe",+ description: "Probe",+ inputSchema: jsonSchema({ type: "object" }),+ execute: vi.fn(),+ });+ const { emit, events } = createEventCollector();+ const run = createToolLoopHarness(createTestConfig(emit, { tools }));+ const ctx = new ContextContainer();+ stashToolInterrupt(ctx, "probe-1", signal);+ const result = await contextStorage.run(ctx, () =>+ run(createPendingBashApprovalSession(), {+ inputResponses: [{ optionId: "approve", requestId: "approval-1" }],+ }),+ );+ expect(runtimeWait(result.session.state)?.tasks[0]?.entry.entryPoint).toBe("execute");+ expect(events.filter((event) => event.type === "authorization.required")).toHaveLength(1);+ const waits = events.filter((event) => event.type === "turn.waiting");+ expect(waits).toHaveLength(1);+ expect(waits[0]?.data.on).toBe("input");+ });+it("parks on both batches when one step carries a workflow task and an approval", async () => {mockApprovalAlongsideWorkflowTask();