Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): report turns waiting on runtime calls (#4498)

Rui3e7bbb2b2ccc
diff --git a/.changeset/runtime-turn-waiting.md b/.changeset/runtime-turn-waiting.md
new file mode 100644
index 000000000..832fdbc54
--- /dev/null
+++ b/.changeset/runtime-turn-waiting.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Emit `turn.waiting` with `on: "tasks"` when a model step parks on blocking workflow execute calls, including calls dispatched after approval. Task and agent receipt dispatches do not emit a wait; their actual task waits report the park.
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md b/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md
index 5b4e4578a..0b5606206 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md
@@ -26,6 +26,8 @@ and [`agent-workflow-tools/evals/hitl`](../../../agent-workflow-tools/evals/hitl
| Rule | Eval |
| ---------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
| Approve runs the call in eve, result before the next step | [approve-runs-the-call](./approvals/approve-runs-the-call.eval.ts) |
+| Approval beside runtime work reports tasks, then input | [approval-runtime-park-emits-waiting](./continuation/approval-runtime-park-emits-waiting.eval.ts) |
+| Runtime workflow parks report waiting on tasks | [runtime-park-emits-waiting](./continuation/runtime-park-emits-waiting.eval.ts) |
| Deny resolves `denied` and records a not-run result | [deny-records-not-run](./approvals/deny-records-not-run.eval.ts) |
| A partial answer keeps the turn held, with no model call | [partial-answer-keeps-turn-held](./approvals/partial-answer-keeps-turn-held.eval.ts) |
| A typed reply naming an option answers | [typed-approve-answers](./approvals/typed-approve-answers.eval.ts) |
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/approval-runtime-park-emits-waiting.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/approval-runtime-park-emits-waiting.eval.ts
new file mode 100644
index 000000000..349fb806f
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/approval-runtime-park-emits-waiting.eval.ts
@@ -0,0 +1,24 @@
+import { defineEval } from "eve/evals";
+import { scriptedSession, requestFrom } from "./helpers.ts";
+
+export default defineEval({
+ description:
+ "An approval beside a running workflow reports tasks, then input when the workflow settles.",
+ tags: ["hitl", "continuation", "workflow", "input-response"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const turn = await t.send(
+ "Prepare change A and read the workflow draft together.",
+ scriptedSession,
+ );
+ requestFrom(turn, "change-a");
+ turn.eventOrder([
+ { type: "turn.started", count: 1 },
+ { type: "turn.waiting", data: { on: "tasks" }, count: 1 },
+ { type: "action.result", data: { result: { toolName: "workflow-draft" } }, count: 1 },
+ { type: "input.requested", count: 1 },
+ { type: "turn.waiting", data: { on: "input" }, count: 1 },
+ ]);
+ turn.notEvent("turn.completed");
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/runtime-park-emits-waiting.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/runtime-park-emits-waiting.eval.ts
new file mode 100644
index 000000000..cc07d7424
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/continuation/runtime-park-emits-waiting.eval.ts
@@ -0,0 +1,19 @@
+import { defineEval } from "eve/evals";
+import { scriptedSession, expectReply } from "./helpers.ts";
+
+export default defineEval({
+ description: "A turn parked on a runtime workflow reports waiting on tasks before its result.",
+ tags: ["hitl", "continuation", "workflow"],
+ timeoutMs: 60_000,
+ async test(t) {
+ const session = await t.session();
+ const live = await session.start("Read the draft through a workflow.", scriptedSession);
+ const turn = await expectReply(t, live, "Workflow draft status: ready.");
+ turn.eventOrder([
+ { type: "turn.started", count: 1 },
+ { type: "turn.waiting", data: { on: "tasks" }, count: 1 },
+ { type: "action.result", data: { result: { toolName: "workflow-draft" } }, count: 1 },
+ { type: "turn.completed", count: 1 },
+ ]);
+ },
+});
diff --git a/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization-close.integration.test.ts b/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization-close.integration.test.ts
index 963b32f6a..8f07aaeef 100644
--- a/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization-close.integration.test.ts
+++ b/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization-close.integration.test.ts
@@ -48,7 +48,9 @@ describe("workflow step authorization failures", () => {
const stream = captureTurnEvents(run);
try {
// The sign-in parks the open turn; the run keeps waiting for its callback.
- const events = await stream.nextUntil((event) => event.type === "turn.waiting");
+ const events = await stream.nextUntil(
+ (event) => event.type === "turn.waiting" && event.data.on === "input",
+ );
expect(events.at(-2)?.type).toBe("authorization.required");
expect(filterEventsByType(events, "turn.completed")).toHaveLength(0);
const required = filterEventsByType(events, "authorization.required")[0]!;
diff --git a/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization.integration.test.ts b/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization.integration.test.ts
index 5da3d89f7..acf60d4aa 100644
--- a/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization.integration.test.ts
+++ b/packages/eve/src/execution/tools/workflow/workflow-tool-run-authorization.integration.test.ts
@@ -87,7 +87,9 @@ describe("workflow step authorization", () => {
const stream = captureTurnEvents(run);
try {
// The sign-in parks the open turn; the run keeps waiting for its callback.
- const events = await stream.nextUntil((event) => event.type === "turn.waiting");
+ const events = await stream.nextUntil(
+ (event) => event.type === "turn.waiting" && event.data.on === "input",
+ );
expect(events.at(-2)?.type).toBe("authorization.required");
expect(filterEventsByType(events, "turn.completed")).toHaveLength(0);
const required = filterEventsByType(events, "authorization.required")[0]!;
diff --git a/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts b/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts
index ed2e45d47..3d77130e7 100644
--- a/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts
+++ b/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts
@@ -202,7 +202,9 @@ describe("workflow tools", () => {
});
expect(request.options?.map((option) => option.id)).toEqual(["approve", "cancel"]);
// The call is still running, so the question parks the open turn.
- const [parked] = filterEventsByType(asked, "turn.waiting");
+ const parked = filterEventsByType(asked, "turn.waiting").find(
+ (event) => event.data.on === "input",
+ );
expect(asked.at(-1)).toBe(parked);
expect(filterEventsByType(asked, "turn.completed")).toHaveLength(0);
diff --git a/packages/eve/src/harness/hitl/index.ts b/packages/eve/src/harness/hitl/index.ts
index 522aef459..9dfa2165e 100644
--- a/packages/eve/src/harness/hitl/index.ts
+++ b/packages/eve/src/harness/hitl/index.ts
@@ -5,7 +5,7 @@ import { AuthKey, SessionKey } from "#context/keys.js";
import { clearPendingAuthorization } from "#harness/authorization.js";
import type { resolveInlineAuthorizationInterrupt } from "#harness/inline-tool-authorization.js";
import { validateHarnessModelMessages } from "#harness/messages.js";
-import { fail } from "#harness/session-machine/transitions.js";
+import { fail, hold } from "#harness/session-machine/transitions.js";
import { readTurnState, writeTurnState } from "#harness/session-machine/state.js";
import type { StepCoordinates } from "#harness/session-machine/view.js";
import type { Step } from "#harness/step/context.js";
@@ -62,8 +62,8 @@ export async function parkOnApprovals(
readonly tasks: readonly RuntimeWorkflowTaskRequest[];
/** The entries the step's calls ran, whose approvals the requests ask for. */
readonly tools: HarnessToolLookup;
- /** The step made calls the runtime runs, so the turn waits on them instead. */
- readonly waitsOnRuntime: boolean;
+ /** Runtime calls run first, or the caller will report a sibling sign-in hold. */
+ readonly waitsOn: "runtime" | "sign-in" | undefined;
},
): Promise<StepResult> {
const transition = parkOnApprovalsTransition(step.view(), {
@@ -75,7 +75,14 @@ export async function parkOnApprovals(
responseAuthRequiredRequestIds: responsePolicyRequestIds(input.tools, input.requests),
});
await step.apply(transition, [...step.session.history, ...(transition.commit ?? [])]);
- if (input.waitsOnRuntime) return { next: null, session: step.session };
+ if (input.waitsOn !== undefined) {
+ if (
+ input.waitsOn === "runtime" &&
+ input.tasks.some((task) => task.entry.entryPoint === "execute")
+ )
+ await step.apply(hold(step.view(), { on: "tasks" }));
+ return { next: null, session: step.session };
+ }
if (hasRunnableQueue(step.view())) return { next: step.runStep, session: step.session };
return holdForInput(step);
}
diff --git a/packages/eve/src/harness/hitl/intake.ts b/packages/eve/src/harness/hitl/intake.ts
index 59555a56c..1edd6dbbd 100644
--- a/packages/eve/src/harness/hitl/intake.ts
+++ b/packages/eve/src/harness/hitl/intake.ts
@@ -282,6 +282,9 @@ export async function dispatchApprovedWorkflows(step: Step, work: ApprovedWork):
await step.apply(
dispatch(step.view(), { approvers: approversOf(approved, step.session.state), tasks }),
);
+ if (tasks.some((task) => task.entry.entryPoint === "execute")) {
+ await step.apply(hold(step.view(), { on: "tasks" }));
+ }
return true;
}
diff --git a/packages/eve/src/harness/step/after-model.ts b/packages/eve/src/harness/step/after-model.ts
index 0cd91470f..69ce18296 100644
--- a/packages/eve/src/harness/step/after-model.ts
+++ b/packages/eve/src/harness/step/after-model.ts
@@ -167,7 +167,11 @@ export async function handleStepResult(step: Step, input: ModelResponse): Promis
...parked,
requests: approvalRequests,
tools: input.catalog,
- waitsOnRuntime: workflowToolCalls.length > 0 || authorizationInterrupt !== undefined,
+ waitsOn: authorizationInterrupt
+ ? "sign-in"
+ : workflowToolCalls.length > 0
+ ? "runtime"
+ : undefined,
});
// Approval only precedes sign-in for the same call, not an executing sibling.
if (authorizationInterrupt) {
@@ -185,6 +189,9 @@ export async function handleStepResult(step: Step, input: ModelResponse): Promis
history: step.session.history,
});
}
+ if (dispatched.workflowRequests.some((task) => task.entry.entryPoint === "execute")) {
+ await step.apply(hold(step.view(), { on: "tasks" }));
+ }
return { next: null, session: step.session };
}
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index 651906695..dfe8c1671 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -1686,6 +1686,82 @@ describe("createToolLoopHarness", () => {
]);
});
+ it("approval, execute, and sign-in siblings emit one input wait for the park", async () => {
+ const signal = requestAuthorization([
+ {
+ attemptId: "sibling-attempt",
+ name: "probe",
+ challenge: {
+ url: "https://idp.example/auth",
+ instructions: "Sign in",
+ userCode: "ABC",
+ },
+ hookUrl: "https://app.example/callback",
+ principal: { type: "app" },
+ resume: { nonce: "n1" },
+ },
+ ]);
+ const output = modelFacingAuthorizationOutput(signal);
+ const gate = { type: "tool-call" as const, toolCallId: "gate-1", toolName: "add", input: {} };
+ const work = {
+ type: "tool-call" as const,
+ toolCallId: "delegate-1",
+ toolName: "delegate",
+ input: { message: "probe" },
+ };
+ const auth = {
+ type: "tool-call" as const,
+ toolCallId: "probe-1",
+ toolName: "probe",
+ input: {},
+ };
+ const approval = {
+ approvalId: "approval-gate",
+ toolCallId: "gate-1",
+ type: "tool-approval-request" as const,
+ };
+ const authResult = {
+ type: "tool-result" as const,
+ toolCallId: "probe-1",
+ toolName: "probe",
+ output,
+ };
+ setupMockAgent({
+ content: [gate, approval, work, auth],
+ finishReason: "tool-calls",
+ response: {
+ messages: [
+ { role: "assistant", content: [gate, approval, work, auth] },
+ { role: "tool", content: [authResult] },
+ ],
+ },
+ text: "",
+ toolCalls: [gate, work, auth],
+ toolResults: [authResult],
+ });
+ const tools = new Map(withBash(createDelegationToolMap()));
+ tools.set("probe", {
+ name: "probe",
+ description: "Probe",
+ inputSchema: jsonSchema({ type: "object" }),
+ execute: vi.fn(),
+ });
+ const { emit, events } = createEventCollector();
+ const run = createToolLoopHarness(createTestConfig(emit, { tools }));
+ const ctx = new ContextContainer();
+ stashToolInterrupt(ctx, "probe-1", signal);
+ const result = await contextStorage.run(ctx, () =>
+ run(createPendingBashApprovalSession(), {
+ inputResponses: [{ optionId: "approve", requestId: "approval-1" }],
+ }),
+ );
+ expect(runtimeWait(result.session.state)?.tasks[0]?.entry.entryPoint).toBe("execute");
+ expect(events.filter((event) => event.type === "authorization.required")).toHaveLength(1);
+ const waits = events.filter((event) => event.type === "turn.waiting");
+ expect(waits).toHaveLength(1);
+ expect(waits[0]?.data.on).toBe("input");
+ });
+
it("parks on both batches when one step carries a workflow task and an approval", async () => {
mockApprovalAlongsideWorkflowTask();