The Open Framework for Building Agents
test(eve): reproduce ctx.cancel() ignored for a typed answer to ctx.ask (#4744)
roboeve[bot]ee0c6c02aeec
diff --git a/.changeset/hook-cancel-typed-answer.md b/.changeset/hook-cancel-typed-answer.mdnew file mode 100644index 000000000..616484851--- /dev/null+++ b/.changeset/hook-cancel-typed-answer.md@@ -0,0 +1,5 @@+---+"eve": patch+---++`ctx.cancel()` in a `message.received` hook now cancels the turn when the message is a plain-text answer to a pending `ask_question` or `ctx.ask()` question. The question resolves as `cancelled` and the tool call never receives the text. Before, eve logged a warning and delivered the answer. `ctx.cancel()` in an `action.partial` hook for a workflow tool's report now cancels the turn waiting on that tool too.diff --git a/docs/guides/hooks.md b/docs/guides/hooks.mdindex 06e29e1fe..8f0ea0721 100644--- a/docs/guides/hooks.md+++ b/docs/guides/hooks.md@@ -219,6 +219,10 @@ export default defineHook({The remaining subscribers for the event still run. Then eve cancels the turn the same way [`session.cancel()`](./client/streaming) does: in-flight model and tool work is aborted, delegated child turns are cancelled, and the turn ends with `turn.cancelled` followed by `session.waiting`. No failure event is emitted. A cancel from `turn.started` or `step.started` takes effect before that model call. In a conversation, the next message starts a new turn. A delegated task reports the cancellation to its caller.+A plain-text message that answers a pending question, such as one from `ask_question`, is recorded as `message.received` before eve delivers it. Calling `ctx.cancel()` from that event's hook cancels the turn instead: the question resolves as `cancelled`, and the waiting tool call never receives the text.++The same goes for the `action.partial` a workflow tool reports while the turn waits on it: `ctx.cancel()` from its hook cancels the turn, and the run's result never reaches the model.+`ctx.cancel()` returns `void` rather than a promise. The turn stops after the hook returns, so there is nothing to await. Call it before the handler's promise settles: eve ignores a call from work the handler does not await and logs a warning.`ctx.cancel()` only stops a running turn. eve logs a warning and ignores the call on `step.failed`, `turn.completed`, `turn.failed`, `turn.cancelled`, `turn.waiting`, `session.waiting`, `session.completed`, `session.failed`, `context.cleared`, `task.started`, `task.settled`, and `agent.started`, and during clear or compact requests.diff --git a/packages/eve/src/execution/proxied-deliver-step.ts b/packages/eve/src/execution/proxied-deliver-step.tsindex 0a51a9ae3..2a2b85fac 100644--- a/packages/eve/src/execution/proxied-deliver-step.ts+++ b/packages/eve/src/execution/proxied-deliver-step.ts@@ -157,8 +157,30 @@ async function routeProxiedDeliver(}}+ let published: PublishedSessionEvents & { readonly cancelled: boolean } = {+ serializedContext,+ sessionState: input.sessionState,+ cancelled: false,+ };+ if (answerMessages.length > 0) {+ // Like a steering message, the answer joins the open turn, so the turn's+ // later events carry its delivery ids too. Its hooks run before the answer+ // goes down, so `ctx.cancel()` stops the turn instead of answering.+ published = await publishSessionEvents(+ {+ serializedContext: joinTurnDeliveryIds(published.serializedContext, answerDeliveryIds),+ sessionState: published.sessionState,+ sessionWritable: input.sessionWritable,+ },+ answerMessages,+ { canCancelTurn: true },+ );+ }+ // The turn owner lives outside this durable step and consumes its result.+ if (published.cancelled) parentAction = { kind: "cancel-turn" };+const answered: { event: PendingInputBatchEvent; resolutions: InputResolution[] }[] = [];- for (const child of children.values()) {+ for (const child of published.cancelled ? [] : children.values()) {if (child.workflowAsk !== undefined) {const responses = coalesceDeliverPayloads(child.payloads).inputResponses ?? [];await sendWorkflowAskAnswers(@@ -211,19 +233,6 @@ async function routeProxiedDeliver(parentAction === undefined &&[...requests.keys()].some((id) => !retired.has(id) && !forwarded.has(id));- let published: PublishedSessionEvents = { serializedContext, sessionState: input.sessionState };- if (answerMessages.length > 0) {- // Like a steering message, the answer joins the open turn, so the turn's- // later events carry its delivery ids too.- published = await publishSessionEvents(- {- serializedContext: joinTurnDeliveryIds(published.serializedContext, answerDeliveryIds),- sessionState: published.sessionState,- sessionWritable: input.sessionWritable,- },- answerMessages,- );- }const context = await commitSessionStep({ ...published, sessionWritable: input.sessionWritable },(view) => [routeAnswer(view, { children: answered, holds })],diff --git a/packages/eve/src/execution/publish-session-events.ts b/packages/eve/src/execution/publish-session-events.tsindex 33f964e70..32a4a61da 100644--- a/packages/eve/src/execution/publish-session-events.ts+++ b/packages/eve/src/execution/publish-session-events.ts@@ -22,6 +22,8 @@ import {import { resolveEffectiveAgentRuntime } from "#execution/effective-agent-config.js";import { reconcileSessionContinuationToken } from "#execution/reconcile-session-continuation-token.js";import { hydrateDurableSession } from "#execution/session.js";+import { isHookCancellableEvent } from "#execution/session/turn-event-handler.js";+import { TurnCancelledError } from "#harness/turn-cancellation.js";import {publishTransition,saveTransition,@@ -92,24 +94,37 @@ export interface PublishedSessionEvents {* Publishes a session step's own events, such as a workflow tool's* `action.partial`, exactly as a turn publishes its events. Call from a step* and adopt the result: hooks run in the session's context and may change it.+ * Set `canCancelTurn` only for events of a running turn. Cancellation is+ * returned with the updated state after the hooks finish, not thrown across+ * the durable step boundary.*/export async function publishSessionEvents(target: SessionStepState,events: readonly UnstampedMessageStreamEvent[],-): Promise<PublishedSessionEvents> {- return await publishEventsFromStep(target, "own", events);+ options?: { readonly canCancelTurn?: boolean },+): Promise<PublishedSessionEvents & { readonly cancelled: boolean }> {+ const hookCancellation = new AbortController();+ const published = await publishEventsFromStep(+ target,+ "own",+ events,+ options?.canCancelTurn ? () => hookCancellation.abort(new TurnCancelledError()) : undefined,+ );+ return { ...published, cancelled: hookCancellation.signal.aborted };}async function publishEventsFromStep(target: SessionStepState,origin: SessionEventOrigin,events: readonly UnstampedMessageStreamEvent[],+ cancelTurn: (() => void) | undefined,): Promise<PublishedSessionEvents> {if (events.length === 0) {return { serializedContext: target.serializedContext, sessionState: target.sessionState };}const restored = await restoreSessionStep(target);const { published } = await publishFromSessionStep(restored, {+ cancelTurn,origin,async publish(emit) {for (const event of events) await emit(event);@@ -145,6 +160,8 @@ export async function restoreSessionStep(step: SessionStepState): Promise<Restor*/export interface SessionStepPublication<T, R, S extends HarnessSessionBase = HarnessSessionBase> {readonly origin: SessionEventOrigin;+ /** Stops the running turn the events belong to, for a hook's `ctx.cancel()`. */+ readonly cancelTurn?: () => void;/** Where a relayed input batch came from; the channel adapter and its forwarding see it. */readonly inputSource?: string;/** Emits the step's events in the session's context scope. */@@ -238,6 +255,7 @@ async function publishInSessionScope<T>(: undefined;const publisher = openSessionEventPublisher({+ cancelTurn: publication.cancelTurn,ctx,inputSource: publication.inputSource,origin: publication.origin,@@ -279,8 +297,8 @@ export interface SessionEventDispatcher {/** The context delivery hands the channel adapter; a turn step also hands it to `adapter.deliver`. */readonly adapterCtx: ChannelAdapterContext;/**- * Runs the stream-event hooks for a written event. Only a turn step passes- * `cancelTurn`; see `turn-event-handler.ts`.+ * Runs the stream-event hooks for a written event. `cancelTurn` is passed+ * only for an event of a running turn; see `turn-event-handler.ts`.*/runHooks(event: MessageStreamEvent, cancelTurn?: () => void): Promise<void>;}@@ -308,6 +326,7 @@ export interface SessionEventPublisher {}export function openSessionEventPublisher(input: {+ readonly cancelTurn?: () => void;readonly ctx: ContextContainer;readonly origin: SessionEventOrigin;readonly sessionWritable: WritableStream<Uint8Array>;@@ -331,7 +350,11 @@ export function openSessionEventPublisher(input: {writer,emit,async publish(event) {- await dispatcher.runHooks(await emit(event));+ const emitted = await emit(event);+ await dispatcher.runHooks(+ emitted,+ isHookCancellableEvent(emitted.type) ? input.cancelTurn : undefined,+ );},};}diff --git a/packages/eve/src/execution/session/active-turn.ts b/packages/eve/src/execution/session/active-turn.tsindex e0f91fb3c..451df3437 100644--- a/packages/eve/src/execution/session/active-turn.ts+++ b/packages/eve/src/execution/session/active-turn.ts@@ -325,7 +325,8 @@ export class ActiveTurn {return steered;}- private abort(): void {+ /** Stops the turn, as `session.cancel()` does; a hook's `ctx.cancel()` uses it too. */+ abort(): void {if (this.controller.signal.aborted) return;this.controller.abort(new TurnCancelledError());}diff --git a/packages/eve/src/execution/session/turn.test.ts b/packages/eve/src/execution/session/turn.test.tsindex e78aa08df..6f9f27660 100644--- a/packages/eve/src/execution/session/turn.test.ts+++ b/packages/eve/src/execution/session/turn.test.ts@@ -1578,6 +1578,7 @@ const PUBLISHED = "test.published";/** Mocks the run-message publishing steps to record what they publish in the session context. */function publishIntoContext(): void {const append = (context: Record<string, unknown>, entry: string) => ({+ cancelled: false,serializedContext: {...context,[PUBLISHED]: [...((context[PUBLISHED] as string[] | undefined) ?? []), entry],diff --git a/packages/eve/src/execution/session/turn.ts b/packages/eve/src/execution/session/turn.tsindex 49e55671f..cbfd1a159 100644--- a/packages/eve/src/execution/session/turn.ts+++ b/packages/eve/src/execution/session/turn.ts@@ -145,23 +145,31 @@ export class SessionExecution {private async handleBoundaryMessages(messages: readonly WorkflowToolRunMessage[],ending = false,+ turn?: ActiveTurn,): Promise<void> {let applied: AppliedRunMessage[] = [];for (const message of messages) {if (!isAppliedRunMessage(message)) {- await this.applyRunMessages(applied);+ await this.applyRunMessages(applied, turn);applied = [];await handleWorkflowToolRunMessage({ cursor: this.input.cursor, message });} else if (!ending || message.kind !== "agent-started") {applied.push(message);}}- await this.applyRunMessages(applied);+ await this.applyRunMessages(applied, turn);}- private async applyRunMessages(messages: readonly AppliedRunMessage[]): Promise<void> {+ /** Hooks on the messages' events may cancel `turn`, the turn that is running. */+ private async applyRunMessages(+ messages: readonly AppliedRunMessage[],+ turn?: ActiveTurn,+ ): Promise<void> {if (messages.length === 0) return;- await this.input.cursor.advance((state) => applyRunMessagesStep({ ...state, messages }));+ const { cancelled } = await this.input.cursor.advance((state) =>+ applyRunMessagesStep({ ...state, canCancelTurn: turn !== undefined, messages }),+ );+ if (cancelled) turn?.abort();}private async runTurnSteps(@@ -197,7 +205,12 @@ export class SessionExecution {// reply to it.const stepAdmission = this.input.queue.nextAdmission;await turn.admitBoundary();- await this.handleBoundaryMessages(turn.takeBoundaryMessages(), result.action === "done");+ const ending = result.action === "done";+ await this.handleBoundaryMessages(+ turn.takeBoundaryMessages(),+ ending,+ ending ? undefined : turn,+ );if (result.action === "cancelled") return await this.finishCancelledTurn(turn);if (!turnCompleted && turn.signal.aborted && pendingCallIds === undefined) {@@ -291,7 +304,7 @@ export class SessionExecution {if (isAppliedRunMessage(message)) {const following =turn?.takeAppliedRunMessages() ?? takeLeadingAppliedRunMessages(this.input.inbox);- await this.applyRunMessages([message, ...following]);+ await this.applyRunMessages([message, ...following], turn);return undefined;}return await handleWorkflowToolRunMessage({diff --git a/packages/eve/src/execution/tools/workflow/apply-run-messages-step.ts b/packages/eve/src/execution/tools/workflow/apply-run-messages-step.tsindex 0607094e7..1c2f733cc 100644--- a/packages/eve/src/execution/tools/workflow/apply-run-messages-step.ts+++ b/packages/eve/src/execution/tools/workflow/apply-run-messages-step.ts@@ -65,19 +65,30 @@ export type AppliedRunMessage =/*** Applies run messages in the order they arrived, in one step however many* there are: a fan-out's starts, reports, and replies would otherwise cost a- * step each.+ * step each. Set `canCancelTurn` while a turn runs: a hook's `ctx.cancel()`+ * on a report then returns `cancelled` for the turn to stop.*/export async function applyRunMessagesStep(- input: SessionStepState & { readonly messages: readonly AppliedRunMessage[] },-): Promise<SessionStateTransition> {+ input: SessionStepState & {+ readonly canCancelTurn?: boolean;+ readonly messages: readonly AppliedRunMessage[];+ },+): Promise<SessionStateTransition & { readonly cancelled: boolean }> {"use step";- return await withSessionStateDelta(input, async ({ messages, ...target }) => {+ return await withSessionStateDelta(input, async ({ canCancelTurn, messages, ...target }) => {let state: SessionStepState = target;+ let cancelled = false;// Consecutive starts and reports publish together, as one turn's events do.let events: UnstampedMessageStreamEvent[] = [];const publish = async (): Promise<void> => {if (events.length === 0) return;- state = { ...state, ...(await publishSessionEvents(state, events)) };+ const published = await publishSessionEvents(state, events, { canCancelTurn });+ state = {+ ...state,+ serializedContext: published.serializedContext,+ sessionState: published.sessionState,+ };+ cancelled ||= published.cancelled;events = [];};for (const message of messages) {@@ -91,6 +102,10 @@ export async function applyRunMessagesStep(}}await publish();- return { serializedContext: state.serializedContext, sessionState: state.sessionState };+ return {+ cancelled,+ serializedContext: state.serializedContext,+ sessionState: state.sessionState,+ };});}diff --git a/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts b/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.tsindex f2d6ddd8b..8210757a2 100644--- a/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts+++ b/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts@@ -26,6 +26,7 @@ import {workflowContextMisuseWorkflow,} from "#internal/testing/workflow-tool-fixtures.js";import type { InputRequestedStreamEvent, MessageStreamEvent } from "#protocol/message.js";+import { defineHook } from "#public/definitions/hook.js";import {buildWorkflowToolSerializedContext,createWorkflowToolRuntime,@@ -323,6 +324,86 @@ describe("workflow tools", () => {});}, 60_000);+ it("lets a message.received hook cancel a typed answer before it resolves ctx.ask", async () => {+ const hooksAfterCancel: string[] = [];+ const runtime = await createWorkflowToolRuntime({+ agentName: "workflow-tool-guarded-answer",+ execute: confirmDeployWorkflow,+ modules: [+ {+ logicalPath: "hooks/guard.ts",+ loadNamespace: async () => ({+ default: defineHook({+ events: {+ "message.received"(event, ctx) {+ if (event.data.message === "Deploy") ctx.cancel();+ },+ },+ }),+ }),+ },+ {+ logicalPath: "hooks/observe.ts",+ loadNamespace: async () => ({+ default: defineHook({+ events: {+ async "message.received"(event) {+ if (event.data.message !== "Deploy") return;+ await Promise.resolve();+ hooksAfterCancel.push(event.data.message);+ },+ },+ }),+ }),+ },+ ],+ toolName: "confirm_deploy",+ });++ await runtime.run(async () => {+ const run = await startSessionOwner(workflowEntry, [+ {+ kind: "initial",+ ownerDeploymentId: "dpl_inline",+ input: { message: 'Run confirm_deploy with service "api"' },+ serializedContext: buildWorkflowToolSerializedContext({+ continuationToken: "http:workflow-tool-guarded-answer",+ requestInput: true,+ }),+ },+ ]);+ const stream = captureTurnEvents(run);+ try {+ const asked = await stream.nextTurn();+ expect(filterEventsByType(asked, "input.requested")).toHaveLength(1);+ await resumeSessionInbox(sessionCommandHookToken(run.runId), {+ delivery: { channelKind: "http", channelName: "test", deliveryId: "delivery-answer" },+ kind: "send",+ payload: { message: "Deploy" },+ });++ const answered = await stream.nextTurn();+ expect(hooksAfterCancel).toEqual(["Deploy"]);+ // The hook rejected the message, so it must not reach the waiting call.+ const outcomes = filterEventsByType(answered, "input.resolved").flatMap((event) =>+ event.data.resolutions.map((resolution) => resolution.outcome),+ );+ expect(outcomes).not.toContain("answered");+ const outputs = filterEventsByType(answered, "action.result").map(+ (event) => event.data.result.output,+ );+ expect(outputs).not.toContainEqual({ approved: true, service: "api" });+ expect(+ containsEventSequence(answered, ["turn.cancelled", "session.waiting"]),+ JSON.stringify(answered.map((event) => event.type)),+ ).toBe(true);+ } finally {+ stream.dispose();+ await run.cancel();+ }+ });+ }, 60_000);+it("exposes the principal that answered ctx.ask", async () => {const alice = {attributes: {},@@ -606,6 +687,58 @@ describe("workflow tools", () => {});}, 30_000);+ it("lets an action.partial hook cancel the turn waiting on the reporting run", async () => {+ const runtime = await createWorkflowToolRuntime({+ agentName: "workflow-tool-guarded-progress",+ execute: reportingDeployWorkflow,+ modules: [+ {+ logicalPath: "hooks/guard.ts",+ loadNamespace: async () => ({+ default: defineHook({+ events: {+ "action.partial"(_event, ctx) {+ ctx.cancel();+ },+ },+ }),+ }),+ },+ ],+ toolName: "deploy_service",+ });++ await runtime.run(async () => {+ const run = await startSessionOwner(workflowEntry, [+ {+ kind: "initial",+ ownerDeploymentId: "dpl_inline",+ input: { message: 'Run deploy_service with service "api"' },+ serializedContext: buildWorkflowToolSerializedContext({+ continuationToken: "http:workflow-tool-guarded-progress",+ }),+ },+ ]);+ const stream = captureTurnEvents(run);+ try {+ const settled = await stream.nextTurn();+ expect(filterEventsByType(settled, "action.partial")).toHaveLength(1);+ // The hook stopped the turn, so the run's return must not reach the model.+ const outputs = filterEventsByType(settled, "action.result").map(+ (event) => event.data.result.output,+ );+ expect(outputs).not.toContainEqual({ plan: "plan:api" });+ expect(+ containsEventSequence(settled, ["turn.cancelled", "session.waiting"]),+ JSON.stringify(settled.map((event) => event.type)),+ ).toBe(true);+ } finally {+ stream.dispose();+ await run.cancel();+ }+ });+ }, 30_000);+it("keeps the conversation out of the step input of a session step that only publishes", async () => {const runtime = await createWorkflowToolRuntime({agentName: "workflow-tool-step-input",diff --git a/packages/eve/src/internal/testing/workflow-tool-run-harness.ts b/packages/eve/src/internal/testing/workflow-tool-run-harness.tsindex f96fd9bd1..d5caa64df 100644--- a/packages/eve/src/internal/testing/workflow-tool-run-harness.ts+++ b/packages/eve/src/internal/testing/workflow-tool-run-harness.ts@@ -1,3 +1,4 @@+import type { ProgrammaticAgentModule } from "#compiler/source-graph.js";import { type TestRuntime, createTestRuntime } from "#internal/testing/app-harness.js";import { createBundledRuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js";import type { ResolvedToolDefinition } from "#runtime/types.js";@@ -49,11 +50,14 @@ export async function createWorkflowToolRuntime(input: {readonly agentName: string;readonly execute: (...args: never[]) => unknown;readonly inputSchema?: ResolvedToolDefinition["inputSchema"];+ /** Other authored modules, such as hooks, compiled next to the tool. */+ readonly modules?: readonly ProgrammaticAgentModule[];readonly toolName: string;}): Promise<TestRuntime> {return await createTestRuntime({agent: { name: input.agentName },modules: [+ ...(input.modules ?? []),{logicalPath: `tools/${input.toolName}.ts`,loadNamespace: async () => ({