The Open Framework for Building Agents
fix(eve): instrument terminal session paths (#2819)
Chad Hietala3c16df6289e6
diff --git a/.changeset/terminal-traces-release.md b/.changeset/terminal-traces-release.mdnew file mode 100644index 000000000..ba8939c7e--- /dev/null+++ b/.changeset/terminal-traces-release.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Publish instrumentation lifecycle events when sessions expire, reset, close, or fail outside an agent turn. Terminal paths now flush providers and release local trace retention state.diff --git a/packages/eve/src/execution/agent-messaging.scenario.test.ts b/packages/eve/src/execution/agent-messaging.scenario.test.tsindex 744c19cef..e52474864 100644--- a/packages/eve/src/execution/agent-messaging.scenario.test.ts+++ b/packages/eve/src/execution/agent-messaging.scenario.test.ts@@ -194,6 +194,7 @@ describe("agent messaging", () => {await expectRetainedChildConversation({childSessionId,client: new Client({ host: server.url }),+ expectedCompletionCount: 0,});expect(await readWorkflowRunStatus(app.appRoot, childSessionId)).toBe("cancelled");} catch (error) {@@ -230,6 +231,7 @@ describe("agent messaging", () => {auth: { bearer: REMOTE_MEMORY_TOKEN },host: remoteServer.url,}),+ expectedCompletionCount: 1,});} catch (error) {throw new Error(@@ -310,6 +312,7 @@ async function runScriptedParentSession(input: {async function expectRetainedChildConversation(input: {readonly childSessionId: string;readonly client: Client;+ readonly expectedCompletionCount: number;}): Promise<void> {const childEvents = await collectStreamToEnd({label: "persisted child events",@@ -321,7 +324,9 @@ async function expectRetainedChildConversation(input: {expect(childTurnStarts).toHaveLength(2);expect(childWaits).toHaveLength(2);expect(childWaits[0]).toBeLessThan(childTurnStarts[1] ?? -1);- expect(filterEventsByType(childEvents, "session.completed")).toHaveLength(0);+ expect(filterEventsByType(childEvents, "session.completed")).toHaveLength(+ input.expectedCompletionCount,+ );expect(filterEventsByType(childEvents, "session.failed")).toHaveLength(0);expect(filterEventsByType(childEvents, "message.completed").some(diff --git a/packages/eve/src/execution/finalize-terminal-session.ts b/packages/eve/src/execution/finalize-terminal-session.tsnew file mode 100644index 000000000..992a999f1--- /dev/null+++ b/packages/eve/src/execution/finalize-terminal-session.ts@@ -0,0 +1,54 @@+import type { TurnCaller } from "#channel/types.js";+import {+ notifyDelegatedParentStep,+ notifyTurnCallerStep,+} from "#execution/delegated-parent-notification.js";+import { createDelegatedSubagentSuccessResult } from "#execution/delegated-parent-result.js";+import type { DurableSessionState } from "#execution/durable-session-store.js";+import { fireSessionCallbackStep } from "#execution/session-callback-step.js";+import { emitTerminalSessionCompletionStep } from "#execution/terminal-session-completion-step.js";+import { terminateChildSessionsStep } from "#execution/terminate-child-sessions-step.js";+import type { RunMode } from "#shared/run-mode.js";++/** Completes a session that terminates outside a turn. */+export async function finalizeTerminalSession(input: {+ readonly caller: TurnCaller | undefined;+ readonly driverWritable: WritableStream<Uint8Array>;+ readonly mode: RunMode;+ readonly notifyCaller: boolean;+ readonly serializedContext: Record<string, unknown>;+ readonly sessionState: DurableSessionState;+ readonly terminalState: { terminalEmitted: boolean };+}): Promise<{ readonly output: unknown }> {+ await terminateChildSessionsStep({+ serializedContext: input.serializedContext,+ sessionState: input.sessionState,+ });+ await emitTerminalSessionCompletionStep({+ parentWritable: input.driverWritable,+ serializedContext: input.serializedContext,+ });+ input.terminalState.terminalEmitted = true;++ if (!input.notifyCaller) return { output: "" };++ if (input.mode === "task") {+ await fireSessionCallbackStep({+ output: "",+ serializedContext: input.serializedContext,+ status: "completed",+ });+ await notifyDelegatedParentStep({+ result: createDelegatedSubagentSuccessResult(input.serializedContext, ""),+ serializedContext: input.serializedContext,+ });+ } else if (input.caller !== undefined) {+ await notifyTurnCallerStep({+ caller: input.caller,+ lifecycle: "terminal",+ sessionId: input.sessionState.sessionId,+ settled: { output: "" },+ });+ }+ return { output: "" };+}diff --git a/packages/eve/src/execution/settle-cancelled-turn-step.ts b/packages/eve/src/execution/settle-cancelled-turn-step.tsindex 54b9079ab..051c610c4 100644--- a/packages/eve/src/execution/settle-cancelled-turn-step.ts+++ b/packages/eve/src/execution/settle-cancelled-turn-step.ts@@ -110,7 +110,7 @@ export async function settleCancelledTurnStep(input: {});};const emit =- instrumentation?.createCancellationHandleEvent({+ instrumentation?.createHandleEvent({handleEvent: baseEmit,turnId: activeTurnId(emissionState),}) ?? baseEmit;diff --git a/packages/eve/src/execution/terminal-session-completion-step.ts b/packages/eve/src/execution/terminal-session-completion-step.tsindex 509765c4c..9ea12182c 100644--- a/packages/eve/src/execution/terminal-session-completion-step.ts+++ b/packages/eve/src/execution/terminal-session-completion-step.ts@@ -1,15 +1,5 @@-import { buildAdapterContext } from "#channel/adapter-context.js";-import { callAdapterEventHandler } from "#channel/adapter.js";-import { deserializeContext } from "#context/serialize.js";-import { createLogger } from "#internal/logging.js";-import {- createSessionCompletedEvent,- encodeMessageStreamEvent,- stampMessageStreamEvent,-} from "#protocol/message.js";-import { ChannelKey } from "#runtime/sessions/runtime-context-keys.js";--const log = createLogger("execution.workflow-entry");+import { emitTerminalSessionEvent } from "#execution/terminal-session-event.js";+import { createSessionCompletedEvent } from "#protocol/message.js";/** Emits a terminal `session.completed` outside a turn. */export async function emitTerminalSessionCompletionStep(input: {@@ -18,34 +8,5 @@ export async function emitTerminalSessionCompletionStep(input: {}): Promise<void> {"use step";- const event = createSessionCompletedEvent();- const sessionId = (input.serializedContext["eve.sessionId"] as string | undefined) ?? "";-- try {- const ctx = await deserializeContext(input.serializedContext);- const adapter = ctx.get(ChannelKey);- if (adapter !== undefined) {- const adapterCtx = buildAdapterContext(adapter, ctx);- await callAdapterEventHandler(adapter, event, adapterCtx);- }- } catch (notificationError) {- log.error("adapter failed to handle terminal session.completed event", {- error: notificationError,- sessionId,- });- }-- try {- const writer = input.parentWritable.getWriter();- try {- await writer.write(encodeMessageStreamEvent(stampMessageStreamEvent(event)));- } finally {- writer.releaseLock();- }- } catch (writeError) {- log.error("failed to write terminal session.completed event to durable stream", {- error: writeError,- sessionId,- });- }+ await emitTerminalSessionEvent({ ...input, event: createSessionCompletedEvent() });}diff --git a/packages/eve/src/execution/terminal-session-event.ts b/packages/eve/src/execution/terminal-session-event.tsnew file mode 100644index 000000000..5d25f020f--- /dev/null+++ b/packages/eve/src/execution/terminal-session-event.ts@@ -0,0 +1,123 @@+import { buildAdapterContext } from "#channel/adapter-context.js";+import { callAdapterEventHandler } from "#channel/adapter.js";+import { ContextContainer, contextStorage } from "#context/container.js";+import { ParentSessionKey } from "#context/keys.js";+import { deserializeContext } from "#context/serialize.js";+import { bindSessionInstrumentation } from "#instrumentation/runtime.js";+import type { HandleEventFn } from "#harness/types.js";+import { createLogger } from "#internal/logging.js";+import {+ encodeMessageStreamEvent,+ stampMessageStreamEvent,+ type UnstampedMessageStreamEvent,+} from "#protocol/message.js";+import { BundleKey, ChannelKey } from "#runtime/sessions/runtime-context-keys.js";++const log = createLogger("execution.workflow-entry");++type TerminalSessionEvent = Extract<+ UnstampedMessageStreamEvent,+ { type: "session.completed" | "session.failed" }+>;++/** Delivers one out-of-turn terminal event through the native instrumentation lifecycle. */+export async function emitTerminalSessionEvent(input: {+ readonly errorId?: string;+ readonly event: TerminalSessionEvent;+ readonly parentWritable: WritableStream<Uint8Array>;+ readonly serializedContext: Record<string, unknown>;+ readonly turnId?: string;+}): Promise<void> {+ const sessionId = (input.serializedContext["eve.sessionId"] as string | undefined) ?? "";+ let ctx: ContextContainer | undefined;++ try {+ ctx = await deserializeContext(input.serializedContext);+ } catch (error) {+ log.error(`failed to restore context for terminal ${input.event.type} event`, {+ error,+ errorId: input.errorId,+ sessionId,+ });+ }++ const handleEvent: HandleEventFn = async (event) => {+ if (ctx !== undefined) {+ const adapter = ctx.get(ChannelKey);+ if (adapter !== undefined) {+ try {+ await callAdapterEventHandler(adapter, event, buildAdapterContext(adapter, ctx));+ } catch (error) {+ log.error(`adapter failed to handle terminal ${event.type} event`, {+ error,+ errorId: input.errorId,+ sessionId,+ });+ }+ }+ }++ try {+ const writer = input.parentWritable.getWriter();+ try {+ await writer.write(encodeMessageStreamEvent(stampMessageStreamEvent(event)));+ } finally {+ writer.releaseLock();+ }+ } catch (error) {+ log.error(`failed to write terminal ${event.type} event to durable stream`, {+ error,+ errorId: input.errorId,+ sessionId,+ });+ }+ };++ let instrumentation: ReturnType<typeof bindSessionInstrumentation>;+ if (ctx !== undefined) {+ try {+ const bundle = ctx.require(BundleKey);+ instrumentation = bindSessionInstrumentation({+ agentName: bundle.turnAgent.id,+ ctx,+ rootSessionId: ctx.get(ParentSessionKey)?.rootSessionId ?? sessionId,+ sessionId,+ });+ } catch (error) {+ log.error(`failed to bind instrumentation for terminal ${input.event.type} event`, {+ error,+ errorId: input.errorId,+ sessionId,+ });+ }+ }++ const emit =+ instrumentation?.createHandleEvent({+ handleEvent,+ turnId: input.turnId,+ }) ?? handleEvent;+ try {+ if (ctx === undefined) {+ await emit(input.event);+ } else {+ await contextStorage.run(ctx, () => emit(input.event));+ }+ } catch (error) {+ log.error(`instrumentation failed to handle terminal ${input.event.type} event`, {+ error,+ errorId: input.errorId,+ sessionId,+ });+ } finally {+ try {+ await instrumentation?.flush();+ } catch (error) {+ log.error(`failed to flush instrumentation after terminal ${input.event.type} event`, {+ error,+ errorId: input.errorId,+ sessionId,+ });+ }+ }+}diff --git a/packages/eve/src/execution/terminal-session-failure-step.ts b/packages/eve/src/execution/terminal-session-failure-step.tsindex 69f61baa6..020b2bba5 100644--- a/packages/eve/src/execution/terminal-session-failure-step.ts+++ b/packages/eve/src/execution/terminal-session-failure-step.ts@@ -1,14 +1,7 @@-import { buildAdapterContext } from "#channel/adapter-context.js";-import { callAdapterEventHandler } from "#channel/adapter.js";-import { deserializeContext } from "#context/serialize.js";+import { emitTerminalSessionEvent } from "#execution/terminal-session-event.js";import { summarizeKnownError } from "#harness/semantic-errors/index.js";import { createLogger, formatError } from "#internal/logging.js";-import {- createSessionFailedEvent,- encodeMessageStreamEvent,- stampMessageStreamEvent,-} from "#protocol/message.js";-import { ChannelKey } from "#runtime/sessions/runtime-context-keys.js";+import { createSessionFailedEvent } from "#protocol/message.js";const log = createLogger("execution.workflow-entry");@@ -17,6 +10,7 @@ export async function emitTerminalSessionFailureStep(input: {readonly error: unknown;readonly parentWritable: WritableStream<Uint8Array>;readonly serializedContext: Record<string, unknown>;+ readonly turnId?: string;}): Promise<void> {"use step";@@ -45,41 +39,11 @@ export async function emitTerminalSessionFailureStep(input: {code,});- const event = createSessionFailedEvent({ code, details, message, sessionId });-- // Best-effort: invoke the adapter handler so channels surface the- // failure. Errors are logged, never rethrown — the outer workflow- // throw must still reach the run handle.- try {- const ctx = await deserializeContext(input.serializedContext);- const adapter = ctx.get(ChannelKey);- if (adapter !== undefined) {- const adapterCtx = buildAdapterContext(adapter, ctx);- await callAdapterEventHandler(adapter, event, adapterCtx);- }- } catch (notificationError) {- log.error("adapter failed to handle terminal session.failed event", {- errorId: typeof details.errorId === "string" ? details.errorId : undefined,- sessionId,- error: notificationError,- });- }-- // Always write the event to the durable stream so downstream- // consumers see a canonical terminal event instead of an abrupt- // stream close.- try {- const writer = input.parentWritable.getWriter();- try {- await writer.write(encodeMessageStreamEvent(stampMessageStreamEvent(event)));- } finally {- writer.releaseLock();- }- } catch (writeError) {- log.error("failed to write terminal session.failed event to durable stream", {- errorId: typeof details.errorId === "string" ? details.errorId : undefined,- sessionId,- error: writeError,- });- }+ await emitTerminalSessionEvent({+ errorId: typeof details.errorId === "string" ? details.errorId : undefined,+ event: createSessionFailedEvent({ code, details, message, sessionId }),+ parentWritable: input.parentWritable,+ serializedContext: input.serializedContext,+ turnId: input.turnId,+ });}diff --git a/packages/eve/src/execution/workflow-entry.test.ts b/packages/eve/src/execution/workflow-entry.test.tsindex 1da20a60c..2ed5c1263 100644--- a/packages/eve/src/execution/workflow-entry.test.ts+++ b/packages/eve/src/execution/workflow-entry.test.ts@@ -7,6 +7,7 @@ import { ChannelRequestIdKey, SubagentDepthKey } from "#context/keys.js";import { createSessionStep } from "#execution/create-session-step.js";import {bindTurnCallerContextStep,+ notifyCancelledTaskCallerStep,notifyDelegatedParentStep,notifyTaskTurnStartedStep,notifyTurnCallerStep,@@ -324,6 +325,56 @@ describe("workflowEntry", () => {serializedContext,sessionState,});+ expect(emitTerminalSessionCompletionStep).toHaveBeenCalledOnce();+ expect(emitTerminalSessionCompletionStep).toHaveBeenCalledWith({+ parentWritable: expect.any(WritableStream),+ serializedContext,+ });+ expect(emitTerminalSessionFailureStep).not.toHaveBeenCalled();+ expect(notifyTurnCallerStep).not.toHaveBeenCalled();+ });++ it("completes a parked session when reset retires it", async () => {+ const sessionState = createBaseSessionState();+ const serializedContext = {+ ...createSerializedContext(),+ "eve.sessionId": "wrun_test_123",+ };+ vi.mocked(createSessionStep).mockResolvedValue(createSessionStepResultForMock(sessionState));+ installHookMocks({+ deliveryHooks: [+ {+ token: "http:test",+ values: [{ kind: "reset", reason: "Start over" }],+ },+ ],+ turnControls: [+ turnResult({+ action: "park",+ serializedContext,+ sessionState,+ }),+ ],+ });++ await expect(+ workflowEntry({+ input: { message: "hello there" },+ serializedContext: createSerializedContext(),+ }),+ ).resolves.toEqual({ output: "" });++ expect(terminateChildSessionsStep).toHaveBeenCalledWith({+ serializedContext,+ sessionState,+ });+ expect(emitTerminalSessionCompletionStep).toHaveBeenCalledOnce();+ expect(emitTerminalSessionCompletionStep).toHaveBeenCalledWith({+ parentWritable: expect.any(WritableStream),+ serializedContext,+ });+ expect(emitTerminalSessionFailureStep).not.toHaveBeenCalled();+ expect(notifyTurnCallerStep).not.toHaveBeenCalled();});it("exits a conflicting initial continuation before dispatching the first turn", async () => {@@ -654,6 +705,7 @@ describe("workflowEntry", () => {expect(emitTerminalSessionFailureStep).toHaveBeenCalledWith(expect.objectContaining({error: expect.objectContaining({ message: "persistent recoverable failure" }),+ turnId: "turn_0",}),);expect(terminateChildSessionsStep).toHaveBeenCalledWith({@@ -680,6 +732,26 @@ describe("workflowEntry", () => {expect(resolveInitialTurnCallerStep).toHaveBeenCalledOnce();});+ it("does not emit session.failed when notification fails after terminal completion", async () => {+ const sessionState = createBaseSessionState();+ vi.mocked(createSessionStep).mockResolvedValue(createSessionStepResultForMock(sessionState));+ vi.mocked(fireSessionCallbackStep).mockRejectedValueOnce(new Error("callback failed"));+ installHookMocks({+ turnControls: [turnResult({ action: "done", output: "ok", sessionState })],+ });++ await expect(+ workflowEntry({+ input: { message: "run task" },+ serializedContext: createSerializedContext({ "eve.mode": "task" }),+ }),+ ).rejects.toMatchObject({ name: "EveWorkflowFailure" });++ expect(fireSessionCallbackStep).toHaveBeenCalledOnce();+ expect(emitTerminalSessionFailureStep).not.toHaveBeenCalled();+ expect(notifyDelegatedParentStep).not.toHaveBeenCalled();+ });+it("rejects the delegated caller when the session fails before the caller is resolved", async () => {const caller = {callId: "call-1",@@ -1156,6 +1228,84 @@ describe("workflowEntry", () => {expect(notifyTurnCallerStep).not.toHaveBeenCalled();});+ it("retains the active turn when cancellation settlement fails", async () => {+ const sessionState = createBaseSessionState();+ vi.mocked(createSessionStep).mockResolvedValue(createSessionStepResultForMock(sessionState));+ vi.mocked(settleCancelledTurnStep).mockRejectedValueOnce(new Error("settlement failed"));+ installHookMocks({+ turnControls: [+ {+ action: {+ cancelled: true,+ kind: "park",+ serializedContext: { "eve.sessionId": "wrun_test_123" },+ sessionState,+ },+ kind: "turn-result",+ },+ ],+ });++ await expect(+ workflowEntry({+ input: { message: "hello" },+ serializedContext: createSerializedContext(),+ }),+ ).rejects.toMatchObject({ name: "EveWorkflowFailure" });++ expect(emitTerminalSessionFailureStep).toHaveBeenCalledWith(+ expect.objectContaining({+ error: expect.objectContaining({ message: "settlement failed" }),+ serializedContext: { "eve.sessionId": "wrun_test_123" },+ turnId: "turn_0",+ }),+ );+ });++ it("adopts cancelled-turn state before notifying its caller", async () => {+ const sessionState = createBaseSessionState();+ const settledState = createBaseSessionState({+ emissionState: { sequence: 1, sessionStarted: true, stepIndex: 0, turnId: "" },+ });+ const settledContext = { "eve.sessionId": "wrun_test_123", settled: true };+ vi.mocked(createSessionStep).mockResolvedValue(createSessionStepResultForMock(sessionState));+ vi.mocked(settleCancelledTurnStep).mockResolvedValue({+ serializedContext: settledContext,+ sessionState: settledState,+ });+ vi.mocked(notifyCancelledTaskCallerStep).mockRejectedValueOnce(+ new Error("caller notification failed"),+ );+ installHookMocks({+ turnControls: [+ {+ action: {+ cancelled: true,+ kind: "park",+ serializedContext: { "eve.sessionId": "wrun_test_123" },+ sessionState,+ },+ kind: "turn-result",+ },+ ],+ });++ await expect(+ workflowEntry({+ input: { message: "hello" },+ serializedContext: createSerializedContext(),+ }),+ ).rejects.toMatchObject({ name: "EveWorkflowFailure" });++ expect(emitTerminalSessionFailureStep).toHaveBeenCalledWith(+ expect.objectContaining({+ error: expect.objectContaining({ message: "caller notification failed" }),+ serializedContext: settledContext,+ turnId: undefined,+ }),+ );+ });+it("does not settle an ordinary park as cancelled", async () => {const sessionState = createBaseSessionState();vi.mocked(createSessionStep).mockResolvedValue(createSessionStepResultForMock(sessionState));diff --git a/packages/eve/src/execution/workflow-entry.ts b/packages/eve/src/execution/workflow-entry.tsindex 72dbae97a..1d72d04d6 100644--- a/packages/eve/src/execution/workflow-entry.ts+++ b/packages/eve/src/execution/workflow-entry.ts@@ -27,10 +27,11 @@ import { cancelDescendantTurnsStep } from "#execution/cancel-descendant-turns-stimport { dispatchAndAwaitTurn } from "#execution/turn-dispatch.js";import type { TurnDriverAction } from "#execution/turn-control-receiver.js";import { normalizeSerializableError } from "#execution/workflow-errors.js";-import { finalizeDone, finalizeExpiredSession } from "#execution/workflow-entry-finalization.js";+import { finalizeDone } from "#execution/workflow-entry-finalization.js";import { createSessionStep } from "#execution/create-session-step.js";import { settleCancelledTurnStep } from "#execution/settle-cancelled-turn-step.js";import { emitTerminalSessionFailureStep } from "#execution/terminal-session-failure-step.js";+import { finalizeTerminalSession } from "#execution/finalize-terminal-session.js";import { fireSessionCallbackStep } from "#execution/session-callback-step.js";import { isHookConflictError } from "#execution/hook-ownership.js";import { createSessionCommandInbox } from "#execution/session-command-inbox.js";@@ -71,7 +72,8 @@ export interface WorkflowEntryResult {type DriverLoopOutcome =| {- readonly kind: "expired";+ readonly kind: "terminal";+ readonly notifyCaller: boolean;readonly serializedContext: Record<string, unknown>;readonly sessionState: DurableSessionState;}@@ -97,6 +99,9 @@ type DriverLoopOutcome =* instead of mirroring it here.*/interface CrashCleanupState {+ // The turn currently owned by `dispatchAndAwaitTurn`, when a crash occurs+ // before its terminal state returns to the driver.+ activeTurnId: string | undefined;// The caller whose awaited reply is still unsettled, so the catch can// reject it with the error instead of leaving it parked forever.// Root sessions leave it undefined; only conversation-mode paths read it.@@ -114,6 +119,12 @@ interface CrashCleanupState {// turn that crashed mid-flight are absent from this snapshot and escape// crash cleanup.lastSessionState: DurableSessionState | undefined;+ // The latest context adopted from a completed turn, which carries provider+ // and trace state needed by terminal instrumentation.+ serializedContext: Record<string, unknown>;+ // Whether the session already emitted a terminal protocol and instrumentation+ // event, so later callback or teardown failures cannot contradict it.+ terminalEmitted: boolean;}/**@@ -150,9 +161,12 @@ export async function workflowEntry(input: WorkflowEntryInput): Promise<Workflowconst driverWritable = getWritable<Uint8Array>();const crashCleanupState: CrashCleanupState = {+ activeTurnId: undefined,caller: undefined,callerResolved: false,lastSessionState: undefined,+ serializedContext: input.serializedContext,+ terminalEmitted: false,};try {@@ -226,12 +240,14 @@ export async function workflowEntry(input: WorkflowEntryInput): Promise<Workflowif (outcome.kind === "result") {return outcome.result;}- return await finalizeExpiredSession({+ return await finalizeTerminalSession({caller: crashCleanupState.caller,driverWritable,mode,+ notifyCaller: outcome.notifyCaller,serializedContext: outcome.serializedContext,sessionState: outcome.sessionState,+ terminalState: crashCleanupState,});} catch (error) {// Safety net for failures the tool-loop harness does not already@@ -240,34 +256,37 @@ export async function workflowEntry(input: WorkflowEntryInput): Promise<Workflow// channel still sees a terminal event.if (crashCleanupState.lastSessionState !== undefined) {await terminateChildSessionsStep({- serializedContext: input.serializedContext,+ serializedContext: crashCleanupState.serializedContext,sessionState: crashCleanupState.lastSessionState,});}- await emitTerminalSessionFailureStep({- error: normalizeSerializableError(error),- parentWritable: driverWritable,- serializedContext: input.serializedContext,- });- if (mode === "task") {- await fireSessionCallbackStep({+ if (!crashCleanupState.terminalEmitted) {+ await emitTerminalSessionFailureStep({error: normalizeSerializableError(error),- serializedContext: input.serializedContext,- status: "failed",+ parentWritable: driverWritable,+ serializedContext: crashCleanupState.serializedContext,+ turnId: crashCleanupState.activeTurnId,});- await notifyDelegatedParentStep({- result: createDelegatedSubagentErrorResult(input.serializedContext, error),- serializedContext: input.serializedContext,- });- } else {- const caller = await resolveCallerForCrash(crashCleanupState, input.serializedContext);- if (caller !== undefined) {- await notifyTurnCallerStep({- caller,- lifecycle: "terminal",- sessionId,- settled: { isError: true, output: error },+ if (mode === "task") {+ await fireSessionCallbackStep({+ error: normalizeSerializableError(error),+ serializedContext: input.serializedContext,+ status: "failed",+ });+ await notifyDelegatedParentStep({+ result: createDelegatedSubagentErrorResult(input.serializedContext, error),+ serializedContext: input.serializedContext,});+ } else {+ const caller = await resolveCallerForCrash(crashCleanupState, input.serializedContext);+ if (caller !== undefined) {+ await notifyTurnCallerStep({+ caller,+ lifecycle: "terminal",+ sessionId,+ settled: { isError: true, output: error },+ });+ }}}throw createSafeOuterWorkflowError();@@ -449,6 +468,8 @@ async function runDriverLoop(input: {caller,serializedContext: stateCursor.serializedContext,});+ const dispatchedTurnId = activeTurnId(stateCursor.sessionState.emissionState);+ input.crashCleanupState.activeTurnId = dispatchedTurnId;const turn = await dispatchAndAwaitTurn({bufferedDeliveries,bufferedSessionControls,@@ -463,10 +484,14 @@ async function runDriverLoop(input: {seenTaskDeliveries,sessionState: stateCursor.sessionState,});+ input.crashCleanupState.activeTurnId =+ turn.action.kind === "park" && turn.action.cancelled === true ? dispatchedTurnId : undefined;+ input.crashCleanupState.lastSessionState = turn.action.sessionState;+ input.crashCleanupState.serializedContext = turn.action.serializedContext;+ if (turn.action.kind === "done") input.crashCleanupState.terminalEmitted = true;await disposeSettledTurnControl?.();disposeSettledTurnControl = turn.dispose;stateCursor.adoptState(turn.action);- input.crashCleanupState.lastSessionState = stateCursor.sessionState;return turn.action;};@@ -513,6 +538,9 @@ async function runDriverLoop(input: {sessionState: stateCursor.sessionState,});stateCursor.adoptState(settled);+ input.crashCleanupState.activeTurnId = undefined;+ input.crashCleanupState.lastSessionState = stateCursor.sessionState;+ input.crashCleanupState.serializedContext = stateCursor.serializedContext;const cancelledCaller = {caller: input.crashCleanupState.caller,sessionId: stateCursor.sessionState.sessionId,@@ -522,7 +550,6 @@ async function runDriverLoop(input: {? cancelledCaller: { ...cancelledCaller, usage: settled.usage },);- input.crashCleanupState.lastSessionState = stateCursor.sessionState;}// Channel-created sessions may rekey their dynamic alias. Sessions@@ -558,6 +585,7 @@ async function runDriverLoop(input: {expectedAttemptIds: action.authorizationAttemptIds ?? [],});input.crashCleanupState.lastSessionState = stateCursor.sessionState;+ input.crashCleanupState.serializedContext = stateCursor.serializedContext;if (next.kind === "authorization-resume") {action = await runTurn({@@ -569,18 +597,20 @@ async function runDriverLoop(input: {if (next.kind === "expired") {return {- kind: "expired",+ kind: "terminal",+ notifyCaller: true,serializedContext: stateCursor.serializedContext,sessionState: stateCursor.sessionState,};}if (next.kind === "reset") {- await terminateChildSessionsStep({+ return {+ kind: "terminal",+ notifyCaller: false,serializedContext: stateCursor.serializedContext,sessionState: stateCursor.sessionState,- });- return { kind: "result", result: { output: "" } };+ };}if (next.kind === "clear" || next.kind === "compact") {@@ -589,11 +619,12 @@ async function runDriverLoop(input: {}if (next.kind === "closed") {- await terminateChildSessionsStep({+ return {+ kind: "terminal",+ notifyCaller: false,serializedContext: stateCursor.serializedContext,sessionState: stateCursor.sessionState,- });- return { kind: "result", result: { output: "" } };+ };}if (next.kind === "cancel-turn") {@@ -613,6 +644,7 @@ async function runDriverLoop(input: {action = { ...action, settled: undefined };input.crashCleanupState.caller = undefined;input.crashCleanupState.lastSessionState = stateCursor.sessionState;+ input.crashCleanupState.serializedContext = stateCursor.serializedContext;continue;}diff --git a/packages/eve/src/execution/workflow-steps.test.ts b/packages/eve/src/execution/workflow-steps.test.tsindex ae38a72b9..bb6119e0a 100644--- a/packages/eve/src/execution/workflow-steps.test.ts+++ b/packages/eve/src/execution/workflow-steps.test.ts@@ -56,7 +56,9 @@ import { runProxySubagentEventStep } from "#execution/subagent-event-proxy-step.import { readLatestTaskView, sendTaskInboundPayload } from "#execution/tasks/parent/run-parent.js";import { recordTaskInputRequestStep } from "#execution/tasks/parent/hitl-proxy-steps.js";import { appendTaskAgentAnnouncement } from "#execution/tasks/parent/agent-views.js";+import { emitTerminalSessionCompletionStep } from "#execution/terminal-session-completion-step.js";import { emitTerminalSessionFailureStep } from "#execution/terminal-session-failure-step.js";+import { registerInstrumentationRuntime } from "#instrumentation/runtime.js";import { resolveEffectiveOutputSchema } from "#execution/effective-output-schema.js";import { turnStep } from "#execution/workflow-steps.js";import { routeProxiedDeliverStep } from "#execution/proxied-deliver-step.js";@@ -247,6 +249,7 @@ function createSerializedContext(}afterEach(() => {+ delete (globalThis as Record<symbol, unknown>)[Symbol.for("eve.instrumentation-runtime")];getRunMock.mockReset();resumeHookMock.mockReset();startMock.mockReset();@@ -3073,7 +3076,7 @@ describe("turnStep", () => {});});-describe("emitTerminalSessionFailureStep", () => {+describe("terminal session event steps", () => {function buildSerializedContextWithAdapter(adapter: ChannelAdapter,sessionId: string,@@ -3115,6 +3118,47 @@ describe("emitTerminalSessionFailureStep", () => {return serialized;}+ function installTerminalInstrumentation() {+ const forceFlush = vi.fn(async () => undefined);+ const publish = vi.fn(async () => undefined);+ registerInstrumentationRuntime({+ forceFlush,+ hooks: { capturesContent: false, publish },+ otelSettings: undefined,+ runInContext: (_operation, execute) => execute(),+ shutdown: async () => undefined,+ });+ return { forceFlush, publish };+ }++ it("publishes and flushes an out-of-turn session completion", async () => {+ const sessionCompleted = vi.fn();+ const adapter: ChannelAdapter = {+ kind: "thread-context",+ "session.completed": sessionCompleted,+ };+ const serializedContext = buildSerializedContextWithAdapter(adapter, "session-completed");+ const instrumentation = installTerminalInstrumentation();++ await emitTerminalSessionCompletionStep({+ parentWritable: createTestWritable(),+ serializedContext,+ });++ expect(sessionCompleted).toHaveBeenCalledOnce();+ expect(instrumentation.publish).toHaveBeenCalledExactlyOnceWith({+ idempotencyKey: "session:session-completed",+ sessionId: "session-completed",+ turnId: undefined,+ type: "session.completed",+ });+ expect(instrumentation.forceFlush).toHaveBeenCalledOnce();+ expect(instrumentation.publish.mock.invocationCallOrder[0]).toBeLessThan(+ instrumentation.forceFlush.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY,+ );+ expect(workflowWritesByNamespace.get(DEFAULT_WORKFLOW_STREAM_NAMESPACE)).toHaveLength(1);+ });+it("invokes the adapter's session.failed handler with a formatted error payload", async () => {// Capture adapter side effects — this is how we verify the step// actually reaches the user-visible notification path. A terminal@@ -3128,6 +3172,7 @@ describe("emitTerminalSessionFailureStep", () => {};const serialized = buildSerializedContextWithAdapter(capturingAdapter, "session-terminal");+ const instrumentation = installTerminalInstrumentation();// Use a plain-object error shape — the workflow body converts// raw Errors to this shape (`normalizeSerializableError`) before@@ -3144,6 +3189,7 @@ describe("emitTerminalSessionFailureStep", () => {error,parentWritable: createTestWritable(),serializedContext: serialized,+ turnId: "turn-active",});expect(sessionFailedCalls).toHaveLength(1);@@ -3163,6 +3209,21 @@ describe("emitTerminalSessionFailureStep", () => {sessionId: "session-terminal",});expect(JSON.stringify(providerLog)).not.toContain("confidential.png");+ expect(instrumentation.publish).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({+ error: expect.objectContaining({+ message: "attachment staging failed for confidential.png",+ }),+ idempotencyKey: "session:session-terminal",+ sessionId: "session-terminal",+ turnId: "turn-active",+ type: "session.failed",+ }),+ );+ expect(instrumentation.forceFlush).toHaveBeenCalledOnce();+ expect(instrumentation.publish.mock.invocationCallOrder[0]).toBeLessThan(+ instrumentation.forceFlush.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY,+ );// The terminal step must also write the event to the durable// stream so event-stream consumers see a canonical tail insteaddiff --git a/packages/eve/src/instrumentation/runtime.ts b/packages/eve/src/instrumentation/runtime.tsindex 9c8c045c3..3d8c50962 100644--- a/packages/eve/src/instrumentation/runtime.ts+++ b/packages/eve/src/instrumentation/runtime.ts@@ -177,9 +177,9 @@ export interface SessionInstrumentation {}export interface ExecutionInstrumentation {- readonly createCancellationHandleEvent: (input: {+ readonly createHandleEvent: (input: {readonly handleEvent?: HandleEventFn;- readonly turnId: string;+ readonly turnId?: string;}) => HandleEventFn | undefined;readonly flush: () => Promise<void>;readonly instrumentChannelDelivery: (@@ -460,7 +460,7 @@ export function bindInstrumentationRuntime(};};return {- createCancellationHandleEvent: (input) => {+ createHandleEvent: (input) => {const sessionContext = readSessionContext();return createInstrumentationHandleEvent({agentName: boundSession.agentName,diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.tsindex a2dcc8b5b..ef8f7ad33 100644--- a/packages/eve/src/tracing/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts@@ -2207,6 +2207,30 @@ describe("createAgentOtelInstrumentation", () => {expect(turn.status).toEqual({ code: SpanStatusCode.ERROR, message: error.message });expect(turn.attributes["error.type"]).toBe("TypeError");});++ it("marks an active invocation failed from the terminal session event", async () => {+ const runtime = createRuntime();+ const error = new TypeError("workflow failed");+ await publishTurnStarted({+ hooks: runtime.hooks,+ sessionId: "session-1",+ turnId: "turn-1",+ turnSequence: 0,+ });+ await runtime.hooks.publish({+ error,+ idempotencyKey: sessionIdempotencyKey("session-1"),+ sessionId: "session-1",+ turnId: "turn-1",+ type: "session.failed",+ });+ await runtime.provider.forceFlush();++ const turn = byName(runtime.exporter.getFinishedSpans(), "invoke_agent weather")[0]!;+ expect(turn.status).toEqual({ code: SpanStatusCode.ERROR, message: error.message });+ expect(turn.attributes["error.type"]).toBe("TypeError");+ expect(turn.events.map((event) => event.name)).toContain("turn.failed");+ });});function spanContext(traceId: string, spanId: string) {diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.tsindex c61daf24a..ee3e16c98 100644--- a/packages/eve/src/tracing/agent-otel-provider.ts+++ b/packages/eve/src/tracing/agent-otel-provider.ts@@ -261,6 +261,12 @@ export function createAgentOtelInstrumentation(const onSessionTransition = async (event: InstrumentationSessionTransitionEvent,): Promise<void> => {+ if (event.type === "session.failed" && event.turnId !== undefined) {+ await input.stateStore.updateTurn(event.sessionId, event.turnId, (turn) => ({+ ...turn,+ terminal: turn.terminal ?? { error: event.error, type: "turn.failed" },+ }));+ }if (event.turnId !== undefined) {const turn = await input.stateStore.getTurn(event.sessionId, event.turnId);if (turn !== undefined) {diff --git a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts b/packages/eve/src/tracing/install-instrumentation-runtime.test.tsindex 4baeb61ae..0c17a6e7e 100644--- a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts+++ b/packages/eve/src/tracing/install-instrumentation-runtime.test.ts@@ -1,7 +1,7 @@import { beforeEach, describe, expect, it, vi } from "vitest";import { ContextContainer, contextStorage } from "#context/container.js";-import { turnIdempotencyKey } from "#instrumentation/lifecycle.js";+import { sessionIdempotencyKey, turnIdempotencyKey } from "#instrumentation/lifecycle.js";import { installInstrumentationRuntime } from "#tracing/install-instrumentation-runtime.js";import { otelIntegration, collectOtelPipeline } from "#tracing/otel-declaration.js";@@ -82,6 +82,37 @@ describe("installInstrumentationRuntime", () => {expect(providerShutdown).toHaveBeenCalledOnce();});+ it.each(["session.completed", "session.failed"] as const)(+ "releases local trace liveness after %s",+ async (type) => {+ const releaseSession = vi.fn(async () => true);+ const processor = {+ forceFlush: vi.fn(async () => undefined),+ onEnd: vi.fn(),+ onStart: vi.fn(),+ releaseSession,+ shutdown: vi.fn(async () => undefined),+ };+ const runtime = installInstrumentationRuntime({+ collected: collectOtelPipeline([otelIntegration({ spanProcessors: [processor] })]),+ frameworkVersion: "test",+ providers: [],+ serviceName: "weather",+ });+ const hooks = runtime.hooks.forTrace!({ agentName: "weather", audience: "unknown" });+ const event = {+ idempotencyKey: sessionIdempotencyKey("session-1"),+ sessionId: "session-1",+ type,+ ...(type === "session.failed" ? { error: new Error("failed") } : undefined),+ };++ await contextStorage.run(new ContextContainer(), () => hooks.publish(event));++ expect(releaseSession).toHaveBeenCalledExactlyOnceWith("session-1");+ },+ );+it("isolates authored state from an internal provider with the same name", async () => {const authoredTerminalState = vi.fn();const runtime = installInstrumentationRuntime({