Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): instrument terminal session paths (#2819)

Chad Hietala3c16df6289e6
diff --git a/.changeset/terminal-traces-release.md b/.changeset/terminal-traces-release.md
new file mode 100644
index 000000000..ba8939c7e
--- /dev/null
+++ b/.changeset/terminal-traces-release.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Publish instrumentation lifecycle events when sessions expire, reset, close, or fail outside an agent turn. Terminal paths now flush providers and release local trace retention state.
diff --git a/packages/eve/src/execution/agent-messaging.scenario.test.ts b/packages/eve/src/execution/agent-messaging.scenario.test.ts
index 744c19cef..e52474864 100644
--- a/packages/eve/src/execution/agent-messaging.scenario.test.ts
+++ b/packages/eve/src/execution/agent-messaging.scenario.test.ts
@@ -194,6 +194,7 @@ describe("agent messaging", () => {
await expectRetainedChildConversation({
childSessionId,
client: new Client({ host: server.url }),
+ expectedCompletionCount: 0,
});
expect(await readWorkflowRunStatus(app.appRoot, childSessionId)).toBe("cancelled");
} catch (error) {
@@ -230,6 +231,7 @@ describe("agent messaging", () => {
auth: { bearer: REMOTE_MEMORY_TOKEN },
host: remoteServer.url,
}),
+ expectedCompletionCount: 1,
});
} catch (error) {
throw new Error(
@@ -310,6 +312,7 @@ async function runScriptedParentSession(input: {
async function expectRetainedChildConversation(input: {
readonly childSessionId: string;
readonly client: Client;
+ readonly expectedCompletionCount: number;
}): Promise<void> {
const childEvents = await collectStreamToEnd({
label: "persisted child events",
@@ -321,7 +324,9 @@ async function expectRetainedChildConversation(input: {
expect(childTurnStarts).toHaveLength(2);
expect(childWaits).toHaveLength(2);
expect(childWaits[0]).toBeLessThan(childTurnStarts[1] ?? -1);
- expect(filterEventsByType(childEvents, "session.completed")).toHaveLength(0);
+ expect(filterEventsByType(childEvents, "session.completed")).toHaveLength(
+ input.expectedCompletionCount,
+ );
expect(filterEventsByType(childEvents, "session.failed")).toHaveLength(0);
expect(
filterEventsByType(childEvents, "message.completed").some(
diff --git a/packages/eve/src/execution/finalize-terminal-session.ts b/packages/eve/src/execution/finalize-terminal-session.ts
new file mode 100644
index 000000000..992a999f1
--- /dev/null
+++ b/packages/eve/src/execution/finalize-terminal-session.ts
@@ -0,0 +1,54 @@
+import type { TurnCaller } from "#channel/types.js";
+import {
+ notifyDelegatedParentStep,
+ notifyTurnCallerStep,
+} from "#execution/delegated-parent-notification.js";
+import { createDelegatedSubagentSuccessResult } from "#execution/delegated-parent-result.js";
+import type { DurableSessionState } from "#execution/durable-session-store.js";
+import { fireSessionCallbackStep } from "#execution/session-callback-step.js";
+import { emitTerminalSessionCompletionStep } from "#execution/terminal-session-completion-step.js";
+import { terminateChildSessionsStep } from "#execution/terminate-child-sessions-step.js";
+import type { RunMode } from "#shared/run-mode.js";
+
+/** Completes a session that terminates outside a turn. */
+export async function finalizeTerminalSession(input: {
+ readonly caller: TurnCaller | undefined;
+ readonly driverWritable: WritableStream<Uint8Array>;
+ readonly mode: RunMode;
+ readonly notifyCaller: boolean;
+ readonly serializedContext: Record<string, unknown>;
+ readonly sessionState: DurableSessionState;
+ readonly terminalState: { terminalEmitted: boolean };
+}): Promise<{ readonly output: unknown }> {
+ await terminateChildSessionsStep({
+ serializedContext: input.serializedContext,
+ sessionState: input.sessionState,
+ });
+ await emitTerminalSessionCompletionStep({
+ parentWritable: input.driverWritable,
+ serializedContext: input.serializedContext,
+ });
+ input.terminalState.terminalEmitted = true;
+
+ if (!input.notifyCaller) return { output: "" };
+
+ if (input.mode === "task") {
+ await fireSessionCallbackStep({
+ output: "",
+ serializedContext: input.serializedContext,
+ status: "completed",
+ });
+ await notifyDelegatedParentStep({
+ result: createDelegatedSubagentSuccessResult(input.serializedContext, ""),
+ serializedContext: input.serializedContext,
+ });
+ } else if (input.caller !== undefined) {
+ await notifyTurnCallerStep({
+ caller: input.caller,
+ lifecycle: "terminal",
+ sessionId: input.sessionState.sessionId,
+ settled: { output: "" },
+ });
+ }
+ return { output: "" };
+}
diff --git a/packages/eve/src/execution/settle-cancelled-turn-step.ts b/packages/eve/src/execution/settle-cancelled-turn-step.ts
index 54b9079ab..051c610c4 100644
--- a/packages/eve/src/execution/settle-cancelled-turn-step.ts
+++ b/packages/eve/src/execution/settle-cancelled-turn-step.ts
@@ -110,7 +110,7 @@ export async function settleCancelledTurnStep(input: {
});
};
const emit =
- instrumentation?.createCancellationHandleEvent({
+ instrumentation?.createHandleEvent({
handleEvent: baseEmit,
turnId: activeTurnId(emissionState),
}) ?? baseEmit;
diff --git a/packages/eve/src/execution/terminal-session-completion-step.ts b/packages/eve/src/execution/terminal-session-completion-step.ts
index 509765c4c..9ea12182c 100644
--- a/packages/eve/src/execution/terminal-session-completion-step.ts
+++ b/packages/eve/src/execution/terminal-session-completion-step.ts
@@ -1,15 +1,5 @@
-import { buildAdapterContext } from "#channel/adapter-context.js";
-import { callAdapterEventHandler } from "#channel/adapter.js";
-import { deserializeContext } from "#context/serialize.js";
-import { createLogger } from "#internal/logging.js";
-import {
- createSessionCompletedEvent,
- encodeMessageStreamEvent,
- stampMessageStreamEvent,
-} from "#protocol/message.js";
-import { ChannelKey } from "#runtime/sessions/runtime-context-keys.js";
-
-const log = createLogger("execution.workflow-entry");
+import { emitTerminalSessionEvent } from "#execution/terminal-session-event.js";
+import { createSessionCompletedEvent } from "#protocol/message.js";
/** Emits a terminal `session.completed` outside a turn. */
export async function emitTerminalSessionCompletionStep(input: {
@@ -18,34 +8,5 @@ export async function emitTerminalSessionCompletionStep(input: {
}): Promise<void> {
"use step";
- const event = createSessionCompletedEvent();
- const sessionId = (input.serializedContext["eve.sessionId"] as string | undefined) ?? "";
-
- try {
- const ctx = await deserializeContext(input.serializedContext);
- const adapter = ctx.get(ChannelKey);
- if (adapter !== undefined) {
- const adapterCtx = buildAdapterContext(adapter, ctx);
- await callAdapterEventHandler(adapter, event, adapterCtx);
- }
- } catch (notificationError) {
- log.error("adapter failed to handle terminal session.completed event", {
- error: notificationError,
- sessionId,
- });
- }
-
- try {
- const writer = input.parentWritable.getWriter();
- try {
- await writer.write(encodeMessageStreamEvent(stampMessageStreamEvent(event)));
- } finally {
- writer.releaseLock();
- }
- } catch (writeError) {
- log.error("failed to write terminal session.completed event to durable stream", {
- error: writeError,
- sessionId,
- });
- }
+ await emitTerminalSessionEvent({ ...input, event: createSessionCompletedEvent() });
}
diff --git a/packages/eve/src/execution/terminal-session-event.ts b/packages/eve/src/execution/terminal-session-event.ts
new file mode 100644
index 000000000..5d25f020f
--- /dev/null
+++ b/packages/eve/src/execution/terminal-session-event.ts
@@ -0,0 +1,123 @@
+import { buildAdapterContext } from "#channel/adapter-context.js";
+import { callAdapterEventHandler } from "#channel/adapter.js";
+import { ContextContainer, contextStorage } from "#context/container.js";
+import { ParentSessionKey } from "#context/keys.js";
+import { deserializeContext } from "#context/serialize.js";
+import { bindSessionInstrumentation } from "#instrumentation/runtime.js";
+import type { HandleEventFn } from "#harness/types.js";
+import { createLogger } from "#internal/logging.js";
+import {
+ encodeMessageStreamEvent,
+ stampMessageStreamEvent,
+ type UnstampedMessageStreamEvent,
+} from "#protocol/message.js";
+import { BundleKey, ChannelKey } from "#runtime/sessions/runtime-context-keys.js";
+
+const log = createLogger("execution.workflow-entry");
+
+type TerminalSessionEvent = Extract<
+ UnstampedMessageStreamEvent,
+ { type: "session.completed" | "session.failed" }
+>;
+
+/** Delivers one out-of-turn terminal event through the native instrumentation lifecycle. */
+export async function emitTerminalSessionEvent(input: {
+ readonly errorId?: string;
+ readonly event: TerminalSessionEvent;
+ readonly parentWritable: WritableStream<Uint8Array>;
+ readonly serializedContext: Record<string, unknown>;
+ readonly turnId?: string;
+}): Promise<void> {
+ const sessionId = (input.serializedContext["eve.sessionId"] as string | undefined) ?? "";
+ let ctx: ContextContainer | undefined;
+
+ try {
+ ctx = await deserializeContext(input.serializedContext);
+ } catch (error) {
+ log.error(`failed to restore context for terminal ${input.event.type} event`, {
+ error,
+ errorId: input.errorId,
+ sessionId,
+ });
+ }
+
+ const handleEvent: HandleEventFn = async (event) => {
+ if (ctx !== undefined) {
+ const adapter = ctx.get(ChannelKey);
+ if (adapter !== undefined) {
+ try {
+ await callAdapterEventHandler(adapter, event, buildAdapterContext(adapter, ctx));
+ } catch (error) {
+ log.error(`adapter failed to handle terminal ${event.type} event`, {
+ error,
+ errorId: input.errorId,
+ sessionId,
+ });
+ }
+ }
+ }
+
+ try {
+ const writer = input.parentWritable.getWriter();
+ try {
+ await writer.write(encodeMessageStreamEvent(stampMessageStreamEvent(event)));
+ } finally {
+ writer.releaseLock();
+ }
+ } catch (error) {
+ log.error(`failed to write terminal ${event.type} event to durable stream`, {
+ error,
+ errorId: input.errorId,
+ sessionId,
+ });
+ }
+ };
+
+ let instrumentation: ReturnType<typeof bindSessionInstrumentation>;
+ if (ctx !== undefined) {
+ try {
+ const bundle = ctx.require(BundleKey);
+ instrumentation = bindSessionInstrumentation({
+ agentName: bundle.turnAgent.id,
+ ctx,
+ rootSessionId: ctx.get(ParentSessionKey)?.rootSessionId ?? sessionId,
+ sessionId,
+ });
+ } catch (error) {
+ log.error(`failed to bind instrumentation for terminal ${input.event.type} event`, {
+ error,
+ errorId: input.errorId,
+ sessionId,
+ });
+ }
+ }
+
+ const emit =
+ instrumentation?.createHandleEvent({
+ handleEvent,
+ turnId: input.turnId,
+ }) ?? handleEvent;
+ try {
+ if (ctx === undefined) {
+ await emit(input.event);
+ } else {
+ await contextStorage.run(ctx, () => emit(input.event));
+ }
+ } catch (error) {
+ log.error(`instrumentation failed to handle terminal ${input.event.type} event`, {
+ error,
+ errorId: input.errorId,
+ sessionId,
+ });
+ } finally {
+ try {
+ await instrumentation?.flush();
+ } catch (error) {
+ log.error(`failed to flush instrumentation after terminal ${input.event.type} event`, {
+ error,
+ errorId: input.errorId,
+ sessionId,
+ });
+ }
+ }
+}
diff --git a/packages/eve/src/execution/terminal-session-failure-step.ts b/packages/eve/src/execution/terminal-session-failure-step.ts
index 69f61baa6..020b2bba5 100644
--- a/packages/eve/src/execution/terminal-session-failure-step.ts
+++ b/packages/eve/src/execution/terminal-session-failure-step.ts
@@ -1,14 +1,7 @@
-import { buildAdapterContext } from "#channel/adapter-context.js";
-import { callAdapterEventHandler } from "#channel/adapter.js";
-import { deserializeContext } from "#context/serialize.js";
+import { emitTerminalSessionEvent } from "#execution/terminal-session-event.js";
import { summarizeKnownError } from "#harness/semantic-errors/index.js";
import { createLogger, formatError } from "#internal/logging.js";
-import {
- createSessionFailedEvent,
- encodeMessageStreamEvent,
- stampMessageStreamEvent,
-} from "#protocol/message.js";
-import { ChannelKey } from "#runtime/sessions/runtime-context-keys.js";
+import { createSessionFailedEvent } from "#protocol/message.js";
const log = createLogger("execution.workflow-entry");
@@ -17,6 +10,7 @@ export async function emitTerminalSessionFailureStep(input: {
readonly error: unknown;
readonly parentWritable: WritableStream<Uint8Array>;
readonly serializedContext: Record<string, unknown>;
+ readonly turnId?: string;
}): Promise<void> {
"use step";
@@ -45,41 +39,11 @@ export async function emitTerminalSessionFailureStep(input: {
code,
});
- const event = createSessionFailedEvent({ code, details, message, sessionId });
-
- // Best-effort: invoke the adapter handler so channels surface the
- // failure. Errors are logged, never rethrown — the outer workflow
- // throw must still reach the run handle.
- try {
- const ctx = await deserializeContext(input.serializedContext);
- const adapter = ctx.get(ChannelKey);
- if (adapter !== undefined) {
- const adapterCtx = buildAdapterContext(adapter, ctx);
- await callAdapterEventHandler(adapter, event, adapterCtx);
- }
- } catch (notificationError) {
- log.error("adapter failed to handle terminal session.failed event", {
- errorId: typeof details.errorId === "string" ? details.errorId : undefined,
- sessionId,
- error: notificationError,
- });
- }
-
- // Always write the event to the durable stream so downstream
- // consumers see a canonical terminal event instead of an abrupt
- // stream close.
- try {
- const writer = input.parentWritable.getWriter();
- try {
- await writer.write(encodeMessageStreamEvent(stampMessageStreamEvent(event)));
- } finally {
- writer.releaseLock();
- }
- } catch (writeError) {
- log.error("failed to write terminal session.failed event to durable stream", {
- errorId: typeof details.errorId === "string" ? details.errorId : undefined,
- sessionId,
- error: writeError,
- });
- }
+ await emitTerminalSessionEvent({
+ errorId: typeof details.errorId === "string" ? details.errorId : undefined,
+ event: createSessionFailedEvent({ code, details, message, sessionId }),
+ parentWritable: input.parentWritable,
+ serializedContext: input.serializedContext,
+ turnId: input.turnId,
+ });
}
diff --git a/packages/eve/src/execution/workflow-entry.test.ts b/packages/eve/src/execution/workflow-entry.test.ts
index 1da20a60c..2ed5c1263 100644
--- a/packages/eve/src/execution/workflow-entry.test.ts
+++ b/packages/eve/src/execution/workflow-entry.test.ts
@@ -7,6 +7,7 @@ import { ChannelRequestIdKey, SubagentDepthKey } from "#context/keys.js";
import { createSessionStep } from "#execution/create-session-step.js";
import {
bindTurnCallerContextStep,
+ notifyCancelledTaskCallerStep,
notifyDelegatedParentStep,
notifyTaskTurnStartedStep,
notifyTurnCallerStep,
@@ -324,6 +325,56 @@ describe("workflowEntry", () => {
serializedContext,
sessionState,
});
+ expect(emitTerminalSessionCompletionStep).toHaveBeenCalledOnce();
+ expect(emitTerminalSessionCompletionStep).toHaveBeenCalledWith({
+ parentWritable: expect.any(WritableStream),
+ serializedContext,
+ });
+ expect(emitTerminalSessionFailureStep).not.toHaveBeenCalled();
+ expect(notifyTurnCallerStep).not.toHaveBeenCalled();
+ });
+
+ it("completes a parked session when reset retires it", async () => {
+ const sessionState = createBaseSessionState();
+ const serializedContext = {
+ ...createSerializedContext(),
+ "eve.sessionId": "wrun_test_123",
+ };
+ vi.mocked(createSessionStep).mockResolvedValue(createSessionStepResultForMock(sessionState));
+ installHookMocks({
+ deliveryHooks: [
+ {
+ token: "http:test",
+ values: [{ kind: "reset", reason: "Start over" }],
+ },
+ ],
+ turnControls: [
+ turnResult({
+ action: "park",
+ serializedContext,
+ sessionState,
+ }),
+ ],
+ });
+
+ await expect(
+ workflowEntry({
+ input: { message: "hello there" },
+ serializedContext: createSerializedContext(),
+ }),
+ ).resolves.toEqual({ output: "" });
+
+ expect(terminateChildSessionsStep).toHaveBeenCalledWith({
+ serializedContext,
+ sessionState,
+ });
+ expect(emitTerminalSessionCompletionStep).toHaveBeenCalledOnce();
+ expect(emitTerminalSessionCompletionStep).toHaveBeenCalledWith({
+ parentWritable: expect.any(WritableStream),
+ serializedContext,
+ });
+ expect(emitTerminalSessionFailureStep).not.toHaveBeenCalled();
+ expect(notifyTurnCallerStep).not.toHaveBeenCalled();
});
it("exits a conflicting initial continuation before dispatching the first turn", async () => {
@@ -654,6 +705,7 @@ describe("workflowEntry", () => {
expect(emitTerminalSessionFailureStep).toHaveBeenCalledWith(
expect.objectContaining({
error: expect.objectContaining({ message: "persistent recoverable failure" }),
+ turnId: "turn_0",
}),
);
expect(terminateChildSessionsStep).toHaveBeenCalledWith({
@@ -680,6 +732,26 @@ describe("workflowEntry", () => {
expect(resolveInitialTurnCallerStep).toHaveBeenCalledOnce();
});
+ it("does not emit session.failed when notification fails after terminal completion", async () => {
+ const sessionState = createBaseSessionState();
+ vi.mocked(createSessionStep).mockResolvedValue(createSessionStepResultForMock(sessionState));
+ vi.mocked(fireSessionCallbackStep).mockRejectedValueOnce(new Error("callback failed"));
+ installHookMocks({
+ turnControls: [turnResult({ action: "done", output: "ok", sessionState })],
+ });
+
+ await expect(
+ workflowEntry({
+ input: { message: "run task" },
+ serializedContext: createSerializedContext({ "eve.mode": "task" }),
+ }),
+ ).rejects.toMatchObject({ name: "EveWorkflowFailure" });
+
+ expect(fireSessionCallbackStep).toHaveBeenCalledOnce();
+ expect(emitTerminalSessionFailureStep).not.toHaveBeenCalled();
+ expect(notifyDelegatedParentStep).not.toHaveBeenCalled();
+ });
+
it("rejects the delegated caller when the session fails before the caller is resolved", async () => {
const caller = {
callId: "call-1",
@@ -1156,6 +1228,84 @@ describe("workflowEntry", () => {
expect(notifyTurnCallerStep).not.toHaveBeenCalled();
});
+ it("retains the active turn when cancellation settlement fails", async () => {
+ const sessionState = createBaseSessionState();
+ vi.mocked(createSessionStep).mockResolvedValue(createSessionStepResultForMock(sessionState));
+ vi.mocked(settleCancelledTurnStep).mockRejectedValueOnce(new Error("settlement failed"));
+ installHookMocks({
+ turnControls: [
+ {
+ action: {
+ cancelled: true,
+ kind: "park",
+ serializedContext: { "eve.sessionId": "wrun_test_123" },
+ sessionState,
+ },
+ kind: "turn-result",
+ },
+ ],
+ });
+
+ await expect(
+ workflowEntry({
+ input: { message: "hello" },
+ serializedContext: createSerializedContext(),
+ }),
+ ).rejects.toMatchObject({ name: "EveWorkflowFailure" });
+
+ expect(emitTerminalSessionFailureStep).toHaveBeenCalledWith(
+ expect.objectContaining({
+ error: expect.objectContaining({ message: "settlement failed" }),
+ serializedContext: { "eve.sessionId": "wrun_test_123" },
+ turnId: "turn_0",
+ }),
+ );
+ });
+
+ it("adopts cancelled-turn state before notifying its caller", async () => {
+ const sessionState = createBaseSessionState();
+ const settledState = createBaseSessionState({
+ emissionState: { sequence: 1, sessionStarted: true, stepIndex: 0, turnId: "" },
+ });
+ const settledContext = { "eve.sessionId": "wrun_test_123", settled: true };
+ vi.mocked(createSessionStep).mockResolvedValue(createSessionStepResultForMock(sessionState));
+ vi.mocked(settleCancelledTurnStep).mockResolvedValue({
+ serializedContext: settledContext,
+ sessionState: settledState,
+ });
+ vi.mocked(notifyCancelledTaskCallerStep).mockRejectedValueOnce(
+ new Error("caller notification failed"),
+ );
+ installHookMocks({
+ turnControls: [
+ {
+ action: {
+ cancelled: true,
+ kind: "park",
+ serializedContext: { "eve.sessionId": "wrun_test_123" },
+ sessionState,
+ },
+ kind: "turn-result",
+ },
+ ],
+ });
+
+ await expect(
+ workflowEntry({
+ input: { message: "hello" },
+ serializedContext: createSerializedContext(),
+ }),
+ ).rejects.toMatchObject({ name: "EveWorkflowFailure" });
+
+ expect(emitTerminalSessionFailureStep).toHaveBeenCalledWith(
+ expect.objectContaining({
+ error: expect.objectContaining({ message: "caller notification failed" }),
+ serializedContext: settledContext,
+ turnId: undefined,
+ }),
+ );
+ });
+
it("does not settle an ordinary park as cancelled", async () => {
const sessionState = createBaseSessionState();
vi.mocked(createSessionStep).mockResolvedValue(createSessionStepResultForMock(sessionState));
diff --git a/packages/eve/src/execution/workflow-entry.ts b/packages/eve/src/execution/workflow-entry.ts
index 72dbae97a..1d72d04d6 100644
--- a/packages/eve/src/execution/workflow-entry.ts
+++ b/packages/eve/src/execution/workflow-entry.ts
@@ -27,10 +27,11 @@ import { cancelDescendantTurnsStep } from "#execution/cancel-descendant-turns-st
import { dispatchAndAwaitTurn } from "#execution/turn-dispatch.js";
import type { TurnDriverAction } from "#execution/turn-control-receiver.js";
import { normalizeSerializableError } from "#execution/workflow-errors.js";
-import { finalizeDone, finalizeExpiredSession } from "#execution/workflow-entry-finalization.js";
+import { finalizeDone } from "#execution/workflow-entry-finalization.js";
import { createSessionStep } from "#execution/create-session-step.js";
import { settleCancelledTurnStep } from "#execution/settle-cancelled-turn-step.js";
import { emitTerminalSessionFailureStep } from "#execution/terminal-session-failure-step.js";
+import { finalizeTerminalSession } from "#execution/finalize-terminal-session.js";
import { fireSessionCallbackStep } from "#execution/session-callback-step.js";
import { isHookConflictError } from "#execution/hook-ownership.js";
import { createSessionCommandInbox } from "#execution/session-command-inbox.js";
@@ -71,7 +72,8 @@ export interface WorkflowEntryResult {
type DriverLoopOutcome =
| {
- readonly kind: "expired";
+ readonly kind: "terminal";
+ readonly notifyCaller: boolean;
readonly serializedContext: Record<string, unknown>;
readonly sessionState: DurableSessionState;
}
@@ -97,6 +99,9 @@ type DriverLoopOutcome =
* instead of mirroring it here.
*/
interface CrashCleanupState {
+ // The turn currently owned by `dispatchAndAwaitTurn`, when a crash occurs
+ // before its terminal state returns to the driver.
+ activeTurnId: string | undefined;
// The caller whose awaited reply is still unsettled, so the catch can
// reject it with the error instead of leaving it parked forever.
// Root sessions leave it undefined; only conversation-mode paths read it.
@@ -114,6 +119,12 @@ interface CrashCleanupState {
// turn that crashed mid-flight are absent from this snapshot and escape
// crash cleanup.
lastSessionState: DurableSessionState | undefined;
+ // The latest context adopted from a completed turn, which carries provider
+ // and trace state needed by terminal instrumentation.
+ serializedContext: Record<string, unknown>;
+ // Whether the session already emitted a terminal protocol and instrumentation
+ // event, so later callback or teardown failures cannot contradict it.
+ terminalEmitted: boolean;
}
/**
@@ -150,9 +161,12 @@ export async function workflowEntry(input: WorkflowEntryInput): Promise<Workflow
const driverWritable = getWritable<Uint8Array>();
const crashCleanupState: CrashCleanupState = {
+ activeTurnId: undefined,
caller: undefined,
callerResolved: false,
lastSessionState: undefined,
+ serializedContext: input.serializedContext,
+ terminalEmitted: false,
};
try {
@@ -226,12 +240,14 @@ export async function workflowEntry(input: WorkflowEntryInput): Promise<Workflow
if (outcome.kind === "result") {
return outcome.result;
}
- return await finalizeExpiredSession({
+ return await finalizeTerminalSession({
caller: crashCleanupState.caller,
driverWritable,
mode,
+ notifyCaller: outcome.notifyCaller,
serializedContext: outcome.serializedContext,
sessionState: outcome.sessionState,
+ terminalState: crashCleanupState,
});
} catch (error) {
// Safety net for failures the tool-loop harness does not already
@@ -240,34 +256,37 @@ export async function workflowEntry(input: WorkflowEntryInput): Promise<Workflow
// channel still sees a terminal event.
if (crashCleanupState.lastSessionState !== undefined) {
await terminateChildSessionsStep({
- serializedContext: input.serializedContext,
+ serializedContext: crashCleanupState.serializedContext,
sessionState: crashCleanupState.lastSessionState,
});
}
- await emitTerminalSessionFailureStep({
- error: normalizeSerializableError(error),
- parentWritable: driverWritable,
- serializedContext: input.serializedContext,
- });
- if (mode === "task") {
- await fireSessionCallbackStep({
+ if (!crashCleanupState.terminalEmitted) {
+ await emitTerminalSessionFailureStep({
error: normalizeSerializableError(error),
- serializedContext: input.serializedContext,
- status: "failed",
+ parentWritable: driverWritable,
+ serializedContext: crashCleanupState.serializedContext,
+ turnId: crashCleanupState.activeTurnId,
});
- await notifyDelegatedParentStep({
- result: createDelegatedSubagentErrorResult(input.serializedContext, error),
- serializedContext: input.serializedContext,
- });
- } else {
- const caller = await resolveCallerForCrash(crashCleanupState, input.serializedContext);
- if (caller !== undefined) {
- await notifyTurnCallerStep({
- caller,
- lifecycle: "terminal",
- sessionId,
- settled: { isError: true, output: error },
+ if (mode === "task") {
+ await fireSessionCallbackStep({
+ error: normalizeSerializableError(error),
+ serializedContext: input.serializedContext,
+ status: "failed",
+ });
+ await notifyDelegatedParentStep({
+ result: createDelegatedSubagentErrorResult(input.serializedContext, error),
+ serializedContext: input.serializedContext,
});
+ } else {
+ const caller = await resolveCallerForCrash(crashCleanupState, input.serializedContext);
+ if (caller !== undefined) {
+ await notifyTurnCallerStep({
+ caller,
+ lifecycle: "terminal",
+ sessionId,
+ settled: { isError: true, output: error },
+ });
+ }
}
}
throw createSafeOuterWorkflowError();
@@ -449,6 +468,8 @@ async function runDriverLoop(input: {
caller,
serializedContext: stateCursor.serializedContext,
});
+ const dispatchedTurnId = activeTurnId(stateCursor.sessionState.emissionState);
+ input.crashCleanupState.activeTurnId = dispatchedTurnId;
const turn = await dispatchAndAwaitTurn({
bufferedDeliveries,
bufferedSessionControls,
@@ -463,10 +484,14 @@ async function runDriverLoop(input: {
seenTaskDeliveries,
sessionState: stateCursor.sessionState,
});
+ input.crashCleanupState.activeTurnId =
+ turn.action.kind === "park" && turn.action.cancelled === true ? dispatchedTurnId : undefined;
+ input.crashCleanupState.lastSessionState = turn.action.sessionState;
+ input.crashCleanupState.serializedContext = turn.action.serializedContext;
+ if (turn.action.kind === "done") input.crashCleanupState.terminalEmitted = true;
await disposeSettledTurnControl?.();
disposeSettledTurnControl = turn.dispose;
stateCursor.adoptState(turn.action);
- input.crashCleanupState.lastSessionState = stateCursor.sessionState;
return turn.action;
};
@@ -513,6 +538,9 @@ async function runDriverLoop(input: {
sessionState: stateCursor.sessionState,
});
stateCursor.adoptState(settled);
+ input.crashCleanupState.activeTurnId = undefined;
+ input.crashCleanupState.lastSessionState = stateCursor.sessionState;
+ input.crashCleanupState.serializedContext = stateCursor.serializedContext;
const cancelledCaller = {
caller: input.crashCleanupState.caller,
sessionId: stateCursor.sessionState.sessionId,
@@ -522,7 +550,6 @@ async function runDriverLoop(input: {
? cancelledCaller
: { ...cancelledCaller, usage: settled.usage },
);
- input.crashCleanupState.lastSessionState = stateCursor.sessionState;
}
// Channel-created sessions may rekey their dynamic alias. Sessions
@@ -558,6 +585,7 @@ async function runDriverLoop(input: {
expectedAttemptIds: action.authorizationAttemptIds ?? [],
});
input.crashCleanupState.lastSessionState = stateCursor.sessionState;
+ input.crashCleanupState.serializedContext = stateCursor.serializedContext;
if (next.kind === "authorization-resume") {
action = await runTurn({
@@ -569,18 +597,20 @@ async function runDriverLoop(input: {
if (next.kind === "expired") {
return {
- kind: "expired",
+ kind: "terminal",
+ notifyCaller: true,
serializedContext: stateCursor.serializedContext,
sessionState: stateCursor.sessionState,
};
}
if (next.kind === "reset") {
- await terminateChildSessionsStep({
+ return {
+ kind: "terminal",
+ notifyCaller: false,
serializedContext: stateCursor.serializedContext,
sessionState: stateCursor.sessionState,
- });
- return { kind: "result", result: { output: "" } };
+ };
}
if (next.kind === "clear" || next.kind === "compact") {
@@ -589,11 +619,12 @@ async function runDriverLoop(input: {
}
if (next.kind === "closed") {
- await terminateChildSessionsStep({
+ return {
+ kind: "terminal",
+ notifyCaller: false,
serializedContext: stateCursor.serializedContext,
sessionState: stateCursor.sessionState,
- });
- return { kind: "result", result: { output: "" } };
+ };
}
if (next.kind === "cancel-turn") {
@@ -613,6 +644,7 @@ async function runDriverLoop(input: {
action = { ...action, settled: undefined };
input.crashCleanupState.caller = undefined;
input.crashCleanupState.lastSessionState = stateCursor.sessionState;
+ input.crashCleanupState.serializedContext = stateCursor.serializedContext;
continue;
}
diff --git a/packages/eve/src/execution/workflow-steps.test.ts b/packages/eve/src/execution/workflow-steps.test.ts
index ae38a72b9..bb6119e0a 100644
--- a/packages/eve/src/execution/workflow-steps.test.ts
+++ b/packages/eve/src/execution/workflow-steps.test.ts
@@ -56,7 +56,9 @@ import { runProxySubagentEventStep } from "#execution/subagent-event-proxy-step.
import { readLatestTaskView, sendTaskInboundPayload } from "#execution/tasks/parent/run-parent.js";
import { recordTaskInputRequestStep } from "#execution/tasks/parent/hitl-proxy-steps.js";
import { appendTaskAgentAnnouncement } from "#execution/tasks/parent/agent-views.js";
+import { emitTerminalSessionCompletionStep } from "#execution/terminal-session-completion-step.js";
import { emitTerminalSessionFailureStep } from "#execution/terminal-session-failure-step.js";
+import { registerInstrumentationRuntime } from "#instrumentation/runtime.js";
import { resolveEffectiveOutputSchema } from "#execution/effective-output-schema.js";
import { turnStep } from "#execution/workflow-steps.js";
import { routeProxiedDeliverStep } from "#execution/proxied-deliver-step.js";
@@ -247,6 +249,7 @@ function createSerializedContext(
}
afterEach(() => {
+ delete (globalThis as Record<symbol, unknown>)[Symbol.for("eve.instrumentation-runtime")];
getRunMock.mockReset();
resumeHookMock.mockReset();
startMock.mockReset();
@@ -3073,7 +3076,7 @@ describe("turnStep", () => {
});
});
-describe("emitTerminalSessionFailureStep", () => {
+describe("terminal session event steps", () => {
function buildSerializedContextWithAdapter(
adapter: ChannelAdapter,
sessionId: string,
@@ -3115,6 +3118,47 @@ describe("emitTerminalSessionFailureStep", () => {
return serialized;
}
+ function installTerminalInstrumentation() {
+ const forceFlush = vi.fn(async () => undefined);
+ const publish = vi.fn(async () => undefined);
+ registerInstrumentationRuntime({
+ forceFlush,
+ hooks: { capturesContent: false, publish },
+ otelSettings: undefined,
+ runInContext: (_operation, execute) => execute(),
+ shutdown: async () => undefined,
+ });
+ return { forceFlush, publish };
+ }
+
+ it("publishes and flushes an out-of-turn session completion", async () => {
+ const sessionCompleted = vi.fn();
+ const adapter: ChannelAdapter = {
+ kind: "thread-context",
+ "session.completed": sessionCompleted,
+ };
+ const serializedContext = buildSerializedContextWithAdapter(adapter, "session-completed");
+ const instrumentation = installTerminalInstrumentation();
+
+ await emitTerminalSessionCompletionStep({
+ parentWritable: createTestWritable(),
+ serializedContext,
+ });
+
+ expect(sessionCompleted).toHaveBeenCalledOnce();
+ expect(instrumentation.publish).toHaveBeenCalledExactlyOnceWith({
+ idempotencyKey: "session:session-completed",
+ sessionId: "session-completed",
+ turnId: undefined,
+ type: "session.completed",
+ });
+ expect(instrumentation.forceFlush).toHaveBeenCalledOnce();
+ expect(instrumentation.publish.mock.invocationCallOrder[0]).toBeLessThan(
+ instrumentation.forceFlush.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY,
+ );
+ expect(workflowWritesByNamespace.get(DEFAULT_WORKFLOW_STREAM_NAMESPACE)).toHaveLength(1);
+ });
+
it("invokes the adapter's session.failed handler with a formatted error payload", async () => {
// Capture adapter side effects — this is how we verify the step
// actually reaches the user-visible notification path. A terminal
@@ -3128,6 +3172,7 @@ describe("emitTerminalSessionFailureStep", () => {
};
const serialized = buildSerializedContextWithAdapter(capturingAdapter, "session-terminal");
+ const instrumentation = installTerminalInstrumentation();
// Use a plain-object error shape — the workflow body converts
// raw Errors to this shape (`normalizeSerializableError`) before
@@ -3144,6 +3189,7 @@ describe("emitTerminalSessionFailureStep", () => {
error,
parentWritable: createTestWritable(),
serializedContext: serialized,
+ turnId: "turn-active",
});
expect(sessionFailedCalls).toHaveLength(1);
@@ -3163,6 +3209,21 @@ describe("emitTerminalSessionFailureStep", () => {
sessionId: "session-terminal",
});
expect(JSON.stringify(providerLog)).not.toContain("confidential.png");
+ expect(instrumentation.publish).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({
+ error: expect.objectContaining({
+ message: "attachment staging failed for confidential.png",
+ }),
+ idempotencyKey: "session:session-terminal",
+ sessionId: "session-terminal",
+ turnId: "turn-active",
+ type: "session.failed",
+ }),
+ );
+ expect(instrumentation.forceFlush).toHaveBeenCalledOnce();
+ expect(instrumentation.publish.mock.invocationCallOrder[0]).toBeLessThan(
+ instrumentation.forceFlush.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY,
+ );
// The terminal step must also write the event to the durable
// stream so event-stream consumers see a canonical tail instead
diff --git a/packages/eve/src/instrumentation/runtime.ts b/packages/eve/src/instrumentation/runtime.ts
index 9c8c045c3..3d8c50962 100644
--- a/packages/eve/src/instrumentation/runtime.ts
+++ b/packages/eve/src/instrumentation/runtime.ts
@@ -177,9 +177,9 @@ export interface SessionInstrumentation {
}
export interface ExecutionInstrumentation {
- readonly createCancellationHandleEvent: (input: {
+ readonly createHandleEvent: (input: {
readonly handleEvent?: HandleEventFn;
- readonly turnId: string;
+ readonly turnId?: string;
}) => HandleEventFn | undefined;
readonly flush: () => Promise<void>;
readonly instrumentChannelDelivery: (
@@ -460,7 +460,7 @@ export function bindInstrumentationRuntime(
};
};
return {
- createCancellationHandleEvent: (input) => {
+ createHandleEvent: (input) => {
const sessionContext = readSessionContext();
return createInstrumentationHandleEvent({
agentName: boundSession.agentName,
diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.ts
index a2dcc8b5b..ef8f7ad33 100644
--- a/packages/eve/src/tracing/agent-otel-provider.test.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts
@@ -2207,6 +2207,30 @@ describe("createAgentOtelInstrumentation", () => {
expect(turn.status).toEqual({ code: SpanStatusCode.ERROR, message: error.message });
expect(turn.attributes["error.type"]).toBe("TypeError");
});
+
+ it("marks an active invocation failed from the terminal session event", async () => {
+ const runtime = createRuntime();
+ const error = new TypeError("workflow failed");
+ await publishTurnStarted({
+ hooks: runtime.hooks,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ turnSequence: 0,
+ });
+ await runtime.hooks.publish({
+ error,
+ idempotencyKey: sessionIdempotencyKey("session-1"),
+ sessionId: "session-1",
+ turnId: "turn-1",
+ type: "session.failed",
+ });
+ await runtime.provider.forceFlush();
+
+ const turn = byName(runtime.exporter.getFinishedSpans(), "invoke_agent weather")[0]!;
+ expect(turn.status).toEqual({ code: SpanStatusCode.ERROR, message: error.message });
+ expect(turn.attributes["error.type"]).toBe("TypeError");
+ expect(turn.events.map((event) => event.name)).toContain("turn.failed");
+ });
});
function spanContext(traceId: string, spanId: string) {
diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.ts
index c61daf24a..ee3e16c98 100644
--- a/packages/eve/src/tracing/agent-otel-provider.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.ts
@@ -261,6 +261,12 @@ export function createAgentOtelInstrumentation(
const onSessionTransition = async (
event: InstrumentationSessionTransitionEvent,
): Promise<void> => {
+ if (event.type === "session.failed" && event.turnId !== undefined) {
+ await input.stateStore.updateTurn(event.sessionId, event.turnId, (turn) => ({
+ ...turn,
+ terminal: turn.terminal ?? { error: event.error, type: "turn.failed" },
+ }));
+ }
if (event.turnId !== undefined) {
const turn = await input.stateStore.getTurn(event.sessionId, event.turnId);
if (turn !== undefined) {
diff --git a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts b/packages/eve/src/tracing/install-instrumentation-runtime.test.ts
index 4baeb61ae..0c17a6e7e 100644
--- a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts
+++ b/packages/eve/src/tracing/install-instrumentation-runtime.test.ts
@@ -1,7 +1,7 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { ContextContainer, contextStorage } from "#context/container.js";
-import { turnIdempotencyKey } from "#instrumentation/lifecycle.js";
+import { sessionIdempotencyKey, turnIdempotencyKey } from "#instrumentation/lifecycle.js";
import { installInstrumentationRuntime } from "#tracing/install-instrumentation-runtime.js";
import { otelIntegration, collectOtelPipeline } from "#tracing/otel-declaration.js";
@@ -82,6 +82,37 @@ describe("installInstrumentationRuntime", () => {
expect(providerShutdown).toHaveBeenCalledOnce();
});
+ it.each(["session.completed", "session.failed"] as const)(
+ "releases local trace liveness after %s",
+ async (type) => {
+ const releaseSession = vi.fn(async () => true);
+ const processor = {
+ forceFlush: vi.fn(async () => undefined),
+ onEnd: vi.fn(),
+ onStart: vi.fn(),
+ releaseSession,
+ shutdown: vi.fn(async () => undefined),
+ };
+ const runtime = installInstrumentationRuntime({
+ collected: collectOtelPipeline([otelIntegration({ spanProcessors: [processor] })]),
+ frameworkVersion: "test",
+ providers: [],
+ serviceName: "weather",
+ });
+ const hooks = runtime.hooks.forTrace!({ agentName: "weather", audience: "unknown" });
+ const event = {
+ idempotencyKey: sessionIdempotencyKey("session-1"),
+ sessionId: "session-1",
+ type,
+ ...(type === "session.failed" ? { error: new Error("failed") } : undefined),
+ };
+
+ await contextStorage.run(new ContextContainer(), () => hooks.publish(event));
+
+ expect(releaseSession).toHaveBeenCalledExactlyOnceWith("session-1");
+ },
+ );
+
it("isolates authored state from an internal provider with the same name", async () => {
const authoredTerminalState = vi.fn();
const runtime = installInstrumentationRuntime({