Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

feat(eve): end stranded sessions on channel delivery, reset, and timeout (#4480)

prha56e814b33294
diff --git a/packages/eve/src/channel/channel-address.test.ts b/packages/eve/src/channel/channel-address.test.ts
index 6ed63f55f..a06968863 100644
--- a/packages/eve/src/channel/channel-address.test.ts
+++ b/packages/eve/src/channel/channel-address.test.ts
@@ -47,6 +47,13 @@ describe("createChannelAddress", () => {
turnPolicy: "steer",
},
continuationToken: "slack:C1:T1",
+ // Fresh creation inputs, used only if the owner's deployment was retired.
+ successor: expect.objectContaining({
+ adapter: { kind: "slack" },
+ channelName: "slack",
+ continuationToken: "slack:C1:T1",
+ input: expect.objectContaining({ message: "hello" }),
+ }),
});
expect(runtime.resolveContinuation).not.toHaveBeenCalled();
expect(runtime.createSession).not.toHaveBeenCalled();
diff --git a/packages/eve/src/channel/channel-address.ts b/packages/eve/src/channel/channel-address.ts
index 0385928ea..5f4e39433 100644
--- a/packages/eve/src/channel/channel-address.ts
+++ b/packages/eve/src/channel/channel-address.ts
@@ -38,7 +38,9 @@ export type ChannelAddressDeliveryOptions<TState = undefined> = [TState] extends
/**
* Dynamic handle for whichever durable session currently owns one channel-local address.
- * Only {@link send} may create a session when the address is unowned.
+ * Only {@link send} may create a session: when the address is unowned, or in
+ * place of a stranded session that another eve version built. Sends to a session whose
+ * owner is only unavailable throw `SessionStrandedError` and deliver nothing.
*/
interface ChannelAddress<TState = undefined> {
readonly continuationToken: string;
@@ -101,27 +103,6 @@ export function createChannelAddress<TState = undefined>(input: {
};
const command: Extract<SessionCommand, { readonly kind: "send" }> =
caller === undefined ? commandWithoutCaller : { ...commandWithoutCaller, caller };
- const dispatch = async (): Promise<Session | undefined> => {
- const result = await input.runtime.dispatchContinuation({
- command,
- continuationToken: namespacedToken,
- });
- return result.status === "accepted"
- ? createSession(result.sessionId, input.runtime, {
- ...metadata,
- turnPolicy: input.turnPolicy,
- })
- : undefined;
- };
-
- const existing = await dispatch();
- if (existing !== undefined) return existing;
- if (payload.inputResponses && payload.inputResponses.length > 0) {
- throw new Error(
- "Cannot deliver inputResponses — the target session was not found via continuation token.",
- );
- }
-
const state = (options as { readonly state?: TState }).state;
const adapter =
state === undefined
@@ -149,11 +130,21 @@ export function createChannelAddress<TState = undefined>(input: {
requestId: metadata.requestId,
title: options.title,
};
- const handle = await input.runtime.createSession(runInput);
- return createSession(handle.sessionId, input.runtime, {
- ...metadata,
- turnPolicy: input.turnPolicy,
+ const toSession = (sessionId: string) =>
+ createSession(sessionId, input.runtime, { ...metadata, turnPolicy: input.turnPolicy });
+
+ const result = await input.runtime.dispatchContinuation({
+ command,
+ continuationToken: namespacedToken,
+ successor: runInput,
});
+ if (result.status === "accepted") return toSession(result.sessionId);
+ if (payload.inputResponses && payload.inputResponses.length > 0) {
+ throw new Error(
+ "Cannot deliver inputResponses — the target session was not found via continuation token.",
+ );
+ }
+ return toSession((await input.runtime.createSession(runInput)).sessionId);
},
async send(message, options) {
return await this.deliver({ message }, options);
diff --git a/packages/eve/src/channel/channel-operations.test.ts b/packages/eve/src/channel/channel-operations.test.ts
index a9156b212..dc2664700 100644
--- a/packages/eve/src/channel/channel-operations.test.ts
+++ b/packages/eve/src/channel/channel-operations.test.ts
@@ -67,6 +67,7 @@ describe("createChannelOperations", () => {
turnPolicy: "steer",
},
continuationToken: "slack:C1:T1",
+ successor: expect.objectContaining({ title: "Support thread" }),
});
expect(runtime.dispatchSession).toHaveBeenCalledWith({
command: { kind: "clear" },
@@ -97,6 +98,9 @@ describe("createChannelOperations", () => {
turnPolicy: "steer",
},
continuationToken: "slack:C1:T1",
+ successor: expect.objectContaining({
+ adapter: { kind: "slack", state: { responder: "U_AUTHOR" } },
+ }),
});
expect(runtime.dispatchContinuation).toHaveBeenNthCalledWith(2, {
command: {
@@ -110,6 +114,9 @@ describe("createChannelOperations", () => {
turnPolicy: undefined,
},
continuationToken: "slack:C1:T1",
+ successor: expect.objectContaining({
+ adapter: { kind: "slack", state: { responder: "U_APPROVER" } },
+ }),
});
});
diff --git a/packages/eve/src/channel/session-stranded-error.ts b/packages/eve/src/channel/session-stranded-error.ts
new file mode 100644
index 000000000..49dbfdfeb
--- /dev/null
+++ b/packages/eve/src/channel/session-stranded-error.ts
@@ -0,0 +1,46 @@
+/** The owner of a stranded session, as far as it recorded itself. */
+interface StrandedSessionOwner {
+ /** eve version that built the owner, when it recorded one. */
+ readonly eveVersion?: string;
+}
+
+/**
+ * Thrown when sending, responding, clearing, or following the live stream of
+ * a session whose owner another eve version built, on a World that only runs
+ * this build, so the owner can never execute again. Nothing is delivered and
+ * nothing is mutated.
+ * Call `reset()` to end the session, then start a new one. Narrow with
+ * {@link SessionStrandedError.is}.
+ */
+export class SessionStrandedError extends Error {
+ readonly owner: StrandedSessionOwner;
+
+ constructor(owner: StrandedSessionOwner, message?: string) {
+ super(message ?? describeStrandedSession(owner));
+ this.name = "SessionStrandedError";
+ this.owner = owner;
+ }
+
+ /** Narrows stranded-session errors across bundle boundaries, where `instanceof` can miss. */
+ static is(error: unknown): error is SessionStrandedError {
+ return (
+ error instanceof SessionStrandedError ||
+ (typeof error === "object" &&
+ error !== null &&
+ Reflect.get(error, "name") === "SessionStrandedError" &&
+ typeof Reflect.get(error, "owner") === "object")
+ );
+ }
+}
+
+/**
+ * One actionable sentence pair that names the cause, the version, and the
+ * next step. `nextStep` replaces the default recovery instruction.
+ */
+export function describeStrandedSession(owner: StrandedSessionOwner, nextStep?: string): string {
+ const builtBy =
+ owner.eveVersion === undefined
+ ? "it was created by an earlier eve release"
+ : `it was created by eve ${owner.eveVersion}`;
+ return `This session is stranded: ${builtBy} on a deployment that is no longer available, so it cannot process messages. ${nextStep ?? "Reset the session to start fresh."}`;
+}
diff --git a/packages/eve/src/channel/session.ts b/packages/eve/src/channel/session.ts
index 7817fe17e..bac61b045 100644
--- a/packages/eve/src/channel/session.ts
+++ b/packages/eve/src/channel/session.ts
@@ -39,12 +39,16 @@ import { attachClientContext, readClientContext } from "#internal/client-context
/** Immutable-ID handle for one exact durable session. */
export interface Session {
readonly id: string;
- /** Sends a message to this exact session ID without creating or following a replacement. */
+ /**
+ * Sends a message to this exact session ID without creating or following a
+ * replacement. Throws `SessionStrandedError` when the session's owner cannot
+ * execute here; nothing is delivered. Call {@link reset} to end it.
+ */
send(
message: string | UserContent,
options: SessionSendOptions,
): Promise<SessionSendCommandResult>;
- /** Answers pending input requests on this exact session ID. */
+ /** Answers pending input requests on this exact session ID. Throws `SessionStrandedError` like {@link send}. */
respond<const TResponses extends readonly InputResponse[]>(
inputResponses: StrictInputResponses<TResponses>,
options: SessionRespondOptions,
@@ -57,6 +61,7 @@ export interface Session {
clear(): Promise<ClearSessionResult>;
/** Terminally retires this exact session ID. */
reset(options?: { reason?: string }): Promise<ResetSessionResult>;
+ /** Opens the recorded stream. Throws `SessionStrandedError` while its owner is stranded. */
getEventStream(options?: { startIndex?: number }): Promise<ReadableStream<MessageStreamEvent>>;
getStreamTailIndex(): Promise<number>;
}
diff --git a/packages/eve/src/channel/types.ts b/packages/eve/src/channel/types.ts
index f3088faf1..9af412170 100644
--- a/packages/eve/src/channel/types.ts
+++ b/packages/eve/src/channel/types.ts
@@ -43,7 +43,10 @@ export type CompactSessionResult =
| { readonly status: "accepted"; readonly sessionId: string }
| { readonly status: "no_active_session" };
-/** Result of queueing a manual context clear for a session. */
+/**
+ * Result of queueing a manual context clear for a session. A stranded session
+ * cannot run the clear, so it throws `SessionStrandedError` instead.
+ */
export type ClearSessionResult =
| { readonly status: "accepted"; readonly sessionId: string }
| { readonly status: "no_active_session" };
@@ -233,6 +236,11 @@ export type SessionCommandResult<TCommand extends SessionCommand = SessionComman
export interface DispatchContinuationInput<TCommand extends SessionCommand = SessionCommand> {
readonly command: TCommand;
readonly continuationToken: string;
+ /**
+ * Fresh creation inputs for a send. Used only when the address's owner is
+ * stranded: the runtime ends it and starts this successor in its place.
+ */
+ readonly successor?: RunInput;
}
export interface DispatchSessionInput<TCommand extends SessionCommand = SessionCommand> {
@@ -596,10 +604,20 @@ export interface Runtime {
*/
createSession(input: RunInput): Promise<RunHandle>;
+ /**
+ * Sends a command to whichever session owns a continuation address. When
+ * that owner is stranded, a send with a `successor` ends it and is accepted
+ * by the successor; a send without one, or a `clear`, throws
+ * `SessionStrandedError`, and a `reset` ends it.
+ */
dispatchContinuation<TCommand extends SessionCommand>(
input: DispatchContinuationInput<TCommand>,
): Promise<SessionCommandResult<TCommand>>;
+ /**
+ * Sends a command to one exact session. A send or `clear` to a stranded
+ * session throws `SessionStrandedError`; a `reset` ends it.
+ */
dispatchSession<TCommand extends SessionCommand>(
input: DispatchSessionInput<TCommand>,
): Promise<SessionCommandResult<TCommand>>;
@@ -622,6 +640,9 @@ export interface Runtime {
* first event to yield. Negative values read relative to the current tail.
* The framework HTTP session-stream route forwards the `startIndex` query
* parameter unchanged.
+ *
+ * Throws `SessionStrandedError` when the session's owner cannot execute
+ * here.
*/
getEventStream(
sessionId: string,
diff --git a/packages/eve/src/execution/agent-sessions/steps.ts b/packages/eve/src/execution/agent-sessions/steps.ts
index 86ad3611a..795d54eb5 100644
--- a/packages/eve/src/execution/agent-sessions/steps.ts
+++ b/packages/eve/src/execution/agent-sessions/steps.ts
@@ -1,4 +1,5 @@
import { getWorkflowMetadata } from "#compiled/@workflow/core/index.js";
+import { SessionStrandedError } from "#channel/session-stranded-error.js";
import { ContextContainer, contextStorage } from "#context/container.js";
import type { SessionAuth } from "#context/session-context.js";
import type { AgentSessionContext } from "#execution/agent-sessions/context.js";
@@ -159,6 +160,11 @@ export async function sendAgentSessionMessageStep(
payload: { message: input.message, outputSchema: input.outputSchema },
},
sessionId: address.sessionId,
+ }).catch((error: unknown) => {
+ if (!SessionStrandedError.is(error)) throw error;
+ throw new Error(`Agent "${address.name}" can no longer receive messages: ${error.message}`, {
+ cause: error,
+ });
});
if (result.status !== "accepted") {
throw new Error(`Agent "${address.name}" can no longer receive messages; its session ended.`);
diff --git a/packages/eve/src/execution/session-inbox/end-stranded-session.ts b/packages/eve/src/execution/session-inbox/end-stranded-session.ts
new file mode 100644
index 000000000..7e175d90d
--- /dev/null
+++ b/packages/eve/src/execution/session-inbox/end-stranded-session.ts
@@ -0,0 +1,201 @@
+import { EVE_SESSION_ATTRIBUTE, EVE_TYPE_ATTRIBUTE } from "#execution/eve-workflow-attributes.js";
+import {
+ logicalSessionInboxToken,
+ sessionCommandHookToken,
+ sessionInboxHookToken,
+ type SessionInboxAddress,
+} from "#execution/session-inbox/address.js";
+import { waitForSessionHooksRelease } from "#execution/session-inbox/hook-release.js";
+import {
+ findSessionHookHolder,
+ type StrandedSessionOwnerError,
+} from "#execution/session-inbox/owner.js";
+import { logicalSessionToken, requireSessionId } from "#execution/session-inbox/resume.js";
+import { resolveInstalledPackageInfo } from "#internal/application/package.js";
+import { createLogger } from "#internal/logging.js";
+import { isInactiveWorkflowRunError } from "#internal/workflow/is-inactive-workflow-run-error.js";
+import { cancelRun, getRun, getWorld } from "#internal/workflow/runtime.js";
+import { sessionFailed } from "#harness/session-machine/transitions.js";
+import { encodeMessageStreamEvent, stampMessageStreamEvent } from "#protocol/message.js";
+
+type World = Awaited<ReturnType<typeof getWorld>>;
+
+const HOOK_PAGE_LIMIT = 100;
+/** A session owns a handful of addresses; the cap only bounds a misbehaving World. */
+const MAX_HOOK_PAGES = 10;
+/** Publication is best effort; a stalled stream write must not hold up the reset. */
+const TERMINAL_EVENT_TIMEOUT_MS = 5_000;
+
+const log = createLogger("execution.session-inbox");
+
+/** What made eve end a stranded session; recorded for operators. */
+export type StrandedSessionEndTrigger = "message" | "reset" | "timeout";
+
+/**
+ * Ends a stranded session without running any of its code, so none of its
+ * terminal hooks fire: cancels its owner run and the original run that
+ * anchors its stream, then waits until the World has released every address
+ * the owner answered to so a fresh session can claim them. Ending cannot be
+ * undone, so each one is logged for operators. Returns the ended session's id.
+ * Does not discover or cancel descendants: their handles live in the
+ * session's state, which only its own code reads. The replay guard retires
+ * incompatible subagent and workflow tool runs when their deliveries arrive;
+ * timers remain runnable.
+ *
+ * Every step tolerates a previous or concurrent attempt having done it
+ * already, so a request that fails part way can simply be retried. Concurrent
+ * attempts are not arbitrated: each may publish and cancel independently.
+ */
+export async function endStrandedSession(
+ stranded: StrandedSessionOwnerError,
+ /** The address the caller used; a session address names the session outright. */
+ address: string | SessionInboxAddress,
+ trigger: StrandedSessionEndTrigger,
+): Promise<string> {
+ const sessionId =
+ typeof address === "string" ? await resolveStrandedSessionId(stranded) : address.sessionId;
+ const { ownerRunId } = stranded;
+ const world = await getWorld();
+ await assertStillServesSession(sessionId, ownerRunId);
+ // Listed before cancelling: a cancelled run's hooks are disposed and no longer listed.
+ const ownedTokens = await listOwnedSessionTokens(world, ownerRunId);
+ // Publish before cancellation can expire zero-retention data.
+ await publishTerminalEvent(sessionId, trigger);
+ // The session id is the original run; after a handoff it parks as the stream anchor.
+ await cancelRuns(world, [ownerRunId, sessionId], "Session ended: its deployment was retired");
+ await waitForSessionHooksRelease(
+ [sessionCommandHookToken(sessionId), logicalSessionToken(address), ...ownedTokens],
+ ownerRunId,
+ );
+ log.warn("Reset stranded session", {
+ currentEveVersion: resolveInstalledPackageInfo().version,
+ previousEveVersion: stranded.eveVersion ?? "unknown",
+ previousSessionId: sessionId,
+ trigger,
+ });
+ return sessionId;
+}
+
+/**
+ * Refuses to touch a session whose stable inbox another run took over since
+ * ingress inspected it: cancelling the anchor would end a session a newer
+ * owner still serves.
+ */
+async function assertStillServesSession(sessionId: string, ownerRunId: string): Promise<void> {
+ const holder = await findSessionHookHolder(
+ sessionInboxHookToken(sessionCommandHookToken(sessionId)),
+ );
+ // Released: an earlier attempt cancelled the owner, or it was mid-handoff.
+ if (holder === undefined) return;
+ if (holder.runId !== ownerRunId) {
+ throw new Error(
+ `Session "${sessionId}" changed owners while it was being reset. Nothing was ended; retry the request.`,
+ );
+ }
+}
+
+/**
+ * Appends `session.failed` and closes the stream without running authored
+ * hooks. Unavailable payload keys, an already-closed stream, or a stalled
+ * write must not prevent the reset.
+ */
+async function publishTerminalEvent(
+ sessionId: string,
+ trigger: StrandedSessionEndTrigger,
+): Promise<void> {
+ const writer = getRun(sessionId).getWritable<Uint8Array>().getWriter();
+ const publication = (async () => {
+ await writer.write(
+ encodeMessageStreamEvent(
+ stampMessageStreamEvent(
+ sessionFailed({
+ code: "session_stranded",
+ // A replacement reading this history skips it when a reset ended the session.
+ details: { trigger },
+ message: "This session is no longer available.",
+ sessionId,
+ usage: undefined,
+ }),
+ ),
+ ),
+ );
+ await writer.close();
+ return true;
+ })();
+ // Settles after the reset moves on when the write stalls.
+ publication.catch(() => {});
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ const timeout = new Promise<false>((resolve) => {
+ timer = setTimeout(() => resolve(false), TERMINAL_EVENT_TIMEOUT_MS);
+ });
+ try {
+ if (!(await Promise.race([publication, timeout]))) {
+ log.warn("Timed out finishing the stranded session stream", { sessionId });
+ }
+ } catch {
+ log.warn("Could not finish the stranded session stream", { sessionId });
+ } finally {
+ clearTimeout(timer);
+ writer.releaseLock();
+ }
+}
+
+/** Cancels exactly these runs; one that already ended counts as cancelled. */
+async function cancelRuns(
+ world: World,
+ runIds: readonly string[],
+ cancelReason: string,
+): Promise<void> {
+ for (const runId of new Set(runIds)) {
+ try {
+ await cancelRun(world, runId, { cancelReason });
+ } catch (error) {
+ if (!isInactiveWorkflowRunError(error)) throw error;
+ }
+ }
+}
+
+/**
+ * Public id of the session a stranded owner reached through a channel address
+ * serves. Initial owners are their own session and successors record theirs,
+ * but successors from builds that predate `$eve.session` name it only in
+ * their encrypted hook metadata. Guessing would cancel the wrong anchor, so an
+ * unknown session fails.
+ */
+async function resolveStrandedSessionId(stranded: StrandedSessionOwnerError): Promise<string> {
+ const { attributes, runId } = stranded.ownerRun;
+ const recorded = attributes[EVE_SESSION_ATTRIBUTE];
+ if (recorded !== undefined) return recorded;
+ if (stranded.hook !== undefined) {
+ try {
+ return requireSessionId(await stranded.hook.metadata);
+ } catch {
+ // A stranded owner's payload key may be gone; the plaintext record may still name it.
+ }
+ }
+ // Only a run started as a session carries `$eve.type`; handoff successors start without it.
+ if (attributes[EVE_TYPE_ATTRIBUTE] !== undefined) return runId;
+ throw new Error(
+ `Cannot reset stranded session owner run "${runId}": it does not record which session it serves. Cancel the run directly.`,
+ );
+}
+
+/** Logical session addresses the owner holds. Hook tokens are plaintext, so no payload is decrypted. */
+async function listOwnedSessionTokens(world: World, ownerRunId: string): Promise<string[]> {
+ const tokens: string[] = [];
+ let cursor: string | undefined;
+ for (let page = 0; page < MAX_HOOK_PAGES; page++) {
+ const hooks = await world.hooks.list({
+ pagination: { cursor, limit: HOOK_PAGE_LIMIT },
+ resolveData: "none",
+ runId: ownerRunId,
+ });
+ for (const hook of hooks.data) {
+ const token = logicalSessionInboxToken(hook.token);
+ if (token !== undefined) tokens.push(token);
+ }
+ if (!hooks.hasMore || hooks.cursor === null) break;
+ cursor = hooks.cursor;
+ }
+ return tokens;
+}
diff --git a/packages/eve/src/execution/session-inbox/hook-release.ts b/packages/eve/src/execution/session-inbox/hook-release.ts
new file mode 100644
index 000000000..e1c634d64
--- /dev/null
+++ b/packages/eve/src/execution/session-inbox/hook-release.ts
@@ -0,0 +1,29 @@
+import { sessionInboxHookToken } from "#execution/session-inbox/address.js";
+import { findSessionHookHolder } from "#execution/session-inbox/owner.js";
+
+/** Longest ingress waits for a session hook to change hands. */
+export const SESSION_HOOK_HANDOVER_TIMEOUT_MS = 30_000;
+
+/** Waits until none of the logical session addresses belongs to `ownerRunId` any more. */
+export async function waitForSessionHooksRelease(
+ logicalTokens: Iterable<string>,
+ ownerRunId: string,
+): Promise<void> {
+ await Promise.all(
+ [...new Set(logicalTokens)].map((token) =>
+ waitForHookRelease(sessionInboxHookToken(token), ownerRunId),
+ ),
+ );
+}
+
+async function waitForHookRelease(token: string, ownerRunId: string): Promise<void> {
+ const deadline = Date.now() + SESSION_HOOK_HANDOVER_TIMEOUT_MS;
+ while (true) {
+ if ((await findSessionHookHolder(token))?.runId !== ownerRunId) return;
+
+ if (Date.now() >= deadline) {
+ throw new Error(`Timed out waiting for session "${ownerRunId}" to release inbox "${token}".`);
+ }
+ await new Promise<void>((resolve) => setTimeout(resolve, 20));
+ }
+}
diff --git a/packages/eve/src/execution/session-stranded.integration.test.ts b/packages/eve/src/execution/session-stranded.integration.test.ts
new file mode 100644
index 000000000..4f345cce5
--- /dev/null
+++ b/packages/eve/src/execution/session-stranded.integration.test.ts
@@ -0,0 +1,413 @@
+import { describe, expect, it, vi } from "vitest";
+
+import { createChannelAddress } from "#channel/channel-address.js";
+import { createSession } from "#channel/session.js";
+import { SessionStrandedError } from "#channel/session-stranded-error.js";
+import type { Runtime } from "#channel/types.js";
+import {
+ EVE_SESSION_ATTRIBUTE,
+ EVE_VERSION_ATTRIBUTE,
+} from "#execution/eve-workflow-attributes.js";
+import { signalSessionTimeoutStep } from "#execution/session/timeout-steps.js";
+import {
+ sessionCommandHookToken,
+ sessionInboxHookToken,
+} from "#execution/session-inbox/address.js";
+import { handleSessionCallbackRequest } from "#subagents/callback-route.js";
+import { readSessionEventStream } from "#execution/session-event-stream.js";
+import { createWorkflowRuntime } from "#execution/workflow-runtime.js";
+import { resolveInstalledPackageInfo } from "#internal/application/package.js";
+import { createTestRuntime } from "#internal/testing/app-harness.js";
+import { captureLogRecords } from "#internal/testing/log-records.js";
+import {
+ parkedAnchorWorkflow,
+ sessionCommandInboxWorkflow,
+} from "#internal/testing/session-inbox-workflow.js";
+import { startSessionOwner, waitForHook } from "#internal/testing/workflow-test-helpers.js";
+import { getWorld, start } from "#internal/workflow/runtime.js";
+import { createBundledRuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js";
+
+/** eve version recorded by the owner an earlier build left parked on the local World. */
+const PREVIOUS_EVE_VERSION = "0.0.1";
+
+type ParkedRun = Awaited<ReturnType<typeof start>>;
+
+/**
+ * Parks a session owner the way an earlier eve build left it: its run records
+ * that build's version, which the running build cannot replay.
+ */
+async function parkStrandedOwner(
+ continuationToken: string,
+ attributes: Readonly<Record<string, string>> = {},
+): Promise<ParkedRun> {
+ const run = await start(sessionCommandInboxWorkflow, [{ token: continuationToken }], {
+ allowReservedAttributes: true,
+ attributes: { ...attributes, [EVE_VERSION_ATTRIBUTE]: PREVIOUS_EVE_VERSION },
+ });
+ await waitForHook(run, { token: sessionInboxHookToken(continuationToken) });
+ return run;
+}
+
+/** Parks a stranded handoff successor that owns the session of a separate anchor run. */
+async function parkStrandedSuccessor(continuationToken: string) {
+ const anchor = await start(parkedAnchorWorkflow, [{ token: `${continuationToken}:anchor` }]);
+ await waitForHook(anchor, { token: `${continuationToken}:anchor` });
+ const owner = await start(
+ sessionCommandInboxWorkflow,
+ [{ sessionId: anchor.runId, token: continuationToken }],
+ {
+ allowReservedAttributes: true,
+ attributes: {
+ [EVE_SESSION_ATTRIBUTE]: anchor.runId,
+ [EVE_VERSION_ATTRIBUTE]: PREVIOUS_EVE_VERSION,
+ },
+ },
+ );
+ await waitForHook(owner, { token: sessionInboxHookToken(continuationToken) });
+ return { anchor, owner };
+}
+
+async function listEventIds(runId: string): Promise<string[]> {
+ const world = await getWorld();
+ const events = await world.events.list({ runId, pagination: { limit: 1000 } });
+ return events.data.map((event) => event.eventId);
+}
+
+async function cancelIfActive(...runs: readonly ParkedRun[]): Promise<void> {
+ for (const run of runs) {
+ const status = await run.status;
+ if (status === "pending" || status === "running") await run.cancel();
+ }
+}
+
+/** Reads a session's public stream until every expected user message has been received. */
+async function waitForReceivedMessages(sessionId: string, expected: readonly string[]) {
+ const reader = readSessionEventStream(sessionId).getReader();
+ const pending = new Set(expected);
+ try {
+ while (pending.size > 0) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ if (value.type === "message.received") {
+ for (const message of pending) {
+ if (value.data.message.includes(message)) pending.delete(message);
+ }
+ }
+ }
+ } finally {
+ await reader.cancel().catch(() => {});
+ }
+ return [...pending];
+}
+
+/** A fresh session claims its address after `createSession` returns. */
+async function waitForAliasOwner(runtime: Runtime, continuationToken: string): Promise<string> {
+ const deadline = Date.now() + 30_000;
+ while (Date.now() < deadline) {
+ const owner = await runtime.resolveContinuation(continuationToken);
+ if (owner !== undefined) return owner.sessionId;
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ }
+ throw new Error(`No session claimed "${continuationToken}".`);
+}
+
+function channelRuntime(): Runtime & { readonly createSession: ReturnType<typeof vi.fn> } {
+ const runtime = createWorkflowRuntime({
+ compiledArtifactsSource: createBundledRuntimeCompiledArtifactsSource(),
+ });
+ return { ...runtime, createSession: vi.fn(runtime.createSession) };
+}
+
+function httpAddress(runtime: Runtime, continuationToken: string) {
+ return createChannelAddress({
+ adapter: { kind: "http" },
+ channelName: "http",
+ continuationToken,
+ runtime,
+ });
+}
+
+async function resetSession(runtime: Runtime, sessionId: string | undefined): Promise<void> {
+ if (sessionId === undefined) return;
+ await runtime.dispatchSession({ command: { kind: "reset" }, sessionId });
+}
+
+describe("stranded sessions", () => {
+ it("ends a stranded session on a channel delivery and starts a fresh one", async () => {
+ const test = await createTestRuntime({ agent: { name: "stranded-session-alias" } });
+ await test.run(async () => {
+ const logs = captureLogRecords();
+ const continuationToken = "http:stranded-session-alias";
+ const { anchor, owner } = await parkStrandedSuccessor(continuationToken);
+ const runtime = channelRuntime();
+ let freshSessionId: string | undefined;
+ const reader = readSessionEventStream(anchor.runId).getReader();
+ try {
+ const waiting = reader.read();
+ const fresh = await httpAddress(runtime, "stranded-session-alias").send(
+ "Alice asks for the weekly summary after the upgrade.",
+ { auth: null },
+ );
+ freshSessionId = fresh.id;
+
+ await expect(waiting).resolves.toMatchObject({
+ done: false,
+ value: {
+ type: "session.failed",
+ data: {
+ code: "session_stranded",
+ details: { trigger: "message" },
+ message: "This session is no longer available.",
+ sessionId: anchor.runId,
+ },
+ },
+ });
+ await expect(reader.read()).resolves.toMatchObject({ done: true });
+ expect(fresh.id).not.toBe(anchor.runId);
+ // The runtime started the successor itself; the channel started nothing.
+ expect(runtime.createSession).not.toHaveBeenCalled();
+ await expect(owner.status).resolves.toBe("cancelled");
+ await expect(anchor.status).resolves.toBe("cancelled");
+ await waitForHook({ runId: fresh.id }, { token: sessionInboxHookToken(continuationToken) });
+ await expect(runtime.resolveContinuation(continuationToken)).resolves.toEqual({
+ sessionId: fresh.id,
+ });
+ await expect(
+ waitForReceivedMessages(fresh.id, ["Alice asks for the weekly summary"]),
+ ).resolves.toEqual([]);
+ // A remote job can finish after replacement. Its fixed parent inbox
+ // must not route the result into the new conversation.
+ const callback = await handleSessionCallbackRequest(
+ new Request("https://eve.test/callback", {
+ method: "POST",
+ body: JSON.stringify({
+ kind: "turn.completed",
+ callId: "old-call",
+ subagentName: "research",
+ outcome: {
+ kind: "parked",
+ result: { kind: "succeeded", output: "Alice's report" },
+ usageDelta: {
+ inputTokens: 0,
+ outputTokens: 0,
+ cacheReadTokens: 0,
+ cacheWriteTokens: 0,
+ },
+ },
+ output: "Alice's report",
+ }),
+ }),
+ {
+ params: { token: sessionInboxHookToken(sessionCommandHookToken(anchor.runId)) },
+ requestIp: null,
+ waitUntil() {},
+ },
+ );
+ expect(callback.status).toBe(404);
+ await expect(runtime.resolveContinuation(continuationToken)).resolves.toEqual({
+ sessionId: fresh.id,
+ });
+
+ const resets = logs.records.filter((record) => record.message === "Reset stranded session");
+ expect(resets).toEqual([
+ expect.objectContaining({
+ fields: {
+ currentEveVersion: resolveInstalledPackageInfo().version,
+ previousEveVersion: PREVIOUS_EVE_VERSION,
+ previousSessionId: anchor.runId,
+ trigger: "message",
+ },
+ level: "warn",
+ }),
+ ]);
+ } finally {
+ await reader.cancel().catch(() => {});
+ reader.releaseLock();
+ await cancelIfActive(owner, anchor);
+ await resetSession(runtime, freshSessionId);
+ }
+ });
+ });
+
+ it("ends a stranded session on a context-only channel delivery", async () => {
+ const test = await createTestRuntime({ agent: { name: "stranded-session-context" } });
+ await test.run(async () => {
+ const continuationToken = "http:stranded-session-context";
+ const stranded = await parkStrandedOwner(continuationToken);
+ const runtime = channelRuntime();
+ let freshSessionId: string | undefined;
+ try {
+ const fresh = await httpAddress(runtime, "stranded-session-context").deliver(
+ { context: ["Alice moved the standup to Thursday."] },
+ { auth: null },
+ );
+ freshSessionId = fresh.id;
+
+ expect(fresh.id).not.toBe(stranded.runId);
+ await expect(stranded.status).resolves.toBe("cancelled");
+ await expect(waitForAliasOwner(runtime, continuationToken)).resolves.toBe(fresh.id);
+ } finally {
+ await cancelIfActive(stranded);
+ await resetSession(runtime, freshSessionId);
+ }
+ });
+ });
+
+ it("starts one fresh session for concurrent deliveries to a stranded session", async () => {
+ const test = await createTestRuntime({ agent: { name: "stranded-session-concurrent" } });
+ await test.run(async () => {
+ const continuationToken = "http:stranded-session-concurrent";
+ const stranded = await parkStrandedOwner(continuationToken);
+ const runtime = channelRuntime();
+ const address = httpAddress(runtime, "stranded-session-concurrent");
+ let freshSessionId: string | undefined;
+ try {
+ const candidates = await Promise.all([
+ address.send("Alice shares the release notes.", { auth: null }),
+ address.send("Bob shares the rollout plan.", { auth: null }),
+ ]);
+ freshSessionId = await waitForAliasOwner(runtime, continuationToken);
+
+ expect(candidates.map((session) => session.id)).toContain(freshSessionId);
+ expect(freshSessionId).not.toBe(stranded.runId);
+ await expect(stranded.status).resolves.toBe("cancelled");
+ await expect(
+ waitForReceivedMessages(freshSessionId!, [
+ "Alice shares the release notes.",
+ "Bob shares the rollout plan.",
+ ]),
+ ).resolves.toEqual([]);
+ } finally {
+ await cancelIfActive(stranded);
+ await resetSession(runtime, freshSessionId);
+ }
+ });
+ });
+
+ it("refuses a send by session id without writing to the stranded run", async () => {
+ const test = await createTestRuntime({ agent: { name: "stranded-session-by-id" } });
+ await test.run(async () => {
+ const stranded = await parkStrandedOwner("http:stranded-session-by-id");
+ const runtime = channelRuntime();
+ try {
+ const eventsBefore = await listEventIds(stranded.runId);
+
+ const refusal = await createSession(stranded.runId, runtime)
+ .send("Alice follows up after the upgrade.", { auth: null })
+ .catch((error: unknown) => error);
+
+ expect(refusal).toBeInstanceOf(SessionStrandedError);
+ expect(refusal).toMatchObject({ owner: { eveVersion: PREVIOUS_EVE_VERSION } });
+ expect((refusal as Error).message).not.toContain(stranded.runId);
+ expect(runtime.createSession).not.toHaveBeenCalled();
+ await expect(stranded.status).resolves.toBe("running");
+ await expect(listEventIds(stranded.runId)).resolves.toEqual(eventsBefore);
+ } finally {
+ await cancelIfActive(stranded);
+ }
+ });
+ });
+
+ it("ends a stranded session on an explicit reset and refuses to clear it", async () => {
+ const test = await createTestRuntime({ agent: { name: "stranded-session-reset" } });
+ await test.run(async () => {
+ const continuationToken = "http:stranded-session-reset";
+ const stranded = await parkStrandedOwner(continuationToken);
+ const runtime = channelRuntime();
+ try {
+ // Clear keeps the session, which cannot run, so the caller must choose reset.
+ await expect(
+ runtime.dispatchSession({ command: { kind: "clear" }, sessionId: stranded.runId }),
+ ).rejects.toBeInstanceOf(SessionStrandedError);
+ await expect(stranded.status).resolves.toBe("running");
+
+ await expect(
+ runtime.dispatchSession({ command: { kind: "reset" }, sessionId: stranded.runId }),
+ ).resolves.toEqual({ previousSessionId: stranded.runId, status: "reset" });
+ await expect(stranded.status).resolves.toBe("cancelled");
+ await expect(runtime.resolveContinuation(continuationToken)).resolves.toBeUndefined();
+ } finally {
+ await cancelIfActive(stranded);
+ }
+ });
+ });
+
+ it("keeps a stranded session when a channel delivery also carries input responses", async () => {
+ const test = await createTestRuntime({ agent: { name: "stranded-session-responses" } });
+ await test.run(async () => {
+ const continuationToken = "http:stranded-session-responses";
+ const stranded = await parkStrandedOwner(continuationToken);
+ const runtime = channelRuntime();
+ try {
+ const refusal = await httpAddress(runtime, "stranded-session-responses")
+ .deliver(
+ {
+ inputResponses: [{ optionId: "approve", requestId: "request_1" }],
+ message: "Alice approves the deploy and adds a note.",
+ },
+ { auth: null },
+ )
+ .catch((error: unknown) => error);
+
+ expect(refusal).toBeInstanceOf(SessionStrandedError);
+ expect((refusal as Error).message).toContain("pending requests can no longer be answered");
+ expect(runtime.createSession).not.toHaveBeenCalled();
+ await expect(stranded.status).resolves.toBe("running");
+ } finally {
+ await cancelIfActive(stranded);
+ }
+ });
+ });
+
+ it("delivers to a session owned by the running eve version", async () => {
+ const test = await createTestRuntime({ agent: { name: "stranded-session-runnable" } });
+ await test.run(async () => {
+ const continuationToken = "http:stranded-session-runnable";
+ const owner = await startSessionOwner(sessionCommandInboxWorkflow, [
+ { token: continuationToken },
+ ]);
+ await waitForHook(owner, { token: sessionInboxHookToken(continuationToken) });
+ const runtime = channelRuntime();
+ try {
+ const session = await httpAddress(runtime, "stranded-session-runnable").send(
+ "Bob checks in on the deploy.",
+ { auth: null },
+ );
+
+ expect(session.id).toBe(owner.runId);
+ expect(runtime.createSession).not.toHaveBeenCalled();
+ await expect(owner.status).resolves.toBe("running");
+ } finally {
+ await cancelIfActive(owner);
+ }
+ });
+ });
+
+ it("ends a retired session when its session timeout elapses", async () => {
+ const test = await createTestRuntime({ agent: { name: "stranded-session-timeout" } });
+ await test.run(async () => {
+ const logs = captureLogRecords();
+ const continuationToken = "http:stranded-session-timeout";
+ const stranded = await parkStrandedOwner(continuationToken);
+ const runtime = channelRuntime();
+ try {
+ await signalSessionTimeoutStep({
+ ownerRunId: stranded.runId,
+ token: sessionCommandHookToken(stranded.runId),
+ });
+
+ await expect(stranded.status).resolves.toBe("cancelled");
+ await expect(runtime.resolveContinuation(continuationToken)).resolves.toBeUndefined();
+ expect(runtime.createSession).not.toHaveBeenCalled();
+ expect(logs.records).toContainEqual(
+ expect.objectContaining({
+ fields: expect.objectContaining({ trigger: "timeout" }),
+ message: "Reset stranded session",
+ }),
+ );
+ } finally {
+ await cancelIfActive(stranded);
+ }
+ });
+ });
+});
diff --git a/packages/eve/src/execution/session/timeout-steps.test.ts b/packages/eve/src/execution/session/timeout-steps.test.ts
index f8fbead61..92619df2f 100644
--- a/packages/eve/src/execution/session/timeout-steps.test.ts
+++ b/packages/eve/src/execution/session/timeout-steps.test.ts
@@ -86,6 +86,20 @@ describe("session timeout steps", () => {
});
});
+ it("drops a signal to a stranded owner without delivering it", async () => {
+ getWorldMock.mockResolvedValue(singleDeploymentWorld("0.0.1"));
+ // Runnable owners are remembered per process, so the stranded owner needs its own run.
+ getHookByTokenMock.mockImplementation(async (token: string) => {
+ if (!token.startsWith(sessionInboxHookToken(""))) throw new HookNotFoundError(token);
+ return { runId: "stranded-owner", specVersion: 6, token };
+ });
+
+ await expect(
+ signalSessionTimeoutStep({ ownerRunId: "session-1", token: "session-1:session-timeout" }),
+ ).resolves.toBeUndefined();
+ expect(resumeHookMock).not.toHaveBeenCalled();
+ });
+
it("ignores a signal after the owning session is gone", async () => {
const { HookNotFoundError } = await import("#compiled/@workflow/errors/index.js");
resumeHookMock.mockRejectedValue(new HookNotFoundError("session-1:session-timeout"));
diff --git a/packages/eve/src/execution/session/timeout-steps.ts b/packages/eve/src/execution/session/timeout-steps.ts
index 8f003ce2a..d82a44e34 100644
--- a/packages/eve/src/execution/session/timeout-steps.ts
+++ b/packages/eve/src/execution/session/timeout-steps.ts
@@ -11,6 +11,8 @@ import {
} from "#execution/workflow-runtime.js";
import type { SessionTimeoutWorkflowInput } from "#execution/session/timeout-workflow.js";
import { resumeSessionInbox } from "#execution/session-inbox/resume.js";
+import { endStrandedSession } from "#execution/session-inbox/end-stranded-session.js";
+import { StrandedSessionOwnerError } from "#execution/session-inbox/owner.js";
import { cancelRun, getWorld } from "#internal/workflow/runtime.js";
import { walkCauseChain } from "#shared/errors.js";
@@ -37,6 +39,17 @@ export async function signalSessionTimeoutStep(input: {
ownerRunId: input.ownerRunId,
});
} catch (error) {
+ if (error instanceof StrandedSessionOwnerError) {
+ // A stranded owner another timer armed is that timer's to end.
+ if (error.ownerRunId !== input.ownerRunId) return;
+ // An owner whose deployment was retired cannot finish itself, so the
+ // timer ends it instead. This bounds a retired session's lifetime: the
+ // timer is its own run, which the stranded replay guard does not skip,
+ // and a timer still sleeping has recorded no step, so it replays on this
+ // build and runs this step.
+ await endStrandedSession(error, input.token, "timeout");
+ return;
+ }
if (!isInactiveTimeoutTarget(error)) {
throw error;
}
diff --git a/packages/eve/src/execution/workflow-runtime.ts b/packages/eve/src/execution/workflow-runtime.ts
index e36490eed..bd315a481 100644
--- a/packages/eve/src/execution/workflow-runtime.ts
+++ b/packages/eve/src/execution/workflow-runtime.ts
@@ -62,15 +62,25 @@ import { isAgentTraceContext } from "#tracing/agent-trace-context.js";
import {
sessionCommandHookToken,
sessionInboxHookToken,
+ type SessionInboxAddress,
} from "#execution/session-inbox/address.js";
+import { endStrandedSession } from "#execution/session-inbox/end-stranded-session.js";
+import {
+ SESSION_HOOK_HANDOVER_TIMEOUT_MS,
+ waitForSessionHooksRelease,
+} from "#execution/session-inbox/hook-release.js";
import {
AcceptedSessionIdentityError,
resolveSessionInbox,
+ logicalSessionToken,
resumeSessionInbox,
SessionHandoffPendingError,
} from "#execution/session-inbox/resume.js";
-import { StrandedSessionOwnerError } from "#execution/session-inbox/owner.js";
-import type { SessionInboxAddress } from "#execution/session-inbox/address.js";
+import {
+ assertRunnableSessionInbox,
+ StrandedSessionOwnerError,
+} from "#execution/session-inbox/owner.js";
+import { describeStrandedSession, SessionStrandedError } from "#channel/session-stranded-error.js";
import type { DynamicSubagentAgentConfig } from "#runtime/subagents/dynamic-agent-config.js";
import { initializeSessionInstrumentation } from "#instrumentation/runtime.js";
import {
@@ -79,7 +89,6 @@ import {
WORKFLOW_ENTRY_NAME,
} from "#execution/stable-workflow-names.js";
const EVE_PACKAGE_INFO = resolveInstalledPackageInfo();
-const COMMAND_HOOK_READY_TIMEOUT_MS = 30_000;
const STABLE_ID_BASE = EVE_PACKAGE_INFO.name;
@@ -122,107 +131,132 @@ export function createWorkflowRuntime(config: {
readonly dynamicSubagentAgentConfig?: DynamicSubagentAgentConfig;
readonly nodeId?: string;
}): Runtime {
- return {
- async createSession(input: RunInput): Promise<RunHandle> {
- const bundle = await getCompiledRuntimeAgentBundle({
- compiledArtifactsSource: config.compiledArtifactsSource,
- nodeId: config.nodeId,
- });
- if (input.toolStubs !== undefined && input.toolStubs.rootSessionId === undefined) {
- validateToolStubTargets(input.toolStubs.rules, bundle.graph);
- }
- const ctx = buildRunContext({
- bundle,
- dynamicSubagentAgentConfig: config.dynamicSubagentAgentConfig,
- run: input,
- });
- const effectiveAgent = resolveEffectiveAgentRuntime(bundle, ctx);
- initializeSessionInstrumentation({
- agentName: effectiveAgent.turnAgent.id,
- ctx,
- });
- const sessionTimeoutMs = effectiveAgent.limits?.sessionTimeoutMs;
- // Retention is always the authored value: `experimental` cannot be
- // selected by a dynamic subagent config, so there is nothing to resolve.
- const retention = bundle.resolvedAgent.config?.experimental?.workflow?.retention;
- const serializedContext = serializeContext(ctx);
- const parentLineage = readParentLineage(serializedContext);
- const workflowInput: {
- -readonly [K in keyof InitialWorkflowEntryInput]: InitialWorkflowEntryInput[K];
- } = {
- kind: "initial",
- input: input.input,
- ownerDeploymentId: await resolveCurrentWorkflowDeploymentId(),
- serializedContext,
- };
- if (input.limits !== undefined) workflowInput.limits = input.limits;
- if (input.continuationConflictCommand !== undefined) {
- workflowInput.continuationConflictCommand = input.continuationConflictCommand;
- }
- if (sessionTimeoutMs !== undefined) {
- workflowInput.sessionTimeoutMs = sessionTimeoutMs;
- }
- if (retention !== undefined) {
- workflowInput.retention = retention;
- }
- const sessionAttributes =
- parentLineage.sessionId === undefined
- ? buildSessionAttributes({
- serializedContext,
- })
- : buildSubagentRootAttributes({
- identity: { nodeId: bundle.nodeId ?? ROOT_RUNTIME_AGENT_NODE_ID },
- parentCallId: parentLineage.callId,
- parentSessionId: parentLineage.sessionId,
- parentTurnId: parentLineage.turnId,
- rootSessionId: parentLineage.rootSessionId ?? parentLineage.sessionId,
- serializedContext,
- });
- const attributes = {
- ...sessionAttributes,
- ...(input.externalInvocation === undefined
- ? {}
- : buildInvocationAttributes(input.externalInvocation)),
+ const startSession = async (input: RunInput): Promise<RunHandle> => {
+ const bundle = await getCompiledRuntimeAgentBundle({
+ compiledArtifactsSource: config.compiledArtifactsSource,
+ nodeId: config.nodeId,
+ });
+ if (input.toolStubs !== undefined && input.toolStubs.rootSessionId === undefined) {
+ validateToolStubTargets(input.toolStubs.rules, bundle.graph);
+ }
+ const ctx = buildRunContext({
+ bundle,
+ dynamicSubagentAgentConfig: config.dynamicSubagentAgentConfig,
+ run: input,
+ });
+ const effectiveAgent = resolveEffectiveAgentRuntime(bundle, ctx);
+ initializeSessionInstrumentation({
+ agentName: effectiveAgent.turnAgent.id,
+ ctx,
+ });
+ const sessionTimeoutMs = effectiveAgent.limits?.sessionTimeoutMs;
+ // Retention is always the authored value: `experimental` cannot be
+ // selected by a dynamic subagent config, so there is nothing to resolve.
+ const retention = bundle.resolvedAgent.config?.experimental?.workflow?.retention;
+ const serializedContext = serializeContext(ctx);
+ const parentLineage = readParentLineage(serializedContext);
+ const workflowInput: {
+ -readonly [K in keyof InitialWorkflowEntryInput]: InitialWorkflowEntryInput[K];
+ } = {
+ kind: "initial",
+ input: input.input,
+ ownerDeploymentId: await resolveCurrentWorkflowDeploymentId(),
+ serializedContext,
+ };
+ if (input.limits !== undefined) workflowInput.limits = input.limits;
+ if (input.continuationConflictCommand !== undefined) {
+ workflowInput.continuationConflictCommand = input.continuationConflictCommand;
+ }
+ if (sessionTimeoutMs !== undefined) {
+ workflowInput.sessionTimeoutMs = sessionTimeoutMs;
+ }
+ if (retention !== undefined) {
+ workflowInput.retention = retention;
+ }
+ const sessionAttributes =
+ parentLineage.sessionId === undefined
+ ? buildSessionAttributes({
+ serializedContext,
+ })
+ : buildSubagentRootAttributes({
+ identity: { nodeId: bundle.nodeId ?? ROOT_RUNTIME_AGENT_NODE_ID },
+ parentCallId: parentLineage.callId,
+ parentSessionId: parentLineage.sessionId,
+ parentTurnId: parentLineage.turnId,
+ rootSessionId: parentLineage.rootSessionId ?? parentLineage.sessionId,
+ serializedContext,
+ });
+ const attributes = {
+ ...sessionAttributes,
+ ...(input.externalInvocation === undefined
+ ? {}
+ : buildInvocationAttributes(input.externalInvocation)),
+ };
+
+ let run: Awaited<ReturnType<typeof startWorkflowOnCurrentDeployment>>;
+ try {
+ const startOptions: StartOptionsWithoutDeploymentId = {
+ allowReservedAttributes: true,
+ attributes: normalizeEveAttributes(attributes),
};
+ if (retention !== undefined) startOptions.experimental_retention = retention;
+ run = await startWorkflowOnDeployment(
+ workflowEntryReference,
+ [workflowInput],
+ workflowInput.ownerDeploymentId,
+ startOptions,
+ );
+ } catch (error) {
+ logError(log, "failed to start workflow run", error, {
+ continuationToken: input.continuationToken,
+ });
+ throw error;
+ }
- let run: Awaited<ReturnType<typeof startWorkflowOnCurrentDeployment>>;
- try {
- const startOptions: StartOptionsWithoutDeploymentId = {
- allowReservedAttributes: true,
- attributes: normalizeEveAttributes(attributes),
- };
- if (retention !== undefined) startOptions.experimental_retention = retention;
- run = await startWorkflowOnDeployment(
- workflowEntryReference,
- [workflowInput],
- workflowInput.ownerDeploymentId,
- startOptions,
- );
- } catch (error) {
- logError(log, "failed to start workflow run", error, {
- continuationToken: input.continuationToken,
- });
- throw error;
- }
-
- let events: ReadableStream<MessageStreamEvent> | undefined;
- const getEvents = () => {
- events ??= readSessionEventStream(run.runId);
- return events;
- };
+ let events: ReadableStream<MessageStreamEvent> | undefined;
+ const getEvents = () => {
+ events ??= readSessionEventStream(run.runId);
+ return events;
+ };
+
+ return {
+ get events() {
+ return getEvents();
+ },
+ sessionId: run.runId,
+ };
+ };
- return {
- get events() {
- return getEvents();
- },
- sessionId: run.runId,
- };
+ return {
+ async createSession(input: RunInput): Promise<RunHandle> {
+ return await startSession(input);
},
async dispatchContinuation<TCommand extends SessionCommand>(
input: DispatchContinuationInput<TCommand>,
): Promise<SessionCommandResult<TCommand>> {
- return await dispatchWorkflowCommand(input.continuationToken, input.command);
+ const { command, continuationToken, successor } = input;
+ try {
+ return await dispatchWorkflowCommand(continuationToken, command);
+ } catch (error) {
+ if (!(error instanceof StrandedSessionOwnerError)) throw error;
+ if (command.kind !== "send" || successor === undefined) {
+ return await settleStrandedCommand(continuationToken, command, error);
+ }
+ // A successor has no request for these answers, so they cannot be delivered anywhere.
+ if ((command.payload.inputResponses?.length ?? 0) > 0) {
+ throw sessionStrandedError(
+ error,
+ "Its pending requests can no longer be answered; send a new message to start a fresh session.",
+ );
+ }
+ await endStrandedSession(error, continuationToken, "message");
+ const started = await startSession(successor);
+ return {
+ sessionId: started.sessionId,
+ status: "accepted",
+ } as SessionCommandResult<TCommand>;
+ }
},
async dispatchSession<TCommand extends SessionCommand>(
@@ -235,6 +269,14 @@ export function createWorkflowRuntime(config: {
sessionId: string,
options?: GetEventStreamOptions,
): Promise<ReadableStream<MessageStreamEvent>> {
+ // An ended session has no inbox, and a dormant `eve dev` run may resume;
+ // either recorded stream stays readable. Only a stranded owner refuses.
+ try {
+ await assertRunnableSessionInbox(sessionInboxHookToken(sessionCommandHookToken(sessionId)));
+ } catch (error) {
+ if (error instanceof StrandedSessionOwnerError) throw sessionStrandedError(error);
+ throw error;
+ }
return readSessionEventStream(sessionId, options?.startIndex);
},
@@ -330,19 +372,48 @@ async function dispatchWorkflowCommand<TCommand extends SessionCommand>(
}
if (command.kind === "reset") {
- const addressedToken =
- typeof token === "string" ? token : sessionCommandHookToken(token.sessionId);
- const tokens = new Set([sessionCommandHookToken(hook.sessionId), addressedToken]);
- await Promise.all(
- [...tokens].map((logicalToken) =>
- waitForHookRelease(sessionInboxHookToken(logicalToken), hook.runId),
- ),
+ await waitForSessionHooksRelease(
+ [sessionCommandHookToken(hook.sessionId), logicalSessionToken(token)],
+ hook.runId,
);
}
return activeCommandResult(command, hook.sessionId);
}
+/**
+ * What a command means for a stranded owner, which can run none. `reset` ends
+ * the session. A send or `clear` needs the session to run, so it is refused
+ * and the caller decides whether to reset. `cancel` and `compact` find no
+ * active session.
+ */
+async function settleStrandedCommand<TCommand extends SessionCommand>(
+ address: string | SessionInboxAddress,
+ command: TCommand,
+ stranded: StrandedSessionOwnerError,
+): Promise<SessionCommandResult<TCommand>> {
+ switch (command.kind) {
+ case "reset": {
+ const previousSessionId = await endStrandedSession(stranded, address, "reset");
+ return { previousSessionId, status: "reset" } as SessionCommandResult<TCommand>;
+ }
+ case "send":
+ case "clear":
+ throw sessionStrandedError(stranded);
+ default:
+ return inactiveCommandResult(command);
+ }
+}
+
+/** Public form of a stranded owner; `nextStep` replaces the default recovery instruction. */
+function sessionStrandedError(
+ stranded: StrandedSessionOwnerError,
+ nextStep?: string,
+): SessionStrandedError {
+ const owner = stranded.eveVersion === undefined ? {} : { eveVersion: stranded.eveVersion };
+ return new SessionStrandedError(owner, describeStrandedSession(owner, nextStep));
+}
+
function activeCommandResult<TCommand extends SessionCommand>(
command: TCommand,
sessionId: string,
@@ -368,11 +439,21 @@ function inactiveCommandResult<TCommand extends SessionCommand>(
return result as SessionCommandResult<TCommand>;
}
-/** Sends one command to a session through its stable command inbox. */
+/**
+ * Sends one command to a session through its stable command inbox. A caller
+ * holding the session id asked for that exact session, so a send to a
+ * stranded one is refused rather than replaced.
+ */
export async function dispatchWorkflowSessionCommand<TCommand extends SessionCommand>(
input: DispatchSessionInput<TCommand>,
): Promise<SessionCommandResult<TCommand>> {
- return await dispatchWorkflowCommand({ sessionId: input.sessionId }, input.command);
+ const address = { sessionId: input.sessionId };
+ try {
+ return await dispatchWorkflowCommand(address, input.command);
+ } catch (error) {
+ if (!(error instanceof StrandedSessionOwnerError)) throw error;
+ return await settleStrandedCommand(address, input.command, error);
+ }
}
/** Requests cancellation through a session's stable command inbox. */
@@ -381,7 +462,7 @@ export async function requestWorkflowTurnCancellation(
): Promise<CancelTurnResult> {
const command: { kind: "cancel"; turnId?: string } = { kind: "cancel" };
if (input.turnId !== undefined) command.turnId = input.turnId;
- return await dispatchWorkflowCommand({ sessionId: input.sessionId }, command);
+ return await dispatchWorkflowSessionCommand({ command, sessionId: input.sessionId });
}
async function isRunActive(runId: string): Promise<boolean> {
@@ -415,7 +496,7 @@ function isInactiveCommandTarget(error: unknown): boolean {
* leave ownership arbitration to the workflow.
*/
export async function waitForCommandHookOwner(token: string): Promise<WorkflowHookRecord> {
- const deadline = Date.now() + COMMAND_HOOK_READY_TIMEOUT_MS;
+ const deadline = Date.now() + SESSION_HOOK_HANDOVER_TIMEOUT_MS;
while (true) {
try {
return normalizeWorkflowHook(await getHookByToken(token));
@@ -426,24 +507,6 @@ export async function waitForCommandHookOwner(token: string): Promise<WorkflowHo
}
}
-async function waitForHookRelease(token: string, ownerRunId: string): Promise<void> {
- const deadline = Date.now() + COMMAND_HOOK_READY_TIMEOUT_MS;
- while (true) {
- try {
- const owner = normalizeWorkflowHook(await getHookByToken(token));
- if (owner.runId !== ownerRunId) return;
- } catch (error) {
- if (HookNotFoundError.is(error)) return;
- throw error;
- }
-
- if (Date.now() >= deadline) {
- throw new Error(`Timed out waiting for session "${ownerRunId}" to release inbox "${token}".`);
- }
- await new Promise<void>((resolve) => setTimeout(resolve, 20));
- }
-}
-
/** Starts a workflow on the deployment executing this call. */
export async function startWorkflowOnCurrentDeployment<TArgs extends unknown[], TResult>(
workflow: WorkflowFunction<TArgs, TResult> | WorkflowMetadata,
diff --git a/packages/eve/test/scenarios/cross-channel-receive.scenario.test.ts b/packages/eve/test/scenarios/cross-channel-receive.scenario.test.ts
index 4adfe2b32..ea02ed925 100644
--- a/packages/eve/test/scenarios/cross-channel-receive.scenario.test.ts
+++ b/packages/eve/test/scenarios/cross-channel-receive.scenario.test.ts
@@ -7,7 +7,7 @@ import {
createCrossChannelToFn,
toCrossChannelTargets,
} from "../../src/channel/cross-channel-receive.js";
-import type { Runtime, SessionCommand, SessionCommandResult } from "../../src/channel/types.js";
+import type { SessionCommandResult, Runtime, SessionCommand } from "../../src/channel/types.js";
import { compileAgent } from "../../src/compiler/compile-agent.js";
import { createDiskRuntimeCompiledArtifactsSource } from "../../src/runtime/compiled-artifacts-source.js";
import { getCompiledRuntimeAgentBundle } from "../../src/runtime/sessions/compiled-agent-cache.js";