The Open Framework for Building Agents
fix(connections): recover dynamic resolver failures (#3405)
roboeve[bot]bf82cfa8837d
diff --git a/.changeset/recover-dynamic-connection-resolvers.md b/.changeset/recover-dynamic-connection-resolvers.mdnew file mode 100644index 000000000..be71a772e--- /dev/null+++ b/.changeset/recover-dynamic-connection-resolvers.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Keep conversation sessions available after a dynamic connection resolver fails, so a later message can retry once the dependency recovers.diff --git a/docs/guides/dynamic-capabilities.md b/docs/guides/dynamic-capabilities.mdindex e8395e6c4..ec00cf123 100644--- a/docs/guides/dynamic-capabilities.md+++ b/docs/guides/dynamic-capabilities.md@@ -221,9 +221,16 @@ same name; namespace one map key to remove the ambiguity.Dynamic connections support `session.started` and `turn.started`. A turn resultreplaces that file's session result for the turn, including when the turn-handler returns `null`. A throwing or invalid handler fails the lifecycle-without rebuilding the registry, so a static connection shadowed by the-dynamic result cannot reappear as a fallback.+handler returns `null`. A throwing or invalid handler fails the current turn+and parks the session, so a later message can retry after the dependency+recovers. The failed lifecycle does not rebuild the registry, so a static+connection shadowed by the dynamic result cannot reappear as a fallback.++Recovery needs another message. eve does not keep the input from the failed+turn or replay it, so the caller must send it again once the dependency is+back. This applies to the first message of a session too. A scheduled run+whose resolver fails is not retried; the session stays idle until the next+scheduled run or message arrives.eve may run the active session and turn handlers again when a parked turnresumes or a durable step retries. This rebuilds live auth, header, approval,diff --git a/packages/eve/src/context/dynamic-connection-lifecycle.ts b/packages/eve/src/context/dynamic-connection-lifecycle.tsindex 08b8cf1e9..9c1d37084 100644--- a/packages/eve/src/context/dynamic-connection-lifecycle.ts+++ b/packages/eve/src/context/dynamic-connection-lifecycle.ts@@ -18,6 +18,27 @@ import type {import { toErrorMessage } from "#shared/errors.js";const log = createLogger("dynamic-connections");+const DYNAMIC_CONNECTION_RESOLUTION_ERROR_CODE = "EVE_DYNAMIC_CONNECTION_RESOLUTION_FAILED";++export class DynamicConnectionResolutionError extends Error {+ readonly code = DYNAMIC_CONNECTION_RESOLUTION_ERROR_CODE;+ override readonly name = "DynamicConnectionResolutionError";++ constructor(error: unknown) {+ super(toErrorMessage(error), { cause: error });+ }+}++export function isDynamicConnectionResolutionError(+ error: unknown,+): error is DynamicConnectionResolutionError {+ return (+ error instanceof DynamicConnectionResolutionError ||+ (typeof error === "object" &&+ error !== null &&+ (error as { readonly code?: unknown }).code === DYNAMIC_CONNECTION_RESOLUTION_ERROR_CODE)+ );+}function qualifyConnectionNames(resolver: ResolvedDynamicConnectionResolver,@@ -95,6 +116,7 @@ export async function dispatchDynamicConnectionEvent(input: {);const updates = new Map<string, readonly ResolvedConnectionDefinition[]>();let failedResolver: ResolvedDynamicConnectionResolver | undefined;+ let failedReason: unknown;for (let index = 0; index < outcomes.length; index += 1) {const outcome = outcomes[index]!;const resolver = matching[index]!;@@ -103,14 +125,20 @@ export async function dispatchDynamicConnectionEvent(input: {error: toErrorMessage(outcome.reason),logicalPath: resolver.logicalPath,});- failedResolver ??= resolver;+ if (failedResolver === undefined) {+ failedResolver = resolver;+ failedReason = outcome.reason;+ }continue;}updates.set(outcome.value.resolver.slug, outcome.value.connections);}if (failedResolver !== undefined) {- throw new Error(- `Dynamic connection resolver "${failedResolver.logicalPath}" failed during "${input.event.type}".`,+ throw new DynamicConnectionResolutionError(+ new Error(+ `Dynamic connection resolver "${failedResolver.logicalPath}" failed during "${input.event.type}".`,+ { cause: failedReason },+ ),);}diff --git a/packages/eve/src/execution/dynamic-connection-recovery.ts b/packages/eve/src/execution/dynamic-connection-recovery.tsnew file mode 100644index 000000000..d27cfef86--- /dev/null+++ b/packages/eve/src/execution/dynamic-connection-recovery.ts@@ -0,0 +1,50 @@+import { isDynamicConnectionResolutionError } from "#context/dynamic-connection-lifecycle.js";+import { applyTransition, sessionView } from "#harness/session-machine/commit.js";+import { fail } from "#harness/session-machine/transitions.js";+import { activeTurnId, turnPosition } from "#harness/session-machine/view.js";+import type { HandleEventFn, HarnessSession, StepResult } from "#harness/types.js";+import { createErrorId, createLogger } from "#internal/logging.js";+import type { SessionProjection } from "#protocol/session-projection.js";+import { toErrorMessage } from "#shared/errors.js";++const log = createLogger("turn-step");++/**+ * Parks the session with `step.failed` → `turn.failed` → `session.waiting`+ * when a dynamic connection resolver throws during rehydration, so the next+ * message can retry once the dependency recovers. Returns `undefined` for any+ * other error so the caller rethrows it.+ */+export async function recoverDynamicConnectionRehydration(input: {+ readonly error: unknown;+ readonly emit: HandleEventFn;+ readonly projection: SessionProjection;+ readonly session: HarnessSession;+}): Promise<StepResult | undefined> {+ if (!isDynamicConnectionResolutionError(input.error)) {+ return undefined;+ }++ const errorId = createErrorId();+ const message = toErrorMessage(input.error);+ log.error("dynamic connection rehydration failed — parking session", {+ error: input.error,+ errorId,+ sessionId: input.session.sessionId,+ turnId: activeTurnId(turnPosition(input.projection)),+ });+ const session = await applyTransition(+ input.session,+ fail(sessionView(input.projection, input.session.state), {+ code: "EVENT_HANDLER_FAILED",+ details: { errorId },+ message,+ }),+ input.emit,+ );+ return {+ next: null,+ session: { ...session, outputSchema: undefined },+ settledTurn: { isError: true, output: message },+ };+}diff --git a/packages/eve/src/execution/session/entry.integration.test.ts b/packages/eve/src/execution/session/entry.integration.test.tsindex 7b09a9550..6b1d70742 100644--- a/packages/eve/src/execution/session/entry.integration.test.ts+++ b/packages/eve/src/execution/session/entry.integration.test.ts@@ -18,6 +18,7 @@ import {import { createWorkflowRuntime } from "#execution/workflow-runtime.js";import { normalizeEveAttributes } from "#runtime/attributes/normalize.js";import { defineHook } from "#public/definitions/hook.js";+import { defineDynamic } from "#dynamic/definition.js";import { sessions } from "#public/server/index.js";import type { MessageStreamEvent } from "#protocol/message.js";import { isEventId } from "#internal/testing/event-id.js";@@ -385,6 +386,135 @@ describe("workflowEntry integration", () => {expect(output.unexpected(workflowSdkNotice.unpinnedDelivery)).toEqual([]);});+ it("parks a failed dynamic connection rehydration and accepts the next message", async () => {+ let shouldFail = false;+ const continuationToken = "http:workflow-entry-dynamic-connection-failure";+ const runtime = await createTestRuntime({+ agent: { name: "workflow-entry-dynamic-connection-failure" },+ modules: [+ {+ logicalPath: "connections/accounts.ts",+ loadNamespace: async () => ({+ default: defineDynamic({+ events: {+ "session.started": () => {+ if (shouldFail) throw new Error("account store unavailable");+ return null;+ },+ },+ }),+ }),+ },+ ],+ });++ await runtime.run(async () => {+ const run = await start(workflowEntry, [+ {+ kind: "initial",+ ownerDeploymentId: "dpl_inline",+ input: { message: "first message" },+ serializedContext: buildSerializedContext({+ channelKind: "http",+ continuationToken,+ }),+ },+ ]);+ const stream = captureTurnEvents(run);+ const inboxToken = sessionInboxHookToken(continuationToken);++ try {+ expect((await stream.nextTurn()).at(-1)?.type).toBe("session.waiting");++ await waitForHook({ runId: run.runId }, { token: inboxToken });+ shouldFail = true;+ await resumeHook(inboxToken, {+ kind: "send",+ payload: { message: "failed message" },+ });++ const failedTurn = await stream.nextTurn();+ expect(failedTurn.at(-1)?.type).toBe("session.waiting");+ expect(filterEventsByType(failedTurn, "turn.failed")).toHaveLength(1);+ expect(filterEventsByType(failedTurn, "session.failed")).toHaveLength(0);++ shouldFail = false;+ await resumeHook(inboxToken, {+ kind: "send",+ payload: { message: "recovered message" },+ });++ const recoveredTurn = await stream.nextTurn();+ expect(recoveredTurn.at(-1)?.type).toBe("session.waiting");+ expect(filterEventsByType(recoveredTurn, "session.failed")).toHaveLength(0);+ } finally {+ stream.dispose();+ await run.cancel();+ }+ });+ });++ it("parks a dynamic connection failure on the first message and accepts the next message", async () => {+ let shouldFail = true;+ const continuationToken = "http:workflow-entry-dynamic-connection-first-failure";+ const runtime = await createTestRuntime({+ agent: { name: "workflow-entry-dynamic-connection-first-failure" },+ modules: [+ {+ logicalPath: "connections/accounts.ts",+ loadNamespace: async () => ({+ default: defineDynamic({+ events: {+ "session.started": () => {+ if (shouldFail) throw new Error("account store unavailable");+ return null;+ },+ },+ }),+ }),+ },+ ],+ });++ await runtime.run(async () => {+ const run = await start(workflowEntry, [+ {+ kind: "initial",+ ownerDeploymentId: "dpl_inline",+ input: { message: "first message" },+ serializedContext: buildSerializedContext({+ channelKind: "http",+ continuationToken,+ }),+ },+ ]);+ const stream = captureTurnEvents(run);+ const inboxToken = sessionInboxHookToken(continuationToken);++ try {+ const failedTurn = await stream.nextTurn();+ expect(failedTurn.at(-1)?.type).toBe("session.waiting");+ expect(filterEventsByType(failedTurn, "turn.failed")).toHaveLength(1);+ expect(filterEventsByType(failedTurn, "session.failed")).toHaveLength(0);++ await waitForHook({ runId: run.runId }, { token: inboxToken });+ shouldFail = false;+ await resumeHook(inboxToken, {+ kind: "send",+ payload: { message: "recovered message" },+ });++ const recoveredTurn = await stream.nextTurn();+ expect(recoveredTurn.at(-1)?.type).toBe("session.waiting");+ expect(filterEventsByType(recoveredTurn, "turn.failed")).toHaveLength(0);+ expect(filterEventsByType(recoveredTurn, "session.failed")).toHaveLength(0);+ } finally {+ stream.dispose();+ await run.cancel();+ }+ });+ });+it("publishes the session ID as the waiting address for an ID-only session", async () => {const runtime = await createTestRuntime({ agent: { name: "workflow-entry-id-only" } });diff --git a/packages/eve/src/execution/session/turn-step.ts b/packages/eve/src/execution/session/turn-step.tsindex 70869ad7c..111ade392 100644--- a/packages/eve/src/execution/session/turn-step.ts+++ b/packages/eve/src/execution/session/turn-step.ts@@ -1,6 +1,7 @@import { bindTurnCallerContext } from "#subagents/parent-notification.js";import type { HandleEventFn } from "#harness/types.js";import { bindDynamicConnections } from "#execution/dynamic-connections.js";+import { recoverDynamicConnectionRehydration } from "#execution/dynamic-connection-recovery.js";import { deriveSessionTitle } from "#execution/eve-workflow-attributes.js";import { setEveAttributes } from "#runtime/attributes/emit.js";import { defaultDeliverResult } from "#channel/adapter.js";@@ -526,13 +527,24 @@ async function runSessionStepBody(? { ...enrichedSession, outputSchema: resolved.outputSchema }: enrichedSession;const connectionState = turnPosition(currentProjection(ctx));- await dynamicConnections.rehydrate(- connectionState,- runtimeIdentity,- isBetweenTurns(currentProjection(ctx))- ? undefined- : { sequence: connectionState.sequence, turnId: activeTurnId(connectionState) },- );+ try {+ await dynamicConnections.rehydrate(+ connectionState,+ runtimeIdentity,+ isBetweenTurns(currentProjection(ctx))+ ? undefined+ : { sequence: connectionState.sequence, turnId: activeTurnId(connectionState) },+ );+ } catch (error) {+ const recovered = await recoverDynamicConnectionRehydration({+ emit: handleEvent,+ error,+ projection: currentProjection(ctx),+ session: schemaSession,+ });+ if (recovered !== undefined) return recovered;+ throw error;+ }// A sign-in completes before the turn it resumes, in the first call only.const completions =firstCall && completedAuths !== undefineddiff --git a/packages/eve/src/harness/step/context.ts b/packages/eve/src/harness/step/context.tsindex 0f7ef622c..57ae9faf5 100644--- a/packages/eve/src/harness/step/context.ts+++ b/packages/eve/src/harness/step/context.ts@@ -5,6 +5,7 @@ import {drainDynamicInstructionUserMessages,prepareDynamicInstructionPreamble,} from "#context/dynamic-instruction-lifecycle.js";+import { isDynamicConnectionResolutionError } from "#context/dynamic-connection-lifecycle.js";import { isDynamicModelSelectionError } from "#context/dynamic-model-lifecycle.js";import { ParentSessionKey, SessionCallbackKey } from "#context/keys.js";import { drainMemoryCommit, prepareMemoryPreamble } from "#context/memory-lifecycle.js";@@ -158,14 +159,42 @@ export async function openTurn(return undefined;}-/** A lifecycle event failed to publish: only a dynamic model selection failure is reported. */+/**+ * A lifecycle event failed to publish: a dynamic model selection failure ends the session, and a+ * dynamic connection resolver failure parks it. Anything else throws.+ */export async function failBoundaryEvent(step: Step, error: unknown): Promise<StepResult> {throwIfTurnAborted(step.config.abortSignal);if (isTurnCancellation(error)) throw error;if (isDynamicModelSelectionError(error)) return failModelSelection(step, error);+ if (isDynamicConnectionResolutionError(error)) return failConnectionResolution(step, error);throw error;}+/** A dynamic connection resolver threw: the session parks so the next message can retry. */+export async function failConnectionResolution(step: Step, error: unknown): Promise<StepResult> {+ if (step.emit === undefined) throw error;+ step.instrumentation?.recordError(error);++ const errorId = createErrorId();+ const message = toErrorMessage(error);+ log.error("dynamic connection resolver failed — parking session", {+ error,+ errorId,+ sessionId: step.session.sessionId,+ turnId: activeTurnId(step.position()),+ });+ await step.apply(+ fail(step.view(), { code: "EVENT_HANDLER_FAILED", details: { errorId }, message }),+ );+ step.session = { ...step.session, outputSchema: undefined };+ return {+ next: null,+ session: step.session,+ settledTurn: { isError: true, output: message },+ };+}+/** No model can serve the turn: the session fails. */export async function failModelSelection(step: Step, error: unknown): Promise<StepResult> {throwIfTurnAborted(step.config.abortSignal);