Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(connections): recover dynamic resolver failures (#3405)

roboeve[bot]bf82cfa8837d
diff --git a/.changeset/recover-dynamic-connection-resolvers.md b/.changeset/recover-dynamic-connection-resolvers.md
new file mode 100644
index 000000000..be71a772e
--- /dev/null
+++ b/.changeset/recover-dynamic-connection-resolvers.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Keep conversation sessions available after a dynamic connection resolver fails, so a later message can retry once the dependency recovers.
diff --git a/docs/guides/dynamic-capabilities.md b/docs/guides/dynamic-capabilities.md
index e8395e6c4..ec00cf123 100644
--- a/docs/guides/dynamic-capabilities.md
+++ b/docs/guides/dynamic-capabilities.md
@@ -221,9 +221,16 @@ same name; namespace one map key to remove the ambiguity.
Dynamic connections support `session.started` and `turn.started`. A turn result
replaces that file's session result for the turn, including when the turn
-handler returns `null`. A throwing or invalid handler fails the lifecycle
-without rebuilding the registry, so a static connection shadowed by the
-dynamic result cannot reappear as a fallback.
+handler returns `null`. A throwing or invalid handler fails the current turn
+and parks the session, so a later message can retry after the dependency
+recovers. The failed lifecycle does not rebuild the registry, so a static
+connection shadowed by the dynamic result cannot reappear as a fallback.
+
+Recovery needs another message. eve does not keep the input from the failed
+turn or replay it, so the caller must send it again once the dependency is
+back. This applies to the first message of a session too. A scheduled run
+whose resolver fails is not retried; the session stays idle until the next
+scheduled run or message arrives.
eve may run the active session and turn handlers again when a parked turn
resumes or a durable step retries. This rebuilds live auth, header, approval,
diff --git a/packages/eve/src/context/dynamic-connection-lifecycle.ts b/packages/eve/src/context/dynamic-connection-lifecycle.ts
index 08b8cf1e9..9c1d37084 100644
--- a/packages/eve/src/context/dynamic-connection-lifecycle.ts
+++ b/packages/eve/src/context/dynamic-connection-lifecycle.ts
@@ -18,6 +18,27 @@ import type {
import { toErrorMessage } from "#shared/errors.js";
const log = createLogger("dynamic-connections");
+const DYNAMIC_CONNECTION_RESOLUTION_ERROR_CODE = "EVE_DYNAMIC_CONNECTION_RESOLUTION_FAILED";
+
+export class DynamicConnectionResolutionError extends Error {
+ readonly code = DYNAMIC_CONNECTION_RESOLUTION_ERROR_CODE;
+ override readonly name = "DynamicConnectionResolutionError";
+
+ constructor(error: unknown) {
+ super(toErrorMessage(error), { cause: error });
+ }
+}
+
+export function isDynamicConnectionResolutionError(
+ error: unknown,
+): error is DynamicConnectionResolutionError {
+ return (
+ error instanceof DynamicConnectionResolutionError ||
+ (typeof error === "object" &&
+ error !== null &&
+ (error as { readonly code?: unknown }).code === DYNAMIC_CONNECTION_RESOLUTION_ERROR_CODE)
+ );
+}
function qualifyConnectionNames(
resolver: ResolvedDynamicConnectionResolver,
@@ -95,6 +116,7 @@ export async function dispatchDynamicConnectionEvent(input: {
);
const updates = new Map<string, readonly ResolvedConnectionDefinition[]>();
let failedResolver: ResolvedDynamicConnectionResolver | undefined;
+ let failedReason: unknown;
for (let index = 0; index < outcomes.length; index += 1) {
const outcome = outcomes[index]!;
const resolver = matching[index]!;
@@ -103,14 +125,20 @@ export async function dispatchDynamicConnectionEvent(input: {
error: toErrorMessage(outcome.reason),
logicalPath: resolver.logicalPath,
});
- failedResolver ??= resolver;
+ if (failedResolver === undefined) {
+ failedResolver = resolver;
+ failedReason = outcome.reason;
+ }
continue;
}
updates.set(outcome.value.resolver.slug, outcome.value.connections);
}
if (failedResolver !== undefined) {
- throw new Error(
- `Dynamic connection resolver "${failedResolver.logicalPath}" failed during "${input.event.type}".`,
+ throw new DynamicConnectionResolutionError(
+ new Error(
+ `Dynamic connection resolver "${failedResolver.logicalPath}" failed during "${input.event.type}".`,
+ { cause: failedReason },
+ ),
);
}
diff --git a/packages/eve/src/execution/dynamic-connection-recovery.ts b/packages/eve/src/execution/dynamic-connection-recovery.ts
new file mode 100644
index 000000000..d27cfef86
--- /dev/null
+++ b/packages/eve/src/execution/dynamic-connection-recovery.ts
@@ -0,0 +1,50 @@
+import { isDynamicConnectionResolutionError } from "#context/dynamic-connection-lifecycle.js";
+import { applyTransition, sessionView } from "#harness/session-machine/commit.js";
+import { fail } from "#harness/session-machine/transitions.js";
+import { activeTurnId, turnPosition } from "#harness/session-machine/view.js";
+import type { HandleEventFn, HarnessSession, StepResult } from "#harness/types.js";
+import { createErrorId, createLogger } from "#internal/logging.js";
+import type { SessionProjection } from "#protocol/session-projection.js";
+import { toErrorMessage } from "#shared/errors.js";
+
+const log = createLogger("turn-step");
+
+/**
+ * Parks the session with `step.failed` → `turn.failed` → `session.waiting`
+ * when a dynamic connection resolver throws during rehydration, so the next
+ * message can retry once the dependency recovers. Returns `undefined` for any
+ * other error so the caller rethrows it.
+ */
+export async function recoverDynamicConnectionRehydration(input: {
+ readonly error: unknown;
+ readonly emit: HandleEventFn;
+ readonly projection: SessionProjection;
+ readonly session: HarnessSession;
+}): Promise<StepResult | undefined> {
+ if (!isDynamicConnectionResolutionError(input.error)) {
+ return undefined;
+ }
+
+ const errorId = createErrorId();
+ const message = toErrorMessage(input.error);
+ log.error("dynamic connection rehydration failed — parking session", {
+ error: input.error,
+ errorId,
+ sessionId: input.session.sessionId,
+ turnId: activeTurnId(turnPosition(input.projection)),
+ });
+ const session = await applyTransition(
+ input.session,
+ fail(sessionView(input.projection, input.session.state), {
+ code: "EVENT_HANDLER_FAILED",
+ details: { errorId },
+ message,
+ }),
+ input.emit,
+ );
+ return {
+ next: null,
+ session: { ...session, outputSchema: undefined },
+ settledTurn: { isError: true, output: message },
+ };
+}
diff --git a/packages/eve/src/execution/session/entry.integration.test.ts b/packages/eve/src/execution/session/entry.integration.test.ts
index 7b09a9550..6b1d70742 100644
--- a/packages/eve/src/execution/session/entry.integration.test.ts
+++ b/packages/eve/src/execution/session/entry.integration.test.ts
@@ -18,6 +18,7 @@ import {
import { createWorkflowRuntime } from "#execution/workflow-runtime.js";
import { normalizeEveAttributes } from "#runtime/attributes/normalize.js";
import { defineHook } from "#public/definitions/hook.js";
+import { defineDynamic } from "#dynamic/definition.js";
import { sessions } from "#public/server/index.js";
import type { MessageStreamEvent } from "#protocol/message.js";
import { isEventId } from "#internal/testing/event-id.js";
@@ -385,6 +386,135 @@ describe("workflowEntry integration", () => {
expect(output.unexpected(workflowSdkNotice.unpinnedDelivery)).toEqual([]);
});
+ it("parks a failed dynamic connection rehydration and accepts the next message", async () => {
+ let shouldFail = false;
+ const continuationToken = "http:workflow-entry-dynamic-connection-failure";
+ const runtime = await createTestRuntime({
+ agent: { name: "workflow-entry-dynamic-connection-failure" },
+ modules: [
+ {
+ logicalPath: "connections/accounts.ts",
+ loadNamespace: async () => ({
+ default: defineDynamic({
+ events: {
+ "session.started": () => {
+ if (shouldFail) throw new Error("account store unavailable");
+ return null;
+ },
+ },
+ }),
+ }),
+ },
+ ],
+ });
+
+ await runtime.run(async () => {
+ const run = await start(workflowEntry, [
+ {
+ kind: "initial",
+ ownerDeploymentId: "dpl_inline",
+ input: { message: "first message" },
+ serializedContext: buildSerializedContext({
+ channelKind: "http",
+ continuationToken,
+ }),
+ },
+ ]);
+ const stream = captureTurnEvents(run);
+ const inboxToken = sessionInboxHookToken(continuationToken);
+
+ try {
+ expect((await stream.nextTurn()).at(-1)?.type).toBe("session.waiting");
+
+ await waitForHook({ runId: run.runId }, { token: inboxToken });
+ shouldFail = true;
+ await resumeHook(inboxToken, {
+ kind: "send",
+ payload: { message: "failed message" },
+ });
+
+ const failedTurn = await stream.nextTurn();
+ expect(failedTurn.at(-1)?.type).toBe("session.waiting");
+ expect(filterEventsByType(failedTurn, "turn.failed")).toHaveLength(1);
+ expect(filterEventsByType(failedTurn, "session.failed")).toHaveLength(0);
+
+ shouldFail = false;
+ await resumeHook(inboxToken, {
+ kind: "send",
+ payload: { message: "recovered message" },
+ });
+
+ const recoveredTurn = await stream.nextTurn();
+ expect(recoveredTurn.at(-1)?.type).toBe("session.waiting");
+ expect(filterEventsByType(recoveredTurn, "session.failed")).toHaveLength(0);
+ } finally {
+ stream.dispose();
+ await run.cancel();
+ }
+ });
+ });
+
+ it("parks a dynamic connection failure on the first message and accepts the next message", async () => {
+ let shouldFail = true;
+ const continuationToken = "http:workflow-entry-dynamic-connection-first-failure";
+ const runtime = await createTestRuntime({
+ agent: { name: "workflow-entry-dynamic-connection-first-failure" },
+ modules: [
+ {
+ logicalPath: "connections/accounts.ts",
+ loadNamespace: async () => ({
+ default: defineDynamic({
+ events: {
+ "session.started": () => {
+ if (shouldFail) throw new Error("account store unavailable");
+ return null;
+ },
+ },
+ }),
+ }),
+ },
+ ],
+ });
+
+ await runtime.run(async () => {
+ const run = await start(workflowEntry, [
+ {
+ kind: "initial",
+ ownerDeploymentId: "dpl_inline",
+ input: { message: "first message" },
+ serializedContext: buildSerializedContext({
+ channelKind: "http",
+ continuationToken,
+ }),
+ },
+ ]);
+ const stream = captureTurnEvents(run);
+ const inboxToken = sessionInboxHookToken(continuationToken);
+
+ try {
+ const failedTurn = await stream.nextTurn();
+ expect(failedTurn.at(-1)?.type).toBe("session.waiting");
+ expect(filterEventsByType(failedTurn, "turn.failed")).toHaveLength(1);
+ expect(filterEventsByType(failedTurn, "session.failed")).toHaveLength(0);
+
+ await waitForHook({ runId: run.runId }, { token: inboxToken });
+ shouldFail = false;
+ await resumeHook(inboxToken, {
+ kind: "send",
+ payload: { message: "recovered message" },
+ });
+
+ const recoveredTurn = await stream.nextTurn();
+ expect(recoveredTurn.at(-1)?.type).toBe("session.waiting");
+ expect(filterEventsByType(recoveredTurn, "turn.failed")).toHaveLength(0);
+ expect(filterEventsByType(recoveredTurn, "session.failed")).toHaveLength(0);
+ } finally {
+ stream.dispose();
+ await run.cancel();
+ }
+ });
+ });
+
it("publishes the session ID as the waiting address for an ID-only session", async () => {
const runtime = await createTestRuntime({ agent: { name: "workflow-entry-id-only" } });
diff --git a/packages/eve/src/execution/session/turn-step.ts b/packages/eve/src/execution/session/turn-step.ts
index 70869ad7c..111ade392 100644
--- a/packages/eve/src/execution/session/turn-step.ts
+++ b/packages/eve/src/execution/session/turn-step.ts
@@ -1,6 +1,7 @@
import { bindTurnCallerContext } from "#subagents/parent-notification.js";
import type { HandleEventFn } from "#harness/types.js";
import { bindDynamicConnections } from "#execution/dynamic-connections.js";
+import { recoverDynamicConnectionRehydration } from "#execution/dynamic-connection-recovery.js";
import { deriveSessionTitle } from "#execution/eve-workflow-attributes.js";
import { setEveAttributes } from "#runtime/attributes/emit.js";
import { defaultDeliverResult } from "#channel/adapter.js";
@@ -526,13 +527,24 @@ async function runSessionStepBody(
? { ...enrichedSession, outputSchema: resolved.outputSchema }
: enrichedSession;
const connectionState = turnPosition(currentProjection(ctx));
- await dynamicConnections.rehydrate(
- connectionState,
- runtimeIdentity,
- isBetweenTurns(currentProjection(ctx))
- ? undefined
- : { sequence: connectionState.sequence, turnId: activeTurnId(connectionState) },
- );
+ try {
+ await dynamicConnections.rehydrate(
+ connectionState,
+ runtimeIdentity,
+ isBetweenTurns(currentProjection(ctx))
+ ? undefined
+ : { sequence: connectionState.sequence, turnId: activeTurnId(connectionState) },
+ );
+ } catch (error) {
+ const recovered = await recoverDynamicConnectionRehydration({
+ emit: handleEvent,
+ error,
+ projection: currentProjection(ctx),
+ session: schemaSession,
+ });
+ if (recovered !== undefined) return recovered;
+ throw error;
+ }
// A sign-in completes before the turn it resumes, in the first call only.
const completions =
firstCall && completedAuths !== undefined
diff --git a/packages/eve/src/harness/step/context.ts b/packages/eve/src/harness/step/context.ts
index 0f7ef622c..57ae9faf5 100644
--- a/packages/eve/src/harness/step/context.ts
+++ b/packages/eve/src/harness/step/context.ts
@@ -5,6 +5,7 @@ import {
drainDynamicInstructionUserMessages,
prepareDynamicInstructionPreamble,
} from "#context/dynamic-instruction-lifecycle.js";
+import { isDynamicConnectionResolutionError } from "#context/dynamic-connection-lifecycle.js";
import { isDynamicModelSelectionError } from "#context/dynamic-model-lifecycle.js";
import { ParentSessionKey, SessionCallbackKey } from "#context/keys.js";
import { drainMemoryCommit, prepareMemoryPreamble } from "#context/memory-lifecycle.js";
@@ -158,14 +159,42 @@ export async function openTurn(
return undefined;
}
-/** A lifecycle event failed to publish: only a dynamic model selection failure is reported. */
+/**
+ * A lifecycle event failed to publish: a dynamic model selection failure ends the session, and a
+ * dynamic connection resolver failure parks it. Anything else throws.
+ */
export async function failBoundaryEvent(step: Step, error: unknown): Promise<StepResult> {
throwIfTurnAborted(step.config.abortSignal);
if (isTurnCancellation(error)) throw error;
if (isDynamicModelSelectionError(error)) return failModelSelection(step, error);
+ if (isDynamicConnectionResolutionError(error)) return failConnectionResolution(step, error);
throw error;
}
+/** A dynamic connection resolver threw: the session parks so the next message can retry. */
+export async function failConnectionResolution(step: Step, error: unknown): Promise<StepResult> {
+ if (step.emit === undefined) throw error;
+ step.instrumentation?.recordError(error);
+
+ const errorId = createErrorId();
+ const message = toErrorMessage(error);
+ log.error("dynamic connection resolver failed — parking session", {
+ error,
+ errorId,
+ sessionId: step.session.sessionId,
+ turnId: activeTurnId(step.position()),
+ });
+ await step.apply(
+ fail(step.view(), { code: "EVENT_HANDLER_FAILED", details: { errorId }, message }),
+ );
+ step.session = { ...step.session, outputSchema: undefined };
+ return {
+ next: null,
+ session: step.session,
+ settledTurn: { isError: true, output: message },
+ };
+}
+
/** No model can serve the turn: the session fails. */
export async function failModelSelection(step: Step, error: unknown): Promise<StepResult> {
throwIfTurnAborted(step.config.abortSignal);