TypeScript
fix(server): prepare restart continuations faster on shutdown (#31807)
Yash Singh620a37b988a5
diff --git a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts b/apps/server/src/orchestration-v2/ProjectionRecovery.test.tsindex 0868b3b6948..e6351eb5f00 100644--- a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts+++ b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts@@ -836,6 +836,28 @@ it.effect("includes shared sessions and provider-owned background rosters in recnew Set(yield* projections.getRecoveryThreadIds("runtime")),new Set([first, second, roster, prepared]),);+ assert.deepEqual(+ new Set(yield* projections.getRecoveryThreadIds("restart-continuations")),+ new Set([first, second, prepared]),+ );+ assert.deepEqual(+ yield* projections.getRecoveryThreadIds("restart-continuations", { include: [] }),+ [],+ );+ assert.deepEqual(+ yield* projections.getRecoveryThreadIds("restart-continuations", {+ exclude: ["project:recovery"],+ }),+ [],+ );+ assert.deepEqual(+ new Set(+ yield* projections.getRecoveryThreadIds("restart-continuations", {+ include: ["project:recovery"],+ }),+ ),+ new Set([first, second, prepared]),+ );const preparedState = yield* projections.getRuntimeRecoveryProjection(prepared);assert.deepEqual(preparedState.providerSessions.map((session) => session.id),diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.tsindex 22f5743c103..b901ea6b0a4 100644--- a/apps/server/src/orchestration-v2/ProjectionStore.ts+++ b/apps/server/src/orchestration-v2/ProjectionStore.ts@@ -168,6 +168,7 @@ export type ProjectionStoreV2Error = typeof ProjectionStoreV2Error.Type;export type ProjectionRecoveryKind =| "queued-runs"| "runtime"+ | "restart-continuations"| "subagent-results"| "delegated-completions";@@ -353,6 +354,7 @@ export interface ProjectionRecordFilter {readonly subagentIds?: ReadonlyArray<NodeId>;readonly subagentRunIds?: ReadonlyArray<RunId>;readonly nodeIds?: ReadonlyArray<NodeId>;+ readonly turnItemRequestIds?: ReadonlyArray<RuntimeRequestId>;readonly turnItemTypes?: ReadonlyArray<OrchestrationV2TurnItem["type"]>;readonly turnItemStatuses?: ReadonlyArray<OrchestrationV2TurnItem["status"]>;readonly omitSettledLeafNodes?: boolean;@@ -678,6 +680,10 @@ export interface ProjectionStoreV2Shape {) => Effect.Effect<ReadonlyArray<OrchestrationV2Run>, ProjectionStoreV2Error>;readonly getRecoveryThreadIds: (kind: ProjectionRecoveryKind,+ projects?: {+ readonly include?: ReadonlyArray<string>;+ readonly exclude?: ReadonlyArray<string>;+ },) => Effect.Effect<ReadonlyArray<ThreadId>, ProjectionStoreV2Error>;readonly getUnreadableThreadIds: () => Effect.Effect<ReadonlyArray<ThreadId>,@@ -3129,6 +3135,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =(fields !== undefined && !fields.includes("turnItems")) ||(window === undefined &&matchesNothing(+ filter?.turnItemRequestIds,filter?.turnItemTypes,filter?.turnItemStatuses,filter?.turnItemRunIds,@@ -3140,6 +3147,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =SELECT payload_jsonFROM orchestration_v2_projection_turn_itemsWHERE thread_id = ${threadId}+ ${filter?.turnItemRequestIds === undefined ? sql`` : sql`AND json_extract(payload_json, '$.requestId') IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemRequestIds)}))`}${filter?.turnItemTypes === undefined ? sql`` : sql`AND type IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemTypes)}))`}${filter?.turnItemStatuses === undefined ? sql`` : sql`AND status IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemStatuses)}))`}${filter?.turnItemRunId === undefined ? sql`` : sql`AND run_id = ${filter.turnItemRunId}`}@@ -4120,7 +4128,13 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =);const getRecoveryThreadIds = Effect.fn("ProjectionStore.getRecoveryThreadIds")(- function* (kind: ProjectionRecoveryKind) {+ function* (+ kind: ProjectionRecoveryKind,+ projects?: {+ readonly include?: ReadonlyArray<string>;+ readonly exclude?: ReadonlyArray<string>;+ },+ ) {const candidates = (() => {switch (kind) {case "queued-runs":@@ -4171,6 +4185,27 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =AND type = 'subagent_result')))`;+ case "restart-continuations":+ return sql`+ SELECT bindings.thread_id+ FROM orchestration_v2_projection_provider_sessions AS sessions+ JOIN orchestration_v2_projection_provider_session_bindings AS bindings+ ON bindings.provider_session_id = sessions.provider_session_id+ WHERE sessions.status NOT IN ('stopped', 'error')+ UNION+ SELECT thread_id FROM orchestration_v2_projection_provider_sessions AS sessions+ WHERE status NOT IN ('stopped', 'error') AND thread_id IS NOT NULL+ AND NOT EXISTS (+ SELECT 1 FROM orchestration_v2_projection_provider_session_bindings AS bindings+ WHERE bindings.provider_session_id = sessions.provider_session_id+ )+ UNION+ SELECT thread_id FROM orchestration_v2_projection_runs+ WHERE status = 'starting'+ AND CASE WHEN json_valid(payload_json)+ THEN json_extract(payload_json, '$.restartContinuationOfRunId') IS NOT NULL+ ELSE 0 END+ `;case "runtime":return sql`WITH pending_provider_threads AS MATERIALIZED (@@ -4257,6 +4292,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =SELECT thread_id FROM orchestration_v2_projection_threadsWHERE deleted_at IS NULLAND thread_id IN (${candidates})+ ${projects?.include === undefined ? sql`` : sql`AND project_id IN (SELECT value FROM json_each(${encodeIdList(projects.include)}))`}+ ${projects?.exclude === undefined ? sql`` : sql`AND project_id NOT IN (SELECT value FROM json_each(${encodeIdList(projects.exclude)}))`}${kind === "queued-runs" ? sql`AND archived_at IS NULL` : sql``}ORDER BY updated_at ASC, thread_id ASC`;diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.tsindex a20b5cbfc29..c374db2abfb 100644--- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts+++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts@@ -875,39 +875,56 @@ export const make = Effect.gen(function* () {const prepareForShutdown = Effect.gen(function* () {const enabled = yield* settings.getSettings.pipe(Effect.orElseSucceed(() => null));if (!enabled) return;- const threadIds = yield* projections.getRecoveryThreadIds("runtime");- for (const threadId of threadIds) {- yield* Effect.gen(function* () {- const projection = yield* projections.getRuntimeRecoveryProjection(threadId);- if (- !resolveProjectSettings(enabled, projection.thread.projectId).settings- .continueThreadsAfterServerUpdate- )- return;- const run = restartContinuationRun(projection, yield* DateTime.now);- if (!run) return;- const commandId = CommandId.make(`command:restart-prepare:${run.id}`);- yield* eventSink.writeWithEffects({- commandId,- events: [],- effects: [- {- id: `effect:restart-continuation:${run.id}`,- commandId,- threadId,- request: { type: "provider-runtime.continue", sourceRunId: run.id },- },- ],- });- }).pipe(- Effect.catchCauseIf(- (cause) => !Cause.hasInterruptsOnly(cause),- (cause) =>- Effect.logWarning("Failed to prepare a restart continuation", { threadId, cause }),+ const enabledProjects = Object.entries(enabled.projectSettingsOverrides)+ .filter(([, overrides]) => overrides.continueThreadsAfterServerUpdate === true)+ .map(([id]) => id);+ if (!enabled.continueThreadsAfterServerUpdate && enabledProjects.length === 0) return;+ const disabledProjects = Object.entries(enabled.projectSettingsOverrides)+ .filter(([, overrides]) => overrides.continueThreadsAfterServerUpdate === false)+ .map(([id]) => id);+ const threadIds = yield* projections.getRecoveryThreadIds(+ "restart-continuations",+ enabled.continueThreadsAfterServerUpdate+ ? { exclude: disabledProjects }+ : { include: enabledProjects },+ );+ yield* Effect.forEach(+ threadIds,+ (threadId) =>+ Effect.gen(function* () {+ const projection = yield* projections.getRuntimeRecoveryProjection(threadId);+ if (+ !resolveProjectSettings(enabled, projection.thread.projectId).settings+ .continueThreadsAfterServerUpdate+ )+ return;+ const run = restartContinuationRun(projection, yield* DateTime.now);+ if (!run) return;+ const commandId = CommandId.make(`command:restart-prepare:${run.id}`);+ yield* eventSink.writeWithEffects({+ commandId,+ events: [],+ effects: [+ {+ id: `effect:restart-continuation:${run.id}`,+ commandId,+ threadId,+ request: { type: "provider-runtime.continue", sourceRunId: run.id },+ },+ ],+ });+ }).pipe(+ Effect.catchCauseIf(+ (cause) => !Cause.hasInterruptsOnly(cause),+ (cause) =>+ Effect.logWarning("Failed to prepare a restart continuation", { threadId, cause }),+ ),),- );- }+ // Leave room in the four-reader pool for other shutdown reads.+ { concurrency: 2, discard: true },+ );}).pipe(+ Effect.withSpan("ProviderRuntimeRecoveryService.prepareForShutdown"),Effect.mapError((cause) => new ProviderRuntimeRecoveryError({ operation: "reconcile", cause })),);diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.tsindex da6774ae235..896de803a38 100644--- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts+++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts@@ -3113,6 +3113,27 @@ it.effect("ProviderSessionManagerV2 marks pending runtime requests non-live on rnow,});yield* eventSink.write({ events: pendingRequest.events });+ const historical = yield* makePendingRuntimeRequestEvents({+ idAllocator,+ threadId,+ providerSessionId,+ providerThread,+ now,+ });+ yield* eventSink.write({+ events: historical.events.map((event) =>+ event.type === "runtime-request.updated"+ ? { ...event, payload: { ...event.payload, status: "resolved", resolvedAt: now } }+ : event,+ ),+ });+ assert.deepEqual(+ (yield* projectionStore.getThreadRecords(threadId, ["nodes", "turnItems"], {+ nodeIds: [pendingRequest.nodeId],+ turnItemRequestIds: [pendingRequest.requestId],+ })).turnItems.map((item) => item.id),+ [pendingRequest.events.find((event) => event.type === "turn-item.updated")!.payload.id],+ );yield* manager.open({threadId,providerSessionId,@@ -3126,7 +3147,13 @@ it.effect("ProviderSessionManagerV2 marks pending runtime requests non-live on r});const projection = yield* projectionStore.getThreadProjection(threadId);- const request = projection.runtimeRequests.at(-1);+ const request = projection.runtimeRequests.find(+ (request) => request.id === pendingRequest.requestId,+ );+ assert.equal(+ projection.runtimeRequests.find((request) => request.id === historical.requestId)?.status,+ "resolved",+ );const requestNode = projection.nodes.find((node) => node.id === request?.nodeId);const requestTurnItem = projection.turnItems.find((item) => item.type === "approval_request" && item.requestId === request?.id,diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.tsindex 3b01de25bc1..39f383666ba 100644--- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts+++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts@@ -655,8 +655,8 @@ export const layerWithOptions = (for (const threadId of input.entry.attachedThreadIds) {const projection = yield* projectionStore.getThreadRecords(threadId,- ["runtimeRequests", "nodes", "turnItems"],- { turnItemTypes: ["approval_request", "user_input_request"] },+ ["runtimeRequests"],+ { runtimeRequestStatuses: ["pending"] },);const releasedRequests = projection.runtimeRequests.filter((request) =>@@ -668,6 +668,16 @@ export const layerWithOptions = (!(input.reason === "server_shutdown" && request.kind === "user_input"),);+ if (releasedRequests.length === 0) continue;+ const records = yield* projectionStore.getThreadRecords(+ threadId,+ ["nodes", "turnItems"],+ {+ nodeIds: releasedRequests.map((request) => request.nodeId),+ turnItemRequestIds: releasedRequests.map((request) => request.id),+ turnItemTypes: ["approval_request", "user_input_request"],+ },+ );for (const request of releasedRequests) {events.push({id: yield* idAllocator.allocate.event({@@ -690,7 +700,7 @@ export const layerWithOptions = (},});- const requestNode = projection.nodes.find((node) => node.id === request.nodeId);+ const requestNode = records.nodes.find((node) => node.id === request.nodeId);if (requestNode !== undefined) {events.push({id: yield* idAllocator.allocate.event({@@ -711,7 +721,7 @@ export const layerWithOptions = (});}- const turnItem = projection.turnItems.find(+ const turnItem = records.turnItems.find((item) =>(item.type === "approval_request" || item.type === "user_input_request") &&item.requestId === request.id,@@ -2120,7 +2130,7 @@ export const layerWithOptions = (),{ concurrency: "unbounded", discard: true },);- });+ }).pipe(Effect.withSpan("ProviderSessionManager.shutdown"));yield* Effect.addFinalizer(() =>shutdown.pipe(Effect.ensuring(closeScopeWithin(sessionScopes, { reason: "server_shutdown" })),diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.tsindex a318130500b..d2f21a772ac 100644--- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts+++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts@@ -17,6 +17,8 @@ import {} from "@t3tools/contracts";import * as DateTime from "effect/DateTime";import * as Effect from "effect/Effect";+import * as Deferred from "effect/Deferred";+import * as Fiber from "effect/Fiber";import * as Layer from "effect/Layer";import * as ServerSettings from "../serverSettings.ts";import { restartContinuationRun, continueRestartedRun } from "./RestartContinuation.ts";@@ -1262,3 +1264,65 @@ it.effect.each(["completed", "failed", "interrupted"] as const)();}),);++it.effect("prepares independent continuations together and waits for durable writes", () =>+ Effect.gen(function* () {+ const bothReading = yield* Deferred.make<void>();+ const bothWriting = yield* Deferred.make<void>();+ const committed = yield* Deferred.make<void>();+ let reads = 0;+ let writes = 0;+ const recovery = yield* ProviderRuntimeRecovery.make.pipe(+ Effect.provide(+ Layer.mergeAll(+ ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),+ Layer.mock(ProjectionStore.ProjectionStoreV2)({+ getRecoveryThreadIds: () => Effect.succeed([threadId, ThreadId.make("thread:second")]),+ getRuntimeRecoveryProjection: () =>+ Effect.gen(function* () {+ if (++reads === 2) yield* Deferred.succeed(bothReading, undefined);+ yield* Deferred.await(bothReading);+ return makeProjection();+ }),+ }),+ Layer.mock(EventSink.EventSinkV2)({+ writeWithEffects: () =>+ Effect.gen(function* () {+ if (++writes === 2) yield* Deferred.succeed(bothWriting, undefined);+ yield* Deferred.await(committed);+ return [];+ }),+ }),+ IdAllocator.layer,+ Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),+ Layer.mock(EffectOutbox.EffectOutboxV2)({}),+ ),+ ),+ );+ const fiber = yield* recovery.prepareForShutdown.pipe(Effect.forkChild);+ yield* Deferred.await(bothWriting);+ assert.equal(writes, 2);+ assert.isUndefined(fiber.pollUnsafe());+ yield* Deferred.succeed(committed, undefined);+ yield* Fiber.join(fiber);+ }),+);++it.effect("skips restart discovery when no project enables continuation", () =>+ Effect.gen(function* () {+ const recovery = yield* ProviderRuntimeRecovery.make.pipe(+ Effect.provide(+ Layer.mergeAll(+ ServerSettings.layerTest({ continueThreadsAfterServerUpdate: false }),+ Layer.mock(ProjectionStore.ProjectionStoreV2)({}),+ Layer.mock(EventSink.EventSinkV2)({}),+ IdAllocator.layer,+ Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),+ Layer.mock(EffectOutbox.EffectOutboxV2)({}),+ ),+ ),+ );+ yield* recovery.prepareForShutdown;+ yield* recovery.prepareForShutdown;+ }),+);diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.tsindex e37f3078548..26f91f59aff 100644--- a/apps/server/src/serverRuntimeStartup.ts+++ b/apps/server/src/serverRuntimeStartup.ts@@ -478,6 +478,7 @@ const make = Effect.fn("serverRuntimeStartup.make")(function* (options?: Startupconst reconciliation = yield* providerRuntimeRecovery.reconcile("shutdown");yield* Effect.logInfo("V2 orchestration shutdown reconciliation completed", reconciliation);}).pipe(+ Effect.withSpan("server.shutdown"),Effect.catchCause((cause) =>Effect.logWarning("V2 orchestration shutdown reconciliation failed", {cause: Cause.pretty(cause),