TypeScript
perf(server): batch shutdown recovery commits (#31854)
Yash Singh1bdb6cf1cfb5
diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.tsindex e1b2e761066..acf3cc27f62 100644--- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts+++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts@@ -563,6 +563,159 @@ it.effect.each(},);+it.effect.each([false, true])("groups shutdown commits with rollback=%s", (fail) => {+ let commits = 0;+ const tracer = Tracer.make({+ span(options) {+ const span = new Tracer.NativeSpan(options);+ const end = span.end.bind(span);+ span.end = (time, exit) => {+ end(time, exit);+ if (span.name === "COMMIT") commits++;+ };+ return span;+ },+ });+ return Effect.gen(function* () {+ const sink = yield* EventSink.EventSinkV2;+ const events = yield* EventStore.EventStoreV2;+ const projections = yield* ProjectionStore.ProjectionStoreV2;+ const sql = yield* SqlClient.SqlClient;+ const outbox = yield* EffectOutbox.EffectOutboxV2;+ const now = yield* DateTime.now;+ const threads = Array.from({ length: 65 }, (_, i) =>+ ThreadId.make(`shutdown-group:${String(i).padStart(2, "0")}`),+ );+ const sharedSession = {+ id: ProviderSessionId.make("shutdown-shared"),+ driver: providerDriver,+ providerInstanceId,+ status: "ready" as const,+ cwd: process.cwd(),+ model: modelSelection.model,+ capabilities: CodexProviderCapabilitiesV2,+ createdAt: now,+ updatedAt: now,+ lastError: null,+ };+ yield* sink.write({+ events: threads.flatMap((threadId, i): OrchestrationV2DomainEvent[] => [+ threadCreatedEvent({ id: `shutdown-create:${i}`, thread: makeThread(threadId, now), now }),+ ...(i < 2+ ? [+ {+ id: EventId.make(`shutdown-session:${i}`),+ type: "provider-session.attached" as const,+ threadId,+ occurredAt: now,+ payload: sharedSession,+ },+ ]+ : []),+ {+ id: EventId.make(`shutdown-provider:${i}`),+ type: "provider-thread.updated",+ threadId,+ occurredAt: now,+ payload: {+ id: ProviderThreadId.make(`shutdown-provider:${i}`),+ driver: providerDriver,+ providerInstanceId,+ providerSessionId: i < 2 ? sharedSession.id : null,+ appThreadId: threadId,+ ownerNodeId: null,+ nativeThreadRef: null,+ nativeConversationHeadRef: null,+ status: i === 1 || i === 32 ? "idle" : "active",+ firstRunOrdinal: null,+ lastRunOrdinal: null,+ handoffIds: [],+ forkedFrom: null,+ createdAt: now,+ updatedAt: now,+ },+ },+ ]),+ });+ yield* outbox.enqueue(+ threads.map((threadId, i) => ({+ id: `shutdown-effect:${i}`,+ commandId: CommandId.make(`shutdown-command:${i}`),+ threadId,+ request: { type: "provider-turn.start" as const, runId: RunId.make(`shutdown-run:${i}`) },+ })),+ );+ let eventlessNotifications = 0;+ let armed = fail;+ const recovery = yield* ProviderRuntimeRecovery.make.pipe(+ Effect.provide(ServerSettings.layerTest()),+ Effect.provideService(EffectOutbox.EffectOutboxV2, {+ ...outbox,+ signalCancellations: (ids) =>+ Effect.gen(function* () {+ assert.isTrue(Option.isNone(yield* Effect.serviceOption(sql.transactionService)));+ eventlessNotifications += ids.length;+ yield* outbox.signalCancellations(ids);+ }),+ }),+ Effect.provideService(+ EffectWorker.OrchestrationEffectWorkerV2,+ EffectWorker.OrchestrationEffectWorkerV2.of({+ awaitWork: Effect.void,+ runRecoveryOnce: Effect.succeed(false),+ runOnce: Effect.succeed(false),+ nextClaimableAt: Effect.succeedNone,+ drain: () => Effect.succeed(0),+ }),+ ),+ Effect.provideService(EventSink.EventSinkV2, {+ ...sink,+ commitCommand: (input) =>+ armed && input.threadId === threads[33]+ ? Effect.die("injected shutdown failure")+ : sink.commitCommand(input),+ }),+ );+ commits = 0;+ const result = yield* recovery.reconcile("shutdown").pipe(Effect.exit);+ assert.equal(Exit.isFailure(result), fail);+ assert.equal(commits, fail ? 1 : 3);+ const receipts =+ yield* sql`SELECT aggregate_id FROM orchestration_command_receipts WHERE command_type = 'provider-runtime.reconcile' ORDER BY aggregate_id`;+ assert.deepEqual(+ receipts.map((r) => r.aggregate_id),+ threads.slice(0, fail ? 32 : 65).filter((_, i) => i !== 1 && i !== 32),+ );+ assert.deepEqual(+ yield* projections.getRecoveryThreadIds("runtime"),+ fail ? threads.slice(32) : [],+ );+ assert.equal(eventlessNotifications, fail ? 1 : 2);+ for (const [i] of threads.entries()) {+ const effect = Option.getOrThrow(yield* outbox.get(`shutdown-effect:${i}`));+ assert.equal(effect.status, !fail || i < 32 ? "cancelled" : "pending");+ }+ assert.equal(+ (yield* events.read({ afterSequence: 0 }).pipe(Stream.runCollect)).filter(+ (e) =>+ e.event.type === "provider-session.updated" && e.event.payload.id === sharedSession.id,+ ).length,+ 1,+ );+ armed = false;+ yield* recovery.reconcile("shutdown");+ assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);+ const highWater = yield* sink.latestSequence();+ yield* recovery.reconcile("shutdown");+ assert.equal(yield* sink.latestSequence(), highWater);+ }).pipe(+ Effect.provide(Layer.fresh(layerTest)),+ Effect.withTracer(tracer),+ Effect.withTracerEnabled(true),+ Effect.provideService(Statement.SpanPropagationEnabled, true),+ );+});+it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {it.effect("projects oversized tool bodies once before replay and live RPC retention", () =>Effect.scoped(diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.tsindex c374db2abfb..222747c4139 100644--- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts+++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts@@ -51,6 +51,8 @@ export class ProviderRuntimeRecoveryError extends Schema.TaggedError<ProviderRun}}+const isProviderRuntimeRecoveryError = Schema.is(ProviderRuntimeRecoveryError);+export interface ProviderRuntimeRecoverySummary {readonly terminalizedRuns: number;readonly stoppedSessions: number;@@ -762,6 +764,7 @@ export const make = Effect.gen(function* () {(candidate) => candidate.status !== "stopped" && candidate.status !== "error",).length;let retiredEffects: number;+ let cancelledEffectIds: ReadonlyArray<string> = [];if (events.length === 0) {const retiredEffectIds = yield* outbox.cancelUnsettled({@@ -779,7 +782,7 @@ export const make = Effect.gen(function* () {}),),);- yield* outbox.signalCancellations(retiredEffectIds);+ cancelledEffectIds = retiredEffectIds;retiredEffects = retiredEffectIds.length;} else {const result = yield* eventSink@@ -812,6 +815,7 @@ export const make = Effect.gen(function* () {stoppedSessions,closedRequests: requests.length,retiredEffects,+ cancelledEffectIds,};},);@@ -832,31 +836,49 @@ export const make = Effect.gen(function* () {let closedRequests = 0;let retiredEffects = 0;const stoppedThisPass = new Map<string, RecoverySession>();- for (const threadId of threadIds) {- const projection = yield* projections.getRuntimeRecoveryProjection(threadId).pipe(- Effect.mapError(- (cause) =>- new ProviderRuntimeRecoveryError({- operation: "read-projections",- threadId,- cause,- }),+ const grouped = trigger === "shutdown" && threadIds.length > 1;+ const groupSize = grouped ? 32 : 1;+ for (let offset = 0; offset < threadIds.length; offset += groupSize) {+ const cancelledEffectIds: string[] = [];+ // Read and decide on the writer so shared sessions see earlier updates in this group.+ const reconcileGroup = Effect.gen(function* () {+ for (const threadId of threadIds.slice(offset, offset + groupSize)) {+ const projection = yield* projections.getRuntimeRecoveryProjection(threadId).pipe(+ Effect.mapError(+ (cause) =>+ new ProviderRuntimeRecoveryError({+ operation: "read-projections",+ threadId,+ cause,+ }),+ ),+ );+ const enabled =+ continueAfterRestart !== null &&+ resolveProjectSettings(continueAfterRestart, projection.thread.projectId).settings+ .continueThreadsAfterServerUpdate;+ const result = yield* reconcileProjection(projection, trigger, enabled, stoppedThisPass);+ for (const session of projection.providerSessions) {+ if (session.status !== "stopped" && session.status !== "error") {+ stoppedThisPass.set(session.id, session);+ }+ }+ cancelledEffectIds.push(...result.cancelledEffectIds);+ terminalizedRuns += result.terminalizedRuns;+ stoppedSessions += result.stoppedSessions;+ closedRequests += result.closedRequests;+ retiredEffects += result.retiredEffects;+ }+ });+ yield* (grouped ? eventSink.withProviderBatch(reconcileGroup) : reconcileGroup).pipe(+ Effect.mapError((cause) =>+ isProviderRuntimeRecoveryError(cause)+ ? cause+ : new ProviderRuntimeRecoveryError({ operation: "reconcile", cause }),),);- const enabled =- continueAfterRestart !== null &&- resolveProjectSettings(continueAfterRestart, projection.thread.projectId).settings- .continueThreadsAfterServerUpdate;- const result = yield* reconcileProjection(projection, trigger, enabled, stoppedThisPass);- for (const session of projection.providerSessions) {- if (session.status !== "stopped" && session.status !== "error") {- stoppedThisPass.set(session.id, session);- }- }- terminalizedRuns += result.terminalizedRuns;- stoppedSessions += result.stoppedSessions;- closedRequests += result.closedRequests;- retiredEffects += result.retiredEffects;+ // Eventless outbox cancellations also wait for the group's durable commit.+ if (cancelledEffectIds.length > 0) yield* outbox.signalCancellations(cancelledEffectIds);}const outboxReconciliation = yield* outbox.reconcileAfterProcessLoss.pipe(Effect.mapError(