Skip to content
TypeScript

perf(server): batch shutdown recovery commits (#31854)

Yash Singh1bdb6cf1cfb5
diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts
index e1b2e761066..acf3cc27f62 100644
--- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts
@@ -563,6 +563,159 @@ it.effect.each(
},
);
+it.effect.each([false, true])("groups shutdown commits with rollback=%s", (fail) => {
+ let commits = 0;
+ const tracer = Tracer.make({
+ span(options) {
+ const span = new Tracer.NativeSpan(options);
+ const end = span.end.bind(span);
+ span.end = (time, exit) => {
+ end(time, exit);
+ if (span.name === "COMMIT") commits++;
+ };
+ return span;
+ },
+ });
+ return Effect.gen(function* () {
+ const sink = yield* EventSink.EventSinkV2;
+ const events = yield* EventStore.EventStoreV2;
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
+ const sql = yield* SqlClient.SqlClient;
+ const outbox = yield* EffectOutbox.EffectOutboxV2;
+ const now = yield* DateTime.now;
+ const threads = Array.from({ length: 65 }, (_, i) =>
+ ThreadId.make(`shutdown-group:${String(i).padStart(2, "0")}`),
+ );
+ const sharedSession = {
+ id: ProviderSessionId.make("shutdown-shared"),
+ driver: providerDriver,
+ providerInstanceId,
+ status: "ready" as const,
+ cwd: process.cwd(),
+ model: modelSelection.model,
+ capabilities: CodexProviderCapabilitiesV2,
+ createdAt: now,
+ updatedAt: now,
+ lastError: null,
+ };
+ yield* sink.write({
+ events: threads.flatMap((threadId, i): OrchestrationV2DomainEvent[] => [
+ threadCreatedEvent({ id: `shutdown-create:${i}`, thread: makeThread(threadId, now), now }),
+ ...(i < 2
+ ? [
+ {
+ id: EventId.make(`shutdown-session:${i}`),
+ type: "provider-session.attached" as const,
+ threadId,
+ occurredAt: now,
+ payload: sharedSession,
+ },
+ ]
+ : []),
+ {
+ id: EventId.make(`shutdown-provider:${i}`),
+ type: "provider-thread.updated",
+ threadId,
+ occurredAt: now,
+ payload: {
+ id: ProviderThreadId.make(`shutdown-provider:${i}`),
+ driver: providerDriver,
+ providerInstanceId,
+ providerSessionId: i < 2 ? sharedSession.id : null,
+ appThreadId: threadId,
+ ownerNodeId: null,
+ nativeThreadRef: null,
+ nativeConversationHeadRef: null,
+ status: i === 1 || i === 32 ? "idle" : "active",
+ firstRunOrdinal: null,
+ lastRunOrdinal: null,
+ handoffIds: [],
+ forkedFrom: null,
+ createdAt: now,
+ updatedAt: now,
+ },
+ },
+ ]),
+ });
+ yield* outbox.enqueue(
+ threads.map((threadId, i) => ({
+ id: `shutdown-effect:${i}`,
+ commandId: CommandId.make(`shutdown-command:${i}`),
+ threadId,
+ request: { type: "provider-turn.start" as const, runId: RunId.make(`shutdown-run:${i}`) },
+ })),
+ );
+ let eventlessNotifications = 0;
+ let armed = fail;
+ const recovery = yield* ProviderRuntimeRecovery.make.pipe(
+ Effect.provide(ServerSettings.layerTest()),
+ Effect.provideService(EffectOutbox.EffectOutboxV2, {
+ ...outbox,
+ signalCancellations: (ids) =>
+ Effect.gen(function* () {
+ assert.isTrue(Option.isNone(yield* Effect.serviceOption(sql.transactionService)));
+ eventlessNotifications += ids.length;
+ yield* outbox.signalCancellations(ids);
+ }),
+ }),
+ Effect.provideService(
+ EffectWorker.OrchestrationEffectWorkerV2,
+ EffectWorker.OrchestrationEffectWorkerV2.of({
+ awaitWork: Effect.void,
+ runRecoveryOnce: Effect.succeed(false),
+ runOnce: Effect.succeed(false),
+ nextClaimableAt: Effect.succeedNone,
+ drain: () => Effect.succeed(0),
+ }),
+ ),
+ Effect.provideService(EventSink.EventSinkV2, {
+ ...sink,
+ commitCommand: (input) =>
+ armed && input.threadId === threads[33]
+ ? Effect.die("injected shutdown failure")
+ : sink.commitCommand(input),
+ }),
+ );
+ commits = 0;
+ const result = yield* recovery.reconcile("shutdown").pipe(Effect.exit);
+ assert.equal(Exit.isFailure(result), fail);
+ assert.equal(commits, fail ? 1 : 3);
+ const receipts =
+ yield* sql`SELECT aggregate_id FROM orchestration_command_receipts WHERE command_type = 'provider-runtime.reconcile' ORDER BY aggregate_id`;
+ assert.deepEqual(
+ receipts.map((r) => r.aggregate_id),
+ threads.slice(0, fail ? 32 : 65).filter((_, i) => i !== 1 && i !== 32),
+ );
+ assert.deepEqual(
+ yield* projections.getRecoveryThreadIds("runtime"),
+ fail ? threads.slice(32) : [],
+ );
+ assert.equal(eventlessNotifications, fail ? 1 : 2);
+ for (const [i] of threads.entries()) {
+ const effect = Option.getOrThrow(yield* outbox.get(`shutdown-effect:${i}`));
+ assert.equal(effect.status, !fail || i < 32 ? "cancelled" : "pending");
+ }
+ assert.equal(
+ (yield* events.read({ afterSequence: 0 }).pipe(Stream.runCollect)).filter(
+ (e) =>
+ e.event.type === "provider-session.updated" && e.event.payload.id === sharedSession.id,
+ ).length,
+ 1,
+ );
+ armed = false;
+ yield* recovery.reconcile("shutdown");
+ assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
+ const highWater = yield* sink.latestSequence();
+ yield* recovery.reconcile("shutdown");
+ assert.equal(yield* sink.latestSequence(), highWater);
+ }).pipe(
+ Effect.provide(Layer.fresh(layerTest)),
+ Effect.withTracer(tracer),
+ Effect.withTracerEnabled(true),
+ Effect.provideService(Statement.SpanPropagationEnabled, true),
+ );
+});
+
it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
it.effect("projects oversized tool bodies once before replay and live RPC retention", () =>
Effect.scoped(
diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
index c374db2abfb..222747c4139 100644
--- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
@@ -51,6 +51,8 @@ export class ProviderRuntimeRecoveryError extends Schema.TaggedError<ProviderRun
}
}
+const isProviderRuntimeRecoveryError = Schema.is(ProviderRuntimeRecoveryError);
+
export interface ProviderRuntimeRecoverySummary {
readonly terminalizedRuns: number;
readonly stoppedSessions: number;
@@ -762,6 +764,7 @@ export const make = Effect.gen(function* () {
(candidate) => candidate.status !== "stopped" && candidate.status !== "error",
).length;
let retiredEffects: number;
+ let cancelledEffectIds: ReadonlyArray<string> = [];
if (events.length === 0) {
const retiredEffectIds = yield* outbox
.cancelUnsettled({
@@ -779,7 +782,7 @@ export const make = Effect.gen(function* () {
}),
),
);
- yield* outbox.signalCancellations(retiredEffectIds);
+ cancelledEffectIds = retiredEffectIds;
retiredEffects = retiredEffectIds.length;
} else {
const result = yield* eventSink
@@ -812,6 +815,7 @@ export const make = Effect.gen(function* () {
stoppedSessions,
closedRequests: requests.length,
retiredEffects,
+ cancelledEffectIds,
};
},
);
@@ -832,31 +836,49 @@ export const make = Effect.gen(function* () {
let closedRequests = 0;
let retiredEffects = 0;
const stoppedThisPass = new Map<string, RecoverySession>();
- for (const threadId of threadIds) {
- const projection = yield* projections.getRuntimeRecoveryProjection(threadId).pipe(
- Effect.mapError(
- (cause) =>
- new ProviderRuntimeRecoveryError({
- operation: "read-projections",
- threadId,
- cause,
- }),
+ const grouped = trigger === "shutdown" && threadIds.length > 1;
+ const groupSize = grouped ? 32 : 1;
+ for (let offset = 0; offset < threadIds.length; offset += groupSize) {
+ const cancelledEffectIds: string[] = [];
+ // Read and decide on the writer so shared sessions see earlier updates in this group.
+ const reconcileGroup = Effect.gen(function* () {
+ for (const threadId of threadIds.slice(offset, offset + groupSize)) {
+ const projection = yield* projections.getRuntimeRecoveryProjection(threadId).pipe(
+ Effect.mapError(
+ (cause) =>
+ new ProviderRuntimeRecoveryError({
+ operation: "read-projections",
+ threadId,
+ cause,
+ }),
+ ),
+ );
+ const enabled =
+ continueAfterRestart !== null &&
+ resolveProjectSettings(continueAfterRestart, projection.thread.projectId).settings
+ .continueThreadsAfterServerUpdate;
+ const result = yield* reconcileProjection(projection, trigger, enabled, stoppedThisPass);
+ for (const session of projection.providerSessions) {
+ if (session.status !== "stopped" && session.status !== "error") {
+ stoppedThisPass.set(session.id, session);
+ }
+ }
+ cancelledEffectIds.push(...result.cancelledEffectIds);
+ terminalizedRuns += result.terminalizedRuns;
+ stoppedSessions += result.stoppedSessions;
+ closedRequests += result.closedRequests;
+ retiredEffects += result.retiredEffects;
+ }
+ });
+ yield* (grouped ? eventSink.withProviderBatch(reconcileGroup) : reconcileGroup).pipe(
+ Effect.mapError((cause) =>
+ isProviderRuntimeRecoveryError(cause)
+ ? cause
+ : new ProviderRuntimeRecoveryError({ operation: "reconcile", cause }),
),
);
- const enabled =
- continueAfterRestart !== null &&
- resolveProjectSettings(continueAfterRestart, projection.thread.projectId).settings
- .continueThreadsAfterServerUpdate;
- const result = yield* reconcileProjection(projection, trigger, enabled, stoppedThisPass);
- for (const session of projection.providerSessions) {
- if (session.status !== "stopped" && session.status !== "error") {
- stoppedThisPass.set(session.id, session);
- }
- }
- terminalizedRuns += result.terminalizedRuns;
- stoppedSessions += result.stoppedSessions;
- closedRequests += result.closedRequests;
- retiredEffects += result.retiredEffects;
+ // Eventless outbox cancellations also wait for the group's durable commit.
+ if (cancelledEffectIds.length > 0) yield* outbox.signalCancellations(cancelledEffectIds);
}
const outboxReconciliation = yield* outbox.reconcileAfterProcessLoss.pipe(
Effect.mapError(