TypeScript
perf(server): batch projection recovery reads into one query (#31842)
Yash Singhc22251163132
diff --git a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts b/apps/server/src/orchestration-v2/ProjectionRecovery.test.tsindex e6351eb5f00..6b1388f0ccf 100644--- a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts+++ b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts@@ -213,55 +213,87 @@ it.effect.each(["startup", "shutdown"] as const)(}).pipe(Effect.provide(layerRecoveryTest)),);-it.effect("stops a session its thread detached, which kept its live status", () =>- Effect.gen(function* () {- const projections = yield* ProjectionStore.ProjectionStoreV2;- const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;- const now = yield* DateTime.now;- const threadId = yield* createThread("detached-session");- const providerSessionId = ProviderSessionId.make("session:recovery:detached");- yield* projections.apply({- id: EventId.make("event:recovery:detached-session:attached"),- type: "provider-session.attached",- threadId,- driver,- providerInstanceId,- occurredAt: now,- payload: {- id: providerSessionId,+it.effect.each(["startup", "shutdown"] as const)(+ "stops a detached session and its orphaned active provider thread on %s",+ (trigger) =>+ Effect.gen(function* () {+ const projections = yield* ProjectionStore.ProjectionStoreV2;+ const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;+ const now = yield* DateTime.now;+ const threadId = yield* createThread("detached-session");+ const providerSessionId = ProviderSessionId.make("session:recovery:detached");+ yield* projections.apply({+ id: EventId.make("event:recovery:detached-session:attached"),+ type: "provider-session.attached",+ threadId,driver,providerInstanceId,- status: "ready",- cwd: "/workspace",- model: modelSelection.model,- capabilities: CodexProviderCapabilitiesV2,- createdAt: now,- updatedAt: now,- lastError: null,- },- });- yield* projections.apply({- id: EventId.make("event:recovery:detached-session:detached"),- type: "provider-session.detached",- threadId,- driver,- providerInstanceId,- occurredAt: now,- payload: { providerSessionId, detachedAt: now, reason: "Thread settled." },- });- assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), [threadId]);+ occurredAt: now,+ payload: {+ id: providerSessionId,+ driver,+ providerInstanceId,+ status: "ready",+ cwd: "/workspace",+ model: modelSelection.model,+ capabilities: CodexProviderCapabilitiesV2,+ createdAt: now,+ updatedAt: now,+ lastError: null,+ },+ });+ yield* projections.apply({+ id: EventId.make("event:recovery:detached-session:detached"),+ type: "provider-session.detached",+ threadId,+ driver,+ providerInstanceId,+ occurredAt: now,+ payload: { providerSessionId, detachedAt: now, reason: "Thread settled." },+ });+ const providerThreadId = ProviderThreadId.make("provider-thread:recovery:detached");+ yield* projections.apply({+ id: EventId.make("event:recovery:detached-provider-thread"),+ type: "provider-thread.updated",+ threadId,+ occurredAt: now,+ payload: {+ id: providerThreadId,+ appThreadId: threadId,+ ownerNodeId: null,+ driver,+ providerInstanceId,+ providerSessionId: null,+ nativeThreadRef: null,+ nativeConversationHeadRef: null,+ status: "active",+ firstRunOrdinal: null,+ lastRunOrdinal: null,+ handoffIds: [],+ forkedFrom: null,+ createdAt: now,+ updatedAt: now,+ },+ });+ assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), [threadId]);- const summary = yield* recovery.reconcile("startup");+ const summary = yield* recovery.reconcile(trigger);- assert.equal(summary.stoppedSessions, 1);- const sql = yield* SqlClient.SqlClient;- const rows = yield* sql<{ readonly status: string }>`+ assert.equal(summary.stoppedSessions, 1);+ const sql = yield* SqlClient.SqlClient;+ const rows = yield* sql<{ readonly status: string }>`SELECT status FROM orchestration_v2_projection_provider_sessionsWHERE provider_session_id = ${providerSessionId}`;- assert.deepEqual(rows, [{ status: "stopped" }]);- assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);- }).pipe(Effect.provide(layerRecoveryTest)),+ assert.deepEqual(rows, [{ status: "stopped" }]);+ const projection = yield* projections.getThreadProjection(threadId);+ assert.equal(projection.providerThreads[0]?.status, "idle");+ const eventStore = yield* EventStore.EventStoreV2;+ const sequence = yield* eventStore.latestSequence();+ assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);+ yield* recovery.reconcile(trigger);+ assert.equal(yield* eventStore.latestSequence(), sequence);+ }).pipe(Effect.provide(layerRecoveryTest)),);it.effect.each(["attached", "detached", "replaced"] as const)(@@ -964,11 +996,18 @@ it.effect("bounds streaming recovery reads by candidate runs and preserves messaassert.equal(statements.length, 2);for (const [query, params] of statements) {const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params);- assert.isTrue(plan[0]?.detail.includes("SEARCH run "), JSON.stringify(plan));+ assert.isTrue(+ plan.some((row) => row.detail.includes("SEARCH message_run ")),+ JSON.stringify(plan),+ );assert.isTrue(plan.some((row) => row.detail.includes("messages_run_idx")),JSON.stringify(plan),);+ assert.isFalse(+ plan.some((row) => /\bSCAN message\b/.test(row.detail)),+ JSON.stringify(plan),+ );assert.isFalse(plan.some((row) => row.detail.includes("messages_thread_created_idx")),JSON.stringify(plan),diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.tsindex acf50f00476..9ae1bc4882d 100644--- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts+++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts@@ -8439,6 +8439,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {.pipe(Effect.withTracer(tracer))).runs.map((run) => run.id),[runId, newerRunId],);+ assert.lengthOf(statements, 2);// Live-item filters must hit the active partial indexes, not walk the thread's history.const sql = yield* SqlClient.SqlClient;for (const statement of statements.filter((query) => query.includes("turn_items"))) {diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.tsindex b901ea6b0a4..2f8c972e7de 100644--- a/apps/server/src/orchestration-v2/ProjectionStore.ts+++ b/apps/server/src/orchestration-v2/ProjectionStore.ts@@ -4608,21 +4608,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =if (threadRows[0] === undefined) {return yield* new ProjectionStoreThreadNotFoundError({ threadId });}- const [- thread,- runRows,- attemptRows,- nodeRows,- subagentRows,- providerSessionRows,- providerThreadRows,- providerTurnRows,- runtimeRequestRows,- messageRows,- turnItemRows,- ] = yield* Effect.all([- decodeThreadPayload(threadRows[0].payload_json),- sql<PayloadRow>`+ const thread = yield* decodeThreadPayload(threadRows[0].payload_json);+ // One worker round trip and one SQLite snapshot; decisions still read after the prior commit.+ const queries = {+ runs: sql<PayloadRow>`SELECT payload_json FROM orchestration_v2_projection_runs AS runWHERE run.thread_id = ${threadId}AND (@@ -4665,7 +4654,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =)ORDER BY run.ordinal ASC`,- sql<PayloadRow>`+ attempts: sql<PayloadRow>`SELECT attempt.payload_jsonFROM orchestration_v2_projection_run_attempts AS attemptWHERE attempt.thread_id = ${threadId}@@ -4676,7 +4665,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =)ORDER BY attempt.run_id ASC, attempt.attempt_ordinal ASC`,- sql<PayloadRow>`+ nodes: sql<PayloadRow>`SELECT node.payload_json FROM orchestration_v2_projection_nodes AS nodeWHERE node.thread_id = ${threadId}AND node.status IN ('pending', 'starting', 'running', 'waiting')@@ -4711,7 +4700,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =)ORDER BY COALESCE(node.started_at, ''), node.node_id ASC`,- sql<PayloadRow>`+ subagents: sql<PayloadRow>`SELECT subagent.payload_json FROM orchestration_v2_projection_subagents AS subagentWHERE subagent.thread_id = ${threadId}AND subagent.status IN ('pending', 'starting', 'running', 'waiting')@@ -4731,7 +4720,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =)ORDER BY COALESCE(subagent.started_at, ''), subagent.subagent_id ASC`,- sql<PayloadRow>`+ providerSessions: sql<PayloadRow>`SELECT session.payload_jsonFROM orchestration_v2_projection_provider_sessions AS sessionWHERE (@@ -4769,7 +4758,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =)ORDER BY session.updated_at ASC, session.provider_session_id ASC`,- sql<PayloadRow>`+ providerThreads: sql<PayloadRow>`SELECT provider_thread.payload_jsonFROM orchestration_v2_projection_provider_threads AS provider_threadWHERE provider_thread.provider_thread_id IN (@@ -4815,7 +4804,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =)ORDER BY COALESCE(provider_thread.first_run_ordinal, 0), provider_thread.provider_thread_id ASC`,- sql<PayloadRow>`+ providerTurns: sql<PayloadRow>`SELECT provider_turn.payload_jsonFROM orchestration_v2_projection_provider_turns AS provider_turnWHERE provider_turn.thread_id = ${threadId}@@ -4843,22 +4832,22 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =)ORDER BY provider_turn.provider_thread_id ASC, provider_turn.ordinal ASC`,- sql<PayloadRow>`+ runtimeRequests: sql<PayloadRow>`SELECT payload_json FROM orchestration_v2_projection_runtime_requestsWHERE thread_id = ${threadId} AND status = 'pending'ORDER BY created_at ASC, runtime_request_id ASC`,// CROSS JOIN keeps empty run cohorts out of message history; the run index bounds each lookup.- sql<PayloadRow>`- SELECT message.payload_json FROM orchestration_v2_projection_runs AS run+ messages: sql<PayloadRow>`+ SELECT message.payload_json FROM orchestration_v2_projection_runs AS message_runCROSS JOIN orchestration_v2_projection_messages AS message- INDEXED BY orchestration_v2_projection_messages_run_idx ON message.run_id = run.run_id- WHERE run.thread_id = ${threadId}- AND run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting')+ INDEXED BY orchestration_v2_projection_messages_run_idx ON message.run_id = message_run.run_id+ WHERE message_run.thread_id = ${threadId}+ AND message_run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting')AND message.thread_id = ${threadId} AND message.streaming = 1ORDER BY message.created_at ASC, message.message_id ASC`,- sql<PayloadRow>`+ turnItems: sql<PayloadRow>`SELECT item.payload_json FROM orchestration_v2_projection_turn_items AS itemWHERE item.thread_id = ${threadId}AND item.status IN ('pending', 'running', 'waiting')@@ -4889,7 +4878,19 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =)ORDER BY item.ordinal ASC, item.turn_item_id ASC`,- ]);+ };+ const sections = Object.entries(queries).map(+ ([section, query]) => sql`+ SELECT ${section} AS section, payload_json, row_number() OVER () AS position+ FROM (${query})+ `,+ );+ const rows = yield* sql<PayloadRow & { readonly section: string }>`+ ${sql.join(" UNION ALL ", false)(sections)}+ ORDER BY section, position+ `;+ const readRows = (section: keyof typeof queries) =>+ rows.filter((row) => row.section === section);const [runs,attempts,@@ -4902,16 +4903,16 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =messages,turnItems,] = yield* Effect.all([- decodeRows(decodeRunPayload, threadId)(runRows),- decodeRows(decodeRunAttemptPayload, threadId)(attemptRows),- decodeRows(decodeNodePayload, threadId)(nodeRows),- decodeRows(decodeSubagentPayload, threadId)(subagentRows),- decodeRows(decodeProviderSessionPayload, threadId)(providerSessionRows),- decodeRows(decodeProviderThreadPayload, threadId)(providerThreadRows),- decodeRows(decodeProviderTurnPayload, threadId)(providerTurnRows),- decodeRows(decodeRuntimeRequestPayload, threadId)(runtimeRequestRows),- decodeRows(decodeMessagePayload, threadId)(messageRows),- decodeRows(decodeTurnItemPayload, threadId)(turnItemRows),+ decodeRows(decodeRunPayload, threadId)(readRows("runs")),+ decodeRows(decodeRunAttemptPayload, threadId)(readRows("attempts")),+ decodeRows(decodeNodePayload, threadId)(readRows("nodes")),+ decodeRows(decodeSubagentPayload, threadId)(readRows("subagents")),+ decodeRows(decodeProviderSessionPayload, threadId)(readRows("providerSessions")),+ decodeRows(decodeProviderThreadPayload, threadId)(readRows("providerThreads")),+ decodeRows(decodeProviderTurnPayload, threadId)(readRows("providerTurns")),+ decodeRows(decodeRuntimeRequestPayload, threadId)(readRows("runtimeRequests")),+ decodeRows(decodeMessagePayload, threadId)(readRows("messages")),+ decodeRows(decodeTurnItemPayload, threadId)(readRows("turnItems")),]);return {thread,