Skip to content
TypeScript

perf(server): index delegated recovery candidates (#30359)

Yash Singh96e887b6a866
diff --git a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts
index 733238e62d2..a4914fdee45 100644
--- a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts
+++ b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts
@@ -24,6 +24,7 @@ import * as Layer from "effect/Layer";
import * as SqlClient from "effect/sql/SqlClient";
import * as Statement from "effect/sql/Statement";
+import delegatedRecoveryIndexes from "../persistence/Migrations/079_DelegatedRecoveryIndexes.ts";
import * as SqlitePersistence from "../persistence/Sqlite.ts";
import * as ServerSettings from "../serverSettings.ts";
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
@@ -357,8 +358,28 @@ it.effect("selects unfinished recovery work without reading settled thread histo
new Set(yield* projections.getRecoveryThreadIds("runtime")),
new Set([queued, archived, blocked, background, outboxOnly, requestOnly]),
);
- assert.deepEqual(yield* projections.getRecoveryThreadIds("delegated-completions"), [delivery]);
- assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), []);
+ for (const [kind, expected, index] of [
+ ["delegated-completions", [delivery], "runs_delegated_delivery_idx"],
+ ["subagent-results", [], "threads_subagent_result_idx"],
+ ] as const) {
+ const statements: Array<ReturnType<Statement.Statement<unknown>["compile"]>> = [];
+ const observe: Statement.Transformer = (statement) => {
+ statements.push(statement.compile());
+ return Effect.succeed(statement);
+ };
+ assert.deepEqual(
+ yield* projections
+ .getRecoveryThreadIds(kind)
+ .pipe(Effect.provideService(Statement.CurrentTransformer, observe)),
+ expected,
+ );
+ const [query, params] = statements[0]!;
+ const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params);
+ assert.isTrue(
+ plan.some((row) => row.detail.includes(index)),
+ JSON.stringify(plan),
+ );
+ }
const recoveryState = yield* projections.getRuntimeRecoveryProjection(queued);
assert.deepEqual(
recoveryState.runs.map((run) => run.id),
@@ -371,6 +392,84 @@ it.effect("selects unfinished recovery work without reading settled thread histo
}).pipe(Effect.provide(layerTest)),
);
+it.effect("maintains delegated candidate indexes across legacy rows and payload transitions", () =>
+ Effect.gen(function* () {
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
+ const sql = yield* SqlClient.SqlClient;
+ const parent = yield* createThread("index-parent");
+ const child = yield* createThread("index-child");
+ const run = yield* createRun(child, "completed");
+ const rows = yield* sql<{ payload_json: string }>`
+ SELECT payload_json FROM orchestration_v2_projection_threads WHERE thread_id = ${child}
+ `;
+ const original = JSON.parse(rows[0]!.payload_json);
+ for (const payload of [
+ "{broken",
+ JSON.stringify({
+ ...original,
+ lineage: { parentThreadId: parent, relationshipToParent: "subagent" },
+ forkedFrom: { type: "provider_thread" },
+ }),
+ JSON.stringify({
+ ...original,
+ lineage: { parentThreadId: parent, relationshipToParent: "subagent" },
+ forkedFrom: { type: "node", nodeId: "node:index" },
+ }),
+ JSON.stringify({
+ ...original,
+ lineage: { parentThreadId: child, relationshipToParent: "subagent" },
+ forkedFrom: { type: "node", nodeId: "node:index" },
+ }),
+ JSON.stringify(original),
+ ]) {
+ yield* sql`UPDATE orchestration_v2_projection_threads SET payload_json = ${payload}
+ WHERE thread_id = ${child}`;
+ const expected = payload.includes('"type":"node"') ? [child] : [];
+ assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), expected);
+ }
+ for (const entry of [
+ ["{broken", false],
+ ["{}", false],
+ ['{"delegatedCompletion":{"delivery":null}}', false],
+ ['{"delegatedCompletion":{"delivery":[]}}', false],
+ ['{"delegatedCompletion":{"delivery":{}}}', true],
+ ["{}", false],
+ ] as const) {
+ const [payload, included] = entry;
+ yield* sql`UPDATE orchestration_v2_projection_runs SET payload_json = ${payload}
+ WHERE run_id = ${run}`;
+ assert.deepEqual(
+ yield* projections.getRecoveryThreadIds("delegated-completions"),
+ included ? [child] : [],
+ );
+ }
+ yield* sql`UPDATE orchestration_v2_projection_threads SET payload_json = '{broken'
+ WHERE thread_id = ${parent}`;
+ yield* sql`UPDATE orchestration_v2_projection_runs SET payload_json = '{broken'
+ WHERE run_id = ${run}`;
+ const existing = yield* createThread("index-existing", {
+ lineage: { parentThreadId: child, relationshipToParent: "subagent", rootThreadId: parent },
+ forkedFrom: { type: "node", nodeId: NodeId.make("node:index-existing") },
+ });
+ yield* createRun(existing, "completed", {
+ delegatedCompletion: {
+ disposition: "open",
+ nextGeneration: 2,
+ delivery: {
+ generation: 1,
+ messageId: MessageId.make("message:index-existing"),
+ taskIds: [],
+ },
+ },
+ });
+ yield* sql`DROP INDEX orchestration_v2_projection_threads_subagent_result_idx`;
+ yield* sql`DROP INDEX orchestration_v2_projection_runs_delegated_delivery_idx`;
+ yield* delegatedRecoveryIndexes;
+ assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), [existing]);
+ assert.deepEqual(yield* projections.getRecoveryThreadIds("delegated-completions"), [existing]);
+ }).pipe(Effect.provide(layerTest)),
+);
+
it.effect("recovers terminal subagent results until their cross-thread transfer exists", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts
index 3e6ca0d75a1..f6be46d28ae 100644
--- a/apps/server/src/orchestration-v2/ProjectionStore.ts
+++ b/apps/server/src/orchestration-v2/ProjectionStore.ts
@@ -4121,6 +4121,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
json_extract(child.payload_json, '$.lineage.relationshipToParent') = 'subagent'
AND json_extract(child.payload_json, '$.lineage.parentThreadId') IS NOT NULL
AND json_extract(child.payload_json, '$.forkedFrom.type') = 'node'
+ ELSE 0 END
AND (EXISTS (
SELECT 1 FROM orchestration_v2_projection_runs AS requested
JOIN orchestration_v2_projection_subagents AS task
@@ -4143,7 +4144,6 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
AND target_thread_id = json_extract(child.payload_json, '$.lineage.parentThreadId')
AND type = 'subagent_result'
)))
- ELSE 0 END
`;
case "runtime":
return sql`
diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts
index 3ee83e5fe01..606770d1b65 100644
--- a/apps/server/src/persistence/Migrations.ts
+++ b/apps/server/src/persistence/Migrations.ts
@@ -82,6 +82,8 @@ import Migration0076 from "./Migrations/076_ProjectionProjectsPinArchive.ts";
import Migration0077 from "./Migrations/077_RepairBooleanSettledOverride.ts";
import Migration0078 from "./Migrations/078_DropOrchestrationEventCorrelationIndex.ts";
+import Migration0079 from "./Migrations/079_DelegatedRecoveryIndexes.ts";
+
const migrationEntries = [
[1, "OrchestrationEvents", Migration0001],
[2, "OrchestrationCommandReceipts", Migration0002],
@@ -162,6 +164,7 @@ const migrationEntries = [
[76, "ProjectionProjectsPinArchive", Migration0076],
[77, "RepairBooleanSettledOverride", Migration0077],
[78, "DropOrchestrationEventCorrelationIndex", Migration0078],
+ [79, "DelegatedRecoveryIndexes", Migration0079],
] as const;
export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);
diff --git a/apps/server/src/persistence/Migrations/079_DelegatedRecoveryIndexes.ts b/apps/server/src/persistence/Migrations/079_DelegatedRecoveryIndexes.ts
new file mode 100644
index 00000000000..3ba72feaf17
--- /dev/null
+++ b/apps/server/src/persistence/Migrations/079_DelegatedRecoveryIndexes.ts
@@ -0,0 +1,26 @@
+import * as Effect from "effect/Effect";
+import * as SqlClient from "effect/sql/SqlClient";
+
+export default Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+
+ // Include the parent ID so transfer checks do not reread each child payload.
+ yield* sql`
+ CREATE INDEX orchestration_v2_projection_threads_subagent_result_idx
+ ON orchestration_v2_projection_threads(
+ thread_id, json_extract(payload_json, '$.lineage.parentThreadId')
+ )
+ WHERE CASE WHEN json_valid(payload_json) THEN
+ json_extract(payload_json, '$.lineage.relationshipToParent') = 'subagent'
+ AND json_extract(payload_json, '$.lineage.parentThreadId') IS NOT NULL
+ AND json_extract(payload_json, '$.forkedFrom.type') = 'node'
+ ELSE 0 END
+ `;
+ yield* sql`
+ CREATE INDEX orchestration_v2_projection_runs_delegated_delivery_idx
+ ON orchestration_v2_projection_runs(thread_id)
+ WHERE CASE WHEN json_valid(payload_json)
+ THEN json_type(payload_json, '$.delegatedCompletion.delivery') = 'object'
+ ELSE 0 END
+ `;
+});
diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
index d062961b3af..5f51c237fd7 100644
--- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
+++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
@@ -58,6 +58,7 @@ describe("V2 preview upgrade", () => {
[76, "ProjectionProjectsPinArchive"],
[77, "RepairBooleanSettledOverride"],
[78, "DropOrchestrationEventCorrelationIndex"],
+ [79, "DelegatedRecoveryIndexes"],
]);
assert.deepStrictEqual(yield* runMigrations(), []);
assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
@@ -159,6 +160,7 @@ describe("V2 preview upgrade", () => {
[76, "ProjectionProjectsPinArchive"],
[77, "RepairBooleanSettledOverride"],
[78, "DropOrchestrationEventCorrelationIndex"],
+ [79, "DelegatedRecoveryIndexes"],
]);
}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
);