TypeScript
fix(server): retry unstarted runs after recovery (#30560)
Yash Singh737afe24bd20
diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.tsindex f20ca8bdc92..f45c63ff27e 100644--- a/apps/server/src/orchestration-v2/Orchestrator.ts+++ b/apps/server/src/orchestration-v2/Orchestrator.ts@@ -2376,7 +2376,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio},})),{- id: `effect:${commandId}:provider-turn.start:${queuedRun.id}`,+ // A restart can return this run to the queue after its previous start effect was cancelled.+ id: `effect:${commandId}:provider-turn.start:${queuedRun.id}:${yield* idAllocator.allocate.event({ threadId, commandId })}`,commandId,threadId,request: { type: "provider-turn.start", runId: queuedRun.id },diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.tsindex 120858ace27..d300cf59ca3 100644--- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts+++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts@@ -501,7 +501,19 @@ it.effect("recovery closes a secret card's form but leaves a link card open", ()}).pipe(Effect.provide(layer));});-it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart", (status) => {+it.effect.each([+ "queued",+ "preparing",+ "starting",+ "starting-other-request",+ "starting-own-request",+ "starting-scheduled",+] as const)("recovers $0 work after restart", (scenario) => {+ const status = scenario.startsWith("starting") ? "starting" : scenario;+ const terminalized =+ status === "preparing" ||+ scenario === "starting-own-request" ||+ scenario === "starting-scheduled";const threadId = ThreadId.make("thread_queued_restart");const runId = RunId.make("run_queued_restart");const attemptId = RunAttemptId.make("attempt_queued_restart");@@ -510,7 +522,18 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restartnull;const projection = {thread: { id: threadId },- runtimeRequests: [],+ runtimeRequests: scenario.endsWith("request")+ ? [+ {+ id: RuntimeRequestId.make("pending-recovery-request"),+ nodeId:+ scenario === "starting-own-request" ? rootNodeId : NodeId.make("other-run-request"),+ kind: "user_input",+ status: "pending",+ responseCapability: { type: "message" },+ },+ ]+ : [],providerSessions: [],providerThreads: [],providerTurns: [],@@ -518,6 +541,9 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart{id: runId,status,+ startedAt: null,+ activeAttemptId: attemptId,+ userMessageId: MessageId.make("message_queued_restart"),queuePosition: 1,providerInstanceId: ProviderInstanceId.make("codex"),...(status === "preparing"@@ -540,6 +566,15 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart},],nodes: [+ ...(scenario === "starting-other-request"+ ? [+ {+ id: NodeId.make("other-run-request"),+ runId: RunId.make("earlier-run"),+ status: "waiting",+ },+ ]+ : []),{id: rootNodeId,runId,@@ -547,7 +582,16 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart},],subagents: [],- messages: [],+ messages:+ status === "starting"+ ? [+ {+ id: MessageId.make("message_queued_restart"),+ createdBy: "user",+ ...(scenario === "starting-scheduled" ? { scheduledTaskId: "scheduled-task" } : {}),+ },+ ]+ : [],turnItems: [],} as unknown as OrchestrationV2ThreadProjection;const layer = ProviderRuntimeRecovery.layer.pipe(@@ -579,7 +623,7 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restartreturn Effect.gen(function* () {const summary =yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");- assert.equal(summary.terminalizedRuns, status === "preparing" ? 1 : 0);+ assert.equal(summary.terminalizedRuns, terminalized ? 1 : 0);const command = committedInput;assert.isNotNull(command);if (command === null) return;@@ -588,17 +632,17 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restartconst nodeEvent = command.events.find((event) => event.type === "node.updated");assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.status : null,- status === "preparing" ? "cancelled" : "queued",+ terminalized ? "cancelled" : "queued",);assert.isEmpty(command.effects);- if (status === "preparing") {+ if (terminalized) {assert.equal(attemptEvent?.type === "run-attempt.updated" ? attemptEvent.payload.status : null,"cancelled",);assert.equal(nodeEvent?.type === "node.updated" ? nodeEvent.payload.status : null,- "cancelled",+ scenario === "starting-own-request" ? null : "cancelled",);} else {assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queuePosition : null, 1);diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.tsindex 80747cb68f2..fa6298ce3f5 100644--- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts+++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts@@ -25,6 +25,7 @@ import * as ProjectionStore from "./ProjectionStore.ts";import * as ServerSettings from "../serverSettings.ts";import { restartContinuationRun } from "./RestartContinuation.ts";import { isBufferedProviderNotification } from "./NotificationMailbox.ts";+import { queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts";import {endOrphanedNativeSubagent,endRunlessRootTurns,@@ -210,7 +211,7 @@ export const make = Effect.gen(function* () {stoppedThisPass: ReadonlyMap<string, RecoverySession> = new Map(),) {const now = yield* DateTime.now;- const runs = [] as Array<OrchestrationV2ThreadProjection["runs"][number]>;+ let runs = [] as Array<OrchestrationV2ThreadProjection["runs"][number]>;for (const run of nonterminalRuns(projection)) {if (run.status === "waiting") {const checkpointEffects = yield* outbox@@ -325,13 +326,25 @@ export const make = Effect.gen(function* () {cancelledBackgroundNativeIds.add(item.nativeItemRef.nativeId);}};+ const unstartedRuns = runs.filter(+ (run) =>+ run.status === "starting" &&+ neverStarted(run) &&+ !projection.providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId) &&+ !projection.runtimeRequests.some(+ (request) =>+ request.status === "pending" &&+ projection.nodes.some((node) => node.id === request.nodeId && node.runId === run.id),+ ),+ );const queuedRuns = projection.runs.filter((run) => run.status === "queued");+ const pendingRuns = [...unstartedRuns, ...queuedRuns];const queuedMessages =- queuedRuns.length === 0+ pendingRuns.length === 0? []: (yield* projections.getThreadRecords(projection.thread.id, ["messages"], {- messageIds: queuedRuns.map((run) => run.userMessageId),+ messageIds: pendingRuns.map((run) => run.userMessageId),}).pipe(Effect.mapError(@@ -346,12 +359,69 @@ export const make = Effect.gen(function* () {const bufferedMessageIds = new Set(queuedMessages.filter(isBufferedProviderNotification).map((message) => message.id),);- for (const run of queuedRuns) {+ const retryableMessageIds = new Set(+ queuedMessages+ .filter(+ (message) =>+ message.createdBy === "user" &&+ message.scheduledTaskId === undefined &&+ message.notification === undefined &&+ message.delegatedCompletion === undefined,+ )+ .map((message) => message.id),+ );+ // A saved user message the provider never accepted stays available through the held queue.+ const retryableRuns = unstartedRuns.filter((run) =>+ retryableMessageIds.has(run.userMessageId),+ );+ const retryableRunIds = new Set(retryableRuns.map((run) => run.id));+ const retryableItems =+ retryableRuns.length === 0+ ? []+ : (yield* projections+ .getThreadRecords(projection.thread.id, ["turnItems"], {+ turnItemTypes: ["user_message"],+ turnItemRunIds: retryableRuns.map((run) => run.id),+ })+ .pipe(+ Effect.mapError(+ (cause) =>+ new ProviderRuntimeRecoveryError({+ operation: "read-projections",+ threadId: projection.thread.id,+ cause,+ }),+ ),+ )).turnItems;+ for (const item of retryableItems) {+ if (item.type !== "user_message" || !retryableMessageIds.has(item.messageId)) continue;+ events.push({+ id: yield* allocateEventId(),+ type: "turn-item.updated",+ threadId: projection.thread.id,+ ...(item.runId === null ? {} : { runId: item.runId }),+ occurredAt: now,+ payload: {+ ...item,+ inputIntent: "queued_turn",+ status: "pending",+ startedAt: null,+ completedAt: null,+ updatedAt: now,+ },+ });+ }+ runs = runs.filter((run) => !retryableRunIds.has(run.id));+ const heldRuns = [+ ...retryableRuns,+ ...queuedRunsInDeliveryOrder({ runs: queuedRuns, messages: queuedMessages }),+ ];+ for (const [index, run] of heldRuns.entries()) {if (bufferedMessageIds.has(run.userMessageId)) {runs.push(run);continue;}- if (run.queueHeld === true) continue;+ if (run.queueHeld === true && retryableRuns.length === 0) continue;events.push({id: yield* allocateEventId(),type: "run.updated",@@ -359,7 +429,12 @@ export const make = Effect.gen(function* () {runId: run.id,providerInstanceId: run.providerInstanceId,occurredAt: now,- payload: { ...run, queueHeld: true },+ payload: {+ ...run,+ status: "queued",+ queueHeld: true,+ ...(retryableRuns.length === 0 ? {} : { queuePosition: index + 1 }),+ },});}for (const question of questions) {diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.tsindex 796a50435ab..9cbb8ead594 100644--- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts+++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts@@ -6310,13 +6310,18 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {}),);- it.effect.each(["startup", "shutdown"] as const)(- "preserves and holds queued messages across %s until explicitly resumed",- (trigger) =>+ it.effect.each([+ { trigger: "startup", started: true },+ { trigger: "shutdown", started: true },+ { trigger: "startup", started: false },+ { trigger: "shutdown", started: false },+ ] as const)(+ "preserves and holds queued messages across $trigger with started=$started until explicitly resumed",+ ({ trigger, started }) =>Effect.gen(function* () {const orchestrator = yield* Orchestrator.OrchestratorV2;const recovery = yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService;- const threadId = ThreadId.make(`queue-hold-${trigger}`);+ const threadId = ThreadId.make(`queue-hold-${trigger}-${started}`);yield* orchestrator.dispatch({type: "thread.create",createdBy: "user",@@ -6346,6 +6351,20 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {});}const before = yield* orchestrator.getThreadProjection(threadId);+ const source = before.runs[0]!;+ if (started) {+ yield* (yield* EventSink.EventSinkV2).write({+ events: [+ {+ id: EventId.make(`${threadId}:started`),+ type: "run.updated",+ threadId,+ occurredAt: yield* DateTime.now,+ payload: { ...source, status: "running", startedAt: yield* DateTime.now },+ },+ ],+ });+ }const queued = before.runs.filter((run) => run.status === "queued");assert.equal(queued.length, 2);yield* recovery.reconcile(trigger);@@ -6356,12 +6375,33 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {const held = yield* orchestrator.getThreadProjection(threadId);assert.deepEqual(held.runs.map((run) => run.status),- ["cancelled", "queued", "queued"],+ [started ? "cancelled" : "queued", "queued", "queued"],);- for (const run of queued) {+ if (!started) {+ assert.deepInclude(+ held.runs.find((run) => run.id === source.id),+ {+ status: "queued",+ queueHeld: true,+ queuePosition: 1,+ },+ );+ }+ if (!started) {+ const prompt = held.turnItems.find(+ (item) => item.type === "user_message" && item.messageId === source.userMessageId,+ );+ assert.deepInclude(prompt, {+ inputIntent: "queued_turn",+ status: "pending",+ startedAt: null,+ completedAt: null,+ });+ }+ for (const [index, run] of queued.entries()) {assert.deepEqual(held.runs.find((row) => row.id === run.id),- { ...run, queueHeld: true },+ { ...run, queueHeld: true, ...(started ? {} : { queuePosition: index + 2 }) },);assert.deepEqual(held.messages.find((row) => row.id === run.userMessageId),@@ -6374,21 +6414,23 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {assert.equal(held.nodes.find((row) => row.id === run.rootNodeId)?.status, "pending");}const first = queued[0]!;- const second = queued[1]!;- yield* orchestrator.dispatch({- type: "queued-run.edit",- commandId: CommandId.make(`${threadId}:edit`),- threadId,- runId: second.id,- text: "Edited second message",- });- yield* orchestrator.dispatch({- type: "queued-run.reorder",- commandId: CommandId.make(`${threadId}:reorder`),- threadId,- runId: second.id,- beforeRunId: first.id,- });+ const second = started ? queued[1]! : source;+ if (started) {+ yield* orchestrator.dispatch({+ type: "queued-run.edit",+ commandId: CommandId.make(`${threadId}:edit`),+ threadId,+ runId: second.id,+ text: "Edited second message",+ });+ yield* orchestrator.dispatch({+ type: "queued-run.reorder",+ commandId: CommandId.make(`${threadId}:reorder`),+ threadId,+ runId: second.id,+ beforeRunId: first.id,+ });+ }assert.equal(yield* orchestrator.resumeQueuedRuns, 0);const resume = {type: "queue.resume" as const,@@ -6403,9 +6445,41 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {assert.isFalse(resumed.runs.some((run) => run.status === "queued" && run.queueHeld));assert.equal(resumed.messages.find((row) => row.id === second.userMessageId)?.text,- "Edited second message",+ started ? "Edited second message" : "Active",+ );+ if (!started) {+ assert.equal(+ resumed.attempts.find((row) => row.id === source.activeAttemptId)?.status,+ "pending",+ );+ assert.equal(+ resumed.nodes.find((row) => row.id === source.rootNodeId)?.status,+ "pending",+ );+ // Recovery command identities include the restart timestamp.+ yield* TestClock.adjust("1 millis");+ yield* recovery.reconcile("startup");+ yield* orchestrator.dispatch({+ ...resume,+ commandId: CommandId.make(`${threadId}:resume-again`),+ });+ const starts = yield* (yield* EffectOutbox.EffectOutboxV2).listByCommandId(+ CommandId.make(`command:system:start-queued:${source.id}`),+ );+ const pendingStarts = starts.filter((effect) => effect.status === "pending");+ assert.lengthOf(pendingStarts, 1);+ assert.deepEqual(pendingStarts[0]?.request, {+ type: "provider-turn.start",+ runId: source.id,+ });+ }+ const afterRetries = yield* orchestrator.getThreadProjection(threadId);+ assert.equal(+ afterRetries.runs.length,+ 3,+ "resume retries must not duplicate messages or runs",);- assert.equal(resumed.runs.length, 3, "resume retries must not duplicate messages or runs");+ assert.lengthOf(afterRetries.messages, before.messages.length);}),);diff --git a/apps/web/src/components/chat/QueuedRunsControl.tsx b/apps/web/src/components/chat/QueuedRunsControl.tsxindex ca98e919ff9..97b74a75a73 100644--- a/apps/web/src/components/chat/QueuedRunsControl.tsx+++ b/apps/web/src/components/chat/QueuedRunsControl.tsx@@ -255,7 +255,7 @@ export function QueuedRunsControl({aria-posinset={position + 1}aria-setsize={items.length}aria-current={isEditing ? "true" : undefined}- className="group/queued flex w-full flex-col items-end gap-1 transition-opacity duration-150 ease-out starting:opacity-0 motion-reduce:transition-none"+ className="group/queued flex w-full flex-col items-end gap-1"><QueuedMessageenvironmentId={props.environmentId}