Skip to content
TypeScript

fix(server): retry unstarted runs after recovery (#30560)

Yash Singh737afe24bd20
diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts
index f20ca8bdc92..f45c63ff27e 100644
--- a/apps/server/src/orchestration-v2/Orchestrator.ts
+++ b/apps/server/src/orchestration-v2/Orchestrator.ts
@@ -2376,7 +2376,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
},
})),
{
- id: `effect:${commandId}:provider-turn.start:${queuedRun.id}`,
+ // A restart can return this run to the queue after its previous start effect was cancelled.
+ id: `effect:${commandId}:provider-turn.start:${queuedRun.id}:${yield* idAllocator.allocate.event({ threadId, commandId })}`,
commandId,
threadId,
request: { type: "provider-turn.start", runId: queuedRun.id },
diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts
index 120858ace27..d300cf59ca3 100644
--- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts
+++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts
@@ -501,7 +501,19 @@ it.effect("recovery closes a secret card's form but leaves a link card open", ()
}).pipe(Effect.provide(layer));
});
-it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart", (status) => {
+it.effect.each([
+ "queued",
+ "preparing",
+ "starting",
+ "starting-other-request",
+ "starting-own-request",
+ "starting-scheduled",
+] as const)("recovers $0 work after restart", (scenario) => {
+ const status = scenario.startsWith("starting") ? "starting" : scenario;
+ const terminalized =
+ status === "preparing" ||
+ scenario === "starting-own-request" ||
+ scenario === "starting-scheduled";
const threadId = ThreadId.make("thread_queued_restart");
const runId = RunId.make("run_queued_restart");
const attemptId = RunAttemptId.make("attempt_queued_restart");
@@ -510,7 +522,18 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
null;
const projection = {
thread: { id: threadId },
- runtimeRequests: [],
+ runtimeRequests: scenario.endsWith("request")
+ ? [
+ {
+ id: RuntimeRequestId.make("pending-recovery-request"),
+ nodeId:
+ scenario === "starting-own-request" ? rootNodeId : NodeId.make("other-run-request"),
+ kind: "user_input",
+ status: "pending",
+ responseCapability: { type: "message" },
+ },
+ ]
+ : [],
providerSessions: [],
providerThreads: [],
providerTurns: [],
@@ -518,6 +541,9 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
{
id: runId,
status,
+ startedAt: null,
+ activeAttemptId: attemptId,
+ userMessageId: MessageId.make("message_queued_restart"),
queuePosition: 1,
providerInstanceId: ProviderInstanceId.make("codex"),
...(status === "preparing"
@@ -540,6 +566,15 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
},
],
nodes: [
+ ...(scenario === "starting-other-request"
+ ? [
+ {
+ id: NodeId.make("other-run-request"),
+ runId: RunId.make("earlier-run"),
+ status: "waiting",
+ },
+ ]
+ : []),
{
id: rootNodeId,
runId,
@@ -547,7 +582,16 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
},
],
subagents: [],
- messages: [],
+ messages:
+ status === "starting"
+ ? [
+ {
+ id: MessageId.make("message_queued_restart"),
+ createdBy: "user",
+ ...(scenario === "starting-scheduled" ? { scheduledTaskId: "scheduled-task" } : {}),
+ },
+ ]
+ : [],
turnItems: [],
} as unknown as OrchestrationV2ThreadProjection;
const layer = ProviderRuntimeRecovery.layer.pipe(
@@ -579,7 +623,7 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
return Effect.gen(function* () {
const summary =
yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
- assert.equal(summary.terminalizedRuns, status === "preparing" ? 1 : 0);
+ assert.equal(summary.terminalizedRuns, terminalized ? 1 : 0);
const command = committedInput;
assert.isNotNull(command);
if (command === null) return;
@@ -588,17 +632,17 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
const nodeEvent = command.events.find((event) => event.type === "node.updated");
assert.equal(
runEvent?.type === "run.updated" ? runEvent.payload.status : null,
- status === "preparing" ? "cancelled" : "queued",
+ terminalized ? "cancelled" : "queued",
);
assert.isEmpty(command.effects);
- if (status === "preparing") {
+ if (terminalized) {
assert.equal(
attemptEvent?.type === "run-attempt.updated" ? attemptEvent.payload.status : null,
"cancelled",
);
assert.equal(
nodeEvent?.type === "node.updated" ? nodeEvent.payload.status : null,
- "cancelled",
+ scenario === "starting-own-request" ? null : "cancelled",
);
} else {
assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queuePosition : null, 1);
diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
index 80747cb68f2..fa6298ce3f5 100644
--- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
@@ -25,6 +25,7 @@ import * as ProjectionStore from "./ProjectionStore.ts";
import * as ServerSettings from "../serverSettings.ts";
import { restartContinuationRun } from "./RestartContinuation.ts";
import { isBufferedProviderNotification } from "./NotificationMailbox.ts";
+import { queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts";
import {
endOrphanedNativeSubagent,
endRunlessRootTurns,
@@ -210,7 +211,7 @@ export const make = Effect.gen(function* () {
stoppedThisPass: ReadonlyMap<string, RecoverySession> = new Map(),
) {
const now = yield* DateTime.now;
- const runs = [] as Array<OrchestrationV2ThreadProjection["runs"][number]>;
+ let runs = [] as Array<OrchestrationV2ThreadProjection["runs"][number]>;
for (const run of nonterminalRuns(projection)) {
if (run.status === "waiting") {
const checkpointEffects = yield* outbox
@@ -325,13 +326,25 @@ export const make = Effect.gen(function* () {
cancelledBackgroundNativeIds.add(item.nativeItemRef.nativeId);
}
};
+ const unstartedRuns = runs.filter(
+ (run) =>
+ run.status === "starting" &&
+ neverStarted(run) &&
+ !projection.providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId) &&
+ !projection.runtimeRequests.some(
+ (request) =>
+ request.status === "pending" &&
+ projection.nodes.some((node) => node.id === request.nodeId && node.runId === run.id),
+ ),
+ );
const queuedRuns = projection.runs.filter((run) => run.status === "queued");
+ const pendingRuns = [...unstartedRuns, ...queuedRuns];
const queuedMessages =
- queuedRuns.length === 0
+ pendingRuns.length === 0
? []
: (yield* projections
.getThreadRecords(projection.thread.id, ["messages"], {
- messageIds: queuedRuns.map((run) => run.userMessageId),
+ messageIds: pendingRuns.map((run) => run.userMessageId),
})
.pipe(
Effect.mapError(
@@ -346,12 +359,69 @@ export const make = Effect.gen(function* () {
const bufferedMessageIds = new Set(
queuedMessages.filter(isBufferedProviderNotification).map((message) => message.id),
);
- for (const run of queuedRuns) {
+ const retryableMessageIds = new Set(
+ queuedMessages
+ .filter(
+ (message) =>
+ message.createdBy === "user" &&
+ message.scheduledTaskId === undefined &&
+ message.notification === undefined &&
+ message.delegatedCompletion === undefined,
+ )
+ .map((message) => message.id),
+ );
+ // A saved user message the provider never accepted stays available through the held queue.
+ const retryableRuns = unstartedRuns.filter((run) =>
+ retryableMessageIds.has(run.userMessageId),
+ );
+ const retryableRunIds = new Set(retryableRuns.map((run) => run.id));
+ const retryableItems =
+ retryableRuns.length === 0
+ ? []
+ : (yield* projections
+ .getThreadRecords(projection.thread.id, ["turnItems"], {
+ turnItemTypes: ["user_message"],
+ turnItemRunIds: retryableRuns.map((run) => run.id),
+ })
+ .pipe(
+ Effect.mapError(
+ (cause) =>
+ new ProviderRuntimeRecoveryError({
+ operation: "read-projections",
+ threadId: projection.thread.id,
+ cause,
+ }),
+ ),
+ )).turnItems;
+ for (const item of retryableItems) {
+ if (item.type !== "user_message" || !retryableMessageIds.has(item.messageId)) continue;
+ events.push({
+ id: yield* allocateEventId(),
+ type: "turn-item.updated",
+ threadId: projection.thread.id,
+ ...(item.runId === null ? {} : { runId: item.runId }),
+ occurredAt: now,
+ payload: {
+ ...item,
+ inputIntent: "queued_turn",
+ status: "pending",
+ startedAt: null,
+ completedAt: null,
+ updatedAt: now,
+ },
+ });
+ }
+ runs = runs.filter((run) => !retryableRunIds.has(run.id));
+ const heldRuns = [
+ ...retryableRuns,
+ ...queuedRunsInDeliveryOrder({ runs: queuedRuns, messages: queuedMessages }),
+ ];
+ for (const [index, run] of heldRuns.entries()) {
if (bufferedMessageIds.has(run.userMessageId)) {
runs.push(run);
continue;
}
- if (run.queueHeld === true) continue;
+ if (run.queueHeld === true && retryableRuns.length === 0) continue;
events.push({
id: yield* allocateEventId(),
type: "run.updated",
@@ -359,7 +429,12 @@ export const make = Effect.gen(function* () {
runId: run.id,
providerInstanceId: run.providerInstanceId,
occurredAt: now,
- payload: { ...run, queueHeld: true },
+ payload: {
+ ...run,
+ status: "queued",
+ queueHeld: true,
+ ...(retryableRuns.length === 0 ? {} : { queuePosition: index + 1 }),
+ },
});
}
for (const question of questions) {
diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts
index 796a50435ab..9cbb8ead594 100644
--- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts
+++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts
@@ -6310,13 +6310,18 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
}),
);
- it.effect.each(["startup", "shutdown"] as const)(
- "preserves and holds queued messages across %s until explicitly resumed",
- (trigger) =>
+ it.effect.each([
+ { trigger: "startup", started: true },
+ { trigger: "shutdown", started: true },
+ { trigger: "startup", started: false },
+ { trigger: "shutdown", started: false },
+ ] as const)(
+ "preserves and holds queued messages across $trigger with started=$started until explicitly resumed",
+ ({ trigger, started }) =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const recovery = yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService;
- const threadId = ThreadId.make(`queue-hold-${trigger}`);
+ const threadId = ThreadId.make(`queue-hold-${trigger}-${started}`);
yield* orchestrator.dispatch({
type: "thread.create",
createdBy: "user",
@@ -6346,6 +6351,20 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
});
}
const before = yield* orchestrator.getThreadProjection(threadId);
+ const source = before.runs[0]!;
+ if (started) {
+ yield* (yield* EventSink.EventSinkV2).write({
+ events: [
+ {
+ id: EventId.make(`${threadId}:started`),
+ type: "run.updated",
+ threadId,
+ occurredAt: yield* DateTime.now,
+ payload: { ...source, status: "running", startedAt: yield* DateTime.now },
+ },
+ ],
+ });
+ }
const queued = before.runs.filter((run) => run.status === "queued");
assert.equal(queued.length, 2);
yield* recovery.reconcile(trigger);
@@ -6356,12 +6375,33 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
const held = yield* orchestrator.getThreadProjection(threadId);
assert.deepEqual(
held.runs.map((run) => run.status),
- ["cancelled", "queued", "queued"],
+ [started ? "cancelled" : "queued", "queued", "queued"],
);
- for (const run of queued) {
+ if (!started) {
+ assert.deepInclude(
+ held.runs.find((run) => run.id === source.id),
+ {
+ status: "queued",
+ queueHeld: true,
+ queuePosition: 1,
+ },
+ );
+ }
+ if (!started) {
+ const prompt = held.turnItems.find(
+ (item) => item.type === "user_message" && item.messageId === source.userMessageId,
+ );
+ assert.deepInclude(prompt, {
+ inputIntent: "queued_turn",
+ status: "pending",
+ startedAt: null,
+ completedAt: null,
+ });
+ }
+ for (const [index, run] of queued.entries()) {
assert.deepEqual(
held.runs.find((row) => row.id === run.id),
- { ...run, queueHeld: true },
+ { ...run, queueHeld: true, ...(started ? {} : { queuePosition: index + 2 }) },
);
assert.deepEqual(
held.messages.find((row) => row.id === run.userMessageId),
@@ -6374,21 +6414,23 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
assert.equal(held.nodes.find((row) => row.id === run.rootNodeId)?.status, "pending");
}
const first = queued[0]!;
- const second = queued[1]!;
- yield* orchestrator.dispatch({
- type: "queued-run.edit",
- commandId: CommandId.make(`${threadId}:edit`),
- threadId,
- runId: second.id,
- text: "Edited second message",
- });
- yield* orchestrator.dispatch({
- type: "queued-run.reorder",
- commandId: CommandId.make(`${threadId}:reorder`),
- threadId,
- runId: second.id,
- beforeRunId: first.id,
- });
+ const second = started ? queued[1]! : source;
+ if (started) {
+ yield* orchestrator.dispatch({
+ type: "queued-run.edit",
+ commandId: CommandId.make(`${threadId}:edit`),
+ threadId,
+ runId: second.id,
+ text: "Edited second message",
+ });
+ yield* orchestrator.dispatch({
+ type: "queued-run.reorder",
+ commandId: CommandId.make(`${threadId}:reorder`),
+ threadId,
+ runId: second.id,
+ beforeRunId: first.id,
+ });
+ }
assert.equal(yield* orchestrator.resumeQueuedRuns, 0);
const resume = {
type: "queue.resume" as const,
@@ -6403,9 +6445,41 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
assert.isFalse(resumed.runs.some((run) => run.status === "queued" && run.queueHeld));
assert.equal(
resumed.messages.find((row) => row.id === second.userMessageId)?.text,
- "Edited second message",
+ started ? "Edited second message" : "Active",
+ );
+ if (!started) {
+ assert.equal(
+ resumed.attempts.find((row) => row.id === source.activeAttemptId)?.status,
+ "pending",
+ );
+ assert.equal(
+ resumed.nodes.find((row) => row.id === source.rootNodeId)?.status,
+ "pending",
+ );
+ // Recovery command identities include the restart timestamp.
+ yield* TestClock.adjust("1 millis");
+ yield* recovery.reconcile("startup");
+ yield* orchestrator.dispatch({
+ ...resume,
+ commandId: CommandId.make(`${threadId}:resume-again`),
+ });
+ const starts = yield* (yield* EffectOutbox.EffectOutboxV2).listByCommandId(
+ CommandId.make(`command:system:start-queued:${source.id}`),
+ );
+ const pendingStarts = starts.filter((effect) => effect.status === "pending");
+ assert.lengthOf(pendingStarts, 1);
+ assert.deepEqual(pendingStarts[0]?.request, {
+ type: "provider-turn.start",
+ runId: source.id,
+ });
+ }
+ const afterRetries = yield* orchestrator.getThreadProjection(threadId);
+ assert.equal(
+ afterRetries.runs.length,
+ 3,
+ "resume retries must not duplicate messages or runs",
);
- assert.equal(resumed.runs.length, 3, "resume retries must not duplicate messages or runs");
+ assert.lengthOf(afterRetries.messages, before.messages.length);
}),
);
diff --git a/apps/web/src/components/chat/QueuedRunsControl.tsx b/apps/web/src/components/chat/QueuedRunsControl.tsx
index ca98e919ff9..97b74a75a73 100644
--- a/apps/web/src/components/chat/QueuedRunsControl.tsx
+++ b/apps/web/src/components/chat/QueuedRunsControl.tsx
@@ -255,7 +255,7 @@ export function QueuedRunsControl({
aria-posinset={position + 1}
aria-setsize={items.length}
aria-current={isEditing ? "true" : undefined}
- className="group/queued flex w-full flex-col items-end gap-1 transition-opacity duration-150 ease-out starting:opacity-0 motion-reduce:transition-none"
+ className="group/queued flex w-full flex-col items-end gap-1"
>
<QueuedMessage
environmentId={props.environmentId}