TypeScript
fix(server): a queued provider switch no longer ends background work (#30401)
Julius Marminge6049196c5004
diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.tsindex 08ba4ab5cc0..f20ca8bdc92 100644--- a/apps/server/src/orchestration-v2/Orchestrator.ts+++ b/apps/server/src/orchestration-v2/Orchestrator.ts@@ -1167,6 +1167,35 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functioreturn withId;});+ // Detaching kills background work running inside the provider process.+ // Probe even with an empty roster: Claude's native subagents never reach+ // pendingBackgroundTasks. Sessions shared across threads probe per thread.+ const sessionHasBackgroundWork = Effect.fn("Orchestrator.sessionHasBackgroundWork")(function* (+ detaching: OrchestrationV2ThreadProjection["providerSessions"][number],+ providerThreads: OrchestrationV2ThreadProjection["providerThreads"],+ ) {+ const session = yield* providerSessions+ .get(detaching.id)+ .pipe(Effect.orElseSucceed(() => Option.none()));+ if (Option.isNone(session)) return false;+ const runtime = session.value;+ const probes =+ !detaching.capabilities.sessions.supportsMultipleProviderThreadsPerSession &&+ runtime.hasPendingBackgroundWork !== undefined+ ? [runtime.hasPendingBackgroundWork]+ : providerThreads.flatMap((providerThread) =>+ providerThread.providerSessionId === detaching.id &&+ runtime.hasPendingBackgroundWorkForThread !== undefined+ ? [runtime.hasPendingBackgroundWorkForThread(providerThread)]+ : [],+ );+ for (const probe of probes) {+ // A failed probe cannot confirm that the work ended.+ if (yield* probe.pipe(Effect.catchCause(() => Effect.succeed(true)))) return true;+ }+ return false;+ });+// Runs in every command transaction; avoid reading thousands of finished leaf nodes.const readCommandProjection = (threadId: ThreadId) =>projectionStore@@ -1827,6 +1856,42 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio),): null;+ const releasableSessions = projection.providerSessions.filter(+ (session) =>+ switchPlan?.releaseProviderSessionIds.includes(session.id) &&+ session.status !== "stopped" &&+ session.status !== "error",+ );+ const busySessionIds = new Set<ProviderSessionId>();+ for (const session of releasableSessions) {+ if (yield* sessionHasBackgroundWork(session, projection.providerThreads)) {+ busySessionIds.add(session.id);+ }+ }+ // Nobody waits on a queued start to refuse it. A handoff runs in its own session, so a+ // busy one stays up; a restart resumes the busy session's native thread, so wait instead.+ if (busySessionIds.size > 0 && switchPlan?.transition.type === "restart_and_resume") {+ yield* Effect.logInfo(+ "Holding the queue: the replaced provider session still runs background work",+ { threadId, runId: queuedRun.id, providerSessionIds: Array.from(busySessionIds) },+ );+ yield* writeSystemEvents(+ projection.runs+ .filter((run) => run.status === "queued")+ .map((run) => ({+ type: "run.updated" as const,+ threadId,+ runId: run.id,+ providerInstanceId: run.providerInstanceId,+ occurredAt: now,+ payload: { ...run, queueHeld: true },+ })),+ );+ return;+ }+ const sessionsToDetach = releasableSessions.filter(+ (session) => !busySessionIds.has(session.id),+ );const canResumeAcrossInstances =switchPlan?.instanceChanged === true &&switchPlan.transition.type === "restart_and_resume" &&@@ -2165,12 +2230,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio},]: [];- const sessionsToDetach = projection.providerSessions.filter(- (session) =>- switchPlan?.releaseProviderSessionIds.includes(session.id) &&- session.status !== "stopped" &&- session.status !== "error",- );yield* writeSystemEvents([...(selectionChanged@@ -6284,40 +6343,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functiosession.status !== "stopped" &&session.status !== "error",);- // Detaching kills background work running inside the provider process.- // Probe even with an empty roster: Claude's native subagents never reach- // pendingBackgroundTasks. Sessions shared across threads probe per thread.for (const detaching of sessionsToDetach) {- const session = yield* providerSessions- .get(detaching.id)- .pipe(Effect.orElseSucceed(() => Option.none()));- if (Option.isNone(session)) {- continue;- }- const runtime = session.value;- const probes =- !detaching.capabilities.sessions.supportsMultipleProviderThreadsPerSession &&- runtime.hasPendingBackgroundWork !== undefined- ? [runtime.hasPendingBackgroundWork]- : projection.providerThreads.flatMap((providerThread) =>- providerThread.providerSessionId === detaching.id &&- runtime.hasPendingBackgroundWorkForThread !== undefined- ? [runtime.hasPendingBackgroundWorkForThread(providerThread)]- : [],- );- for (const probe of probes) {- const stillRunning = yield* probe.pipe(- // A failed probe cannot confirm that the work ended.- Effect.catchCause(() => Effect.succeed(true)),- );- if (stillRunning) {- return yield* new OrchestratorDispatchError({- commandId: command.commandId,- commandType: command.type,- cause:- "The previous provider is still running background agents or commands, and switching would end them. Wait for them to finish, or press Stop, then send the message again.",- });- }+ if (yield* sessionHasBackgroundWork(detaching, projection.providerThreads)) {+ return yield* new OrchestratorDispatchError({+ commandId: command.commandId,+ commandType: command.type,+ cause:+ "The previous provider is still running background agents or commands, and switching would end them. Wait for them to finish, or press Stop, then send the message again.",+ });}}diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.tsindex 2570c28e36b..629b3902e76 100644--- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts+++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts@@ -2529,21 +2529,36 @@ describe("orchestration v2 provider switching", () => {// regression: #20343 sending to an instance with separate history must release the old sessionit.live.each([- ["detaches the previous instance's session when a send hands off to another instance", false],- ["refuses the hand-off while the previous instance still runs background work", true],- ] as const)("%s", ([, backgroundRunning]) =>+ [+ "detaches the previous instance's session when a send hands off to another instance",+ { background: false, queued: false, sharedHistory: false },+ ],+ [+ "refuses the hand-off while the previous instance still runs background work",+ { background: true, queued: false, sharedHistory: false },+ ],+ [+ "keeps a busy previous session running when a queued hand-off starts",+ { background: true, queued: true, sharedHistory: false },+ ],+ [+ "holds a queued account switch while the replaced session still runs background work",+ { background: true, queued: true, sharedHistory: true },+ ],+ ] as const)("%s", ([, { background, queued, sharedHistory }]) =>Effect.scoped(Effect.gen(function* () {const cwd = yield* checkpointWorkspace("handoff-releases-session");const capturedTurns = yield* Ref.make<ReadonlyArray<CapturedTurn>>([]);- const backgroundWork = yield* Ref.make(backgroundRunning);+ const backgroundWork = yield* Ref.make(background);const closedSessions = yield* Ref.make<ReadonlyArray<string>>([]);+ const firstStarted = yield* Deferred.make<void>();+ const releaseFirst = yield* Deferred.make<void>();const secondSelection: ModelSelection = {...CLAUDE_MODEL_SELECTION,instanceId: ProviderInstanceId.make("claude-second-account"),};- // No shared continuation key: each Claude instance owns its own native history.- const layerRegistry = ProviderAdapterRegistry.layerFromAdapters([+ const adapters = [makeTestAdapter({instanceId: CLAUDE_MODEL_SELECTION.instanceId,driver: CLAUDE_DRIVER,@@ -2553,6 +2568,7 @@ describe("orchestration v2 provider switching", () => {capturedTurns,backgroundWork,closedSessions,+ ...(queued ? { holdFirstTurn: firstStarted, releaseFirstTurn: releaseFirst } : {}),}),makeTestAdapter({instanceId: secondSelection.instanceId,@@ -2562,7 +2578,33 @@ describe("orchestration v2 provider switching", () => {responseByRunOrdinal: { 2: "Second account complete" },capturedTurns,}),- ]);+ ];+ // Without a shared continuation key, each Claude instance owns its own native history.+ const layerRegistry = sharedHistory+ ? Layer.succeed(+ ProviderAdapterRegistry.ProviderAdapterRegistryV2,+ ProviderAdapterRegistry.ProviderAdapterRegistryV2.of({+ get: (instanceId) => {+ const adapter = adapters.find((candidate) => candidate.instanceId === instanceId);+ return adapter === undefined+ ? Effect.fail(+ new ProviderAdapterRegistry.ProviderAdapterRegistryLookupError({+ instanceId,+ }),+ )+ : Effect.succeed(adapter);+ },+ list: () => Effect.succeed(adapters.map((adapter) => adapter.instanceId)),+ getMetadata: () =>+ Effect.succeed({+ driver: CLAUDE_DRIVER,+ continuationKey: "claude:shared-native-account-history",+ enabled: true,+ capabilities: ClaudeProviderCapabilitiesV2,+ }),+ }),+ )+ : ProviderAdapterRegistry.layerFromAdapters(adapters);const handoffThreadId = ThreadId.make("thread:handoff-releases-session");yield* Effect.gen(function* () {const orchestrator = yield* Orchestrator.OrchestratorV2;@@ -2578,7 +2620,7 @@ describe("orchestration v2 provider switching", () => {text: `Prompt ${ordinal}`,attachments: [],modelSelection,- dispatchMode: { type: "start_immediately" },+ dispatchMode: { type: queued ? "queue_after_active" : "start_immediately" },});yield* orchestrator.dispatch({type: "thread.create",@@ -2595,6 +2637,69 @@ describe("orchestration v2 provider switching", () => {worktreePath: cwd,});yield* send(1, CLAUDE_MODEL_SELECTION);++ if (queued) {+ yield* Deferred.await(firstStarted);+ yield* send(2, secondSelection);+ const queuedRunId = (yield* orchestrator.getThreadProjection(handoffThreadId)).runs[1]+ ?.id;+ assert.isDefined(queuedRunId);+ yield* Deferred.succeed(releaseFirst, undefined);+ yield* orchestrator.streamStoredEvents.pipe(+ Stream.filter(+ ({ event }) =>+ event.type === "run.updated" &&+ event.runId === queuedRunId &&+ (event.payload.queueHeld === true || event.payload.status === "completed"),+ ),+ Stream.runHead,+ );+ yield* worker.drain();+ const afterQueue = yield* orchestrator.getThreadProjection(handoffThreadId);+ const firstSession = afterQueue.providerSessions.find(+ (session) => session.providerInstanceId === CLAUDE_MODEL_SELECTION.instanceId,+ );+ // Detaching would end the background agents, so the busy session must stay up.+ assert.deepEqual(yield* Ref.get(closedSessions), []);+ assert.equal(firstSession?.status, "ready");+ if (!sharedHistory) {+ assert.deepEqual(+ afterQueue.runs.map((run) => [run.providerInstanceId, run.status]),+ [+ [CLAUDE_MODEL_SELECTION.instanceId, "completed"],+ [secondSelection.instanceId, "completed"],+ ],+ );+ return;+ }+ // Resuming the native thread beside the busy session would fork its history.+ assert.deepEqual(+ afterQueue.runs.map((run) => [run.status, run.queueHeld === true]),+ [+ ["completed", false],+ ["queued", true],+ ],+ );+ assert.lengthOf(yield* Ref.get(capturedTurns), 1);+ yield* Ref.set(backgroundWork, false);+ yield* orchestrator.dispatch({+ type: "queue.resume",+ commandId: CommandId.make("command:handoff-releases-session:resume"),+ threadId: handoffThreadId,+ });+ yield* worker.drain();+ const resumed = yield* waitForIdle(handoffThreadId);+ assert.deepEqual(+ resumed.runs.map((run) => [run.providerInstanceId, run.status]),+ [+ [CLAUDE_MODEL_SELECTION.instanceId, "completed"],+ [secondSelection.instanceId, "completed"],+ ],+ );+ assert.deepEqual(yield* Ref.get(closedSessions), [firstSession!.id]);+ return;+ }+yield* worker.drain();const afterFirst = yield* waitForIdle(handoffThreadId);const firstSession = afterFirst.providerSessions.find(@@ -2602,7 +2707,7 @@ describe("orchestration v2 provider switching", () => {);assert.equal(firstSession?.status, "ready");- if (backgroundRunning) {+ if (background) {// Native subagents live only in the adapter's roster, so the projection's// pendingBackgroundTasks stays empty while the probe still reports work.assert.deepEqual(afterFirst.providerThreads[0]?.pendingBackgroundTasks ?? [], []);