Skip to content
TypeScript

fix(server): a queued provider switch no longer ends background work (#30401)

Julius Marminge6049196c5004
diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts
index 08ba4ab5cc0..f20ca8bdc92 100644
--- a/apps/server/src/orchestration-v2/Orchestrator.ts
+++ b/apps/server/src/orchestration-v2/Orchestrator.ts
@@ -1167,6 +1167,35 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
return withId;
});
+ // Detaching kills background work running inside the provider process.
+ // Probe even with an empty roster: Claude's native subagents never reach
+ // pendingBackgroundTasks. Sessions shared across threads probe per thread.
+ const sessionHasBackgroundWork = Effect.fn("Orchestrator.sessionHasBackgroundWork")(function* (
+ detaching: OrchestrationV2ThreadProjection["providerSessions"][number],
+ providerThreads: OrchestrationV2ThreadProjection["providerThreads"],
+ ) {
+ const session = yield* providerSessions
+ .get(detaching.id)
+ .pipe(Effect.orElseSucceed(() => Option.none()));
+ if (Option.isNone(session)) return false;
+ const runtime = session.value;
+ const probes =
+ !detaching.capabilities.sessions.supportsMultipleProviderThreadsPerSession &&
+ runtime.hasPendingBackgroundWork !== undefined
+ ? [runtime.hasPendingBackgroundWork]
+ : providerThreads.flatMap((providerThread) =>
+ providerThread.providerSessionId === detaching.id &&
+ runtime.hasPendingBackgroundWorkForThread !== undefined
+ ? [runtime.hasPendingBackgroundWorkForThread(providerThread)]
+ : [],
+ );
+ for (const probe of probes) {
+ // A failed probe cannot confirm that the work ended.
+ if (yield* probe.pipe(Effect.catchCause(() => Effect.succeed(true)))) return true;
+ }
+ return false;
+ });
+
// Runs in every command transaction; avoid reading thousands of finished leaf nodes.
const readCommandProjection = (threadId: ThreadId) =>
projectionStore
@@ -1827,6 +1856,42 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
),
)
: null;
+ const releasableSessions = projection.providerSessions.filter(
+ (session) =>
+ switchPlan?.releaseProviderSessionIds.includes(session.id) &&
+ session.status !== "stopped" &&
+ session.status !== "error",
+ );
+ const busySessionIds = new Set<ProviderSessionId>();
+ for (const session of releasableSessions) {
+ if (yield* sessionHasBackgroundWork(session, projection.providerThreads)) {
+ busySessionIds.add(session.id);
+ }
+ }
+ // Nobody waits on a queued start to refuse it. A handoff runs in its own session, so a
+ // busy one stays up; a restart resumes the busy session's native thread, so wait instead.
+ if (busySessionIds.size > 0 && switchPlan?.transition.type === "restart_and_resume") {
+ yield* Effect.logInfo(
+ "Holding the queue: the replaced provider session still runs background work",
+ { threadId, runId: queuedRun.id, providerSessionIds: Array.from(busySessionIds) },
+ );
+ yield* writeSystemEvents(
+ projection.runs
+ .filter((run) => run.status === "queued")
+ .map((run) => ({
+ type: "run.updated" as const,
+ threadId,
+ runId: run.id,
+ providerInstanceId: run.providerInstanceId,
+ occurredAt: now,
+ payload: { ...run, queueHeld: true },
+ })),
+ );
+ return;
+ }
+ const sessionsToDetach = releasableSessions.filter(
+ (session) => !busySessionIds.has(session.id),
+ );
const canResumeAcrossInstances =
switchPlan?.instanceChanged === true &&
switchPlan.transition.type === "restart_and_resume" &&
@@ -2165,12 +2230,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
},
]
: [];
- const sessionsToDetach = projection.providerSessions.filter(
- (session) =>
- switchPlan?.releaseProviderSessionIds.includes(session.id) &&
- session.status !== "stopped" &&
- session.status !== "error",
- );
yield* writeSystemEvents(
[
...(selectionChanged
@@ -6284,40 +6343,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
session.status !== "stopped" &&
session.status !== "error",
);
- // Detaching kills background work running inside the provider process.
- // Probe even with an empty roster: Claude's native subagents never reach
- // pendingBackgroundTasks. Sessions shared across threads probe per thread.
for (const detaching of sessionsToDetach) {
- const session = yield* providerSessions
- .get(detaching.id)
- .pipe(Effect.orElseSucceed(() => Option.none()));
- if (Option.isNone(session)) {
- continue;
- }
- const runtime = session.value;
- const probes =
- !detaching.capabilities.sessions.supportsMultipleProviderThreadsPerSession &&
- runtime.hasPendingBackgroundWork !== undefined
- ? [runtime.hasPendingBackgroundWork]
- : projection.providerThreads.flatMap((providerThread) =>
- providerThread.providerSessionId === detaching.id &&
- runtime.hasPendingBackgroundWorkForThread !== undefined
- ? [runtime.hasPendingBackgroundWorkForThread(providerThread)]
- : [],
- );
- for (const probe of probes) {
- const stillRunning = yield* probe.pipe(
- // A failed probe cannot confirm that the work ended.
- Effect.catchCause(() => Effect.succeed(true)),
- );
- if (stillRunning) {
- return yield* new OrchestratorDispatchError({
- commandId: command.commandId,
- commandType: command.type,
- cause:
- "The previous provider is still running background agents or commands, and switching would end them. Wait for them to finish, or press Stop, then send the message again.",
- });
- }
+ if (yield* sessionHasBackgroundWork(detaching, projection.providerThreads)) {
+ return yield* new OrchestratorDispatchError({
+ commandId: command.commandId,
+ commandType: command.type,
+ cause:
+ "The previous provider is still running background agents or commands, and switching would end them. Wait for them to finish, or press Stop, then send the message again.",
+ });
}
}
diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts
index 2570c28e36b..629b3902e76 100644
--- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts
+++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts
@@ -2529,21 +2529,36 @@ describe("orchestration v2 provider switching", () => {
// regression: #20343 sending to an instance with separate history must release the old session
it.live.each([
- ["detaches the previous instance's session when a send hands off to another instance", false],
- ["refuses the hand-off while the previous instance still runs background work", true],
- ] as const)("%s", ([, backgroundRunning]) =>
+ [
+ "detaches the previous instance's session when a send hands off to another instance",
+ { background: false, queued: false, sharedHistory: false },
+ ],
+ [
+ "refuses the hand-off while the previous instance still runs background work",
+ { background: true, queued: false, sharedHistory: false },
+ ],
+ [
+ "keeps a busy previous session running when a queued hand-off starts",
+ { background: true, queued: true, sharedHistory: false },
+ ],
+ [
+ "holds a queued account switch while the replaced session still runs background work",
+ { background: true, queued: true, sharedHistory: true },
+ ],
+ ] as const)("%s", ([, { background, queued, sharedHistory }]) =>
Effect.scoped(
Effect.gen(function* () {
const cwd = yield* checkpointWorkspace("handoff-releases-session");
const capturedTurns = yield* Ref.make<ReadonlyArray<CapturedTurn>>([]);
- const backgroundWork = yield* Ref.make(backgroundRunning);
+ const backgroundWork = yield* Ref.make(background);
const closedSessions = yield* Ref.make<ReadonlyArray<string>>([]);
+ const firstStarted = yield* Deferred.make<void>();
+ const releaseFirst = yield* Deferred.make<void>();
const secondSelection: ModelSelection = {
...CLAUDE_MODEL_SELECTION,
instanceId: ProviderInstanceId.make("claude-second-account"),
};
- // No shared continuation key: each Claude instance owns its own native history.
- const layerRegistry = ProviderAdapterRegistry.layerFromAdapters([
+ const adapters = [
makeTestAdapter({
instanceId: CLAUDE_MODEL_SELECTION.instanceId,
driver: CLAUDE_DRIVER,
@@ -2553,6 +2568,7 @@ describe("orchestration v2 provider switching", () => {
capturedTurns,
backgroundWork,
closedSessions,
+ ...(queued ? { holdFirstTurn: firstStarted, releaseFirstTurn: releaseFirst } : {}),
}),
makeTestAdapter({
instanceId: secondSelection.instanceId,
@@ -2562,7 +2578,33 @@ describe("orchestration v2 provider switching", () => {
responseByRunOrdinal: { 2: "Second account complete" },
capturedTurns,
}),
- ]);
+ ];
+ // Without a shared continuation key, each Claude instance owns its own native history.
+ const layerRegistry = sharedHistory
+ ? Layer.succeed(
+ ProviderAdapterRegistry.ProviderAdapterRegistryV2,
+ ProviderAdapterRegistry.ProviderAdapterRegistryV2.of({
+ get: (instanceId) => {
+ const adapter = adapters.find((candidate) => candidate.instanceId === instanceId);
+ return adapter === undefined
+ ? Effect.fail(
+ new ProviderAdapterRegistry.ProviderAdapterRegistryLookupError({
+ instanceId,
+ }),
+ )
+ : Effect.succeed(adapter);
+ },
+ list: () => Effect.succeed(adapters.map((adapter) => adapter.instanceId)),
+ getMetadata: () =>
+ Effect.succeed({
+ driver: CLAUDE_DRIVER,
+ continuationKey: "claude:shared-native-account-history",
+ enabled: true,
+ capabilities: ClaudeProviderCapabilitiesV2,
+ }),
+ }),
+ )
+ : ProviderAdapterRegistry.layerFromAdapters(adapters);
const handoffThreadId = ThreadId.make("thread:handoff-releases-session");
yield* Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
@@ -2578,7 +2620,7 @@ describe("orchestration v2 provider switching", () => {
text: `Prompt ${ordinal}`,
attachments: [],
modelSelection,
- dispatchMode: { type: "start_immediately" },
+ dispatchMode: { type: queued ? "queue_after_active" : "start_immediately" },
});
yield* orchestrator.dispatch({
type: "thread.create",
@@ -2595,6 +2637,69 @@ describe("orchestration v2 provider switching", () => {
worktreePath: cwd,
});
yield* send(1, CLAUDE_MODEL_SELECTION);
+
+ if (queued) {
+ yield* Deferred.await(firstStarted);
+ yield* send(2, secondSelection);
+ const queuedRunId = (yield* orchestrator.getThreadProjection(handoffThreadId)).runs[1]
+ ?.id;
+ assert.isDefined(queuedRunId);
+ yield* Deferred.succeed(releaseFirst, undefined);
+ yield* orchestrator.streamStoredEvents.pipe(
+ Stream.filter(
+ ({ event }) =>
+ event.type === "run.updated" &&
+ event.runId === queuedRunId &&
+ (event.payload.queueHeld === true || event.payload.status === "completed"),
+ ),
+ Stream.runHead,
+ );
+ yield* worker.drain();
+ const afterQueue = yield* orchestrator.getThreadProjection(handoffThreadId);
+ const firstSession = afterQueue.providerSessions.find(
+ (session) => session.providerInstanceId === CLAUDE_MODEL_SELECTION.instanceId,
+ );
+ // Detaching would end the background agents, so the busy session must stay up.
+ assert.deepEqual(yield* Ref.get(closedSessions), []);
+ assert.equal(firstSession?.status, "ready");
+ if (!sharedHistory) {
+ assert.deepEqual(
+ afterQueue.runs.map((run) => [run.providerInstanceId, run.status]),
+ [
+ [CLAUDE_MODEL_SELECTION.instanceId, "completed"],
+ [secondSelection.instanceId, "completed"],
+ ],
+ );
+ return;
+ }
+ // Resuming the native thread beside the busy session would fork its history.
+ assert.deepEqual(
+ afterQueue.runs.map((run) => [run.status, run.queueHeld === true]),
+ [
+ ["completed", false],
+ ["queued", true],
+ ],
+ );
+ assert.lengthOf(yield* Ref.get(capturedTurns), 1);
+ yield* Ref.set(backgroundWork, false);
+ yield* orchestrator.dispatch({
+ type: "queue.resume",
+ commandId: CommandId.make("command:handoff-releases-session:resume"),
+ threadId: handoffThreadId,
+ });
+ yield* worker.drain();
+ const resumed = yield* waitForIdle(handoffThreadId);
+ assert.deepEqual(
+ resumed.runs.map((run) => [run.providerInstanceId, run.status]),
+ [
+ [CLAUDE_MODEL_SELECTION.instanceId, "completed"],
+ [secondSelection.instanceId, "completed"],
+ ],
+ );
+ assert.deepEqual(yield* Ref.get(closedSessions), [firstSession!.id]);
+ return;
+ }
+
yield* worker.drain();
const afterFirst = yield* waitForIdle(handoffThreadId);
const firstSession = afterFirst.providerSessions.find(
@@ -2602,7 +2707,7 @@ describe("orchestration v2 provider switching", () => {
);
assert.equal(firstSession?.status, "ready");
- if (backgroundRunning) {
+ if (background) {
// Native subagents live only in the adapter's roster, so the projection's
// pendingBackgroundTasks stays empty while the probe still reports work.
assert.deepEqual(afterFirst.providerThreads[0]?.pendingBackgroundTasks ?? [], []);