TypeScript
fix(server): Claude subagents left by a stopped process no longer stay Running (#28356)
Julius Marminge709cfb3a1f50
diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.tsindex 19720096837..f3cf3c43c3d 100644--- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts+++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts@@ -11780,6 +11780,313 @@ describe("ClaudeAdapterV2 background wake turns", () => {),);+ // A Stop closes the first process, and a turn replaces it before its stream ends.+ const stopRacedByReplacementTurn = (label: string) =>+ Effect.gen(function* () {+ const changes = yield* makeChangeSignal;+ const fileSystem = yield* FileSystem.FileSystem;+ const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({+ prefix: `t3-claude-v2-${label}-`,+ });+ const processQueues: Array<Queue.Queue<SDKMessage>> = [];+ // The first two processes exit only when the test ends their streams.+ const closeRequested = [yield* Deferred.make<void>(), yield* Deferred.make<void>()];+ const events: Array<ProviderAdapter.ProviderAdapterV2Event> = [];+ const adapter = yield* ClaudeAdapterV2.makeClaudeAdapterV2({+ instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID,+ settings: DEFAULT_CLAUDE_SETTINGS,+ environment: {},+ attachmentsDir,+ fileSystem,+ path: yield* Path.Path,+ random: yield* SecureRandom.SecureRandom,+ idAllocator: yield* IdAllocator.IdAllocatorV2,+ continuationRequests: { offer: () => Effect.void },+ queryRunner: {+ allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION),+ open: () =>+ Effect.gen(function* () {+ const sdkMessages = yield* Queue.unbounded<SDKMessage>();+ const closeSignal = closeRequested[processQueues.length];+ processQueues.push(sdkMessages);+ return {+ messages: Stream.fromQueue(sdkMessages),+ offer: () => Effect.void,+ setModel: () => Effect.void,+ applyFlagSettings: () => Effect.void,+ setPermissionMode: () => Effect.void,+ backgroundTasks: () => Effect.succeed(false),+ stopTask: () => Effect.void,+ interrupt: Effect.void,+ close:+ closeSignal === undefined+ ? Queue.shutdown(sdkMessages)+ : Deferred.succeed(closeSignal, undefined).pipe(Effect.asVoid),+ };+ }),+ forkSession: () => Effect.die("unused forkSession"),+ subagentLaunchToolUseId: () => Effect.succeed(null),+ assertComplete: Effect.void,+ },+ });+ const threadId = ThreadId.make(`thread-claude-${label}`);+ const runtime = yield* adapter.openSession({+ threadId,+ providerSessionId: ProviderSessionId.make(`provider-session-claude-${label}`),+ modelSelection: CLAUDE_TEST_MODEL_SELECTION,+ runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY,+ });+ const providerThread = yield* runtime.ensureThread({+ threadId,+ modelSelection: CLAUDE_TEST_MODEL_SELECTION,+ runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY,+ });+ yield* runtime.events.pipe(+ Stream.runForEach((event) =>+ Effect.sync(() => {+ events.push(event);+ }).pipe(Effect.andThen(changes.notify)),+ ),+ Effect.forkScoped,+ );+ const terminals = () => events.filter((event) => event.type === "turn.terminal");+ const subagentStatus = (taskId: string) => {+ const latest = events.findLast(+ (event) =>+ event.type === "subagent.updated" && event.subagent.nativeTaskRef?.nativeId === taskId,+ );+ return latest?.type === "subagent.updated" ? latest.subagent.status : undefined;+ };+ const now = yield* DateTime.now;++ yield* runtime.startTurn(+ makeClaudeTestTurnInput({+ threadId,+ providerThread,+ now,+ attemptId: RunAttemptId.make(`attempt-claude-${label}-a`),+ text: "Audit the last commits in the background.",+ attachments: [],+ }),+ );+ yield* Queue.offer(+ processQueues[0]!,+ makeSubagentTaskStartedFrame({+ taskId: "raced-agent",+ toolUseId: "toolu-raced-agent",+ uuid: "00000000-0000-4000-8000-000000002801",+ }),+ );+ yield* Queue.offer(+ processQueues[0]!,+ makeResultFrame({+ uuid: "00000000-0000-4000-8000-000000002802",+ result: "Launched the audit.",+ }),+ );+ yield* changes.until(+ () => terminals().length === 1 && subagentStatus("raced-agent") === "running",+ );++ const stop = yield* runtime+ .interruptTurn({+ providerThread,+ providerTurnId: terminals()[0]!.providerTurnId,+ requestRuntimeRestart: true,+ })+ .pipe(Effect.forkScoped);+ yield* Deferred.await(closeRequested[0]!);+ yield* runtime.startTurn(+ makeClaudeTestTurnInput({+ threadId,+ providerThread: { ...providerThread, status: "active" },+ now,+ attemptId: RunAttemptId.make(`attempt-claude-${label}-b`),+ text: "Resume after the limit reset.",+ attachments: [],+ providerTurnOrdinal: 2,+ }),+ );+ assert.lengthOf(processQueues, 2);+ yield* Queue.offer(+ processQueues[1]!,+ makeSubagentTaskStartedFrame({+ taskId: "own-agent",+ toolUseId: "toolu-own-agent",+ uuid: "00000000-0000-4000-8000-000000002804",+ }),+ );+ yield* changes.until(() => subagentStatus("own-agent") === "running");+ const exitProcess = <E>(index: number, stopping: Fiber.Fiber<void, E>) =>+ Queue.shutdown(processQueues[index]!).pipe(Effect.andThen(Fiber.join(stopping)));+ return {+ runtime,+ threadId,+ providerThread,+ processQueues,+ closeRequested,+ changes,+ terminals,+ subagentStatus,+ exitFirstProcess: exitProcess(0, stop),+ exitProcess,+ now,+ };+ });++ // regression: #28331 a subagent whose process is gone must not stay running forever+ it.effect("ends a subagent whose process a Stop closed while a new turn replaced it", () =>+ Effect.scoped(+ Effect.gen(function* () {+ const { processQueues, changes, terminals, subagentStatus, exitFirstProcess } =+ yield* stopRacedByReplacementTurn("raced-stop-replacement-close");+ yield* exitFirstProcess;+ // The replacement's result follows the stopped process's exit on the event stream.+ yield* Queue.offer(+ processQueues[1]!,+ makeResultFrame({+ uuid: "00000000-0000-4000-8000-000000002803",+ result: "Resumed.",+ }),+ );+ yield* changes.until(() => terminals().length === 2);++ assert.strictEqual(+ subagentStatus("raced-agent"),+ "cancelled",+ "the Stop ended the subagent's process, but the subagent is not cancelled",+ );+ assert.strictEqual(+ subagentStatus("own-agent"),+ "running",+ "the replacement process still runs its own subagent",+ );+ }).pipe(+ Effect.provide(+ Layer.mergeAll(+ IdAllocator.layer,+ McpProviderSessions.layer,+ NodeServices.layer,+ NodeCrypto.layer,+ ),+ ),+ ),+ ),+ );++ // regression: #28331 Stop must clear a subagent whose process is gone+ it.effect("stops a subagent whose process a Stop closed while a new turn replaced it", () =>+ Effect.scoped(+ Effect.gen(function* () {+ const { runtime, providerThread, exitFirstProcess } = yield* stopRacedByReplacementTurn(+ "raced-stop-replacement-stop-subagent",+ );+ yield* exitFirstProcess;+ const stopped = yield* runtime.stopSubagent!({+ providerThread: { ...providerThread, status: "active" },+ nativeTaskId: "raced-agent",+ }).pipe(+ Effect.map((outcome) => outcome?.status ?? "no outcome"),+ Effect.catch((error) =>+ Effect.succeed(`refused: ${"detail" in error ? error.detail : error.message}`),+ ),+ );++ assert.strictEqual(stopped, "cancelled", "Stop of a subagent whose process is gone");+ }).pipe(+ Effect.provide(+ Layer.mergeAll(+ IdAllocator.layer,+ McpProviderSessions.layer,+ NodeServices.layer,+ NodeCrypto.layer,+ ),+ ),+ ),+ ),+ );++ // regression: #28331 a process that exits late ends only the subagents it ran+ it.effect("leaves a stopping replacement's subagent to that replacement's exit", () =>+ Effect.scoped(+ Effect.gen(function* () {+ const {+ runtime,+ threadId,+ providerThread,+ processQueues,+ closeRequested,+ changes,+ terminals,+ subagentStatus,+ exitFirstProcess,+ exitProcess,+ now,+ } = yield* stopRacedByReplacementTurn("raced-stop-replacement-chain");+ const activeThread = { ...providerThread, status: "active" as const };+ const finishTurn = (index: number, turns: number, uuid: string) =>+ Queue.offer(processQueues[index]!, makeResultFrame({ uuid, result: "Done." })).pipe(+ Effect.andThen(changes.until(() => terminals().length === turns)),+ );+ const startTurn = (label: string, providerTurnOrdinal: number) =>+ runtime.startTurn(+ makeClaudeTestTurnInput({+ threadId,+ providerThread: activeThread,+ now,+ attemptId: RunAttemptId.make(`attempt-claude-raced-stop-replacement-chain-${label}`),+ text: "Carry on.",+ attachments: [],+ providerTurnOrdinal,+ }),+ );++ yield* finishTurn(1, 2, "00000000-0000-4000-8000-000000002805");+ const stopSecond = yield* runtime+ .interruptTurn({+ providerThread: activeThread,+ providerTurnId: terminals()[1]!.providerTurnId,+ requestRuntimeRestart: true,+ })+ .pipe(Effect.forkScoped);+ yield* Deferred.await(closeRequested[1]!);+ yield* startTurn("c", 3);+ assert.lengthOf(processQueues, 3);++ yield* exitFirstProcess;+ yield* finishTurn(2, 3, "00000000-0000-4000-8000-000000002806");+ assert.strictEqual(+ subagentStatus("raced-agent"),+ "cancelled",+ "the first process exited after a Stop, but its subagent is not cancelled",+ );+ assert.strictEqual(+ subagentStatus("own-agent"),+ "running",+ "the first process ended a subagent the second process still ran",+ );++ yield* exitProcess(1, stopSecond);+ yield* startTurn("d", 4);+ yield* finishTurn(2, 4, "00000000-0000-4000-8000-000000002807");+ assert.strictEqual(+ subagentStatus("own-agent"),+ "cancelled",+ "the second process exited after a Stop, but its subagent is not cancelled",+ );+ }).pipe(+ Effect.provide(+ Layer.mergeAll(+ IdAllocator.layer,+ McpProviderSessions.layer,+ NodeServices.layer,+ NodeCrypto.layer,+ ),+ ),+ ),+ ),+ );+it.effect("streams a background subagent's work while the root turn is idle", () =>Effect.scoped(Effect.gen(function* () {diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.tsindex add71bfd165..ef5849edb8f 100644--- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts+++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts@@ -4133,6 +4133,8 @@ interface ClaudeLiveQueryContext {permissionMode: PermissionMode;stopping: boolean;readonly subagentsFromEarlierProcesses: ReadonlySet<ActiveClaudeSubagent>;+ // Subagents this process still ran when a replacement took over; only its exit can end them.+ handedOffSubagents: ReadonlySet<ActiveClaudeSubagent>;// The SDK's per-process model ledger at the last result; a new process starts at zero.modelLedger: ClaudeModelLedger | undefined;// Per process, not per session: a relative temp dir follows the cwd this CLI@@ -6361,6 +6363,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (SDKMessage,{ readonly type: "system"; readonly subtype: "task_notification" }>;+ // The process that ran it exited, which confirms the outcome as a native one would.+ readonly processExited?: true;readonly onlyIfRunningInRun?: RunId | null;}) {const registeredSubagent = (yield* Ref.get(sessionSubagentsByTaskId)).get(input.taskId);@@ -6498,7 +6502,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (task,nativeTaskType: input.nativeTaskType ?? existingSubagent?.nativeTaskType,confirmedOutcome:- input.nativeOutcome !== undefined+ input.nativeOutcome !== undefined || input.processExited === true? confirmedSubagentOutcome(task): isReopen? undefined@@ -7787,6 +7791,69 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ();});+ const endSubagentsOfClosedQuery = Effect.fnUntraced(function* (input: {+ readonly closedQuery: ClaudeLiveQueryContext;+ readonly context: ActiveClaudeTurnContext;+ readonly subagents: Iterable<readonly [string, ActiveClaudeSubagent]>;+ readonly buffered: ReadonlyArray<SDKMessage> | undefined;+ readonly interrupted: boolean;+ }) {+ for (const [taskId, subagent] of input.subagents) {+ if (+ subagent.task.status !== "running" ||+ input.closedQuery.subagentsFromEarlierProcesses.has(subagent)+ )+ continue;+ const latestLifecycle = input.buffered?.findLast(+ (+ message,+ ): message is Extract<+ SDKMessage,+ { readonly subtype: "task_started" | "task_notification" }+ > =>+ message.type === "system" &&+ (message.subtype === "task_started" || message.subtype === "task_notification") &&+ message.task_id === taskId,+ );+ const completion =+ latestLifecycle?.subtype === "task_notification" ? latestLifecycle : undefined;+ yield* updateClaudeSubagentNode({+ context: input.context,+ taskId,+ status: completion+ ? claudeTaskOutcome(completion.status)+ : input.interrupted+ ? "cancelled"+ : "failed",+ ...(completion === undefined+ ? {}+ : { workflowFrame: completion, result: completion.summary }),+ processExited: true,+ onlyIfRunningInRun: subagent.task.runId,+ });+ }+ });++ // A query opened while this one was closing inherited the subagents it ran, so+ // nothing else ends them. Leave the live query's own turn, roster and subagents alone.+ const endSubagentsOfReplacedQuery = Effect.fnUntraced(function* (+ closedQuery: ClaudeLiveQueryContext,+ ) {+ const context = closedQuery.lastTurn;+ if (context === null) return;+ yield* endSubagentsOfClosedQuery({+ closedQuery,+ context,+ subagents: [...(yield* Ref.get(sessionSubagentsByTaskId))].filter(([, subagent]) =>+ closedQuery.handedOffSubagents.has(subagent),+ ),+ buffered: (yield* Ref.get(wakeBuffers)).get(closedQuery.nativeThreadId)?.messages,+ interrupted:+ closedQuery.stopping ||+ (yield* Ref.get(interruptedTurns)).has(context.providerTurnId),+ });+ });+const finalizeActiveTurnAfterQueryExit = Effect.fnUntraced(function* (closedQuery: ClaudeLiveQueryContext,cause?: Cause.Cause<ClaudeAgentSdkQueryRunnerError>,@@ -7818,39 +7885,13 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (yield* resetBackgroundTaskStateForNativeThreadProcess(closedQuery.nativeThreadId);}// Preserve buffered outcomes before Stop clears the wake buffer.- for (const [taskId, subagent] of subagents) {- if (- subagent.task.status !== "running" ||- closedQuery.subagentsFromEarlierProcesses.has(subagent)- )- continue;- const latestLifecycle = buffered?.findLast(- (- message,- ): message is Extract<- SDKMessage,- { readonly subtype: "task_started" | "task_notification" }- > =>- message.type === "system" &&- (message.subtype === "task_started" || message.subtype === "task_notification") &&- message.task_id === taskId,- );- const completion =- latestLifecycle?.subtype === "task_notification" ? latestLifecycle : undefined;- yield* updateClaudeSubagentNode({- context,- taskId,- status: completion- ? claudeTaskOutcome(completion.status)- : interrupted- ? "cancelled"- : "failed",- ...(completion === undefined- ? {}- : { workflowFrame: completion, result: completion.summary }),- onlyIfRunningInRun: subagent.task.runId,- });- }+ yield* endSubagentsOfClosedQuery({+ closedQuery,+ context,+ subagents,+ buffered,+ interrupted,+ });yield* Ref.update(interruptedTurns, (current) => {const next = new Set(current);next.delete(context.providerTurnId);@@ -10434,6 +10475,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (receivedInit: false,permissionMode: queryOptions.permissionMode,stopping: false,+ handedOffSubagents: new Set(),subagentsFromEarlierProcesses: new Set([...(yield* Ref.get(sessionSubagentsByTaskId)).values()].filter((subagent) => subagent.task.status === "running",@@ -10447,7 +10489,17 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (hostPlatform,),};- yield* Ref.set(queryContext, context);+ // One step, so the replaced process cannot exit between the install and the handoff.+ yield* Ref.update(queryContext, (replaced) => {+ if (replaced !== null) {+ replaced.handedOffSubagents = new Set(+ [...context.subagentsFromEarlierProcesses].filter(+ (subagent) => !replaced.subagentsFromEarlierProcesses.has(subagent),+ ),+ );+ }+ return context;+ });yield* querySession.messages.pipe(Stream.runForEach((message) =>sdkMessageLock.withPermits(1)(@@ -10505,6 +10557,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (context.nativeThreadId,context.stopping ? "interrupted" : "failed",);+ } else {+ yield* endSubagentsOfReplacedQuery(context);}}),),@@ -11120,6 +11174,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (if (ownedLiveQuery) {yield* finalizeActiveTurnAfterQueryExit(existing);yield* endSubagentCallsOfClosedQuery(nativeThreadId, "interrupted");+ } else {+ yield* endSubagentsOfReplacedQuery(existing);}yield* Deferred.succeed(existing.closed, undefined);});