Claude Code. Any Model. The most powerful AI coding agent now speaks every language.
fix(team): send SIGKILL to spawned slots when the spawn loop throws
Jack Rudenkoa63d51723ab2
diff --git a/ai-docs/architecture/team-lifecycle.md b/ai-docs/architecture/team-lifecycle.mdindex 93cd89ce..d9726a97 100644--- a/ai-docs/architecture/team-lifecycle.md+++ b/ai-docs/architecture/team-lifecycle.md@@ -178,11 +178,15 @@ on `handle.done.then(…)` in the handler would depend on code that runs afterregistry entry and `done` are all built after the loop, so a synchronous`spawn()` error or a failed `status.json` write for slot N used to leave slots1..N-1 running and billing, unreachable, with the SIGINT handler still installed.-The loop's `catch` sends SIGTERM to every spawned slot's process group-(`signalProcessTree`, as the SIGINT handler does), removes the SIGINT handler and-rethrows; the handler's `catch` writes `failed`, `reason: start-failed` with-counts from `status.json` (`summarise(readTeamStatus(path))`), then rethrows the-ORIGINAL error. `spawnChild` is the seam a test uses to make one spawn throw.+The loop's `catch` stops every spawned slot's process group with+`terminateChildTree` — SIGTERM, then SIGKILL after `TERMINATE_GRACE_MS`, as+`cancel_session` does — and waits for that before it removes the SIGINT handler+and rethrows. SIGTERM alone left a slot that traps it running and billing under+a record that already said `start-failed`. The handler's `catch` then writes+`failed`, `reason: start-failed` with counts from `status.json`+(`summarise(readTeamStatus(path))`), and rethrows the ORIGINAL error. `spawnChild`+is the seam a test uses to make one spawn throw, `terminateGraceMs` the one that+shortens the grace period (`team-start-failure.test.ts`).`summarise` counts `COMPLETED` as ok, a `cancelled` reason as cancelled, and everyother state — `PENDING` and `RUNNING` included — as failed; the verdict isdiff --git a/packages/cli/src/team-orchestrator.ts b/packages/cli/src/team-orchestrator.tsindex 84025e79..02ac60b9 100644--- a/packages/cli/src/team-orchestrator.ts+++ b/packages/cli/src/team-orchestrator.ts@@ -12,7 +12,12 @@ import { type SpawnPlan, prehydrateCredentialsForSpawn } from "./auth/credentialimport { StreamJsonReducer } from "./channel/stream-json-reducer.js";import { ENV } from "./config.js";import { UPSTREAM_ERROR_LOG_ENV } from "./handlers/shared/upstream-error-capture.js";-import { KILL_PROCESS_GROUP, signalProcessTree, terminateChildTree } from "./process-tree.js";+import {+ KILL_PROCESS_GROUP,+ TERMINATE_GRACE_MS,+ signalProcessTree,+ terminateChildTree,+} from "./process-tree.js";import { redactSecrets } from "./redact.js";import { resolveClaudishSpawn } from "./spawn-claudish.js";import { decodeChunk, newStdioDecoder } from "./stdio-decode.js";@@ -205,6 +210,11 @@ export interface TeamRunOptions {* `spawn`). Lets a test make one slot's spawn throw synchronously.*/spawnChild?: typeof spawn;+ /**+ * Pause between SIGTERM and SIGKILL when the spawn loop throws and the slots+ * already spawned are stopped. Default `TERMINATE_GRACE_MS`. A test seam.+ */+ terminateGraceMs?: number;/*** Called exactly once, when the run SETTLES: after the last slot finished and* the settled `status.txt` render, from inside `done`'s `finally`. Receives@@ -1517,13 +1527,15 @@ export async function startModels(completionPromises.push(completionPromise);}} catch (err) {- // Nothing reported as failed to start may keep running or billing: signal- // every slot already spawned, by process group, exactly as the SIGINT- // handler does. `onSettled` is NOT called — no run started; the caller- // records the failure.- for (const [, proc] of processes) {- signalProcessTree(proc, "SIGTERM");- }+ // Nothing reported as failed to start may keep running or billing: stop+ // every slot already spawned, by process group — SIGTERM, then SIGKILL+ // after the grace period, as `cancel_session` does — and only then let+ // the error out, so the caller's `start-failed` record is true when it+ // is written. A slot that traps SIGTERM would otherwise run on. The SIGINT+ // handler stays installed while we wait. `onSettled` is NOT called — no run+ // started; the caller records the failure.+ const graceMs = opts.terminateGraceMs ?? TERMINATE_GRACE_MS;+ await Promise.all([...processes.values()].map((proc) => terminateChildTree(proc, graceMs)));process.off("SIGINT", sigintHandler);throw err;}diff --git a/packages/cli/src/team-start-failure.test.ts b/packages/cli/src/team-start-failure.test.tsnew file mode 100644index 00000000..028f2f48--- /dev/null+++ b/packages/cli/src/team-start-failure.test.ts@@ -0,0 +1,106 @@+/**+ * A team run that fails to START: the spawn loop throws after some slots+ * exist. Every slot already spawned must be STOPPED before the error reaches+ * the caller — SIGTERM, then SIGKILL after the grace period — so nothing the+ * `team(mode:"run")` handler records as `failed reason=start-failed` keeps+ * running and billing. A slot that ignores SIGTERM is the case that matters.+ */++import { afterEach, beforeEach, describe, expect, it } from "bun:test";+import type { ChildProcess, spawn } from "node:child_process";+import { EventEmitter } from "node:events";+import { mkdtempSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { PassThrough } from "node:stream";++import { getStatus, setupSession, startModels } from "./team-orchestrator.js";++const spawnPlanner = async () => ({ pinned: new Map<string, string>() });+const delay = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));++/**+ * Until the one spawned slot's end is recorded in `status.json`. The slot whose+ * spawn threw stays `RUNNING` there: it was marked before the throw.+ */+async function waitForSpawnedSlotEnd(sessionPath: string, limitMs = 5_000): Promise<void> {+ const deadline = Date.now() + limitMs;+ const ended = () =>+ Object.values(getStatus(sessionPath).models).some((m) => Boolean(m.completedAt));+ while (!ended()) {+ if (Date.now() > deadline) throw new Error(`the spawned slot never ended: ${sessionPath}`);+ await delay(10);+ }+}++/**+ * A child that ignores SIGTERM and dies only on SIGKILL. No pid, so+ * `signalProcessTree` never addresses a real process group; `killed` stays+ * false so every signal reaches `kill`, as the group kill would in production.+ */+class StubbornChild extends EventEmitter {+ readonly pid = undefined;+ readonly killed = false;+ exitCode: number | null = null;+ signalCode: NodeJS.Signals | null = null;+ readonly stdin = new PassThrough();+ readonly stdout = new PassThrough();+ readonly stderr = new PassThrough();+ readonly signals: string[] = [];++ kill(signal: NodeJS.Signals = "SIGTERM"): boolean {+ this.signals.push(signal);+ if (signal === "SIGKILL" && this.signalCode === null) {+ this.signalCode = "SIGKILL";+ this.stdout.end();+ this.stderr.end();+ this.emit("exit", null, "SIGKILL");+ }+ return true;+ }+}++let tempRoot: string;++beforeEach(() => {+ tempRoot = mkdtempSync(join(tmpdir(), "team-start-failure-test-"));+});++afterEach(() => {+ rmSync(tempRoot, { recursive: true, force: true });+});++describe("team spawn-loop failure", () => {+ it("escalates to SIGKILL for a spawned slot that ignores SIGTERM, before rethrowing", async () => {+ const sessionPath = join(tempRoot, "stubborn");+ setupSession(sessionPath, ["model-a", "model-b"], "ping");++ const stubborn = new StubbornChild();+ let calls = 0;+ const spawnChild = (() => {+ calls++;+ if (calls === 1) return stubborn as unknown as ChildProcess;+ throw new Error("spawn EAGAIN");+ }) as unknown as typeof spawn;++ let settledCalls = 0;+ const error = await startModels(sessionPath, {+ captureMode: "print",+ spawnPlanner,+ spawnChild,+ terminateGraceMs: 50,+ onSettled: () => {+ settledCalls++;+ },+ }).catch((err: unknown) => err);++ expect(error).toBeInstanceOf(Error);+ expect((error as Error).message).toBe("spawn EAGAIN");+ // Stopped BEFORE the error reached us, not after.+ expect(stubborn.signals).toEqual(["SIGTERM", "SIGKILL"]);+ expect(stubborn.signalCode).toBe("SIGKILL");+ // Its close handler records the kill; let it land before the directory goes.+ await waitForSpawnedSlotEnd(sessionPath);+ expect(settledCalls).toBe(0);+ });+});