Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

fix(team): send SIGKILL to spawned slots when the spawn loop throws

Jack Rudenkoa63d51723ab2
diff --git a/ai-docs/architecture/team-lifecycle.md b/ai-docs/architecture/team-lifecycle.md
index 93cd89ce..d9726a97 100644
--- a/ai-docs/architecture/team-lifecycle.md
+++ b/ai-docs/architecture/team-lifecycle.md
@@ -178,11 +178,15 @@ on `handle.done.then(…)` in the handler would depend on code that runs after
registry entry and `done` are all built after the loop, so a synchronous
`spawn()` error or a failed `status.json` write for slot N used to leave slots
1..N-1 running and billing, unreachable, with the SIGINT handler still installed.
-The loop's `catch` sends SIGTERM to every spawned slot's process group
-(`signalProcessTree`, as the SIGINT handler does), removes the SIGINT handler and
-rethrows; the handler's `catch` writes `failed`, `reason: start-failed` with
-counts from `status.json` (`summarise(readTeamStatus(path))`), then rethrows the
-ORIGINAL error. `spawnChild` is the seam a test uses to make one spawn throw.
+The loop's `catch` stops every spawned slot's process group with
+`terminateChildTree` — SIGTERM, then SIGKILL after `TERMINATE_GRACE_MS`, as
+`cancel_session` does — and waits for that before it removes the SIGINT handler
+and rethrows. SIGTERM alone left a slot that traps it running and billing under
+a record that already said `start-failed`. The handler's `catch` then writes
+`failed`, `reason: start-failed` with counts from `status.json`
+(`summarise(readTeamStatus(path))`), and rethrows the ORIGINAL error. `spawnChild`
+is the seam a test uses to make one spawn throw, `terminateGraceMs` the one that
+shortens the grace period (`team-start-failure.test.ts`).
`summarise` counts `COMPLETED` as ok, a `cancelled` reason as cancelled, and every
other state — `PENDING` and `RUNNING` included — as failed; the verdict is
diff --git a/packages/cli/src/team-orchestrator.ts b/packages/cli/src/team-orchestrator.ts
index 84025e79..02ac60b9 100644
--- a/packages/cli/src/team-orchestrator.ts
+++ b/packages/cli/src/team-orchestrator.ts
@@ -12,7 +12,12 @@ import { type SpawnPlan, prehydrateCredentialsForSpawn } from "./auth/credential
import { StreamJsonReducer } from "./channel/stream-json-reducer.js";
import { ENV } from "./config.js";
import { UPSTREAM_ERROR_LOG_ENV } from "./handlers/shared/upstream-error-capture.js";
-import { KILL_PROCESS_GROUP, signalProcessTree, terminateChildTree } from "./process-tree.js";
+import {
+ KILL_PROCESS_GROUP,
+ TERMINATE_GRACE_MS,
+ signalProcessTree,
+ terminateChildTree,
+} from "./process-tree.js";
import { redactSecrets } from "./redact.js";
import { resolveClaudishSpawn } from "./spawn-claudish.js";
import { decodeChunk, newStdioDecoder } from "./stdio-decode.js";
@@ -205,6 +210,11 @@ export interface TeamRunOptions {
* `spawn`). Lets a test make one slot's spawn throw synchronously.
*/
spawnChild?: typeof spawn;
+ /**
+ * Pause between SIGTERM and SIGKILL when the spawn loop throws and the slots
+ * already spawned are stopped. Default `TERMINATE_GRACE_MS`. A test seam.
+ */
+ terminateGraceMs?: number;
/**
* Called exactly once, when the run SETTLES: after the last slot finished and
* the settled `status.txt` render, from inside `done`'s `finally`. Receives
@@ -1517,13 +1527,15 @@ export async function startModels(
completionPromises.push(completionPromise);
}
} catch (err) {
- // Nothing reported as failed to start may keep running or billing: signal
- // every slot already spawned, by process group, exactly as the SIGINT
- // handler does. `onSettled` is NOT called — no run started; the caller
- // records the failure.
- for (const [, proc] of processes) {
- signalProcessTree(proc, "SIGTERM");
- }
+ // Nothing reported as failed to start may keep running or billing: stop
+ // every slot already spawned, by process group — SIGTERM, then SIGKILL
+ // after the grace period, as `cancel_session` does — and only then let
+ // the error out, so the caller's `start-failed` record is true when it
+ // is written. A slot that traps SIGTERM would otherwise run on. The SIGINT
+ // handler stays installed while we wait. `onSettled` is NOT called — no run
+ // started; the caller records the failure.
+ const graceMs = opts.terminateGraceMs ?? TERMINATE_GRACE_MS;
+ await Promise.all([...processes.values()].map((proc) => terminateChildTree(proc, graceMs)));
process.off("SIGINT", sigintHandler);
throw err;
}
diff --git a/packages/cli/src/team-start-failure.test.ts b/packages/cli/src/team-start-failure.test.ts
new file mode 100644
index 00000000..028f2f48
--- /dev/null
+++ b/packages/cli/src/team-start-failure.test.ts
@@ -0,0 +1,106 @@
+/**
+ * A team run that fails to START: the spawn loop throws after some slots
+ * exist. Every slot already spawned must be STOPPED before the error reaches
+ * the caller — SIGTERM, then SIGKILL after the grace period — so nothing the
+ * `team(mode:"run")` handler records as `failed reason=start-failed` keeps
+ * running and billing. A slot that ignores SIGTERM is the case that matters.
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import type { ChildProcess, spawn } from "node:child_process";
+import { EventEmitter } from "node:events";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { PassThrough } from "node:stream";
+
+import { getStatus, setupSession, startModels } from "./team-orchestrator.js";
+
+const spawnPlanner = async () => ({ pinned: new Map<string, string>() });
+const delay = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Until the one spawned slot's end is recorded in `status.json`. The slot whose
+ * spawn threw stays `RUNNING` there: it was marked before the throw.
+ */
+async function waitForSpawnedSlotEnd(sessionPath: string, limitMs = 5_000): Promise<void> {
+ const deadline = Date.now() + limitMs;
+ const ended = () =>
+ Object.values(getStatus(sessionPath).models).some((m) => Boolean(m.completedAt));
+ while (!ended()) {
+ if (Date.now() > deadline) throw new Error(`the spawned slot never ended: ${sessionPath}`);
+ await delay(10);
+ }
+}
+
+/**
+ * A child that ignores SIGTERM and dies only on SIGKILL. No pid, so
+ * `signalProcessTree` never addresses a real process group; `killed` stays
+ * false so every signal reaches `kill`, as the group kill would in production.
+ */
+class StubbornChild extends EventEmitter {
+ readonly pid = undefined;
+ readonly killed = false;
+ exitCode: number | null = null;
+ signalCode: NodeJS.Signals | null = null;
+ readonly stdin = new PassThrough();
+ readonly stdout = new PassThrough();
+ readonly stderr = new PassThrough();
+ readonly signals: string[] = [];
+
+ kill(signal: NodeJS.Signals = "SIGTERM"): boolean {
+ this.signals.push(signal);
+ if (signal === "SIGKILL" && this.signalCode === null) {
+ this.signalCode = "SIGKILL";
+ this.stdout.end();
+ this.stderr.end();
+ this.emit("exit", null, "SIGKILL");
+ }
+ return true;
+ }
+}
+
+let tempRoot: string;
+
+beforeEach(() => {
+ tempRoot = mkdtempSync(join(tmpdir(), "team-start-failure-test-"));
+});
+
+afterEach(() => {
+ rmSync(tempRoot, { recursive: true, force: true });
+});
+
+describe("team spawn-loop failure", () => {
+ it("escalates to SIGKILL for a spawned slot that ignores SIGTERM, before rethrowing", async () => {
+ const sessionPath = join(tempRoot, "stubborn");
+ setupSession(sessionPath, ["model-a", "model-b"], "ping");
+
+ const stubborn = new StubbornChild();
+ let calls = 0;
+ const spawnChild = (() => {
+ calls++;
+ if (calls === 1) return stubborn as unknown as ChildProcess;
+ throw new Error("spawn EAGAIN");
+ }) as unknown as typeof spawn;
+
+ let settledCalls = 0;
+ const error = await startModels(sessionPath, {
+ captureMode: "print",
+ spawnPlanner,
+ spawnChild,
+ terminateGraceMs: 50,
+ onSettled: () => {
+ settledCalls++;
+ },
+ }).catch((err: unknown) => err);
+
+ expect(error).toBeInstanceOf(Error);
+ expect((error as Error).message).toBe("spawn EAGAIN");
+ // Stopped BEFORE the error reached us, not after.
+ expect(stubborn.signals).toEqual(["SIGTERM", "SIGKILL"]);
+ expect(stubborn.signalCode).toBe("SIGKILL");
+ // Its close handler records the kill; let it land before the directory goes.
+ await waitForSpawnedSlotEnd(sessionPath);
+ expect(settledCalls).toBe(0);
+ });
+});