Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
fix(daemon): actually kill agent processes instead of silently deadlocking
Guse8159eba6b0f
diff --git a/src/cli/daemon/kill-tree.ts b/src/cli/daemon/kill-tree.tsindex 29c908d32..30e7dbd3d 100644--- a/src/cli/daemon/kill-tree.ts+++ b/src/cli/daemon/kill-tree.ts@@ -35,28 +35,33 @@ export function isAlive(pid: number): boolean {}}-/** Send `signal` to the whole process group; fall back to the plain pid. */+/**+ * Send `signal` to the whole process group, ALWAYS ALSO followed by a direct+ * pid signal — regardless of whether the group signal succeeded, threw+ * `ESRCH` (no such process group — e.g. a driver stopped passing `detached`),+ * or threw anything else. A group-signal failure must never be read as "the+ * pid is dead": that conflates two unrelated failure modes and was the root+ * cause of a sibling bug in `src/daemon/src/runtime/killTree.ts` where+ * stopped agents kept running forever (see+ * plans/fix-daemon-agent-process-kill.md). Signaling an already-dead pid is+ * safe — it just throws too, caught and ignored below.+ */function signalTree(pid: number, signal: NodeJS.Signals): void {- if (isPosix) {- try {- // Negative pid = the process group led by `pid`.- process.kill(-pid, signal);- return;- } catch (e) {- const code = (e as NodeJS.ErrnoException)?.code;- if (code === "ESRCH") return; // group already gone- // EPERM / no-group (legacy non-detached spawn): fall through to plain pid.- }- }if (!isPosix) {try {execSync(`taskkill /PID ${pid} /T /F`, { stdio: "ignore" });- return;} catch {// process may already be dead}return;}+ try {+ // Negative pid = the process group led by `pid`.+ process.kill(-pid, signal);+ } catch {+ // No such process group (not detached, or already gone) — fall through+ // to signal the pid directly.+ }try {process.kill(pid, signal);} catch {diff --git a/src/daemon/eslint.config.ts b/src/daemon/eslint.config.tsindex f6e9d8cb6..0a0bbe8ca 100644--- a/src/daemon/eslint.config.ts+++ b/src/daemon/eslint.config.ts@@ -11,6 +11,28 @@ const eslintConfig = defineConfig(["@typescript-eslint/no-unused-vars": ["error", { argsIgnorePattern: "^_", varsIgnorePattern: "^_" }],},},+ {+ // Drivers must spawn agent CLIs through `spawnAgentProcess` (src/runtime/killTree.ts)+ // so the detached-process-group contract killProcessTree relies on can't be+ // silently skipped by a new/edited driver — see plans/fix-daemon-agent-process-kill.md.+ files: ["src/drivers/**/*.ts"],+ ignores: ["src/drivers/**/*.test.ts"],+ rules: {+ "no-restricted-imports": [+ "error",+ {+ paths: [+ {+ name: "child_process",+ importNames: ["spawn"],+ message:+ "Use spawnAgentProcess from '../runtime/killTree.js' instead — it guarantees the detached process-group contract killProcessTree relies on to actually terminate the process.",+ },+ ],+ },+ ],+ },+ },{files: ["**/*.test.ts"],rules: {diff --git a/src/daemon/src/drivers/antigravity.ts b/src/daemon/src/drivers/antigravity.tsindex b4a449cb4..b9854a776 100644--- a/src/daemon/src/drivers/antigravity.ts+++ b/src/daemon/src/drivers/antigravity.ts@@ -6,11 +6,11 @@* `text` event unless it matches an error pattern. Models are suggestion-only* (not passed at launch). The prompt is written to stdin then closed.*/-import { spawn } from "child_process";import { randomUUID } from "crypto";import type { Driver, LaunchConfig, LaunchContext, ParsedEvent, SpawnResult } from "../types.js";import { prepareCliTransport, buildCliTransportSystemPrompt } from "./cliTransport.js";import { probeCliRuntime, resolveSpawnSpec } from "./probe.js";+import { spawnAgentProcess } from "../runtime/killTree.js";const ERROR_LINE_PATTERNS: RegExp[] = [/^error[:\s]/i, /\bfatal\b/i, /\bpanic\b/i, /unable to/i];@@ -60,9 +60,8 @@ export class AntigravityDriver implements Driver {// Cross-platform spawn: on Windows the agy entry is often a `.cmd`// shim, which `child_process.spawn` can't exec without a shell.const spec = resolveSpawnSpec("agy", buildAntigravityArgs(ctx));- const proc = spawn(spec.command, spec.args, {+ const proc = spawnAgentProcess(spec.command, spec.args, {cwd: ctx.workingDirectory,- stdio: ["pipe", "pipe", "pipe"],env: spawnEnv,shell: spec.shell,});diff --git a/src/daemon/src/drivers/claude.ts b/src/daemon/src/drivers/claude.tsindex 625a3d1d0..231a6fc79 100644--- a/src/daemon/src/drivers/claude.ts+++ b/src/daemon/src/drivers/claude.ts@@ -8,7 +8,6 @@* delivery is `gated` — held until a safe boundary (see runtime/apmStateMachine* and runtime/turnState).*/-import { spawn } from "child_process";import type { Driver, EncodeOpts, LaunchConfig, LaunchContext, ParsedEvent, SpawnResult } from "../types.js";import { prepareCliTransport, buildCliTransportSystemPrompt, DEFAULT_CLI_CONFIG } from "./cliTransport.js";import { buildClaudeProviderIsolationEnv } from "./claudeProviderIsolation.js";@@ -19,6 +18,7 @@ import {} from "./claudeLaunch.js";import { ClaudeEventNormalizer } from "./claudeEventNormalizer.js";import { probeClaude } from "./probe.js";+import { spawnAgentProcess } from "../runtime/killTree.js";export class ClaudeDriver implements Driver {readonly id = "claude";@@ -54,9 +54,8 @@ export class ClaudeDriver implements Driver {const claudeCommand = resolveClaudeLaunchCommand(ctx.config);const spawnSpec = buildClaudeSpawnSpec(claudeCommand);- const proc = spawn(spawnSpec.command, args, {+ const proc = spawnAgentProcess(spawnSpec.command, args, {cwd: ctx.workingDirectory,- stdio: ["pipe", "pipe", "pipe"],env: spawnEnv,shell: spawnSpec.shell,});diff --git a/src/daemon/src/drivers/codex.ts b/src/daemon/src/drivers/codex.tsindex d7279bc56..fe2cab617 100644--- a/src/daemon/src/drivers/codex.ts+++ b/src/daemon/src/drivers/codex.ts@@ -13,13 +13,13 @@* Handshake (queued on spawn): `initialize` → then `thread/start` (or* `thread/resume` with the prior threadId). The thread id is the session id.*/-import { spawn } from "child_process";import type { Driver, EncodeOpts, LaunchConfig, LaunchContext, ParsedEvent, SpawnResult } from "../types.js";import { prepareCliTransport, buildCliTransportSystemPrompt } from "./cliTransport.js";import { CodexEventNormalizer } from "./codexEventNormalizer.js";import { probeCliRuntime, resolveSpawnSpec } from "./probe.js";import { resolveCodexHomeRootFromEnv } from "./codexHome.js";import { resolveLaunchFieldsOrDefault } from "../runtimeConfig.js";+import { spawnAgentProcess } from "../runtime/killTree.js";/** True if a resume error means the prior thread rollout is gone. */export function isCodexMissingRolloutError(message: string): boolean {@@ -85,9 +85,8 @@ export class CodexDriver implements Driver {this.codexHomeRoot = resolveCodexHomeRootFromEnv(spawnEnv, { cwd: ctx.workingDirectory });// Cross-platform spawn: on Windows the codex entry is often a `.cmd` shim.const spec = resolveSpawnSpec("codex", ["app-server", "--listen", "stdio://"]);- const proc = spawn(spec.command, spec.args, {+ const proc = spawnAgentProcess(spec.command, spec.args, {cwd: ctx.workingDirectory,- stdio: ["pipe", "pipe", "pipe"],env: spawnEnv,shell: spec.shell,});diff --git a/src/daemon/src/drivers/copilot.ts b/src/daemon/src/drivers/copilot.tsindex 04dfe002b..721214bdf 100644--- a/src/daemon/src/drivers/copilot.ts+++ b/src/daemon/src/drivers/copilot.ts@@ -5,11 +5,11 @@* is launched per wake; the prompt is passed as the `-p` argument (nothing is* written to stdin). Emits a JSON event stream and exits.*/-import { spawn } from "child_process";import type { Driver, LaunchConfig, LaunchContext, ParsedEvent, SpawnResult } from "../types.js";import { prepareCliTransport, buildCliTransportSystemPrompt } from "./cliTransport.js";import { probeCliRuntime, resolveSpawnSpec } from "./probe.js";import { resolveLaunchFieldsOrDefault } from "../runtimeConfig.js";+import { spawnAgentProcess } from "../runtime/killTree.js";export class CopilotDriver implements Driver {readonly id = "copilot";@@ -46,9 +46,8 @@ export class CopilotDriver implements Driver {// Cross-platform spawn: on Windows the copilot entry is often a `.cmd`// shim, which `child_process.spawn` can't exec without a shell.const spec = resolveSpawnSpec("copilot", args);- const proc = spawn(spec.command, spec.args, {+ const proc = spawnAgentProcess(spec.command, spec.args, {cwd: ctx.workingDirectory,- stdio: ["pipe", "pipe", "pipe"],env: spawnEnv,shell: spec.shell,});diff --git a/src/daemon/src/drivers/cursor.ts b/src/daemon/src/drivers/cursor.tsindex d8883cadf..064d3136a 100644--- a/src/daemon/src/drivers/cursor.ts+++ b/src/daemon/src/drivers/cursor.ts@@ -5,11 +5,11 @@* --trust <prompt>` is launched per wake. Emits Anthropic-style stream-json* (system/assistant/result envelopes) and exits.*/-import { spawn } from "child_process";import type { Driver, LaunchConfig, LaunchContext, ParsedEvent, SpawnResult } from "../types.js";import { prepareCliTransport, buildCliTransportSystemPrompt } from "./cliTransport.js";import { probeCliRuntime, resolveSpawnSpec } from "./probe.js";import { resolveLaunchFieldsOrDefault } from "../runtimeConfig.js";+import { spawnAgentProcess } from "../runtime/killTree.js";export class CursorDriver implements Driver {readonly id = "cursor";@@ -46,9 +46,8 @@ export class CursorDriver implements Driver {// Cross-platform spawn: on Windows the cursor-agent entry is often a// `.cmd` shim, which `child_process.spawn` can't exec without a shell.const spec = resolveSpawnSpec("cursor-agent", args);- const proc = spawn(spec.command, spec.args, {+ const proc = spawnAgentProcess(spec.command, spec.args, {cwd: ctx.workingDirectory,- stdio: ["pipe", "pipe", "pipe"],env: spawnEnv,shell: spec.shell,});diff --git a/src/daemon/src/drivers/gemini.ts b/src/daemon/src/drivers/gemini.tsindex 92e16896f..6b7aba033 100644--- a/src/daemon/src/drivers/gemini.ts+++ b/src/daemon/src/drivers/gemini.ts@@ -7,11 +7,11 @@* boundary. No mid-session input is possible (`encodeStdinMessage` → null), so* new messages mean a brand-new process and the agent polls the inbox.*/-import { spawn } from "child_process";import type { Driver, LaunchConfig, LaunchContext, ParsedEvent, SpawnResult } from "../types.js";import { prepareCliTransport, buildCliTransportSystemPrompt } from "./cliTransport.js";import { probeCliRuntime, resolveSpawnSpec } from "./probe.js";import { resolveLaunchFieldsOrDefault } from "../runtimeConfig.js";+import { spawnAgentProcess } from "../runtime/killTree.js";export function buildGeminiArgs(config: LaunchConfig): string[] {const f = resolveLaunchFieldsOrDefault(config.runtimeConfig);@@ -58,9 +58,8 @@ export class GeminiDriver implements Driver {// Cross-platform spawn: on Windows the gemini entry is often a `.cmd` shim.const spec = resolveSpawnSpec("gemini", buildGeminiArgs(ctx.config));- const proc = spawn(spec.command, spec.args, {+ const proc = spawnAgentProcess(spec.command, spec.args, {cwd: ctx.workingDirectory,- stdio: ["pipe", "pipe", "pipe"],env: spawnEnv,shell: spec.shell,});diff --git a/src/daemon/src/drivers/kimi.ts b/src/daemon/src/drivers/kimi.tsindex fa43f4207..173299ae5 100644--- a/src/daemon/src/drivers/kimi.ts+++ b/src/daemon/src/drivers/kimi.ts@@ -7,12 +7,12 @@* workdir (Kimi auto-reads it from cwd). On spawn we send `initialize` then a* `prompt`; thereafter idle messages are `prompt` and busy messages are `steer`.*/-import { spawn } from "child_process";import { randomUUID } from "crypto";import type { Driver, EncodeOpts, LaunchConfig, LaunchContext, ParsedEvent, SpawnResult } from "../types.js";import { prepareCliTransport, buildCliTransportSystemPrompt } from "./cliTransport.js";import { probeCliRuntime, resolveSpawnSpec } from "./probe.js";import { resolveLaunchFieldsOrDefault } from "../runtimeConfig.js";+import { spawnAgentProcess } from "../runtime/killTree.js";function parseToolArguments(args: unknown): unknown {if (typeof args !== "string") return args ?? {};@@ -58,9 +58,8 @@ export class KimiDriver implements Driver {// Cross-platform spawn: on Windows the kimi entry is often a `.cmd`// shim, which `child_process.spawn` can't exec without a shell.const spec = resolveSpawnSpec("kimi", args);- const proc = spawn(spec.command, spec.args, {+ const proc = spawnAgentProcess(spec.command, spec.args, {cwd: ctx.workingDirectory,- stdio: ["pipe", "pipe", "pipe"],env: spawnEnv,shell: spec.shell,});diff --git a/src/daemon/src/drivers/opencode.ts b/src/daemon/src/drivers/opencode.tsindex 6439cd8a0..8f4215f96 100644--- a/src/daemon/src/drivers/opencode.ts+++ b/src/daemon/src/drivers/opencode.ts@@ -8,11 +8,11 @@* writes into the workdir (OpenCode auto-reads it from cwd); the user message* is the trailing `-- <prompt>` positional.*/-import { spawn } from "child_process";import type { Driver, LaunchConfig, LaunchContext, ParsedEvent, SpawnResult } from "../types.js";import { prepareCliTransport, buildCliTransportSystemPrompt } from "./cliTransport.js";import { probeCliRuntime, resolveSpawnSpec } from "./probe.js";import { resolveLaunchFieldsOrDefault } from "../runtimeConfig.js";+import { spawnAgentProcess } from "../runtime/killTree.js";export class OpenCodeDriver implements Driver {readonly id = "opencode";@@ -60,9 +60,8 @@ export class OpenCodeDriver implements Driver {// Cross-platform spawn: on Windows the opencode entry is often a `.cmd`// shim, which `child_process.spawn` can't exec without a shell.const spec = resolveSpawnSpec("opencode", args);- const proc = spawn(spec.command, spec.args, {+ const proc = spawnAgentProcess(spec.command, spec.args, {cwd: ctx.workingDirectory,- stdio: ["pipe", "pipe", "pipe"],env: spawnEnv,shell: spec.shell,});diff --git a/src/daemon/src/runtime/killTree.test.ts b/src/daemon/src/runtime/killTree.test.tsnew file mode 100644index 000000000..d5e0b6cdf--- /dev/null+++ b/src/daemon/src/runtime/killTree.test.ts@@ -0,0 +1,134 @@+import { describe, it, expect, afterEach } from "vitest";+import { spawn, type ChildProcess } from "child_process";+import { spawnAgentProcess, killProcessTree, isAlive } from "./killTree.js";++/**+ * These tests spawn real child processes rather than mocking+ * `child_process` — the bug this file guards against (silent no-op kill on+ * a non-detached child, see plans/fix-daemon-agent-process-kill.md) is a+ * real OS process-group interaction that a mock would trivially hide.+ */++const spawned: ChildProcess[] = [];++/** A child that just idles until signaled. */+function spawnIdleChild(opts: { detached?: boolean } = {}): ChildProcess {+ const proc = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {+ stdio: "ignore",+ detached: opts.detached,+ });+ spawned.push(proc);+ return proc;+}++/** A child that installs a no-op SIGTERM handler, so only SIGKILL kills it. */+function spawnSigtermImmuneChild(opts: { detached?: boolean } = {}): ChildProcess {+ const proc = spawn(+ process.execPath,+ ["-e", "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)"],+ { stdio: "ignore", detached: opts.detached },+ );+ spawned.push(proc);+ return proc;+}++afterEach(() => {+ // Belt-and-suspenders cleanup in case a test fails before its own kill.+ for (const proc of spawned.splice(0)) {+ if (proc.pid && isAlive(proc.pid)) {+ try {+ process.kill(proc.pid, "SIGKILL");+ } catch {+ // already dead+ }+ }+ }+});++describe("killProcessTree", () => {+ it("kills a detached child via the process-group signal", async () => {+ const proc = spawnAgentProcess(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {+ cwd: process.cwd(),+ env: process.env,+ });+ spawned.push(proc);+ await new Promise((r) => proc.once("spawn", r));+ expect(isAlive(proc.pid!)).toBe(true);++ await killProcessTree(proc.pid!, { graceMs: 1000 });++ expect(isAlive(proc.pid!)).toBe(false);+ });++ it("still kills a NON-detached child — regression test for the ESRCH-treated-as-dead bug", async () => {+ // Deliberately spawned WITHOUT detached, reproducing every driver's+ // pre-fix spawn() call. `process.kill(-pid, sig)` throws ESRCH here+ // because the child isn't its own process-group leader; killProcessTree+ // must fall back to signaling the pid directly instead of assuming+ // ESRCH means "already dead".+ const proc = spawnIdleChild({ detached: false });+ await new Promise((r) => proc.once("spawn", r));+ expect(isAlive(proc.pid!)).toBe(true);++ await killProcessTree(proc.pid!, { graceMs: 1000 });++ expect(isAlive(proc.pid!)).toBe(false);+ });++ it("resolves immediately for an already-dead pid without throwing", async () => {+ const proc = spawnIdleChild({ detached: false });+ await new Promise((r) => proc.once("spawn", r));+ const pid = proc.pid!;+ process.kill(pid, "SIGKILL");+ await new Promise((r) => proc.once("exit", r));+ expect(isAlive(pid)).toBe(false);++ await expect(killProcessTree(pid, { graceMs: 1000 })).resolves.toBeUndefined();+ });++ it("no-ops on an invalid pid (0 or negative)", async () => {+ await expect(killProcessTree(0)).resolves.toBeUndefined();+ await expect(killProcessTree(-1)).resolves.toBeUndefined();+ });++ it("escalates to SIGKILL after graceMs when the child ignores SIGTERM", async () => {+ const proc = spawnSigtermImmuneChild({ detached: false });+ await new Promise((r) => proc.once("spawn", r));+ expect(isAlive(proc.pid!)).toBe(true);++ await killProcessTree(proc.pid!, { graceMs: 300 });++ expect(isAlive(proc.pid!)).toBe(false);+ });+});++describe("spawnAgentProcess", () => {+ it("spawns detached on POSIX so killProcessTree's group signal has a group to hit", async () => {+ if (process.platform === "win32") return;+ const proc = spawnAgentProcess(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {+ cwd: process.cwd(),+ env: process.env,+ });+ spawned.push(proc);+ await new Promise((r) => proc.once("spawn", r));++ // A detached child is its own process-group leader: pgid === pid.+ const { execFileSync } = await import("child_process");+ const out = execFileSync("ps", ["-o", "pgid=", "-p", String(proc.pid)], { encoding: "utf8" }).trim();+ expect(Number(out)).toBe(proc.pid);++ process.kill(proc.pid!, "SIGKILL");+ });++ it("pipes stdio and forwards cwd/env/shell", () => {+ const proc = spawnAgentProcess(process.execPath, ["-e", "process.exit(0)"], {+ cwd: process.cwd(),+ env: { ...process.env, FOO: "bar" },+ shell: false,+ });+ spawned.push(proc);+ expect(proc.stdin).not.toBeNull();+ expect(proc.stdout).not.toBeNull();+ expect(proc.stderr).not.toBeNull();+ });+});diff --git a/src/daemon/src/runtime/killTree.ts b/src/daemon/src/runtime/killTree.tsindex ed587795c..d22eb8e98 100644--- a/src/daemon/src/runtime/killTree.ts+++ b/src/daemon/src/runtime/killTree.ts@@ -1,18 +1,51 @@/**- * Process-tree termination with SIGKILL escalation.+ * Agent process spawn + process-tree termination with SIGKILL escalation.*- * Agent CLIs are spawned detached on POSIX, so each becomes the leader of its- * own process group (pgid = pid). Signalling the negative pid reaches the whole- * group — the CLI plus any MCP servers / tool subprocesses it spawned — instead- * of just the leader. A plain positive-pid SIGTERM leaves grandchildren orphaned.+ * These two live together on purpose: they're opposite ends of the SAME+ * contract. `spawnAgentProcess` is the ONLY way a driver may start an agent+ * CLI — it always spawns `detached` on POSIX, making the child the leader of+ * its own process group (pgid = pid). That's what lets `killProcessTree`+ * signal the negative pid to reach the whole group — the CLI plus any MCP+ * servers / tool subprocesses it spawns — instead of just the leader, which+ * would otherwise leave grandchildren orphaned.+ *+ * Driver files must NOT call `child_process.spawn` directly for the agent CLI+ * — always go through `spawnAgentProcess` here, so the detached contract+ * can't be silently skipped by a new (or edited) driver.** SIGTERM is a request; after a grace window we escalate to SIGKILL.*/+import { spawn, type ChildProcess } from "child_process";const POLL_MS = 100;const DEFAULT_GRACE_MS = 2000;const isPosix = process.platform !== "win32";+export interface AgentSpawnOptions {+ cwd: string;+ env: NodeJS.ProcessEnv;+ /** Run through a shell — needed on Windows for `.cmd`/`.bat` shims. */+ shell?: boolean;+}++/**+ * The only sanctioned way to spawn an agent CLI child process. Always pipes+ * stdio and (on POSIX) spawns `detached` so the child becomes its own+ * process-group leader — required for `killProcessTree`'s group signal to+ * actually reach it (and its grandchildren) instead of silently no-oping.+ * See the module doc comment above for why this must be the single spawn+ * entry point rather than each driver calling `child_process.spawn` itself.+ */+export function spawnAgentProcess(command: string, args: string[], opts: AgentSpawnOptions): ChildProcess {+ return spawn(command, args, {+ cwd: opts.cwd,+ stdio: ["pipe", "pipe", "pipe"],+ env: opts.env,+ shell: opts.shell ?? false,+ detached: isPosix,+ });+}+export function isAlive(pid: number): boolean {try {process.kill(pid, 0);@@ -22,14 +55,24 @@ export function isAlive(pid: number): boolean {}}+/**+ * Best-effort group signal, ALWAYS followed by a direct pid signal —+ * regardless of whether the group signal succeeded, threw `ESRCH` (no such+ * process group — e.g. the child wasn't spawned detached), or threw anything+ * else. A group signal failure must never be mistaken for "the pid is dead":+ * that conflates two unrelated failure semantics and was the root cause of a+ * bug where stopped agents kept running forever (see+ * plans/fix-daemon-agent-process-kill.md). Signaling an already-dead pid is+ * safe — it just throws ESRCH too, caught and ignored below.+ */function signalTree(pid: number, signal: NodeJS.Signals): void {if (isPosix) {try {process.kill(-pid, signal);- return;- } catch (e) {- const code = (e as NodeJS.ErrnoException)?.code;- if (code === "ESRCH") return;+ } catch {+ // Most commonly ESRCH (no such process group — not detached, or+ // already gone), but any failure here falls through the same way:+ // never treat it as proof the pid itself is dead.}}try {