Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

Kernels exit when their owner dies (#1559)

Sebastian Mülleraddfc23ffcff
diff --git a/packages/coding-agent/.changes/eng-5310.md b/packages/coding-agent/.changes/eng-5310.md
new file mode 100644
index 0000000000..35e23911e5
--- /dev/null
+++ b/packages/coding-agent/.changes/eng-5310.md
@@ -0,0 +1,2 @@
+- Fixed IPython kernels and forkserver processes outliving their owner after a hard crash: kernels now arm ipykernel's parent-death poller via JPY_PARENT_PID, the forkserver watches its parent pid, and both pids are registered in the orphan process journal for supervisor recovery.
+- Fixed a pid-reuse race for forked IPython kernels: signaling and liveness now go through the forkserver (the kernels' parent) instead of raw pid operations from Node, and the orphan journal's inactive record is only written on a confirmed kill outcome.
diff --git a/packages/coding-agent/src/core/kernel/fork-server-script.ts b/packages/coding-agent/src/core/kernel/fork-server-script.ts
index d08f8e0139..a6e286d0f9 100644
--- a/packages/coding-agent/src/core/kernel/fork-server-script.ts
+++ b/packages/coding-agent/src/core/kernel/fork-server-script.ts
@@ -5,13 +5,21 @@
//
// Embedded as a string rather than shipped as a package asset so it can never be
// missing from a release layout (see the built-in-skills packaging gap). Run via
-// `python -c <this> <control-socket-path>`.
+// `python -c <this> <control-socket-path> [<history-bound>]`.
//
// Protocol (newline-delimited JSON over the unix socket, forkserver is the client):
// -> { "id": <n>, "connectionPath": "<abs path>" } spawn request from Node
+// -> { "id": <n>, "kill": <fork-id>, "signal": "TERM"|"KILL" } kill a forked child
+// -> { "id": <n>, "alive": <fork-id> } liveness query for a child
// <- { "type": "ready" } once, after imports finish
// <- { "id": <n>, "pid": <pid> } fork succeeded
// <- { "id": <n>, "error": "<message>" } fork failed
+// <- { "id": <n>, "outcome": "signaled"|"already-exited"|"unknown-pid" } kill reply
+// <- { "id": <n>, "alive": true|false } alive reply
+//
+// Kill/alive are keyed by the never-reused fork request id, so a handle can only
+// name the one child incarnation it forked (a recycled pid can't alias), and the
+// forkserver signals only its own un-reaped children — POSIX-race-free.
export const FORK_SERVER_SCRIPT = String.raw`
import gc
import json
@@ -19,6 +27,15 @@ import os
import signal
import socket
import sys
+import threading
+import time
+
+# fork id -> [pid, alive]; _pid_to_id holds un-reaped, un-evicted children only.
+# Only the main thread mutates these (the SIGCHLD handler runs between its bytecodes).
+_children = {}
+_pid_to_id = {}
+# FIFO history bound; kill/alive for an evicted id fail closed.
+_history_bound = 4096
def _reap_children(*_args):
@@ -30,10 +47,22 @@ def _reap_children(*_args):
pid, _status = os.waitpid(-1, os.WNOHANG)
if pid == 0:
break
+ child_id = _pid_to_id.pop(pid, None)
+ if child_id is not None:
+ _children[child_id][1] = False
except ChildProcessError:
pass
+def _watch_parent(original_ppid):
+ # A SIGKILLed owner can't close our socket while a forked child still holds the
+ # fd; poll ppid so the forkserver dies with its owner regardless.
+ while True:
+ if os.getppid() != original_ppid:
+ os._exit(1)
+ time.sleep(1.0)
+
+
def _import_template():
# Everything a kernel touches at import time. Paid once; shared COW by children.
import IPython # noqa: F401
@@ -71,7 +100,12 @@ def _run_child(connection_path, cwd, env):
# instance (and, critically, a jupyter_client Session created in *this* pid;
# a Session inherited from the template silently drops messages via check_pid).
IPKernelApp.clear_instance()
- app = IPKernelApp.instance(connection_file=connection_path)
+ # Watch our real parent (the forkserver): ipykernel distrusts a handle that
+ # differs from getppid(), and its pid-1 fallback breaks under subreapers.
+ app = IPKernelApp.instance(
+ connection_file=connection_path,
+ parent_handle=os.getppid(),
+ )
# initialize() binds the 5 ZMQ ports, writes the resolved ports back into
# connection.json, and starts the heartbeat thread + ioloop — all post-fork,
# so no thread/loop/socket is ever inherited across the fork boundary.
@@ -84,8 +118,18 @@ def _serve(control_path):
sock.connect(control_path)
control_fd = sock.fileno()
+ # Block SIGCHLD before spawning the watcher so it inherits the blocked mask;
+ # otherwise the kernel could deliver SIGCHLD there while the main thread is masked.
+ signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGCHLD})
+
+ # Compare against the original ppid (not ==1) so this holds under subreapers.
+ # Threads aren't inherited across fork, so children never carry this watcher.
+ threading.Thread(target=_watch_parent, args=(os.getppid(),), daemon=True).start()
+
# Reap forked kernels as they exit, independent of the request loop.
signal.signal(signal.SIGCHLD, _reap_children)
+ # Unblock after handler install; the main thread's mask is now authoritative.
+ signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGCHLD})
_import_template()
# Freeze the heap so the cyclic GC doesn't write to (and thus COW-copy) the
@@ -109,19 +153,57 @@ def _serve(control_path):
except ValueError:
continue
req_id = req.get("id")
+
+ kill_id = req.get("kill")
+ if kill_id is not None:
+ # Blocked across check+kill: an alive entry then means an un-reaped
+ # own child (zombie-pinned pid), so os.kill is race-free.
+ signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGCHLD})
+ try:
+ entry = _children.get(kill_id)
+ if entry is None:
+ outcome = "unknown-pid"
+ elif not entry[1]:
+ outcome = "already-exited"
+ else:
+ sig = signal.SIGKILL if req.get("signal") == "KILL" else signal.SIGTERM
+ try:
+ os.kill(entry[0], sig)
+ outcome = "signaled"
+ except ProcessLookupError:
+ outcome = "already-exited"
+ finally:
+ signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGCHLD})
+ f.write(json.dumps({"id": req_id, "outcome": outcome}).encode() + b"\n")
+ f.flush()
+ continue
+
+ alive_id = req.get("alive")
+ if alive_id is not None:
+ entry = _children.get(alive_id)
+ f.write(json.dumps({"id": req_id, "alive": bool(entry and entry[1])}).encode() + b"\n")
+ f.flush()
+ continue
+
connection_path = req.get("connectionPath")
cwd = req.get("cwd")
env = req.get("env")
+ # Blocked across fork+bookkeeping: a fast-exiting child could otherwise
+ # be reaped before its registry entry exists, recording a stale pid as alive.
+ signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGCHLD})
try:
pid = os.fork()
except OSError as exc:
+ signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGCHLD})
f.write(json.dumps({"id": req_id, "error": str(exc)}).encode() + b"\n")
f.flush()
continue
if pid == 0:
- # Child: shed every inherited fd tied to the control channel, then run.
+ # The child inherits the blocked mask; the kernel needs SIGCHLD.
+ signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGCHLD})
+ # Shed every inherited fd tied to the control channel, then run.
try:
sock.close()
f.close()
@@ -139,10 +221,23 @@ def _serve(control_path):
os._exit(0)
# Parent: stay pristine (no loop/threads/ZMQ ever) so the next fork is clean.
+ _children[req_id] = [pid, True]
+ _pid_to_id[pid] = req_id
+ # FIFO-evict only exited entries: dropping a live child would make its
+ # liveness read false and its kill unroutable — the exact leak this
+ # forkserver exists to prevent. Live entries are bounded by real kernels.
+ if len(_children) > _history_bound:
+ for evicted_id in [i for i, e in _children.items() if not e[1]]:
+ if len(_children) <= _history_bound:
+ break
+ _children.pop(evicted_id)
+ signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGCHLD})
f.write(json.dumps({"id": req_id, "pid": pid}).encode() + b"\n")
f.flush()
if __name__ == "__main__":
+ if len(sys.argv) > 2:
+ _history_bound = int(sys.argv[2])
_serve(sys.argv[1])
`;
diff --git a/packages/coding-agent/src/core/kernel/fork-server.ts b/packages/coding-agent/src/core/kernel/fork-server.ts
index d5b5423ac2..a6c1240d74 100644
--- a/packages/coding-agent/src/core/kernel/fork-server.ts
+++ b/packages/coding-agent/src/core/kernel/fork-server.ts
@@ -12,6 +12,7 @@ import { createServer, type Server, type Socket } from "node:net";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { registerSessionResourceCleanup } from "@earendil-works/pi-ai";
+import { recordOrphanProcessState } from "../orphan-process-journal.js";
import { FORK_SERVER_SCRIPT } from "./fork-server-script.js";
const READY_TIMEOUT_MS = 30_000;
@@ -33,9 +34,13 @@ function affectsInterpreterStartup(key: string): boolean {
}
export class ForkServerUnavailable extends Error {
- constructor(message: string) {
+ /** True for a request timeout: the forkserver may just be stalled (e.g. mid-fork), not gone. */
+ readonly timedOut: boolean;
+
+ constructor(message: string, opts?: { timedOut?: boolean }) {
super(message);
this.name = "ForkServerUnavailable";
+ this.timedOut = opts?.timedOut ?? false;
}
}
@@ -51,6 +56,8 @@ export function isForkServerEnabled(): boolean {
// forked child, so every kernel for a given python shares ONE template.
interface ForkServerParams {
python: string;
+ // Registry-history bound override (argv[2] to the script); tests only.
+ historyBound?: number;
}
interface SpawnParams {
@@ -61,13 +68,37 @@ interface SpawnParams {
env?: Record<string, string | undefined>;
}
-type PendingSpawn = {
- resolve: (pid: number) => void;
+export type ForkedKernelKillSignal = "TERM" | "KILL";
+export type ForkedKernelKillOutcome = "signaled" | "already-exited" | "unknown-pid";
+
+/**
+ * Handle to a forkserver-forked kernel: signaling/liveness go through the
+ * forkserver by fork request id, never process.kill (see the protocol header
+ * in fork-server-script.ts). `pid` exists for journal writes only.
+ */
+export interface ForkedKernelHandle {
+ readonly pid: number;
+ kill(signal: ForkedKernelKillSignal): Promise<ForkedKernelKillOutcome>;
+ isAlive(): Promise<boolean>;
+}
+
+interface ForkServerReply {
+ id: number;
+ pid?: number;
+ error?: string;
+ outcome?: string;
+ alive?: boolean;
+}
+
+type PendingRequest = {
+ resolve: (msg: ForkServerReply) => void;
reject: (err: Error) => void;
timer: ReturnType<typeof setTimeout>;
};
-class ForkServer {
+// Exported for protocol-level tests (which construct it directly, bypassing the
+// isForkServerEnabled platform gate); production entry stays forkKernel().
+export class ForkServer {
private readonly params: ForkServerParams;
// The env the template process was launched with, snapshotted at construction so
// it exactly matches what the template's interpreter imported with. The
@@ -85,7 +116,7 @@ class ForkServer {
// child's import/startup traceback lands here; surfaced in error messages.
private stderrTail = "";
private nextId = 1;
- private readonly pending = new Map<number, PendingSpawn>();
+ private readonly pending = new Map<number, PendingRequest>();
// Request ids whose caller timed out; a late pid reply for one is an orphan to kill.
private readonly abandoned = new Set<number>();
private dead = false;
@@ -172,11 +203,14 @@ class ForkServer {
server.listen(socketPath, () => {
// The template only imports; its own cwd/env are irrelevant since each
// forked child applies the per-kernel cwd/env itself. Inherit the daemon's.
- const proc = spawn(this.params.python, ["-c", FORK_SERVER_SCRIPT, socketPath], {
+ const args = ["-c", FORK_SERVER_SCRIPT, socketPath];
+ if (this.params.historyBound !== undefined) args.push(String(this.params.historyBound));
+ const proc = spawn(this.params.python, args, {
env: this.launchEnv,
stdio: ["ignore", "ignore", "pipe"],
});
this.proc = proc;
+ if (proc.pid !== undefined) recordOrphanProcessState(proc.pid, true);
proc.stderr?.on("data", (buf: Buffer) => {
this.stderrTail = `${this.stderrTail}${buf.toString()}`.slice(-STDERR_TAIL_MAX);
});
@@ -194,7 +228,7 @@ class ForkServer {
const line = this.buffer.slice(0, idx);
this.buffer = this.buffer.slice(idx + 1);
if (!line.trim()) continue;
- let msg: { type?: string; id?: number; pid?: number; error?: string };
+ let msg: ForkServerReply & { type?: string };
try {
msg = JSON.parse(line);
} catch {
@@ -207,46 +241,73 @@ class ForkServer {
if (typeof msg.id !== "number") continue;
const p = this.pending.get(msg.id);
if (!p) {
- // A pid for a request the caller already abandoned (timed out): the
- // fork succeeded but nobody owns it, so kill the orphan here.
+ // A fork the caller already abandoned (timed out): nobody owns it,
+ // so have the forkserver kill it.
if (this.abandoned.delete(msg.id) && typeof msg.pid === "number") {
- try {
- process.kill(msg.pid, "SIGTERM");
- } catch {
- // Orphan already exited.
- }
+ void this.killChild(msg.id, "TERM").catch(() => {});
}
continue;
}
this.pending.delete(msg.id);
clearTimeout(p.timer);
- if (typeof msg.pid === "number") {
- p.resolve(msg.pid);
- } else {
- p.reject(new ForkServerUnavailable(this.withStderr(msg.error ?? "forkserver fork failed")));
- }
+ p.resolve(msg);
}
}
- async spawnKernel(spawn: SpawnParams): Promise<number> {
- await this.ensureReady();
- if (this.dead || !this.conn) throw new ForkServerUnavailable("forkserver connection unavailable");
+ private request(payload: Record<string, unknown>, opts?: { abandonOnTimeout?: boolean }): Promise<ForkServerReply> {
+ if (this.dead || !this.conn) {
+ return Promise.reject(new ForkServerUnavailable("forkserver connection unavailable"));
+ }
const id = this.nextId++;
- return new Promise<number>((resolve, reject) => {
+ return new Promise<ForkServerReply>((resolve, reject) => {
const timer = setTimeout(() => {
this.pending.delete(id);
- // The fork may still land after we give up; remember the id so its late
+ // A spawn may still land after we give up; remember the id so its late
// pid reply gets the orphan killed instead of leaked.
- this.abandoned.add(id);
- reject(new ForkServerUnavailable(`forkserver spawn timed out after ${SPAWN_TIMEOUT_MS}ms`));
+ if (opts?.abandonOnTimeout) this.abandoned.add(id);
+ reject(
+ new ForkServerUnavailable(`forkserver request timed out after ${SPAWN_TIMEOUT_MS}ms`, {
+ timedOut: true,
+ }),
+ );
}, SPAWN_TIMEOUT_MS);
this.pending.set(id, { resolve, reject, timer });
- this.conn?.write(
- `${JSON.stringify({ id, connectionPath: spawn.connectionPath, cwd: spawn.cwd, env: spawn.env })}\n`,
- );
+ this.conn?.write(`${JSON.stringify({ id, ...payload })}\n`);
});
}
+ async spawnKernel(spawn: SpawnParams): Promise<ForkedKernelHandle> {
+ await this.ensureReady();
+ const msg = await this.request(
+ { connectionPath: spawn.connectionPath, cwd: spawn.cwd, env: spawn.env },
+ { abandonOnTimeout: true },
+ );
+ if (typeof msg.pid !== "number") {
+ throw new ForkServerUnavailable(this.withStderr(msg.error ?? "forkserver fork failed"));
+ }
+ const pid = msg.pid;
+ const forkId = msg.id;
+ return {
+ pid,
+ kill: (signal) => this.killChild(forkId, signal),
+ isAlive: () => this.isChildAlive(forkId),
+ };
+ }
+
+ async killChild(forkId: number, signal: ForkedKernelKillSignal): Promise<ForkedKernelKillOutcome> {
+ const msg = await this.request({ kill: forkId, signal });
+ const outcome = msg.outcome;
+ if (outcome === "signaled" || outcome === "already-exited" || outcome === "unknown-pid") {
+ return outcome;
+ }
+ throw new ForkServerUnavailable(`forkserver kill reply malformed: ${JSON.stringify(msg)}`);
+ }
+
+ async isChildAlive(forkId: number): Promise<boolean> {
+ const msg = await this.request({ alive: forkId });
+ return msg.alive === true;
+ }
+
private withStderr(message: string): string {
const tail = this.stderrTail.trim();
return tail ? `${message}\nforkserver stderr:\n${tail}` : message;
@@ -284,10 +345,18 @@ class ForkServer {
} catch {
// Already closed.
}
- try {
- this.proc?.kill("SIGTERM");
- } catch {
- // Already exited.
+ const proc = this.proc;
+ if (proc) {
+ // Inactive only when the pid is soundly ours at write time (handle saw the
+ // exit, or handle-based kill delivered); otherwise leave the record active.
+ const observedExit = proc.exitCode !== null || proc.signalCode !== null;
+ let delivered = false;
+ try {
+ delivered = proc.kill("SIGTERM");
+ } catch {
+ // Already exited.
+ }
+ if (proc.pid !== undefined && (observedExit || delivered)) recordOrphanProcessState(proc.pid, false);
}
if (this.socketDir) {
try {
@@ -330,9 +399,9 @@ function registerForkServerCleanupOnce(): void {
* Fork a kernel onto `spawn.connectionPath` from the shared template for this
* interpreter, applying `spawn.cwd`/`spawn.env` in the forked child. Throws
* ForkServerUnavailable if forking is disabled or fails — callers fall back to
- * direct spawn. Returns the forked child's pid (owned/killed by the caller).
+ * direct spawn. Returns a handle whose kill/liveness go through the forkserver.
*/
-export async function forkKernel(python: string, spawn: SpawnParams): Promise<number> {
+export async function forkKernel(python: string, spawn: SpawnParams): Promise<ForkedKernelHandle> {
if (!isForkServerEnabled()) throw new ForkServerUnavailable("forkserver disabled");
registerForkServerCleanupOnce();
const key = keyFor({ python });
diff --git a/packages/coding-agent/src/core/kernel/index.ts b/packages/coding-agent/src/core/kernel/index.ts
index 1f5af4f1b3..e12cde055d 100644
--- a/packages/coding-agent/src/core/kernel/index.ts
+++ b/packages/coding-agent/src/core/kernel/index.ts
@@ -7,8 +7,9 @@ import { setTimeout as sleep } from "node:timers/promises";
import { registerSessionResourceCleanup } from "@earendil-works/pi-ai";
import { v4 as uuid } from "uuid";
import { Dealer, Subscriber } from "zeromq";
+import { recordOrphanProcessState } from "../orphan-process-journal.js";
import { ensureKernelPython, type KernelBootstrapProgressHandler, type KernelPythonSkill } from "./bootstrap.js";
-import { ForkServerUnavailable, forkKernel, isForkServerEnabled } from "./fork-server.js";
+import { type ForkedKernelHandle, ForkServerUnavailable, forkKernel, isForkServerEnabled } from "./fork-server.js";
import {
buildListNamesCode,
buildRestoreCode,
@@ -594,11 +595,12 @@ export class KernelManager {
private readonly commTargets = new Map<string, string>();
private readonly handledHostRequestCommIds = new Set<string>();
private kernel?: ChildProcess;
- // Set instead of `kernel` when the kernel was forked from the forkserver: it is
- // not a direct child, so it has no ChildProcess handle and is killed by pid.
- private kernelPid?: number;
- /** Polls a forked kernel's pid for death (no "exit" event on a non-child). */
+ // Set instead of `kernel` for forkserver-forked kernels (not our child):
+ // signaling/liveness go through the forkserver, never process.kill.
+ private forkedKernel?: ForkedKernelHandle;
+ /** Polls a forked kernel for death (no "exit" event on a non-child). */
private forkedLivenessTimer?: ReturnType<typeof globalThis.setInterval>;
+ private forkedLivenessProbeInFlight = false;
private shell?: Dealer;
private iopub?: Subscriber;
private control?: Dealer;
@@ -619,6 +621,8 @@ export class KernelManager {
private lastCellCode?: string;
private readonly inFlightHostRequests = new Set<Promise<void>>();
private state: "idle" | "starting" | "running" | "shutdown" = "idle";
+ /** Bumped by every teardown so a stale in-flight doStart can never touch a newer kernel. */
+ private startGeneration = 0;
/** Memoized so concurrent callers all await the same in-flight startup. */
private startPromise?: Promise<void>;
/** Pending debounced auto-snapshot, if one has been scheduled. */
@@ -650,16 +654,19 @@ export class KernelManager {
throw createKernelStartupAbortError();
}
if (!this.startPromise) {
- this.startPromise = this.doStart({ onBootstrapProgress: options.onBootstrapProgress }).catch((error) => {
- this.startPromise = undefined;
+ const startPromise = this.doStart({ onBootstrapProgress: options.onBootstrapProgress }).catch((error) => {
+ // Only clear our own memoization: a stale start must not evict a newer one.
+ if (this.startPromise === startPromise) this.startPromise = undefined;
throw error;
});
+ this.startPromise = startPromise;
}
return raceStartupWithAbort(this.startPromise, options.signal);
}
private async doStart(startOptions: KernelStartOptions): Promise<void> {
if (this.state !== "idle") return;
+ const generation = ++this.startGeneration;
this.state = "starting";
installSignalHandlersOnce();
// Tracked from the moment startup begins so session cleanup and signal
@@ -674,8 +681,10 @@ export class KernelManager {
pythonSkills: this.options.pythonSkills,
onProgress: startOptions.onBootstrapProgress,
}));
+ if (this.startStale(generation)) throw new Error("Kernel start superseded");
this.options.python = python;
} catch (error) {
+ if (this.startStale(generation)) throw error; // never touch a newer start's state
liveKernels.delete(this);
if ((this.state as string) !== "shutdown") this.state = "idle";
throw error;
@@ -694,19 +703,26 @@ export class KernelManager {
let forked = false;
if (isForkServerEnabled()) {
try {
- this.kernelPid = await forkKernel(python, {
+ const handle = await forkKernel(python, {
connectionPath: connection.path,
cwd: this.options.cwd,
- // Match the direct-spawn env exactly: merge the current host env with
- // the per-kernel overrides, applied fresh in the child (the template's
- // inherited env snapshot may be stale by fork time).
- env: this.options.env ? { ...process.env, ...this.options.env } : { ...process.env },
+ // Applied fresh in the child (the template's env snapshot may be stale).
+ // No JPY_PARENT_PID: forked children watch the forkserver by getppid().
+ env: { ...process.env, ...this.options.env },
});
+ if (this.startStale(generation)) {
+ // Nobody owns this kernel; the protocol kill is id-keyed and safe.
+ void handle.kill("TERM").catch(() => {});
+ throw new Error("Kernel start superseded");
+ }
+ this.forkedKernel = handle;
+ recordOrphanProcessState(handle.pid, true);
forked = true;
} catch (err) {
+ if (this.startStale(generation)) throw err; // never touch a newer start's state
if (!(err instanceof ForkServerUnavailable)) throw err;
this.appendKernelDiagnostic(`forkserver unavailable, spawning directly: ${err.message}`);
- this.kernelPid = undefined;
+ this.forkedKernel = undefined;
// A fork request that times out or loses its pid reply may still have
// forked a child that binds the ports in this connection file. Mint a
// fresh connection for the direct spawn so a possible orphan can never
@@ -725,10 +741,12 @@ export class KernelManager {
if (!forked) {
const kernel = spawn(python, ["-m", "ipykernel_launcher", "-f", connection.path], {
cwd: this.options.cwd,
- env: this.options.env ? { ...process.env, ...this.options.env } : process.env,
+ // ipykernel's parent poller exits the kernel if this pid dies (covers SIGKILL of the owner).
+ env: { ...process.env, ...this.options.env, JPY_PARENT_PID: String(process.pid) },
stdio: ["ignore", "pipe", "pipe"],
});
this.kernel = kernel;
+ if (kernel.pid !== undefined) recordOrphanProcessState(kernel.pid, true);
kernel.stderr?.on("data", (buf: Buffer) => {
const s = buf.toString();
@@ -758,11 +776,14 @@ export class KernelManager {
let conn: ConnectionInfo;
try {
conn = await this.waitForResolvedConnection(connectionPath);
+ if (this.startStale(generation)) throw new Error("Kernel start superseded");
this.connection = conn;
} catch (e) {
+ if (this.startStale(generation)) throw e; // never tear down a newer start's kernel
const canRetryStartup = (this.state as string) !== "shutdown";
- await this.shutdown();
- if (canRetryStartup) this.state = "idle";
+ // Only the call that performed the cleanup may resurrect to idle; a
+ // concurrent kill()/teardown owns the state otherwise.
+ if ((await this.shutdown()) && canRetryStartup) this.state = "idle";
throw e;
}
@@ -777,14 +798,18 @@ export class KernelManager {
// ZMQ PUB/SUB slow-joiner: give the subscription a brief chance to reach the kernel before first execute.
await sleep(IOPUB_SUBSCRIBE_DELAY_MS);
+ if (this.startStale(generation)) throw new Error("Kernel start superseded");
this.startIopubPump();
try {
await this.probeReady();
+ if (this.startStale(generation)) throw new Error("Kernel start superseded");
} catch (e) {
+ if (this.startStale(generation)) throw e; // never tear down a newer start's kernel
const canRetryStartup = (this.state as string) !== "shutdown";
- await this.shutdown();
- if (canRetryStartup) this.state = "idle";
+ // Only the call that performed the cleanup may resurrect to idle; a
+ // concurrent kill()/teardown owns the state otherwise.
+ if ((await this.shutdown()) && canRetryStartup) this.state = "idle";
throw e;
}
@@ -792,40 +817,64 @@ export class KernelManager {
this.startForkedLivenessMonitor();
}
- // A forked kernel isn't a direct child, so no "exit" fires when it dies. Poll its
- // pid so a mid-run death tears down like the direct-spawn exit handler: mark
- // shutdown, drop from liveKernels, and reject any in-flight execution.
+ /** True when a teardown (or newer start) superseded the start that captured `generation`. */
+ private startStale(generation: number): boolean {
+ return generation !== this.startGeneration;
+ }
+
+ // No "exit" event fires for a non-child; poll the forkserver so a mid-run
+ // death tears down like the direct-spawn exit handler.
private startForkedLivenessMonitor(): void {
- if (this.kernelPid === undefined) return;
+ if (!this.forkedKernel) return;
this.forkedLivenessTimer = globalThis.setInterval(() => {
- if (this.state !== "running") return;
- if (!this.forkedKernelDied()) return;
- this.appendKernelDiagnostic("forked kernel exited unexpectedly");
- this.state = "shutdown";
- liveKernels.delete(this);
- this.cleanupResources();
+ void this.checkForkedKernelDeath();
}, FORKED_LIVENESS_POLL_MS);
this.forkedLivenessTimer.unref?.();
}
- // A forked kernel is not a direct child, so it emits no "exit" event; poll its
- // pid so a dead child fails fast instead of burning the full resolve timeout.
- private forkedKernelDied(): boolean {
- if (this.kernelPid === undefined) return false;
+ private async checkForkedKernelDeath(): Promise<void> {
+ if (this.state !== "running" || this.forkedLivenessProbeInFlight) return;
+ const probed = this.forkedKernel;
+ this.forkedLivenessProbeInFlight = true;
try {
- process.kill(this.kernelPid, 0);
- return false;
+ if (!(await this.forkedKernelDead(probed))) return;
+ } finally {
+ this.forkedLivenessProbeInFlight = false;
+ }
+ // Re-check after the await: teardown or a restart may have raced this poll.
+ if (this.state !== "running" || this.forkedKernel !== probed) return;
+ this.appendKernelDiagnostic("forked kernel exited unexpectedly");
+ this.state = "shutdown";
+ liveKernels.delete(this);
+ this.cleanupResources();
+ }
+
+ // Liveness from the forkserver's reap table; a pid-0 probe would race reuse.
+ // `timeoutMs` bounds the probe (timeout counts as alive so the caller's own
+ // deadline decides); without it the protocol request timeout applies.
+ private async forkedKernelDead(probed: ForkedKernelHandle | undefined, timeoutMs?: number): Promise<boolean> {
+ if (!probed) return false;
+ try {
+ const alive = probed.isAlive();
+ if (timeoutMs === undefined) return !(await alive);
+ alive.catch(() => {}); // absorb a rejection that lands after the race is lost
+ return !(await Promise.race([alive, sleep(timeoutMs, true, { ref: false })]));
} catch (error) {
- // EPERM means the pid exists but isn't signalable by us — still alive.
- // Only ESRCH (no such process) is genuine death.
- return !(error instanceof Error && (error as NodeJS.ErrnoException).code === "EPERM");
+ // A timeout is unknown liveness, not proven death (the forkserver may just be stalled in a slow fork).
+ if (error instanceof ForkServerUnavailable && error.timedOut) return false;
+ // Forkserver gone: its kernels' parent_handle watchdogs exit them too.
+ return true;
}
}
private async waitForResolvedConnection(connectionPath: string): Promise<ConnectionInfo> {
const startedAt = Date.now();
while (Date.now() - startedAt < PORTS_RESOLVE_TIMEOUT_MS) {
- if ((this.state as string) === "shutdown" || this.forkedKernelDied()) {
+ const remainingBudget = PORTS_RESOLVE_TIMEOUT_MS - (Date.now() - startedAt);
+ if (
+ (this.state as string) === "shutdown" ||
+ (await this.forkedKernelDead(this.forkedKernel, remainingBudget))
+ ) {
const tail = this.kernelStderr.slice(-1024);
throw new Error(`Kernel exited before resolving ports. stderr:\n${tail || "(empty)"}`);
}
@@ -854,7 +903,11 @@ export class KernelManager {
const startedAt = Date.now();
while (Date.now() - startedAt < READY_TIMEOUT_MS) {
- if ((this.state as string) === "shutdown" || this.forkedKernelDied()) {
+ const remainingBudget = READY_TIMEOUT_MS - (Date.now() - startedAt);
+ if (
+ (this.state as string) === "shutdown" ||
+ (await this.forkedKernelDead(this.forkedKernel, remainingBudget))
+ ) {
const tail = this.kernelStderr.slice(-1024);
throw new Error(`Kernel exited during startup. stderr:\n${tail || "(empty)"}`);
}
@@ -1462,6 +1515,7 @@ export class KernelManager {
}
private cleanupResources(killSignal: NodeJS.Signals = "SIGTERM"): void {
+ this.startGeneration++; // any teardown invalidates in-flight starts
this.clearSnapshotTimer();
this.lateSentAgentMessageHandlers.clear();
if (this.forkedLivenessTimer) {
@@ -1478,19 +1532,32 @@ export class KernelManager {
this.control = undefined;
this.iopubPumpPromise = undefined;
this.controlPumpPromise = undefined;
- try {
- if (this.kernel) {
- this.kernel.kill(killSignal);
- } else if (this.kernelPid !== undefined && !this.forkedKernelDied()) {
- // Only signal a forked kernel confirmed still alive: a dead pid may have
- // been recycled by the OS, and a kill would then hit an unrelated process.
- process.kill(this.kernelPid, killSignal);
+ if (this.kernel) {
+ const directPid = this.kernel.pid;
+ let signaled = false;
+ try {
+ signaled = this.kernel.kill(killSignal);
+ } catch {
+ // The kernel has already exited.
}
- } catch {
- // The kernel has already exited.
+ // Same rule as the forked branch below: inactive only when the signal proved the pid still ours.
+ if (directPid !== undefined && signaled) recordOrphanProcessState(directPid, false);
+ } else if (this.forkedKernel) {
+ const forked = this.forkedKernel;
+ // The journal is raw-pid keyed, so inactive is written only on "signaled"
+ // — the one outcome proving the pid still named our un-reaped child. Any
+ // other outcome leaves the record stale-active: the reaper's startId check
+ // neutralizes it, while a wrong inactive write could mask a sibling's
+ // record for a reused pid.
+ void forked
+ .kill(killSignal === "SIGKILL" ? "KILL" : "TERM")
+ .then((outcome) => {
+ if (outcome === "signaled") recordOrphanProcessState(forked.pid, false);
+ })
+ .catch(() => this.appendKernelDiagnostic("forkserver kill unconfirmed; leaving orphan record active"));
}
this.kernel = undefined;
- this.kernelPid = undefined;
+ this.forkedKernel = undefined;
this.connection = undefined;
if (this.tempDir) {
try {
@@ -1510,9 +1577,9 @@ export class KernelManager {
await new Promise<void>((resolve) => kernel.once("exit", () => resolve()));
return;
}
- const pid = this.kernelPid;
- if (pid === undefined) return;
- while (this.kernelPid === pid && !this.forkedKernelDied()) {
+ const forked = this.forkedKernel;
+ if (!forked) return;
+ while (this.forkedKernel === forked && !(await this.forkedKernelDead(forked))) {
await sleep(25);
}
}
@@ -1537,22 +1604,27 @@ export class KernelManager {
}
}
- async shutdown(opts: { snapshot?: boolean } = {}): Promise<void> {
+ /** Resolves true when this call performed the cleanup (false: a concurrent teardown won). */
+ async shutdown(opts: { snapshot?: boolean } = {}): Promise<boolean> {
if (this.state === "shutdown") {
liveKernels.delete(this);
this.cleanupResources();
- return;
+ return true;
}
+ // Captured before any await: teardowns and newer starts bump the counter.
+ const generation = this.startGeneration;
// Best-effort final flush (bounded) before teardown — used by signal handlers
// so a SIGINT/SIGTERM exit doesn't lose work the debounced snapshot hasn't saved.
if (opts.snapshot) {
await this.flushSnapshotForDispose();
+ if (this.startStale(generation)) return false; // superseded mid-flush: the newer owner already cleaned this kernel
}
this.state = "shutdown";
liveKernels.delete(this);
let replyWait: { promise: Promise<void>; cancel: () => void } | undefined;
let shutdownTimer: ReturnType<typeof globalThis.setTimeout> | undefined;
+ let performedCleanup = false;
const shutdownDeadline = new Promise<never>((_resolve, reject) => {
shutdownTimer = globalThis.setTimeout(
() => reject(new Error(`Kernel did not shut down within ${KERNEL_SHUTDOWN_TIMEOUT_MS}ms`)),
@@ -1581,8 +1653,15 @@ export class KernelManager {
} finally {
if (shutdownTimer) globalThis.clearTimeout(shutdownTimer);
replyWait?.cancel();
- this.cleanupResources();
+ // A superseded shutdown must not tear down the newer start's sockets. Ownership is decided
+ // here, before cleanupResources bumps the generation and would misread this call as superseded.
+ if (!this.startStale(generation)) {
+ this.cleanupResources();
+ performedCleanup = true;
+ }
}
+
+ return performedCleanup;
}
async restart(): Promise<void> {
@@ -1733,8 +1812,11 @@ export class KernelManager {
/** Graceful cleanup. Waits briefly for in-flight host request handlers before closing sockets. */
dispose(): Promise<void> {
return (async () => {
+ // Captured before any await: teardowns and newer starts bump the counter.
+ const generation = this.startGeneration;
// Final namespace flush while the kernel is still live (session end / reload).
await this.flushSnapshotForDispose();
+ if (this.startStale(generation)) return; // superseded mid-flush: the newer owner already cleaned this kernel
this.state = "shutdown";
liveKernels.delete(this);
const inFlightHostRequests = [...this.inFlightHostRequests];
@@ -1744,7 +1826,7 @@ export class KernelManager {
await this.waitForHostRequestsToSettle(inFlightHostRequests, HOST_REQUEST_DISPOSE_TIMEOUT_MS);
}
} finally {
- this.cleanupResources();
+ if (!this.startStale(generation)) this.cleanupResources(); // else: superseded, the newer owner already cleaned
}
})();
}
diff --git a/packages/coding-agent/test/kernel-fork-server-protocol.test.ts b/packages/coding-agent/test/kernel-fork-server-protocol.test.ts
new file mode 100644
index 0000000000..3967132779
--- /dev/null
+++ b/packages/coding-agent/test/kernel-fork-server-protocol.test.ts
@@ -0,0 +1,366 @@
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { type ForkedKernelHandle, ForkServer, ForkServerUnavailable } from "../src/core/kernel/fork-server.js";
+import { ORPHAN_PROCESS_JOURNAL_ENV } from "../src/core/orphan-process-journal.js";
+
+// Drives the REAL Python forkserver script through the REAL ForkServer class.
+// Stub Python modules stand in for IPython/ipykernel so the forked child stays
+// alive without a real kernel — fork itself is safe here even on macOS because
+// the child never touches the frameworks that make fork-without-exec unsafe.
+const STUB_KERNELAPP = [
+ "import os",
+ "import time",
+ "",
+ "",
+ "class IPKernelApp:",
+ " @classmethod",
+ " def clear_instance(cls):",
+ " pass",
+ "",
+ " @classmethod",
+ " def instance(cls, **_kwargs):",
+ " return cls()",
+ "",
+ " def initialize(self, _argv):",
+ " pass",
+ "",
+ " def start(self):",
+ " # Env knob: lets tests fork a child that exits immediately (the",
+ " # per-kernel env is applied in the child before start()).",
+ " if os.environ.get('STUB_KERNEL_EXIT'):",
+ " return",
+ " while True:",
+ " time.sleep(1)",
+ "",
+].join("\n");
+
+function writeStubModules(dir: string): void {
+ writeFileSync(join(dir, "IPython.py"), "");
+ writeFileSync(join(dir, "jupyter_client.py"), "");
+ writeFileSync(join(dir, "nest_asyncio.py"), "");
+ const ipykernelDir = join(dir, "ipykernel");
+ mkdirSync(ipykernelDir);
+ writeFileSync(join(ipykernelDir, "__init__.py"), "");
+ writeFileSync(join(ipykernelDir, "kernelapp.py"), STUB_KERNELAPP);
+}
+
+function killQuietly(pid: number | undefined): void {
+ if (pid === undefined) return;
+ try {
+ process.kill(pid, "SIGKILL");
+ } catch {
+ // Already exited.
+ }
+}
+
+const havePython3 = process.platform !== "win32" && spawnSync("python3", ["-V"]).status === 0;
+const describeIf = havePython3 ? describe : describe.skip;
+
+describeIf("forkserver kill/liveness protocol (stub python)", () => {
+ let tempDir = "";
+ let server: ForkServer | undefined;
+ let savedPythonPath: string | undefined;
+ const leakedPids: number[] = [];
+
+ beforeEach(() => {
+ tempDir = mkdtempSync(join(tmpdir(), "prime-agent-forkserver-proto-"));
+ const stubDir = join(tempDir, "stubs");
+ mkdirSync(stubDir);
+ writeStubModules(stubDir);
+ // launchEnv is snapshotted at ForkServer construction, so the template
+ // imports the stubs.
+ savedPythonPath = process.env.PYTHONPATH;
+ process.env.PYTHONPATH = stubDir;
+ server = new ForkServer({ python: "python3" });
+ });
+
+ afterEach(() => {
+ if (savedPythonPath === undefined) delete process.env.PYTHONPATH;
+ else process.env.PYTHONPATH = savedPythonPath;
+ server?.dispose();
+ server = undefined;
+ // The stub app ignores parent_handle, so leak-proof the test itself.
+ for (const pid of leakedPids.splice(0)) killQuietly(pid);
+ rmSync(tempDir, { recursive: true, force: true });
+ });
+
+ async function spawnStubKernel(
+ target: ForkServer = server!,
+ env?: Record<string, string | undefined>,
+ ): Promise<ForkedKernelHandle> {
+ const handle = await target.spawnKernel({ connectionPath: join(tempDir, "conn.json"), env });
+ leakedPids.push(handle.pid);
+ return handle;
+ }
+
+ it("kills its own child through the protocol and observes the reap", async () => {
+ const handle = await spawnStubKernel();
+ expect(await handle.isAlive()).toBe(true);
+ expect(await handle.kill("TERM")).toBe("signaled");
+ await vi.waitFor(async () => {
+ expect(await handle.isAlive()).toBe(false);
+ });
+ // OS-level confirmation (test-side observation only, never a signal path).
+ await vi.waitFor(() => {
+ expect(() => process.kill(handle.pid, 0)).toThrow();
+ });
+ }, 15_000);
+
+ it("reports already-exited after reap and fails closed on an unknown fork id", async () => {
+ const handle = await spawnStubKernel();
+ expect(await handle.kill("TERM")).toBe("signaled");
+ await vi.waitFor(async () => {
+ expect(await handle.kill("TERM")).toBe("already-exited");
+ });
+ // A fork id the forkserver never issued: nothing may be signaled.
+ expect(await server!.killChild(999_999, "TERM")).toBe("unknown-pid");
+ expect(await server!.isChildAlive(999_999)).toBe(false);
+ }, 15_000);
+
+ it("liveness reflects the reap table on external child death", async () => {
+ const handle = await spawnStubKernel();
+ expect(await handle.isAlive()).toBe(true);
+ // External death (not via the protocol): proves the SIGCHLD reaper drives
+ // the registry independently of the kill path.
+ process.kill(handle.pid, "SIGTERM");
+ await vi.waitFor(async () => {
+ expect(await handle.isAlive()).toBe(false);
+ });
+ }, 15_000);
+
+ it("kill outcomes stay correct while sibling children churn and get reaped", async () => {
+ // Regression for the SIGCHLD-in-watcher-thread race: external deaths storm
+ // the reaper while kill requests are in flight; every reply must still match
+ // the child's true state (no false "signaled" for a freed pid).
+ const keep = await Promise.all([spawnStubKernel(), spawnStubKernel(), spawnStubKernel()]);
+ const churn = await Promise.all([spawnStubKernel(), spawnStubKernel(), spawnStubKernel()]);
+ for (const handle of churn) process.kill(handle.pid, "SIGKILL");
+ const outcomes = await Promise.all(keep.map((handle) => handle.kill("TERM")));
+ expect(outcomes).toEqual(["signaled", "signaled", "signaled"]);
+ for (const handle of churn) {
+ await vi.waitFor(async () => {
+ expect(await handle.kill("TERM")).toBe("already-exited");
+ });
+ }
+ }, 15_000);
+
+ it("a fast-exiting child never lands alive; its handle stays already-exited across new forks", async () => {
+ // STUB_KERNEL_EXIT makes the forked child return from start() immediately,
+ // so it can be reaped as early as the OS allows relative to the parent-side
+ // registry insertion — the SIGCHLD-blocked fork bookkeeping must still
+ // record it under its own id and mark exactly that entry not-alive.
+ const fast = await spawnStubKernel(server!, { STUB_KERNEL_EXIT: "1" });
+ await vi.waitFor(async () => {
+ expect(await fast.isAlive()).toBe(false);
+ });
+ expect(await fast.kill("TERM")).toBe("already-exited");
+ // Id-keying invariant: a later fork gets a fresh id (even if the OS reuses
+ // the pid, which we can't force), and the old handle's answers don't change.
+ const next = await spawnStubKernel();
+ expect(await next.isAlive()).toBe(true);
+ expect(await fast.isAlive()).toBe(false);
+ expect(await fast.kill("TERM")).toBe("already-exited");
+ expect(await next.kill("TERM")).toBe("signaled");
+ }, 15_000);
+
+ it("evicted registry entries fail closed (unknown-pid/false), recent ones stay answerable", async () => {
+ // A tiny history bound (argv[2] to the script) makes FIFO eviction reachable.
+ const bounded = new ForkServer({ python: "python3", historyBound: 2 });
+ try {
+ const oldest = await spawnStubKernel(bounded, { STUB_KERNEL_EXIT: "1" });
+ await vi.waitFor(async () => {
+ expect(await oldest.kill("TERM")).toBe("already-exited");
+ });
+ const middle = await spawnStubKernel(bounded, { STUB_KERNEL_EXIT: "1" });
+ const newest = await spawnStubKernel(bounded, { STUB_KERNEL_EXIT: "1" });
+ // Three entries against a bound of 2: the oldest id has been evicted.
+ expect(await oldest.kill("TERM")).toBe("unknown-pid");
+ expect(await oldest.isAlive()).toBe(false);
+ for (const handle of [middle, newest]) {
+ await vi.waitFor(async () => {
+ expect(await handle.kill("TERM")).toBe("already-exited");
+ });
+ expect(await handle.isAlive()).toBe(false);
+ }
+ } finally {
+ bounded.dispose();
+ }
+ }, 15_000);
+
+ it("keeps live entries answerable past the history bound and evicts only exited ones", async () => {
+ // A live child must never be evicted: an evicted-live entry would read
+ // dead to the liveness monitor and be unroutable for kill — the exact
+ // orphan leak this forkserver exists to prevent.
+ const bounded = new ForkServer({ python: "python3", historyBound: 2 });
+ try {
+ const oldest = await spawnStubKernel(bounded);
+ expect(await oldest.isAlive()).toBe(true);
+ const middle = await spawnStubKernel(bounded);
+ const newest = await spawnStubKernel(bounded);
+ // Three live entries against a bound of 2: none may be evicted.
+ expect(await oldest.isAlive()).toBe(true);
+ expect(await middle.isAlive()).toBe(true);
+ expect(await newest.isAlive()).toBe(true);
+ // Once dead entries exist, the next fork sweeps them out instead.
+ expect(await oldest.kill("TERM")).toBe("signaled");
+ expect(await middle.kill("TERM")).toBe("signaled");
+ await vi.waitFor(async () => {
+ expect(await oldest.isAlive()).toBe(false);
+ expect(await middle.isAlive()).toBe(false);
+ });
+ const extra = await spawnStubKernel(bounded);
+ // Bound 2 with two dead entries: both are evicted, the live ones stay.
+ expect(await oldest.kill("TERM")).toBe("unknown-pid");
+ expect(await middle.kill("TERM")).toBe("unknown-pid");
+ expect(await newest.isAlive()).toBe(true);
+ expect(await extra.isAlive()).toBe(true);
+ expect(await newest.kill("TERM")).toBe("signaled");
+ expect(await extra.kill("TERM")).toBe("signaled");
+ } finally {
+ bounded.dispose();
+ }
+ }, 15_000);
+
+ it("handle kill/isAlive reject with ForkServerUnavailable when the server is dead", async () => {
+ const handle = await spawnStubKernel();
+ server!.dispose();
+ await expect(handle.kill("TERM")).rejects.toBeInstanceOf(ForkServerUnavailable);
+ await expect(handle.isAlive()).rejects.toBeInstanceOf(ForkServerUnavailable);
+ }, 15_000);
+
+ describe("forkserver orphan journal", () => {
+ let journalPath = "";
+ const savedJournal = process.env[ORPHAN_PROCESS_JOURNAL_ENV];
+
+ interface JournalRecord {
+ pid: number;
+ active: boolean;
+ }
+
+ function readJournal(): JournalRecord[] {
+ if (!existsSync(journalPath)) return [];
+ return readFileSync(journalPath, "utf8")
+ .split("\n")
+ .filter(Boolean)
+ .map((line) => JSON.parse(line) as JournalRecord);
+ }
+
+ beforeEach(() => {
+ journalPath = join(tempDir, "orphans.jsonl");
+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;
+ });
+
+ afterEach(() => {
+ if (savedJournal === undefined) delete process.env[ORPHAN_PROCESS_JOURNAL_ENV];
+ else process.env[ORPHAN_PROCESS_JOURNAL_ENV] = savedJournal;
+ });
+
+ it("disposing a live forkserver delivers the kill and writes inactive", async () => {
+ await spawnStubKernel();
+ const records = readJournal();
+ expect(records).toHaveLength(1);
+ expect(records[0]).toMatchObject({ active: true });
+ server!.dispose();
+ const after = readJournal();
+ expect(after).toHaveLength(2);
+ expect(after[1]).toMatchObject({ pid: records[0]!.pid, active: false });
+ }, 15_000);
+
+ it("a forkserver exit observed by the handle writes inactive", async () => {
+ await spawnStubKernel();
+ const forkserverPid = readJournal()[0]!.pid;
+ process.kill(forkserverPid, "SIGKILL");
+ // The exit event drives markDead → dispose with the exit already observed.
+ await vi.waitFor(() => {
+ const records = readJournal();
+ expect(records).toHaveLength(2);
+ expect(records[1]).toMatchObject({ pid: forkserverPid, active: false });
+ });
+ }, 15_000);
+
+ it("a never-started forkserver writes no journal records on dispose", () => {
+ const idle = new ForkServer({ python: "python3" });
+ idle.dispose();
+ expect(readJournal()).toHaveLength(0);
+ });
+
+ it("an unconfirmed kill with no observed exit leaves no inactive record", () => {
+ const unconfirmed = new ForkServer({ python: "python3" });
+ const internals = unconfirmed as unknown as {
+ proc?: { pid?: number; exitCode: number | null; signalCode: string | null; kill(): boolean };
+ };
+ internals.proc = { pid: 424242, exitCode: null, signalCode: null, kill: () => false };
+ unconfirmed.dispose();
+ expect(readJournal().some((r) => r.pid === 424242)).toBe(false);
+ });
+ });
+});
+
+function resolveKernelPython(): string | null {
+ const candidates = [
+ process.env.PRIME_AGENT_KERNEL_PYTHON,
+ join(homedir(), ".prime", "agent", "kernel-venv", "bin", "python"),
+ ].filter((p): p is string => Boolean(p));
+ for (const python of candidates) {
+ if (!existsSync(python)) continue;
+ const check = spawnSync(python, ["-c", "import ipykernel"], { encoding: "utf8" });
+ if (check.status === 0) return python;
+ }
+ return null;
+}
+
+const kernelPython = resolveKernelPython();
+// Real ipykernel fork round-trip is linux-only: on darwin the forked child dies
+// immediately (fork-without-exec is unsafe there), matching isForkServerEnabled.
+const describeIfRealKernel = process.platform === "linux" && kernelPython ? describe : describe.skip;
+
+describeIfRealKernel("forkserver kill/liveness protocol (real kernel)", { tags: ["kernel-heavy"] }, () => {
+ it("forks a real ipykernel, resolves ports, kills it via the protocol", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "prime-agent-forkserver-real-"));
+ const connectionPath = join(dir, "connection.json");
+ writeFileSync(
+ connectionPath,
+ JSON.stringify({
+ ip: "127.0.0.1",
+ transport: "tcp",
+ shell_port: 0,
+ iopub_port: 0,
+ stdin_port: 0,
+ control_port: 0,
+ hb_port: 0,
+ signature_scheme: "hmac-sha256",
+ key: "test-key",
+ kernel_name: "python3",
+ }),
+ { mode: 0o600 },
+ );
+ const server = new ForkServer({ python: kernelPython! });
+ let pid: number | undefined;
+ try {
+ const handle = await server.spawnKernel({ connectionPath });
+ pid = handle.pid;
+ await vi.waitFor(
+ () => {
+ const info = JSON.parse(readFileSync(connectionPath, "utf8")) as { shell_port: number };
+ expect(info.shell_port).toBeGreaterThan(0);
+ },
+ { timeout: 20_000, interval: 250 },
+ );
+ expect(await handle.isAlive()).toBe(true);
+ expect(await handle.kill("TERM")).toBe("signaled");
+ await vi.waitFor(
+ async () => {
+ expect(await handle.isAlive()).toBe(false);
+ },
+ { timeout: 20_000, interval: 250 },
+ );
+ } finally {
+ server.dispose();
+ killQuietly(pid);
+ rmSync(dir, { recursive: true, force: true });
+ }
+ }, 60_000);
+});
diff --git a/packages/coding-agent/test/kernel-parent-watchdog.test.ts b/packages/coding-agent/test/kernel-parent-watchdog.test.ts
new file mode 100644
index 0000000000..565b312ee7
--- /dev/null
+++ b/packages/coding-agent/test/kernel-parent-watchdog.test.ts
@@ -0,0 +1,548 @@
+import { spawn, spawnSync } from "node:child_process";
+import { EventEmitter } from "node:events";
+import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { type ForkedKernelHandle, ForkServerUnavailable } from "../src/core/kernel/fork-server.js";
+import { KernelManager } from "../src/core/kernel/index.js";
+import { ORPHAN_PROCESS_JOURNAL_ENV } from "../src/core/orphan-process-journal.js";
+
+const forkKernelMock = vi.hoisted(() => vi.fn());
+const forkEnabledMock = vi.hoisted(() => vi.fn(() => false));
+
+vi.mock("../src/core/kernel/fork-server.js", async (importOriginal) => {
+ const original = await importOriginal<typeof import("../src/core/kernel/fork-server.js")>();
+ return { ...original, forkKernel: forkKernelMock, isForkServerEnabled: forkEnabledMock };
+});
+
+let tempDir = "";
+const savedForkFlag = process.env.PRIME_AGENT_KERNEL_FORKSERVER;
+const savedJournalPath = process.env[ORPHAN_PROCESS_JOURNAL_ENV];
+
+beforeAll(() => {
+ process.env.PRIME_AGENT_KERNEL_FORKSERVER = "0";
+});
+afterAll(() => {
+ if (savedForkFlag === undefined) delete process.env.PRIME_AGENT_KERNEL_FORKSERVER;
+ else process.env.PRIME_AGENT_KERNEL_FORKSERVER = savedForkFlag;
+});
+
+function writeFakePython(script: string[]): string {
+ const python = join(tempDir, "python");
+ writeFileSync(python, script.join("\n"));
+ chmodSync(python, 0o755);
+ return python;
+}
+
+interface JournalRecord {
+ pid: number;
+ ownerPid: number;
+ active: boolean;
+}
+
+function readJournalRecords(path: string): JournalRecord[] {
+ return readFileSync(path, "utf8")
+ .split("\n")
+ .filter(Boolean)
+ .map((line) => JSON.parse(line) as JournalRecord);
+}
+
+describe("kernel parent watchdog", () => {
+ beforeEach(() => {
+ tempDir = mkdtempSync(join(tmpdir(), "prime-agent-kernel-watchdog-"));
+ });
+
+ afterEach(() => {
+ forkEnabledMock.mockReturnValue(false);
+ forkKernelMock.mockReset();
+ if (savedJournalPath === undefined) delete process.env[ORPHAN_PROCESS_JOURNAL_ENV];
+ else process.env[ORPHAN_PROCESS_JOURNAL_ENV] = savedJournalPath;
+ if (tempDir) {
+ rmSync(tempDir, { recursive: true, force: true });
+ tempDir = "";
+ }
+ });
+
+ it("direct spawn sets JPY_PARENT_PID and journals the kernel pid", async () => {
+ const envDump = join(tempDir, "kernel-env");
+ const python = writeFakePython(["#!/bin/sh", `env > "${envDump}"`, "exit 42", ""]);
+ const journalPath = join(tempDir, "orphans.jsonl");
+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;
+ const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+ const manager = new KernelManager({ python, cwd: tempDir });
+
+ try {
+ await expect(manager.execute("x")).rejects.toThrow(/Kernel exited before resolving ports/);
+ } finally {
+ errorSpy.mockRestore();
+ await manager.dispose();
+ }
+
+ expect(readFileSync(envDump, "utf8")).toMatch(new RegExp(`^JPY_PARENT_PID=${process.pid}$`, "m"));
+
+ // Self-exited child: the handle-based kill signals nothing, so the record must stay active.
+ await vi.waitFor(() => {
+ const records = readJournalRecords(journalPath);
+ expect(records).toHaveLength(1);
+ expect(records[0]?.ownerPid).toBe(process.pid);
+ expect(records[0]?.active).toBe(true);
+ });
+ });
+
+ it("writes the inactive journal record only on a signaled forkserver kill outcome", async () => {
+ const journalPath = join(tempDir, "orphans.jsonl");
+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;
+ forkEnabledMock.mockReturnValue(true);
+ const killMock = vi.fn(async (): Promise<"signaled"> => "signaled");
+ forkKernelMock.mockResolvedValue({
+ pid: 999999,
+ isAlive: async () => false,
+ kill: killMock,
+ });
+ const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });
+
+ try {
+ await expect(manager.execute("x")).rejects.toThrow(/Kernel exited before resolving ports/);
+ } finally {
+ errorSpy.mockRestore();
+ await manager.dispose();
+ }
+
+ await vi.waitFor(() => {
+ expect(killMock).toHaveBeenCalledTimes(1);
+ expect(killMock).toHaveBeenCalledWith("TERM");
+ const records = readJournalRecords(journalPath);
+ expect(records).toHaveLength(2);
+ expect(records[0]).toMatchObject({ pid: 999999, active: true });
+ expect(records[1]).toMatchObject({ pid: 999999, active: false });
+ });
+ });
+
+ it("leaves the journal record active on an already-exited kill outcome (pid may be reused)", async () => {
+ const journalPath = join(tempDir, "orphans.jsonl");
+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;
+ forkEnabledMock.mockReturnValue(true);
+ const killMock = vi.fn(async (): Promise<"already-exited"> => "already-exited");
+ forkKernelMock.mockResolvedValue({
+ pid: 999999,
+ isAlive: async () => false,
+ kill: killMock,
+ });
+ const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });
+
+ try {
+ await expect(manager.execute("x")).rejects.toThrow(/Kernel exited before resolving ports/);
+ } finally {
+ errorSpy.mockRestore();
+ await manager.dispose();
+ }
+
+ await vi.waitFor(() => expect(killMock).toHaveBeenCalledTimes(1));
+ // Let the resolved kill settle: no inactive record may ever follow.
+ await new Promise((resolve) => setTimeout(resolve, 200));
+ const records = readJournalRecords(journalPath);
+ expect(records).toHaveLength(1);
+ expect(records[0]).toMatchObject({ pid: 999999, active: true });
+ });
+
+ it("leaves the journal record active and never signals the pid when the kill is unconfirmed", async () => {
+ const journalPath = join(tempDir, "orphans.jsonl");
+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;
+ forkEnabledMock.mockReturnValue(true);
+ const killMock = vi.fn(async (): Promise<never> => {
+ throw new ForkServerUnavailable("dead");
+ });
+ forkKernelMock.mockResolvedValue({
+ pid: 999999,
+ isAlive: async () => {
+ throw new ForkServerUnavailable("dead");
+ },
+ kill: killMock,
+ });
+ const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+ const killSpy = vi.spyOn(process, "kill");
+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });
+
+ try {
+ await expect(manager.execute("x")).rejects.toThrow(/Kernel exited before resolving ports/);
+ await manager.dispose();
+ await vi.waitFor(() => expect(killMock).toHaveBeenCalled());
+ // Let the rejected kill settle: no inactive record may ever follow.
+ await new Promise((resolve) => setTimeout(resolve, 200));
+ const records = readJournalRecords(journalPath);
+ expect(records).toHaveLength(1);
+ expect(records[0]).toMatchObject({ pid: 999999, active: true });
+ expect(killSpy.mock.calls.some((call) => call[0] === 999999)).toBe(false);
+ } finally {
+ killSpy.mockRestore();
+ errorSpy.mockRestore();
+ await manager.dispose();
+ }
+ });
+
+ it("leaves the direct-spawn journal record active when the kill signals nothing", async () => {
+ const journalPath = join(tempDir, "orphans.jsonl");
+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;
+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });
+ const internals = manager as unknown as {
+ kernel?: { pid?: number; kill(signal: string): boolean };
+ cleanupResources(): void;
+ };
+
+ try {
+ internals.kernel = { pid: 999999, kill: () => false };
+ internals.cleanupResources();
+
+ const records = existsSync(journalPath) ? readJournalRecords(journalPath) : [];
+ expect(records.some((r) => r.pid === 999999 && !r.active)).toBe(false);
+ } finally {
+ await manager.dispose();
+ }
+ });
+
+ it("maps SIGKILL to the forkserver KILL signal", async () => {
+ forkEnabledMock.mockReturnValue(true);
+ const isAliveMock = vi.fn(async () => true);
+ const killMock = vi.fn(async (): Promise<"signaled"> => "signaled");
+ forkKernelMock.mockResolvedValue({ pid: 999999, isAlive: isAliveMock, kill: killMock });
+ const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });
+
+ try {
+ const execution = manager.execute("x");
+ execution.catch(() => {});
+ // The startup poll calls isAlive only after the handle is assigned, so
+ // kill() below is guaranteed to reach the forked branch.
+ await vi.waitFor(() => expect(isAliveMock).toHaveBeenCalled());
+ await manager.kill();
+ expect(killMock).toHaveBeenCalledWith("KILL");
+ await expect(execution).rejects.toThrow();
+ } finally {
+ errorSpy.mockRestore();
+ await manager.dispose();
+ }
+ });
+
+ it("a stale liveness probe resumed after restart never tears down the new kernel", async () => {
+ const journalPath = join(tempDir, "orphans.jsonl");
+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;
+ let resolveProbe: (alive: boolean) => void = () => {};
+ const killA = vi.fn(async (): Promise<"signaled"> => "signaled");
+ const killB = vi.fn(async (): Promise<"signaled"> => "signaled");
+ const handleA: ForkedKernelHandle = {
+ pid: 111111,
+ isAlive: () =>
+ new Promise<boolean>((resolve) => {
+ resolveProbe = resolve;
+ }),
+ kill: killA,
+ };
+ const handleB: ForkedKernelHandle = { pid: 222222, isAlive: async () => true, kill: killB };
+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });
+ const internals = manager as unknown as {
+ state: string;
+ forkedKernel?: ForkedKernelHandle;
+ checkForkedKernelDeath(): Promise<void>;
+ };
+
+ try {
+ internals.state = "running";
+ internals.forkedKernel = handleA;
+ const probe = internals.checkForkedKernelDeath();
+ // A restart completes while the probe is in flight: the manager is
+ // "running" again, but on a different kernel.
+ internals.forkedKernel = handleB;
+ resolveProbe(false);
+ await probe;
+
+ expect(internals.state).toBe("running");
+ expect(killB).not.toHaveBeenCalled();
+ const records = existsSync(journalPath) ? readJournalRecords(journalPath) : [];
+ expect(records.some((r) => r.pid === handleB.pid && !r.active)).toBe(false);
+ } finally {
+ internals.forkedKernel = undefined;
+ internals.state = "idle";
+ await manager.dispose();
+ }
+ });
+
+ it("a timed-out liveness probe never tears down a possibly-healthy kernel", async () => {
+ const kill = vi.fn(async (): Promise<"signaled"> => "signaled");
+ const handle: ForkedKernelHandle = {
+ pid: 111111,
+ isAlive: async () => {
+ throw new ForkServerUnavailable("forkserver request timed out after 10000ms", { timedOut: true });
+ },
+ kill,
+ };
+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });
+ const internals = manager as unknown as {
+ state: string;
+ forkedKernel?: ForkedKernelHandle;
+ checkForkedKernelDeath(): Promise<void>;
+ };
+
+ try {
+ internals.state = "running";
+ internals.forkedKernel = handle;
+ await internals.checkForkedKernelDeath();
+
+ expect(internals.state).toBe("running");
+ expect(kill).not.toHaveBeenCalled();
+ } finally {
+ internals.forkedKernel = undefined;
+ internals.state = "idle";
+ await manager.dispose();
+ }
+ });
+
+ it("a stale doStart resumed after a concurrent teardown never touches the new kernel", async () => {
+ const journalPath = join(tempDir, "orphans.jsonl");
+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;
+ forkEnabledMock.mockReturnValue(true);
+ let resolveFork: (handle: ForkedKernelHandle) => void = () => {};
+ const killA = vi.fn(async (): Promise<"signaled"> => "signaled");
+ const killB = vi.fn(async (): Promise<"signaled"> => "signaled");
+ const handleA: ForkedKernelHandle = { pid: 111111, isAlive: async () => true, kill: killA };
+ const handleB: ForkedKernelHandle = { pid: 222222, isAlive: async () => true, kill: killB };
+ forkKernelMock.mockReturnValue(
+ new Promise<ForkedKernelHandle>((resolve) => {
+ resolveFork = resolve;
+ }),
+ );
+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });
+ const internals = manager as unknown as { state: string; forkedKernel?: ForkedKernelHandle };
+
+ try {
+ // Start A blocks inside `await forkKernel(...)`.
+ const staleStart = manager.start();
+ staleStart.catch(() => {});
+ await vi.waitFor(() => expect(forkKernelMock).toHaveBeenCalledTimes(1));
+
+ // A concurrent restart tears the starting kernel down (bumping the start
+ // generation) and brings up a new running kernel B.
+ await manager.kill();
+ internals.state = "running";
+ internals.forkedKernel = handleB;
+
+ // The stale doStart resumes with handle A: it must fail without touching B.
+ resolveFork(handleA);
+ await expect(staleStart).rejects.toThrow(/Kernel start superseded/);
+
+ expect(internals.state).toBe("running");
+ expect(internals.forkedKernel).toBe(handleB);
+ expect(killB).not.toHaveBeenCalled();
+ // A is reclaimed via its own handle, never journaled.
+ await vi.waitFor(() => expect(killA).toHaveBeenCalledWith("TERM"));
+ const records = existsSync(journalPath) ? readJournalRecords(journalPath) : [];
+ expect(records.some((r) => r.pid === handleA.pid)).toBe(false);
+ } finally {
+ internals.forkedKernel = undefined;
+ internals.state = "idle";
+ await manager.dispose();
+ }
+ });
+
+ it("a shutdown superseded by a concurrent kill reports not-owner so recovery cannot resurrect to idle", async () => {
+ const manager = new KernelManager({ python: "/nonexistent-python", cwd: tmpdir() });
+ const internals = manager as unknown as {
+ state: string;
+ connection: unknown;
+ control: unknown;
+ kernel: unknown;
+ shutdown(): Promise<boolean>;
+ kill(): Promise<void>;
+ };
+ internals.state = "starting";
+ internals.connection = { key: "" };
+ // A live kernel handle keeps waitForKernelExit parked so the send actually blocks the shutdown.
+ const kernel = Object.assign(new EventEmitter(), { exitCode: null, signalCode: null, kill: () => true });
+ internals.kernel = kernel;
+ let releaseSend: () => void = () => {};
+ internals.control = {
+ send: () => new Promise<void>((resolve) => (releaseSend = resolve)),
+ close: () => {},
+ };
+ const shutdownResult = internals.shutdown();
+ await new Promise((resolve) => setTimeout(resolve, 10)); // park in the control send
+ await internals.kill(); // concurrent teardown wins ownership
+ releaseSend();
+ kernel.emit("exit", 0, null);
+ expect(await shutdownResult).toBe(false); // recovery must not set idle
+ expect(internals.state).toBe("shutdown");
+ });
+
+ it("a stale shutdown parked in its control-send await never cleans up a successor kernel", async () => {
+ const journalPath = join(tempDir, "orphans.jsonl");
+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;
+ let releaseSend: () => void = () => {};
+ const parkedSend = new Promise<void>((resolve) => {
+ releaseSend = resolve;
+ });
+ const killA = vi.fn(async (): Promise<"signaled"> => "signaled");
+ const killB = vi.fn(async (): Promise<"signaled"> => "signaled");
+ const handleA: ForkedKernelHandle = { pid: 111111, isAlive: async () => true, kill: killA };
+ const handleB: ForkedKernelHandle = { pid: 222222, isAlive: async () => true, kill: killB };
+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });
+ const internals = manager as unknown as {
+ state: string;
+ forkedKernel?: ForkedKernelHandle;
+ control?: { send(frames: Buffer[]): Promise<void>; close(): void };
+ connection?: { ip: string; transport: string; control_port: number; key: string };
+ startPromise?: Promise<void>;
+ };
+
+ try {
+ // Kernel A is running with a control channel whose send parks forever
+ // until released — shutdown() will suspend inside its await window
+ // after having synchronously set state = "shutdown".
+ internals.state = "running";
+ internals.forkedKernel = handleA;
+ internals.control = { send: () => parkedSend, close: () => {} };
+ internals.connection = { ip: "127.0.0.1", transport: "tcp", control_port: 1, key: "test-key" };
+ const staleShutdown = manager.shutdown();
+
+ // While A's shutdown is parked, a concurrent teardown reclaims A (this is
+ // the cleanup restart() reaches via shutdown's already-shutdown fast path)
+ // and a new start brings up kernel B.
+ await manager.kill();
+ internals.state = "running";
+ internals.forkedKernel = handleB;
+ const startPromiseB = Promise.resolve();
+ internals.startPromise = startPromiseB;
+
+ // A's stale shutdown resumes: it must not clean up B or clear B's start.
+ releaseSend();
+ await staleShutdown;
+
+ expect(internals.state).toBe("running");
+ expect(internals.forkedKernel).toBe(handleB);
+ expect(killB).not.toHaveBeenCalled();
+ expect(internals.startPromise).toBe(startPromiseB);
+ // A was reclaimed by kill(); B must never gain an inactive journal record.
+ await vi.waitFor(() => expect(killA).toHaveBeenCalledWith("KILL"));
+ const records = existsSync(journalPath) ? readJournalRecords(journalPath) : [];
+ expect(records.some((r) => r.pid === handleB.pid && !r.active)).toBe(false);
+ } finally {
+ internals.forkedKernel = undefined;
+ internals.startPromise = undefined;
+ internals.state = "idle";
+ await manager.dispose();
+ }
+ });
+
+ it("a hung liveness probe resolves alive once its budget expires", async () => {
+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });
+ const internals = manager as unknown as {
+ forkedKernelDead(probed: ForkedKernelHandle, timeoutMs?: number): Promise<boolean>;
+ };
+ const handle: ForkedKernelHandle = {
+ pid: 999999,
+ isAlive: () => new Promise<boolean>(() => {}),
+ kill: async () => "signaled",
+ };
+
+ try {
+ const started = Date.now();
+ // Unknown is not death, and the caller's budget bounds the wait.
+ await expect(internals.forkedKernelDead(handle, 250)).resolves.toBe(false);
+ expect(Date.now() - started).toBeLessThan(5_000);
+ } finally {
+ await manager.dispose();
+ }
+ });
+
+ it("fork request env does not carry JPY_PARENT_PID (forked children watch the forkserver)", async () => {
+ const python = writeFakePython(["#!/bin/sh", "exit 42", ""]);
+ forkEnabledMock.mockReturnValue(true);
+ forkKernelMock.mockRejectedValue(new ForkServerUnavailable("test"));
+ const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+ const manager = new KernelManager({ python, cwd: tempDir });
+
+ try {
+ await expect(manager.execute("x")).rejects.toThrow(/Kernel exited before resolving ports/);
+ } finally {
+ errorSpy.mockRestore();
+ await manager.dispose();
+ }
+
+ expect(forkKernelMock).toHaveBeenCalledTimes(1);
+ const spawnParams = forkKernelMock.mock.calls[0]?.[1] as { env?: Record<string, string | undefined> };
+ expect(spawnParams.env?.JPY_PARENT_PID).toBeUndefined();
+ });
+});
+
+function resolveKernelPython(): string | null {
+ const candidates = [
+ process.env.PRIME_AGENT_KERNEL_PYTHON,
+ join(homedir(), ".prime", "agent", "kernel-venv", "bin", "python"),
+ ].filter((p): p is string => Boolean(p));
+ for (const python of candidates) {
+ if (!existsSync(python)) continue;
+ const check = spawnSync(python, ["-c", "import ipykernel"], { encoding: "utf8" });
+ if (check.status === 0) return python;
+ }
+ return null;
+}
+
+const kernelPython = resolveKernelPython();
+const describeIf = kernelPython && process.platform !== "win32" ? describe : describe.skip;
+
+describeIf("kernel outlives-owner watchdog (real kernel)", { tags: ["kernel-heavy"] }, () => {
+ it("kernel exits after its owner is SIGKILLed", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "prime-agent-watchdog-int-"));
+ const pidFile = join(dir, "kernel.pid");
+ const connectionFile = join(dir, "connection.json");
+ // The owner must be a separate killable process; it replicates KernelManager's
+ // exact spawn line (T1 above proves the manager emits that env).
+ const ownerScript = [
+ `const { spawn } = require("node:child_process");`,
+ `const { writeFileSync } = require("node:fs");`,
+ `const k = spawn(${JSON.stringify(kernelPython)}, ["-m", "ipykernel_launcher", "-f", ${JSON.stringify(connectionFile)}], {`,
+ ` env: { ...process.env, JPY_PARENT_PID: String(process.pid) },`,
+ ` stdio: "ignore",`,
+ `});`,
+ `writeFileSync(${JSON.stringify(pidFile)}, String(k.pid));`,
+ `setInterval(() => {}, 1000);`,
+ ].join("\n");
+ const owner = spawn(process.execPath, ["-e", ownerScript], { stdio: ["ignore", "ignore", "inherit"] });
+ let kernelPid = 0;
+
+ try {
+ await vi.waitFor(
+ () => {
+ kernelPid = Number(readFileSync(pidFile, "utf8"));
+ expect(kernelPid).toBeGreaterThan(0);
+ expect(() => process.kill(kernelPid, 0)).not.toThrow();
+ },
+ { timeout: 20_000, interval: 500 },
+ );
+
+ owner.kill("SIGKILL");
+
+ await vi.waitFor(
+ () => {
+ expect(() => process.kill(kernelPid, 0)).toThrow();
+ },
+ { timeout: 20_000, interval: 500 },
+ );
+ } finally {
+ if (kernelPid > 0) {
+ try {
+ process.kill(kernelPid, "SIGKILL");
+ } catch {
+ // Already exited (the expected outcome).
+ }
+ }
+ try {
+ owner.kill("SIGKILL");
+ } catch {
+ // Already exited.
+ }
+ rmSync(dir, { recursive: true, force: true });
+ }
+ }, 30_000);
+});
diff --git a/packages/coding-agent/test/kernel-shutdown.test.ts b/packages/coding-agent/test/kernel-shutdown.test.ts
index 50a65b9a0b..3f002ffb3e 100644
--- a/packages/coding-agent/test/kernel-shutdown.test.ts
+++ b/packages/coding-agent/test/kernel-shutdown.test.ts
@@ -103,7 +103,8 @@ describe("KernelManager graceful shutdown", () => {
try {
const shutdown = manager.shutdown();
await vi.advanceTimersByTimeAsync(100);
- await shutdown;
+ // True = this call performed the cleanup: startup-failure recovery relies on it to resurrect to idle.
+ await expect(shutdown).resolves.toBe(true);
expect(internals.kernel).toBeUndefined();
} finally {
vi.useRealTimers();