A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
Kernels exit when their owner dies (#1559)
Sebastian Mülleraddfc23ffcff
diff --git a/packages/coding-agent/.changes/eng-5310.md b/packages/coding-agent/.changes/eng-5310.mdnew file mode 100644index 0000000000..35e23911e5--- /dev/null+++ b/packages/coding-agent/.changes/eng-5310.md@@ -0,0 +1,2 @@+- Fixed IPython kernels and forkserver processes outliving their owner after a hard crash: kernels now arm ipykernel's parent-death poller via JPY_PARENT_PID, the forkserver watches its parent pid, and both pids are registered in the orphan process journal for supervisor recovery.+- Fixed a pid-reuse race for forked IPython kernels: signaling and liveness now go through the forkserver (the kernels' parent) instead of raw pid operations from Node, and the orphan journal's inactive record is only written on a confirmed kill outcome.diff --git a/packages/coding-agent/src/core/kernel/fork-server-script.ts b/packages/coding-agent/src/core/kernel/fork-server-script.tsindex d08f8e0139..a6e286d0f9 100644--- a/packages/coding-agent/src/core/kernel/fork-server-script.ts+++ b/packages/coding-agent/src/core/kernel/fork-server-script.ts@@ -5,13 +5,21 @@//// Embedded as a string rather than shipped as a package asset so it can never be// missing from a release layout (see the built-in-skills packaging gap). Run via-// `python -c <this> <control-socket-path>`.+// `python -c <this> <control-socket-path> [<history-bound>]`.//// Protocol (newline-delimited JSON over the unix socket, forkserver is the client):// -> { "id": <n>, "connectionPath": "<abs path>" } spawn request from Node+// -> { "id": <n>, "kill": <fork-id>, "signal": "TERM"|"KILL" } kill a forked child+// -> { "id": <n>, "alive": <fork-id> } liveness query for a child// <- { "type": "ready" } once, after imports finish// <- { "id": <n>, "pid": <pid> } fork succeeded// <- { "id": <n>, "error": "<message>" } fork failed+// <- { "id": <n>, "outcome": "signaled"|"already-exited"|"unknown-pid" } kill reply+// <- { "id": <n>, "alive": true|false } alive reply+//+// Kill/alive are keyed by the never-reused fork request id, so a handle can only+// name the one child incarnation it forked (a recycled pid can't alias), and the+// forkserver signals only its own un-reaped children — POSIX-race-free.export const FORK_SERVER_SCRIPT = String.raw`import gcimport json@@ -19,6 +27,15 @@ import osimport signalimport socketimport sys+import threading+import time++# fork id -> [pid, alive]; _pid_to_id holds un-reaped, un-evicted children only.+# Only the main thread mutates these (the SIGCHLD handler runs between its bytecodes).+_children = {}+_pid_to_id = {}+# FIFO history bound; kill/alive for an evicted id fail closed.+_history_bound = 4096def _reap_children(*_args):@@ -30,10 +47,22 @@ def _reap_children(*_args):pid, _status = os.waitpid(-1, os.WNOHANG)if pid == 0:break+ child_id = _pid_to_id.pop(pid, None)+ if child_id is not None:+ _children[child_id][1] = Falseexcept ChildProcessError:pass+def _watch_parent(original_ppid):+ # A SIGKILLed owner can't close our socket while a forked child still holds the+ # fd; poll ppid so the forkserver dies with its owner regardless.+ while True:+ if os.getppid() != original_ppid:+ os._exit(1)+ time.sleep(1.0)++def _import_template():# Everything a kernel touches at import time. Paid once; shared COW by children.import IPython # noqa: F401@@ -71,7 +100,12 @@ def _run_child(connection_path, cwd, env):# instance (and, critically, a jupyter_client Session created in *this* pid;# a Session inherited from the template silently drops messages via check_pid).IPKernelApp.clear_instance()- app = IPKernelApp.instance(connection_file=connection_path)+ # Watch our real parent (the forkserver): ipykernel distrusts a handle that+ # differs from getppid(), and its pid-1 fallback breaks under subreapers.+ app = IPKernelApp.instance(+ connection_file=connection_path,+ parent_handle=os.getppid(),+ )# initialize() binds the 5 ZMQ ports, writes the resolved ports back into# connection.json, and starts the heartbeat thread + ioloop — all post-fork,# so no thread/loop/socket is ever inherited across the fork boundary.@@ -84,8 +118,18 @@ def _serve(control_path):sock.connect(control_path)control_fd = sock.fileno()+ # Block SIGCHLD before spawning the watcher so it inherits the blocked mask;+ # otherwise the kernel could deliver SIGCHLD there while the main thread is masked.+ signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGCHLD})++ # Compare against the original ppid (not ==1) so this holds under subreapers.+ # Threads aren't inherited across fork, so children never carry this watcher.+ threading.Thread(target=_watch_parent, args=(os.getppid(),), daemon=True).start()+# Reap forked kernels as they exit, independent of the request loop.signal.signal(signal.SIGCHLD, _reap_children)+ # Unblock after handler install; the main thread's mask is now authoritative.+ signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGCHLD})_import_template()# Freeze the heap so the cyclic GC doesn't write to (and thus COW-copy) the@@ -109,19 +153,57 @@ def _serve(control_path):except ValueError:continuereq_id = req.get("id")++ kill_id = req.get("kill")+ if kill_id is not None:+ # Blocked across check+kill: an alive entry then means an un-reaped+ # own child (zombie-pinned pid), so os.kill is race-free.+ signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGCHLD})+ try:+ entry = _children.get(kill_id)+ if entry is None:+ outcome = "unknown-pid"+ elif not entry[1]:+ outcome = "already-exited"+ else:+ sig = signal.SIGKILL if req.get("signal") == "KILL" else signal.SIGTERM+ try:+ os.kill(entry[0], sig)+ outcome = "signaled"+ except ProcessLookupError:+ outcome = "already-exited"+ finally:+ signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGCHLD})+ f.write(json.dumps({"id": req_id, "outcome": outcome}).encode() + b"\n")+ f.flush()+ continue++ alive_id = req.get("alive")+ if alive_id is not None:+ entry = _children.get(alive_id)+ f.write(json.dumps({"id": req_id, "alive": bool(entry and entry[1])}).encode() + b"\n")+ f.flush()+ continue+connection_path = req.get("connectionPath")cwd = req.get("cwd")env = req.get("env")+ # Blocked across fork+bookkeeping: a fast-exiting child could otherwise+ # be reaped before its registry entry exists, recording a stale pid as alive.+ signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGCHLD})try:pid = os.fork()except OSError as exc:+ signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGCHLD})f.write(json.dumps({"id": req_id, "error": str(exc)}).encode() + b"\n")f.flush()continueif pid == 0:- # Child: shed every inherited fd tied to the control channel, then run.+ # The child inherits the blocked mask; the kernel needs SIGCHLD.+ signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGCHLD})+ # Shed every inherited fd tied to the control channel, then run.try:sock.close()f.close()@@ -139,10 +221,23 @@ def _serve(control_path):os._exit(0)# Parent: stay pristine (no loop/threads/ZMQ ever) so the next fork is clean.+ _children[req_id] = [pid, True]+ _pid_to_id[pid] = req_id+ # FIFO-evict only exited entries: dropping a live child would make its+ # liveness read false and its kill unroutable — the exact leak this+ # forkserver exists to prevent. Live entries are bounded by real kernels.+ if len(_children) > _history_bound:+ for evicted_id in [i for i, e in _children.items() if not e[1]]:+ if len(_children) <= _history_bound:+ break+ _children.pop(evicted_id)+ signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGCHLD})f.write(json.dumps({"id": req_id, "pid": pid}).encode() + b"\n")f.flush()if __name__ == "__main__":+ if len(sys.argv) > 2:+ _history_bound = int(sys.argv[2])_serve(sys.argv[1])`;diff --git a/packages/coding-agent/src/core/kernel/fork-server.ts b/packages/coding-agent/src/core/kernel/fork-server.tsindex d5b5423ac2..a6c1240d74 100644--- a/packages/coding-agent/src/core/kernel/fork-server.ts+++ b/packages/coding-agent/src/core/kernel/fork-server.ts@@ -12,6 +12,7 @@ import { createServer, type Server, type Socket } from "node:net";import { tmpdir } from "node:os";import { join } from "node:path";import { registerSessionResourceCleanup } from "@earendil-works/pi-ai";+import { recordOrphanProcessState } from "../orphan-process-journal.js";import { FORK_SERVER_SCRIPT } from "./fork-server-script.js";const READY_TIMEOUT_MS = 30_000;@@ -33,9 +34,13 @@ function affectsInterpreterStartup(key: string): boolean {}export class ForkServerUnavailable extends Error {- constructor(message: string) {+ /** True for a request timeout: the forkserver may just be stalled (e.g. mid-fork), not gone. */+ readonly timedOut: boolean;++ constructor(message: string, opts?: { timedOut?: boolean }) {super(message);this.name = "ForkServerUnavailable";+ this.timedOut = opts?.timedOut ?? false;}}@@ -51,6 +56,8 @@ export function isForkServerEnabled(): boolean {// forked child, so every kernel for a given python shares ONE template.interface ForkServerParams {python: string;+ // Registry-history bound override (argv[2] to the script); tests only.+ historyBound?: number;}interface SpawnParams {@@ -61,13 +68,37 @@ interface SpawnParams {env?: Record<string, string | undefined>;}-type PendingSpawn = {- resolve: (pid: number) => void;+export type ForkedKernelKillSignal = "TERM" | "KILL";+export type ForkedKernelKillOutcome = "signaled" | "already-exited" | "unknown-pid";++/**+ * Handle to a forkserver-forked kernel: signaling/liveness go through the+ * forkserver by fork request id, never process.kill (see the protocol header+ * in fork-server-script.ts). `pid` exists for journal writes only.+ */+export interface ForkedKernelHandle {+ readonly pid: number;+ kill(signal: ForkedKernelKillSignal): Promise<ForkedKernelKillOutcome>;+ isAlive(): Promise<boolean>;+}++interface ForkServerReply {+ id: number;+ pid?: number;+ error?: string;+ outcome?: string;+ alive?: boolean;+}++type PendingRequest = {+ resolve: (msg: ForkServerReply) => void;reject: (err: Error) => void;timer: ReturnType<typeof setTimeout>;};-class ForkServer {+// Exported for protocol-level tests (which construct it directly, bypassing the+// isForkServerEnabled platform gate); production entry stays forkKernel().+export class ForkServer {private readonly params: ForkServerParams;// The env the template process was launched with, snapshotted at construction so// it exactly matches what the template's interpreter imported with. The@@ -85,7 +116,7 @@ class ForkServer {// child's import/startup traceback lands here; surfaced in error messages.private stderrTail = "";private nextId = 1;- private readonly pending = new Map<number, PendingSpawn>();+ private readonly pending = new Map<number, PendingRequest>();// Request ids whose caller timed out; a late pid reply for one is an orphan to kill.private readonly abandoned = new Set<number>();private dead = false;@@ -172,11 +203,14 @@ class ForkServer {server.listen(socketPath, () => {// The template only imports; its own cwd/env are irrelevant since each// forked child applies the per-kernel cwd/env itself. Inherit the daemon's.- const proc = spawn(this.params.python, ["-c", FORK_SERVER_SCRIPT, socketPath], {+ const args = ["-c", FORK_SERVER_SCRIPT, socketPath];+ if (this.params.historyBound !== undefined) args.push(String(this.params.historyBound));+ const proc = spawn(this.params.python, args, {env: this.launchEnv,stdio: ["ignore", "ignore", "pipe"],});this.proc = proc;+ if (proc.pid !== undefined) recordOrphanProcessState(proc.pid, true);proc.stderr?.on("data", (buf: Buffer) => {this.stderrTail = `${this.stderrTail}${buf.toString()}`.slice(-STDERR_TAIL_MAX);});@@ -194,7 +228,7 @@ class ForkServer {const line = this.buffer.slice(0, idx);this.buffer = this.buffer.slice(idx + 1);if (!line.trim()) continue;- let msg: { type?: string; id?: number; pid?: number; error?: string };+ let msg: ForkServerReply & { type?: string };try {msg = JSON.parse(line);} catch {@@ -207,46 +241,73 @@ class ForkServer {if (typeof msg.id !== "number") continue;const p = this.pending.get(msg.id);if (!p) {- // A pid for a request the caller already abandoned (timed out): the- // fork succeeded but nobody owns it, so kill the orphan here.+ // A fork the caller already abandoned (timed out): nobody owns it,+ // so have the forkserver kill it.if (this.abandoned.delete(msg.id) && typeof msg.pid === "number") {- try {- process.kill(msg.pid, "SIGTERM");- } catch {- // Orphan already exited.- }+ void this.killChild(msg.id, "TERM").catch(() => {});}continue;}this.pending.delete(msg.id);clearTimeout(p.timer);- if (typeof msg.pid === "number") {- p.resolve(msg.pid);- } else {- p.reject(new ForkServerUnavailable(this.withStderr(msg.error ?? "forkserver fork failed")));- }+ p.resolve(msg);}}- async spawnKernel(spawn: SpawnParams): Promise<number> {- await this.ensureReady();- if (this.dead || !this.conn) throw new ForkServerUnavailable("forkserver connection unavailable");+ private request(payload: Record<string, unknown>, opts?: { abandonOnTimeout?: boolean }): Promise<ForkServerReply> {+ if (this.dead || !this.conn) {+ return Promise.reject(new ForkServerUnavailable("forkserver connection unavailable"));+ }const id = this.nextId++;- return new Promise<number>((resolve, reject) => {+ return new Promise<ForkServerReply>((resolve, reject) => {const timer = setTimeout(() => {this.pending.delete(id);- // The fork may still land after we give up; remember the id so its late+ // A spawn may still land after we give up; remember the id so its late// pid reply gets the orphan killed instead of leaked.- this.abandoned.add(id);- reject(new ForkServerUnavailable(`forkserver spawn timed out after ${SPAWN_TIMEOUT_MS}ms`));+ if (opts?.abandonOnTimeout) this.abandoned.add(id);+ reject(+ new ForkServerUnavailable(`forkserver request timed out after ${SPAWN_TIMEOUT_MS}ms`, {+ timedOut: true,+ }),+ );}, SPAWN_TIMEOUT_MS);this.pending.set(id, { resolve, reject, timer });- this.conn?.write(- `${JSON.stringify({ id, connectionPath: spawn.connectionPath, cwd: spawn.cwd, env: spawn.env })}\n`,- );+ this.conn?.write(`${JSON.stringify({ id, ...payload })}\n`);});}+ async spawnKernel(spawn: SpawnParams): Promise<ForkedKernelHandle> {+ await this.ensureReady();+ const msg = await this.request(+ { connectionPath: spawn.connectionPath, cwd: spawn.cwd, env: spawn.env },+ { abandonOnTimeout: true },+ );+ if (typeof msg.pid !== "number") {+ throw new ForkServerUnavailable(this.withStderr(msg.error ?? "forkserver fork failed"));+ }+ const pid = msg.pid;+ const forkId = msg.id;+ return {+ pid,+ kill: (signal) => this.killChild(forkId, signal),+ isAlive: () => this.isChildAlive(forkId),+ };+ }++ async killChild(forkId: number, signal: ForkedKernelKillSignal): Promise<ForkedKernelKillOutcome> {+ const msg = await this.request({ kill: forkId, signal });+ const outcome = msg.outcome;+ if (outcome === "signaled" || outcome === "already-exited" || outcome === "unknown-pid") {+ return outcome;+ }+ throw new ForkServerUnavailable(`forkserver kill reply malformed: ${JSON.stringify(msg)}`);+ }++ async isChildAlive(forkId: number): Promise<boolean> {+ const msg = await this.request({ alive: forkId });+ return msg.alive === true;+ }+private withStderr(message: string): string {const tail = this.stderrTail.trim();return tail ? `${message}\nforkserver stderr:\n${tail}` : message;@@ -284,10 +345,18 @@ class ForkServer {} catch {// Already closed.}- try {- this.proc?.kill("SIGTERM");- } catch {- // Already exited.+ const proc = this.proc;+ if (proc) {+ // Inactive only when the pid is soundly ours at write time (handle saw the+ // exit, or handle-based kill delivered); otherwise leave the record active.+ const observedExit = proc.exitCode !== null || proc.signalCode !== null;+ let delivered = false;+ try {+ delivered = proc.kill("SIGTERM");+ } catch {+ // Already exited.+ }+ if (proc.pid !== undefined && (observedExit || delivered)) recordOrphanProcessState(proc.pid, false);}if (this.socketDir) {try {@@ -330,9 +399,9 @@ function registerForkServerCleanupOnce(): void {* Fork a kernel onto `spawn.connectionPath` from the shared template for this* interpreter, applying `spawn.cwd`/`spawn.env` in the forked child. Throws* ForkServerUnavailable if forking is disabled or fails — callers fall back to- * direct spawn. Returns the forked child's pid (owned/killed by the caller).+ * direct spawn. Returns a handle whose kill/liveness go through the forkserver.*/-export async function forkKernel(python: string, spawn: SpawnParams): Promise<number> {+export async function forkKernel(python: string, spawn: SpawnParams): Promise<ForkedKernelHandle> {if (!isForkServerEnabled()) throw new ForkServerUnavailable("forkserver disabled");registerForkServerCleanupOnce();const key = keyFor({ python });diff --git a/packages/coding-agent/src/core/kernel/index.ts b/packages/coding-agent/src/core/kernel/index.tsindex 1f5af4f1b3..e12cde055d 100644--- a/packages/coding-agent/src/core/kernel/index.ts+++ b/packages/coding-agent/src/core/kernel/index.ts@@ -7,8 +7,9 @@ import { setTimeout as sleep } from "node:timers/promises";import { registerSessionResourceCleanup } from "@earendil-works/pi-ai";import { v4 as uuid } from "uuid";import { Dealer, Subscriber } from "zeromq";+import { recordOrphanProcessState } from "../orphan-process-journal.js";import { ensureKernelPython, type KernelBootstrapProgressHandler, type KernelPythonSkill } from "./bootstrap.js";-import { ForkServerUnavailable, forkKernel, isForkServerEnabled } from "./fork-server.js";+import { type ForkedKernelHandle, ForkServerUnavailable, forkKernel, isForkServerEnabled } from "./fork-server.js";import {buildListNamesCode,buildRestoreCode,@@ -594,11 +595,12 @@ export class KernelManager {private readonly commTargets = new Map<string, string>();private readonly handledHostRequestCommIds = new Set<string>();private kernel?: ChildProcess;- // Set instead of `kernel` when the kernel was forked from the forkserver: it is- // not a direct child, so it has no ChildProcess handle and is killed by pid.- private kernelPid?: number;- /** Polls a forked kernel's pid for death (no "exit" event on a non-child). */+ // Set instead of `kernel` for forkserver-forked kernels (not our child):+ // signaling/liveness go through the forkserver, never process.kill.+ private forkedKernel?: ForkedKernelHandle;+ /** Polls a forked kernel for death (no "exit" event on a non-child). */private forkedLivenessTimer?: ReturnType<typeof globalThis.setInterval>;+ private forkedLivenessProbeInFlight = false;private shell?: Dealer;private iopub?: Subscriber;private control?: Dealer;@@ -619,6 +621,8 @@ export class KernelManager {private lastCellCode?: string;private readonly inFlightHostRequests = new Set<Promise<void>>();private state: "idle" | "starting" | "running" | "shutdown" = "idle";+ /** Bumped by every teardown so a stale in-flight doStart can never touch a newer kernel. */+ private startGeneration = 0;/** Memoized so concurrent callers all await the same in-flight startup. */private startPromise?: Promise<void>;/** Pending debounced auto-snapshot, if one has been scheduled. */@@ -650,16 +654,19 @@ export class KernelManager {throw createKernelStartupAbortError();}if (!this.startPromise) {- this.startPromise = this.doStart({ onBootstrapProgress: options.onBootstrapProgress }).catch((error) => {- this.startPromise = undefined;+ const startPromise = this.doStart({ onBootstrapProgress: options.onBootstrapProgress }).catch((error) => {+ // Only clear our own memoization: a stale start must not evict a newer one.+ if (this.startPromise === startPromise) this.startPromise = undefined;throw error;});+ this.startPromise = startPromise;}return raceStartupWithAbort(this.startPromise, options.signal);}private async doStart(startOptions: KernelStartOptions): Promise<void> {if (this.state !== "idle") return;+ const generation = ++this.startGeneration;this.state = "starting";installSignalHandlersOnce();// Tracked from the moment startup begins so session cleanup and signal@@ -674,8 +681,10 @@ export class KernelManager {pythonSkills: this.options.pythonSkills,onProgress: startOptions.onBootstrapProgress,}));+ if (this.startStale(generation)) throw new Error("Kernel start superseded");this.options.python = python;} catch (error) {+ if (this.startStale(generation)) throw error; // never touch a newer start's stateliveKernels.delete(this);if ((this.state as string) !== "shutdown") this.state = "idle";throw error;@@ -694,19 +703,26 @@ export class KernelManager {let forked = false;if (isForkServerEnabled()) {try {- this.kernelPid = await forkKernel(python, {+ const handle = await forkKernel(python, {connectionPath: connection.path,cwd: this.options.cwd,- // Match the direct-spawn env exactly: merge the current host env with- // the per-kernel overrides, applied fresh in the child (the template's- // inherited env snapshot may be stale by fork time).- env: this.options.env ? { ...process.env, ...this.options.env } : { ...process.env },+ // Applied fresh in the child (the template's env snapshot may be stale).+ // No JPY_PARENT_PID: forked children watch the forkserver by getppid().+ env: { ...process.env, ...this.options.env },});+ if (this.startStale(generation)) {+ // Nobody owns this kernel; the protocol kill is id-keyed and safe.+ void handle.kill("TERM").catch(() => {});+ throw new Error("Kernel start superseded");+ }+ this.forkedKernel = handle;+ recordOrphanProcessState(handle.pid, true);forked = true;} catch (err) {+ if (this.startStale(generation)) throw err; // never touch a newer start's stateif (!(err instanceof ForkServerUnavailable)) throw err;this.appendKernelDiagnostic(`forkserver unavailable, spawning directly: ${err.message}`);- this.kernelPid = undefined;+ this.forkedKernel = undefined;// A fork request that times out or loses its pid reply may still have// forked a child that binds the ports in this connection file. Mint a// fresh connection for the direct spawn so a possible orphan can never@@ -725,10 +741,12 @@ export class KernelManager {if (!forked) {const kernel = spawn(python, ["-m", "ipykernel_launcher", "-f", connection.path], {cwd: this.options.cwd,- env: this.options.env ? { ...process.env, ...this.options.env } : process.env,+ // ipykernel's parent poller exits the kernel if this pid dies (covers SIGKILL of the owner).+ env: { ...process.env, ...this.options.env, JPY_PARENT_PID: String(process.pid) },stdio: ["ignore", "pipe", "pipe"],});this.kernel = kernel;+ if (kernel.pid !== undefined) recordOrphanProcessState(kernel.pid, true);kernel.stderr?.on("data", (buf: Buffer) => {const s = buf.toString();@@ -758,11 +776,14 @@ export class KernelManager {let conn: ConnectionInfo;try {conn = await this.waitForResolvedConnection(connectionPath);+ if (this.startStale(generation)) throw new Error("Kernel start superseded");this.connection = conn;} catch (e) {+ if (this.startStale(generation)) throw e; // never tear down a newer start's kernelconst canRetryStartup = (this.state as string) !== "shutdown";- await this.shutdown();- if (canRetryStartup) this.state = "idle";+ // Only the call that performed the cleanup may resurrect to idle; a+ // concurrent kill()/teardown owns the state otherwise.+ if ((await this.shutdown()) && canRetryStartup) this.state = "idle";throw e;}@@ -777,14 +798,18 @@ export class KernelManager {// ZMQ PUB/SUB slow-joiner: give the subscription a brief chance to reach the kernel before first execute.await sleep(IOPUB_SUBSCRIBE_DELAY_MS);+ if (this.startStale(generation)) throw new Error("Kernel start superseded");this.startIopubPump();try {await this.probeReady();+ if (this.startStale(generation)) throw new Error("Kernel start superseded");} catch (e) {+ if (this.startStale(generation)) throw e; // never tear down a newer start's kernelconst canRetryStartup = (this.state as string) !== "shutdown";- await this.shutdown();- if (canRetryStartup) this.state = "idle";+ // Only the call that performed the cleanup may resurrect to idle; a+ // concurrent kill()/teardown owns the state otherwise.+ if ((await this.shutdown()) && canRetryStartup) this.state = "idle";throw e;}@@ -792,40 +817,64 @@ export class KernelManager {this.startForkedLivenessMonitor();}- // A forked kernel isn't a direct child, so no "exit" fires when it dies. Poll its- // pid so a mid-run death tears down like the direct-spawn exit handler: mark- // shutdown, drop from liveKernels, and reject any in-flight execution.+ /** True when a teardown (or newer start) superseded the start that captured `generation`. */+ private startStale(generation: number): boolean {+ return generation !== this.startGeneration;+ }++ // No "exit" event fires for a non-child; poll the forkserver so a mid-run+ // death tears down like the direct-spawn exit handler.private startForkedLivenessMonitor(): void {- if (this.kernelPid === undefined) return;+ if (!this.forkedKernel) return;this.forkedLivenessTimer = globalThis.setInterval(() => {- if (this.state !== "running") return;- if (!this.forkedKernelDied()) return;- this.appendKernelDiagnostic("forked kernel exited unexpectedly");- this.state = "shutdown";- liveKernels.delete(this);- this.cleanupResources();+ void this.checkForkedKernelDeath();}, FORKED_LIVENESS_POLL_MS);this.forkedLivenessTimer.unref?.();}- // A forked kernel is not a direct child, so it emits no "exit" event; poll its- // pid so a dead child fails fast instead of burning the full resolve timeout.- private forkedKernelDied(): boolean {- if (this.kernelPid === undefined) return false;+ private async checkForkedKernelDeath(): Promise<void> {+ if (this.state !== "running" || this.forkedLivenessProbeInFlight) return;+ const probed = this.forkedKernel;+ this.forkedLivenessProbeInFlight = true;try {- process.kill(this.kernelPid, 0);- return false;+ if (!(await this.forkedKernelDead(probed))) return;+ } finally {+ this.forkedLivenessProbeInFlight = false;+ }+ // Re-check after the await: teardown or a restart may have raced this poll.+ if (this.state !== "running" || this.forkedKernel !== probed) return;+ this.appendKernelDiagnostic("forked kernel exited unexpectedly");+ this.state = "shutdown";+ liveKernels.delete(this);+ this.cleanupResources();+ }++ // Liveness from the forkserver's reap table; a pid-0 probe would race reuse.+ // `timeoutMs` bounds the probe (timeout counts as alive so the caller's own+ // deadline decides); without it the protocol request timeout applies.+ private async forkedKernelDead(probed: ForkedKernelHandle | undefined, timeoutMs?: number): Promise<boolean> {+ if (!probed) return false;+ try {+ const alive = probed.isAlive();+ if (timeoutMs === undefined) return !(await alive);+ alive.catch(() => {}); // absorb a rejection that lands after the race is lost+ return !(await Promise.race([alive, sleep(timeoutMs, true, { ref: false })]));} catch (error) {- // EPERM means the pid exists but isn't signalable by us — still alive.- // Only ESRCH (no such process) is genuine death.- return !(error instanceof Error && (error as NodeJS.ErrnoException).code === "EPERM");+ // A timeout is unknown liveness, not proven death (the forkserver may just be stalled in a slow fork).+ if (error instanceof ForkServerUnavailable && error.timedOut) return false;+ // Forkserver gone: its kernels' parent_handle watchdogs exit them too.+ return true;}}private async waitForResolvedConnection(connectionPath: string): Promise<ConnectionInfo> {const startedAt = Date.now();while (Date.now() - startedAt < PORTS_RESOLVE_TIMEOUT_MS) {- if ((this.state as string) === "shutdown" || this.forkedKernelDied()) {+ const remainingBudget = PORTS_RESOLVE_TIMEOUT_MS - (Date.now() - startedAt);+ if (+ (this.state as string) === "shutdown" ||+ (await this.forkedKernelDead(this.forkedKernel, remainingBudget))+ ) {const tail = this.kernelStderr.slice(-1024);throw new Error(`Kernel exited before resolving ports. stderr:\n${tail || "(empty)"}`);}@@ -854,7 +903,11 @@ export class KernelManager {const startedAt = Date.now();while (Date.now() - startedAt < READY_TIMEOUT_MS) {- if ((this.state as string) === "shutdown" || this.forkedKernelDied()) {+ const remainingBudget = READY_TIMEOUT_MS - (Date.now() - startedAt);+ if (+ (this.state as string) === "shutdown" ||+ (await this.forkedKernelDead(this.forkedKernel, remainingBudget))+ ) {const tail = this.kernelStderr.slice(-1024);throw new Error(`Kernel exited during startup. stderr:\n${tail || "(empty)"}`);}@@ -1462,6 +1515,7 @@ export class KernelManager {}private cleanupResources(killSignal: NodeJS.Signals = "SIGTERM"): void {+ this.startGeneration++; // any teardown invalidates in-flight startsthis.clearSnapshotTimer();this.lateSentAgentMessageHandlers.clear();if (this.forkedLivenessTimer) {@@ -1478,19 +1532,32 @@ export class KernelManager {this.control = undefined;this.iopubPumpPromise = undefined;this.controlPumpPromise = undefined;- try {- if (this.kernel) {- this.kernel.kill(killSignal);- } else if (this.kernelPid !== undefined && !this.forkedKernelDied()) {- // Only signal a forked kernel confirmed still alive: a dead pid may have- // been recycled by the OS, and a kill would then hit an unrelated process.- process.kill(this.kernelPid, killSignal);+ if (this.kernel) {+ const directPid = this.kernel.pid;+ let signaled = false;+ try {+ signaled = this.kernel.kill(killSignal);+ } catch {+ // The kernel has already exited.}- } catch {- // The kernel has already exited.+ // Same rule as the forked branch below: inactive only when the signal proved the pid still ours.+ if (directPid !== undefined && signaled) recordOrphanProcessState(directPid, false);+ } else if (this.forkedKernel) {+ const forked = this.forkedKernel;+ // The journal is raw-pid keyed, so inactive is written only on "signaled"+ // — the one outcome proving the pid still named our un-reaped child. Any+ // other outcome leaves the record stale-active: the reaper's startId check+ // neutralizes it, while a wrong inactive write could mask a sibling's+ // record for a reused pid.+ void forked+ .kill(killSignal === "SIGKILL" ? "KILL" : "TERM")+ .then((outcome) => {+ if (outcome === "signaled") recordOrphanProcessState(forked.pid, false);+ })+ .catch(() => this.appendKernelDiagnostic("forkserver kill unconfirmed; leaving orphan record active"));}this.kernel = undefined;- this.kernelPid = undefined;+ this.forkedKernel = undefined;this.connection = undefined;if (this.tempDir) {try {@@ -1510,9 +1577,9 @@ export class KernelManager {await new Promise<void>((resolve) => kernel.once("exit", () => resolve()));return;}- const pid = this.kernelPid;- if (pid === undefined) return;- while (this.kernelPid === pid && !this.forkedKernelDied()) {+ const forked = this.forkedKernel;+ if (!forked) return;+ while (this.forkedKernel === forked && !(await this.forkedKernelDead(forked))) {await sleep(25);}}@@ -1537,22 +1604,27 @@ export class KernelManager {}}- async shutdown(opts: { snapshot?: boolean } = {}): Promise<void> {+ /** Resolves true when this call performed the cleanup (false: a concurrent teardown won). */+ async shutdown(opts: { snapshot?: boolean } = {}): Promise<boolean> {if (this.state === "shutdown") {liveKernels.delete(this);this.cleanupResources();- return;+ return true;}+ // Captured before any await: teardowns and newer starts bump the counter.+ const generation = this.startGeneration;// Best-effort final flush (bounded) before teardown — used by signal handlers// so a SIGINT/SIGTERM exit doesn't lose work the debounced snapshot hasn't saved.if (opts.snapshot) {await this.flushSnapshotForDispose();+ if (this.startStale(generation)) return false; // superseded mid-flush: the newer owner already cleaned this kernel}this.state = "shutdown";liveKernels.delete(this);let replyWait: { promise: Promise<void>; cancel: () => void } | undefined;let shutdownTimer: ReturnType<typeof globalThis.setTimeout> | undefined;+ let performedCleanup = false;const shutdownDeadline = new Promise<never>((_resolve, reject) => {shutdownTimer = globalThis.setTimeout(() => reject(new Error(`Kernel did not shut down within ${KERNEL_SHUTDOWN_TIMEOUT_MS}ms`)),@@ -1581,8 +1653,15 @@ export class KernelManager {} finally {if (shutdownTimer) globalThis.clearTimeout(shutdownTimer);replyWait?.cancel();- this.cleanupResources();+ // A superseded shutdown must not tear down the newer start's sockets. Ownership is decided+ // here, before cleanupResources bumps the generation and would misread this call as superseded.+ if (!this.startStale(generation)) {+ this.cleanupResources();+ performedCleanup = true;+ }}++ return performedCleanup;}async restart(): Promise<void> {@@ -1733,8 +1812,11 @@ export class KernelManager {/** Graceful cleanup. Waits briefly for in-flight host request handlers before closing sockets. */dispose(): Promise<void> {return (async () => {+ // Captured before any await: teardowns and newer starts bump the counter.+ const generation = this.startGeneration;// Final namespace flush while the kernel is still live (session end / reload).await this.flushSnapshotForDispose();+ if (this.startStale(generation)) return; // superseded mid-flush: the newer owner already cleaned this kernelthis.state = "shutdown";liveKernels.delete(this);const inFlightHostRequests = [...this.inFlightHostRequests];@@ -1744,7 +1826,7 @@ export class KernelManager {await this.waitForHostRequestsToSettle(inFlightHostRequests, HOST_REQUEST_DISPOSE_TIMEOUT_MS);}} finally {- this.cleanupResources();+ if (!this.startStale(generation)) this.cleanupResources(); // else: superseded, the newer owner already cleaned}})();}diff --git a/packages/coding-agent/test/kernel-fork-server-protocol.test.ts b/packages/coding-agent/test/kernel-fork-server-protocol.test.tsnew file mode 100644index 0000000000..3967132779--- /dev/null+++ b/packages/coding-agent/test/kernel-fork-server-protocol.test.ts@@ -0,0 +1,366 @@+import { spawnSync } from "node:child_process";+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { homedir, tmpdir } from "node:os";+import { join } from "node:path";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { type ForkedKernelHandle, ForkServer, ForkServerUnavailable } from "../src/core/kernel/fork-server.js";+import { ORPHAN_PROCESS_JOURNAL_ENV } from "../src/core/orphan-process-journal.js";++// Drives the REAL Python forkserver script through the REAL ForkServer class.+// Stub Python modules stand in for IPython/ipykernel so the forked child stays+// alive without a real kernel — fork itself is safe here even on macOS because+// the child never touches the frameworks that make fork-without-exec unsafe.+const STUB_KERNELAPP = [+ "import os",+ "import time",+ "",+ "",+ "class IPKernelApp:",+ " @classmethod",+ " def clear_instance(cls):",+ " pass",+ "",+ " @classmethod",+ " def instance(cls, **_kwargs):",+ " return cls()",+ "",+ " def initialize(self, _argv):",+ " pass",+ "",+ " def start(self):",+ " # Env knob: lets tests fork a child that exits immediately (the",+ " # per-kernel env is applied in the child before start()).",+ " if os.environ.get('STUB_KERNEL_EXIT'):",+ " return",+ " while True:",+ " time.sleep(1)",+ "",+].join("\n");++function writeStubModules(dir: string): void {+ writeFileSync(join(dir, "IPython.py"), "");+ writeFileSync(join(dir, "jupyter_client.py"), "");+ writeFileSync(join(dir, "nest_asyncio.py"), "");+ const ipykernelDir = join(dir, "ipykernel");+ mkdirSync(ipykernelDir);+ writeFileSync(join(ipykernelDir, "__init__.py"), "");+ writeFileSync(join(ipykernelDir, "kernelapp.py"), STUB_KERNELAPP);+}++function killQuietly(pid: number | undefined): void {+ if (pid === undefined) return;+ try {+ process.kill(pid, "SIGKILL");+ } catch {+ // Already exited.+ }+}++const havePython3 = process.platform !== "win32" && spawnSync("python3", ["-V"]).status === 0;+const describeIf = havePython3 ? describe : describe.skip;++describeIf("forkserver kill/liveness protocol (stub python)", () => {+ let tempDir = "";+ let server: ForkServer | undefined;+ let savedPythonPath: string | undefined;+ const leakedPids: number[] = [];++ beforeEach(() => {+ tempDir = mkdtempSync(join(tmpdir(), "prime-agent-forkserver-proto-"));+ const stubDir = join(tempDir, "stubs");+ mkdirSync(stubDir);+ writeStubModules(stubDir);+ // launchEnv is snapshotted at ForkServer construction, so the template+ // imports the stubs.+ savedPythonPath = process.env.PYTHONPATH;+ process.env.PYTHONPATH = stubDir;+ server = new ForkServer({ python: "python3" });+ });++ afterEach(() => {+ if (savedPythonPath === undefined) delete process.env.PYTHONPATH;+ else process.env.PYTHONPATH = savedPythonPath;+ server?.dispose();+ server = undefined;+ // The stub app ignores parent_handle, so leak-proof the test itself.+ for (const pid of leakedPids.splice(0)) killQuietly(pid);+ rmSync(tempDir, { recursive: true, force: true });+ });++ async function spawnStubKernel(+ target: ForkServer = server!,+ env?: Record<string, string | undefined>,+ ): Promise<ForkedKernelHandle> {+ const handle = await target.spawnKernel({ connectionPath: join(tempDir, "conn.json"), env });+ leakedPids.push(handle.pid);+ return handle;+ }++ it("kills its own child through the protocol and observes the reap", async () => {+ const handle = await spawnStubKernel();+ expect(await handle.isAlive()).toBe(true);+ expect(await handle.kill("TERM")).toBe("signaled");+ await vi.waitFor(async () => {+ expect(await handle.isAlive()).toBe(false);+ });+ // OS-level confirmation (test-side observation only, never a signal path).+ await vi.waitFor(() => {+ expect(() => process.kill(handle.pid, 0)).toThrow();+ });+ }, 15_000);++ it("reports already-exited after reap and fails closed on an unknown fork id", async () => {+ const handle = await spawnStubKernel();+ expect(await handle.kill("TERM")).toBe("signaled");+ await vi.waitFor(async () => {+ expect(await handle.kill("TERM")).toBe("already-exited");+ });+ // A fork id the forkserver never issued: nothing may be signaled.+ expect(await server!.killChild(999_999, "TERM")).toBe("unknown-pid");+ expect(await server!.isChildAlive(999_999)).toBe(false);+ }, 15_000);++ it("liveness reflects the reap table on external child death", async () => {+ const handle = await spawnStubKernel();+ expect(await handle.isAlive()).toBe(true);+ // External death (not via the protocol): proves the SIGCHLD reaper drives+ // the registry independently of the kill path.+ process.kill(handle.pid, "SIGTERM");+ await vi.waitFor(async () => {+ expect(await handle.isAlive()).toBe(false);+ });+ }, 15_000);++ it("kill outcomes stay correct while sibling children churn and get reaped", async () => {+ // Regression for the SIGCHLD-in-watcher-thread race: external deaths storm+ // the reaper while kill requests are in flight; every reply must still match+ // the child's true state (no false "signaled" for a freed pid).+ const keep = await Promise.all([spawnStubKernel(), spawnStubKernel(), spawnStubKernel()]);+ const churn = await Promise.all([spawnStubKernel(), spawnStubKernel(), spawnStubKernel()]);+ for (const handle of churn) process.kill(handle.pid, "SIGKILL");+ const outcomes = await Promise.all(keep.map((handle) => handle.kill("TERM")));+ expect(outcomes).toEqual(["signaled", "signaled", "signaled"]);+ for (const handle of churn) {+ await vi.waitFor(async () => {+ expect(await handle.kill("TERM")).toBe("already-exited");+ });+ }+ }, 15_000);++ it("a fast-exiting child never lands alive; its handle stays already-exited across new forks", async () => {+ // STUB_KERNEL_EXIT makes the forked child return from start() immediately,+ // so it can be reaped as early as the OS allows relative to the parent-side+ // registry insertion — the SIGCHLD-blocked fork bookkeeping must still+ // record it under its own id and mark exactly that entry not-alive.+ const fast = await spawnStubKernel(server!, { STUB_KERNEL_EXIT: "1" });+ await vi.waitFor(async () => {+ expect(await fast.isAlive()).toBe(false);+ });+ expect(await fast.kill("TERM")).toBe("already-exited");+ // Id-keying invariant: a later fork gets a fresh id (even if the OS reuses+ // the pid, which we can't force), and the old handle's answers don't change.+ const next = await spawnStubKernel();+ expect(await next.isAlive()).toBe(true);+ expect(await fast.isAlive()).toBe(false);+ expect(await fast.kill("TERM")).toBe("already-exited");+ expect(await next.kill("TERM")).toBe("signaled");+ }, 15_000);++ it("evicted registry entries fail closed (unknown-pid/false), recent ones stay answerable", async () => {+ // A tiny history bound (argv[2] to the script) makes FIFO eviction reachable.+ const bounded = new ForkServer({ python: "python3", historyBound: 2 });+ try {+ const oldest = await spawnStubKernel(bounded, { STUB_KERNEL_EXIT: "1" });+ await vi.waitFor(async () => {+ expect(await oldest.kill("TERM")).toBe("already-exited");+ });+ const middle = await spawnStubKernel(bounded, { STUB_KERNEL_EXIT: "1" });+ const newest = await spawnStubKernel(bounded, { STUB_KERNEL_EXIT: "1" });+ // Three entries against a bound of 2: the oldest id has been evicted.+ expect(await oldest.kill("TERM")).toBe("unknown-pid");+ expect(await oldest.isAlive()).toBe(false);+ for (const handle of [middle, newest]) {+ await vi.waitFor(async () => {+ expect(await handle.kill("TERM")).toBe("already-exited");+ });+ expect(await handle.isAlive()).toBe(false);+ }+ } finally {+ bounded.dispose();+ }+ }, 15_000);++ it("keeps live entries answerable past the history bound and evicts only exited ones", async () => {+ // A live child must never be evicted: an evicted-live entry would read+ // dead to the liveness monitor and be unroutable for kill — the exact+ // orphan leak this forkserver exists to prevent.+ const bounded = new ForkServer({ python: "python3", historyBound: 2 });+ try {+ const oldest = await spawnStubKernel(bounded);+ expect(await oldest.isAlive()).toBe(true);+ const middle = await spawnStubKernel(bounded);+ const newest = await spawnStubKernel(bounded);+ // Three live entries against a bound of 2: none may be evicted.+ expect(await oldest.isAlive()).toBe(true);+ expect(await middle.isAlive()).toBe(true);+ expect(await newest.isAlive()).toBe(true);+ // Once dead entries exist, the next fork sweeps them out instead.+ expect(await oldest.kill("TERM")).toBe("signaled");+ expect(await middle.kill("TERM")).toBe("signaled");+ await vi.waitFor(async () => {+ expect(await oldest.isAlive()).toBe(false);+ expect(await middle.isAlive()).toBe(false);+ });+ const extra = await spawnStubKernel(bounded);+ // Bound 2 with two dead entries: both are evicted, the live ones stay.+ expect(await oldest.kill("TERM")).toBe("unknown-pid");+ expect(await middle.kill("TERM")).toBe("unknown-pid");+ expect(await newest.isAlive()).toBe(true);+ expect(await extra.isAlive()).toBe(true);+ expect(await newest.kill("TERM")).toBe("signaled");+ expect(await extra.kill("TERM")).toBe("signaled");+ } finally {+ bounded.dispose();+ }+ }, 15_000);++ it("handle kill/isAlive reject with ForkServerUnavailable when the server is dead", async () => {+ const handle = await spawnStubKernel();+ server!.dispose();+ await expect(handle.kill("TERM")).rejects.toBeInstanceOf(ForkServerUnavailable);+ await expect(handle.isAlive()).rejects.toBeInstanceOf(ForkServerUnavailable);+ }, 15_000);++ describe("forkserver orphan journal", () => {+ let journalPath = "";+ const savedJournal = process.env[ORPHAN_PROCESS_JOURNAL_ENV];++ interface JournalRecord {+ pid: number;+ active: boolean;+ }++ function readJournal(): JournalRecord[] {+ if (!existsSync(journalPath)) return [];+ return readFileSync(journalPath, "utf8")+ .split("\n")+ .filter(Boolean)+ .map((line) => JSON.parse(line) as JournalRecord);+ }++ beforeEach(() => {+ journalPath = join(tempDir, "orphans.jsonl");+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;+ });++ afterEach(() => {+ if (savedJournal === undefined) delete process.env[ORPHAN_PROCESS_JOURNAL_ENV];+ else process.env[ORPHAN_PROCESS_JOURNAL_ENV] = savedJournal;+ });++ it("disposing a live forkserver delivers the kill and writes inactive", async () => {+ await spawnStubKernel();+ const records = readJournal();+ expect(records).toHaveLength(1);+ expect(records[0]).toMatchObject({ active: true });+ server!.dispose();+ const after = readJournal();+ expect(after).toHaveLength(2);+ expect(after[1]).toMatchObject({ pid: records[0]!.pid, active: false });+ }, 15_000);++ it("a forkserver exit observed by the handle writes inactive", async () => {+ await spawnStubKernel();+ const forkserverPid = readJournal()[0]!.pid;+ process.kill(forkserverPid, "SIGKILL");+ // The exit event drives markDead → dispose with the exit already observed.+ await vi.waitFor(() => {+ const records = readJournal();+ expect(records).toHaveLength(2);+ expect(records[1]).toMatchObject({ pid: forkserverPid, active: false });+ });+ }, 15_000);++ it("a never-started forkserver writes no journal records on dispose", () => {+ const idle = new ForkServer({ python: "python3" });+ idle.dispose();+ expect(readJournal()).toHaveLength(0);+ });++ it("an unconfirmed kill with no observed exit leaves no inactive record", () => {+ const unconfirmed = new ForkServer({ python: "python3" });+ const internals = unconfirmed as unknown as {+ proc?: { pid?: number; exitCode: number | null; signalCode: string | null; kill(): boolean };+ };+ internals.proc = { pid: 424242, exitCode: null, signalCode: null, kill: () => false };+ unconfirmed.dispose();+ expect(readJournal().some((r) => r.pid === 424242)).toBe(false);+ });+ });+});++function resolveKernelPython(): string | null {+ const candidates = [+ process.env.PRIME_AGENT_KERNEL_PYTHON,+ join(homedir(), ".prime", "agent", "kernel-venv", "bin", "python"),+ ].filter((p): p is string => Boolean(p));+ for (const python of candidates) {+ if (!existsSync(python)) continue;+ const check = spawnSync(python, ["-c", "import ipykernel"], { encoding: "utf8" });+ if (check.status === 0) return python;+ }+ return null;+}++const kernelPython = resolveKernelPython();+// Real ipykernel fork round-trip is linux-only: on darwin the forked child dies+// immediately (fork-without-exec is unsafe there), matching isForkServerEnabled.+const describeIfRealKernel = process.platform === "linux" && kernelPython ? describe : describe.skip;++describeIfRealKernel("forkserver kill/liveness protocol (real kernel)", { tags: ["kernel-heavy"] }, () => {+ it("forks a real ipykernel, resolves ports, kills it via the protocol", async () => {+ const dir = mkdtempSync(join(tmpdir(), "prime-agent-forkserver-real-"));+ const connectionPath = join(dir, "connection.json");+ writeFileSync(+ connectionPath,+ JSON.stringify({+ ip: "127.0.0.1",+ transport: "tcp",+ shell_port: 0,+ iopub_port: 0,+ stdin_port: 0,+ control_port: 0,+ hb_port: 0,+ signature_scheme: "hmac-sha256",+ key: "test-key",+ kernel_name: "python3",+ }),+ { mode: 0o600 },+ );+ const server = new ForkServer({ python: kernelPython! });+ let pid: number | undefined;+ try {+ const handle = await server.spawnKernel({ connectionPath });+ pid = handle.pid;+ await vi.waitFor(+ () => {+ const info = JSON.parse(readFileSync(connectionPath, "utf8")) as { shell_port: number };+ expect(info.shell_port).toBeGreaterThan(0);+ },+ { timeout: 20_000, interval: 250 },+ );+ expect(await handle.isAlive()).toBe(true);+ expect(await handle.kill("TERM")).toBe("signaled");+ await vi.waitFor(+ async () => {+ expect(await handle.isAlive()).toBe(false);+ },+ { timeout: 20_000, interval: 250 },+ );+ } finally {+ server.dispose();+ killQuietly(pid);+ rmSync(dir, { recursive: true, force: true });+ }+ }, 60_000);+});diff --git a/packages/coding-agent/test/kernel-parent-watchdog.test.ts b/packages/coding-agent/test/kernel-parent-watchdog.test.tsnew file mode 100644index 0000000000..565b312ee7--- /dev/null+++ b/packages/coding-agent/test/kernel-parent-watchdog.test.ts@@ -0,0 +1,548 @@+import { spawn, spawnSync } from "node:child_process";+import { EventEmitter } from "node:events";+import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { homedir, tmpdir } from "node:os";+import { join } from "node:path";+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";+import { type ForkedKernelHandle, ForkServerUnavailable } from "../src/core/kernel/fork-server.js";+import { KernelManager } from "../src/core/kernel/index.js";+import { ORPHAN_PROCESS_JOURNAL_ENV } from "../src/core/orphan-process-journal.js";++const forkKernelMock = vi.hoisted(() => vi.fn());+const forkEnabledMock = vi.hoisted(() => vi.fn(() => false));++vi.mock("../src/core/kernel/fork-server.js", async (importOriginal) => {+ const original = await importOriginal<typeof import("../src/core/kernel/fork-server.js")>();+ return { ...original, forkKernel: forkKernelMock, isForkServerEnabled: forkEnabledMock };+});++let tempDir = "";+const savedForkFlag = process.env.PRIME_AGENT_KERNEL_FORKSERVER;+const savedJournalPath = process.env[ORPHAN_PROCESS_JOURNAL_ENV];++beforeAll(() => {+ process.env.PRIME_AGENT_KERNEL_FORKSERVER = "0";+});+afterAll(() => {+ if (savedForkFlag === undefined) delete process.env.PRIME_AGENT_KERNEL_FORKSERVER;+ else process.env.PRIME_AGENT_KERNEL_FORKSERVER = savedForkFlag;+});++function writeFakePython(script: string[]): string {+ const python = join(tempDir, "python");+ writeFileSync(python, script.join("\n"));+ chmodSync(python, 0o755);+ return python;+}++interface JournalRecord {+ pid: number;+ ownerPid: number;+ active: boolean;+}++function readJournalRecords(path: string): JournalRecord[] {+ return readFileSync(path, "utf8")+ .split("\n")+ .filter(Boolean)+ .map((line) => JSON.parse(line) as JournalRecord);+}++describe("kernel parent watchdog", () => {+ beforeEach(() => {+ tempDir = mkdtempSync(join(tmpdir(), "prime-agent-kernel-watchdog-"));+ });++ afterEach(() => {+ forkEnabledMock.mockReturnValue(false);+ forkKernelMock.mockReset();+ if (savedJournalPath === undefined) delete process.env[ORPHAN_PROCESS_JOURNAL_ENV];+ else process.env[ORPHAN_PROCESS_JOURNAL_ENV] = savedJournalPath;+ if (tempDir) {+ rmSync(tempDir, { recursive: true, force: true });+ tempDir = "";+ }+ });++ it("direct spawn sets JPY_PARENT_PID and journals the kernel pid", async () => {+ const envDump = join(tempDir, "kernel-env");+ const python = writeFakePython(["#!/bin/sh", `env > "${envDump}"`, "exit 42", ""]);+ const journalPath = join(tempDir, "orphans.jsonl");+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;+ const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});+ const manager = new KernelManager({ python, cwd: tempDir });++ try {+ await expect(manager.execute("x")).rejects.toThrow(/Kernel exited before resolving ports/);+ } finally {+ errorSpy.mockRestore();+ await manager.dispose();+ }++ expect(readFileSync(envDump, "utf8")).toMatch(new RegExp(`^JPY_PARENT_PID=${process.pid}$`, "m"));++ // Self-exited child: the handle-based kill signals nothing, so the record must stay active.+ await vi.waitFor(() => {+ const records = readJournalRecords(journalPath);+ expect(records).toHaveLength(1);+ expect(records[0]?.ownerPid).toBe(process.pid);+ expect(records[0]?.active).toBe(true);+ });+ });++ it("writes the inactive journal record only on a signaled forkserver kill outcome", async () => {+ const journalPath = join(tempDir, "orphans.jsonl");+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;+ forkEnabledMock.mockReturnValue(true);+ const killMock = vi.fn(async (): Promise<"signaled"> => "signaled");+ forkKernelMock.mockResolvedValue({+ pid: 999999,+ isAlive: async () => false,+ kill: killMock,+ });+ const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });++ try {+ await expect(manager.execute("x")).rejects.toThrow(/Kernel exited before resolving ports/);+ } finally {+ errorSpy.mockRestore();+ await manager.dispose();+ }++ await vi.waitFor(() => {+ expect(killMock).toHaveBeenCalledTimes(1);+ expect(killMock).toHaveBeenCalledWith("TERM");+ const records = readJournalRecords(journalPath);+ expect(records).toHaveLength(2);+ expect(records[0]).toMatchObject({ pid: 999999, active: true });+ expect(records[1]).toMatchObject({ pid: 999999, active: false });+ });+ });++ it("leaves the journal record active on an already-exited kill outcome (pid may be reused)", async () => {+ const journalPath = join(tempDir, "orphans.jsonl");+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;+ forkEnabledMock.mockReturnValue(true);+ const killMock = vi.fn(async (): Promise<"already-exited"> => "already-exited");+ forkKernelMock.mockResolvedValue({+ pid: 999999,+ isAlive: async () => false,+ kill: killMock,+ });+ const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });++ try {+ await expect(manager.execute("x")).rejects.toThrow(/Kernel exited before resolving ports/);+ } finally {+ errorSpy.mockRestore();+ await manager.dispose();+ }++ await vi.waitFor(() => expect(killMock).toHaveBeenCalledTimes(1));+ // Let the resolved kill settle: no inactive record may ever follow.+ await new Promise((resolve) => setTimeout(resolve, 200));+ const records = readJournalRecords(journalPath);+ expect(records).toHaveLength(1);+ expect(records[0]).toMatchObject({ pid: 999999, active: true });+ });++ it("leaves the journal record active and never signals the pid when the kill is unconfirmed", async () => {+ const journalPath = join(tempDir, "orphans.jsonl");+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;+ forkEnabledMock.mockReturnValue(true);+ const killMock = vi.fn(async (): Promise<never> => {+ throw new ForkServerUnavailable("dead");+ });+ forkKernelMock.mockResolvedValue({+ pid: 999999,+ isAlive: async () => {+ throw new ForkServerUnavailable("dead");+ },+ kill: killMock,+ });+ const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});+ const killSpy = vi.spyOn(process, "kill");+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });++ try {+ await expect(manager.execute("x")).rejects.toThrow(/Kernel exited before resolving ports/);+ await manager.dispose();+ await vi.waitFor(() => expect(killMock).toHaveBeenCalled());+ // Let the rejected kill settle: no inactive record may ever follow.+ await new Promise((resolve) => setTimeout(resolve, 200));+ const records = readJournalRecords(journalPath);+ expect(records).toHaveLength(1);+ expect(records[0]).toMatchObject({ pid: 999999, active: true });+ expect(killSpy.mock.calls.some((call) => call[0] === 999999)).toBe(false);+ } finally {+ killSpy.mockRestore();+ errorSpy.mockRestore();+ await manager.dispose();+ }+ });++ it("leaves the direct-spawn journal record active when the kill signals nothing", async () => {+ const journalPath = join(tempDir, "orphans.jsonl");+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });+ const internals = manager as unknown as {+ kernel?: { pid?: number; kill(signal: string): boolean };+ cleanupResources(): void;+ };++ try {+ internals.kernel = { pid: 999999, kill: () => false };+ internals.cleanupResources();++ const records = existsSync(journalPath) ? readJournalRecords(journalPath) : [];+ expect(records.some((r) => r.pid === 999999 && !r.active)).toBe(false);+ } finally {+ await manager.dispose();+ }+ });++ it("maps SIGKILL to the forkserver KILL signal", async () => {+ forkEnabledMock.mockReturnValue(true);+ const isAliveMock = vi.fn(async () => true);+ const killMock = vi.fn(async (): Promise<"signaled"> => "signaled");+ forkKernelMock.mockResolvedValue({ pid: 999999, isAlive: isAliveMock, kill: killMock });+ const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });++ try {+ const execution = manager.execute("x");+ execution.catch(() => {});+ // The startup poll calls isAlive only after the handle is assigned, so+ // kill() below is guaranteed to reach the forked branch.+ await vi.waitFor(() => expect(isAliveMock).toHaveBeenCalled());+ await manager.kill();+ expect(killMock).toHaveBeenCalledWith("KILL");+ await expect(execution).rejects.toThrow();+ } finally {+ errorSpy.mockRestore();+ await manager.dispose();+ }+ });++ it("a stale liveness probe resumed after restart never tears down the new kernel", async () => {+ const journalPath = join(tempDir, "orphans.jsonl");+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;+ let resolveProbe: (alive: boolean) => void = () => {};+ const killA = vi.fn(async (): Promise<"signaled"> => "signaled");+ const killB = vi.fn(async (): Promise<"signaled"> => "signaled");+ const handleA: ForkedKernelHandle = {+ pid: 111111,+ isAlive: () =>+ new Promise<boolean>((resolve) => {+ resolveProbe = resolve;+ }),+ kill: killA,+ };+ const handleB: ForkedKernelHandle = { pid: 222222, isAlive: async () => true, kill: killB };+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });+ const internals = manager as unknown as {+ state: string;+ forkedKernel?: ForkedKernelHandle;+ checkForkedKernelDeath(): Promise<void>;+ };++ try {+ internals.state = "running";+ internals.forkedKernel = handleA;+ const probe = internals.checkForkedKernelDeath();+ // A restart completes while the probe is in flight: the manager is+ // "running" again, but on a different kernel.+ internals.forkedKernel = handleB;+ resolveProbe(false);+ await probe;++ expect(internals.state).toBe("running");+ expect(killB).not.toHaveBeenCalled();+ const records = existsSync(journalPath) ? readJournalRecords(journalPath) : [];+ expect(records.some((r) => r.pid === handleB.pid && !r.active)).toBe(false);+ } finally {+ internals.forkedKernel = undefined;+ internals.state = "idle";+ await manager.dispose();+ }+ });++ it("a timed-out liveness probe never tears down a possibly-healthy kernel", async () => {+ const kill = vi.fn(async (): Promise<"signaled"> => "signaled");+ const handle: ForkedKernelHandle = {+ pid: 111111,+ isAlive: async () => {+ throw new ForkServerUnavailable("forkserver request timed out after 10000ms", { timedOut: true });+ },+ kill,+ };+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });+ const internals = manager as unknown as {+ state: string;+ forkedKernel?: ForkedKernelHandle;+ checkForkedKernelDeath(): Promise<void>;+ };++ try {+ internals.state = "running";+ internals.forkedKernel = handle;+ await internals.checkForkedKernelDeath();++ expect(internals.state).toBe("running");+ expect(kill).not.toHaveBeenCalled();+ } finally {+ internals.forkedKernel = undefined;+ internals.state = "idle";+ await manager.dispose();+ }+ });++ it("a stale doStart resumed after a concurrent teardown never touches the new kernel", async () => {+ const journalPath = join(tempDir, "orphans.jsonl");+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;+ forkEnabledMock.mockReturnValue(true);+ let resolveFork: (handle: ForkedKernelHandle) => void = () => {};+ const killA = vi.fn(async (): Promise<"signaled"> => "signaled");+ const killB = vi.fn(async (): Promise<"signaled"> => "signaled");+ const handleA: ForkedKernelHandle = { pid: 111111, isAlive: async () => true, kill: killA };+ const handleB: ForkedKernelHandle = { pid: 222222, isAlive: async () => true, kill: killB };+ forkKernelMock.mockReturnValue(+ new Promise<ForkedKernelHandle>((resolve) => {+ resolveFork = resolve;+ }),+ );+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });+ const internals = manager as unknown as { state: string; forkedKernel?: ForkedKernelHandle };++ try {+ // Start A blocks inside `await forkKernel(...)`.+ const staleStart = manager.start();+ staleStart.catch(() => {});+ await vi.waitFor(() => expect(forkKernelMock).toHaveBeenCalledTimes(1));++ // A concurrent restart tears the starting kernel down (bumping the start+ // generation) and brings up a new running kernel B.+ await manager.kill();+ internals.state = "running";+ internals.forkedKernel = handleB;++ // The stale doStart resumes with handle A: it must fail without touching B.+ resolveFork(handleA);+ await expect(staleStart).rejects.toThrow(/Kernel start superseded/);++ expect(internals.state).toBe("running");+ expect(internals.forkedKernel).toBe(handleB);+ expect(killB).not.toHaveBeenCalled();+ // A is reclaimed via its own handle, never journaled.+ await vi.waitFor(() => expect(killA).toHaveBeenCalledWith("TERM"));+ const records = existsSync(journalPath) ? readJournalRecords(journalPath) : [];+ expect(records.some((r) => r.pid === handleA.pid)).toBe(false);+ } finally {+ internals.forkedKernel = undefined;+ internals.state = "idle";+ await manager.dispose();+ }+ });++ it("a shutdown superseded by a concurrent kill reports not-owner so recovery cannot resurrect to idle", async () => {+ const manager = new KernelManager({ python: "/nonexistent-python", cwd: tmpdir() });+ const internals = manager as unknown as {+ state: string;+ connection: unknown;+ control: unknown;+ kernel: unknown;+ shutdown(): Promise<boolean>;+ kill(): Promise<void>;+ };+ internals.state = "starting";+ internals.connection = { key: "" };+ // A live kernel handle keeps waitForKernelExit parked so the send actually blocks the shutdown.+ const kernel = Object.assign(new EventEmitter(), { exitCode: null, signalCode: null, kill: () => true });+ internals.kernel = kernel;+ let releaseSend: () => void = () => {};+ internals.control = {+ send: () => new Promise<void>((resolve) => (releaseSend = resolve)),+ close: () => {},+ };+ const shutdownResult = internals.shutdown();+ await new Promise((resolve) => setTimeout(resolve, 10)); // park in the control send+ await internals.kill(); // concurrent teardown wins ownership+ releaseSend();+ kernel.emit("exit", 0, null);+ expect(await shutdownResult).toBe(false); // recovery must not set idle+ expect(internals.state).toBe("shutdown");+ });++ it("a stale shutdown parked in its control-send await never cleans up a successor kernel", async () => {+ const journalPath = join(tempDir, "orphans.jsonl");+ process.env[ORPHAN_PROCESS_JOURNAL_ENV] = journalPath;+ let releaseSend: () => void = () => {};+ const parkedSend = new Promise<void>((resolve) => {+ releaseSend = resolve;+ });+ const killA = vi.fn(async (): Promise<"signaled"> => "signaled");+ const killB = vi.fn(async (): Promise<"signaled"> => "signaled");+ const handleA: ForkedKernelHandle = { pid: 111111, isAlive: async () => true, kill: killA };+ const handleB: ForkedKernelHandle = { pid: 222222, isAlive: async () => true, kill: killB };+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });+ const internals = manager as unknown as {+ state: string;+ forkedKernel?: ForkedKernelHandle;+ control?: { send(frames: Buffer[]): Promise<void>; close(): void };+ connection?: { ip: string; transport: string; control_port: number; key: string };+ startPromise?: Promise<void>;+ };++ try {+ // Kernel A is running with a control channel whose send parks forever+ // until released — shutdown() will suspend inside its await window+ // after having synchronously set state = "shutdown".+ internals.state = "running";+ internals.forkedKernel = handleA;+ internals.control = { send: () => parkedSend, close: () => {} };+ internals.connection = { ip: "127.0.0.1", transport: "tcp", control_port: 1, key: "test-key" };+ const staleShutdown = manager.shutdown();++ // While A's shutdown is parked, a concurrent teardown reclaims A (this is+ // the cleanup restart() reaches via shutdown's already-shutdown fast path)+ // and a new start brings up kernel B.+ await manager.kill();+ internals.state = "running";+ internals.forkedKernel = handleB;+ const startPromiseB = Promise.resolve();+ internals.startPromise = startPromiseB;++ // A's stale shutdown resumes: it must not clean up B or clear B's start.+ releaseSend();+ await staleShutdown;++ expect(internals.state).toBe("running");+ expect(internals.forkedKernel).toBe(handleB);+ expect(killB).not.toHaveBeenCalled();+ expect(internals.startPromise).toBe(startPromiseB);+ // A was reclaimed by kill(); B must never gain an inactive journal record.+ await vi.waitFor(() => expect(killA).toHaveBeenCalledWith("KILL"));+ const records = existsSync(journalPath) ? readJournalRecords(journalPath) : [];+ expect(records.some((r) => r.pid === handleB.pid && !r.active)).toBe(false);+ } finally {+ internals.forkedKernel = undefined;+ internals.startPromise = undefined;+ internals.state = "idle";+ await manager.dispose();+ }+ });++ it("a hung liveness probe resolves alive once its budget expires", async () => {+ const manager = new KernelManager({ python: "/nonexistent/python", cwd: tempDir });+ const internals = manager as unknown as {+ forkedKernelDead(probed: ForkedKernelHandle, timeoutMs?: number): Promise<boolean>;+ };+ const handle: ForkedKernelHandle = {+ pid: 999999,+ isAlive: () => new Promise<boolean>(() => {}),+ kill: async () => "signaled",+ };++ try {+ const started = Date.now();+ // Unknown is not death, and the caller's budget bounds the wait.+ await expect(internals.forkedKernelDead(handle, 250)).resolves.toBe(false);+ expect(Date.now() - started).toBeLessThan(5_000);+ } finally {+ await manager.dispose();+ }+ });++ it("fork request env does not carry JPY_PARENT_PID (forked children watch the forkserver)", async () => {+ const python = writeFakePython(["#!/bin/sh", "exit 42", ""]);+ forkEnabledMock.mockReturnValue(true);+ forkKernelMock.mockRejectedValue(new ForkServerUnavailable("test"));+ const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});+ const manager = new KernelManager({ python, cwd: tempDir });++ try {+ await expect(manager.execute("x")).rejects.toThrow(/Kernel exited before resolving ports/);+ } finally {+ errorSpy.mockRestore();+ await manager.dispose();+ }++ expect(forkKernelMock).toHaveBeenCalledTimes(1);+ const spawnParams = forkKernelMock.mock.calls[0]?.[1] as { env?: Record<string, string | undefined> };+ expect(spawnParams.env?.JPY_PARENT_PID).toBeUndefined();+ });+});++function resolveKernelPython(): string | null {+ const candidates = [+ process.env.PRIME_AGENT_KERNEL_PYTHON,+ join(homedir(), ".prime", "agent", "kernel-venv", "bin", "python"),+ ].filter((p): p is string => Boolean(p));+ for (const python of candidates) {+ if (!existsSync(python)) continue;+ const check = spawnSync(python, ["-c", "import ipykernel"], { encoding: "utf8" });+ if (check.status === 0) return python;+ }+ return null;+}++const kernelPython = resolveKernelPython();+const describeIf = kernelPython && process.platform !== "win32" ? describe : describe.skip;++describeIf("kernel outlives-owner watchdog (real kernel)", { tags: ["kernel-heavy"] }, () => {+ it("kernel exits after its owner is SIGKILLed", async () => {+ const dir = mkdtempSync(join(tmpdir(), "prime-agent-watchdog-int-"));+ const pidFile = join(dir, "kernel.pid");+ const connectionFile = join(dir, "connection.json");+ // The owner must be a separate killable process; it replicates KernelManager's+ // exact spawn line (T1 above proves the manager emits that env).+ const ownerScript = [+ `const { spawn } = require("node:child_process");`,+ `const { writeFileSync } = require("node:fs");`,+ `const k = spawn(${JSON.stringify(kernelPython)}, ["-m", "ipykernel_launcher", "-f", ${JSON.stringify(connectionFile)}], {`,+ ` env: { ...process.env, JPY_PARENT_PID: String(process.pid) },`,+ ` stdio: "ignore",`,+ `});`,+ `writeFileSync(${JSON.stringify(pidFile)}, String(k.pid));`,+ `setInterval(() => {}, 1000);`,+ ].join("\n");+ const owner = spawn(process.execPath, ["-e", ownerScript], { stdio: ["ignore", "ignore", "inherit"] });+ let kernelPid = 0;++ try {+ await vi.waitFor(+ () => {+ kernelPid = Number(readFileSync(pidFile, "utf8"));+ expect(kernelPid).toBeGreaterThan(0);+ expect(() => process.kill(kernelPid, 0)).not.toThrow();+ },+ { timeout: 20_000, interval: 500 },+ );++ owner.kill("SIGKILL");++ await vi.waitFor(+ () => {+ expect(() => process.kill(kernelPid, 0)).toThrow();+ },+ { timeout: 20_000, interval: 500 },+ );+ } finally {+ if (kernelPid > 0) {+ try {+ process.kill(kernelPid, "SIGKILL");+ } catch {+ // Already exited (the expected outcome).+ }+ }+ try {+ owner.kill("SIGKILL");+ } catch {+ // Already exited.+ }+ rmSync(dir, { recursive: true, force: true });+ }+ }, 30_000);+});diff --git a/packages/coding-agent/test/kernel-shutdown.test.ts b/packages/coding-agent/test/kernel-shutdown.test.tsindex 50a65b9a0b..3f002ffb3e 100644--- a/packages/coding-agent/test/kernel-shutdown.test.ts+++ b/packages/coding-agent/test/kernel-shutdown.test.ts@@ -103,7 +103,8 @@ describe("KernelManager graceful shutdown", () => {try {const shutdown = manager.shutdown();await vi.advanceTimersByTimeAsync(100);- await shutdown;+ // True = this call performed the cleanup: startup-failure recovery relies on it to resurrect to idle.+ await expect(shutdown).resolves.toBe(true);expect(internals.kernel).toBeUndefined();} finally {vi.useRealTimers();