Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

fast kernel spawning via a python forkserver (#298)

kt2b6fe2f9eff7
diff --git a/packages/coding-agent/src/core/kernel/boot-gate.ts b/packages/coding-agent/src/core/kernel/boot-gate.ts
index 5813d5b755..3fe2211edd 100644
--- a/packages/coding-agent/src/core/kernel/boot-gate.ts
+++ b/packages/coding-agent/src/core/kernel/boot-gate.ts
@@ -1,27 +1,50 @@
import { cpus } from "node:os";
import { Semaphore } from "../../utils/semaphore.js";
+import { isForkServerEnabled } from "./fork-server.js";
// Above core count because boots are IO-bound (cold imports), but capped so a
// fan-out can't thrash the FS past the port-resolve window.
const DEFAULT_KERNEL_BOOT_CONCURRENCY = Math.min(16, Math.max(4, (cpus().length || 4) * 2));
const MAX_KERNEL_BOOT_CONCURRENCY = 64;
+// Fork-per-child is ~ms and bypasses the FS, so we can admit well past the
+// direct-spawn cap. But fork only removes the *import* cost — every admitted
+// kernel still starts a live ioloop + heartbeat thread + rlm bootstrap, and
+// letting all N do that at once trips the ready timeout (measured: 256 collapses
+// to ~28% boot at N=200; core*4 holds 100%). So the gate stays a real bound.
+// Explicit env overrides may go higher than the auto default, but not unbounded —
+// past this the live-kernel startup storm regresses boot rate.
+const FORKSERVER_KERNEL_BOOT_CONCURRENCY_MAX = 128;
+const FORKSERVER_KERNEL_BOOT_CONCURRENCY = Math.min(
+ FORKSERVER_KERNEL_BOOT_CONCURRENCY_MAX,
+ Math.max(32, (cpus().length || 4) * 4),
+);
export function resolveKernelBootConcurrency(): number {
const raw = process.env.PRIME_AGENT_MAX_CONCURRENT_KERNEL_BOOTS;
+ const fallback = isForkServerEnabled() ? FORKSERVER_KERNEL_BOOT_CONCURRENCY : DEFAULT_KERNEL_BOOT_CONCURRENCY;
if (raw === undefined || !/^\d+$/.test(raw)) {
- return DEFAULT_KERNEL_BOOT_CONCURRENCY;
+ return fallback;
}
const parsed = Number.parseInt(raw, 10);
// A malformed or out-of-range value (incl. 0, e.g. "00") falls back to the
// default rather than mis-bounding the gate or throwing at module load.
if (parsed < 1) {
- return DEFAULT_KERNEL_BOOT_CONCURRENCY;
+ return fallback;
}
- return Math.min(MAX_KERNEL_BOOT_CONCURRENCY, parsed);
+ // An explicit override may exceed the auto default (that's its purpose), but is
+ // still clamped: the FS-contention cap on direct spawn, a looser backstop on fork.
+ const max = isForkServerEnabled() ? FORKSERVER_KERNEL_BOOT_CONCURRENCY_MAX : MAX_KERNEL_BOOT_CONCURRENCY;
+ return Math.min(max, parsed);
}
-const kernelBootSemaphore = new Semaphore(resolveKernelBootConcurrency());
+// Resolved lazily on first boot, not at module load, so the fork-server flag is
+// honored whenever it is set before the first kernel starts — not just if it
+// happened to be set at import time.
+let kernelBootSemaphore: Semaphore | undefined;
export function withKernelBootPermit<T>(boot: () => Promise<T>, signal?: AbortSignal): Promise<T> {
+ if (!kernelBootSemaphore) {
+ kernelBootSemaphore = new Semaphore(resolveKernelBootConcurrency());
+ }
return kernelBootSemaphore.run(boot, signal);
}
diff --git a/packages/coding-agent/src/core/kernel/fork-server-script.ts b/packages/coding-agent/src/core/kernel/fork-server-script.ts
new file mode 100644
index 0000000000..d08f8e0139
--- /dev/null
+++ b/packages/coding-agent/src/core/kernel/fork-server-script.ts
@@ -0,0 +1,148 @@
+// The Python "kernel forkserver": a long-lived template process that pays the
+// ~1.2s IPython/ipykernel/rlm import cost once, then forks a ready-to-run kernel
+// per request in ~ms. Children inherit the imported module objects via
+// copy-on-write, bypassing the (slow, virtiofs-backed) per-file import path.
+//
+// Embedded as a string rather than shipped as a package asset so it can never be
+// missing from a release layout (see the built-in-skills packaging gap). Run via
+// `python -c <this> <control-socket-path>`.
+//
+// Protocol (newline-delimited JSON over the unix socket, forkserver is the client):
+// -> { "id": <n>, "connectionPath": "<abs path>" } spawn request from Node
+// <- { "type": "ready" } once, after imports finish
+// <- { "id": <n>, "pid": <pid> } fork succeeded
+// <- { "id": <n>, "error": "<message>" } fork failed
+export const FORK_SERVER_SCRIPT = String.raw`
+import gc
+import json
+import os
+import signal
+import socket
+import sys
+
+
+def _reap_children(*_args):
+ # Reap every exited child so disposed kernels never linger as zombies. Wired to
+ # SIGCHLD so reaping happens on child exit, not only when the next request wakes
+ # the accept loop. Safe under PEP 475: the interrupted socket read auto-retries.
+ try:
+ while True:
+ pid, _status = os.waitpid(-1, os.WNOHANG)
+ if pid == 0:
+ break
+ except ChildProcessError:
+ pass
+
+
+def _import_template():
+ # Everything a kernel touches at import time. Paid once; shared COW by children.
+ import IPython # noqa: F401
+ import ipykernel # noqa: F401
+ import ipykernel.kernelapp # noqa: F401
+ import jupyter_client # noqa: F401
+ import nest_asyncio # noqa: F401
+ try:
+ import rlm # noqa: F401
+ except Exception:
+ # rlm may not import cleanly outside a live kernel namespace; the Node-side
+ # bootstrap cell wires it up per-child regardless. Preloading is a best-effort
+ # speedup, not a correctness requirement.
+ pass
+
+
+def _run_child(connection_path, cwd, env):
+ # We are the forked child; become the ipykernel server on the given connection.
+ from ipykernel.kernelapp import IPKernelApp
+
+ # Drop the inherited SIGCHLD reaper so it can't interfere with ipykernel's own
+ # child/signal handling.
+ signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+ # cwd/env are per-kernel and applied here (not at template import), so all
+ # kernels can share one warm template regardless of their working dir / env.
+ if env:
+ os.environ.update(env)
+ if cwd:
+ # Don't swallow a bad cwd: direct spawn fails fast on ENOENT, so match that
+ # (the OSError propagates, the child exits non-zero, Node falls back).
+ os.chdir(cwd)
+
+ # Drop any singleton the template happened to build so the child owns a fresh
+ # instance (and, critically, a jupyter_client Session created in *this* pid;
+ # a Session inherited from the template silently drops messages via check_pid).
+ IPKernelApp.clear_instance()
+ app = IPKernelApp.instance(connection_file=connection_path)
+ # initialize() binds the 5 ZMQ ports, writes the resolved ports back into
+ # connection.json, and starts the heartbeat thread + ioloop — all post-fork,
+ # so no thread/loop/socket is ever inherited across the fork boundary.
+ app.initialize([])
+ app.start()
+
+
+def _serve(control_path):
+ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ sock.connect(control_path)
+ control_fd = sock.fileno()
+
+ # Reap forked kernels as they exit, independent of the request loop.
+ signal.signal(signal.SIGCHLD, _reap_children)
+
+ _import_template()
+ # Freeze the heap so the cyclic GC doesn't write to (and thus COW-copy) the
+ # shared module pages, keeping memory genuinely shared across children.
+ gc.freeze()
+
+ f = sock.makefile("rwb", buffering=0)
+ f.write(json.dumps({"type": "ready"}).encode() + b"\n")
+ f.flush()
+
+ while True:
+ # Belt-and-suspenders: the SIGCHLD handler is the primary reaper, but sweep
+ # again each turn in case a signal was missed (coalesced) while handling one.
+ _reap_children()
+
+ line = f.readline()
+ if not line:
+ break
+ try:
+ req = json.loads(line)
+ except ValueError:
+ continue
+ req_id = req.get("id")
+ connection_path = req.get("connectionPath")
+ cwd = req.get("cwd")
+ env = req.get("env")
+
+ try:
+ pid = os.fork()
+ except OSError as exc:
+ f.write(json.dumps({"id": req_id, "error": str(exc)}).encode() + b"\n")
+ f.flush()
+ continue
+
+ if pid == 0:
+ # Child: shed every inherited fd tied to the control channel, then run.
+ try:
+ sock.close()
+ f.close()
+ except Exception:
+ pass
+ try:
+ os.close(control_fd)
+ except OSError:
+ pass
+ try:
+ _run_child(connection_path, cwd, env)
+ except BaseException as exc: # never return to the accept loop
+ sys.stderr.write("forked kernel failed: %r\n" % (exc,))
+ os._exit(1)
+ os._exit(0)
+
+ # Parent: stay pristine (no loop/threads/ZMQ ever) so the next fork is clean.
+ f.write(json.dumps({"id": req_id, "pid": pid}).encode() + b"\n")
+ f.flush()
+
+
+if __name__ == "__main__":
+ _serve(sys.argv[1])
+`;
diff --git a/packages/coding-agent/src/core/kernel/fork-server.ts b/packages/coding-agent/src/core/kernel/fork-server.ts
new file mode 100644
index 0000000000..139c8aa64f
--- /dev/null
+++ b/packages/coding-agent/src/core/kernel/fork-server.ts
@@ -0,0 +1,350 @@
+// Node-side client for the Python kernel forkserver (see fork-server-script.ts).
+// One forkserver process per (python, cwd, env) profile, spawned lazily and kept
+// alive for the agent process. Each KernelManager asks it to fork a kernel onto a
+// connection file instead of paying a full `python -m ipykernel_launcher` cold boot.
+//
+// Everything degrades to direct spawn: if the forkserver is disabled, unavailable,
+// or a spawn request fails/times out, callers catch ForkServerUnavailable and fall
+// back to the existing path, so correctness never depends on fork.
+import { type ChildProcess, spawn } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { createServer, type Server, type Socket } from "node:net";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { registerSessionResourceCleanup } from "@earendil-works/pi-ai";
+import { FORK_SERVER_SCRIPT } from "./fork-server-script.js";
+
+const READY_TIMEOUT_MS = 30_000;
+const SPAWN_TIMEOUT_MS = 10_000;
+const STDERR_TAIL_MAX = 4096;
+
+// Vars the Python interpreter consumes at startup (before any user code runs), so
+// they can't be honored post-fork via os.environ.update — the forked child inherits
+// the template's already-initialized sys.path / site config. A kernel overriding any
+// of these to a value the template didn't launch with must take the direct-spawn
+// path. We treat the entire PYTHON* family as startup-affecting rather than
+// enumerating individual vars: the set is large and version-dependent, and a false
+// positive merely (safely) routes a kernel to direct spawn. Plus the non-PYTHON*
+// startup vars that also steer interpreter/site resolution.
+const INTERPRETER_STARTUP_ENV_EXACT = ["VIRTUAL_ENV", "CONDA_PREFIX", "__PYVENV_LAUNCHER__"];
+
+function affectsInterpreterStartup(key: string): boolean {
+ return key.startsWith("PYTHON") || INTERPRETER_STARTUP_ENV_EXACT.includes(key);
+}
+
+export class ForkServerUnavailable extends Error {
+ constructor(message: string) {
+ super(message);
+ this.name = "ForkServerUnavailable";
+ }
+}
+
+/** Fork is a Linux-only, in-process fan-out optimization. */
+export function isForkServerEnabled(): boolean {
+ return process.platform === "linux" && process.env.PRIME_AGENT_KERNEL_FORKSERVER === "1";
+}
+
+// A forkserver template is defined solely by the interpreter — the imported
+// module graph doesn't depend on cwd/env. Those are per-kernel and applied in the
+// forked child, so every kernel for a given python shares ONE template.
+interface ForkServerParams {
+ python: string;
+}
+
+interface SpawnParams {
+ connectionPath: string;
+ cwd?: string;
+ // Mirrors process.env shape; undefined values are dropped when JSON-serialized
+ // to the child, matching how spawn() treats them on the direct path.
+ env?: Record<string, string | undefined>;
+}
+
+type PendingSpawn = {
+ resolve: (pid: number) => void;
+ reject: (err: Error) => void;
+ timer: ReturnType<typeof setTimeout>;
+};
+
+class ForkServer {
+ private readonly params: ForkServerParams;
+ // The env the template process was launched with, snapshotted at construction so
+ // it exactly matches what the template's interpreter imported with. The
+ // startup-env guard compares against THIS, not live process.env, so a later
+ // mutation of process.env can't make a stale template look compatible.
+ private readonly launchEnv: NodeJS.ProcessEnv;
+ private proc?: ChildProcess;
+ private server?: Server;
+ private conn?: Socket;
+ private socketDir?: string;
+ private readyPromise?: Promise<void>;
+ private failReady?: (err: Error) => void;
+ private buffer = "";
+ // Rolling tail of forkserver stderr. Forked children inherit this fd, so a
+ // child's import/startup traceback lands here; surfaced in error messages.
+ private stderrTail = "";
+ private nextId = 1;
+ private readonly pending = new Map<number, PendingSpawn>();
+ // Request ids whose caller timed out; a late pid reply for one is an orphan to kill.
+ private readonly abandoned = new Set<number>();
+ private dead = false;
+
+ constructor(params: ForkServerParams) {
+ this.params = params;
+ // Snapshot now and launch the template with this same object, so the guard's
+ // comparison uses exactly the env the interpreter imported with.
+ this.launchEnv = { ...process.env };
+ }
+
+ get isDead(): boolean {
+ return this.dead;
+ }
+
+ /**
+ * True if `env` overrides an interpreter-startup var to a value the template
+ * didn't launch with — such a kernel can't be forked (sys.path is baked in at
+ * import), so the caller must direct-spawn it.
+ */
+ needsStartupEnvNotInTemplate(env: SpawnParams["env"]): boolean {
+ if (!env) return false;
+ return Object.keys(env).some((k) => affectsInterpreterStartup(k) && env[k] !== this.launchEnv[k]);
+ }
+
+ private async ensureReady(): Promise<void> {
+ if (this.dead) throw new ForkServerUnavailable("forkserver is dead");
+ if (!this.readyPromise) {
+ this.readyPromise = this.start().catch((err) => {
+ this.markDead();
+ throw err instanceof ForkServerUnavailable ? err : new ForkServerUnavailable(String(err));
+ });
+ }
+ return this.readyPromise;
+ }
+
+ private start(): Promise<void> {
+ this.socketDir = mkdtempSync(join(tmpdir(), "prime-agent-forkserver-"));
+ const socketPath = join(this.socketDir, "control.sock");
+
+ return new Promise<void>((resolve, reject) => {
+ const server = createServer();
+ this.server = server;
+
+ let settled = false;
+ const readyTimer = setTimeout(() => {
+ if (settled) return;
+ settled = true;
+ reject(new ForkServerUnavailable(`forkserver did not become ready within ${READY_TIMEOUT_MS}ms`));
+ }, READY_TIMEOUT_MS);
+
+ // Lets markDead() fail a still-pending start (interpreter crashed / socket
+ // closed before "ready") instead of waiting out the full ready timeout.
+ this.failReady = (err) => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(readyTimer);
+ reject(err instanceof ForkServerUnavailable ? err : new ForkServerUnavailable(String(err)));
+ };
+
+ server.on("connection", (socket) => {
+ this.conn = socket;
+ socket.setEncoding("utf8");
+ socket.on("data", (chunk: string) => this.onData(chunk));
+ socket.on("close", () => this.markDead());
+ socket.on("error", () => this.markDead());
+ });
+
+ server.on("error", (err) => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(readyTimer);
+ reject(new ForkServerUnavailable(`forkserver control socket failed: ${err.message}`));
+ });
+
+ // Buffered until the "ready" line; flip the promise from there.
+ this.onReady = () => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(readyTimer);
+ resolve();
+ };
+
+ server.listen(socketPath, () => {
+ // The template only imports; its own cwd/env are irrelevant since each
+ // forked child applies the per-kernel cwd/env itself. Inherit the daemon's.
+ const proc = spawn(this.params.python, ["-c", FORK_SERVER_SCRIPT, socketPath], {
+ env: this.launchEnv,
+ stdio: ["ignore", "ignore", "pipe"],
+ });
+ this.proc = proc;
+ proc.stderr?.on("data", (buf: Buffer) => {
+ this.stderrTail = `${this.stderrTail}${buf.toString()}`.slice(-STDERR_TAIL_MAX);
+ });
+ proc.on("error", () => this.markDead());
+ proc.on("exit", () => this.markDead());
+ });
+ });
+ }
+
+ private onReady: () => void = () => {};
+
+ private onData(chunk: string): void {
+ this.buffer += chunk;
+ for (let idx = this.buffer.indexOf("\n"); idx !== -1; idx = this.buffer.indexOf("\n")) {
+ const line = this.buffer.slice(0, idx);
+ this.buffer = this.buffer.slice(idx + 1);
+ if (!line.trim()) continue;
+ let msg: { type?: string; id?: number; pid?: number; error?: string };
+ try {
+ msg = JSON.parse(line);
+ } catch {
+ continue;
+ }
+ if (msg.type === "ready") {
+ this.onReady();
+ continue;
+ }
+ if (typeof msg.id !== "number") continue;
+ const p = this.pending.get(msg.id);
+ if (!p) {
+ // A pid for a request the caller already abandoned (timed out): the
+ // fork succeeded but nobody owns it, so kill the orphan here.
+ if (this.abandoned.delete(msg.id) && typeof msg.pid === "number") {
+ try {
+ process.kill(msg.pid, "SIGTERM");
+ } catch {}
+ }
+ continue;
+ }
+ this.pending.delete(msg.id);
+ clearTimeout(p.timer);
+ if (typeof msg.pid === "number") {
+ p.resolve(msg.pid);
+ } else {
+ p.reject(new ForkServerUnavailable(this.withStderr(msg.error ?? "forkserver fork failed")));
+ }
+ }
+ }
+
+ async spawnKernel(spawn: SpawnParams): Promise<number> {
+ await this.ensureReady();
+ if (this.dead || !this.conn) throw new ForkServerUnavailable("forkserver connection unavailable");
+ const id = this.nextId++;
+ return new Promise<number>((resolve, reject) => {
+ const timer = setTimeout(() => {
+ this.pending.delete(id);
+ // The fork may still land after we give up; remember the id so its late
+ // pid reply gets the orphan killed instead of leaked.
+ this.abandoned.add(id);
+ reject(new ForkServerUnavailable(`forkserver spawn timed out after ${SPAWN_TIMEOUT_MS}ms`));
+ }, SPAWN_TIMEOUT_MS);
+ this.pending.set(id, { resolve, reject, timer });
+ this.conn?.write(
+ `${JSON.stringify({ id, connectionPath: spawn.connectionPath, cwd: spawn.cwd, env: spawn.env })}\n`,
+ );
+ });
+ }
+
+ private withStderr(message: string): string {
+ const tail = this.stderrTail.trim();
+ return tail ? `${message}\nforkserver stderr:\n${tail}` : message;
+ }
+
+ /** Reject any in-flight ensureReady/spawnKernel callers so none wait out a timeout. */
+ private rejectPending(reason: string): void {
+ this.failReady?.(new ForkServerUnavailable(reason));
+ for (const p of this.pending.values()) {
+ clearTimeout(p.timer);
+ p.reject(new ForkServerUnavailable(reason));
+ }
+ this.pending.clear();
+ }
+
+ private markDead(): void {
+ this.dispose(this.withStderr("forkserver died"));
+ }
+
+ dispose(reason = "forkserver disposed"): void {
+ if (this.dead) return;
+ // Flip `dead` at entry so the close/exit events our own teardown triggers
+ // don't re-enter. Pending callers are rejected directly here (not deferred to
+ // those events), so this early flag flip can't strand them on a timeout.
+ this.dead = true;
+ this.rejectPending(reason);
+ try {
+ this.conn?.destroy();
+ } catch {}
+ try {
+ this.server?.close();
+ } catch {}
+ try {
+ this.proc?.kill("SIGTERM");
+ } catch {}
+ if (this.socketDir) {
+ try {
+ rmSync(this.socketDir, { recursive: true, force: true });
+ } catch {}
+ this.socketDir = undefined;
+ }
+ }
+}
+
+// Keyed by interpreter only: cwd/env are per-kernel (applied in the child), so all
+// kernels for a given python share ONE warm template.
+const servers = new Map<string, ForkServer>();
+let cleanupRegistered = false;
+
+function keyFor(params: ForkServerParams): string {
+ return params.python;
+}
+
+function registerForkServerCleanupOnce(): void {
+ if (cleanupRegistered) return;
+ cleanupRegistered = true;
+ // A forkserver is process-lived and shared across sessions, so per-session
+ // cleanup must not yank the warm template from other live sessions — only a full
+ // process shutdown (no sessionId) disposes it here.
+ registerSessionResourceCleanup((sessionId) => {
+ if (!sessionId) disposeAllForkServers();
+ });
+ // cleanupSessionResources(undefined) is never called on exit, so also tear the
+ // forkserver down directly on process teardown or it leaks as an orphan.
+ process.once("exit", disposeAllForkServers);
+ for (const signal of ["SIGINT", "SIGTERM", "beforeExit"] as const) {
+ process.once(signal, () => disposeAllForkServers());
+ }
+}
+
+/**
+ * Fork a kernel onto `spawn.connectionPath` from the shared template for this
+ * interpreter, applying `spawn.cwd`/`spawn.env` in the forked child. Throws
+ * ForkServerUnavailable if forking is disabled or fails — callers fall back to
+ * direct spawn. Returns the forked child's pid (owned/killed by the caller).
+ */
+export async function forkKernel(python: string, spawn: SpawnParams): Promise<number> {
+ if (!isForkServerEnabled()) throw new ForkServerUnavailable("forkserver disabled");
+ registerForkServerCleanupOnce();
+ const key = keyFor({ python });
+ let server = servers.get(key);
+ if (!server || server.isDead) {
+ server = new ForkServer({ python });
+ servers.set(key, server);
+ }
+ // Compare against the template's own launch env (not live process.env): a kernel
+ // overriding an interpreter-startup var to a value the template didn't import
+ // with can't be forked — defer to direct spawn rather than boot with wrong sys.path.
+ if (server.needsStartupEnvNotInTemplate(spawn.env)) {
+ throw new ForkServerUnavailable("kernel overrides interpreter-startup env; using direct spawn");
+ }
+ try {
+ return await server.spawnKernel(spawn);
+ } catch (err) {
+ // Only evict if the map still points at this dead instance: a concurrent
+ // caller may have already replaced it with a fresh live server under this key.
+ if (server.isDead && servers.get(key) === server) servers.delete(key);
+ throw err instanceof ForkServerUnavailable ? err : new ForkServerUnavailable(String(err));
+ }
+}
+
+export function disposeAllForkServers(): void {
+ for (const server of servers.values()) server.dispose();
+ servers.clear();
+}
diff --git a/packages/coding-agent/src/core/kernel/index.ts b/packages/coding-agent/src/core/kernel/index.ts
index 1adb22cca2..5a95bbc74e 100644
--- a/packages/coding-agent/src/core/kernel/index.ts
+++ b/packages/coding-agent/src/core/kernel/index.ts
@@ -9,6 +9,7 @@ import { registerSessionResourceCleanup } from "@earendil-works/pi-ai";
import { v4 as uuid } from "uuid";
import { Dealer, Subscriber } from "zeromq";
import { ensureKernelPython, type KernelBootstrapProgressHandler, type KernelPythonSkill } from "./bootstrap.js";
+import { ForkServerUnavailable, forkKernel, isForkServerEnabled } from "./fork-server.js";
import {
buildListNamesCode,
buildRestoreCode,
@@ -30,6 +31,8 @@ const IOPUB_SUBSCRIBE_DELAY_MS = 50;
const DEFAULT_MAX_OUTPUT_CHARS = 65536;
const HOST_REQUEST_DISPOSE_TIMEOUT_MS = 5000;
const DEFAULT_SNAPSHOT_DEBOUNCE_MS = 1500;
+// How often to poll a forked kernel's pid for unexpected death.
+const FORKED_LIVENESS_POLL_MS = 1000;
// Snapshot/restore cells can be large to (de)serialize; give them room beyond the user cap.
const SNAPSHOT_MAX_OUTPUT_CHARS = 1_000_000;
// Cap how long a graceful dispose waits on the final snapshot; the debounced
@@ -409,6 +412,11 @@ export class KernelManager {
private readonly commTargets = new Map<string, string>();
private readonly handledHostRequestCommIds = new Set<string>();
private kernel?: ChildProcess;
+ // Set instead of `kernel` when the kernel was forked from the forkserver: it is
+ // not a direct child, so it has no ChildProcess handle and is killed by pid.
+ private kernelPid?: number;
+ /** Polls a forked kernel's pid for death (no "exit" event on a non-child). */
+ private forkedLivenessTimer?: ReturnType<typeof globalThis.setInterval>;
private shell?: Dealer;
private iopub?: Subscriber;
private control?: Dealer;
@@ -485,37 +493,71 @@ export class KernelManager {
throw new Error("Kernel was disposed during startup");
}
- const { path: connectionPath, tempDir } = makeConnection();
- this.tempDir = tempDir;
+ let connection = makeConnection();
+ this.tempDir = connection.tempDir;
- const kernel = spawn(python, ["-m", "ipykernel_launcher", "-f", connectionPath], {
- cwd: this.options.cwd,
- env: this.options.env ? { ...process.env, ...this.options.env } : process.env,
- stdio: ["ignore", "pipe", "pipe"],
- });
- this.kernel = kernel;
+ // Fast path: fork a pre-imported kernel from the forkserver. Any failure
+ // (disabled, unavailable, fork error) degrades to the direct-spawn path so
+ // correctness never depends on fork.
+ let forked = false;
+ if (isForkServerEnabled()) {
+ try {
+ this.kernelPid = await forkKernel(python, {
+ connectionPath: connection.path,
+ cwd: this.options.cwd,
+ // Match the direct-spawn env exactly: merge the current host env with
+ // the per-kernel overrides, applied fresh in the child (the template's
+ // inherited env snapshot may be stale by fork time).
+ env: this.options.env ? { ...process.env, ...this.options.env } : { ...process.env },
+ });
+ forked = true;
+ } catch (err) {
+ if (!(err instanceof ForkServerUnavailable)) throw err;
+ this.appendKernelDiagnostic(`forkserver unavailable, spawning directly: ${err.message}`);
+ this.kernelPid = undefined;
+ // A fork request that times out or loses its pid reply may still have
+ // forked a child that binds the ports in this connection file. Mint a
+ // fresh connection for the direct spawn so a possible orphan can never
+ // collide with it (write the same file / re-bind the same ports).
+ try {
+ rmSync(connection.tempDir, { recursive: true, force: true });
+ } catch {}
+ connection = makeConnection();
+ this.tempDir = connection.tempDir;
+ }
+ }
- kernel.stderr?.on("data", (buf: Buffer) => {
- const s = buf.toString();
- this.kernelStderr += s;
- });
+ if (!forked) {
+ const kernel = spawn(python, ["-m", "ipykernel_launcher", "-f", connection.path], {
+ cwd: this.options.cwd,
+ env: this.options.env ? { ...process.env, ...this.options.env } : process.env,
+ stdio: ["ignore", "pipe", "pipe"],
+ });
+ this.kernel = kernel;
- kernel.on("error", (err) => {
- if (this.kernel !== kernel) return;
- this.appendKernelDiagnostic(`spawn error: ${err.message}`);
- this.state = "shutdown";
- liveKernels.delete(this);
- });
+ kernel.stderr?.on("data", (buf: Buffer) => {
+ const s = buf.toString();
+ this.kernelStderr += s;
+ });
- kernel.on("exit", (code, signal) => {
- if (this.kernel !== kernel) return;
- if (this.state !== "shutdown") {
- this.appendKernelDiagnostic(`unexpected exit code=${code} signal=${signal}`);
- }
- this.state = "shutdown";
- liveKernels.delete(this);
- });
+ kernel.on("error", (err) => {
+ if (this.kernel !== kernel) return;
+ this.appendKernelDiagnostic(`spawn error: ${err.message}`);
+ this.state = "shutdown";
+ liveKernels.delete(this);
+ });
+
+ kernel.on("exit", (code, signal) => {
+ if (this.kernel !== kernel) return;
+ if (this.state !== "shutdown") {
+ this.appendKernelDiagnostic(`unexpected exit code=${code} signal=${signal}`);
+ }
+ this.state = "shutdown";
+ liveKernels.delete(this);
+ });
+ }
+ const connectionPath = connection.path;
let conn: ConnectionInfo;
try {
conn = await this.waitForResolvedConnection(connectionPath);
@@ -549,12 +591,43 @@ export class KernelManager {
}
this.state = "running";
+ this.startForkedLivenessMonitor();
+ }
+
+ // A forked kernel isn't a direct child, so no "exit" fires when it dies. Poll its
+ // pid so a mid-run death tears down like the direct-spawn exit handler: mark
+ // shutdown, drop from liveKernels, and reject any in-flight execution.
+ private startForkedLivenessMonitor(): void {
+ if (this.kernelPid === undefined) return;
+ this.forkedLivenessTimer = globalThis.setInterval(() => {
+ if (this.state !== "running") return;
+ if (!this.forkedKernelDied()) return;
+ this.appendKernelDiagnostic("forked kernel exited unexpectedly");
+ this.state = "shutdown";
+ liveKernels.delete(this);
+ this.cleanupResources();
+ }, FORKED_LIVENESS_POLL_MS);
+ this.forkedLivenessTimer.unref?.();
+ }
+
+ // A forked kernel is not a direct child, so it emits no "exit" event; poll its
+ // pid so a dead child fails fast instead of burning the full resolve timeout.
+ private forkedKernelDied(): boolean {
+ if (this.kernelPid === undefined) return false;
+ try {
+ process.kill(this.kernelPid, 0);
+ return false;
+ } catch (error) {
+ // EPERM means the pid exists but isn't signalable by us — still alive.
+ // Only ESRCH (no such process) is genuine death.
+ return !(error instanceof Error && (error as NodeJS.ErrnoException).code === "EPERM");
+ }
}
private async waitForResolvedConnection(connectionPath: string): Promise<ConnectionInfo> {
const startedAt = Date.now();
while (Date.now() - startedAt < PORTS_RESOLVE_TIMEOUT_MS) {
- if ((this.state as string) === "shutdown") {
+ if ((this.state as string) === "shutdown" || this.forkedKernelDied()) {
const tail = this.kernelStderr.slice(-1024);
throw new Error(`Kernel exited before resolving ports. stderr:\n${tail || "(empty)"}`);
}
@@ -583,7 +656,7 @@ export class KernelManager {
const startedAt = Date.now();
while (Date.now() - startedAt < READY_TIMEOUT_MS) {
- if ((this.state as string) === "shutdown") {
+ if ((this.state as string) === "shutdown" || this.forkedKernelDied()) {
const tail = this.kernelStderr.slice(-1024);
throw new Error(`Kernel exited during startup. stderr:\n${tail || "(empty)"}`);
}
@@ -950,6 +1023,10 @@ export class KernelManager {
private cleanupResources(): void {
this.clearSnapshotTimer();
+ if (this.forkedLivenessTimer) {
+ globalThis.clearInterval(this.forkedLivenessTimer);
+ this.forkedLivenessTimer = undefined;
+ }
this.rejectActiveExecution(new Error("Kernel has been shut down"));
this.shell?.close();
this.iopub?.close();
@@ -959,9 +1036,16 @@ export class KernelManager {
this.control = undefined;
this.iopubPumpPromise = undefined;
try {
- this.kernel?.kill("SIGTERM");
+ if (this.kernel) {
+ this.kernel.kill("SIGTERM");
+ } else if (this.kernelPid !== undefined && !this.forkedKernelDied()) {
+ // Only signal a forked kernel confirmed still alive: a dead pid may have
+ // been recycled by the OS, and SIGTERM would then hit an unrelated process.
+ process.kill(this.kernelPid, "SIGTERM");
+ }
} catch {}
this.kernel = undefined;
+ this.kernelPid = undefined;
this.connection = undefined;
if (this.tempDir) {
try {
diff --git a/packages/coding-agent/test/boot-gate.test.ts b/packages/coding-agent/test/boot-gate.test.ts
index 795d3db907..31d1c8b33c 100644
--- a/packages/coding-agent/test/boot-gate.test.ts
+++ b/packages/coding-agent/test/boot-gate.test.ts
@@ -2,10 +2,12 @@ import { afterEach, describe, expect, it } from "vitest";
import { resolveKernelBootConcurrency } from "../src/core/kernel/boot-gate.js";
const ENV = "PRIME_AGENT_MAX_CONCURRENT_KERNEL_BOOTS";
+const FORK_ENV = "PRIME_AGENT_KERNEL_FORKSERVER";
describe("resolveKernelBootConcurrency", () => {
afterEach(() => {
delete process.env[ENV];
+ delete process.env[FORK_ENV];
});
it("never returns a value below 1 (so the semaphore can't throw at load)", () => {
@@ -15,10 +17,22 @@ describe("resolveKernelBootConcurrency", () => {
}
});
- it("honors a clean positive integer, clamped to the max", () => {
+ it("honors a clean positive integer, clamped to the direct-spawn max", () => {
process.env[ENV] = "8";
expect(resolveKernelBootConcurrency()).toBe(8);
process.env[ENV] = "999999";
expect(resolveKernelBootConcurrency()).toBe(64);
});
+
+ it("raises the ceiling when the forkserver is enabled (linux only)", () => {
+ if (process.platform !== "linux") return;
+ process.env[FORK_ENV] = "1";
+ // Auto default is well above the direct-spawn default but still bounded.
+ const auto = resolveKernelBootConcurrency();
+ expect(auto).toBeGreaterThanOrEqual(32);
+ expect(auto).toBeLessThanOrEqual(128);
+ // A huge override is clamped to the fork backstop, not the direct-spawn cap.
+ process.env[ENV] = "999999";
+ expect(resolveKernelBootConcurrency()).toBe(128);
+ });
});
diff --git a/packages/coding-agent/test/kernel-fork-server.test.ts b/packages/coding-agent/test/kernel-fork-server.test.ts
new file mode 100644
index 0000000000..8c3a0b0587
--- /dev/null
+++ b/packages/coding-agent/test/kernel-fork-server.test.ts
@@ -0,0 +1,49 @@
+import { afterEach, describe, expect, it } from "vitest";
+import { ForkServerUnavailable, forkKernel, isForkServerEnabled } from "../src/core/kernel/fork-server.js";
+
+const FORK_ENV = "PRIME_AGENT_KERNEL_FORKSERVER";
+
+describe("fork-server gating", () => {
+ afterEach(() => {
+ delete process.env[FORK_ENV];
+ });
+
+ it("is disabled unless the flag is set on linux", () => {
+ delete process.env[FORK_ENV];
+ expect(isForkServerEnabled()).toBe(false);
+ process.env[FORK_ENV] = "1";
+ expect(isForkServerEnabled()).toBe(process.platform === "linux");
+ });
+
+ it("rejects with ForkServerUnavailable when disabled so callers fall back", async () => {
+ delete process.env[FORK_ENV];
+ await expect(forkKernel("python3", { connectionPath: "/tmp/nope/connection.json" })).rejects.toBeInstanceOf(
+ ForkServerUnavailable,
+ );
+ });
+
+ it("degrades to ForkServerUnavailable when the interpreter can't start", async () => {
+ if (process.platform !== "linux") return;
+ process.env[FORK_ENV] = "1";
+ // The spawn errors immediately (ENOENT), so markDead fails the ready promise
+ // fast rather than waiting out the ready timeout.
+ await expect(
+ forkKernel("/nonexistent/python-binary", { connectionPath: "/tmp/nope/connection.json" }),
+ ).rejects.toBeInstanceOf(ForkServerUnavailable);
+ }, 15_000);
+
+ it("falls back to direct spawn for any PYTHON* startup-env override", async () => {
+ if (process.platform !== "linux") return;
+ process.env[FORK_ENV] = "1";
+ // The guard treats the whole PYTHON* family as startup-affecting, so even a var
+ // not explicitly enumerated diverts to direct spawn (no var can be "missed").
+ for (const key of ["PYTHONPATH", "PYTHONUSERBASE", "PYTHONDONTWRITEBYTECODE"]) {
+ await expect(
+ forkKernel("python3", {
+ connectionPath: "/tmp/nope/connection.json",
+ env: { [key]: "/some/custom/value" },
+ }),
+ ).rejects.toBeInstanceOf(ForkServerUnavailable);
+ }
+ });
+});
diff --git a/scripts/boot-bench.mjs b/scripts/boot-bench.mjs
new file mode 100644
index 0000000000..c71bc931ee
--- /dev/null
+++ b/scripts/boot-bench.mjs
@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+/**
+ * Kernel boot fan-out benchmark for ENG-4387. Boots N IPython kernels concurrently
+ * and reports boot success rate, per-kernel latency, peak RSS, and wall time.
+ *
+ * node scripts/boot-bench.mjs --mode forkserver --n 100
+ *
+ * Modes:
+ * ungated direct `python -m ipykernel_launcher`, no boot gate
+ * gated direct spawn through the boot-gate semaphore (today's default)
+ * forkserver fork each kernel from the pre-imported template (PRIME_AGENT_KERNEL_FORKSERVER=1)
+ *
+ * Also asserts a forked kernel is a real, reachable kernel (runs code over ZMQ)
+ * and that namespaces are isolated (a var set in kernel A is absent in kernel B).
+ *
+ * Requires the coding-agent package to be built (dist/). Run from the repo root.
+ */
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const repoRoot = join(fileURLToPath(import.meta.url), "..", "..");
+const kernelMod = join(repoRoot, "packages", "coding-agent", "dist", "core", "kernel", "index.js");
+const bootstrapMod = join(repoRoot, "packages", "coding-agent", "dist", "core", "kernel", "bootstrap.js");
+const bootGateMod = join(repoRoot, "packages", "coding-agent", "dist", "core", "kernel", "boot-gate.js");
+
+function arg(name, def) {
+ const i = process.argv.indexOf(`--${name}`);
+ return i !== -1 ? process.argv[i + 1] : def;
+}
+
+const mode = arg("mode", "gated");
+const n = Number(arg("n", 50)) || 50;
+
+if (mode === "forkserver") process.env.PRIME_AGENT_KERNEL_FORKSERVER = "1";
+
+const { KernelManager } = await import(kernelMod);
+const { ensureKernelPython } = await import(bootstrapMod);
+const { withKernelBootPermit } = await import(bootGateMod);
+
+function peakRssMb() {
+ // Sum Pss across the process tree if available (COW-aware); fall back to RSS.
+ return Math.round(process.memoryUsage().rss / 1024 / 1024);
+}
+
+function pct(sorted, p) {
+ if (!sorted.length) return 0;
+ const idx = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length));
+ return Math.round(sorted[idx]);
+}
+
+async function bootOne(python) {
+ const m = new KernelManager({ python });
+ const started = performance.now();
+ const boot = () => m.start();
+ if (mode === "ungated") await boot();
+ else await withKernelBootPermit(boot);
+ const latency = performance.now() - started;
+ return { m, latency };
+}
+
+async function main() {
+ console.log(`mode=${mode} n=${n} platform=${process.platform}`);
+ const python = await ensureKernelPython({});
+
+ const wallStart = performance.now();
+ const results = await Promise.allSettled(Array.from({ length: n }, () => bootOne(python)));
+ const wallMs = performance.now() - wallStart;
+
+ const ok = results.filter((r) => r.status === "fulfilled");
+ const latencies = ok.map((r) => r.value.latency).sort((a, b) => a - b);
+ const managers = ok.map((r) => r.value.m);
+
+ console.log(`booted: ${ok.length}/${n} (${Math.round((100 * ok.length) / n)}%)`);
+ console.log(`latency: p50=${pct(latencies, 50)}ms p90=${pct(latencies, 90)}ms max=${pct(latencies, 100)}ms`);
+ console.log(`wall: ${Math.round(wallMs)}ms`);
+ console.log(`rss: ${peakRssMb()}MB (node process only)`);
+
+ const firstErr = results.find((r) => r.status === "rejected");
+ if (firstErr) console.log(`first error: ${firstErr.reason?.message ?? firstErr.reason}`);
+
+ // Reachability + isolation check on the first two booted kernels.
+ if (managers.length >= 2) {
+ const [a, b] = managers;
+ const setA = await a.execute("bench_marker = 4387\nprint(bench_marker)");
+ const seeB = await b.execute("print('bench_marker' in dir())");
+ const reachable = setA.status === "ok" && setA.stdout.includes("4387");
+ const isolated = seeB.status === "ok" && /False/.test(seeB.stdout);
+ console.log(`reachable: ${reachable ? "yes" : "NO"} isolated: ${isolated ? "yes" : "NO"}`);
+ }
+
+ await Promise.allSettled(managers.map((m) => m.dispose()));
+}
+
+await main();
+process.exit(0);