A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
fast kernel spawning via a python forkserver (#298)
kt2b6fe2f9eff7
diff --git a/packages/coding-agent/src/core/kernel/boot-gate.ts b/packages/coding-agent/src/core/kernel/boot-gate.tsindex 5813d5b755..3fe2211edd 100644--- a/packages/coding-agent/src/core/kernel/boot-gate.ts+++ b/packages/coding-agent/src/core/kernel/boot-gate.ts@@ -1,27 +1,50 @@import { cpus } from "node:os";import { Semaphore } from "../../utils/semaphore.js";+import { isForkServerEnabled } from "./fork-server.js";// Above core count because boots are IO-bound (cold imports), but capped so a// fan-out can't thrash the FS past the port-resolve window.const DEFAULT_KERNEL_BOOT_CONCURRENCY = Math.min(16, Math.max(4, (cpus().length || 4) * 2));const MAX_KERNEL_BOOT_CONCURRENCY = 64;+// Fork-per-child is ~ms and bypasses the FS, so we can admit well past the+// direct-spawn cap. But fork only removes the *import* cost — every admitted+// kernel still starts a live ioloop + heartbeat thread + rlm bootstrap, and+// letting all N do that at once trips the ready timeout (measured: 256 collapses+// to ~28% boot at N=200; core*4 holds 100%). So the gate stays a real bound.+// Explicit env overrides may go higher than the auto default, but not unbounded —+// past this the live-kernel startup storm regresses boot rate.+const FORKSERVER_KERNEL_BOOT_CONCURRENCY_MAX = 128;+const FORKSERVER_KERNEL_BOOT_CONCURRENCY = Math.min(+ FORKSERVER_KERNEL_BOOT_CONCURRENCY_MAX,+ Math.max(32, (cpus().length || 4) * 4),+);export function resolveKernelBootConcurrency(): number {const raw = process.env.PRIME_AGENT_MAX_CONCURRENT_KERNEL_BOOTS;+ const fallback = isForkServerEnabled() ? FORKSERVER_KERNEL_BOOT_CONCURRENCY : DEFAULT_KERNEL_BOOT_CONCURRENCY;if (raw === undefined || !/^\d+$/.test(raw)) {- return DEFAULT_KERNEL_BOOT_CONCURRENCY;+ return fallback;}const parsed = Number.parseInt(raw, 10);// A malformed or out-of-range value (incl. 0, e.g. "00") falls back to the// default rather than mis-bounding the gate or throwing at module load.if (parsed < 1) {- return DEFAULT_KERNEL_BOOT_CONCURRENCY;+ return fallback;}- return Math.min(MAX_KERNEL_BOOT_CONCURRENCY, parsed);+ // An explicit override may exceed the auto default (that's its purpose), but is+ // still clamped: the FS-contention cap on direct spawn, a looser backstop on fork.+ const max = isForkServerEnabled() ? FORKSERVER_KERNEL_BOOT_CONCURRENCY_MAX : MAX_KERNEL_BOOT_CONCURRENCY;+ return Math.min(max, parsed);}-const kernelBootSemaphore = new Semaphore(resolveKernelBootConcurrency());+// Resolved lazily on first boot, not at module load, so the fork-server flag is+// honored whenever it is set before the first kernel starts — not just if it+// happened to be set at import time.+let kernelBootSemaphore: Semaphore | undefined;export function withKernelBootPermit<T>(boot: () => Promise<T>, signal?: AbortSignal): Promise<T> {+ if (!kernelBootSemaphore) {+ kernelBootSemaphore = new Semaphore(resolveKernelBootConcurrency());+ }return kernelBootSemaphore.run(boot, signal);}diff --git a/packages/coding-agent/src/core/kernel/fork-server-script.ts b/packages/coding-agent/src/core/kernel/fork-server-script.tsnew file mode 100644index 0000000000..d08f8e0139--- /dev/null+++ b/packages/coding-agent/src/core/kernel/fork-server-script.ts@@ -0,0 +1,148 @@+// The Python "kernel forkserver": a long-lived template process that pays the+// ~1.2s IPython/ipykernel/rlm import cost once, then forks a ready-to-run kernel+// per request in ~ms. Children inherit the imported module objects via+// copy-on-write, bypassing the (slow, virtiofs-backed) per-file import path.+//+// Embedded as a string rather than shipped as a package asset so it can never be+// missing from a release layout (see the built-in-skills packaging gap). Run via+// `python -c <this> <control-socket-path>`.+//+// Protocol (newline-delimited JSON over the unix socket, forkserver is the client):+// -> { "id": <n>, "connectionPath": "<abs path>" } spawn request from Node+// <- { "type": "ready" } once, after imports finish+// <- { "id": <n>, "pid": <pid> } fork succeeded+// <- { "id": <n>, "error": "<message>" } fork failed+export const FORK_SERVER_SCRIPT = String.raw`+import gc+import json+import os+import signal+import socket+import sys+++def _reap_children(*_args):+ # Reap every exited child so disposed kernels never linger as zombies. Wired to+ # SIGCHLD so reaping happens on child exit, not only when the next request wakes+ # the accept loop. Safe under PEP 475: the interrupted socket read auto-retries.+ try:+ while True:+ pid, _status = os.waitpid(-1, os.WNOHANG)+ if pid == 0:+ break+ except ChildProcessError:+ pass+++def _import_template():+ # Everything a kernel touches at import time. Paid once; shared COW by children.+ import IPython # noqa: F401+ import ipykernel # noqa: F401+ import ipykernel.kernelapp # noqa: F401+ import jupyter_client # noqa: F401+ import nest_asyncio # noqa: F401+ try:+ import rlm # noqa: F401+ except Exception:+ # rlm may not import cleanly outside a live kernel namespace; the Node-side+ # bootstrap cell wires it up per-child regardless. Preloading is a best-effort+ # speedup, not a correctness requirement.+ pass+++def _run_child(connection_path, cwd, env):+ # We are the forked child; become the ipykernel server on the given connection.+ from ipykernel.kernelapp import IPKernelApp++ # Drop the inherited SIGCHLD reaper so it can't interfere with ipykernel's own+ # child/signal handling.+ signal.signal(signal.SIGCHLD, signal.SIG_DFL)++ # cwd/env are per-kernel and applied here (not at template import), so all+ # kernels can share one warm template regardless of their working dir / env.+ if env:+ os.environ.update(env)+ if cwd:+ # Don't swallow a bad cwd: direct spawn fails fast on ENOENT, so match that+ # (the OSError propagates, the child exits non-zero, Node falls back).+ os.chdir(cwd)++ # Drop any singleton the template happened to build so the child owns a fresh+ # instance (and, critically, a jupyter_client Session created in *this* pid;+ # a Session inherited from the template silently drops messages via check_pid).+ IPKernelApp.clear_instance()+ app = IPKernelApp.instance(connection_file=connection_path)+ # initialize() binds the 5 ZMQ ports, writes the resolved ports back into+ # connection.json, and starts the heartbeat thread + ioloop — all post-fork,+ # so no thread/loop/socket is ever inherited across the fork boundary.+ app.initialize([])+ app.start()+++def _serve(control_path):+ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)+ sock.connect(control_path)+ control_fd = sock.fileno()++ # Reap forked kernels as they exit, independent of the request loop.+ signal.signal(signal.SIGCHLD, _reap_children)++ _import_template()+ # Freeze the heap so the cyclic GC doesn't write to (and thus COW-copy) the+ # shared module pages, keeping memory genuinely shared across children.+ gc.freeze()++ f = sock.makefile("rwb", buffering=0)+ f.write(json.dumps({"type": "ready"}).encode() + b"\n")+ f.flush()++ while True:+ # Belt-and-suspenders: the SIGCHLD handler is the primary reaper, but sweep+ # again each turn in case a signal was missed (coalesced) while handling one.+ _reap_children()++ line = f.readline()+ if not line:+ break+ try:+ req = json.loads(line)+ except ValueError:+ continue+ req_id = req.get("id")+ connection_path = req.get("connectionPath")+ cwd = req.get("cwd")+ env = req.get("env")++ try:+ pid = os.fork()+ except OSError as exc:+ f.write(json.dumps({"id": req_id, "error": str(exc)}).encode() + b"\n")+ f.flush()+ continue++ if pid == 0:+ # Child: shed every inherited fd tied to the control channel, then run.+ try:+ sock.close()+ f.close()+ except Exception:+ pass+ try:+ os.close(control_fd)+ except OSError:+ pass+ try:+ _run_child(connection_path, cwd, env)+ except BaseException as exc: # never return to the accept loop+ sys.stderr.write("forked kernel failed: %r\n" % (exc,))+ os._exit(1)+ os._exit(0)++ # Parent: stay pristine (no loop/threads/ZMQ ever) so the next fork is clean.+ f.write(json.dumps({"id": req_id, "pid": pid}).encode() + b"\n")+ f.flush()+++if __name__ == "__main__":+ _serve(sys.argv[1])+`;diff --git a/packages/coding-agent/src/core/kernel/fork-server.ts b/packages/coding-agent/src/core/kernel/fork-server.tsnew file mode 100644index 0000000000..139c8aa64f--- /dev/null+++ b/packages/coding-agent/src/core/kernel/fork-server.ts@@ -0,0 +1,350 @@+// Node-side client for the Python kernel forkserver (see fork-server-script.ts).+// One forkserver process per (python, cwd, env) profile, spawned lazily and kept+// alive for the agent process. Each KernelManager asks it to fork a kernel onto a+// connection file instead of paying a full `python -m ipykernel_launcher` cold boot.+//+// Everything degrades to direct spawn: if the forkserver is disabled, unavailable,+// or a spawn request fails/times out, callers catch ForkServerUnavailable and fall+// back to the existing path, so correctness never depends on fork.+import { type ChildProcess, spawn } from "node:child_process";+import { mkdtempSync, rmSync } from "node:fs";+import { createServer, type Server, type Socket } from "node:net";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { registerSessionResourceCleanup } from "@earendil-works/pi-ai";+import { FORK_SERVER_SCRIPT } from "./fork-server-script.js";++const READY_TIMEOUT_MS = 30_000;+const SPAWN_TIMEOUT_MS = 10_000;+const STDERR_TAIL_MAX = 4096;++// Vars the Python interpreter consumes at startup (before any user code runs), so+// they can't be honored post-fork via os.environ.update — the forked child inherits+// the template's already-initialized sys.path / site config. A kernel overriding any+// of these to a value the template didn't launch with must take the direct-spawn+// path. We treat the entire PYTHON* family as startup-affecting rather than+// enumerating individual vars: the set is large and version-dependent, and a false+// positive merely (safely) routes a kernel to direct spawn. Plus the non-PYTHON*+// startup vars that also steer interpreter/site resolution.+const INTERPRETER_STARTUP_ENV_EXACT = ["VIRTUAL_ENV", "CONDA_PREFIX", "__PYVENV_LAUNCHER__"];++function affectsInterpreterStartup(key: string): boolean {+ return key.startsWith("PYTHON") || INTERPRETER_STARTUP_ENV_EXACT.includes(key);+}++export class ForkServerUnavailable extends Error {+ constructor(message: string) {+ super(message);+ this.name = "ForkServerUnavailable";+ }+}++/** Fork is a Linux-only, in-process fan-out optimization. */+export function isForkServerEnabled(): boolean {+ return process.platform === "linux" && process.env.PRIME_AGENT_KERNEL_FORKSERVER === "1";+}++// A forkserver template is defined solely by the interpreter — the imported+// module graph doesn't depend on cwd/env. Those are per-kernel and applied in the+// forked child, so every kernel for a given python shares ONE template.+interface ForkServerParams {+ python: string;+}++interface SpawnParams {+ connectionPath: string;+ cwd?: string;+ // Mirrors process.env shape; undefined values are dropped when JSON-serialized+ // to the child, matching how spawn() treats them on the direct path.+ env?: Record<string, string | undefined>;+}++type PendingSpawn = {+ resolve: (pid: number) => void;+ reject: (err: Error) => void;+ timer: ReturnType<typeof setTimeout>;+};++class ForkServer {+ private readonly params: ForkServerParams;+ // The env the template process was launched with, snapshotted at construction so+ // it exactly matches what the template's interpreter imported with. The+ // startup-env guard compares against THIS, not live process.env, so a later+ // mutation of process.env can't make a stale template look compatible.+ private readonly launchEnv: NodeJS.ProcessEnv;+ private proc?: ChildProcess;+ private server?: Server;+ private conn?: Socket;+ private socketDir?: string;+ private readyPromise?: Promise<void>;+ private failReady?: (err: Error) => void;+ private buffer = "";+ // Rolling tail of forkserver stderr. Forked children inherit this fd, so a+ // child's import/startup traceback lands here; surfaced in error messages.+ private stderrTail = "";+ private nextId = 1;+ private readonly pending = new Map<number, PendingSpawn>();+ // Request ids whose caller timed out; a late pid reply for one is an orphan to kill.+ private readonly abandoned = new Set<number>();+ private dead = false;++ constructor(params: ForkServerParams) {+ this.params = params;+ // Snapshot now and launch the template with this same object, so the guard's+ // comparison uses exactly the env the interpreter imported with.+ this.launchEnv = { ...process.env };+ }++ get isDead(): boolean {+ return this.dead;+ }++ /**+ * True if `env` overrides an interpreter-startup var to a value the template+ * didn't launch with — such a kernel can't be forked (sys.path is baked in at+ * import), so the caller must direct-spawn it.+ */+ needsStartupEnvNotInTemplate(env: SpawnParams["env"]): boolean {+ if (!env) return false;+ return Object.keys(env).some((k) => affectsInterpreterStartup(k) && env[k] !== this.launchEnv[k]);+ }++ private async ensureReady(): Promise<void> {+ if (this.dead) throw new ForkServerUnavailable("forkserver is dead");+ if (!this.readyPromise) {+ this.readyPromise = this.start().catch((err) => {+ this.markDead();+ throw err instanceof ForkServerUnavailable ? err : new ForkServerUnavailable(String(err));+ });+ }+ return this.readyPromise;+ }++ private start(): Promise<void> {+ this.socketDir = mkdtempSync(join(tmpdir(), "prime-agent-forkserver-"));+ const socketPath = join(this.socketDir, "control.sock");++ return new Promise<void>((resolve, reject) => {+ const server = createServer();+ this.server = server;++ let settled = false;+ const readyTimer = setTimeout(() => {+ if (settled) return;+ settled = true;+ reject(new ForkServerUnavailable(`forkserver did not become ready within ${READY_TIMEOUT_MS}ms`));+ }, READY_TIMEOUT_MS);++ // Lets markDead() fail a still-pending start (interpreter crashed / socket+ // closed before "ready") instead of waiting out the full ready timeout.+ this.failReady = (err) => {+ if (settled) return;+ settled = true;+ clearTimeout(readyTimer);+ reject(err instanceof ForkServerUnavailable ? err : new ForkServerUnavailable(String(err)));+ };++ server.on("connection", (socket) => {+ this.conn = socket;+ socket.setEncoding("utf8");+ socket.on("data", (chunk: string) => this.onData(chunk));+ socket.on("close", () => this.markDead());+ socket.on("error", () => this.markDead());+ });++ server.on("error", (err) => {+ if (settled) return;+ settled = true;+ clearTimeout(readyTimer);+ reject(new ForkServerUnavailable(`forkserver control socket failed: ${err.message}`));+ });++ // Buffered until the "ready" line; flip the promise from there.+ this.onReady = () => {+ if (settled) return;+ settled = true;+ clearTimeout(readyTimer);+ resolve();+ };++ server.listen(socketPath, () => {+ // The template only imports; its own cwd/env are irrelevant since each+ // forked child applies the per-kernel cwd/env itself. Inherit the daemon's.+ const proc = spawn(this.params.python, ["-c", FORK_SERVER_SCRIPT, socketPath], {+ env: this.launchEnv,+ stdio: ["ignore", "ignore", "pipe"],+ });+ this.proc = proc;+ proc.stderr?.on("data", (buf: Buffer) => {+ this.stderrTail = `${this.stderrTail}${buf.toString()}`.slice(-STDERR_TAIL_MAX);+ });+ proc.on("error", () => this.markDead());+ proc.on("exit", () => this.markDead());+ });+ });+ }++ private onReady: () => void = () => {};++ private onData(chunk: string): void {+ this.buffer += chunk;+ for (let idx = this.buffer.indexOf("\n"); idx !== -1; idx = this.buffer.indexOf("\n")) {+ const line = this.buffer.slice(0, idx);+ this.buffer = this.buffer.slice(idx + 1);+ if (!line.trim()) continue;+ let msg: { type?: string; id?: number; pid?: number; error?: string };+ try {+ msg = JSON.parse(line);+ } catch {+ continue;+ }+ if (msg.type === "ready") {+ this.onReady();+ continue;+ }+ if (typeof msg.id !== "number") continue;+ const p = this.pending.get(msg.id);+ if (!p) {+ // A pid for a request the caller already abandoned (timed out): the+ // fork succeeded but nobody owns it, so kill the orphan here.+ if (this.abandoned.delete(msg.id) && typeof msg.pid === "number") {+ try {+ process.kill(msg.pid, "SIGTERM");+ } catch {}+ }+ continue;+ }+ this.pending.delete(msg.id);+ clearTimeout(p.timer);+ if (typeof msg.pid === "number") {+ p.resolve(msg.pid);+ } else {+ p.reject(new ForkServerUnavailable(this.withStderr(msg.error ?? "forkserver fork failed")));+ }+ }+ }++ async spawnKernel(spawn: SpawnParams): Promise<number> {+ await this.ensureReady();+ if (this.dead || !this.conn) throw new ForkServerUnavailable("forkserver connection unavailable");+ const id = this.nextId++;+ return new Promise<number>((resolve, reject) => {+ const timer = setTimeout(() => {+ this.pending.delete(id);+ // The fork may still land after we give up; remember the id so its late+ // pid reply gets the orphan killed instead of leaked.+ this.abandoned.add(id);+ reject(new ForkServerUnavailable(`forkserver spawn timed out after ${SPAWN_TIMEOUT_MS}ms`));+ }, SPAWN_TIMEOUT_MS);+ this.pending.set(id, { resolve, reject, timer });+ this.conn?.write(+ `${JSON.stringify({ id, connectionPath: spawn.connectionPath, cwd: spawn.cwd, env: spawn.env })}\n`,+ );+ });+ }++ private withStderr(message: string): string {+ const tail = this.stderrTail.trim();+ return tail ? `${message}\nforkserver stderr:\n${tail}` : message;+ }++ /** Reject any in-flight ensureReady/spawnKernel callers so none wait out a timeout. */+ private rejectPending(reason: string): void {+ this.failReady?.(new ForkServerUnavailable(reason));+ for (const p of this.pending.values()) {+ clearTimeout(p.timer);+ p.reject(new ForkServerUnavailable(reason));+ }+ this.pending.clear();+ }++ private markDead(): void {+ this.dispose(this.withStderr("forkserver died"));+ }++ dispose(reason = "forkserver disposed"): void {+ if (this.dead) return;+ // Flip `dead` at entry so the close/exit events our own teardown triggers+ // don't re-enter. Pending callers are rejected directly here (not deferred to+ // those events), so this early flag flip can't strand them on a timeout.+ this.dead = true;+ this.rejectPending(reason);+ try {+ this.conn?.destroy();+ } catch {}+ try {+ this.server?.close();+ } catch {}+ try {+ this.proc?.kill("SIGTERM");+ } catch {}+ if (this.socketDir) {+ try {+ rmSync(this.socketDir, { recursive: true, force: true });+ } catch {}+ this.socketDir = undefined;+ }+ }+}++// Keyed by interpreter only: cwd/env are per-kernel (applied in the child), so all+// kernels for a given python share ONE warm template.+const servers = new Map<string, ForkServer>();+let cleanupRegistered = false;++function keyFor(params: ForkServerParams): string {+ return params.python;+}++function registerForkServerCleanupOnce(): void {+ if (cleanupRegistered) return;+ cleanupRegistered = true;+ // A forkserver is process-lived and shared across sessions, so per-session+ // cleanup must not yank the warm template from other live sessions — only a full+ // process shutdown (no sessionId) disposes it here.+ registerSessionResourceCleanup((sessionId) => {+ if (!sessionId) disposeAllForkServers();+ });+ // cleanupSessionResources(undefined) is never called on exit, so also tear the+ // forkserver down directly on process teardown or it leaks as an orphan.+ process.once("exit", disposeAllForkServers);+ for (const signal of ["SIGINT", "SIGTERM", "beforeExit"] as const) {+ process.once(signal, () => disposeAllForkServers());+ }+}++/**+ * Fork a kernel onto `spawn.connectionPath` from the shared template for this+ * interpreter, applying `spawn.cwd`/`spawn.env` in the forked child. Throws+ * ForkServerUnavailable if forking is disabled or fails — callers fall back to+ * direct spawn. Returns the forked child's pid (owned/killed by the caller).+ */+export async function forkKernel(python: string, spawn: SpawnParams): Promise<number> {+ if (!isForkServerEnabled()) throw new ForkServerUnavailable("forkserver disabled");+ registerForkServerCleanupOnce();+ const key = keyFor({ python });+ let server = servers.get(key);+ if (!server || server.isDead) {+ server = new ForkServer({ python });+ servers.set(key, server);+ }+ // Compare against the template's own launch env (not live process.env): a kernel+ // overriding an interpreter-startup var to a value the template didn't import+ // with can't be forked — defer to direct spawn rather than boot with wrong sys.path.+ if (server.needsStartupEnvNotInTemplate(spawn.env)) {+ throw new ForkServerUnavailable("kernel overrides interpreter-startup env; using direct spawn");+ }+ try {+ return await server.spawnKernel(spawn);+ } catch (err) {+ // Only evict if the map still points at this dead instance: a concurrent+ // caller may have already replaced it with a fresh live server under this key.+ if (server.isDead && servers.get(key) === server) servers.delete(key);+ throw err instanceof ForkServerUnavailable ? err : new ForkServerUnavailable(String(err));+ }+}++export function disposeAllForkServers(): void {+ for (const server of servers.values()) server.dispose();+ servers.clear();+}diff --git a/packages/coding-agent/src/core/kernel/index.ts b/packages/coding-agent/src/core/kernel/index.tsindex 1adb22cca2..5a95bbc74e 100644--- a/packages/coding-agent/src/core/kernel/index.ts+++ b/packages/coding-agent/src/core/kernel/index.ts@@ -9,6 +9,7 @@ import { registerSessionResourceCleanup } from "@earendil-works/pi-ai";import { v4 as uuid } from "uuid";import { Dealer, Subscriber } from "zeromq";import { ensureKernelPython, type KernelBootstrapProgressHandler, type KernelPythonSkill } from "./bootstrap.js";+import { ForkServerUnavailable, forkKernel, isForkServerEnabled } from "./fork-server.js";import {buildListNamesCode,buildRestoreCode,@@ -30,6 +31,8 @@ const IOPUB_SUBSCRIBE_DELAY_MS = 50;const DEFAULT_MAX_OUTPUT_CHARS = 65536;const HOST_REQUEST_DISPOSE_TIMEOUT_MS = 5000;const DEFAULT_SNAPSHOT_DEBOUNCE_MS = 1500;+// How often to poll a forked kernel's pid for unexpected death.+const FORKED_LIVENESS_POLL_MS = 1000;// Snapshot/restore cells can be large to (de)serialize; give them room beyond the user cap.const SNAPSHOT_MAX_OUTPUT_CHARS = 1_000_000;// Cap how long a graceful dispose waits on the final snapshot; the debounced@@ -409,6 +412,11 @@ export class KernelManager {private readonly commTargets = new Map<string, string>();private readonly handledHostRequestCommIds = new Set<string>();private kernel?: ChildProcess;+ // Set instead of `kernel` when the kernel was forked from the forkserver: it is+ // not a direct child, so it has no ChildProcess handle and is killed by pid.+ private kernelPid?: number;+ /** Polls a forked kernel's pid for death (no "exit" event on a non-child). */+ private forkedLivenessTimer?: ReturnType<typeof globalThis.setInterval>;private shell?: Dealer;private iopub?: Subscriber;private control?: Dealer;@@ -485,37 +493,71 @@ export class KernelManager {throw new Error("Kernel was disposed during startup");}- const { path: connectionPath, tempDir } = makeConnection();- this.tempDir = tempDir;+ let connection = makeConnection();+ this.tempDir = connection.tempDir;- const kernel = spawn(python, ["-m", "ipykernel_launcher", "-f", connectionPath], {- cwd: this.options.cwd,- env: this.options.env ? { ...process.env, ...this.options.env } : process.env,- stdio: ["ignore", "pipe", "pipe"],- });- this.kernel = kernel;+ // Fast path: fork a pre-imported kernel from the forkserver. Any failure+ // (disabled, unavailable, fork error) degrades to the direct-spawn path so+ // correctness never depends on fork.+ let forked = false;+ if (isForkServerEnabled()) {+ try {+ this.kernelPid = await forkKernel(python, {+ connectionPath: connection.path,+ cwd: this.options.cwd,+ // Match the direct-spawn env exactly: merge the current host env with+ // the per-kernel overrides, applied fresh in the child (the template's+ // inherited env snapshot may be stale by fork time).+ env: this.options.env ? { ...process.env, ...this.options.env } : { ...process.env },+ });+ forked = true;+ } catch (err) {+ if (!(err instanceof ForkServerUnavailable)) throw err;+ this.appendKernelDiagnostic(`forkserver unavailable, spawning directly: ${err.message}`);+ this.kernelPid = undefined;+ // A fork request that times out or loses its pid reply may still have+ // forked a child that binds the ports in this connection file. Mint a+ // fresh connection for the direct spawn so a possible orphan can never+ // collide with it (write the same file / re-bind the same ports).+ try {+ rmSync(connection.tempDir, { recursive: true, force: true });+ } catch {}+ connection = makeConnection();+ this.tempDir = connection.tempDir;+ }+ }- kernel.stderr?.on("data", (buf: Buffer) => {- const s = buf.toString();- this.kernelStderr += s;- });+ if (!forked) {+ const kernel = spawn(python, ["-m", "ipykernel_launcher", "-f", connection.path], {+ cwd: this.options.cwd,+ env: this.options.env ? { ...process.env, ...this.options.env } : process.env,+ stdio: ["ignore", "pipe", "pipe"],+ });+ this.kernel = kernel;- kernel.on("error", (err) => {- if (this.kernel !== kernel) return;- this.appendKernelDiagnostic(`spawn error: ${err.message}`);- this.state = "shutdown";- liveKernels.delete(this);- });+ kernel.stderr?.on("data", (buf: Buffer) => {+ const s = buf.toString();+ this.kernelStderr += s;+ });- kernel.on("exit", (code, signal) => {- if (this.kernel !== kernel) return;- if (this.state !== "shutdown") {- this.appendKernelDiagnostic(`unexpected exit code=${code} signal=${signal}`);- }- this.state = "shutdown";- liveKernels.delete(this);- });+ kernel.on("error", (err) => {+ if (this.kernel !== kernel) return;+ this.appendKernelDiagnostic(`spawn error: ${err.message}`);+ this.state = "shutdown";+ liveKernels.delete(this);+ });++ kernel.on("exit", (code, signal) => {+ if (this.kernel !== kernel) return;+ if (this.state !== "shutdown") {+ this.appendKernelDiagnostic(`unexpected exit code=${code} signal=${signal}`);+ }+ this.state = "shutdown";+ liveKernels.delete(this);+ });+ }+ const connectionPath = connection.path;let conn: ConnectionInfo;try {conn = await this.waitForResolvedConnection(connectionPath);@@ -549,12 +591,43 @@ export class KernelManager {}this.state = "running";+ this.startForkedLivenessMonitor();+ }++ // A forked kernel isn't a direct child, so no "exit" fires when it dies. Poll its+ // pid so a mid-run death tears down like the direct-spawn exit handler: mark+ // shutdown, drop from liveKernels, and reject any in-flight execution.+ private startForkedLivenessMonitor(): void {+ if (this.kernelPid === undefined) return;+ this.forkedLivenessTimer = globalThis.setInterval(() => {+ if (this.state !== "running") return;+ if (!this.forkedKernelDied()) return;+ this.appendKernelDiagnostic("forked kernel exited unexpectedly");+ this.state = "shutdown";+ liveKernels.delete(this);+ this.cleanupResources();+ }, FORKED_LIVENESS_POLL_MS);+ this.forkedLivenessTimer.unref?.();+ }++ // A forked kernel is not a direct child, so it emits no "exit" event; poll its+ // pid so a dead child fails fast instead of burning the full resolve timeout.+ private forkedKernelDied(): boolean {+ if (this.kernelPid === undefined) return false;+ try {+ process.kill(this.kernelPid, 0);+ return false;+ } catch (error) {+ // EPERM means the pid exists but isn't signalable by us — still alive.+ // Only ESRCH (no such process) is genuine death.+ return !(error instanceof Error && (error as NodeJS.ErrnoException).code === "EPERM");+ }}private async waitForResolvedConnection(connectionPath: string): Promise<ConnectionInfo> {const startedAt = Date.now();while (Date.now() - startedAt < PORTS_RESOLVE_TIMEOUT_MS) {- if ((this.state as string) === "shutdown") {+ if ((this.state as string) === "shutdown" || this.forkedKernelDied()) {const tail = this.kernelStderr.slice(-1024);throw new Error(`Kernel exited before resolving ports. stderr:\n${tail || "(empty)"}`);}@@ -583,7 +656,7 @@ export class KernelManager {const startedAt = Date.now();while (Date.now() - startedAt < READY_TIMEOUT_MS) {- if ((this.state as string) === "shutdown") {+ if ((this.state as string) === "shutdown" || this.forkedKernelDied()) {const tail = this.kernelStderr.slice(-1024);throw new Error(`Kernel exited during startup. stderr:\n${tail || "(empty)"}`);}@@ -950,6 +1023,10 @@ export class KernelManager {private cleanupResources(): void {this.clearSnapshotTimer();+ if (this.forkedLivenessTimer) {+ globalThis.clearInterval(this.forkedLivenessTimer);+ this.forkedLivenessTimer = undefined;+ }this.rejectActiveExecution(new Error("Kernel has been shut down"));this.shell?.close();this.iopub?.close();@@ -959,9 +1036,16 @@ export class KernelManager {this.control = undefined;this.iopubPumpPromise = undefined;try {- this.kernel?.kill("SIGTERM");+ if (this.kernel) {+ this.kernel.kill("SIGTERM");+ } else if (this.kernelPid !== undefined && !this.forkedKernelDied()) {+ // Only signal a forked kernel confirmed still alive: a dead pid may have+ // been recycled by the OS, and SIGTERM would then hit an unrelated process.+ process.kill(this.kernelPid, "SIGTERM");+ }} catch {}this.kernel = undefined;+ this.kernelPid = undefined;this.connection = undefined;if (this.tempDir) {try {diff --git a/packages/coding-agent/test/boot-gate.test.ts b/packages/coding-agent/test/boot-gate.test.tsindex 795d3db907..31d1c8b33c 100644--- a/packages/coding-agent/test/boot-gate.test.ts+++ b/packages/coding-agent/test/boot-gate.test.ts@@ -2,10 +2,12 @@ import { afterEach, describe, expect, it } from "vitest";import { resolveKernelBootConcurrency } from "../src/core/kernel/boot-gate.js";const ENV = "PRIME_AGENT_MAX_CONCURRENT_KERNEL_BOOTS";+const FORK_ENV = "PRIME_AGENT_KERNEL_FORKSERVER";describe("resolveKernelBootConcurrency", () => {afterEach(() => {delete process.env[ENV];+ delete process.env[FORK_ENV];});it("never returns a value below 1 (so the semaphore can't throw at load)", () => {@@ -15,10 +17,22 @@ describe("resolveKernelBootConcurrency", () => {}});- it("honors a clean positive integer, clamped to the max", () => {+ it("honors a clean positive integer, clamped to the direct-spawn max", () => {process.env[ENV] = "8";expect(resolveKernelBootConcurrency()).toBe(8);process.env[ENV] = "999999";expect(resolveKernelBootConcurrency()).toBe(64);});++ it("raises the ceiling when the forkserver is enabled (linux only)", () => {+ if (process.platform !== "linux") return;+ process.env[FORK_ENV] = "1";+ // Auto default is well above the direct-spawn default but still bounded.+ const auto = resolveKernelBootConcurrency();+ expect(auto).toBeGreaterThanOrEqual(32);+ expect(auto).toBeLessThanOrEqual(128);+ // A huge override is clamped to the fork backstop, not the direct-spawn cap.+ process.env[ENV] = "999999";+ expect(resolveKernelBootConcurrency()).toBe(128);+ });});diff --git a/packages/coding-agent/test/kernel-fork-server.test.ts b/packages/coding-agent/test/kernel-fork-server.test.tsnew file mode 100644index 0000000000..8c3a0b0587--- /dev/null+++ b/packages/coding-agent/test/kernel-fork-server.test.ts@@ -0,0 +1,49 @@+import { afterEach, describe, expect, it } from "vitest";+import { ForkServerUnavailable, forkKernel, isForkServerEnabled } from "../src/core/kernel/fork-server.js";++const FORK_ENV = "PRIME_AGENT_KERNEL_FORKSERVER";++describe("fork-server gating", () => {+ afterEach(() => {+ delete process.env[FORK_ENV];+ });++ it("is disabled unless the flag is set on linux", () => {+ delete process.env[FORK_ENV];+ expect(isForkServerEnabled()).toBe(false);+ process.env[FORK_ENV] = "1";+ expect(isForkServerEnabled()).toBe(process.platform === "linux");+ });++ it("rejects with ForkServerUnavailable when disabled so callers fall back", async () => {+ delete process.env[FORK_ENV];+ await expect(forkKernel("python3", { connectionPath: "/tmp/nope/connection.json" })).rejects.toBeInstanceOf(+ ForkServerUnavailable,+ );+ });++ it("degrades to ForkServerUnavailable when the interpreter can't start", async () => {+ if (process.platform !== "linux") return;+ process.env[FORK_ENV] = "1";+ // The spawn errors immediately (ENOENT), so markDead fails the ready promise+ // fast rather than waiting out the ready timeout.+ await expect(+ forkKernel("/nonexistent/python-binary", { connectionPath: "/tmp/nope/connection.json" }),+ ).rejects.toBeInstanceOf(ForkServerUnavailable);+ }, 15_000);++ it("falls back to direct spawn for any PYTHON* startup-env override", async () => {+ if (process.platform !== "linux") return;+ process.env[FORK_ENV] = "1";+ // The guard treats the whole PYTHON* family as startup-affecting, so even a var+ // not explicitly enumerated diverts to direct spawn (no var can be "missed").+ for (const key of ["PYTHONPATH", "PYTHONUSERBASE", "PYTHONDONTWRITEBYTECODE"]) {+ await expect(+ forkKernel("python3", {+ connectionPath: "/tmp/nope/connection.json",+ env: { [key]: "/some/custom/value" },+ }),+ ).rejects.toBeInstanceOf(ForkServerUnavailable);+ }+ });+});diff --git a/scripts/boot-bench.mjs b/scripts/boot-bench.mjsnew file mode 100644index 0000000000..c71bc931ee--- /dev/null+++ b/scripts/boot-bench.mjs@@ -0,0 +1,96 @@+#!/usr/bin/env node+/**+ * Kernel boot fan-out benchmark for ENG-4387. Boots N IPython kernels concurrently+ * and reports boot success rate, per-kernel latency, peak RSS, and wall time.+ *+ * node scripts/boot-bench.mjs --mode forkserver --n 100+ *+ * Modes:+ * ungated direct `python -m ipykernel_launcher`, no boot gate+ * gated direct spawn through the boot-gate semaphore (today's default)+ * forkserver fork each kernel from the pre-imported template (PRIME_AGENT_KERNEL_FORKSERVER=1)+ *+ * Also asserts a forked kernel is a real, reachable kernel (runs code over ZMQ)+ * and that namespaces are isolated (a var set in kernel A is absent in kernel B).+ *+ * Requires the coding-agent package to be built (dist/). Run from the repo root.+ */+import { readFileSync } from "node:fs";+import { join } from "node:path";+import { fileURLToPath } from "node:url";++const repoRoot = join(fileURLToPath(import.meta.url), "..", "..");+const kernelMod = join(repoRoot, "packages", "coding-agent", "dist", "core", "kernel", "index.js");+const bootstrapMod = join(repoRoot, "packages", "coding-agent", "dist", "core", "kernel", "bootstrap.js");+const bootGateMod = join(repoRoot, "packages", "coding-agent", "dist", "core", "kernel", "boot-gate.js");++function arg(name, def) {+ const i = process.argv.indexOf(`--${name}`);+ return i !== -1 ? process.argv[i + 1] : def;+}++const mode = arg("mode", "gated");+const n = Number(arg("n", 50)) || 50;++if (mode === "forkserver") process.env.PRIME_AGENT_KERNEL_FORKSERVER = "1";++const { KernelManager } = await import(kernelMod);+const { ensureKernelPython } = await import(bootstrapMod);+const { withKernelBootPermit } = await import(bootGateMod);++function peakRssMb() {+ // Sum Pss across the process tree if available (COW-aware); fall back to RSS.+ return Math.round(process.memoryUsage().rss / 1024 / 1024);+}++function pct(sorted, p) {+ if (!sorted.length) return 0;+ const idx = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length));+ return Math.round(sorted[idx]);+}++async function bootOne(python) {+ const m = new KernelManager({ python });+ const started = performance.now();+ const boot = () => m.start();+ if (mode === "ungated") await boot();+ else await withKernelBootPermit(boot);+ const latency = performance.now() - started;+ return { m, latency };+}++async function main() {+ console.log(`mode=${mode} n=${n} platform=${process.platform}`);+ const python = await ensureKernelPython({});++ const wallStart = performance.now();+ const results = await Promise.allSettled(Array.from({ length: n }, () => bootOne(python)));+ const wallMs = performance.now() - wallStart;++ const ok = results.filter((r) => r.status === "fulfilled");+ const latencies = ok.map((r) => r.value.latency).sort((a, b) => a - b);+ const managers = ok.map((r) => r.value.m);++ console.log(`booted: ${ok.length}/${n} (${Math.round((100 * ok.length) / n)}%)`);+ console.log(`latency: p50=${pct(latencies, 50)}ms p90=${pct(latencies, 90)}ms max=${pct(latencies, 100)}ms`);+ console.log(`wall: ${Math.round(wallMs)}ms`);+ console.log(`rss: ${peakRssMb()}MB (node process only)`);++ const firstErr = results.find((r) => r.status === "rejected");+ if (firstErr) console.log(`first error: ${firstErr.reason?.message ?? firstErr.reason}`);++ // Reachability + isolation check on the first two booted kernels.+ if (managers.length >= 2) {+ const [a, b] = managers;+ const setA = await a.execute("bench_marker = 4387\nprint(bench_marker)");+ const seeB = await b.execute("print('bench_marker' in dir())");+ const reachable = setA.status === "ok" && setA.stdout.includes("4387");+ const isolated = seeB.status === "ok" && /False/.test(seeB.stdout);+ console.log(`reachable: ${reachable ? "yes" : "NO"} isolated: ${isolated ? "yes" : "NO"}`);+ }++ await Promise.allSettled(managers.map((m) => m.dispose()));+}++await main();+process.exit(0);