Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix(ingest): stop ix map segfaulting at teardown by asking parse workers to end themselves (#598)
Joseph Mikhail20e1daedc6a6
diff --git a/core-ingestion/src/parse-worker.ts b/core-ingestion/src/parse-worker.tsindex 3485d01e..e99fa73e 100644--- a/core-ingestion/src/parse-worker.ts+++ b/core-ingestion/src/parse-worker.ts@@ -1,7 +1,7 @@/*** Worker thread entry point for parallel file parsing.* Each worker maintains its own Parser singleton (safe — module state is per-thread).- * Receives: { filePath: string, source: string }+ * Receives: { filePath: string, source: string } | { __shutdown: true }* Posts: { ok: true, result: FileParseResult } | { ok: false }*/import { parentPort } from 'node:worker_threads';@@ -9,7 +9,46 @@ import { parseFile } from './index.js';if (!parentPort) throw new Error('parse-worker must run inside a worker thread');-parentPort.on('message', ({ filePath, source }: { filePath: string; source: string }) => {+/**+ * A message asking this thread to end itself, rather than be terminated.+ *+ * `Worker.terminate()` tears a thread down from the outside, and doing that to+ * a thread that has loaded the tree-sitter native bindings SEGFAULTS the whole+ * process -- the parses need not even be in flight, an idle worker that has+ * parsed once is enough. Measured on Windows/Node 26 at ~8% per teardown, and+ * the pool terminates every worker at the end of every run, so roughly one+ * `ix map` in twelve exited 139 after a completely successful ingest: patches+ * committed, summary printed, non-zero status for anything reading `$?`.+ *+ * Closing the port from INSIDE lets the thread unwind its own event loop and+ * dispose its isolate in order. Measured 0 crashes in the same experiment.+ *+ * The teardown contract, precisely, because the pool's half of it changed and+ * this is the only place the worker states it: `ParsePool` never terminates a+ * worker. If one does not answer, the pool stops waiting and `unref()`s it, so+ * the CLI can exit while the thread is still alive. Nothing reclaims a wedged+ * thread before the process ends -- do not write code here that relies on+ * being killed, and do not "restore" a `terminate()` fallback in the pool,+ * which is the segfault itself.+ *+ * Not exported: `ix-cli` does not depend on this package and inlines the+ * literal, and this module throws at load outside a worker thread, so an+ * importer could not use the type anyway. The binding between the two is the+ * test that runs `ParsePool` against this built file.+ */+type ShutdownMessage = { __shutdown: true };++type ParseMessage = { filePath: string; source: string };++parentPort.on('message', (msg: ParseMessage | ShutdownMessage) => {+ if ((msg as ShutdownMessage).__shutdown) {+ // `close()`, not `process.exit()`. Both avoid the crash, but `exit()` from+ // a worker takes the process's exit code with it if the main thread is+ // already on its way out, and this runs during teardown.+ parentPort!.close();+ return;+ }+ const { filePath, source } = msg as ParseMessage;try {const result = parseFile(filePath, source);parentPort!.postMessage({ ok: result !== null, result: result ?? null });diff --git a/ix-cli/src/cli/__tests__/parse-pool.test.ts b/ix-cli/src/cli/__tests__/parse-pool.test.tsindex 07f08201..be88eb02 100644--- a/ix-cli/src/cli/__tests__/parse-pool.test.ts+++ b/ix-cli/src/cli/__tests__/parse-pool.test.ts@@ -1,5 +1,7 @@import { afterEach, beforeEach, describe, expect, it } from "vitest";-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { dirname } from "node:path";+import { fileURLToPath } from "node:url";import { tmpdir } from "node:os";import { join } from "node:path";@@ -42,10 +44,20 @@ describe("ParsePool", () => {return path;}+ /**+ * Answers parses, and honours the shutdown request like the real worker.+ *+ * The `__shutdown` branch is not decoration. Without it this fixture ignores+ * the request, so `destroy()` waits out the full grace and then gives up on+ * the worker -- which meant the test named "shuts down without hanging" was+ * silently exercising the FALLBACK rather than the clean path it claims, and+ * paying 2s to do it.+ */const ECHO = `import { parentPort } from 'node:worker_threads';- parentPort.on('message', ({ filePath }) => {- parentPort.postMessage({ ok: true, result: { filePath } });+ parentPort.on('message', (msg) => {+ if (msg && msg.__shutdown) { parentPort.close(); return; }+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });});`;@@ -64,8 +76,9 @@ describe("ParsePool", () => {const IDLE_FAULT = `import { parentPort } from 'node:worker_threads';let served = 0;- parentPort.on('message', ({ filePath }) => {- parentPort.postMessage({ ok: true, result: { filePath } });+ parentPort.on('message', (msg) => {+ if (msg && msg.__shutdown) { parentPort.close(); return; }+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });if (++served === 1) setTimeout(() => { throw new Error('idle fault'); }, 20);});`;@@ -79,8 +92,8 @@ describe("ParsePool", () => {expect(results).toEqual([{ filePath: "a.ts" }, { filePath: "b.ts" }]);expect(pool.crashedTasks()).toBe(0);- // The hang: `destroy()` terminates every worker, each emits 'exit', and an- // exit handler that respawns hands back a pool of live threads that nothing+ // The hang: shutting a worker down makes it emit 'exit', and an exit+ // handler that respawns hands back a pool of live threads that nothing// owns. If that happens this test still passes -- the process is what hangs// -- so `crashedTasks()` is asserted as the observable proxy: a deliberate// teardown must not be recorded as a crash.@@ -113,7 +126,7 @@ describe("ParsePool", () => {expect(await pool.parse("first.ts", "x")).toEqual({ filePath: "first.ts" });// Let the fault land while the pool is idle.- await new Promise(resolve => setTimeout(resolve, 120));+ await new Promise((resolve) => setTimeout(resolve, 120));// TWO at once, deliberately. `drain()` pops the free list, so a single// parse takes the replacement worker that `onError` just pushed and never@@ -137,7 +150,7 @@ describe("ParsePool", () => {const first = await Promise.all(Array.from({ length: 20 }, (_, i) => pool.parse(`f${i}.ts`, "x")),);- expect(first.every(r => r === null)).toBe(true);+ expect(first.every((r) => r === null)).toBe(true);// The regression: these arrive after the pool is dead.await expect(pool.parse("later.ts", "x")).resolves.toBeNull();@@ -150,4 +163,356 @@ describe("ParsePool", () => {await pool.destroy();});++ it("asks a worker to end itself rather than terminating it", async () => {+ // The fourth way this file has gone wrong, and the only one that was not a+ // hang. `Worker.terminate()` tears a thread down from outside, and doing+ // that to one that has loaded the tree-sitter native bindings segfaults the+ // PROCESS -- an idle worker that parsed a single file is enough, because+ // the crash is in disposing an isolate that still holds the addon. Twenty+ // teardowns per process, six runs each, on Windows/Node 26:+ //+ // terminate() 5 of 6 runs died with SIGSEGV (139)+ // worker closes its own port 0 of 6+ //+ // `destroy()` runs in `ingestFiles`'s outermost `finally`, so this was one+ // `ix map` in roughly twelve exiting 139 with every patch committed and the+ // summary printed -- which is why nobody reported it.+ //+ // Asserted through a marker the worker writes when ASKED to go, because the+ // crash itself is probabilistic: a test that just tore pools down would+ // pass against the bug most of the time. This one pins the mechanism.+ const marker = join(dir, "asked-to-go");+ const path = worker(+ "polite",+ `+ import { parentPort } from 'node:worker_threads';+ import { writeFileSync } from 'node:fs';+ parentPort.on('message', (msg) => {+ if (msg && msg.__shutdown) {+ writeFileSync(${JSON.stringify(marker)}, 'bye', 'utf8');+ parentPort.close();+ return;+ }+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });+ });+ `,+ );++ const pool = new ParsePool(path, 2);+ pool.init();+ // Parse first: an untouched worker has not loaded the addon, and it is the+ // loaded-then-idle thread that crashes.+ await Promise.all([pool.parse("a.ts", "x"), pool.parse("b.ts", "x")]);+ await pool.destroy();++ expect(existsSync(marker), "destroy() must ask, not terminate").toBe(true);+ });++ it("returns, rather than waiting forever, when an IDLE worker ignores the request", async () => {+ // The fallback, for the case it is actually for: a worker sitting in its+ // event loop that simply will not answer. Waiting on it forever is the CLI+ // hang the three tests above exist for, so the pool stops waiting and+ // unrefs it. The busy case is a later test, and is deliberately different.+ const path = worker(+ "stubborn",+ `+ import { parentPort } from 'node:worker_threads';+ parentPort.on('message', () => { /* answers nothing, leaves nothing */ });+ setInterval(() => {}, 1000);+ `,+ );++ const pool = new ParsePool(path, 1);+ pool.init();++ const started = Date.now();+ await pool.destroy();+ const elapsed = Date.now() - started;++ // Derived from the constant, not restated as a magic number. Tuning the+ // grace down is a legitimate change and must not fail a test about+ // waiting-then-returning.+ expect(elapsed).toBeGreaterThanOrEqual(ParsePool.SHUTDOWN_GRACE_MS * 0.75);+ // ...and it did NOT wait forever. Without the timeout this never resolves+ // and the failure is a suite timeout with no explanation attached.+ expect(elapsed).toBeLessThan(ParsePool.SHUTDOWN_GRACE_MS * 4);+ }, 15000);++ it("still gives up on a worker that was busy when the grace expired, once it goes idle", async () => {+ // The hang I introduced while fixing the previous finding. The busy branch+ // `return`ed instead of re-arming, so a worker that happened to be busy at+ // the ONE moment the timer fired permanently disarmed the fallback: it+ // finished its parse, went idle, refused the shutdown, and nothing was left+ // to terminate it. `destroy()` never resolved.+ //+ // Reproduced against the broken version at a 5s cutoff before writing this.+ const path = worker(+ "busy-then-stubborn",+ `+ import { parentPort } from 'node:worker_threads';+ parentPort.on('message', (msg) => {+ if (msg && msg.__shutdown) return; // refuses, once idle+ const end = Date.now() + 400; while (Date.now() < end);+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });+ });+ setInterval(() => {}, 1000); // and stays alive+ `,+ );++ const pool = new ParsePool(path, 1, 100);+ pool.init();+ const parsing = pool.parse("slow.ts", "x");+ // Destroy while it is mid-parse, so the first expiry lands on a BUSY worker.+ await new Promise(resolve => setTimeout(resolve, 50));++ const started = Date.now();+ await pool.destroy();++ // It returned at all -- that is the regression. And it waited for the parse+ // rather than killing it, which is the rule the previous test pins.+ expect(Date.now() - started).toBeGreaterThan(300);+ await expect(parsing).resolves.toEqual({ filePath: "slow.ts" });+ }, 20000);++ it("gives a worker that has just gone idle a full grace before giving up", async () => {+ // Idleness has to be OBSERVED for a grace period, not read as an instant.+ // The worker posts its result, the main thread deletes its `active` entry,+ // and only then does the worker reach the queued `__shutdown` and start+ // unwinding its isolate. An expiry landing in that window sees a worker+ // that looks idle and unresponsive and abandons it -- one that was about to+ // answer. Under the old `terminate()` this was the ~8% per-teardown+ // segfault itself; the pool unrefs now, so the cost is a lost parse result+ // rather than the process, and it is still wrong.+ //+ // The timings are chosen so the two rules give different answers, which is+ // the only way to catch this. Grace 300ms, so ticks land at 300/600/900. A+ // 580ms parse puts the result 20ms before a tick, and the worker then+ // spends 150ms in its shutdown handler:+ //+ // instantaneous idleness -> abandons at 600, mid-handler, no marker+ // observed idleness -> first idle tick at 600 starts the clock, so+ // giving up would be 900; the worker finishes+ // at 730 and exits on its own, marker written+ const marker = join(dir, "unwound-cleanly");+ const path = worker(+ "slow-to-unwind",+ `+ import { parentPort } from 'node:worker_threads';+ import { writeFileSync } from 'node:fs';+ const spin = (ms) => { const end = Date.now() + ms; while (Date.now() < end); };+ parentPort.on('message', (msg) => {+ if (msg && msg.__shutdown) {+ spin(150); // stands in for isolate teardown+ writeFileSync(${JSON.stringify(marker)}, 'clean', 'utf8');+ parentPort.close();+ return;+ }+ spin(580);+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });+ });+ `,+ );++ const pool = new ParsePool(path, 1, 300);+ pool.init();+ const parsing = pool.parse("slow.ts", "x");+ await pool.destroy();++ await expect(parsing).resolves.toEqual({ filePath: "slow.ts" });+ expect(+ existsSync(marker),+ "a worker that just went idle must not be killed mid-teardown",+ ).toBe(true);+ }, 20000);++ it("gives up on a worker that stays busy past the hard deadline", async () => {+ // The bound on the busy branch. Waiting while a worker is mid-parse is+ // right -- it is about to answer -- but waiting FOREVER is a hang, and+ // `destroy()` is the first statement of `ingestFiles`'s outermost+ // `finally`, so it stops the run before it can print anything at all.+ //+ // An earlier revision did wait forever, reasoning that killing a busy+ // worker "buys nothing". True, but the conclusion did not follow: the pool+ // does not have to kill it to stop waiting on it. This fixture is a worker+ // that never reaches its message loop, and the assertion is simply that+ // `destroy()` still returns.+ const path = worker(+ "never-finishes",+ `+ import { parentPort } from 'node:worker_threads';+ parentPort.on('message', () => {+ // Busy well past the ceiling, then gone. Bounded on purpose: the pool+ // unrefs rather than terminates now, so an endless loop here would burn+ // a core for the rest of the file.+ const end = Date.now() + 5000; while (Date.now() < end);+ process.exit(0);+ });+ `,+ );++ // A 500ms ceiling rather than the real 10s: what is being asserted is that+ // a ceiling EXISTS and is honoured, which is precisely what the unbounded+ // version lacked. Spending the production value here would add ten seconds+ // to every leg of the matrix and prove nothing extra.+ const pool = new ParsePool(path, 1, 100, 500);+ pool.init();+ const wedged = pool.parse("spin.ts", "x");+ await new Promise(resolve => setTimeout(resolve, 50));++ const started = Date.now();+ await Promise.race([+ pool.destroy(),+ new Promise((_, reject) =>+ setTimeout(() => reject(new Error("destroy() never returned")), 10000),+ ),+ ]);++ // It returned, by letting go of the worker once the deadline passed --+ // long before that 5s spin finishes, which is the point.+ expect(Date.now() - started).toBeLessThan(3000);+ // And the task that worker was holding still settles, so the `Promise.all`+ // over the parse batch cannot hang either.+ await expect(wedged).resolves.toBeNull();+ }, 20000);++ it("resolves parses still queued when the pool is destroyed", async () => {+ // `destroy()` left `this.queue` untouched while `onResult` still called+ // `drain()`, so a worker finishing its last parse could be handed a queued+ // file BEHIND the `__shutdown` already sitting in its port. It closed+ // first, and that task's promise never settled -- and `onError`+ // early-returns once `destroyed` is set, so it was not counted either. A+ // silently lost file that `crashedTasks()` reported as zero, which is the+ // number the stitch gate and the mtime baseline trust.+ const path = worker(+ "slowecho",+ `+ import { parentPort } from 'node:worker_threads';+ parentPort.on('message', (msg) => {+ if (msg && msg.__shutdown) { parentPort.close(); return; }+ const end = Date.now() + 300; while (Date.now() < end);+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });+ });+ `,+ );++ // One worker: the first parse is dispatched, the second stays queued.+ const pool = new ParsePool(path, 1, 200);+ pool.init();+ const dispatched = pool.parse("a.ts", "x");+ const queued = pool.parse("b.ts", "x");+ await new Promise(resolve => setTimeout(resolve, 50));++ await pool.destroy();++ await expect(dispatched).resolves.toEqual({ filePath: "a.ts" });+ // Settles rather than hanging forever...+ await expect(queued).resolves.toBeNull();+ // ...and is NOT counted. Every gate that reads `crashedParses()` -- the+ // mtime baseline, the pre-migration delete guard, both stitch gates -- runs+ // inside `ingestFiles`'s `try`, strictly before the `finally` that calls+ // `destroy()`. Counting here could only move the printed summary, making it+ // disagree with the baseline decision already taken.+ expect(pool.crashedTasks(), "too late for any gate to see it").toBe(0);+ }, 20000);++ it("waits for a worker that is mid-parse instead of terminating it", async () => {+ // The rule the previous test cannot check, because its fixture wedges in+ // JAVASCRIPT and `terminate()` kills that instantly. A real parse blocks+ // inside the addon, where a V8 termination interrupt has no JS boundary to+ // fire at, so `terminate()` waits for the call to return anyway -- measured+ // at 3981ms against 4s of CPU-bound native work on Node 26. Terminating a+ // busy worker therefore costs exactly as much as asking does and adds back+ // the segfault, on a thread that was about to answer.+ //+ // `pbkdf2Sync` stands in for a long parse: same shape, no tree-sitter+ // needed. The grace is 200ms and the work is ~1.3s, so the margin holds+ // even on a machine several times faster than this one.+ const marker = join(dir, "finished-its-parse");+ const path = worker(+ "busy",+ `+ import { parentPort } from 'node:worker_threads';+ import { writeFileSync } from 'node:fs';+ import { pbkdf2Sync } from 'node:crypto';+ parentPort.on('message', (msg) => {+ if (msg && msg.__shutdown) {+ writeFileSync(${JSON.stringify(marker)}, 'graceful', 'utf8');+ parentPort.close();+ return;+ }+ pbkdf2Sync('p', 's', 4000000, 64, 'sha512');+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });+ });+ `,+ );++ // An explicit, huge ceiling. Without it this test rides the production 10s+ // `SHUTDOWN_MAX_WAIT_MS`, and the ~1.5s KDF only has to be 6.5x slower --+ // an instrumented coverage leg, a loaded macos-14 runner -- for the pool to+ // give up first and the marker never to be written. It would then fail as+ // "a busy worker must be asked", reading as a real regression rather than a+ // timing miss.+ const pool = new ParsePool(path, 1, 200, 120000);+ pool.init();++ // Do NOT await: destroy() has to run while the parse is still in flight.+ const parsing = pool.parse("slow.ts", "x");+ await new Promise((resolve) => setTimeout(resolve, 300));++ const started = Date.now();+ await pool.destroy();+ const elapsed = Date.now() - started;++ // It outlasted the grace rather than terminating at it...+ expect(elapsed).toBeGreaterThan(250);+ // ...the parse it was in the middle of still returned its result...+ await expect(parsing).resolves.toEqual({ filePath: "slow.ts" });+ // ...and the worker went by ANSWERING, which is the whole point: under the+ // old rule it was terminated mid-native-call, which is the segfault.+ expect(existsSync(marker), "a busy worker must be asked, never terminated").toBe(true);+ expect(pool.crashedTasks(), "waiting for a parse is not a crash").toBe(0);+ }, 20000);++ it("shuts the REAL parse worker down through the same protocol", async () => {+ // The one test that binds the two packages. `ix-cli` does not depend on+ // `@ix/core-ingestion` as a package -- it loads the built worker by+ // relative path -- so `__shutdown` is a bare literal in three unlinked+ // places: here, `parse-pool.ts`, and `core-ingestion/src/parse-worker.ts`.+ // Every other test in this file uses an inline fixture that hardcodes the+ // literal itself, so renaming or dropping the handler in the real worker+ // would leave the whole suite green while every `ix map` teardown quietly+ // reverted to grace-then-terminate: the exact segfault this is all for.+ const real = join(+ dirname(fileURLToPath(import.meta.url)),+ "../../../../core-ingestion/dist/parse-worker.js",+ );+ expect(existsSync(real), `build core-ingestion first: ${real}`).toBe(true);++ // A deliberately huge grace, so this cannot pass or fail on how fast the+ // machine is. If the real worker still understands `__shutdown` it exits in+ // milliseconds; if it does not, teardown waits the full 10s and misses the+ // bound by a factor of three, on any runner.+ const pool = new ParsePool(real, 2, 10000);+ pool.init();+ // Parse for real, so the threads have the tree-sitter addon loaded -- an+ // untouched worker does not, and it is the loaded-then-idle thread that+ // crashes under `terminate()`.+ const results = await Promise.all([+ pool.parse("a.ts", "export function a(): number { return 1; }"),+ pool.parse("b.ts", "export function b(): number { return 2; }"),+ ]);+ expect(+ results.every((r) => r !== null),+ "the real worker should parse these",+ ).toBe(true);++ const started = Date.now();+ await pool.destroy();+ const elapsed = Date.now() - started;++ // It answered rather than being terminated.+ expect(elapsed).toBeLessThan(3000);+ expect(pool.crashedTasks(), "a clean teardown is not a crash").toBe(0);+ }, 20000);});diff --git a/ix-cli/src/cli/commands/parse-pool.ts b/ix-cli/src/cli/commands/parse-pool.tsindex e25470c7..be080c3d 100644--- a/ix-cli/src/cli/commands/parse-pool.ts+++ b/ix-cli/src/cli/commands/parse-pool.ts@@ -23,7 +23,21 @@ export class ParsePool {private queue: Task[] = [];private active = new Map<Worker, Task>();- constructor(private workerPath: string, private concurrency: number) {}+ /**+ * @param graceMs How long an UNRESPONSIVE IDLE worker gets before the pool+ * stops waiting for it.+ * @param maxWaitMs The ceiling on waiting for a BUSY one.+ *+ * Both injectable only so the tests can pin the behaviour without spending+ * the real periods -- the ceiling test would otherwise cost ten seconds on+ * every leg of the matrix. `ingestFiles` passes neither.+ */+ constructor(+ private workerPath: string,+ private concurrency: number,+ private graceMs: number = ParsePool.SHUTDOWN_GRACE_MS,+ private maxWaitMs: number = ParsePool.SHUTDOWN_MAX_WAIT_MS,+ ) {}init(): void {for (let i = 0; i < this.concurrency; i++) {@@ -61,12 +75,12 @@ export class ParsePool {}async destroy(): Promise<void> {- // Set FIRST. `terminate()` makes every worker emit 'exit', and the handler- // for that treats an exit as a crash and spawns a replacement -- so- // shutting the pool down span up a fresh set of threads, `this.workers = []`- // orphaned them, and because a worker thread refs the event loop and the- // CLI has no `process.exit(0)` on its success path, `ix map` printed its- // summary and then hung forever.+ // Set FIRST. A worker that answers `__shutdown` emits 'exit', and the+ // handler for that treats+ // an exit as a crash and spawns a replacement. So closing the pool spun up+ // a fresh set of threads, `this.workers = []` orphaned them, and because a+ // worker thread refs the event loop and the CLI has no `process.exit(0)` on+ // its success path, `ix map` printed its summary and then hung forever.this.destroyed = true;// ...and `dead`, so a `parse()` after teardown resolves instead of queueing// onto a pool with no workers left to run it. Not reachable from@@ -74,11 +88,240 @@ export class ParsePool {// is the identical hang this file has already produced three times, and one// line closes it.this.dead = true;- await Promise.all(this.workers.map(w => w.terminate()));+ // Resolve what is still QUEUED. `destroy()` left the queue untouched while+ // `onResult` still calls `drain()`, so a worker finishing its last parse+ // could be handed a queued file behind the `__shutdown` already sitting in+ // its port: the worker closes first, and that task's promise never settled.+ // Empty on every path where `ingestFiles` completes; reachable when it+ // throws with parses still queued.+ //+ // Resolved but deliberately NOT counted as crashed. An earlier revision+ // incremented `crashed` here and justified it as "the input the stitch gate+ // and the mtime baseline trust" -- which is wrong: `destroy()` runs in the+ // outermost `finally`, while the baseline, the pre-migration delete guard+ // and both stitch gates read `crashedParses()` inside the `try`, strictly+ // earlier. They cannot observe this. The only readers left are the summary+ // fields, so counting it there made the printed `parseErrors` disagree with+ // the baseline decision already taken, on the one path that reaches it --+ // a run that is throwing anyway, where the exception is the story.+ const stranded = this.queue.splice(0, this.queue.length);+ for (const t of stranded) t.resolve(null);+ await Promise.all(this.workers.map(w => this.shutdown(w)));+ // ...and resolve whatever was still IN FLIGHT when the last worker went.+ //+ // `onError` early-returns once `destroyed` is set -- deliberately, so a+ // deliberate teardown is not counted as a crash and does not respawn the+ // pool it is closing -- which means the in-flight task of a worker that+ // `shutdown` gave up on was never settled by anything. Its promise stayed+ // pending for the life of the process, and the `Promise.all` over the parse+ // batch with it. Found by a test asserting that such a worker's parse still+ // settles: `destroy()` returned on schedule and the test then hung on the+ // task.+ //+ // Not counted, for the same reason the stranded queue is not: every gate+ // that reads `crashedParses()` has already run by the time `destroy()` is+ // called from the outermost `finally`.+ for (const task of this.active.values()) task.resolve(null);+ this.active.clear();this.workers = [];this.idle = [];}+ /**+ * How long an IDLE worker gets to answer before the pool gives up on it.+ *+ * Only ever paid by one whose JS event loop is wedged. A responsive worker+ * closes its port and emits 'exit' in about a millisecond, and `shutdown`+ * resolves on that event, so the common path does not wait. Public because+ * the test that pins the fallback derives its bound from this rather than+ * restating it: tuning the grace should not fail a test about+ * waiting-then-returning.+ *+ * A worker that is mid-parse keeps being given more time instead, up to+ * `SHUTDOWN_MAX_WAIT_MS`. Note this is checked on tick boundaries, so one+ * that goes idle just after a tick can wait up to twice this.+ */+ static readonly SHUTDOWN_GRACE_MS = 2000;++ /**+ * The same, for a worker that has ALREADY faulted.+ *+ * Shorter than the ordinary grace, though what it buys is now modest and+ * worth stating exactly: since nothing is terminated, the faulted thread+ * survives regardless of this value. All it controls is how long the pool+ * holds a timer and an 'exit' listener for a worker it has already replaced.+ * An earlier revision justified it as keeping the pool from running+ * `concurrency + 1` threads, which was true only while the fallback killed+ * things. Do not tune it on that reasoning.+ */+ static readonly FAULTED_GRACE_MS = 250;++ /**+ * The ceiling on waiting for a BUSY worker, after which the pool lets go.+ *+ * Generous, because the common reason a worker is still busy at teardown is+ * a genuine parse that will finish on its own and answer. Finite, because+ * the alternative is an unbounded wait, and `destroy()` is the first+ * statement of `ingestFiles`'s outermost `finally` -- a hang there stops the+ * run before it can even print its summary. Only reached on the path where+ * `ingestFiles` throws with parses still in flight; on every completing path+ * the batch is awaited long before `destroy()`.+ */+ static readonly SHUTDOWN_MAX_WAIT_MS = 10000;++ /**+ * End one worker by ASKING, and let go of it if it will not answer.+ *+ * `terminate()` tears a thread down from outside, and doing that to a thread+ * that has loaded the tree-sitter native bindings segfaults the process. The+ * thread does not have to be busy: an idle worker that has parsed a single+ * file is enough, because the crash is in disposing an isolate that still+ * holds the addon. Measured on Windows/Node 26, twenty pool teardowns per+ * process, six runs each:+ *+ * terminate() 5 of 6 runs died with SIGSEGV (139)+ * worker closes its own port 0 of 6+ * worker calls process.exit(0) 0 of 6+ *+ * `destroy()` runs in `ingestFiles`'s outermost `finally`, so this was one+ * `ix map` in roughly twelve exiting 139 with every patch committed and the+ * summary already printed -- invisible unless something reads the status.+ *+ * What the grace period bounds, precisely: the wait for a reply from a+ * worker that is IDLE and does not answer -- one whose JS event loop is+ * wedged. It does not bound teardown in general, and an earlier revision of+ * this comment claimed it did.+ *+ * NOTHING here terminates a worker, and that is the point. Four review+ * rounds went into rules for when killing a thread was safe -- never while+ * busy, only after observed idleness, unless past a deadline -- and every+ * rule had a counterexample, because `terminate()` is simply the wrong verb+ * for teardown. It is what segfaults. Against a worker inside a native call+ * it does not even preempt: V8's termination interrupt is only checked at JS+ * boundaries, so it resolves when the call returns on its own (3981ms+ * against ~4s of CPU-bound native work, measured), making it slower AND+ * crash-prone there. Only a JS-wedged worker dies promptly, in about 2ms.+ *+ * Teardown does not need the thread to die. It needs the pool to stop+ * waiting on it and the thread to stop holding the process open, which is+ * exactly `unref()`. Measured on the real parse worker, four addon-loaded+ * threads left live and unref'd across process exit: 0 failures in 10 runs,+ * against 5 of 6 for `terminate()`.+ *+ * So the clocks below decide WHEN to give up, never whether it is safe to+ * kill -- and the `onError` path needs no special case either, which is what+ * removed the caveat this comment used to carry.+ *+ * The cost, stated rather than hidden: a genuinely wedged worker survives+ * until the process exits, burning a core if it is spinning. For `ix map` or+ * `ix ingest` that is nothing -- the process is about to end -- and `ix+ * watch` is safe too, because it runs each map as a CHILD PROCESS+ * (`watch.ts`), so every pool dies with its own process.+ *+ * The consumer that does hold the leak is the MCP server's in-process+ * runner: `createInProcessRunner` in `src/mcp/runner.ts` is the default+ * (`IX_MCP_SUBPROCESS=1` opts out), so repeated `ix_map` calls against a repo+ * with a wedging grammar would accumulate unref'd, still-spinning threads for+ * the life of the server. An earlier version of this comment named `ix watch`+ * and missed that one, which is exactly backwards. It is still the right+ * trade -- a leaked thread is recoverable, a SIGSEGV is not, and the crash+ * would take the whole server down -- but that is where to look if threads+ * ever accumulate.+ */+ private shutdown(w: Worker, graceMs = this.graceMs): Promise<void> {+ // A worker that has already gone: settle NOW, synchronously.+ //+ // Checked rather than inferred from a failed post. An earlier revision+ // relied on `postMessage()` throwing once the thread was gone; it does not+ // -- on Node 26 it is a silent no-op -- so for every worker that had+ // already exited (`process.exit()` inside it, the respawn loop, a crash)+ // this held a live timer and a listener for an 'exit' that had already+ // fired and would never fire again, then terminated a corpse. `threadId`+ // is -1 once the thread is gone, which is the only signal here that is+ // actually true.+ if (w.threadId === -1) return Promise.resolve();+ return new Promise<void>(resolve => {+ let settled = false;+ let timer: ReturnType<typeof setTimeout>;+ const done = (): void => {+ if (settled) return;+ settled = true;+ clearTimeout(timer);+ resolve();+ };+ // Two clocks, and every path ends at one of them.+ //+ // `idleSince` is when the worker was last OBSERVED idle, not the instant+ // it left `active`. Checking `active` alone gave up on a worker that had+ // only just posted its result: the main thread deletes the entry, and the+ // worker reaches the queued `__shutdown` a moment later, so an expiry+ // landing in that window read a busy-but-untracked worker as silent.+ // Restarting the clock when it goes idle gives it a full `graceMs` of+ // quiet before anything is concluded from the silence. Under the old+ // `terminate()` this was a segfault; it now costs a parse result rather+ // than the process, but abandoning a worker that was about to answer is+ // still wrong.+ //+ // `hardDeadline` bounds the busy case, and exists because an earlier+ // revision re-armed forever. `destroy()` is the first statement of+ // `ingestFiles`'s outermost `finally`, so waiting on a worker that never+ // finishes stopped the run before it could print its summary at all --+ // strictly worse than the exit-139 it replaced. Past the deadline the+ // pool lets go, which costs nothing now that letting go is `unref()`+ // rather than a kill.+ // The ceiling as given. `Math.max(graceMs, maxWaitMs)` meant a ceiling+ // could never be shorter than the grace -- `new ParsePool(p, 1, 2000,+ // 500)` silently got 2000ms, not 500ms. It is still only CHECKED on tick+ // boundaries, so the effective ceiling rounds up to the next multiple of+ // `graceMs`; that is why the tests bound loosely rather than exactly.+ const hardDeadline = Date.now() + this.maxWaitMs;+ let idleSince: number | null = this.active.has(w) ? null : Date.now();+ const arm = (): void => {+ timer = setTimeout(tick, graceMs);+ // `unref` so a pool that is torn down early cannot hold the process+ // open for a grace period after everything else has finished.+ timer.unref?.();+ };+ const tick = (): void => {+ const now = Date.now();+ const busy = this.active.has(w);+ if (busy) idleSince = null;+ else if (idleSince === null) idleSince = now;++ const quietLongEnough = idleSince !== null && now - idleSince >= graceMs;+ if (!quietLongEnough && now < hardDeadline) {+ arm();+ return;+ }+ // Given up on: stop waiting for it, and stop it holding the process+ // open. NOT terminated.+ //+ // `terminate()` was the wrong verb for this whole branch. It is what+ // segfaults -- that is the bug this file exists to fix -- and against a+ // worker inside a native call it does not even preempt: it resolves+ // when the call returns (4457ms against ~4.5s of work, measured), so it+ // was strictly slower AND crash-prone there. `unref()` gives the only+ // thing teardown actually needs: the thread stops keeping the event+ // loop alive, so the CLI exits, and nobody disposes an isolate that+ // still holds the addon. Measured on the real parse worker, four+ // addon-loaded threads left live and unref'd across process exit: 0+ // failures in 10 runs, against 5 of 6 for `terminate()`.+ //+ // The cost is written up on `shutdown` above: a wedged worker survives+ // until the process exits. Nothing for the CLI, and `ix watch` runs+ // each map as a child process, but the MCP in-process runner can+ // accumulate them.+ w.unref();+ done();+ };+ arm();+ w.once('exit', done);+ // No try/catch: the post cannot throw for a live thread, and the dead+ // case is handled above. Wrapping it here is what disguised the bug.+ w.postMessage({ __shutdown: true });+ });+ }+private spawnWorker(): Worker {const w = new Worker(this.workerPath);w.on('message', (msg: { ok: boolean; result: unknown }) => this.onResult(w, msg));@@ -156,13 +399,13 @@ export class ParsePool {private static readonly MAX_RESPAWNS = 16;private onError(w: Worker, _err: Error): void {- // Not during shutdown. `destroy()` terminates every worker on purpose, and+ // Not during shutdown. `destroy()` ends every worker on purpose, and// treating that as a crash respawns the pool it is trying to close: the// replacements outlive `this.workers = []`, a worker thread refs the event// loop, and the CLI prints its summary and then hangs forever. The guard// lives HERE rather than on the 'exit' listener because 'error' reaches the- // same code: a worker that faults while `destroy()` is awaiting- // `terminate()` took the identical path.+ // same code: a worker that faults while `destroy()` is waiting on it takes+ // the identical path.if (this.destroyed) return;const task = this.active.get(w);if (task) {@@ -179,7 +422,18 @@ export class ParsePool {// lost task is still counted for the stitch gate.const idx = this.workers.indexOf(w);if (idx === -1) return;- w.terminate().catch(() => {});+ // Asked rather than killed, for the same reason `destroy()` asks, and+ // fire-and-forget as the `terminate()` here always was.+ //+ // Defensive, and measured to be so: a worker that emits 'error' is already+ // being torn down by V8, 'exit' follows immediately, and it never processes+ // a `__shutdown` posted from this handler -- a probe printed the 'error'+ // event with `threadId=1`, then 'exit' with `threadId=-1`, and the worker's+ // message handler never ran. So on both arms this settles via 'exit' or the+ // `threadId` fast path, and the post is a no-op. Kept because it costs+ // nothing and is correct if a worker ever does survive an 'error'; do not+ // read it as evidence that a faulted thread can still answer.+ this.shutdown(w, ParsePool.FAULTED_GRACE_MS).catch(() => {});this.workers.splice(idx, 1);// ...and out of `idle` too, ALWAYS, cap or no cap. A worker can emit 'error'// while it is IDLE -- an uncaught async throw, or ERR_WORKER_OUT_OF_MEMORY