Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

fix(ingest): stop ix map segfaulting at teardown by asking parse workers to end themselves (#598)

Joseph Mikhail20e1daedc6a6
diff --git a/core-ingestion/src/parse-worker.ts b/core-ingestion/src/parse-worker.ts
index 3485d01e..e99fa73e 100644
--- a/core-ingestion/src/parse-worker.ts
+++ b/core-ingestion/src/parse-worker.ts
@@ -1,7 +1,7 @@
/**
* Worker thread entry point for parallel file parsing.
* Each worker maintains its own Parser singleton (safe — module state is per-thread).
- * Receives: { filePath: string, source: string }
+ * Receives: { filePath: string, source: string } | { __shutdown: true }
* Posts: { ok: true, result: FileParseResult } | { ok: false }
*/
import { parentPort } from 'node:worker_threads';
@@ -9,7 +9,46 @@ import { parseFile } from './index.js';
if (!parentPort) throw new Error('parse-worker must run inside a worker thread');
-parentPort.on('message', ({ filePath, source }: { filePath: string; source: string }) => {
+/**
+ * A message asking this thread to end itself, rather than be terminated.
+ *
+ * `Worker.terminate()` tears a thread down from the outside, and doing that to
+ * a thread that has loaded the tree-sitter native bindings SEGFAULTS the whole
+ * process -- the parses need not even be in flight, an idle worker that has
+ * parsed once is enough. Measured on Windows/Node 26 at ~8% per teardown, and
+ * the pool terminates every worker at the end of every run, so roughly one
+ * `ix map` in twelve exited 139 after a completely successful ingest: patches
+ * committed, summary printed, non-zero status for anything reading `$?`.
+ *
+ * Closing the port from INSIDE lets the thread unwind its own event loop and
+ * dispose its isolate in order. Measured 0 crashes in the same experiment.
+ *
+ * The teardown contract, precisely, because the pool's half of it changed and
+ * this is the only place the worker states it: `ParsePool` never terminates a
+ * worker. If one does not answer, the pool stops waiting and `unref()`s it, so
+ * the CLI can exit while the thread is still alive. Nothing reclaims a wedged
+ * thread before the process ends -- do not write code here that relies on
+ * being killed, and do not "restore" a `terminate()` fallback in the pool,
+ * which is the segfault itself.
+ *
+ * Not exported: `ix-cli` does not depend on this package and inlines the
+ * literal, and this module throws at load outside a worker thread, so an
+ * importer could not use the type anyway. The binding between the two is the
+ * test that runs `ParsePool` against this built file.
+ */
+type ShutdownMessage = { __shutdown: true };
+
+type ParseMessage = { filePath: string; source: string };
+
+parentPort.on('message', (msg: ParseMessage | ShutdownMessage) => {
+ if ((msg as ShutdownMessage).__shutdown) {
+ // `close()`, not `process.exit()`. Both avoid the crash, but `exit()` from
+ // a worker takes the process's exit code with it if the main thread is
+ // already on its way out, and this runs during teardown.
+ parentPort!.close();
+ return;
+ }
+ const { filePath, source } = msg as ParseMessage;
try {
const result = parseFile(filePath, source);
parentPort!.postMessage({ ok: result !== null, result: result ?? null });
diff --git a/ix-cli/src/cli/__tests__/parse-pool.test.ts b/ix-cli/src/cli/__tests__/parse-pool.test.ts
index 07f08201..be88eb02 100644
--- a/ix-cli/src/cli/__tests__/parse-pool.test.ts
+++ b/ix-cli/src/cli/__tests__/parse-pool.test.ts
@@ -1,5 +1,7 @@
import { afterEach, beforeEach, describe, expect, it } from "vitest";
-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import { fileURLToPath } from "node:url";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -42,10 +44,20 @@ describe("ParsePool", () => {
return path;
}
+ /**
+ * Answers parses, and honours the shutdown request like the real worker.
+ *
+ * The `__shutdown` branch is not decoration. Without it this fixture ignores
+ * the request, so `destroy()` waits out the full grace and then gives up on
+ * the worker -- which meant the test named "shuts down without hanging" was
+ * silently exercising the FALLBACK rather than the clean path it claims, and
+ * paying 2s to do it.
+ */
const ECHO = `
import { parentPort } from 'node:worker_threads';
- parentPort.on('message', ({ filePath }) => {
- parentPort.postMessage({ ok: true, result: { filePath } });
+ parentPort.on('message', (msg) => {
+ if (msg && msg.__shutdown) { parentPort.close(); return; }
+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
});
`;
@@ -64,8 +76,9 @@ describe("ParsePool", () => {
const IDLE_FAULT = `
import { parentPort } from 'node:worker_threads';
let served = 0;
- parentPort.on('message', ({ filePath }) => {
- parentPort.postMessage({ ok: true, result: { filePath } });
+ parentPort.on('message', (msg) => {
+ if (msg && msg.__shutdown) { parentPort.close(); return; }
+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
if (++served === 1) setTimeout(() => { throw new Error('idle fault'); }, 20);
});
`;
@@ -79,8 +92,8 @@ describe("ParsePool", () => {
expect(results).toEqual([{ filePath: "a.ts" }, { filePath: "b.ts" }]);
expect(pool.crashedTasks()).toBe(0);
- // The hang: `destroy()` terminates every worker, each emits 'exit', and an
- // exit handler that respawns hands back a pool of live threads that nothing
+ // The hang: shutting a worker down makes it emit 'exit', and an exit
+ // handler that respawns hands back a pool of live threads that nothing
// owns. If that happens this test still passes -- the process is what hangs
// -- so `crashedTasks()` is asserted as the observable proxy: a deliberate
// teardown must not be recorded as a crash.
@@ -113,7 +126,7 @@ describe("ParsePool", () => {
expect(await pool.parse("first.ts", "x")).toEqual({ filePath: "first.ts" });
// Let the fault land while the pool is idle.
- await new Promise(resolve => setTimeout(resolve, 120));
+ await new Promise((resolve) => setTimeout(resolve, 120));
// TWO at once, deliberately. `drain()` pops the free list, so a single
// parse takes the replacement worker that `onError` just pushed and never
@@ -137,7 +150,7 @@ describe("ParsePool", () => {
const first = await Promise.all(
Array.from({ length: 20 }, (_, i) => pool.parse(`f${i}.ts`, "x")),
);
- expect(first.every(r => r === null)).toBe(true);
+ expect(first.every((r) => r === null)).toBe(true);
// The regression: these arrive after the pool is dead.
await expect(pool.parse("later.ts", "x")).resolves.toBeNull();
@@ -150,4 +163,356 @@ describe("ParsePool", () => {
await pool.destroy();
});
+
+ it("asks a worker to end itself rather than terminating it", async () => {
+ // The fourth way this file has gone wrong, and the only one that was not a
+ // hang. `Worker.terminate()` tears a thread down from outside, and doing
+ // that to one that has loaded the tree-sitter native bindings segfaults the
+ // PROCESS -- an idle worker that parsed a single file is enough, because
+ // the crash is in disposing an isolate that still holds the addon. Twenty
+ // teardowns per process, six runs each, on Windows/Node 26:
+ //
+ // terminate() 5 of 6 runs died with SIGSEGV (139)
+ // worker closes its own port 0 of 6
+ //
+ // `destroy()` runs in `ingestFiles`'s outermost `finally`, so this was one
+ // `ix map` in roughly twelve exiting 139 with every patch committed and the
+ // summary printed -- which is why nobody reported it.
+ //
+ // Asserted through a marker the worker writes when ASKED to go, because the
+ // crash itself is probabilistic: a test that just tore pools down would
+ // pass against the bug most of the time. This one pins the mechanism.
+ const marker = join(dir, "asked-to-go");
+ const path = worker(
+ "polite",
+ `
+ import { parentPort } from 'node:worker_threads';
+ import { writeFileSync } from 'node:fs';
+ parentPort.on('message', (msg) => {
+ if (msg && msg.__shutdown) {
+ writeFileSync(${JSON.stringify(marker)}, 'bye', 'utf8');
+ parentPort.close();
+ return;
+ }
+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
+ });
+ `,
+ );
+
+ const pool = new ParsePool(path, 2);
+ pool.init();
+ // Parse first: an untouched worker has not loaded the addon, and it is the
+ // loaded-then-idle thread that crashes.
+ await Promise.all([pool.parse("a.ts", "x"), pool.parse("b.ts", "x")]);
+ await pool.destroy();
+
+ expect(existsSync(marker), "destroy() must ask, not terminate").toBe(true);
+ });
+
+ it("returns, rather than waiting forever, when an IDLE worker ignores the request", async () => {
+ // The fallback, for the case it is actually for: a worker sitting in its
+ // event loop that simply will not answer. Waiting on it forever is the CLI
+ // hang the three tests above exist for, so the pool stops waiting and
+ // unrefs it. The busy case is a later test, and is deliberately different.
+ const path = worker(
+ "stubborn",
+ `
+ import { parentPort } from 'node:worker_threads';
+ parentPort.on('message', () => { /* answers nothing, leaves nothing */ });
+ setInterval(() => {}, 1000);
+ `,
+ );
+
+ const pool = new ParsePool(path, 1);
+ pool.init();
+
+ const started = Date.now();
+ await pool.destroy();
+ const elapsed = Date.now() - started;
+
+ // Derived from the constant, not restated as a magic number. Tuning the
+ // grace down is a legitimate change and must not fail a test about
+ // waiting-then-returning.
+ expect(elapsed).toBeGreaterThanOrEqual(ParsePool.SHUTDOWN_GRACE_MS * 0.75);
+ // ...and it did NOT wait forever. Without the timeout this never resolves
+ // and the failure is a suite timeout with no explanation attached.
+ expect(elapsed).toBeLessThan(ParsePool.SHUTDOWN_GRACE_MS * 4);
+ }, 15000);
+
+ it("still gives up on a worker that was busy when the grace expired, once it goes idle", async () => {
+ // The hang I introduced while fixing the previous finding. The busy branch
+ // `return`ed instead of re-arming, so a worker that happened to be busy at
+ // the ONE moment the timer fired permanently disarmed the fallback: it
+ // finished its parse, went idle, refused the shutdown, and nothing was left
+ // to terminate it. `destroy()` never resolved.
+ //
+ // Reproduced against the broken version at a 5s cutoff before writing this.
+ const path = worker(
+ "busy-then-stubborn",
+ `
+ import { parentPort } from 'node:worker_threads';
+ parentPort.on('message', (msg) => {
+ if (msg && msg.__shutdown) return; // refuses, once idle
+ const end = Date.now() + 400; while (Date.now() < end);
+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
+ });
+ setInterval(() => {}, 1000); // and stays alive
+ `,
+ );
+
+ const pool = new ParsePool(path, 1, 100);
+ pool.init();
+ const parsing = pool.parse("slow.ts", "x");
+ // Destroy while it is mid-parse, so the first expiry lands on a BUSY worker.
+ await new Promise(resolve => setTimeout(resolve, 50));
+
+ const started = Date.now();
+ await pool.destroy();
+
+ // It returned at all -- that is the regression. And it waited for the parse
+ // rather than killing it, which is the rule the previous test pins.
+ expect(Date.now() - started).toBeGreaterThan(300);
+ await expect(parsing).resolves.toEqual({ filePath: "slow.ts" });
+ }, 20000);
+
+ it("gives a worker that has just gone idle a full grace before giving up", async () => {
+ // Idleness has to be OBSERVED for a grace period, not read as an instant.
+ // The worker posts its result, the main thread deletes its `active` entry,
+ // and only then does the worker reach the queued `__shutdown` and start
+ // unwinding its isolate. An expiry landing in that window sees a worker
+ // that looks idle and unresponsive and abandons it -- one that was about to
+ // answer. Under the old `terminate()` this was the ~8% per-teardown
+ // segfault itself; the pool unrefs now, so the cost is a lost parse result
+ // rather than the process, and it is still wrong.
+ //
+ // The timings are chosen so the two rules give different answers, which is
+ // the only way to catch this. Grace 300ms, so ticks land at 300/600/900. A
+ // 580ms parse puts the result 20ms before a tick, and the worker then
+ // spends 150ms in its shutdown handler:
+ //
+ // instantaneous idleness -> abandons at 600, mid-handler, no marker
+ // observed idleness -> first idle tick at 600 starts the clock, so
+ // giving up would be 900; the worker finishes
+ // at 730 and exits on its own, marker written
+ const marker = join(dir, "unwound-cleanly");
+ const path = worker(
+ "slow-to-unwind",
+ `
+ import { parentPort } from 'node:worker_threads';
+ import { writeFileSync } from 'node:fs';
+ const spin = (ms) => { const end = Date.now() + ms; while (Date.now() < end); };
+ parentPort.on('message', (msg) => {
+ if (msg && msg.__shutdown) {
+ spin(150); // stands in for isolate teardown
+ writeFileSync(${JSON.stringify(marker)}, 'clean', 'utf8');
+ parentPort.close();
+ return;
+ }
+ spin(580);
+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
+ });
+ `,
+ );
+
+ const pool = new ParsePool(path, 1, 300);
+ pool.init();
+ const parsing = pool.parse("slow.ts", "x");
+ await pool.destroy();
+
+ await expect(parsing).resolves.toEqual({ filePath: "slow.ts" });
+ expect(
+ existsSync(marker),
+ "a worker that just went idle must not be killed mid-teardown",
+ ).toBe(true);
+ }, 20000);
+
+ it("gives up on a worker that stays busy past the hard deadline", async () => {
+ // The bound on the busy branch. Waiting while a worker is mid-parse is
+ // right -- it is about to answer -- but waiting FOREVER is a hang, and
+ // `destroy()` is the first statement of `ingestFiles`'s outermost
+ // `finally`, so it stops the run before it can print anything at all.
+ //
+ // An earlier revision did wait forever, reasoning that killing a busy
+ // worker "buys nothing". True, but the conclusion did not follow: the pool
+ // does not have to kill it to stop waiting on it. This fixture is a worker
+ // that never reaches its message loop, and the assertion is simply that
+ // `destroy()` still returns.
+ const path = worker(
+ "never-finishes",
+ `
+ import { parentPort } from 'node:worker_threads';
+ parentPort.on('message', () => {
+ // Busy well past the ceiling, then gone. Bounded on purpose: the pool
+ // unrefs rather than terminates now, so an endless loop here would burn
+ // a core for the rest of the file.
+ const end = Date.now() + 5000; while (Date.now() < end);
+ process.exit(0);
+ });
+ `,
+ );
+
+ // A 500ms ceiling rather than the real 10s: what is being asserted is that
+ // a ceiling EXISTS and is honoured, which is precisely what the unbounded
+ // version lacked. Spending the production value here would add ten seconds
+ // to every leg of the matrix and prove nothing extra.
+ const pool = new ParsePool(path, 1, 100, 500);
+ pool.init();
+ const wedged = pool.parse("spin.ts", "x");
+ await new Promise(resolve => setTimeout(resolve, 50));
+
+ const started = Date.now();
+ await Promise.race([
+ pool.destroy(),
+ new Promise((_, reject) =>
+ setTimeout(() => reject(new Error("destroy() never returned")), 10000),
+ ),
+ ]);
+
+ // It returned, by letting go of the worker once the deadline passed --
+ // long before that 5s spin finishes, which is the point.
+ expect(Date.now() - started).toBeLessThan(3000);
+ // And the task that worker was holding still settles, so the `Promise.all`
+ // over the parse batch cannot hang either.
+ await expect(wedged).resolves.toBeNull();
+ }, 20000);
+
+ it("resolves parses still queued when the pool is destroyed", async () => {
+ // `destroy()` left `this.queue` untouched while `onResult` still called
+ // `drain()`, so a worker finishing its last parse could be handed a queued
+ // file BEHIND the `__shutdown` already sitting in its port. It closed
+ // first, and that task's promise never settled -- and `onError`
+ // early-returns once `destroyed` is set, so it was not counted either. A
+ // silently lost file that `crashedTasks()` reported as zero, which is the
+ // number the stitch gate and the mtime baseline trust.
+ const path = worker(
+ "slowecho",
+ `
+ import { parentPort } from 'node:worker_threads';
+ parentPort.on('message', (msg) => {
+ if (msg && msg.__shutdown) { parentPort.close(); return; }
+ const end = Date.now() + 300; while (Date.now() < end);
+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
+ });
+ `,
+ );
+
+ // One worker: the first parse is dispatched, the second stays queued.
+ const pool = new ParsePool(path, 1, 200);
+ pool.init();
+ const dispatched = pool.parse("a.ts", "x");
+ const queued = pool.parse("b.ts", "x");
+ await new Promise(resolve => setTimeout(resolve, 50));
+
+ await pool.destroy();
+
+ await expect(dispatched).resolves.toEqual({ filePath: "a.ts" });
+ // Settles rather than hanging forever...
+ await expect(queued).resolves.toBeNull();
+ // ...and is NOT counted. Every gate that reads `crashedParses()` -- the
+ // mtime baseline, the pre-migration delete guard, both stitch gates -- runs
+ // inside `ingestFiles`'s `try`, strictly before the `finally` that calls
+ // `destroy()`. Counting here could only move the printed summary, making it
+ // disagree with the baseline decision already taken.
+ expect(pool.crashedTasks(), "too late for any gate to see it").toBe(0);
+ }, 20000);
+
+ it("waits for a worker that is mid-parse instead of terminating it", async () => {
+ // The rule the previous test cannot check, because its fixture wedges in
+ // JAVASCRIPT and `terminate()` kills that instantly. A real parse blocks
+ // inside the addon, where a V8 termination interrupt has no JS boundary to
+ // fire at, so `terminate()` waits for the call to return anyway -- measured
+ // at 3981ms against 4s of CPU-bound native work on Node 26. Terminating a
+ // busy worker therefore costs exactly as much as asking does and adds back
+ // the segfault, on a thread that was about to answer.
+ //
+ // `pbkdf2Sync` stands in for a long parse: same shape, no tree-sitter
+ // needed. The grace is 200ms and the work is ~1.3s, so the margin holds
+ // even on a machine several times faster than this one.
+ const marker = join(dir, "finished-its-parse");
+ const path = worker(
+ "busy",
+ `
+ import { parentPort } from 'node:worker_threads';
+ import { writeFileSync } from 'node:fs';
+ import { pbkdf2Sync } from 'node:crypto';
+ parentPort.on('message', (msg) => {
+ if (msg && msg.__shutdown) {
+ writeFileSync(${JSON.stringify(marker)}, 'graceful', 'utf8');
+ parentPort.close();
+ return;
+ }
+ pbkdf2Sync('p', 's', 4000000, 64, 'sha512');
+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
+ });
+ `,
+ );
+
+ // An explicit, huge ceiling. Without it this test rides the production 10s
+ // `SHUTDOWN_MAX_WAIT_MS`, and the ~1.5s KDF only has to be 6.5x slower --
+ // an instrumented coverage leg, a loaded macos-14 runner -- for the pool to
+ // give up first and the marker never to be written. It would then fail as
+ // "a busy worker must be asked", reading as a real regression rather than a
+ // timing miss.
+ const pool = new ParsePool(path, 1, 200, 120000);
+ pool.init();
+
+ // Do NOT await: destroy() has to run while the parse is still in flight.
+ const parsing = pool.parse("slow.ts", "x");
+ await new Promise((resolve) => setTimeout(resolve, 300));
+
+ const started = Date.now();
+ await pool.destroy();
+ const elapsed = Date.now() - started;
+
+ // It outlasted the grace rather than terminating at it...
+ expect(elapsed).toBeGreaterThan(250);
+ // ...the parse it was in the middle of still returned its result...
+ await expect(parsing).resolves.toEqual({ filePath: "slow.ts" });
+ // ...and the worker went by ANSWERING, which is the whole point: under the
+ // old rule it was terminated mid-native-call, which is the segfault.
+ expect(existsSync(marker), "a busy worker must be asked, never terminated").toBe(true);
+ expect(pool.crashedTasks(), "waiting for a parse is not a crash").toBe(0);
+ }, 20000);
+
+ it("shuts the REAL parse worker down through the same protocol", async () => {
+ // The one test that binds the two packages. `ix-cli` does not depend on
+ // `@ix/core-ingestion` as a package -- it loads the built worker by
+ // relative path -- so `__shutdown` is a bare literal in three unlinked
+ // places: here, `parse-pool.ts`, and `core-ingestion/src/parse-worker.ts`.
+ // Every other test in this file uses an inline fixture that hardcodes the
+ // literal itself, so renaming or dropping the handler in the real worker
+ // would leave the whole suite green while every `ix map` teardown quietly
+ // reverted to grace-then-terminate: the exact segfault this is all for.
+ const real = join(
+ dirname(fileURLToPath(import.meta.url)),
+ "../../../../core-ingestion/dist/parse-worker.js",
+ );
+ expect(existsSync(real), `build core-ingestion first: ${real}`).toBe(true);
+
+ // A deliberately huge grace, so this cannot pass or fail on how fast the
+ // machine is. If the real worker still understands `__shutdown` it exits in
+ // milliseconds; if it does not, teardown waits the full 10s and misses the
+ // bound by a factor of three, on any runner.
+ const pool = new ParsePool(real, 2, 10000);
+ pool.init();
+ // Parse for real, so the threads have the tree-sitter addon loaded -- an
+ // untouched worker does not, and it is the loaded-then-idle thread that
+ // crashes under `terminate()`.
+ const results = await Promise.all([
+ pool.parse("a.ts", "export function a(): number { return 1; }"),
+ pool.parse("b.ts", "export function b(): number { return 2; }"),
+ ]);
+ expect(
+ results.every((r) => r !== null),
+ "the real worker should parse these",
+ ).toBe(true);
+
+ const started = Date.now();
+ await pool.destroy();
+ const elapsed = Date.now() - started;
+
+ // It answered rather than being terminated.
+ expect(elapsed).toBeLessThan(3000);
+ expect(pool.crashedTasks(), "a clean teardown is not a crash").toBe(0);
+ }, 20000);
});
diff --git a/ix-cli/src/cli/commands/parse-pool.ts b/ix-cli/src/cli/commands/parse-pool.ts
index e25470c7..be080c3d 100644
--- a/ix-cli/src/cli/commands/parse-pool.ts
+++ b/ix-cli/src/cli/commands/parse-pool.ts
@@ -23,7 +23,21 @@ export class ParsePool {
private queue: Task[] = [];
private active = new Map<Worker, Task>();
- constructor(private workerPath: string, private concurrency: number) {}
+ /**
+ * @param graceMs How long an UNRESPONSIVE IDLE worker gets before the pool
+ * stops waiting for it.
+ * @param maxWaitMs The ceiling on waiting for a BUSY one.
+ *
+ * Both injectable only so the tests can pin the behaviour without spending
+ * the real periods -- the ceiling test would otherwise cost ten seconds on
+ * every leg of the matrix. `ingestFiles` passes neither.
+ */
+ constructor(
+ private workerPath: string,
+ private concurrency: number,
+ private graceMs: number = ParsePool.SHUTDOWN_GRACE_MS,
+ private maxWaitMs: number = ParsePool.SHUTDOWN_MAX_WAIT_MS,
+ ) {}
init(): void {
for (let i = 0; i < this.concurrency; i++) {
@@ -61,12 +75,12 @@ export class ParsePool {
}
async destroy(): Promise<void> {
- // Set FIRST. `terminate()` makes every worker emit 'exit', and the handler
- // for that treats an exit as a crash and spawns a replacement -- so
- // shutting the pool down span up a fresh set of threads, `this.workers = []`
- // orphaned them, and because a worker thread refs the event loop and the
- // CLI has no `process.exit(0)` on its success path, `ix map` printed its
- // summary and then hung forever.
+ // Set FIRST. A worker that answers `__shutdown` emits 'exit', and the
+ // handler for that treats
+ // an exit as a crash and spawns a replacement. So closing the pool spun up
+ // a fresh set of threads, `this.workers = []` orphaned them, and because a
+ // worker thread refs the event loop and the CLI has no `process.exit(0)` on
+ // its success path, `ix map` printed its summary and then hung forever.
this.destroyed = true;
// ...and `dead`, so a `parse()` after teardown resolves instead of queueing
// onto a pool with no workers left to run it. Not reachable from
@@ -74,11 +88,240 @@ export class ParsePool {
// is the identical hang this file has already produced three times, and one
// line closes it.
this.dead = true;
- await Promise.all(this.workers.map(w => w.terminate()));
+ // Resolve what is still QUEUED. `destroy()` left the queue untouched while
+ // `onResult` still calls `drain()`, so a worker finishing its last parse
+ // could be handed a queued file behind the `__shutdown` already sitting in
+ // its port: the worker closes first, and that task's promise never settled.
+ // Empty on every path where `ingestFiles` completes; reachable when it
+ // throws with parses still queued.
+ //
+ // Resolved but deliberately NOT counted as crashed. An earlier revision
+ // incremented `crashed` here and justified it as "the input the stitch gate
+ // and the mtime baseline trust" -- which is wrong: `destroy()` runs in the
+ // outermost `finally`, while the baseline, the pre-migration delete guard
+ // and both stitch gates read `crashedParses()` inside the `try`, strictly
+ // earlier. They cannot observe this. The only readers left are the summary
+ // fields, so counting it there made the printed `parseErrors` disagree with
+ // the baseline decision already taken, on the one path that reaches it --
+ // a run that is throwing anyway, where the exception is the story.
+ const stranded = this.queue.splice(0, this.queue.length);
+ for (const t of stranded) t.resolve(null);
+ await Promise.all(this.workers.map(w => this.shutdown(w)));
+ // ...and resolve whatever was still IN FLIGHT when the last worker went.
+ //
+ // `onError` early-returns once `destroyed` is set -- deliberately, so a
+ // deliberate teardown is not counted as a crash and does not respawn the
+ // pool it is closing -- which means the in-flight task of a worker that
+ // `shutdown` gave up on was never settled by anything. Its promise stayed
+ // pending for the life of the process, and the `Promise.all` over the parse
+ // batch with it. Found by a test asserting that such a worker's parse still
+ // settles: `destroy()` returned on schedule and the test then hung on the
+ // task.
+ //
+ // Not counted, for the same reason the stranded queue is not: every gate
+ // that reads `crashedParses()` has already run by the time `destroy()` is
+ // called from the outermost `finally`.
+ for (const task of this.active.values()) task.resolve(null);
+ this.active.clear();
this.workers = [];
this.idle = [];
}
+ /**
+ * How long an IDLE worker gets to answer before the pool gives up on it.
+ *
+ * Only ever paid by one whose JS event loop is wedged. A responsive worker
+ * closes its port and emits 'exit' in about a millisecond, and `shutdown`
+ * resolves on that event, so the common path does not wait. Public because
+ * the test that pins the fallback derives its bound from this rather than
+ * restating it: tuning the grace should not fail a test about
+ * waiting-then-returning.
+ *
+ * A worker that is mid-parse keeps being given more time instead, up to
+ * `SHUTDOWN_MAX_WAIT_MS`. Note this is checked on tick boundaries, so one
+ * that goes idle just after a tick can wait up to twice this.
+ */
+ static readonly SHUTDOWN_GRACE_MS = 2000;
+
+ /**
+ * The same, for a worker that has ALREADY faulted.
+ *
+ * Shorter than the ordinary grace, though what it buys is now modest and
+ * worth stating exactly: since nothing is terminated, the faulted thread
+ * survives regardless of this value. All it controls is how long the pool
+ * holds a timer and an 'exit' listener for a worker it has already replaced.
+ * An earlier revision justified it as keeping the pool from running
+ * `concurrency + 1` threads, which was true only while the fallback killed
+ * things. Do not tune it on that reasoning.
+ */
+ static readonly FAULTED_GRACE_MS = 250;
+
+ /**
+ * The ceiling on waiting for a BUSY worker, after which the pool lets go.
+ *
+ * Generous, because the common reason a worker is still busy at teardown is
+ * a genuine parse that will finish on its own and answer. Finite, because
+ * the alternative is an unbounded wait, and `destroy()` is the first
+ * statement of `ingestFiles`'s outermost `finally` -- a hang there stops the
+ * run before it can even print its summary. Only reached on the path where
+ * `ingestFiles` throws with parses still in flight; on every completing path
+ * the batch is awaited long before `destroy()`.
+ */
+ static readonly SHUTDOWN_MAX_WAIT_MS = 10000;
+
+ /**
+ * End one worker by ASKING, and let go of it if it will not answer.
+ *
+ * `terminate()` tears a thread down from outside, and doing that to a thread
+ * that has loaded the tree-sitter native bindings segfaults the process. The
+ * thread does not have to be busy: an idle worker that has parsed a single
+ * file is enough, because the crash is in disposing an isolate that still
+ * holds the addon. Measured on Windows/Node 26, twenty pool teardowns per
+ * process, six runs each:
+ *
+ * terminate() 5 of 6 runs died with SIGSEGV (139)
+ * worker closes its own port 0 of 6
+ * worker calls process.exit(0) 0 of 6
+ *
+ * `destroy()` runs in `ingestFiles`'s outermost `finally`, so this was one
+ * `ix map` in roughly twelve exiting 139 with every patch committed and the
+ * summary already printed -- invisible unless something reads the status.
+ *
+ * What the grace period bounds, precisely: the wait for a reply from a
+ * worker that is IDLE and does not answer -- one whose JS event loop is
+ * wedged. It does not bound teardown in general, and an earlier revision of
+ * this comment claimed it did.
+ *
+ * NOTHING here terminates a worker, and that is the point. Four review
+ * rounds went into rules for when killing a thread was safe -- never while
+ * busy, only after observed idleness, unless past a deadline -- and every
+ * rule had a counterexample, because `terminate()` is simply the wrong verb
+ * for teardown. It is what segfaults. Against a worker inside a native call
+ * it does not even preempt: V8's termination interrupt is only checked at JS
+ * boundaries, so it resolves when the call returns on its own (3981ms
+ * against ~4s of CPU-bound native work, measured), making it slower AND
+ * crash-prone there. Only a JS-wedged worker dies promptly, in about 2ms.
+ *
+ * Teardown does not need the thread to die. It needs the pool to stop
+ * waiting on it and the thread to stop holding the process open, which is
+ * exactly `unref()`. Measured on the real parse worker, four addon-loaded
+ * threads left live and unref'd across process exit: 0 failures in 10 runs,
+ * against 5 of 6 for `terminate()`.
+ *
+ * So the clocks below decide WHEN to give up, never whether it is safe to
+ * kill -- and the `onError` path needs no special case either, which is what
+ * removed the caveat this comment used to carry.
+ *
+ * The cost, stated rather than hidden: a genuinely wedged worker survives
+ * until the process exits, burning a core if it is spinning. For `ix map` or
+ * `ix ingest` that is nothing -- the process is about to end -- and `ix
+ * watch` is safe too, because it runs each map as a CHILD PROCESS
+ * (`watch.ts`), so every pool dies with its own process.
+ *
+ * The consumer that does hold the leak is the MCP server's in-process
+ * runner: `createInProcessRunner` in `src/mcp/runner.ts` is the default
+ * (`IX_MCP_SUBPROCESS=1` opts out), so repeated `ix_map` calls against a repo
+ * with a wedging grammar would accumulate unref'd, still-spinning threads for
+ * the life of the server. An earlier version of this comment named `ix watch`
+ * and missed that one, which is exactly backwards. It is still the right
+ * trade -- a leaked thread is recoverable, a SIGSEGV is not, and the crash
+ * would take the whole server down -- but that is where to look if threads
+ * ever accumulate.
+ */
+ private shutdown(w: Worker, graceMs = this.graceMs): Promise<void> {
+ // A worker that has already gone: settle NOW, synchronously.
+ //
+ // Checked rather than inferred from a failed post. An earlier revision
+ // relied on `postMessage()` throwing once the thread was gone; it does not
+ // -- on Node 26 it is a silent no-op -- so for every worker that had
+ // already exited (`process.exit()` inside it, the respawn loop, a crash)
+ // this held a live timer and a listener for an 'exit' that had already
+ // fired and would never fire again, then terminated a corpse. `threadId`
+ // is -1 once the thread is gone, which is the only signal here that is
+ // actually true.
+ if (w.threadId === -1) return Promise.resolve();
+ return new Promise<void>(resolve => {
+ let settled = false;
+ let timer: ReturnType<typeof setTimeout>;
+ const done = (): void => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(timer);
+ resolve();
+ };
+ // Two clocks, and every path ends at one of them.
+ //
+ // `idleSince` is when the worker was last OBSERVED idle, not the instant
+ // it left `active`. Checking `active` alone gave up on a worker that had
+ // only just posted its result: the main thread deletes the entry, and the
+ // worker reaches the queued `__shutdown` a moment later, so an expiry
+ // landing in that window read a busy-but-untracked worker as silent.
+ // Restarting the clock when it goes idle gives it a full `graceMs` of
+ // quiet before anything is concluded from the silence. Under the old
+ // `terminate()` this was a segfault; it now costs a parse result rather
+ // than the process, but abandoning a worker that was about to answer is
+ // still wrong.
+ //
+ // `hardDeadline` bounds the busy case, and exists because an earlier
+ // revision re-armed forever. `destroy()` is the first statement of
+ // `ingestFiles`'s outermost `finally`, so waiting on a worker that never
+ // finishes stopped the run before it could print its summary at all --
+ // strictly worse than the exit-139 it replaced. Past the deadline the
+ // pool lets go, which costs nothing now that letting go is `unref()`
+ // rather than a kill.
+ // The ceiling as given. `Math.max(graceMs, maxWaitMs)` meant a ceiling
+ // could never be shorter than the grace -- `new ParsePool(p, 1, 2000,
+ // 500)` silently got 2000ms, not 500ms. It is still only CHECKED on tick
+ // boundaries, so the effective ceiling rounds up to the next multiple of
+ // `graceMs`; that is why the tests bound loosely rather than exactly.
+ const hardDeadline = Date.now() + this.maxWaitMs;
+ let idleSince: number | null = this.active.has(w) ? null : Date.now();
+ const arm = (): void => {
+ timer = setTimeout(tick, graceMs);
+ // `unref` so a pool that is torn down early cannot hold the process
+ // open for a grace period after everything else has finished.
+ timer.unref?.();
+ };
+ const tick = (): void => {
+ const now = Date.now();
+ const busy = this.active.has(w);
+ if (busy) idleSince = null;
+ else if (idleSince === null) idleSince = now;
+
+ const quietLongEnough = idleSince !== null && now - idleSince >= graceMs;
+ if (!quietLongEnough && now < hardDeadline) {
+ arm();
+ return;
+ }
+ // Given up on: stop waiting for it, and stop it holding the process
+ // open. NOT terminated.
+ //
+ // `terminate()` was the wrong verb for this whole branch. It is what
+ // segfaults -- that is the bug this file exists to fix -- and against a
+ // worker inside a native call it does not even preempt: it resolves
+ // when the call returns (4457ms against ~4.5s of work, measured), so it
+ // was strictly slower AND crash-prone there. `unref()` gives the only
+ // thing teardown actually needs: the thread stops keeping the event
+ // loop alive, so the CLI exits, and nobody disposes an isolate that
+ // still holds the addon. Measured on the real parse worker, four
+ // addon-loaded threads left live and unref'd across process exit: 0
+ // failures in 10 runs, against 5 of 6 for `terminate()`.
+ //
+ // The cost is written up on `shutdown` above: a wedged worker survives
+ // until the process exits. Nothing for the CLI, and `ix watch` runs
+ // each map as a child process, but the MCP in-process runner can
+ // accumulate them.
+ w.unref();
+ done();
+ };
+ arm();
+ w.once('exit', done);
+ // No try/catch: the post cannot throw for a live thread, and the dead
+ // case is handled above. Wrapping it here is what disguised the bug.
+ w.postMessage({ __shutdown: true });
+ });
+ }
+
private spawnWorker(): Worker {
const w = new Worker(this.workerPath);
w.on('message', (msg: { ok: boolean; result: unknown }) => this.onResult(w, msg));
@@ -156,13 +399,13 @@ export class ParsePool {
private static readonly MAX_RESPAWNS = 16;
private onError(w: Worker, _err: Error): void {
- // Not during shutdown. `destroy()` terminates every worker on purpose, and
+ // Not during shutdown. `destroy()` ends every worker on purpose, and
// treating that as a crash respawns the pool it is trying to close: the
// replacements outlive `this.workers = []`, a worker thread refs the event
// loop, and the CLI prints its summary and then hangs forever. The guard
// lives HERE rather than on the 'exit' listener because 'error' reaches the
- // same code: a worker that faults while `destroy()` is awaiting
- // `terminate()` took the identical path.
+ // same code: a worker that faults while `destroy()` is waiting on it takes
+ // the identical path.
if (this.destroyed) return;
const task = this.active.get(w);
if (task) {
@@ -179,7 +422,18 @@ export class ParsePool {
// lost task is still counted for the stitch gate.
const idx = this.workers.indexOf(w);
if (idx === -1) return;
- w.terminate().catch(() => {});
+ // Asked rather than killed, for the same reason `destroy()` asks, and
+ // fire-and-forget as the `terminate()` here always was.
+ //
+ // Defensive, and measured to be so: a worker that emits 'error' is already
+ // being torn down by V8, 'exit' follows immediately, and it never processes
+ // a `__shutdown` posted from this handler -- a probe printed the 'error'
+ // event with `threadId=1`, then 'exit' with `threadId=-1`, and the worker's
+ // message handler never ran. So on both arms this settles via 'exit' or the
+ // `threadId` fast path, and the post is a no-op. Kept because it costs
+ // nothing and is correct if a worker ever does survive an 'error'; do not
+ // read it as evidence that a faulted thread can still answer.
+ this.shutdown(w, ParsePool.FAULTED_GRACE_MS).catch(() => {});
this.workers.splice(idx, 1);
// ...and out of `idle` too, ALWAYS, cap or no cap. A worker can emit 'error'
// while it is IDLE -- an uncaught async throw, or ERR_WORKER_OUT_OF_MEMORY