Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
// Copyright 2026 Ix Infrastructure Inc.
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { ParsePool } from "../commands/parse-pool.js";
/**
* These exist because this file has produced a HANG three separate ways, and
* every one of them was found by a reviewer or by timing a real `ix ingest`
* rather than by the suite:
*
* - a worker that emitted 'error' while idle was spliced out of `workers` but
* left in `idle`, so a later `drain()` posted to a dead thread;
* - an 'exit' listener added to close that hole treated `destroy()`'s own
* `terminate()` as a crash and respawned the pool it was closing, and a
* worker thread refs the event loop, so the CLI hung after printing its
* summary;
* - the respawn cap that bounded THAT could empty the pool, and only the
* queue existing at that instant was resolved -- every later `parse()`
* waited forever.
*
* Each test below is one of those. They use real worker threads, because every
* one of these bugs is in the interaction with the thread lifecycle and a fake
* would have reproduced none of them.
*/
describe("ParsePool", () => {
let dir: string;
beforeEach(() => {
dir = mkdtempSync(join(tmpdir(), "ix-parse-pool-"));
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
});
/** Write a worker and return its path. */
function worker(name: string, body: string): string {
const path = join(dir, `${name}.mjs`);
writeFileSync(path, body, "utf8");
return path;
}
/**
* Answers parses, and honours the shutdown request like the real worker.
*
* The `__shutdown` branch is not decoration. Without it this fixture ignores
* the request, so `destroy()` waits out the full grace and then gives up on
* the worker -- which meant the test named "shuts down without hanging" was
* silently exercising the FALLBACK rather than the clean path it claims, and
* paying 2s to do it.
*/
const ECHO = `
import { parentPort } from 'node:worker_threads';
parentPort.on('message', (msg) => {
if (msg && msg.__shutdown) { parentPort.close(); return; }
parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
});
`;
/** Exits on the first message, without ever emitting 'error'. */
const QUIET_DEATH = `
import { parentPort } from 'node:worker_threads';
parentPort.on('message', () => { process.exit(1); });
`;
/** Dies on construction, deterministically — the spawn-loop shape. */
const BORN_DEAD = `
process.exit(1);
`;
/**
* Faults ONCE per pool, not once per worker.
*
* `let served = 0` was per-thread, so the replacement armed its own fault
* shortly after serving `a2.ts` -- and `b2.ts` is only posted once the
* parent has received a2's result and re-drained. Miss that window on a
* busy machine and the replacement dies with `b2.ts` in flight, which
* `onError`
* resolves to null: the exact Ix#567 signature, from the harness rather
* than the pool. Fixing only the first wait would have left this half of
* the race in place.
*
* The marker is a file because the arming has to be visible to the NEXT
* thread, and these fixtures share nothing else.
*/
const IDLE_FAULT_ONCE = (marker: string): string => `
import { parentPort, threadId } from 'node:worker_threads';
import { writeFileSync } from 'node:fs';
const marker = ${JSON.stringify(marker)};
// Claim at MODULE SCOPE, once per thread, before any message is
// served. The previous revision claimed inside the message handler,
// which put a filesystem write on the path of every parse: the
// replacement re-attempted it while serving \`b2.ts\`, and a transient
// EPERM there faulted a worker with a task in flight -- producing
// \`b2.ts -> null\`, which is the Ix#567 signature this whole test exists
// to tell apart from a harness problem. The old fixture did no I/O at
// all, so that was exposure this PR introduced. Here the write happens
// once, before the pool can dispatch anything, and the handler touches
// no filesystem at all.
//
// \`wx\` because it is one atomic syscall: \`existsSync\` then write is two,
// and two threads can both pass the check. That cannot happen at this
// pool's concurrency of 1, but the guarantee is stated
// unconditionally, and this is what makes it true at any size.
let isFaulter = false;
try {
writeFileSync(marker, threadId + '\\n', { flag: 'wx' });
isFaulter = true;
} catch (err) {
// EEXIST is the expected answer: another thread holds the claim.
// Anything else means THIS thread failed to claim for an unrelated
// reason, and the two cases differ:
//
// the FIRST thread fails -- at concurrency 1 it is the only worker,
// so isFaulter stays false, nothing ever throws, nothing dies
// and nothing respawns. The run is NOT green; it fails in
// waitUntil, which reads this file to say so.
// a REPLACEMENT fails -- the fault already happened, so the test
// reaches its assertions, and the .failed check below is what
// catches it.
//
// Recorded rather than rethrown for both. A previous revision rethrew
// and claimed that surfaced the problem; it does not -- it only moves
// which thread faults. Best-effort: if this write fails too there is
// nothing left to say with.
if (err.code !== 'EEXIST') {
try {
writeFileSync(marker + '.failed', threadId + ' ' + err.code + '\\n', { flag: 'a' });
} catch {}
}
}
let armed = false;
parentPort.on('message', (msg) => {
if (msg && msg.__shutdown) { parentPort.close(); return; }
parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
// The delay exists so the parent has consumed this reply before the
// fault lands: 'message' and 'error' reach it on different channels,
// so a parent descheduled across both can process the 'error' first,
// find the task still in \`active\`, and resolve it null -- failing the
// first.ts assertion with the very signature this test is meant to
// distinguish. It was 20ms, the same order as the scheduling delays
// that caused the original flake; it is 250ms now.
//
// What this does NOT do is remove the ordering dependency, and it
// cannot: the worker has no way to learn that its reply was consumed,
// and a fault raised while a task IS in flight never leaves a stale
// entry in \`idle\`, which is the whole bug. So the premise needs an
// idle fault, an idle fault needs a delay, and this only makes the
// required parent stall implausible rather than merely unlikely.
if (isFaulter && !armed) {
armed = true;
setTimeout(() => { throw new Error('idle fault'); }, 250);
}
});
`;
/**
* Poll until `cond` holds. Deliberately not a fixed sleep: every wait in
* this file that is really "wait for the pool to observe something" should
* be bounded by the observation, so it costs a few ms when idle and still
* passes on a runner that is thrashing. The timeout only decides how long
* to wait before calling it a failure, so it can be generous.
*/
const waitUntil = async (
cond: () => boolean,
// A thunk, not just a string, so the message can be built when the wait
// FAILS. A caller waiting on something the fixture arranges needs to say
// "the fixture never armed" rather than "the pool never reacted", and it
// can only tell them apart at that moment.
what: string | (() => string),
timeoutMs = 10000,
): Promise<void> => {
const deadline = Date.now() + timeoutMs;
while (!cond()) {
if (Date.now() > deadline) {
throw new Error(`waitUntil timed out: ${typeof what === "string" ? what : what()}`);
}
await new Promise((resolve) => setTimeout(resolve, 5));
}
};
it("parses through the pool and shuts down without hanging", async () => {
const pool = new ParsePool(worker("echo", ECHO), 2);
pool.init();
const results = await Promise.all([pool.parse("a.ts", "x"), pool.parse("b.ts", "y")]);
expect(results).toEqual([{ filePath: "a.ts" }, { filePath: "b.ts" }]);
expect(pool.crashedTasks()).toBe(0);
// The hang: shutting a worker down makes it emit 'exit', and an exit
// handler that respawns hands back a pool of live threads that nothing
// owns. If that happens this test still passes -- the process is what hangs
// -- so `crashedTasks()` is asserted as the observable proxy: a deliberate
// teardown must not be recorded as a crash.
await pool.destroy();
expect(pool.crashedTasks(), "destroy() must not read as a crash").toBe(0);
});
it("resolves the in-flight task of a worker that dies without an error", async () => {
// `parseFile` catches its own exceptions, so a worker that goes quiet --
// `process.exit()`, or the thread killed under the runtime -- emits no
// 'error' at all. Before the 'exit' listener its task never settled and the
// `Promise.all` over the parse batch waited forever.
const pool = new ParsePool(worker("quiet", QUIET_DEATH), 1);
pool.init();
await expect(pool.parse("a.ts", "x")).resolves.toBeNull();
expect(pool.crashedTasks()).toBeGreaterThan(0);
await pool.destroy();
});
it("takes a worker that faults while IDLE out of the free list", async () => {
// The first of the three hangs. A worker can emit 'error' with no task in
// flight -- an uncaught async throw, ERR_WORKER_OUT_OF_MEMORY between
// tasks -- and it was spliced out of `workers` but left in `idle`, so the
// next `drain()` popped the terminated thread and posted to nothing: that
// task's promise never settled.
const marker = join(dir, "idle-fault-armed");
const pool = new ParsePool(worker("idlefault", IDLE_FAULT_ONCE(marker)), 1);
pool.init();
expect(await pool.parse("first.ts", "x")).toEqual({ filePath: "first.ts" });
// Wait for the fault to LAND, not for a duration. The fixture throws
// 250ms after serving, and this was `setTimeout(120)` -- ample when the
// machine is idle (0 failures in 10 runs) and is not when it is busy: 1 of
// 8 runs under saturating CPU load, where the 'error' event had not been
// delivered before the two parses below went out. The test then failed on
// `b2.ts` coming back null, which looks exactly like the bug it guards.
//
// `workerDeaths()` is the pool's own signal that `onError` has REACTED to
// a death -- it advances before `spawnWorker()` and `drain()`, so it means
// begun, not finished. That is enough here, and not because of the poll
// interval: `waitUntil` evaluates its condition once SYNCHRONOUSLY before
// any timer, so a 5ms tick guarantees nothing. What guarantees it is that
// this test body is async and so cannot be running inside `onError` --
// the handler has always returned before any line here executes. The
// distinction matters to anyone copying the `workerDeaths() > before`
// pattern into a synchronous callback, which the accessor's own doc
// warns against.
// `> 0` is only correct because this is the FIRST death of the run. The
// counter is monotonic and never resets, so a second wait written this
// way returns immediately -- a silent no-op, which is the same class of
// bug this test was fixed for. A later fault must capture a baseline
// first: `const before = pool.workerDeaths()` then
// `() => pool.workerDeaths() > before`.
await waitUntil(
() => pool.workerDeaths() > 0,
// Ask the fixture first. If its claim failed there is no fault to wait
// for, and reporting that the pool never reacted would blame the pool
// for a harness problem -- the misdiagnosis this whole test exists to
// prevent. At concurrency 1 that is the ONLY worker, so nothing
// respawns and this wait is where it surfaces; the `.failed` assertion
// below never gets to run.
() =>
existsSync(`${marker}.failed`)
? `the fixture could not claim the fault: ${readFileSync(`${marker}.failed`, "utf8").trim()}`
: "the idle fault never reached the pool",
);
// TWO at once, deliberately. `drain()` pops the free list, so a single
// parse takes the replacement worker that `onError` just pushed and never
// touches the dead entry underneath it -- the bug hides completely at
// depth 1. The second parse is the one that reaches the terminated thread,
// posts to nothing, and never settles.
const both = await Promise.all([pool.parse("a2.ts", "x"), pool.parse("b2.ts", "x")]);
expect(both).toEqual([{ filePath: "a2.ts" }, { filePath: "b2.ts" }]);
// The once-per-pool contract is ENFORCED by the fixture's `wx` flag, which
// makes a second arming impossible rather than detectable. This asserts
// that the enforcement is still there and did its job: exactly one thread
// armed, so `b2.ts` was never racing a replacement's timer.
//
// Be precise about what it can and cannot fail on, because two earlier
// revisions got this wrong in opposite directions. It fails if the `wx`
// claim is weakened to an append, and if the marker mechanism is replaced
// by per-thread counting (both verified by mutation, 3 of 3). It does NOT
// independently detect "a replacement armed too" -- with `wx` in place
// that cannot happen, so adding a redundant per-thread guard leaves this
// green, which is correct and not a gap.
//
// What it replaced was an assertion on `workerDeaths()`, which pinned
// nothing: the replacement's fault is a 250ms timer and the parses above
// take a few ms, so the count still reads 1 either way. Reverting the
// fixture passed that one 5 runs out of 5.
// Read through `existsSync`, so a marker that was never written fails as
// "arm exactly one fault" rather than as a raw ENOENT. `waitUntil` above
// only proves SOME respawn happened -- a fixture that died during module
// evaluation would satisfy it and never arm -- and the ENOENT would then
// point at this line instead of at the contract.
// No thread failed to claim for an unrelated reason. This catches a
// REPLACEMENT's claim failing -- it does not catch the first thread's,
// because at concurrency 1 that is the only worker, nothing respawns, and
// the `waitUntil` thunk above reports it before execution ever reaches
// here. The two checks cover different threads; neither covers both.
expect(
existsSync(`${marker}.failed`) ? readFileSync(`${marker}.failed`, "utf8") : "",
"a worker failed to claim the fault for a reason other than EEXIST",
).toBe("");
const armings = (existsSync(marker) ? readFileSync(marker, "utf8") : "")
.split("\n")
// `.filter(Boolean)`, not `.trim()`: an empty file trims to "" and then
// splits to [""], so ZERO armings would satisfy `toHaveLength(1)` and
// this check would be inert. Unreachable today only because the fault
// demonstrably landed above -- but the obvious future fix for the
// ENOENT path is to pre-create the marker, which would walk straight
// into it.
.filter(Boolean);
expect(armings, "the fixture must arm exactly one fault for the whole pool").toHaveLength(1);
// Kept, but for what it is: a cheap check that no EXTRA fault landed
// during the two parses. It is not the guard for the line above.
expect(pool.workerDeaths(), "no further worker died during the parses").toBe(1);
await pool.destroy();
});
it("does not queue forever once the respawn cap has emptied the pool", async () => {
// A worker that dies deterministically on construction burns through the
// cap. Draining only the queue that existed at that instant left every
// LATER parse waiting on a `drain()` that is a no-op with no idle workers.
// Named, because the death count below is derived from it as well as from
// the cap -- see there.
const concurrency = 1;
const pool = new ParsePool(worker("dead", BORN_DEAD), concurrency);
pool.init();
// Enough calls to outlast the cap, DERIVED from it. A hard-coded depth
// makes the derivation below one-directional: lowering `MAX_RESPAWNS`
// still works, but raising it past that depth means the queue drains
// before the pool reaches the latch, and the death assertion then fails
// with a message accusing the counter -- which is the exact failure the
// derivation exists to prevent. `MAX_RESPAWNS` is documented as tunable,
// so that is a reachable edit, not a hypothetical one.
const postLatch = 3; // the parses issued below, after the pool is dead
// Headroom, so the queue is still non-empty when `dead` latches. Unrelated
// to `postLatch` above; they are equal by coincidence, and unifying them
// would invent a coupling that does not exist.
//
// This is the constant that makes this test exercise the branch it is
// named for, and it is worth knowing that no other assertion here
// protects it: both of the ones below hold at `headroom = 0` too, because
// then every queued task is consumed by a death and none is stranded.
// Mutation-checked -- deleting the strand-and-resolve body from the cap
// branch in `parse-pool.ts` SURVIVES at 0 and is killed at 3, where the
// three stranded parses never settle and this hangs to the timeout. So
// trimming it silently turns a queue-stranding test into one that only
// covers in-flight losses.
const headroom = 3;
const queued = ParsePool.MAX_RESPAWNS + concurrency + headroom;
expect(
headroom,
"headroom is this test's coverage of the stranded-queue path, not slack",
).toBeGreaterThan(0);
const first = await Promise.all(
Array.from({ length: queued }, (_, i) => pool.parse(`f${i}.ts`, "x")),
);
expect(first.every((r) => r === null)).toBe(true);
// The regression: these arrive after the pool is dead.
await expect(pool.parse("later.ts", "x")).resolves.toBeNull();
await expect(
Promise.all([pool.parse("l1.ts", "x"), pool.parse("l2.ts", "x")]),
).resolves.toEqual([null, null]);
// And every one of them is counted, so the stitch gate sees the loss: the
// whole queued batch plus every parse issued after the latch. EXACT, not
// a floor -- this was the last loose assertion in a test tightened
// everywhere else, and a floor cannot see an over-count. A regression
// that counted the stranded queue twice (the `else if` branch's
// `crashed += stranded` plus `parse()`'s own `if (this.dead)`) raises the
// number and slides under a `>=`.
expect(pool.crashedTasks()).toBe(queued + postLatch);
// The death that hits the cap still counts. This is the only place that
// pins it: every other use of `workerDeaths()` watches the FIRST death of
// a healthy pool, where a counter incremented inside the respawn branch
// and one incremented outside it both read 1, so neither placement is
// distinguishable there. Here they are not equal -- `MAX_RESPAWNS` is 16,
// so the pool takes `MAX_RESPAWNS + concurrency` deaths: every worker it
// ever starts dies, and it starts `concurrency` up front plus one per
// respawn until the budget is gone. The last `concurrency` of them fall
// outside the branch -- one only when the pool is size 1, which is what
// it is here. Derived from BOTH constants, because at concurrency N the
// initial N-1 extra workers also die before `workers.length === 0` can
// latch `dead` -- so a bare `MAX_RESPAWNS + 1` silently means "and the
// pool is size 1", and raising the size here would fail this assertion
// with a message accusing the death counter.
//
// Asserted EXACTLY, and DERIVED from the constant rather than restated.
// A loose `>` form pins the placement only by accident of the cap's
// value: an increment moved back inside the branch yields exactly
// `MAX_RESPAWNS`, so `> MAX_RESPAWNS` happens to catch it, but any
// "more than roughly the cap" shape stops discriminating the moment
// someone reaches for a rounder number. And a hard-coded 17 fails a cap
// change with a message about the counter, where the tempting repair is
// to loosen the assertion into one that no longer catches the bug.
// `MAX_RESPAWNS` is public for exactly this, the way
// `SHUTDOWN_GRACE_MS` already is for the teardown-bound test below.
//
// Without this the increment can be tidied back into the branch with the
// suite green, and the `const before = ...` / `> before` wait that
// `workerDeaths()`'s own doc prescribes then hangs forever past the cap.
expect(
pool.workerDeaths(),
"expected MAX_RESPAWNS + concurrency deaths: every worker the pool " +
"starts dies. The shortfall is the deaths PAST the cap -- the ones " +
"that exhaust the budget are still counted either way, so a count of " +
"exactly MAX_RESPAWNS means the post-cap deaths stopped being counted",
).toBe(ParsePool.MAX_RESPAWNS + concurrency);
await pool.destroy();
});
it("asks a worker to end itself rather than terminating it", async () => {
// The fourth way this file has gone wrong, and the only one that was not a
// hang. `Worker.terminate()` tears a thread down from outside, and doing
// that to one that has loaded the tree-sitter native bindings segfaults the
// PROCESS -- an idle worker that parsed a single file is enough, because
// the crash is in disposing an isolate that still holds the addon. The
// comparison that settles it is on `ParsePool.shutdown` in
// `../commands/parse-pool.ts`; rates and populations are in
// `docs/parse-pool-teardown.md`, not restated here. What matters
// for this file: a pool built on one of the inline `.mjs` fixtures never
// imports `core-ingestion`, so its teardown cannot crash, and only a pool
// pointed at the real built worker can. Stated as a property of the
// fixture rather than as "only the test at the bottom", which is true
// today and goes stale silently the first time a second real-worker test
// is added -- and goes stale in the dangerous direction, telling whoever
// added it that their teardown cannot crash. Deliberately no tally
// either, for the same reason: a count goes stale with the suite green.
//
// And on the shipped build nothing here calls `terminate()` at all --
// Ix#598 removed it. A worker that will not
// answer `__shutdown` is left alive and `unref()`'d, and process exit
// does NOT run the shutdown path that `terminate()` drives. Probed three
// times, in a standalone script rather than through this pool: the parent
// never receives an 'exit' event for such a worker, and the worker's own
// `process.on('exit')` never runs -- the process leaves at code 0 with the
// thread still live.
//
// Standalone matters. The give-up path calls `removeAllListeners('exit')`
// before it unrefs, so measured through `ParsePool` the missing event
// would be guaranteed by construction and would show nothing. It is the
// worker-side handler that carries this.
//
// The give-up branch of `shutdown` in `../commands/parse-pool.ts` says the
// same thing -- "nobody disposes an isolate that still holds the addon" --
// though that is in the branch body, not in `shutdown`'s doc header, which
// only covers how long a wedged worker survives. An earlier revision of
// THIS comment said the opposite ("its isolate is still disposed when the
// process exits"), which would have told a reader the give-up path is
// exposed to the crash. It was the one claim in this PR that two files
// answered differently.
//
// The arm behind it is weak on its own -- four addon-loaded threads
// unref'd across exit, 0 failures in 10 runs, which at the per-isolate
// hazard is the expected result either way -- so the mechanism is what
// carries it, not the count.
//
// Nor is the repo clear: `core-ingestion`'s own suite runs on vitest's
// threads pool, which does terminate threads that have loaded the addon.
// The doc covers both.
//
// Asserted through a marker the worker writes when ASKED to go, because the
// crash itself is probabilistic: a test that just tore pools down would
// pass against the bug most of the time. This one pins the mechanism.
const marker = join(dir, "asked-to-go");
const path = worker(
"polite",
`
import { parentPort } from 'node:worker_threads';
import { writeFileSync } from 'node:fs';
parentPort.on('message', (msg) => {
if (msg && msg.__shutdown) {
writeFileSync(${JSON.stringify(marker)}, 'bye', 'utf8');
parentPort.close();
return;
}
parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
});
`,
);
const pool = new ParsePool(path, 2);
pool.init();
// Parse first so the worker is idle-after-work, which is the state
// `destroy()` meets in a real run. Note this fixture is an inline .mjs that
// never imports `core-ingestion`, so no addon is loaded here and the crash
// itself cannot occur -- what is pinned is the MECHANISM, that the pool
// asks rather than terminates. The real-worker test lower down is the one
// that runs against the addon.
await Promise.all([pool.parse("a.ts", "x"), pool.parse("b.ts", "x")]);
await pool.destroy();
expect(existsSync(marker), "destroy() must ask, not terminate").toBe(true);
});
it("returns, rather than waiting forever, when an IDLE worker ignores the request", async () => {
// The fallback, for the case it is actually for: a worker sitting in its
// event loop that simply will not answer. Waiting on it forever is the CLI
// hang the three tests above exist for, so the pool stops waiting and
// unrefs it. The busy case is a later test, and is deliberately different.
const path = worker(
"stubborn",
`
import { parentPort } from 'node:worker_threads';
parentPort.on('message', () => { /* answers nothing, leaves nothing */ });
setInterval(() => {}, 1000);
`,
);
const pool = new ParsePool(path, 1);
pool.init();
const started = Date.now();
await pool.destroy();
const elapsed = Date.now() - started;
// Derived from the constant, not restated as a magic number. Tuning the
// grace down is a legitimate change and must not fail a test about
// waiting-then-returning.
expect(elapsed).toBeGreaterThanOrEqual(ParsePool.SHUTDOWN_GRACE_MS * 0.75);
// ...and it did NOT wait forever. Without the timeout this never resolves
// and the failure is a suite timeout with no explanation attached.
expect(elapsed).toBeLessThan(ParsePool.SHUTDOWN_GRACE_MS * 4);
}, 15000);
it("still gives up on a worker that was busy when the grace expired, once it goes idle", async () => {
// The hang I introduced while fixing the previous finding. The busy branch
// `return`ed instead of re-arming, so a worker that happened to be busy at
// the ONE moment the timer fired permanently disarmed the fallback: it
// finished its parse, went idle, refused the shutdown, and nothing was left
// to terminate it. `destroy()` never resolved.
//
// Reproduced against the broken version at a 5s cutoff before writing this.
const path = worker(
"busy-then-stubborn",
`
import { parentPort } from 'node:worker_threads';
parentPort.on('message', (msg) => {
if (msg && msg.__shutdown) return; // refuses, once idle
const end = Date.now() + 400; while (Date.now() < end);
parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
});
setInterval(() => {}, 1000); // and stays alive
`,
);
const pool = new ParsePool(path, 1, 100);
pool.init();
const parsing = pool.parse("slow.ts", "x");
// Destroy while it is mid-parse, so the first expiry lands on a BUSY
// worker.
await new Promise(resolve => setTimeout(resolve, 50));
const started = Date.now();
await pool.destroy();
// It returned at all -- that is the regression. And it waited for the parse
// rather than killing it, which is the rule the previous test pins.
expect(Date.now() - started).toBeGreaterThan(300);
await expect(parsing).resolves.toEqual({ filePath: "slow.ts" });
}, 20000);
it("gives a worker that has just gone idle a full grace before giving up", async () => {
// Idleness has to be OBSERVED for a grace period, not read as an instant.
// The worker posts its result, the main thread deletes its `active` entry,
// and only then does the worker reach the queued `__shutdown` and start
// unwinding its isolate. An expiry landing in that window sees a worker
// that looks idle and unresponsive and abandons it -- one that was about to
// answer. Under the old `terminate()` this was the segfault itself; the
// pool unrefs now, so the cost is a lost parse result rather than the
// process, and it is still wrong.
//
// The timings are chosen so the two rules give different answers, which is
// the only way to catch this. Grace 300ms, so ticks land at 300/600/900. A
// 580ms parse puts the result 20ms before a tick, and the worker then
// spends 150ms in its shutdown handler:
//
// instantaneous idleness -> abandons at 600, mid-handler, no marker
// observed idleness -> first idle tick at 600 starts the clock, so
// giving up would be 900; the worker finishes
// at 730 and exits on its own, marker written
const marker = join(dir, "unwound-cleanly");
const path = worker(
"slow-to-unwind",
`
import { parentPort } from 'node:worker_threads';
import { writeFileSync } from 'node:fs';
const spin = (ms) => { const end = Date.now() + ms; while (Date.now() < end); };
parentPort.on('message', (msg) => {
if (msg && msg.__shutdown) {
spin(150); // stands in for isolate teardown
writeFileSync(${JSON.stringify(marker)}, 'clean', 'utf8');
parentPort.close();
return;
}
spin(580);
parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
});
`,
);
const pool = new ParsePool(path, 1, 300);
pool.init();
const parsing = pool.parse("slow.ts", "x");
await pool.destroy();
await expect(parsing).resolves.toEqual({ filePath: "slow.ts" });
expect(
existsSync(marker),
"a worker that just went idle must not be killed mid-teardown",
).toBe(true);
}, 20000);
it("gives up on a worker that stays busy past the hard deadline", async () => {
// The bound on the busy branch. Waiting while a worker is mid-parse is
// right -- it is about to answer -- but waiting FOREVER is a hang, and
// `destroy()` is the first statement of `ingestFiles`'s outermost
// `finally`, so it stops the run before it can print anything at all.
//
// An earlier revision did wait forever, reasoning that killing a busy
// worker "buys nothing". True, but the conclusion did not follow: the pool
// does not have to kill it to stop waiting on it. This fixture is a worker
// that never reaches its message loop, and the assertion is simply that
// `destroy()` still returns.
const path = worker(
"never-finishes",
`
import { parentPort } from 'node:worker_threads';
parentPort.on('message', () => {
// Busy well past the 500ms ceiling, then gone. Bounded on purpose, and
// kept short: the pool unrefs rather than terminates now, so this burns
// a core until it returns, overlapping the tests that follow.
const end = Date.now() + 2000; while (Date.now() < end);
process.exit(0);
});
`,
);
// A 500ms ceiling rather than the real 10s: what is being asserted is that
// a ceiling EXISTS and is honoured, which is precisely what the unbounded
// version lacked. Spending the production value here would add ten seconds
// to every leg of the matrix and prove nothing extra.
const pool = new ParsePool(path, 1, 100, 500);
pool.init();
const wedged = pool.parse("spin.ts", "x");
await new Promise(resolve => setTimeout(resolve, 50));
const started = Date.now();
await Promise.race([
pool.destroy(),
new Promise((_, reject) =>
setTimeout(() => reject(new Error("destroy() never returned")), 10000),
),
]);
// It returned by letting go once the 500ms ceiling passed, WELL before the
// 2s spin finishes -- and the margin is the whole gate. An earlier revision
// paired a 3000ms bound with this shortened fixture, which quietly made the
// test vacuous: with the deadline removed, `destroy()` simply waits out the
// spin and returns at ~2.05s, still under 3000. Measured pass time here is
// ~600ms, so this bound separates "let go at the ceiling" from "waited for
// the worker" instead of accepting both.
expect(Date.now() - started).toBeLessThan(1500);
// And the task that worker was holding still settles, so the `Promise.all`
// over the parse batch cannot hang either.
await expect(wedged).resolves.toBeNull();
}, 20000);
it("survives an abandoned worker that throws after the pool let go", async () => {
// Letting go means the thread is still ALIVE, so its listeners matter.
// Dropping them all -- which is the obvious way to stop an abandoned worker
// pinning the whole `ParsePool` object graph -- leaves a live Worker with
// no 'error' listener, and an unhandled 'error' event is a process-level
// crash. A wedged parse that eventually throws would take an `ix mcp`
// server down with it.
//
// So the give-up path drops the pool's listeners and attaches a no-op
// 'error' handler that closes over nothing. Measured both ways with a
// worker that throws 900ms after being abandoned: without the handler 3 of
// 3 runs died, with it 0 of 3.
//
// The assertion below is not what guards this: survival is. Measured, so
// that the next reader does not have to guess at the failure shape -- with
// the handler removed the run exits 1 with an "Unhandled Errors: late boom"
// section while still printing "Tests 13 passed". Reading the Tests line
// alone would call that green.
const path = worker(
"late-thrower",
`
import { parentPort } from 'node:worker_threads';
parentPort.on('message', () => { /* never answers */ });
setTimeout(() => { throw new Error('late boom from an abandoned worker'); }, 400);
`,
);
const pool = new ParsePool(path, 1, 100, 300);
pool.init();
await pool.destroy();
// Outlive the throw, which lands after the pool has already let go.
await new Promise(resolve => setTimeout(resolve, 700));
expect(pool.crashedTasks(), "nothing was dispatched to it").toBe(0);
}, 20000);
it("resolves parses still queued when the pool is destroyed", async () => {
// `destroy()` left `this.queue` untouched while `onResult` still called
// `drain()`, so a worker finishing its last parse could be handed a queued
// file BEHIND the `__shutdown` already sitting in its port. It closed
// first, and that task's promise never settled -- and `onError`
// early-returns once `destroyed` is set, so it was not counted either. A
// silently lost file that `crashedTasks()` reported as zero, which is the
// number the stitch gate and the mtime baseline trust.
const path = worker(
"slowecho",
`
import { parentPort } from 'node:worker_threads';
parentPort.on('message', (msg) => {
if (msg && msg.__shutdown) { parentPort.close(); return; }
const end = Date.now() + 300; while (Date.now() < end);
parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
});
`,
);
// One worker: the first parse is dispatched, the second stays queued.
const pool = new ParsePool(path, 1, 200);
pool.init();
const dispatched = pool.parse("a.ts", "x");
const queued = pool.parse("b.ts", "x");
await new Promise(resolve => setTimeout(resolve, 50));
await pool.destroy();
await expect(dispatched).resolves.toEqual({ filePath: "a.ts" });
// Settles rather than hanging forever...
await expect(queued).resolves.toBeNull();
// ...and is NOT counted. Every gate that reads `crashedParses()` -- the
// mtime baseline, the pre-migration delete guard, both stitch gates -- runs
// inside `ingestFiles`'s `try`, strictly before the `finally` that calls
// `destroy()`. Counting here could only move the printed summary, making it
// disagree with the baseline decision already taken.
expect(pool.crashedTasks(), "too late for any gate to see it").toBe(0);
}, 20000);
it("waits for a worker that is mid-parse instead of terminating it", async () => {
// The rule the previous test cannot check, because its fixture wedges in
// JAVASCRIPT and `terminate()` kills that instantly. A real parse blocks
// inside the addon, where a V8 termination interrupt has no JS boundary to
// fire at, so `terminate()` waits for the call to return anyway -- measured
// at 3981ms against 4s of CPU-bound native work on Node 26. Terminating a
// busy worker therefore costs exactly as much as asking does and adds back
// the segfault, on a thread that was about to answer.
//
// `pbkdf2Sync` stands in for a long parse: same shape, no tree-sitter
// needed. The grace is 200ms and the work is ~1.3s, so the margin holds
// even on a machine several times faster than this one.
const marker = join(dir, "finished-its-parse");
const path = worker(
"busy",
`
import { parentPort } from 'node:worker_threads';
import { writeFileSync } from 'node:fs';
import { pbkdf2Sync } from 'node:crypto';
parentPort.on('message', (msg) => {
if (msg && msg.__shutdown) {
writeFileSync(${JSON.stringify(marker)}, 'graceful', 'utf8');
parentPort.close();
return;
}
pbkdf2Sync('p', 's', 4000000, 64, 'sha512');
parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
});
`,
);
// An explicit, huge ceiling. Without it this test rides the production 10s
// `SHUTDOWN_MAX_WAIT_MS`, and the KDF -- ~1.5s alone, measured at 2.5s when
// an abandoned fixture from an earlier test is still spinning -- only has
// to be a few times slower again --
// an instrumented coverage leg, a loaded macos-14 runner -- for the pool to
// give up first and the marker never to be written. It would then fail as
// "a busy worker must be asked", reading as a real regression rather than a
// timing miss.
const pool = new ParsePool(path, 1, 200, 120000);
pool.init();
// Do NOT await: destroy() has to run while the parse is still in flight.
const parsing = pool.parse("slow.ts", "x");
await new Promise((resolve) => setTimeout(resolve, 300));
const started = Date.now();
await pool.destroy();
const elapsed = Date.now() - started;
// It outlasted the grace rather than terminating at it...
expect(elapsed).toBeGreaterThan(250);
// ...the parse it was in the middle of still returned its result...
await expect(parsing).resolves.toEqual({ filePath: "slow.ts" });
// ...and the worker went by ANSWERING, which is the whole point: under the
// old rule it was terminated mid-native-call, which is the segfault.
expect(existsSync(marker), "a busy worker must be asked, never terminated").toBe(true);
expect(pool.crashedTasks(), "waiting for a parse is not a crash").toBe(0);
}, 20000);
it("shuts the REAL parse worker down through the same protocol", async () => {
// The one test that binds the two packages. `ix-cli` does not depend on
// `@ix/core-ingestion` as a package -- it loads the built worker by
// relative path -- so `__shutdown` is a bare literal in three unlinked
// places: here, `parse-pool.ts`, and `core-ingestion/src/parse-worker.ts`.
// Every other test in this file uses an inline fixture that hardcodes the
// literal itself, so renaming or dropping the handler in the real worker
// would leave the whole suite green while every `ix map` teardown quietly
// reverted to grace-then-terminate: the exact segfault this is all for.
const real = join(
dirname(fileURLToPath(import.meta.url)),
"../../../../core-ingestion/dist/parse-worker.js",
);
expect(existsSync(real), `build core-ingestion first: ${real}`).toBe(true);
// A deliberately huge grace, so this cannot pass or fail on how fast the
// machine is. If the real worker still understands `__shutdown` it exits in
// milliseconds; if it does not, teardown waits the full 10s and misses the
// bound by a factor of three, on any runner.
const pool = new ParsePool(real, 2, 10000);
pool.init();
// Parse for real, because having PARSED is what was observed to arm the
// crash. Not because parsing loads the addon: `index.ts` resolves
// its grammars at module scope -- some by static import, the rest eagerly
// through helpers -- so these threads hold the core and the static ones
// within moments of spawn, well before that module finishes evaluating.
// The helper-loaded ones return null wherever a platform has no prebuild,
// so the guaranteed floor is the core plus the statically imported
// grammars, which is all this test needs. (Being a REQUIRED dependency
// does not put a grammar in that floor: `tree-sitter-powershell` is
// required and still loads through a helper.) The counts are in
// `docs/parse-pool-teardown.md` rather than here.
//
// Workers that had parsed were the ones observed to crash under
// `terminate()`, and spawn-then-destroy was not -- but the mechanism was
// never isolated, and the addon is held either way, so that does not
// establish an undispatched worker is safe to terminate. Parse here
// because the real run does.
const results = await Promise.all([
pool.parse("a.ts", "export function a(): number { return 1; }"),
pool.parse("b.ts", "export function b(): number { return 2; }"),
]);
expect(
results.every((r) => r !== null),
"the real worker should parse these",
).toBe(true);
const started = Date.now();
await pool.destroy();
const elapsed = Date.now() - started;
// It answered rather than being terminated.
expect(elapsed).toBeLessThan(3000);
expect(pool.crashedTasks(), "a clean teardown is not a crash").toBe(0);
}, 20000);
/**
* Blocks its event loop for 1.5 s on `slow.ts` -- standing in for a parse
* stuck in native code, which no budget check inside the worker can reach --
* then answers. Every other file answers at once.
*/
const SLOW_ON_ONE = `
import { parentPort } from 'node:worker_threads';
parentPort.on('message', (msg) => {
if (msg && msg.__shutdown) { parentPort.close(); return; }
if (msg.filePath === 'slow.ts') Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 6000);
parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
});
`;
it("settles a parse that holds its worker past the backstop as a timeout, and carries on", async () => {
// The backstop also runs for `a.ts` from the moment it is handed to the
// replacement worker, whose start-up counts against it. 1 s leaves room for
// that start on a busy runner; the slow parse holds its worker for 6 s.
const pool = new ParsePool(worker("slow", SLOW_ON_ONE), 1, undefined, undefined, 1000);
pool.init();
const start = Date.now();
const [slow, next] = await Promise.all([pool.parse("slow.ts", "x"), pool.parse("a.ts", "y")]);
expect(slow, "settled, not left pending").toBeNull();
expect(next, "a replacement worker took the queue").toEqual({ filePath: "a.ts" });
expect(Date.now() - start, "did not wait for the slow parse").toBeLessThan(5000);
expect(pool.timedOutFiles()).toEqual(["slow.ts"]);
expect(pool.crashedTasks(), "a timeout is not a crash: it recurs every run").toBe(0);
await pool.destroy();
});
/** Reports a budget timeout for `t.ts`, as the real worker does. */
const REPORTS_TIMEOUT = `
import { parentPort } from 'node:worker_threads';
parentPort.on('message', (msg) => {
if (msg && msg.__shutdown) { parentPort.close(); return; }
if (msg.filePath === 't.ts') {
parentPort.postMessage({ ok: false, result: null, reason: 'timeout', message: 'budget' });
} else {
parentPort.postMessage({ ok: false, result: null, reason: 'error', message: 'boom' });
}
});
`;
/** Echoes, and appends a line to `marker` when it starts: one line per worker. */
const COUNTS_STARTS = (marker: string) => `
import { parentPort, threadId } from 'node:worker_threads';
import { appendFileSync } from 'node:fs';
appendFileSync(${JSON.stringify(marker)}, threadId + '\\n');
parentPort.on('message', (msg) => {
if (msg && msg.__shutdown) { parentPort.close(); return; }
parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
});
`;
it("starts workers only as work arrives, up to its size", async () => {
// A one-file edit used to start a worker per core to parse one file.
const marker = join(dir, "starts");
const pool = new ParsePool(worker("counts", COUNTS_STARTS(marker)), 4);
const started = () => (existsSync(marker) ? readFileSync(marker, "utf8").trim().split("\n").length : 0);
await pool.parse("a.ts", "x");
expect(started(), "one task, one worker").toBe(1);
await Promise.all(["b", "c", "d", "e", "f", "g"].map((f) => pool.parse(`${f}.ts`, "x")));
expect(started(), "never more than the pool's size").toBeLessThanOrEqual(4);
await pool.destroy();
});
it("grows when a later phase asks for more, and never shrinks", async () => {
const marker = join(dir, "grows");
const pool = new ParsePool(worker("grows", COUNTS_STARTS(marker)), 1);
const started = () => readFileSync(marker, "utf8").trim().split("\n").length;
await Promise.all(["a", "b", "c"].map((f) => pool.parse(`${f}.ts`, "x")));
expect(started()).toBe(1);
pool.growTo(3);
pool.growTo(2);
await Promise.all(["d", "e", "f", "g"].map((f) => pool.parse(`${f}.ts`, "x")));
expect(started()).toBeGreaterThan(1);
expect(started()).toBeLessThanOrEqual(3);
await pool.destroy();
});
it("names a file the worker reports as timed out, and only one that counts", async () => {
const pool = new ParsePool(worker("reports", REPORTS_TIMEOUT), 1);
pool.init();
const results = await Promise.all([
pool.parse("t.ts", "x"),
pool.parse("e.ts", "x"),
// The index prescan does not count its losses; see `parse`.
pool.parse("t.ts", "x", false),
]);
expect(results).toEqual([null, null, null]);
expect(pool.timedOutFiles()).toEqual(["t.ts"]);
expect(pool.crashedTasks()).toBe(0);
await pool.destroy();
});
});