Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix(ingest): drop the abandoned worker's listeners too, keeping a no-op error handler (#607)
Joseph Mikhail5f352b699bc4
diff --git a/ix-cli/src/cli/__tests__/parse-pool.test.ts b/ix-cli/src/cli/__tests__/parse-pool.test.tsindex be88eb02..82494974 100644--- a/ix-cli/src/cli/__tests__/parse-pool.test.ts+++ b/ix-cli/src/cli/__tests__/parse-pool.test.ts@@ -342,10 +342,10 @@ describe("ParsePool", () => {`import { parentPort } from 'node:worker_threads';parentPort.on('message', () => {- // Busy well past the ceiling, then gone. Bounded on purpose: the pool- // unrefs rather than terminates now, so an endless loop here would burn- // a core for the rest of the file.- const end = Date.now() + 5000; while (Date.now() < end);+ // Busy well past the 500ms ceiling, then gone. Bounded on purpose, and+ // kept short: the pool unrefs rather than terminates now, so this burns+ // a core until it returns, overlapping the tests that follow.+ const end = Date.now() + 2000; while (Date.now() < end);process.exit(0);});`,@@ -368,14 +368,55 @@ describe("ParsePool", () => {),]);- // It returned, by letting go of the worker once the deadline passed --- // long before that 5s spin finishes, which is the point.- expect(Date.now() - started).toBeLessThan(3000);+ // It returned by letting go once the 500ms ceiling passed, WELL before the+ // 2s spin finishes -- and the margin is the whole gate. An earlier revision+ // paired a 3000ms bound with this shortened fixture, which quietly made the+ // test vacuous: with the deadline removed, `destroy()` simply waits out the+ // spin and returns at ~2.05s, still under 3000. Measured pass time here is+ // ~600ms, so this bound separates "let go at the ceiling" from "waited for+ // the worker" instead of accepting both.+ expect(Date.now() - started).toBeLessThan(1500);// And the task that worker was holding still settles, so the `Promise.all`// over the parse batch cannot hang either.await expect(wedged).resolves.toBeNull();}, 20000);+ it("survives an abandoned worker that throws after the pool let go", async () => {+ // Letting go means the thread is still ALIVE, so its listeners matter.+ // Dropping them all -- which is the obvious way to stop an abandoned worker+ // pinning the whole `ParsePool` object graph -- leaves a live Worker with+ // no 'error' listener, and an unhandled 'error' event is a process-level+ // crash. A wedged parse that eventually throws would take an `ix mcp`+ // server down with it.+ //+ // So the give-up path drops the pool's listeners and attaches a no-op+ // 'error' handler that closes over nothing. Measured both ways with a+ // worker that throws 900ms after being abandoned: without the handler 3 of+ // 3 runs died, with it 0 of 3.+ //+ // The assertion below is not what guards this: survival is. Measured, so+ // that the next reader does not have to guess at the failure shape -- with+ // the handler removed the run exits 1 with an "Unhandled Errors: late boom"+ // section while still printing "Tests 13 passed". Reading the Tests line+ // alone would call that green.+ const path = worker(+ "late-thrower",+ `+ import { parentPort } from 'node:worker_threads';+ parentPort.on('message', () => { /* never answers */ });+ setTimeout(() => { throw new Error('late boom from an abandoned worker'); }, 400);+ `,+ );++ const pool = new ParsePool(path, 1, 100, 300);+ pool.init();+ await pool.destroy();++ // Outlive the throw, which lands after the pool has already let go.+ await new Promise(resolve => setTimeout(resolve, 700));+ expect(pool.crashedTasks(), "nothing was dispatched to it").toBe(0);+ }, 20000);+it("resolves parses still queued when the pool is destroyed", async () => {// `destroy()` left `this.queue` untouched while `onResult` still called// `drain()`, so a worker finishing its last parse could be handed a queued@@ -448,7 +489,9 @@ describe("ParsePool", () => {);// An explicit, huge ceiling. Without it this test rides the production 10s- // `SHUTDOWN_MAX_WAIT_MS`, and the ~1.5s KDF only has to be 6.5x slower --+ // `SHUTDOWN_MAX_WAIT_MS`, and the KDF -- ~1.5s alone, measured at 2.5s when+ // an abandoned fixture from an earlier test is still spinning -- only has+ // to be a few times slower again --// an instrumented coverage leg, a loaded macos-14 runner -- for the pool to// give up first and the marker never to be written. It would then fail as// "a busy worker must be asked", reading as a real regression rather than adiff --git a/ix-cli/src/cli/commands/parse-pool.ts b/ix-cli/src/cli/commands/parse-pool.tsindex be080c3d..f7714cfb 100644--- a/ix-cli/src/cli/commands/parse-pool.ts+++ b/ix-cli/src/cli/commands/parse-pool.ts@@ -9,6 +9,15 @@*/import { Worker } from 'node:worker_threads';+/**+ * The 'error' listener left on a worker the pool has let go of.+ *+ * Module level on purpose. See `shutdown`: a handler defined inside that+ * method would capture its scope and keep the whole pool alive, defeating the+ * listener removal it accompanies.+ */+const SWALLOW_ERROR = (): void => {};+type Task = {filePath: string;source: string;@@ -76,11 +85,11 @@ export class ParsePool {async destroy(): Promise<void> {// Set FIRST. A worker that answers `__shutdown` emits 'exit', and the- // handler for that treats- // an exit as a crash and spawns a replacement. So closing the pool spun up- // a fresh set of threads, `this.workers = []` orphaned them, and because a- // worker thread refs the event loop and the CLI has no `process.exit(0)` on- // its success path, `ix map` printed its summary and then hung forever.+ // handler for that treats an exit as a crash and spawns a replacement. So+ // closing the pool spun up a fresh set of threads, `this.workers = []`+ // orphaned them, and because a worker thread refs the event loop and the+ // CLI has no `process.exit(0)` on its success path, `ix map` printed its+ // summary and then hung forever.this.destroyed = true;// ...and `dead`, so a `parse()` after teardown resolves instead of queueing// onto a pool with no workers left to run it. Not reachable from@@ -311,6 +320,31 @@ export class ParsePool {// until the process exits. Nothing for the CLI, and `ix watch` runs// each map as a child process, but the MCP in-process runner can// accumulate them.+ //+ // So let go of the OBJECT GRAPH too, not just the thread. Every+ // listener `spawnWorker` attached is a closure over `this`, and+ // `destroy()` only drops the pool's reference to the worker -- the+ // worker still referenced the pool, so one abandoned thread pinned its+ // whole `ParsePool`, its `Worker[]`, its `active` map and every+ // remaining `Task` closure for the life of the process. In a long-lived+ // `ix mcp` that is the bigger of the two leaks.+ w.removeAllListeners('message');+ w.removeAllListeners('exit');+ w.removeAllListeners('error');+ // ...but never leave a live Worker with NO 'error' listener. An+ // unhandled 'error' event is a process-level crash, and this thread is+ // still running -- a wedged parse that eventually throws would take the+ // MCP server down.+ //+ // `SWALLOW_ERROR` rather than an inline `() => {}`, and that is the+ // whole point rather than a style preference. A function literal+ // written HERE closes over this scope -- `this`, `w`, the task+ // closures -- whether or not it names any of them, so attaching one+ // would have re-pinned the exact object graph the three lines above+ // just released, and the removals would have bought nothing.+ // Confirmed with `--expose-gc` and a `WeakRef` on the pool: inline+ // handler, POOL STILL RETAINED; hoisted, POOL COLLECTED.+ w.on('error', SWALLOW_ERROR);w.unref();done();};