Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix(tests): drain cancelled lifecycle watchers before removing fixtures (#13314)

[._.]/ Adam Eivyfac26895759d
diff --git a/scripts/lib/vitestTempRootFixture.js b/scripts/lib/vitestTempRootFixture.js
index 4b9ebcaaa3..8fa6d2a2cb 100644
--- a/scripts/lib/vitestTempRootFixture.js
+++ b/scripts/lib/vitestTempRootFixture.js
@@ -207,9 +207,11 @@ export function runFixtureToCompletion({ args, options }, ceilingMs = FIXTURE_RU
* a restart ("Restarting due to config changes"), and hand the restarted run's
* observations to `inspect`. The fixture test records `{ root, owner }` into a
* numbered marker per run, so readiness is event-driven and bounded by
- * WATCH_READY_MS. The process tree is always killed before returning.
+ * WATCH_READY_MS. Caller cancellation stops readiness; returning waits for the
+ * killed watcher to close. FIXTURE_CLOSE_UNCONFIRMED means retain the host.
*/
-export async function runWatchConfigRestart(host, workspace, inspect) {
+export async function runWatchConfigRestart(host, workspace, inspect, { signal } = {}) {
+ if (signal?.aborted) throw fixtureAborted(signal);
const marks = join(host, 'marks');
mkdirSync(marks);
const { args, options } = createVitestTempFixture(host, workspace, `
@@ -237,10 +239,21 @@ export async function runWatchConfigRestart(host, workspace, inspect) {
detached: process.platform !== 'win32',
stdio: ['ignore', 'pipe', 'pipe'],
});
+ // Register before any signal: removal of the host must follow process AND
+ // pipe settlement, not merely the synchronous kill request.
+ const abort = new AbortController();
+ let closed = false;
+ let spawnError;
+ const completion = new Promise(resolve => {
+ child.once('close', () => { closed = true; resolve(); });
+ });
+ child.once('error', error => { spawnError = error; abort.abort(error); });
let output = '';
child.stdout.on('data', (chunk) => { output += chunk; });
child.stderr.on('data', (chunk) => { output += chunk; });
- const abort = new AbortController();
+ const onAbort = () => abort.abort(signal.reason);
+ signal?.addEventListener('abort', onAbort, { once: true });
+ if (signal?.aborted) onAbort();
const timer = setTimeout(() => abort.abort(new Error('watch fixture readiness timed out')), WATCH_READY_MS);
child.once('exit', () => abort.abort(new Error('watch fixture exited before readiness')));
const readMark = async (n) => {
@@ -248,6 +261,7 @@ export async function runWatchConfigRestart(host, workspace, inspect) {
await waitForFixtureReady(file, { signal: abort.signal });
return JSON.parse(readFileSync(file, 'utf8'));
};
+ let failure;
try {
const first = await readMark(1);
// run-1 is written from inside the first test, possibly before the watcher has
@@ -267,10 +281,28 @@ export async function runWatchConfigRestart(host, workspace, inspect) {
}
return await inspect({ first, restarted, pid: child.pid });
} catch (error) {
- error.message += `\n${output}`;
- throw error;
+ failure = spawnError ?? error;
+ failure.message += `\n${output}`;
+ throw failure;
} finally {
clearTimeout(timer);
- killProcessTree(child);
+ signal?.removeEventListener('abort', onAbort);
+ killProcessTree(child, { sweepGroup: true });
+ if (!closed) {
+ let closeTimer;
+ try {
+ await Promise.race([completion, new Promise((_, reject) => {
+ closeTimer = setTimeout(() => reject(new Error('watch fixture process close unconfirmed')), 2000);
+ })]);
+ } catch (error) {
+ const cleanupError = failure
+ ? new AggregateError([failure, error], 'watch fixture failed and process close is unconfirmed', { cause: failure })
+ : error;
+ cleanupError.code = 'FIXTURE_CLOSE_UNCONFIRMED';
+ throw cleanupError;
+ } finally {
+ clearTimeout(closeTimer);
+ }
+ }
}
}
diff --git a/scripts/lib/vitestWatchFixtureSettlement.test.js b/scripts/lib/vitestWatchFixtureSettlement.test.js
new file mode 100644
index 0000000000..833d4f34b2
--- /dev/null
+++ b/scripts/lib/vitestWatchFixtureSettlement.test.js
@@ -0,0 +1,81 @@
+import { EventEmitter } from 'node:events';
+import { PassThrough } from 'node:stream';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+vi.mock('node:child_process', () => ({ spawn: vi.fn(), spawnSync: vi.fn() }));
+import { spawn } from 'node:child_process';
+import { runWatchConfigRestart } from './vitestTempRootFixture.js';
+
+const tick = () => new Promise(resolve => setImmediate(resolve));
+
+describe('watch fixture process settlement', () => {
+ let host;
+ afterEach(() => {
+ vi.restoreAllMocks();
+ vi.useRealTimers();
+ if (host) rmSync(host, { recursive: true, force: true });
+ });
+
+ function fixture({ ready = true } = {}) {
+ host = mkdtempSync(join(tmpdir(), 'watch-settlement-'));
+ const child = new EventEmitter();
+ Object.assign(child, { pid: 12345, exitCode: null, stdout: new PassThrough(), stderr: new PassThrough(), kill: vi.fn() });
+ vi.spyOn(process, 'kill').mockReturnValue(true);
+ spawn.mockImplementation(() => {
+ if (ready) {
+ for (const n of [1, 2]) writeFileSync(join(host, 'marks', `run-${n}.json`), JSON.stringify({ root: 'synthetic', owner: '12345' }));
+ setImmediate(() => child.stdout.write('Waiting for file changes'));
+ }
+ return child;
+ });
+ return child;
+ }
+
+ it('does not release the host to its caller until the killed watcher closes', async () => {
+ const child = fixture();
+ let inspected;
+ const inspection = new Promise(resolve => { inspected = resolve; });
+ let settled = false;
+ const pending = runWatchConfigRestart(host, 'server', () => { inspected(); return 'inspected'; });
+ pending.then(() => { settled = true; }, () => { settled = true; });
+ await inspection;
+ await tick();
+ const releasedBeforeClose = settled;
+ child.emit('exit', null, 'SIGKILL');
+ await tick();
+ const releasedBeforePipesClosed = settled;
+ child.emit('close', null, 'SIGKILL');
+ await expect(pending).resolves.toBe('inspected');
+ expect(releasedBeforeClose).toBe(false);
+ expect(releasedBeforePipesClosed).toBe(false);
+ });
+
+ it('cancels readiness but still waits for process close before returning', async () => {
+ const child = fixture({ ready: false });
+ const controller = new AbortController();
+ const pending = runWatchConfigRestart(host, 'server', () => {}, { signal: controller.signal });
+ const result = pending.catch(error => error);
+ controller.abort();
+ await tick();
+ child.emit('exit', null, 'SIGKILL');
+ child.emit('close', null, 'SIGKILL');
+ await expect(result).resolves.toMatchObject({ name: 'AbortError' });
+ });
+
+ it('reports unconfirmed close as cleanup failure even after cancellation', async () => {
+ const child = fixture({ ready: false });
+ vi.useFakeTimers();
+ const controller = new AbortController();
+ const pending = runWatchConfigRestart(host, 'server', () => {}, { signal: controller.signal });
+ const result = pending.catch(error => error);
+ controller.abort();
+ await vi.advanceTimersByTimeAsync(2000);
+ child.emit('close', null, 'SIGKILL');
+ const error = await result;
+ expect(error).toMatchObject({ code: 'FIXTURE_CLOSE_UNCONFIRMED', cause: { name: 'AbortError' } });
+ expect(error.errors[1].message).toBe('watch fixture process close unconfirmed');
+ });
+});
diff --git a/scripts/vitestTempRoot.test.js b/scripts/vitestTempRoot.test.js
index 9b7aa4e9b9..58f37e674a 100644
--- a/scripts/vitestTempRoot.test.js
+++ b/scripts/vitestTempRoot.test.js
@@ -1,4 +1,4 @@
-import { describe, expect, it } from 'vitest';
+import { afterAll, describe, expect, it } from 'vitest';
import { spawn, spawnSync } from 'node:child_process';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, utimesSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
@@ -6,14 +6,20 @@ import { join, relative } from 'node:path';
import { fileURLToPath } from 'node:url';
import { createVitestTempFixture, runWatchConfigRestart } from './lib/vitestTempRootFixture.js';
import { STALE_ROOT_AGE_MS, killTestBrowsersUnder, sweepStaleRunRoots, writeOwnerFile } from './lib/vitestStaleRunRoots.js';
+import { ownTestBodies, recordTempRootOwner } from '../server/lib/mockPathsDataRoot.js';
import { isProcessAlive } from '../server/test/processAlive.js';
+// Vitest can cancel a wrapper before its async body reaches finally.
+const owned = ownTestBodies(it);
+afterAll(() => owned.drain());
+
describe('real workspace runner temp lifecycle', () => {
const workspace = 'server';
describe.each(['vitest.config.js', 'vitest.config.db.js'])('%s', (config) => {
for (const outcome of ['success', 'failure', 'leak']) {
it(`${workspace} removes framework scratch on ${outcome}, preserving host artifacts`, () => {
const host = mkdtempSync(join(tmpdir(), 'vrt-'));
+ recordTempRootOwner(host);
try {
// Even nanoid-looking pre-existing directories belong to the host.
const untouched = 'A'.repeat(21);
@@ -38,6 +44,7 @@ describe('real workspace runner temp lifecycle', () => {
it.each(['source-owned', 'outside-repo'])('retains composition provenance after a passing %s worker ends, without changing the global leak failure', mode => {
const repo = fileURLToPath(new URL('../', import.meta.url));
const host = mkdtempSync(mode === 'source-owned' ? join(repo, 'server/test/fixtures/provenance-worker-') : join(tmpdir(), 'vrt-'));
+ recordTempRootOwner(host);
try {
const { args, options } = createVitestTempFixture(host, 'server', `
const { channel } = await import('node:diagnostics_channel');
@@ -69,6 +76,7 @@ describe('real workspace runner temp lifecycle', () => {
it('reuses config ownership on reimport and reclaims it after interruption, preserving a live owner', async () => {
const host = mkdtempSync(join(tmpdir(), 'vrt-'));
+ recordTempRootOwner(host);
let child;
try {
const live = join(host, 'pvt-live');
@@ -110,8 +118,10 @@ describe('real workspace runner temp lifecycle', () => {
rmSync(host, { recursive: true, force: true });
}
}, 20000);
- it('server watch-config restart re-stamps the live owner so a later sweep keeps the root', async () => {
+ owned.it('server watch-config restart re-stamps the live owner so a later sweep keeps the root', async ({ signal }) => {
const host = mkdtempSync(join(tmpdir(), 'vrt-'));
+ recordTempRootOwner(host);
+ let cleanupConfirmed = true;
try {
await runWatchConfigRestart(host, 'server', ({ first, restarted, pid }) => {
// The restart reused the pathname and the main process re-stamped it.
@@ -124,9 +134,13 @@ describe('real workspace runner temp lifecycle', () => {
sweepStaleRunRoots(host);
expect(existsSync(first.root)).toBe(true);
rmSync(mkdtempSync(join(first.root, 'after-sweep-')), { recursive: true });
- });
+ }, { signal });
+ } catch (error) {
+ if (error.code === 'FIXTURE_CLOSE_UNCONFIRMED') cleanupConfirmed = false;
+ // Cancellation is expected, but failed process settlement is still fatal.
+ if (!signal.aborted || error.name !== 'AbortError') throw error;
} finally {
- rmSync(host, { recursive: true, force: true });
+ if (cleanupConfirmed) rmSync(host, { recursive: true, force: true });
}
}, 150000); // must exceed the fixture's readiness bound (WATCH_READY_MS)
});
@@ -151,6 +165,7 @@ describe.skipIf(process.platform === 'win32')('orphaned test browser sweep', ()
it('stops a browser left by a dead run and keeps one a live run still owns', async () => {
const host = mkdtempSync(join(tmpdir(), 'vrt-'));
+ recordTempRootOwner(host);
const pids = [];
try {
const dead = join(host, 'pvt-dead');
@@ -182,6 +197,7 @@ describe.skipIf(process.platform === 'win32')('orphaned test browser sweep', ()
it('stops a browser whose run root was already removed', async () => {
const host = mkdtempSync(join(tmpdir(), 'vrt-'));
+ recordTempRootOwner(host);
let pid;
try {
({ pid } = orphanBrowser(join(host, 'pvt-removed', 'profile')));
diff --git a/scripts/vitestTempRootCancellation.test.js b/scripts/vitestTempRootCancellation.test.js
new file mode 100644
index 0000000000..3dd2e19fd4
--- /dev/null
+++ b/scripts/vitestTempRootCancellation.test.js
@@ -0,0 +1,97 @@
+import { afterAll, expect, it } from 'vitest';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { ownTestBodies } from '../server/lib/mockPathsDataRoot.js';
+import { createVitestTempFixture, fixtureReadyFailureSource, runFixtureToCompletion } from './lib/vitestTempRootFixture.js';
+import { writeOwnerFile } from './lib/vitestStaleRunRoots.js';
+
+const sourcePath = fileURLToPath(new URL('./vitestTempRoot.test.js', import.meta.url));
+const lifecycleSource = readFileSync(sourcePath, 'utf8')
+ .replace(/(['"])(\.\.?\/[^'"]+)\1/g, (_, _quote, specifier) => JSON.stringify(resolve(dirname(sourcePath), specifier)));
+
+const owned = ownTestBodies(it);
+afterAll(() => owned.drain());
+
+// Exercise the actual watch test's ownership/cleanup with a different worker
+// triggering bail while its nested watcher is active. The injected inspection
+// gate waits for cancellation, so a fast restart cannot finish the scenario early.
+owned.it('drains the active lifecycle watcher after a sibling fails, preserving an external owner', async ({ signal }) => {
+ const host = mkdtempSync(join(tmpdir(), 'watch-cancel-'));
+ try {
+ const run = createVitestTempFixture(host, 'server', '');
+ const fixture = join(host, 'fixture');
+ const ready = join(fixture, 'ready');
+ const finished = join(fixture, 'finished');
+ const live = join(host, 'pvt-live');
+ mkdirSync(live);
+ writeOwnerFile(live);
+ writeFileSync(join(live, 'keep'), 'synthetic external owner');
+ const source = lifecycleSource
+ .replace("runWatchConfigRestart(host, 'server', ({ first, restarted, pid }) => {", `runWatchConfigRestart(host, 'server', async ({ first, restarted, pid }) => {
+ const cancelled = new Promise(resolve => signal.addEventListener('abort', () => setImmediate(resolve), { once: true }));
+ writeFileSync(${JSON.stringify(ready)}, 'watch-inspection-active');
+ await cancelled;`)
+ .replace(" }, 150000); // must exceed", ` writeFileSync(${JSON.stringify(finished)}, 'watch-closed-and-host-removed');
+ }, 150000); // must exceed`);
+ expect(source).toContain('await cancelled;');
+ expect(source).toContain('watch-closed-and-host-removed');
+ writeFileSync(join(fixture, 'lifecycle.test.js'), source);
+ writeFileSync(join(fixture, 'failure.test.js'), fixtureReadyFailureSource(ready));
+ run.args[run.args.indexOf('--maxWorkers') + 1] = '2';
+ run.args.push('--bail=1', '--testNamePattern', 'server watch-config restart|controlled sibling failure');
+ const result = await runFixtureToCompletion(run, 120000, { signal });
+ if (signal.aborted) return;
+ // Deliberately bounded diagnostics: no nested output, paths, PIDs or env.
+ const state = JSON.stringify({ ready: existsSync(ready), finished: existsSync(finished), status: result.status, error: result.error?.name ?? null });
+ expect(result.error, state).toBeUndefined();
+ expect(result.status, state).toBe(1);
+ expect(result.stderr, state).toContain('controlled sibling failure');
+ expect(existsSync(finished), state).toBe(true);
+ expect(result.stderr, state).not.toMatch(/test temp leak:|test process leak:|Unhandled|ENOTEMPTY|ENOENT/);
+ expect(readFileSync(join(live, 'keep'), 'utf8')).toBe('synthetic external owner');
+ } finally {
+ rmSync(host, { recursive: true, force: true });
+ }
+}, 135000);
+
+owned.it('retains the lifecycle host and fails when watcher settlement is unconfirmed', async ({ signal }) => {
+ const host = mkdtempSync(join(tmpdir(), 'watch-unconfirmed-'));
+ try {
+ const run = createVitestTempFixture(host, 'server', '');
+ const fixture = join(host, 'fixture');
+ const allocated = join(fixture, 'allocated');
+ const retained = join(fixture, 'retained');
+ const helperPath = fileURLToPath(new URL('./lib/vitestTempRootFixture.js', import.meta.url));
+ writeFileSync(join(fixture, 'lifecycle.test.js'), `
+import { vi } from 'vitest';
+vi.mock(${JSON.stringify(helperPath)}, async importOriginal => ({
+ ...await importOriginal(),
+ runWatchConfigRestart: async host => {
+ writeFileSync(${JSON.stringify(allocated)}, host);
+ writeFileSync(join(host, 'keep'), 'synthetic original bytes');
+ throw Object.assign(new Error('synthetic close unconfirmed'), { code: 'FIXTURE_CLOSE_UNCONFIRMED' });
+ },
+}));
+${lifecycleSource}
+afterAll(() => {
+ const allocatedHost = readFileSync(${JSON.stringify(allocated)}, 'utf8');
+ const present = existsSync(join(allocatedHost, 'keep'));
+ const original = present && readFileSync(join(allocatedHost, 'keep'), 'utf8') === 'synthetic original bytes';
+ writeFileSync(${JSON.stringify(retained)}, JSON.stringify({ present, original }));
+});
+`);
+ run.args.push('--testNamePattern', 'server watch-config restart');
+ const result = await runFixtureToCompletion(run, 120000, { signal });
+ if (signal.aborted) return;
+ const state = JSON.stringify({ status: result.status, retained: existsSync(retained), error: result.error?.name ?? null });
+ expect(result.error, state).toBeUndefined();
+ expect(result.status, state).toBe(1);
+ expect(result.stderr, state).toContain('synthetic close unconfirmed');
+ expect(JSON.parse(readFileSync(retained, 'utf8'))).toEqual({ present: true, original: true });
+ expect(result.stderr, state).toContain('test temp leak: vrt-');
+ } finally {
+ rmSync(host, { recursive: true, force: true });
+ }
+}, 135000);