Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): stop teardown rmSync failing green runs (PRD-8274, PRD-8276) (#3836)

Andrew Mikofalvy4e0637f7f125
diff --git a/packages/app/tests/integration/e2e-stop-rules.test.ts b/packages/app/tests/integration/e2e-stop-rules.test.ts
index 32533501d..c3efbb9dd 100644
--- a/packages/app/tests/integration/e2e-stop-rules.test.ts
+++ b/packages/app/tests/integration/e2e-stop-rules.test.ts
@@ -442,7 +442,7 @@ describe('E2E STOP rule — zero allowlist', () => {
}
if (violations.length > 0) {
throw new Error(
- `dev-server spawn without shared-state isolation — pass OK_TEST_VITE_CACHE_DIR (via prepareViteCacheDir from ./_helpers, rmSync in teardown) and OK_TEST_SKIP_I18N_COMPILE: '1' in the spawn env:\n${violations.join('\n')}`,
+ `dev-server spawn without shared-state isolation — pass OK_TEST_VITE_CACHE_DIR (via prepareViteCacheDir from ./_helpers, removeAllDuringTeardown in teardown) and OK_TEST_SKIP_I18N_COMPILE: '1' in the spawn env:\n${violations.join('\n')}`,
);
}
});
diff --git a/packages/app/tests/integration/teardown-fs-errno.test.ts b/packages/app/tests/integration/teardown-fs-errno.test.ts
new file mode 100644
index 000000000..99850556a
--- /dev/null
+++ b/packages/app/tests/integration/teardown-fs-errno.test.ts
@@ -0,0 +1,167 @@
+/**
+ * Errno policy for the e2e teardown filesystem-removal path.
+ *
+ * The sibling `kill-gracefully-errno.test.ts` pins which SIGNAL errnos
+ * teardown absorbs. This file pins the same question one layer over, for the
+ * removals that reclaim a just-killed dev server's contentDir, Vite cacheDir
+ * and log file: which removal errnos teardown absorbs, which it still lets
+ * out, and that one failing target cannot stop the others being reclaimed.
+ *
+ * The motivating failure is the shape the signal policy was written for, one
+ * layer over — a Playwright run where every test passed and the run still
+ * reported failure, because a teardown throw escaped after the last test had
+ * finished. Playwright counts that as "N errors were not a part of any test":
+ * a red run with a completely green report, and in the merge queue an ejection
+ * that rebuilds everything behind it.
+ *
+ * `rmSync`'s `force` option only suppresses ENOENT; EBUSY, ENOTEMPTY and EPERM
+ * still throw, and its `maxRetries` defaults to 0. Those three are what a Vite
+ * optimizer or file-watcher still holding a handle produces while teardown
+ * deletes the directory under it — so they are injected here at the `rmSync`
+ * boundary rather than raced for against a real watcher.
+ *
+ * The mock factory spreads the real `node:fs`, so every other call in this file
+ * (`mkdtempSync`, `existsSync`) is the genuine syscall and `rmSync` itself
+ * falls back to the real one whenever a test does not override it.
+ */
+
+import { existsSync, mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
+import { removeAllDuringTeardown } from '../stress/_helpers/teardown-fs.ts';
+
+vi.mock('node:fs', async (importOriginal) => {
+ const actual = await importOriginal<typeof import('node:fs')>();
+ return { ...actual, rmSync: vi.fn(actual.rmSync) };
+});
+
+const mockedRm = vi.mocked(rmSync);
+
+function errnoError(code: string): NodeJS.ErrnoException {
+ return Object.assign(new Error(`rm ${code}`), { code });
+}
+
+/** Fail only on `failing`; every other target removes cleanly. */
+function rmFailingOn(failing: string, err: NodeJS.ErrnoException) {
+ mockedRm.mockImplementation((target) => {
+ if (target === failing) throw err;
+ });
+ return mockedRm;
+}
+
+let warn: ReturnType<typeof vi.spyOn<Console, 'warn'>>;
+
+beforeEach(() => {
+ warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+});
+
+afterEach(() => {
+ // `mockReset` restores the implementation `vi.fn(actual.rmSync)` was built
+ // with, so the next test starts from the real syscall again.
+ mockedRm.mockReset();
+ vi.restoreAllMocks();
+});
+
+describe('removeAllDuringTeardown removal policy', () => {
+ test('removes every target, recursively and forcefully', () => {
+ mockedRm.mockImplementation(() => {});
+ removeAllDuringTeardown('/tmp/a', '/tmp/b', '/tmp/c');
+ expect(mockedRm).toHaveBeenCalledTimes(3);
+ for (const target of ['/tmp/a', '/tmp/b', '/tmp/c']) {
+ expect(mockedRm).toHaveBeenCalledWith(
+ target,
+ expect.objectContaining({ recursive: true, force: true }),
+ );
+ }
+ });
+
+ test('retries transient removals instead of giving up on the first refusal', () => {
+ // `maxRetries` engages Node's own linear backoff over the errnos it treats
+ // as transient, so a watcher that closes its handle a moment later is
+ // actually reclaimed rather than tolerated and leaked.
+ mockedRm.mockImplementation(() => {});
+ removeAllDuringTeardown('/tmp/a');
+ const options = mockedRm.mock.calls[0]?.[1] as { maxRetries?: number } | undefined;
+ expect(options?.maxRetries).toBeGreaterThan(0);
+ });
+
+ test.each([
+ 'EBUSY',
+ 'ENOTEMPTY',
+ 'EPERM',
+ ])('tolerates %s rather than failing a run whose tests have all passed', (code) => {
+ rmFailingOn('/tmp/busy', errnoError(code));
+ expect(() => removeAllDuringTeardown('/tmp/busy')).not.toThrow();
+ });
+
+ test.each([
+ 'EBUSY',
+ 'ENOTEMPTY',
+ 'EPERM',
+ ])('still reclaims the remaining targets after a tolerated %s', (code) => {
+ rmFailingOn('/tmp/busy', errnoError(code));
+ removeAllDuringTeardown('/tmp/busy', '/tmp/second', '/tmp/third');
+ // The bug: a throw on the first target skipped the two that followed it,
+ // leaking a contentDir and a Vite cacheDir per teardown.
+ expect(mockedRm).toHaveBeenCalledWith('/tmp/second', expect.anything());
+ expect(mockedRm).toHaveBeenCalledWith('/tmp/third', expect.anything());
+ });
+
+ test('warns on a tolerated failure so a genuinely leaked directory leaves a trace', () => {
+ rmFailingOn('/tmp/busy', errnoError('EBUSY'));
+ removeAllDuringTeardown('/tmp/busy');
+ expect(warn).toHaveBeenCalledTimes(1);
+ expect(String(warn.mock.calls[0]?.[0])).toContain('EBUSY');
+ expect(String(warn.mock.calls[0]?.[0])).toContain('/tmp/busy');
+ });
+
+ test.each([
+ 'EMFILE',
+ 'ENFILE',
+ 'EACCES',
+ 'ENOTDIR',
+ 'EINVAL',
+ undefined,
+ ])('rethrows %s rather than hiding it', (code) => {
+ const err = code === undefined ? new Error('not an errno at all') : errnoError(code);
+ mockedRm.mockImplementation(() => {
+ throw err;
+ });
+ expect(() => removeAllDuringTeardown('/tmp/a')).toThrow(err);
+ });
+
+ test('reclaims every remaining target before rethrowing an untolerated errno', () => {
+ // Losing the diagnosis is bad; leaking the other directories too is worse.
+ const err = errnoError('EMFILE');
+ rmFailingOn('/tmp/broken', err);
+ expect(() => removeAllDuringTeardown('/tmp/broken', '/tmp/second', '/tmp/third')).toThrow(err);
+ expect(mockedRm).toHaveBeenCalledWith('/tmp/second', expect.anything());
+ expect(mockedRm).toHaveBeenCalledWith('/tmp/third', expect.anything());
+ });
+
+ test('rethrows the FIRST untolerated errno when several targets fail', () => {
+ const first = errnoError('EMFILE');
+ const second = errnoError('EACCES');
+ mockedRm.mockImplementation((target) => {
+ if (target === '/tmp/one') throw first;
+ if (target === '/tmp/two') throw second;
+ });
+ expect(() => removeAllDuringTeardown('/tmp/one', '/tmp/two')).toThrow(first);
+ });
+
+ test('is silent on an absent path, which `force` already suppresses', () => {
+ // Real syscalls, no injection: ENOENT must not reach the tolerate path and
+ // must not warn, or every ordinary teardown would log.
+ const dir = mkdtempSync(join(tmpdir(), 'ok-teardown-fs-'));
+ expect(() => removeAllDuringTeardown(join(dir, 'never-existed'), dir)).not.toThrow();
+ expect(warn).not.toHaveBeenCalled();
+ expect(existsSync(dir)).toBe(false);
+ });
+
+ test('accepts an empty target list without touching the filesystem', () => {
+ mockedRm.mockImplementation(() => {});
+ expect(() => removeAllDuringTeardown()).not.toThrow();
+ expect(mockedRm).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/app/tests/integration/vite-worker-cachedir-isolation.test.ts b/packages/app/tests/integration/vite-worker-cachedir-isolation.test.ts
index 34bc809b2..f4c23ecd7 100644
--- a/packages/app/tests/integration/vite-worker-cachedir-isolation.test.ts
+++ b/packages/app/tests/integration/vite-worker-cachedir-isolation.test.ts
@@ -43,10 +43,16 @@
* references so the fixture can bind the path to a local variable
* (`const viteCacheDir = mkdtempSync(...)`) rather than inlining
* the expression. Test B3 pins the teardown counterpart: every
- * `mkdtempSync` allocation must be paired with an `rmSync` so
- * per-worker cache directories under `node_modules/` are reclaimed
- * on both the happy path (after the spawned server is released)
- * and the failure path (caught launch errors that abort `use`).
+ * `mkdtempSync` allocation must be reclaimed, so per-worker cache
+ * directories under `node_modules/` do not survive on the happy path
+ * (after the spawned server is released) or the failure path (caught
+ * launch errors that abort `use`). B3 asserts the WIRING — that both
+ * teardown sites reclaim the cacheDir alongside the contentDir. That
+ * a removal actually deletes the directory, and that a tolerated
+ * errno does not abort the removals after it, is pinned
+ * behaviourally against real syscalls in `teardown-fs-errno.test.ts`;
+ * exercising the wiring itself would mean booting a per-worker dev
+ * server, which is the cost this file exists to avoid.
*
* Why a meta-test rather than reproducing the race itself: the bug is a
* cross-process TOCTOU over the shared directory; it needs CI-grade
@@ -279,7 +285,7 @@ describe('per-worker Vite cacheDir isolation — workerServer fixture side', ()
// Step 1: resolve the per-worker cacheDir identifier from the
// OK_TEST_VITE_CACHE_DIR property assignment. Cleanup needs a stable
- // name to pass to `rmSync` at teardown — an inline expression as the
+ // name to hand the removal at teardown — an inline expression as the
// property value (e.g. `OK_TEST_VITE_CACHE_DIR: mkdtempSync(...)`)
// discards the resolved path the moment the env-object literal
// closes, leaving no handle for teardown to reach. The
@@ -291,7 +297,7 @@ describe('per-worker Vite cacheDir isolation — workerServer fixture side', ()
if (props.length === 0) {
throw new Error(
`B3 prerequisite missing: no \`${CACHE_DIR_ENV_VAR}\` property in fixtures.ts (B1 should have failed first).\n` +
- `Without the property, there is no identifier to pass to \`rmSync\` at teardown — so the per-worker Vite cacheDir cannot be reclaimed and CI accumulates orphan directories under tmpdir() until the runner's filesystem fills.`,
+ `Without the property, there is no identifier to hand the removal at teardown — so the per-worker Vite cacheDir cannot be reclaimed and CI accumulates orphan directories under tmpdir() until the runner's filesystem fills.`,
);
}
const initializer = props[0]?.getInitializer();
@@ -302,48 +308,59 @@ describe('per-worker Vite cacheDir isolation — workerServer fixture side', ()
}
if (!initializer.isKind(SyntaxKind.Identifier)) {
throw new Error(
- `B3 requires \`${CACHE_DIR_ENV_VAR}\` to be bound to a named local variable so the workerServer fixture can \`rmSync\` the same path at teardown.\n` +
+ `B3 requires \`${CACHE_DIR_ENV_VAR}\` to be bound to a named local variable so the workerServer fixture can reclaim the same path at teardown.\n` +
`Found initializer at line ${initializer.getStartLineNumber()}: \`${initializer.getText().slice(0, 80)}\` (kind: ${initializer.getKindName()}).\n` +
`Acceptable shape: \`const viteCacheDir = mkdtempSync(...); ...; ${CACHE_DIR_ENV_VAR}: viteCacheDir,\`.`,
);
}
const cacheDirVar = initializer.getText();
- // Step 2: enumerate rmSync call sites in the fixture. The two
- // existing `rmSync(contentDir, ...)` calls anchor the failure-path
- // catch block and the happy-path after-`use` block — together they
- // are the two teardown sites the fix must mirror with a paired
- // `rmSync(${cacheDirVar}, ...)`.
+ // Step 2: enumerate the teardown sites. A teardown site is a removal
+ // call that reclaims `contentDir`; the fixture has one on the failure
+ // path and one on the happy path, and each must reclaim the cacheDir
+ // too.
+ //
+ // The removal primitive is matched as a SET, and the reclaimed path may
+ // sit at ANY argument position, because neither is the invariant. The
+ // fixture reclaims through the variadic `removeAllDuringTeardown`, which
+ // applies the shared teardown errno policy and takes every target in one
+ // call; a bare `rmSync` stays legal for a path whose removal failure must
+ // abort rather than be tolerated. Pinning one callee and argument 0 made
+ // this assertion fail on a refactor that preserved the invariant exactly,
+ // so what is pinned now is "both directories are reclaimed at both
+ // sites", however the fixture spells the removal.
+ const REMOVAL_CALLEES = new Set(['rmSync', 'removeAllDuringTeardown']);
const allCalls = sf.getDescendantsOfKind(SyntaxKind.CallExpression);
- const rmSyncMatching = (targetName: string) =>
+ const reclaimsOf = (targetName: string) =>
allCalls.filter((call) => {
- if (call.getExpression().getText() !== 'rmSync') return false;
- const firstArg = call.getArguments()[0];
- if (!firstArg) return false;
- return firstArg.isKind(SyntaxKind.Identifier) && firstArg.getText() === targetName;
+ if (!REMOVAL_CALLEES.has(call.getExpression().getText())) return false;
+ return call
+ .getArguments()
+ .some((arg) => arg.isKind(SyntaxKind.Identifier) && arg.getText() === targetName);
});
- const contentDirTeardowns = rmSyncMatching('contentDir');
- const cacheDirTeardowns = rmSyncMatching(cacheDirVar);
+ const contentDirTeardowns = reclaimsOf('contentDir');
+ const cacheDirTeardowns = reclaimsOf(cacheDirVar);
if (contentDirTeardowns.length < 2) {
throw new Error(
- `B3 expected at least 2 \`rmSync(contentDir, ...)\` calls to anchor the failure-path + happy-path teardown sites; found ${contentDirTeardowns.length}.\n` +
- `If the fixture's teardown structure was refactored, update this assertion to reflect the new pattern.`,
+ `B3 expected at least 2 removal calls reclaiming \`contentDir\` to anchor the failure-path + happy-path teardown sites; found ${contentDirTeardowns.length}.\n` +
+ `Recognised removal primitives: ${[...REMOVAL_CALLEES].join(', ')}.\n` +
+ `If the fixture now reclaims through a different primitive, add it to REMOVAL_CALLEES rather than reverting the fixture — the invariant is that both directories are reclaimed at both teardown sites, not which function does it.`,
);
}
- // Step 3: pair each contentDir teardown with a sibling
- // `rmSync(${cacheDirVar}, ...)` in the SAME Block ancestor. This
- // binds the assertion to BOTH the catch-block (failure path) and
- // the function-body Block (happy path) without coupling to exact
- // line numbers — the same nearby-block proximity required
- // ("directly after each existing `rmSync(contentDir, ...)`").
+ // Step 3: require each contentDir teardown to be accompanied by a
+ // reclaim of ${cacheDirVar} in the SAME Block ancestor. This binds the
+ // assertion to BOTH the catch-block (failure path) and the function-body
+ // Block (happy path) without coupling to exact line numbers. A single
+ // variadic call that reclaims both satisfies this by being its own
+ // match — it appears in both arrays with the same Block ancestor.
const missing: string[] = [];
for (const cTeardown of contentDirTeardowns) {
const parentBlock = cTeardown.getFirstAncestorByKind(SyntaxKind.Block);
if (!parentBlock) {
missing.push(
- `\`rmSync(contentDir, ...)\` at line ${cTeardown.getStartLineNumber()} has no Block ancestor; cannot locate sibling teardown for ${cacheDirVar}`,
+ `removal call reclaiming \`contentDir\` at line ${cTeardown.getStartLineNumber()} has no Block ancestor; cannot locate the ${cacheDirVar} reclaim for that site`,
);
continue;
}
@@ -352,7 +369,7 @@ describe('per-worker Vite cacheDir isolation — workerServer fixture side', ()
);
if (!sibling) {
missing.push(
- `teardown site near \`rmSync(contentDir, ...)\` at line ${cTeardown.getStartLineNumber()} is missing a sibling \`rmSync(${cacheDirVar}, ...)\` in the same block`,
+ `teardown site at line ${cTeardown.getStartLineNumber()} reclaims \`contentDir\` but nothing in the same block reclaims \`${cacheDirVar}\``,
);
}
}
@@ -360,7 +377,7 @@ describe('per-worker Vite cacheDir isolation — workerServer fixture side', ()
if (missing.length > 0) {
throw new Error(
[
- `tests/stress/_helpers/fixtures.ts must \`rmSync(${cacheDirVar}, { recursive: true, force: true })\` at BOTH teardown sites — the failure-path catch block AND the happy-path after-\`use\` block — mirroring the existing \`rmSync(contentDir, ...)\` pattern.`,
+ `tests/stress/_helpers/fixtures.ts must reclaim \`${cacheDirVar}\` at BOTH teardown sites — the failure-path catch block AND the happy-path after-\`use\` block — alongside the \`contentDir\` reclaim already there.`,
`Without cleanup at both sites, CI accumulates orphan per-worker Vite cache directories under tmpdir() until the runner's filesystem fills.`,
``,
`Missing:`,
diff --git a/packages/app/tests/stress/_helpers/fixtures.ts b/packages/app/tests/stress/_helpers/fixtures.ts
index ace6ea7a6..249666e84 100644
--- a/packages/app/tests/stress/_helpers/fixtures.ts
+++ b/packages/app/tests/stress/_helpers/fixtures.ts
@@ -47,7 +47,7 @@
*/
import { spawn } from 'node:child_process';
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { test as base } from '@playwright/test';
@@ -63,6 +63,7 @@ import {
tailServerLog,
waitForHttpReady,
} from './server-process.ts';
+import { removeAllDuringTeardown } from './teardown-fs.ts';
export interface WorkerServer {
/** Port the dev server is listening on. */
@@ -476,15 +477,17 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
} catch (err) {
const reason = err instanceof Error ? err.message : String(err);
// Same guard shape as the teardown below: cleanup must run even if
- // the kill throws. Which errnos it absorbs is `tolerateDuringTeardown`
- // in server-process.ts, not a set restated here. The log file is
- // deliberately NOT removed — the thrown message cites its path.
+ // the kill throws, and every directory is reclaimed even if an earlier
+ // one cannot be. Which errnos each half absorbs is
+ // `tolerateDuringTeardown` in server-process.ts and
+ // `removeAllDuringTeardown` in teardown-fs.ts, not a set restated here.
+ // The log file is deliberately NOT removed — the thrown message cites
+ // its path.
try {
await killGracefully(proc);
} finally {
closeServerLog(serverLog);
- rmSync(contentDir, { recursive: true, force: true });
- rmSync(viteCacheDir, { recursive: true, force: true });
+ removeAllDuringTeardown(contentDir, viteCacheDir);
}
throw new Error(
`${reason}\n--- dev server log tail (${serverLog.path}) ---\n${tailServerLog(serverLog)}`,
@@ -493,17 +496,17 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
await use({ port, baseURL, contentDir });
- // Cleanup must run even if killGracefully throws — a leaked
- // viteCacheDir under node_modules accumulates prebundled chunks with no
- // reaper. Which errnos it absorbs is `tolerateDuringTeardown` in
- // server-process.ts. Mirrors the per-test fixtures' teardown shape.
+ // Cleanup must run even if killGracefully throws, and every target is
+ // attempted even if an earlier one refuses — a leaked viteCacheDir under
+ // node_modules accumulates prebundled chunks with no reaper. Which
+ // errnos each half absorbs is `tolerateDuringTeardown` in
+ // server-process.ts and `removeAllDuringTeardown` in teardown-fs.ts.
+ // Mirrors the per-test fixtures' teardown shape.
try {
await killGracefully(proc);
} finally {
closeServerLog(serverLog);
- rmSync(serverLog.path, { force: true });
- rmSync(contentDir, { recursive: true, force: true });
- rmSync(viteCacheDir, { recursive: true, force: true });
+ removeAllDuringTeardown(serverLog.path, contentDir, viteCacheDir);
}
},
// 240s: server boot probe (≤72s) + app first-load warmup (≤60s goto +
diff --git a/packages/app/tests/stress/_helpers/global-warm-cache.ts b/packages/app/tests/stress/_helpers/global-warm-cache.ts
index 7a59dc9c7..6ae7d5dfe 100644
--- a/packages/app/tests/stress/_helpers/global-warm-cache.ts
+++ b/packages/app/tests/stress/_helpers/global-warm-cache.ts
@@ -44,6 +44,7 @@ import {
VITE_E2E_SEED_DIR,
waitForHttpReady,
} from './server-process.ts';
+import { removeAllDuringTeardown } from './teardown-fs.ts';
const SEED_KEY_FILENAME = '.seed-key';
const OPTIMIZER_SETTLE_BUDGET_MS = 90_000;
@@ -159,23 +160,27 @@ async function buildSeedOnce(key: string): Promise<void> {
// Kill BEFORE any buildDir cleanup: the server writes into buildDir (its
// Vite cacheDir) until it exits, and the optimizer recreates a deleted
// dir mid-write — orphaning it under node_modules/ with nothing to reap it.
- // Cleanup still runs if the kill throws; which errnos it absorbs is
- // `tolerateDuringTeardown` in server-process.ts (same guard shape as the
+ // Cleanup still runs if the kill throws, and a directory that refuses to
+ // go does not strand the others; which errnos each half absorbs is
+ // `tolerateDuringTeardown` in server-process.ts and
+ // `removeAllDuringTeardown` in teardown-fs.ts (same guard shape as the
// worker-fixture teardown in fixtures.ts).
try {
await killGracefully(proc);
} finally {
closeServerLog(log);
- rmSync(contentDir, { recursive: true, force: true });
- if (!succeeded) {
- rmSync(buildDir, { recursive: true, force: true });
- }
+ // buildDir is only reclaimed on failure: a successful build is renamed
+ // into the seed dir below, so removing it here would destroy the seed.
+ removeAllDuringTeardown(contentDir, ...(succeeded ? [] : [buildDir]));
}
}
try {
+ // Bare `rmSync`: this one is a PRECONDITION for the rename, not teardown.
+ // Its failure must abort the promotion rather than be tolerated, or the
+ // rename lands on top of a half-cleared seed.
rmSync(VITE_E2E_SEED_DIR, { recursive: true, force: true });
renameSync(buildDir, VITE_E2E_SEED_DIR);
- rmSync(log.path, { force: true });
+ removeAllDuringTeardown(log.path);
} catch (promoteErr) {
// A failed promotion (e.g. ENOTEMPTY from a concurrent local run's
// globalSetup re-creating the seed) would otherwise orphan buildDir
@@ -183,8 +188,7 @@ async function buildSeedOnce(key: string): Promise<void> {
// on !succeeded. The build itself succeeded, so the log holds no
// failure diagnostics; remove both and rethrow for the fail-open
// wrapper to handle.
- rmSync(buildDir, { recursive: true, force: true });
- rmSync(log.path, { force: true });
+ removeAllDuringTeardown(buildDir, log.path);
throw promoteErr;
}
}
diff --git a/packages/app/tests/stress/_helpers/index.ts b/packages/app/tests/stress/_helpers/index.ts
index 202be5840..ff9a9d7ba 100644
--- a/packages/app/tests/stress/_helpers/index.ts
+++ b/packages/app/tests/stress/_helpers/index.ts
@@ -75,6 +75,7 @@ export {
waitForSlashMenuOpen,
} from './slash-menu.ts';
export { blockMarker, generateTallDoc } from './tall-doc-fixture.ts';
+export { removeAllDuringTeardown } from './teardown-fs.ts';
export {
createMp3Buffer,
createMp4Buffer,
diff --git a/packages/app/tests/stress/_helpers/server-process.ts b/packages/app/tests/stress/_helpers/server-process.ts
index 8fab91e43..de93b0f80 100644
--- a/packages/app/tests/stress/_helpers/server-process.ts
+++ b/packages/app/tests/stress/_helpers/server-process.ts
@@ -319,10 +319,14 @@ export async function killGracefully(proc: ChildProcess, timeoutMs = 5000): Prom
if (!signalTree(proc, 'SIGTERM')) return;
await Promise.race([exited, wait(timeoutMs)]);
if (proc.exitCode === null && proc.signalCode === null) {
- // signalTree may report the tree already gone (the child exited during
- // the timeout window); `exited` was armed before the first signal, so
- // the await below cannot hang either way. The group-SIGKILL inside
- // signalTree already reaped any descendants — no further sweep needed.
+ // signalTree can report false for either tolerated errno, and neither
+ // can hang the await. ESRCH means the child exited during the timeout
+ // window. EPERM means the pid it holds is no longer ours — which requires
+ // that pid to have been recycled, and a pid is only recycled once the
+ // process holding it was reaped, so our child has exited on that branch
+ // too. `exited` was armed before the first signal, so it is already
+ // settled either way. The group-SIGKILL inside signalTree already reaped
+ // any descendants — no further sweep needed.
signalTree(proc, 'SIGKILL');
await exited;
} else if (proc.pid !== undefined) {
diff --git a/packages/app/tests/stress/_helpers/teardown-fs.ts b/packages/app/tests/stress/_helpers/teardown-fs.ts
new file mode 100644
index 000000000..0ed667ee6
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/teardown-fs.ts
@@ -0,0 +1,103 @@
+/**
+ * Filesystem-removal policy for e2e teardown — the `rmSync` sibling of
+ * `tolerateDuringTeardown` in `server-process.ts`, written for the same
+ * failure class one layer over.
+ *
+ * Every fixture here kills a dev server and then reclaims the directories it
+ * was writing into. Both halves can fail for the same reason (the tree is not
+ * quite gone yet), and both failures land in a `finally` that runs after the
+ * last test has finished — so a throw is attributed to no test at all.
+ * Playwright reports that as "N errors were not a part of any test": a fully
+ * green report on a red run, and in the merge queue an ejection that rebuilds
+ * everything queued behind it.
+ *
+ * `rmSync`'s `force` option is not the guard it looks like: it suppresses
+ * ENOENT and nothing else, and `maxRetries` defaults to 0. A bare
+ * `rmSync(dir, { recursive: true, force: true })` in a teardown therefore
+ * throws on exactly the errnos a still-settling dev server produces, and — the
+ * part that also leaks — the throw skips every sibling removal after it.
+ *
+ * This module answers both: retry what the kernel calls transient, tolerate
+ * what means "someone else is holding this path", attempt every target
+ * regardless, and rethrow anything else.
+ */
+
+import { rmSync } from 'node:fs';
+
+/**
+ * The removal errnos teardown reads as "the path is not ours to reclaim right
+ * now". Anything else rethrows.
+ *
+ * EBUSY — a live handle or mount point. During teardown that is the Vite
+ * optimizer or a file-watcher that has not finished unwinding.
+ * ENOTEMPTY — a concurrent writer created entries inside a directory while
+ * the recursive walk was deleting it; same racing writer, seen
+ * from the other end.
+ * EPERM — the platform's refusal-to-unlink code. On Windows it is what a
+ * locked file reports, so it is the same "held by something
+ * else" case rather than a distinct one.
+ *
+ * ENOENT is absent because `force` already suppresses it: it is the ordinary
+ * outcome (nothing was left to remove) and warning on it would make every
+ * teardown log.
+ *
+ * Deliberately NOT tolerated, because during teardown they are real news
+ * rather than noise:
+ *
+ * EMFILE / ENFILE — file-descriptor exhaustion. Node's own retry list
+ * includes these, so they are retried below, but a run that has exhausted
+ * its descriptors leaked them, and that is a defect this suite should
+ * report rather than absorb.
+ * ENOTDIR / EINVAL — a malformed path. A teardown pointed at the wrong
+ * variable must fail loudly; swallowing it would silently stop reclaiming
+ * anything.
+ * EACCES and everything else — no teardown story explains them on a tmpdir
+ * this process created.
+ *
+ * The tolerance is scoped to teardown, and the scoping is structural rather
+ * than a convention someone has to remember: this function returns void, so no
+ * caller can consume a "did the removal succeed?" answer, and there is nothing
+ * for a swallowed failure to corrupt downstream. Removals whose success IS
+ * consumed by the next statement — the per-test content reset in
+ * `content-reset.ts`, the pre-rename clearing of the promotion target in
+ * `global-warm-cache.ts` — keep a bare `rmSync` and keep throwing, because
+ * there "the directory is still there" changes what happens next.
+ *
+ * A tolerated failure warns rather than passing silently. The residual risk is
+ * a tmpdir that really is unreclaimable and accumulates across CI runs; this
+ * line is the only trace such a leak would leave.
+ */
+const TOLERATED_REMOVAL_ERRNOS: ReadonlySet<string> = new Set(['EBUSY', 'ENOTEMPTY', 'EPERM']);
+
+/**
+ * Node retries EBUSY/EMFILE/ENFILE/ENOTEMPTY/EPERM internally with a linear
+ * backoff of `retryDelay` (100ms default) per attempt, but only when
+ * `recursive` is set. Retrying first means a watcher that closes its handle a
+ * moment later is actually reclaimed, so tolerance stays the last resort
+ * rather than the first one. Three attempts bounds a stuck target at well
+ * under a second — teardown budget, not a test's.
+ */
+const REMOVAL_RETRIES = 3;
+
+/**
+ * Remove every target, then report. Never lets a tolerated errno end the run,
+ * never lets one failing target stop the others being reclaimed, and never
+ * hides an errno that has no teardown explanation — the first such error is
+ * rethrown once every target has been attempted.
+ */
+export function removeAllDuringTeardown(...targets: string[]): void {
+ let firstUntolerated: unknown;
+ for (const target of targets) {
+ try {
+ rmSync(target, { recursive: true, force: true, maxRetries: REMOVAL_RETRIES });
+ } catch (err) {
+ const code = (err as NodeJS.ErrnoException).code;
+ if (code !== undefined && TOLERATED_REMOVAL_ERRNOS.has(code)) {
+ console.warn(`[e2e teardown] rm ${target} reported ${code}; leaving it behind`);
+ continue;
+ }
+ firstUntolerated ??= err;
+ }
+ }
+ if (firstUntolerated !== undefined) throw firstUntolerated;
+}