Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): stop teardown rmSync failing green runs (PRD-8274, PRD-8276) (#3836)
Andrew Mikofalvy4e0637f7f125
diff --git a/packages/app/tests/integration/e2e-stop-rules.test.ts b/packages/app/tests/integration/e2e-stop-rules.test.tsindex 32533501d..c3efbb9dd 100644--- a/packages/app/tests/integration/e2e-stop-rules.test.ts+++ b/packages/app/tests/integration/e2e-stop-rules.test.ts@@ -442,7 +442,7 @@ describe('E2E STOP rule — zero allowlist', () => {}if (violations.length > 0) {throw new Error(- `dev-server spawn without shared-state isolation — pass OK_TEST_VITE_CACHE_DIR (via prepareViteCacheDir from ./_helpers, rmSync in teardown) and OK_TEST_SKIP_I18N_COMPILE: '1' in the spawn env:\n${violations.join('\n')}`,+ `dev-server spawn without shared-state isolation — pass OK_TEST_VITE_CACHE_DIR (via prepareViteCacheDir from ./_helpers, removeAllDuringTeardown in teardown) and OK_TEST_SKIP_I18N_COMPILE: '1' in the spawn env:\n${violations.join('\n')}`,);}});diff --git a/packages/app/tests/integration/teardown-fs-errno.test.ts b/packages/app/tests/integration/teardown-fs-errno.test.tsnew file mode 100644index 000000000..99850556a--- /dev/null+++ b/packages/app/tests/integration/teardown-fs-errno.test.ts@@ -0,0 +1,167 @@+/**+ * Errno policy for the e2e teardown filesystem-removal path.+ *+ * The sibling `kill-gracefully-errno.test.ts` pins which SIGNAL errnos+ * teardown absorbs. This file pins the same question one layer over, for the+ * removals that reclaim a just-killed dev server's contentDir, Vite cacheDir+ * and log file: which removal errnos teardown absorbs, which it still lets+ * out, and that one failing target cannot stop the others being reclaimed.+ *+ * The motivating failure is the shape the signal policy was written for, one+ * layer over — a Playwright run where every test passed and the run still+ * reported failure, because a teardown throw escaped after the last test had+ * finished. Playwright counts that as "N errors were not a part of any test":+ * a red run with a completely green report, and in the merge queue an ejection+ * that rebuilds everything behind it.+ *+ * `rmSync`'s `force` option only suppresses ENOENT; EBUSY, ENOTEMPTY and EPERM+ * still throw, and its `maxRetries` defaults to 0. Those three are what a Vite+ * optimizer or file-watcher still holding a handle produces while teardown+ * deletes the directory under it — so they are injected here at the `rmSync`+ * boundary rather than raced for against a real watcher.+ *+ * The mock factory spreads the real `node:fs`, so every other call in this file+ * (`mkdtempSync`, `existsSync`) is the genuine syscall and `rmSync` itself+ * falls back to the real one whenever a test does not override it.+ */++import { existsSync, mkdtempSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';+import { removeAllDuringTeardown } from '../stress/_helpers/teardown-fs.ts';++vi.mock('node:fs', async (importOriginal) => {+ const actual = await importOriginal<typeof import('node:fs')>();+ return { ...actual, rmSync: vi.fn(actual.rmSync) };+});++const mockedRm = vi.mocked(rmSync);++function errnoError(code: string): NodeJS.ErrnoException {+ return Object.assign(new Error(`rm ${code}`), { code });+}++/** Fail only on `failing`; every other target removes cleanly. */+function rmFailingOn(failing: string, err: NodeJS.ErrnoException) {+ mockedRm.mockImplementation((target) => {+ if (target === failing) throw err;+ });+ return mockedRm;+}++let warn: ReturnType<typeof vi.spyOn<Console, 'warn'>>;++beforeEach(() => {+ warn = vi.spyOn(console, 'warn').mockImplementation(() => {});+});++afterEach(() => {+ // `mockReset` restores the implementation `vi.fn(actual.rmSync)` was built+ // with, so the next test starts from the real syscall again.+ mockedRm.mockReset();+ vi.restoreAllMocks();+});++describe('removeAllDuringTeardown removal policy', () => {+ test('removes every target, recursively and forcefully', () => {+ mockedRm.mockImplementation(() => {});+ removeAllDuringTeardown('/tmp/a', '/tmp/b', '/tmp/c');+ expect(mockedRm).toHaveBeenCalledTimes(3);+ for (const target of ['/tmp/a', '/tmp/b', '/tmp/c']) {+ expect(mockedRm).toHaveBeenCalledWith(+ target,+ expect.objectContaining({ recursive: true, force: true }),+ );+ }+ });++ test('retries transient removals instead of giving up on the first refusal', () => {+ // `maxRetries` engages Node's own linear backoff over the errnos it treats+ // as transient, so a watcher that closes its handle a moment later is+ // actually reclaimed rather than tolerated and leaked.+ mockedRm.mockImplementation(() => {});+ removeAllDuringTeardown('/tmp/a');+ const options = mockedRm.mock.calls[0]?.[1] as { maxRetries?: number } | undefined;+ expect(options?.maxRetries).toBeGreaterThan(0);+ });++ test.each([+ 'EBUSY',+ 'ENOTEMPTY',+ 'EPERM',+ ])('tolerates %s rather than failing a run whose tests have all passed', (code) => {+ rmFailingOn('/tmp/busy', errnoError(code));+ expect(() => removeAllDuringTeardown('/tmp/busy')).not.toThrow();+ });++ test.each([+ 'EBUSY',+ 'ENOTEMPTY',+ 'EPERM',+ ])('still reclaims the remaining targets after a tolerated %s', (code) => {+ rmFailingOn('/tmp/busy', errnoError(code));+ removeAllDuringTeardown('/tmp/busy', '/tmp/second', '/tmp/third');+ // The bug: a throw on the first target skipped the two that followed it,+ // leaking a contentDir and a Vite cacheDir per teardown.+ expect(mockedRm).toHaveBeenCalledWith('/tmp/second', expect.anything());+ expect(mockedRm).toHaveBeenCalledWith('/tmp/third', expect.anything());+ });++ test('warns on a tolerated failure so a genuinely leaked directory leaves a trace', () => {+ rmFailingOn('/tmp/busy', errnoError('EBUSY'));+ removeAllDuringTeardown('/tmp/busy');+ expect(warn).toHaveBeenCalledTimes(1);+ expect(String(warn.mock.calls[0]?.[0])).toContain('EBUSY');+ expect(String(warn.mock.calls[0]?.[0])).toContain('/tmp/busy');+ });++ test.each([+ 'EMFILE',+ 'ENFILE',+ 'EACCES',+ 'ENOTDIR',+ 'EINVAL',+ undefined,+ ])('rethrows %s rather than hiding it', (code) => {+ const err = code === undefined ? new Error('not an errno at all') : errnoError(code);+ mockedRm.mockImplementation(() => {+ throw err;+ });+ expect(() => removeAllDuringTeardown('/tmp/a')).toThrow(err);+ });++ test('reclaims every remaining target before rethrowing an untolerated errno', () => {+ // Losing the diagnosis is bad; leaking the other directories too is worse.+ const err = errnoError('EMFILE');+ rmFailingOn('/tmp/broken', err);+ expect(() => removeAllDuringTeardown('/tmp/broken', '/tmp/second', '/tmp/third')).toThrow(err);+ expect(mockedRm).toHaveBeenCalledWith('/tmp/second', expect.anything());+ expect(mockedRm).toHaveBeenCalledWith('/tmp/third', expect.anything());+ });++ test('rethrows the FIRST untolerated errno when several targets fail', () => {+ const first = errnoError('EMFILE');+ const second = errnoError('EACCES');+ mockedRm.mockImplementation((target) => {+ if (target === '/tmp/one') throw first;+ if (target === '/tmp/two') throw second;+ });+ expect(() => removeAllDuringTeardown('/tmp/one', '/tmp/two')).toThrow(first);+ });++ test('is silent on an absent path, which `force` already suppresses', () => {+ // Real syscalls, no injection: ENOENT must not reach the tolerate path and+ // must not warn, or every ordinary teardown would log.+ const dir = mkdtempSync(join(tmpdir(), 'ok-teardown-fs-'));+ expect(() => removeAllDuringTeardown(join(dir, 'never-existed'), dir)).not.toThrow();+ expect(warn).not.toHaveBeenCalled();+ expect(existsSync(dir)).toBe(false);+ });++ test('accepts an empty target list without touching the filesystem', () => {+ mockedRm.mockImplementation(() => {});+ expect(() => removeAllDuringTeardown()).not.toThrow();+ expect(mockedRm).not.toHaveBeenCalled();+ });+});diff --git a/packages/app/tests/integration/vite-worker-cachedir-isolation.test.ts b/packages/app/tests/integration/vite-worker-cachedir-isolation.test.tsindex 34bc809b2..f4c23ecd7 100644--- a/packages/app/tests/integration/vite-worker-cachedir-isolation.test.ts+++ b/packages/app/tests/integration/vite-worker-cachedir-isolation.test.ts@@ -43,10 +43,16 @@* references so the fixture can bind the path to a local variable* (`const viteCacheDir = mkdtempSync(...)`) rather than inlining* the expression. Test B3 pins the teardown counterpart: every- * `mkdtempSync` allocation must be paired with an `rmSync` so- * per-worker cache directories under `node_modules/` are reclaimed- * on both the happy path (after the spawned server is released)- * and the failure path (caught launch errors that abort `use`).+ * `mkdtempSync` allocation must be reclaimed, so per-worker cache+ * directories under `node_modules/` do not survive on the happy path+ * (after the spawned server is released) or the failure path (caught+ * launch errors that abort `use`). B3 asserts the WIRING — that both+ * teardown sites reclaim the cacheDir alongside the contentDir. That+ * a removal actually deletes the directory, and that a tolerated+ * errno does not abort the removals after it, is pinned+ * behaviourally against real syscalls in `teardown-fs-errno.test.ts`;+ * exercising the wiring itself would mean booting a per-worker dev+ * server, which is the cost this file exists to avoid.** Why a meta-test rather than reproducing the race itself: the bug is a* cross-process TOCTOU over the shared directory; it needs CI-grade@@ -279,7 +285,7 @@ describe('per-worker Vite cacheDir isolation — workerServer fixture side', ()// Step 1: resolve the per-worker cacheDir identifier from the// OK_TEST_VITE_CACHE_DIR property assignment. Cleanup needs a stable- // name to pass to `rmSync` at teardown — an inline expression as the+ // name to hand the removal at teardown — an inline expression as the// property value (e.g. `OK_TEST_VITE_CACHE_DIR: mkdtempSync(...)`)// discards the resolved path the moment the env-object literal// closes, leaving no handle for teardown to reach. The@@ -291,7 +297,7 @@ describe('per-worker Vite cacheDir isolation — workerServer fixture side', ()if (props.length === 0) {throw new Error(`B3 prerequisite missing: no \`${CACHE_DIR_ENV_VAR}\` property in fixtures.ts (B1 should have failed first).\n` +- `Without the property, there is no identifier to pass to \`rmSync\` at teardown — so the per-worker Vite cacheDir cannot be reclaimed and CI accumulates orphan directories under tmpdir() until the runner's filesystem fills.`,+ `Without the property, there is no identifier to hand the removal at teardown — so the per-worker Vite cacheDir cannot be reclaimed and CI accumulates orphan directories under tmpdir() until the runner's filesystem fills.`,);}const initializer = props[0]?.getInitializer();@@ -302,48 +308,59 @@ describe('per-worker Vite cacheDir isolation — workerServer fixture side', ()}if (!initializer.isKind(SyntaxKind.Identifier)) {throw new Error(- `B3 requires \`${CACHE_DIR_ENV_VAR}\` to be bound to a named local variable so the workerServer fixture can \`rmSync\` the same path at teardown.\n` ++ `B3 requires \`${CACHE_DIR_ENV_VAR}\` to be bound to a named local variable so the workerServer fixture can reclaim the same path at teardown.\n` +`Found initializer at line ${initializer.getStartLineNumber()}: \`${initializer.getText().slice(0, 80)}\` (kind: ${initializer.getKindName()}).\n` +`Acceptable shape: \`const viteCacheDir = mkdtempSync(...); ...; ${CACHE_DIR_ENV_VAR}: viteCacheDir,\`.`,);}const cacheDirVar = initializer.getText();- // Step 2: enumerate rmSync call sites in the fixture. The two- // existing `rmSync(contentDir, ...)` calls anchor the failure-path- // catch block and the happy-path after-`use` block — together they- // are the two teardown sites the fix must mirror with a paired- // `rmSync(${cacheDirVar}, ...)`.+ // Step 2: enumerate the teardown sites. A teardown site is a removal+ // call that reclaims `contentDir`; the fixture has one on the failure+ // path and one on the happy path, and each must reclaim the cacheDir+ // too.+ //+ // The removal primitive is matched as a SET, and the reclaimed path may+ // sit at ANY argument position, because neither is the invariant. The+ // fixture reclaims through the variadic `removeAllDuringTeardown`, which+ // applies the shared teardown errno policy and takes every target in one+ // call; a bare `rmSync` stays legal for a path whose removal failure must+ // abort rather than be tolerated. Pinning one callee and argument 0 made+ // this assertion fail on a refactor that preserved the invariant exactly,+ // so what is pinned now is "both directories are reclaimed at both+ // sites", however the fixture spells the removal.+ const REMOVAL_CALLEES = new Set(['rmSync', 'removeAllDuringTeardown']);const allCalls = sf.getDescendantsOfKind(SyntaxKind.CallExpression);- const rmSyncMatching = (targetName: string) =>+ const reclaimsOf = (targetName: string) =>allCalls.filter((call) => {- if (call.getExpression().getText() !== 'rmSync') return false;- const firstArg = call.getArguments()[0];- if (!firstArg) return false;- return firstArg.isKind(SyntaxKind.Identifier) && firstArg.getText() === targetName;+ if (!REMOVAL_CALLEES.has(call.getExpression().getText())) return false;+ return call+ .getArguments()+ .some((arg) => arg.isKind(SyntaxKind.Identifier) && arg.getText() === targetName);});- const contentDirTeardowns = rmSyncMatching('contentDir');- const cacheDirTeardowns = rmSyncMatching(cacheDirVar);+ const contentDirTeardowns = reclaimsOf('contentDir');+ const cacheDirTeardowns = reclaimsOf(cacheDirVar);if (contentDirTeardowns.length < 2) {throw new Error(- `B3 expected at least 2 \`rmSync(contentDir, ...)\` calls to anchor the failure-path + happy-path teardown sites; found ${contentDirTeardowns.length}.\n` +- `If the fixture's teardown structure was refactored, update this assertion to reflect the new pattern.`,+ `B3 expected at least 2 removal calls reclaiming \`contentDir\` to anchor the failure-path + happy-path teardown sites; found ${contentDirTeardowns.length}.\n` ++ `Recognised removal primitives: ${[...REMOVAL_CALLEES].join(', ')}.\n` ++ `If the fixture now reclaims through a different primitive, add it to REMOVAL_CALLEES rather than reverting the fixture — the invariant is that both directories are reclaimed at both teardown sites, not which function does it.`,);}- // Step 3: pair each contentDir teardown with a sibling- // `rmSync(${cacheDirVar}, ...)` in the SAME Block ancestor. This- // binds the assertion to BOTH the catch-block (failure path) and- // the function-body Block (happy path) without coupling to exact- // line numbers — the same nearby-block proximity required- // ("directly after each existing `rmSync(contentDir, ...)`").+ // Step 3: require each contentDir teardown to be accompanied by a+ // reclaim of ${cacheDirVar} in the SAME Block ancestor. This binds the+ // assertion to BOTH the catch-block (failure path) and the function-body+ // Block (happy path) without coupling to exact line numbers. A single+ // variadic call that reclaims both satisfies this by being its own+ // match — it appears in both arrays with the same Block ancestor.const missing: string[] = [];for (const cTeardown of contentDirTeardowns) {const parentBlock = cTeardown.getFirstAncestorByKind(SyntaxKind.Block);if (!parentBlock) {missing.push(- `\`rmSync(contentDir, ...)\` at line ${cTeardown.getStartLineNumber()} has no Block ancestor; cannot locate sibling teardown for ${cacheDirVar}`,+ `removal call reclaiming \`contentDir\` at line ${cTeardown.getStartLineNumber()} has no Block ancestor; cannot locate the ${cacheDirVar} reclaim for that site`,);continue;}@@ -352,7 +369,7 @@ describe('per-worker Vite cacheDir isolation — workerServer fixture side', ());if (!sibling) {missing.push(- `teardown site near \`rmSync(contentDir, ...)\` at line ${cTeardown.getStartLineNumber()} is missing a sibling \`rmSync(${cacheDirVar}, ...)\` in the same block`,+ `teardown site at line ${cTeardown.getStartLineNumber()} reclaims \`contentDir\` but nothing in the same block reclaims \`${cacheDirVar}\``,);}}@@ -360,7 +377,7 @@ describe('per-worker Vite cacheDir isolation — workerServer fixture side', ()if (missing.length > 0) {throw new Error([- `tests/stress/_helpers/fixtures.ts must \`rmSync(${cacheDirVar}, { recursive: true, force: true })\` at BOTH teardown sites — the failure-path catch block AND the happy-path after-\`use\` block — mirroring the existing \`rmSync(contentDir, ...)\` pattern.`,+ `tests/stress/_helpers/fixtures.ts must reclaim \`${cacheDirVar}\` at BOTH teardown sites — the failure-path catch block AND the happy-path after-\`use\` block — alongside the \`contentDir\` reclaim already there.`,`Without cleanup at both sites, CI accumulates orphan per-worker Vite cache directories under tmpdir() until the runner's filesystem fills.`,``,`Missing:`,diff --git a/packages/app/tests/stress/_helpers/fixtures.ts b/packages/app/tests/stress/_helpers/fixtures.tsindex ace6ea7a6..249666e84 100644--- a/packages/app/tests/stress/_helpers/fixtures.ts+++ b/packages/app/tests/stress/_helpers/fixtures.ts@@ -47,7 +47,7 @@*/import { spawn } from 'node:child_process';-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';import { test as base } from '@playwright/test';@@ -63,6 +63,7 @@ import {tailServerLog,waitForHttpReady,} from './server-process.ts';+import { removeAllDuringTeardown } from './teardown-fs.ts';export interface WorkerServer {/** Port the dev server is listening on. */@@ -476,15 +477,17 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({} catch (err) {const reason = err instanceof Error ? err.message : String(err);// Same guard shape as the teardown below: cleanup must run even if- // the kill throws. Which errnos it absorbs is `tolerateDuringTeardown`- // in server-process.ts, not a set restated here. The log file is- // deliberately NOT removed — the thrown message cites its path.+ // the kill throws, and every directory is reclaimed even if an earlier+ // one cannot be. Which errnos each half absorbs is+ // `tolerateDuringTeardown` in server-process.ts and+ // `removeAllDuringTeardown` in teardown-fs.ts, not a set restated here.+ // The log file is deliberately NOT removed — the thrown message cites+ // its path.try {await killGracefully(proc);} finally {closeServerLog(serverLog);- rmSync(contentDir, { recursive: true, force: true });- rmSync(viteCacheDir, { recursive: true, force: true });+ removeAllDuringTeardown(contentDir, viteCacheDir);}throw new Error(`${reason}\n--- dev server log tail (${serverLog.path}) ---\n${tailServerLog(serverLog)}`,@@ -493,17 +496,17 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({await use({ port, baseURL, contentDir });- // Cleanup must run even if killGracefully throws — a leaked- // viteCacheDir under node_modules accumulates prebundled chunks with no- // reaper. Which errnos it absorbs is `tolerateDuringTeardown` in- // server-process.ts. Mirrors the per-test fixtures' teardown shape.+ // Cleanup must run even if killGracefully throws, and every target is+ // attempted even if an earlier one refuses — a leaked viteCacheDir under+ // node_modules accumulates prebundled chunks with no reaper. Which+ // errnos each half absorbs is `tolerateDuringTeardown` in+ // server-process.ts and `removeAllDuringTeardown` in teardown-fs.ts.+ // Mirrors the per-test fixtures' teardown shape.try {await killGracefully(proc);} finally {closeServerLog(serverLog);- rmSync(serverLog.path, { force: true });- rmSync(contentDir, { recursive: true, force: true });- rmSync(viteCacheDir, { recursive: true, force: true });+ removeAllDuringTeardown(serverLog.path, contentDir, viteCacheDir);}},// 240s: server boot probe (≤72s) + app first-load warmup (≤60s goto +diff --git a/packages/app/tests/stress/_helpers/global-warm-cache.ts b/packages/app/tests/stress/_helpers/global-warm-cache.tsindex 7a59dc9c7..6ae7d5dfe 100644--- a/packages/app/tests/stress/_helpers/global-warm-cache.ts+++ b/packages/app/tests/stress/_helpers/global-warm-cache.ts@@ -44,6 +44,7 @@ import {VITE_E2E_SEED_DIR,waitForHttpReady,} from './server-process.ts';+import { removeAllDuringTeardown } from './teardown-fs.ts';const SEED_KEY_FILENAME = '.seed-key';const OPTIMIZER_SETTLE_BUDGET_MS = 90_000;@@ -159,23 +160,27 @@ async function buildSeedOnce(key: string): Promise<void> {// Kill BEFORE any buildDir cleanup: the server writes into buildDir (its// Vite cacheDir) until it exits, and the optimizer recreates a deleted// dir mid-write — orphaning it under node_modules/ with nothing to reap it.- // Cleanup still runs if the kill throws; which errnos it absorbs is- // `tolerateDuringTeardown` in server-process.ts (same guard shape as the+ // Cleanup still runs if the kill throws, and a directory that refuses to+ // go does not strand the others; which errnos each half absorbs is+ // `tolerateDuringTeardown` in server-process.ts and+ // `removeAllDuringTeardown` in teardown-fs.ts (same guard shape as the// worker-fixture teardown in fixtures.ts).try {await killGracefully(proc);} finally {closeServerLog(log);- rmSync(contentDir, { recursive: true, force: true });- if (!succeeded) {- rmSync(buildDir, { recursive: true, force: true });- }+ // buildDir is only reclaimed on failure: a successful build is renamed+ // into the seed dir below, so removing it here would destroy the seed.+ removeAllDuringTeardown(contentDir, ...(succeeded ? [] : [buildDir]));}}try {+ // Bare `rmSync`: this one is a PRECONDITION for the rename, not teardown.+ // Its failure must abort the promotion rather than be tolerated, or the+ // rename lands on top of a half-cleared seed.rmSync(VITE_E2E_SEED_DIR, { recursive: true, force: true });renameSync(buildDir, VITE_E2E_SEED_DIR);- rmSync(log.path, { force: true });+ removeAllDuringTeardown(log.path);} catch (promoteErr) {// A failed promotion (e.g. ENOTEMPTY from a concurrent local run's// globalSetup re-creating the seed) would otherwise orphan buildDir@@ -183,8 +188,7 @@ async function buildSeedOnce(key: string): Promise<void> {// on !succeeded. The build itself succeeded, so the log holds no// failure diagnostics; remove both and rethrow for the fail-open// wrapper to handle.- rmSync(buildDir, { recursive: true, force: true });- rmSync(log.path, { force: true });+ removeAllDuringTeardown(buildDir, log.path);throw promoteErr;}}diff --git a/packages/app/tests/stress/_helpers/index.ts b/packages/app/tests/stress/_helpers/index.tsindex 202be5840..ff9a9d7ba 100644--- a/packages/app/tests/stress/_helpers/index.ts+++ b/packages/app/tests/stress/_helpers/index.ts@@ -75,6 +75,7 @@ export {waitForSlashMenuOpen,} from './slash-menu.ts';export { blockMarker, generateTallDoc } from './tall-doc-fixture.ts';+export { removeAllDuringTeardown } from './teardown-fs.ts';export {createMp3Buffer,createMp4Buffer,diff --git a/packages/app/tests/stress/_helpers/server-process.ts b/packages/app/tests/stress/_helpers/server-process.tsindex 8fab91e43..de93b0f80 100644--- a/packages/app/tests/stress/_helpers/server-process.ts+++ b/packages/app/tests/stress/_helpers/server-process.ts@@ -319,10 +319,14 @@ export async function killGracefully(proc: ChildProcess, timeoutMs = 5000): Promif (!signalTree(proc, 'SIGTERM')) return;await Promise.race([exited, wait(timeoutMs)]);if (proc.exitCode === null && proc.signalCode === null) {- // signalTree may report the tree already gone (the child exited during- // the timeout window); `exited` was armed before the first signal, so- // the await below cannot hang either way. The group-SIGKILL inside- // signalTree already reaped any descendants — no further sweep needed.+ // signalTree can report false for either tolerated errno, and neither+ // can hang the await. ESRCH means the child exited during the timeout+ // window. EPERM means the pid it holds is no longer ours — which requires+ // that pid to have been recycled, and a pid is only recycled once the+ // process holding it was reaped, so our child has exited on that branch+ // too. `exited` was armed before the first signal, so it is already+ // settled either way. The group-SIGKILL inside signalTree already reaped+ // any descendants — no further sweep needed.signalTree(proc, 'SIGKILL');await exited;} else if (proc.pid !== undefined) {diff --git a/packages/app/tests/stress/_helpers/teardown-fs.ts b/packages/app/tests/stress/_helpers/teardown-fs.tsnew file mode 100644index 000000000..0ed667ee6--- /dev/null+++ b/packages/app/tests/stress/_helpers/teardown-fs.ts@@ -0,0 +1,103 @@+/**+ * Filesystem-removal policy for e2e teardown — the `rmSync` sibling of+ * `tolerateDuringTeardown` in `server-process.ts`, written for the same+ * failure class one layer over.+ *+ * Every fixture here kills a dev server and then reclaims the directories it+ * was writing into. Both halves can fail for the same reason (the tree is not+ * quite gone yet), and both failures land in a `finally` that runs after the+ * last test has finished — so a throw is attributed to no test at all.+ * Playwright reports that as "N errors were not a part of any test": a fully+ * green report on a red run, and in the merge queue an ejection that rebuilds+ * everything queued behind it.+ *+ * `rmSync`'s `force` option is not the guard it looks like: it suppresses+ * ENOENT and nothing else, and `maxRetries` defaults to 0. A bare+ * `rmSync(dir, { recursive: true, force: true })` in a teardown therefore+ * throws on exactly the errnos a still-settling dev server produces, and — the+ * part that also leaks — the throw skips every sibling removal after it.+ *+ * This module answers both: retry what the kernel calls transient, tolerate+ * what means "someone else is holding this path", attempt every target+ * regardless, and rethrow anything else.+ */++import { rmSync } from 'node:fs';++/**+ * The removal errnos teardown reads as "the path is not ours to reclaim right+ * now". Anything else rethrows.+ *+ * EBUSY — a live handle or mount point. During teardown that is the Vite+ * optimizer or a file-watcher that has not finished unwinding.+ * ENOTEMPTY — a concurrent writer created entries inside a directory while+ * the recursive walk was deleting it; same racing writer, seen+ * from the other end.+ * EPERM — the platform's refusal-to-unlink code. On Windows it is what a+ * locked file reports, so it is the same "held by something+ * else" case rather than a distinct one.+ *+ * ENOENT is absent because `force` already suppresses it: it is the ordinary+ * outcome (nothing was left to remove) and warning on it would make every+ * teardown log.+ *+ * Deliberately NOT tolerated, because during teardown they are real news+ * rather than noise:+ *+ * EMFILE / ENFILE — file-descriptor exhaustion. Node's own retry list+ * includes these, so they are retried below, but a run that has exhausted+ * its descriptors leaked them, and that is a defect this suite should+ * report rather than absorb.+ * ENOTDIR / EINVAL — a malformed path. A teardown pointed at the wrong+ * variable must fail loudly; swallowing it would silently stop reclaiming+ * anything.+ * EACCES and everything else — no teardown story explains them on a tmpdir+ * this process created.+ *+ * The tolerance is scoped to teardown, and the scoping is structural rather+ * than a convention someone has to remember: this function returns void, so no+ * caller can consume a "did the removal succeed?" answer, and there is nothing+ * for a swallowed failure to corrupt downstream. Removals whose success IS+ * consumed by the next statement — the per-test content reset in+ * `content-reset.ts`, the pre-rename clearing of the promotion target in+ * `global-warm-cache.ts` — keep a bare `rmSync` and keep throwing, because+ * there "the directory is still there" changes what happens next.+ *+ * A tolerated failure warns rather than passing silently. The residual risk is+ * a tmpdir that really is unreclaimable and accumulates across CI runs; this+ * line is the only trace such a leak would leave.+ */+const TOLERATED_REMOVAL_ERRNOS: ReadonlySet<string> = new Set(['EBUSY', 'ENOTEMPTY', 'EPERM']);++/**+ * Node retries EBUSY/EMFILE/ENFILE/ENOTEMPTY/EPERM internally with a linear+ * backoff of `retryDelay` (100ms default) per attempt, but only when+ * `recursive` is set. Retrying first means a watcher that closes its handle a+ * moment later is actually reclaimed, so tolerance stays the last resort+ * rather than the first one. Three attempts bounds a stuck target at well+ * under a second — teardown budget, not a test's.+ */+const REMOVAL_RETRIES = 3;++/**+ * Remove every target, then report. Never lets a tolerated errno end the run,+ * never lets one failing target stop the others being reclaimed, and never+ * hides an errno that has no teardown explanation — the first such error is+ * rethrown once every target has been attempted.+ */+export function removeAllDuringTeardown(...targets: string[]): void {+ let firstUntolerated: unknown;+ for (const target of targets) {+ try {+ rmSync(target, { recursive: true, force: true, maxRetries: REMOVAL_RETRIES });+ } catch (err) {+ const code = (err as NodeJS.ErrnoException).code;+ if (code !== undefined && TOLERATED_REMOVAL_ERRNOS.has(code)) {+ console.warn(`[e2e teardown] rm ${target} reported ${code}; leaving it behind`);+ continue;+ }+ firstUntolerated ??= err;+ }+ }+ if (firstUntolerated !== undefined) throw firstUntolerated;+}