Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): tolerate EPERM in e2e teardown kills (PRD-8170) (#3793)

Andrew Mikofalvyf65004eb5f1a
diff --git a/packages/app/tests/integration/kill-gracefully-errno.test.ts b/packages/app/tests/integration/kill-gracefully-errno.test.ts
new file mode 100644
index 000000000..a1bb181d4
--- /dev/null
+++ b/packages/app/tests/integration/kill-gracefully-errno.test.ts
@@ -0,0 +1,240 @@
+/**
+ * Errno policy for the e2e teardown kill path.
+ *
+ * The sibling `kill-gracefully-tree.test.ts` proves the tree really dies, with
+ * real processes. This file proves the narrower thing real processes cannot be
+ * made to demonstrate on demand: which signal errnos teardown absorbs, which it
+ * still lets out, and that nothing here can ever address our own process group.
+ *
+ * The motivating failure was a Playwright run that reported `100 passed, 0
+ * failed` and still exited 1, because a `kill EPERM` escaped a worker fixture's
+ * teardown after every test had finished. Playwright counts that as "N errors
+ * were not a part of any test" — a red run with a completely green report, and
+ * in the merge queue an ejection that rebuilds everything behind it.
+ *
+ * `process.kill` and `ChildProcess.kill` fail in different shapes, so both are
+ * exercised: the former throws, the latter returns false on ESRCH, throws only
+ * for EINVAL/ENOSYS, and routes everything else (in practice EPERM) to an
+ * 'error' EVENT — which is an uncaught throw when nothing is listening.
+ */
+
+import type { ChildProcess } from 'node:child_process';
+import { EventEmitter } from 'node:events';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
+import { killGracefully, killGroup, signalTree } from '../stress/_helpers/server-process.ts';
+
+function errnoError(code: string): NodeJS.ErrnoException {
+ return Object.assign(new Error(`kill ${code}`), { code });
+}
+
+/**
+ * A ChildProcess-shaped stub whose `kill` each test supplies, so a test can
+ * produce any of the three failure shapes Node's real `kill` has: return
+ * false, throw, or emit an 'error' event (via `emitError`) and return false.
+ */
+function fakeChild(options: {
+ pid?: number;
+ kill: () => boolean;
+}): ChildProcess & { emitError: (err: Error) => void } {
+ const emitter = new EventEmitter() as EventEmitter & Record<string, unknown>;
+ emitter.pid = options.pid ?? 424_242;
+ emitter.exitCode = null;
+ emitter.signalCode = null;
+ emitter.kill = options.kill;
+ (emitter as unknown as { emitError: (err: Error) => void }).emitError = (err: Error) => {
+ emitter.emit('error', err);
+ };
+ return emitter as unknown as ChildProcess & { emitError: (err: Error) => void };
+}
+
+function silenceWarn() {
+ return vi.spyOn(console, 'warn').mockImplementation(() => {});
+}
+
+let warn: ReturnType<typeof silenceWarn>;
+
+beforeEach(() => {
+ warn = silenceWarn();
+});
+
+afterEach(() => {
+ vi.restoreAllMocks();
+});
+
+describe('killGroup errno policy', () => {
+ test('reports success when the group signal lands', () => {
+ const kill = vi.spyOn(process, 'kill').mockReturnValue(true);
+ expect(killGroup(9_001, 'SIGTERM')).toBe(true);
+ expect(kill).toHaveBeenCalledWith(-9_001, 'SIGTERM');
+ });
+
+ test('swallows ESRCH and reports that nothing was signalled', () => {
+ vi.spyOn(process, 'kill').mockImplementation(() => {
+ throw errnoError('ESRCH');
+ });
+ expect(killGroup(9_001, 'SIGTERM')).toBe(false);
+ // ESRCH is the ordinary end-of-life case; warning on it would be noise.
+ expect(warn).not.toHaveBeenCalled();
+ });
+
+ test('swallows EPERM and reports that nothing was signalled', () => {
+ vi.spyOn(process, 'kill').mockImplementation(() => {
+ throw errnoError('EPERM');
+ });
+ expect(killGroup(9_001, 'SIGKILL')).toBe(false);
+ });
+
+ test('warns on EPERM so a genuinely leaked tree still leaves a trace', () => {
+ vi.spyOn(process, 'kill').mockImplementation(() => {
+ throw errnoError('EPERM');
+ });
+ killGroup(9_001, 'SIGKILL');
+ expect(warn).toHaveBeenCalledTimes(1);
+ expect(String(warn.mock.calls[0]?.[0])).toContain('EPERM');
+ });
+
+ test.each([
+ 'EACCES',
+ 'EINVAL',
+ 'ENOSYS',
+ undefined,
+ ])('rethrows %s rather than hiding it', (code) => {
+ const err = code === undefined ? new Error('not an errno at all') : errnoError(code);
+ vi.spyOn(process, 'kill').mockImplementation(() => {
+ throw err;
+ });
+ expect(() => killGroup(9_001, 'SIGTERM')).toThrow(err);
+ });
+
+ test.each([
+ 0,
+ 1,
+ -5,
+ 1.5,
+ Number.NaN,
+ ])('refuses pid %p instead of signalling our own process group', (pid) => {
+ // kill(-0) collapses to kill(0), which signals the caller's own group —
+ // here, the whole test run. kill(-1) broadcasts. Neither may ever be
+ // reached, so the guard must fire before the syscall.
+ const kill = vi.spyOn(process, 'kill').mockReturnValue(true);
+ expect(killGroup(pid, 'SIGKILL')).toBe(false);
+ expect(kill).not.toHaveBeenCalled();
+ });
+});
+
+describe('signalTree errno policy', () => {
+ beforeEach(() => {
+ // Force the direct-child fallback: the group attempt must miss so each
+ // test below exercises the ChildProcess.kill half.
+ vi.spyOn(process, 'kill').mockImplementation(() => {
+ throw errnoError('ESRCH');
+ });
+ });
+
+ test('reports the direct kill result instead of assuming success', () => {
+ // ChildProcess.kill returns false on ESRCH without throwing, so a caller
+ // that ignores the return value claims a signal landed when none did.
+ const proc = fakeChild({ kill: () => false });
+ expect(signalTree(proc, 'SIGTERM')).toBe(false);
+ });
+
+ test('reports success when the direct kill lands', () => {
+ const proc = fakeChild({ kill: () => true });
+ expect(signalTree(proc, 'SIGTERM')).toBe(true);
+ });
+
+ test('tolerates EPERM delivered as an error event rather than a throw', () => {
+ const proc = fakeChild({
+ kill: () => {
+ proc.emitError(errnoError('EPERM'));
+ return false;
+ },
+ });
+ expect(signalTree(proc, 'SIGKILL')).toBe(false);
+ expect(warn).toHaveBeenCalledTimes(1);
+ });
+
+ test('rethrows a non-teardown errno delivered as an error event', () => {
+ const err = errnoError('EACCES');
+ const proc = fakeChild({
+ kill: () => {
+ proc.emitError(err);
+ return false;
+ },
+ });
+ // Converting the unhandled 'error' event into a synchronous throw at the
+ // call site is the point: same failure, diagnosable stack.
+ expect(() => signalTree(proc, 'SIGKILL')).toThrow(err);
+ });
+
+ test('rethrows a synchronous EINVAL from the direct kill', () => {
+ const err = errnoError('EINVAL');
+ const proc = fakeChild({
+ kill: () => {
+ throw err;
+ },
+ });
+ expect(() => signalTree(proc, 'SIGTERM')).toThrow(err);
+ });
+
+ test.each([
+ ['a landed signal', () => true],
+ ['a missed signal', () => false],
+ ])('leaves no error listener behind after %s', (_label, kill) => {
+ const proc = fakeChild({ kill });
+ signalTree(proc, 'SIGTERM');
+ expect(proc.listenerCount('error')).toBe(0);
+ });
+
+ test('leaves no error listener behind when the direct kill throws', () => {
+ const proc = fakeChild({
+ kill: () => {
+ throw errnoError('EINVAL');
+ },
+ });
+ expect(() => signalTree(proc, 'SIGTERM')).toThrow();
+ expect(proc.listenerCount('error')).toBe(0);
+ });
+
+ test('does not suppress a pre-existing error listener', () => {
+ // The worker fixture and the warm-cache setup both register their own
+ // 'error' listener for spawn failures; the temporary one must be additive.
+ const seen: Error[] = [];
+ const proc = fakeChild({
+ kill: () => {
+ proc.emitError(errnoError('EPERM'));
+ return false;
+ },
+ });
+ proc.on('error', (err) => seen.push(err));
+ signalTree(proc, 'SIGKILL');
+ expect(seen).toHaveLength(1);
+ expect(seen[0]?.message).toContain('EPERM');
+ expect(proc.listenerCount('error')).toBe(1);
+ });
+});
+
+describe('killGracefully under a kill surface that reports EPERM everywhere', () => {
+ test('resolves instead of failing a run whose tests have all passed', async () => {
+ vi.spyOn(process, 'kill').mockImplementation(() => {
+ throw errnoError('EPERM');
+ });
+ const proc = fakeChild({
+ kill: () => {
+ proc.emitError(errnoError('EPERM'));
+ return false;
+ },
+ });
+ // The regression: this rejected, and Playwright attributed the rejection to
+ // no test at all.
+ await expect(killGracefully(proc, 500)).resolves.toBeUndefined();
+ });
+
+ test('still sweeps the group when the direct child has already exited', async () => {
+ const kill = vi.spyOn(process, 'kill').mockReturnValue(true);
+ const proc = fakeChild({ kill: () => true });
+ (proc as unknown as { exitCode: number | null }).exitCode = 0;
+ await killGracefully(proc, 500);
+ expect(kill).toHaveBeenCalledWith(-424_242, 'SIGKILL');
+ });
+});
diff --git a/packages/app/tests/stress/_helpers/fixtures.ts b/packages/app/tests/stress/_helpers/fixtures.ts
index a057e360c..ace6ea7a6 100644
--- a/packages/app/tests/stress/_helpers/fixtures.ts
+++ b/packages/app/tests/stress/_helpers/fixtures.ts
@@ -476,8 +476,9 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
} catch (err) {
const reason = err instanceof Error ? err.message : String(err);
// Same guard shape as the teardown below: cleanup must run even if
- // the kill throws a non-ESRCH error. The log file is deliberately
- // NOT removed here — the thrown message cites its path for triage.
+ // the kill throws. Which errnos it absorbs is `tolerateDuringTeardown`
+ // in server-process.ts, not a set restated here. The log file is
+ // deliberately NOT removed — the thrown message cites its path.
try {
await killGracefully(proc);
} finally {
@@ -492,9 +493,10 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
await use({ port, baseURL, contentDir });
- // Cleanup must run even if killGracefully throws a non-ESRCH error —
- // a leaked viteCacheDir under node_modules accumulates prebundled
- // chunks with no reaper. Mirrors the per-test fixtures' teardown shape.
+ // Cleanup must run even if killGracefully throws — a leaked
+ // viteCacheDir under node_modules accumulates prebundled chunks with no
+ // reaper. Which errnos it absorbs is `tolerateDuringTeardown` in
+ // server-process.ts. Mirrors the per-test fixtures' teardown shape.
try {
await killGracefully(proc);
} finally {
diff --git a/packages/app/tests/stress/_helpers/global-warm-cache.ts b/packages/app/tests/stress/_helpers/global-warm-cache.ts
index 04c648cf3..7a59dc9c7 100644
--- a/packages/app/tests/stress/_helpers/global-warm-cache.ts
+++ b/packages/app/tests/stress/_helpers/global-warm-cache.ts
@@ -159,8 +159,9 @@ async function buildSeedOnce(key: string): Promise<void> {
// Kill BEFORE any buildDir cleanup: the server writes into buildDir (its
// Vite cacheDir) until it exits, and the optimizer recreates a deleted
// dir mid-write — orphaning it under node_modules/ with nothing to reap it.
- // Cleanup still runs if the kill throws a non-ESRCH error (same guard
- // shape as the worker-fixture teardown in fixtures.ts).
+ // Cleanup still runs if the kill throws; which errnos it absorbs is
+ // `tolerateDuringTeardown` in server-process.ts (same guard shape as the
+ // worker-fixture teardown in fixtures.ts).
try {
await killGracefully(proc);
} finally {
diff --git a/packages/app/tests/stress/_helpers/server-process.ts b/packages/app/tests/stress/_helpers/server-process.ts
index f8873d7c6..8fab91e43 100644
--- a/packages/app/tests/stress/_helpers/server-process.ts
+++ b/packages/app/tests/stress/_helpers/server-process.ts
@@ -3,11 +3,14 @@
* file-scoped e2e fixtures.
*
* Replaces previously file-private copies that drifted slightly:
- * - `killGracefully` here is the ESRCH-safe variant. `proc.kill()` can
+ * - `killGracefully` here is the errno-safe variant. `proc.kill()` can
* race a process exit between the `exitCode` check and the actual
* kill syscall — the unwrapped variant in earlier copies would throw
* `ESRCH` from cleanup teardown and replace the real test failure
- * with a misleading post-test error.
+ * with a misleading post-test error. `EPERM` is the same race one step
+ * further along (the pgid was released and re-taken) and gets the same
+ * treatment; see `tolerateDuringTeardown` for why that is safe HERE and
+ * nowhere a supervisor lives.
* - `waitForHttpReady` requires an explicit `timeoutMs` so each fixture
* names its tolerance at the call site (worker-scoped fixtures pick
* ~30s for shared cached server; per-test fixtures pick ~60s for
@@ -190,18 +193,59 @@ export async function waitForHttpReady(baseURL: string, timeoutMs: number): Prom
}
/**
- * Group-kill half of the tree contract: `kill(-pid)` + ESRCH-swallow, the one
- * place that discipline lives. While any member lives, POSIX reserves the
- * pgid, so this cannot hit a recycled pid; with no members it reports ESRCH
- * and does nothing. Returns false when the group no longer exists.
+ * The two signal errnos teardown reads as "nothing of ours is left to
+ * signal". Anything else rethrows.
+ *
+ * ESRCH — no process matched. The tree is already gone; the common case.
+ * EPERM — a process matched but is not ours to signal. `kill(2)` reports
+ * this for a whole group when no member could be signalled, so during
+ * teardown it means the pid being held is no longer the group that was
+ * spawned: the leader exited, the kernel released the pgid, and an
+ * unrelated process — on macOS often a hardened system one — now holds
+ * it. Either way nothing of ours survives, and throwing turns a finished,
+ * fully-passing run into a failed one.
+ *
+ * The tolerance is scoped to teardown, and the scoping is structural rather
+ * than a convention someone has to remember: `killGracefully` is the only
+ * caller of `killGroup`/`signalTree`, and every caller of `killGracefully` is
+ * a fixture cleanup path. No live-supervision caller exists whose "is my child
+ * still running?" answer this could corrupt. (`killGroup` and `signalTree` are
+ * exported only so `tests/integration/kill-gracefully-errno.test.ts` can drive
+ * this policy directly; a production caller appearing in that list is the
+ * signal that this reasoning needs redoing.) Code that has to KEEP a child
+ * alive must not copy this — there EPERM is real news (the pid was recycled
+ * out from under it) and swallowing it would hide the bug.
+ *
+ * EPERM warns rather than passing silently. The residual risk is a tree that
+ * really is still ours, still alive, and unsignalable, which leaks a dev
+ * server; this line is the only trace such a leak would leave.
*/
-function killGroup(pid: number, signal: NodeJS.Signals): boolean {
+function tolerateDuringTeardown(err: unknown, attempt: string): false {
+ const code = (err as NodeJS.ErrnoException).code;
+ if (code === 'ESRCH') return false;
+ if (code === 'EPERM') {
+ console.warn(`[e2e teardown] ${attempt} reported EPERM; treating the group as already gone`);
+ return false;
+ }
+ throw err;
+}
+
+/**
+ * Group-kill half of the tree contract: `kill(-pid)`, the one place that
+ * discipline lives. Returns false when nothing of ours was signalled.
+ */
+export function killGroup(pid: number, signal: NodeJS.Signals): boolean {
+ // `kill(-pid)` only addresses a process group for a genuine pgid. `-0`
+ // collapses to `kill(0, …)`, which signals OUR OWN group — the whole
+ // Playwright run — and `-1` broadcasts to every process this user owns.
+ // Node never yields pid 0 or 1 for a child, but the blast radius if it ever
+ // did is the entire run, so refuse rather than trust the caller.
+ if (!Number.isInteger(pid) || pid <= 1) return false;
try {
process.kill(-pid, signal);
return true;
} catch (err) {
- if ((err as NodeJS.ErrnoException).code !== 'ESRCH') throw err;
- return false;
+ return tolerateDuringTeardown(err, `kill(-${pid}, ${signal})`);
}
}
@@ -218,17 +262,46 @@ function killGroup(pid: number, signal: NodeJS.Signals): boolean {
*
* Returns false when nothing was signalled (tree already gone).
*/
-function signalTree(proc: ChildProcess, signal: NodeJS.Signals): boolean {
+export function signalTree(proc: ChildProcess, signal: NodeJS.Signals): boolean {
const pid = proc.pid;
if (pid === undefined) return false;
if (killGroup(pid, signal)) return true;
+
+ // `ChildProcess.kill()` does not report failure the way `process.kill()`
+ // does. It returns false on ESRCH without throwing, throws only for
+ // EINVAL/ENOSYS (a bad signal name — a programming error, so let it out),
+ // and routes every other errno, in practice EPERM, to an 'error' EVENT on
+ // `proc`.
+ //
+ // Every spawn site does register an 'error' listener, but not one of them
+ // routes the event through the policy above, and each fails differently:
+ // - `fixtures.ts` / `global-warm-cache.ts` hold a permanent `on('error')`
+ // that only logs, so a teardown EPERM prints as a `spawn error:` line
+ // blaming a spawn that in fact succeeded.
+ // - the two `.private.e2e.ts` fixtures arm `once('error', reject)` inside
+ // their readiness race. Reached from teardown that listener is either
+ // already spent (a pre-readiness spawn failure fired it, so the
+ // catch-path kill has no listener at all and an 'error' event with no
+ // listener is an uncaught throw), or still armed but pointed at a
+ // settled promise, where `reject` is a silent no-op.
+ // So: crash on one path, misattributed log on the second, total silence on
+ // the third. Capturing the event for the duration of this call is what
+ // makes all three land on one policy. The boolean result is the honest
+ // answer about whether a signal landed; the earlier shape discarded it and
+ // reported success unconditionally.
+ let emitted: Error | undefined;
+ const capture = (err: Error) => {
+ emitted = err;
+ };
+ proc.on('error', capture);
+ let signalled: boolean;
try {
- proc.kill(signal);
- return true;
- } catch (err) {
- if ((err as NodeJS.ErrnoException).code !== 'ESRCH') throw err;
- return false;
+ signalled = proc.kill(signal);
+ } finally {
+ proc.off('error', capture);
}
+ if (emitted !== undefined) return tolerateDuringTeardown(emitted, `child.kill(${signal})`);
+ return signalled;
}
export async function killGracefully(proc: ChildProcess, timeoutMs = 5000): Promise<void> {
@@ -239,9 +312,10 @@ export async function killGracefully(proc: ChildProcess, timeoutMs = 5000): Prom
return;
}
const exited = new Promise<void>((resolve) => proc.once('exit', () => resolve()));
- // ESRCH races: the process can exit between the exitCode check above and
- // either kill() call. Swallow ESRCH so cleanup teardown does not replace
- // the real test result (and the post-use rmSync still runs).
+ // Exit races: the process can exit between the exitCode check above and
+ // either kill() call. `signalTree` absorbs the errnos that mean the tree is
+ // already gone, so cleanup teardown does not replace the real test result
+ // (and the post-use rmSync still runs).
if (!signalTree(proc, 'SIGTERM')) return;
await Promise.race([exited, wait(timeoutMs)]);
if (proc.exitCode === null && proc.signalCode === null) {