Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): tolerate EPERM in e2e teardown kills (PRD-8170) (#3793)
Andrew Mikofalvyf65004eb5f1a
diff --git a/packages/app/tests/integration/kill-gracefully-errno.test.ts b/packages/app/tests/integration/kill-gracefully-errno.test.tsnew file mode 100644index 000000000..a1bb181d4--- /dev/null+++ b/packages/app/tests/integration/kill-gracefully-errno.test.ts@@ -0,0 +1,240 @@+/**+ * Errno policy for the e2e teardown kill path.+ *+ * The sibling `kill-gracefully-tree.test.ts` proves the tree really dies, with+ * real processes. This file proves the narrower thing real processes cannot be+ * made to demonstrate on demand: which signal errnos teardown absorbs, which it+ * still lets out, and that nothing here can ever address our own process group.+ *+ * The motivating failure was a Playwright run that reported `100 passed, 0+ * failed` and still exited 1, because a `kill EPERM` escaped a worker fixture's+ * teardown after every test had finished. Playwright counts that as "N errors+ * were not a part of any test" — a red run with a completely green report, and+ * in the merge queue an ejection that rebuilds everything behind it.+ *+ * `process.kill` and `ChildProcess.kill` fail in different shapes, so both are+ * exercised: the former throws, the latter returns false on ESRCH, throws only+ * for EINVAL/ENOSYS, and routes everything else (in practice EPERM) to an+ * 'error' EVENT — which is an uncaught throw when nothing is listening.+ */++import type { ChildProcess } from 'node:child_process';+import { EventEmitter } from 'node:events';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';+import { killGracefully, killGroup, signalTree } from '../stress/_helpers/server-process.ts';++function errnoError(code: string): NodeJS.ErrnoException {+ return Object.assign(new Error(`kill ${code}`), { code });+}++/**+ * A ChildProcess-shaped stub whose `kill` each test supplies, so a test can+ * produce any of the three failure shapes Node's real `kill` has: return+ * false, throw, or emit an 'error' event (via `emitError`) and return false.+ */+function fakeChild(options: {+ pid?: number;+ kill: () => boolean;+}): ChildProcess & { emitError: (err: Error) => void } {+ const emitter = new EventEmitter() as EventEmitter & Record<string, unknown>;+ emitter.pid = options.pid ?? 424_242;+ emitter.exitCode = null;+ emitter.signalCode = null;+ emitter.kill = options.kill;+ (emitter as unknown as { emitError: (err: Error) => void }).emitError = (err: Error) => {+ emitter.emit('error', err);+ };+ return emitter as unknown as ChildProcess & { emitError: (err: Error) => void };+}++function silenceWarn() {+ return vi.spyOn(console, 'warn').mockImplementation(() => {});+}++let warn: ReturnType<typeof silenceWarn>;++beforeEach(() => {+ warn = silenceWarn();+});++afterEach(() => {+ vi.restoreAllMocks();+});++describe('killGroup errno policy', () => {+ test('reports success when the group signal lands', () => {+ const kill = vi.spyOn(process, 'kill').mockReturnValue(true);+ expect(killGroup(9_001, 'SIGTERM')).toBe(true);+ expect(kill).toHaveBeenCalledWith(-9_001, 'SIGTERM');+ });++ test('swallows ESRCH and reports that nothing was signalled', () => {+ vi.spyOn(process, 'kill').mockImplementation(() => {+ throw errnoError('ESRCH');+ });+ expect(killGroup(9_001, 'SIGTERM')).toBe(false);+ // ESRCH is the ordinary end-of-life case; warning on it would be noise.+ expect(warn).not.toHaveBeenCalled();+ });++ test('swallows EPERM and reports that nothing was signalled', () => {+ vi.spyOn(process, 'kill').mockImplementation(() => {+ throw errnoError('EPERM');+ });+ expect(killGroup(9_001, 'SIGKILL')).toBe(false);+ });++ test('warns on EPERM so a genuinely leaked tree still leaves a trace', () => {+ vi.spyOn(process, 'kill').mockImplementation(() => {+ throw errnoError('EPERM');+ });+ killGroup(9_001, 'SIGKILL');+ expect(warn).toHaveBeenCalledTimes(1);+ expect(String(warn.mock.calls[0]?.[0])).toContain('EPERM');+ });++ test.each([+ 'EACCES',+ 'EINVAL',+ 'ENOSYS',+ undefined,+ ])('rethrows %s rather than hiding it', (code) => {+ const err = code === undefined ? new Error('not an errno at all') : errnoError(code);+ vi.spyOn(process, 'kill').mockImplementation(() => {+ throw err;+ });+ expect(() => killGroup(9_001, 'SIGTERM')).toThrow(err);+ });++ test.each([+ 0,+ 1,+ -5,+ 1.5,+ Number.NaN,+ ])('refuses pid %p instead of signalling our own process group', (pid) => {+ // kill(-0) collapses to kill(0), which signals the caller's own group —+ // here, the whole test run. kill(-1) broadcasts. Neither may ever be+ // reached, so the guard must fire before the syscall.+ const kill = vi.spyOn(process, 'kill').mockReturnValue(true);+ expect(killGroup(pid, 'SIGKILL')).toBe(false);+ expect(kill).not.toHaveBeenCalled();+ });+});++describe('signalTree errno policy', () => {+ beforeEach(() => {+ // Force the direct-child fallback: the group attempt must miss so each+ // test below exercises the ChildProcess.kill half.+ vi.spyOn(process, 'kill').mockImplementation(() => {+ throw errnoError('ESRCH');+ });+ });++ test('reports the direct kill result instead of assuming success', () => {+ // ChildProcess.kill returns false on ESRCH without throwing, so a caller+ // that ignores the return value claims a signal landed when none did.+ const proc = fakeChild({ kill: () => false });+ expect(signalTree(proc, 'SIGTERM')).toBe(false);+ });++ test('reports success when the direct kill lands', () => {+ const proc = fakeChild({ kill: () => true });+ expect(signalTree(proc, 'SIGTERM')).toBe(true);+ });++ test('tolerates EPERM delivered as an error event rather than a throw', () => {+ const proc = fakeChild({+ kill: () => {+ proc.emitError(errnoError('EPERM'));+ return false;+ },+ });+ expect(signalTree(proc, 'SIGKILL')).toBe(false);+ expect(warn).toHaveBeenCalledTimes(1);+ });++ test('rethrows a non-teardown errno delivered as an error event', () => {+ const err = errnoError('EACCES');+ const proc = fakeChild({+ kill: () => {+ proc.emitError(err);+ return false;+ },+ });+ // Converting the unhandled 'error' event into a synchronous throw at the+ // call site is the point: same failure, diagnosable stack.+ expect(() => signalTree(proc, 'SIGKILL')).toThrow(err);+ });++ test('rethrows a synchronous EINVAL from the direct kill', () => {+ const err = errnoError('EINVAL');+ const proc = fakeChild({+ kill: () => {+ throw err;+ },+ });+ expect(() => signalTree(proc, 'SIGTERM')).toThrow(err);+ });++ test.each([+ ['a landed signal', () => true],+ ['a missed signal', () => false],+ ])('leaves no error listener behind after %s', (_label, kill) => {+ const proc = fakeChild({ kill });+ signalTree(proc, 'SIGTERM');+ expect(proc.listenerCount('error')).toBe(0);+ });++ test('leaves no error listener behind when the direct kill throws', () => {+ const proc = fakeChild({+ kill: () => {+ throw errnoError('EINVAL');+ },+ });+ expect(() => signalTree(proc, 'SIGTERM')).toThrow();+ expect(proc.listenerCount('error')).toBe(0);+ });++ test('does not suppress a pre-existing error listener', () => {+ // The worker fixture and the warm-cache setup both register their own+ // 'error' listener for spawn failures; the temporary one must be additive.+ const seen: Error[] = [];+ const proc = fakeChild({+ kill: () => {+ proc.emitError(errnoError('EPERM'));+ return false;+ },+ });+ proc.on('error', (err) => seen.push(err));+ signalTree(proc, 'SIGKILL');+ expect(seen).toHaveLength(1);+ expect(seen[0]?.message).toContain('EPERM');+ expect(proc.listenerCount('error')).toBe(1);+ });+});++describe('killGracefully under a kill surface that reports EPERM everywhere', () => {+ test('resolves instead of failing a run whose tests have all passed', async () => {+ vi.spyOn(process, 'kill').mockImplementation(() => {+ throw errnoError('EPERM');+ });+ const proc = fakeChild({+ kill: () => {+ proc.emitError(errnoError('EPERM'));+ return false;+ },+ });+ // The regression: this rejected, and Playwright attributed the rejection to+ // no test at all.+ await expect(killGracefully(proc, 500)).resolves.toBeUndefined();+ });++ test('still sweeps the group when the direct child has already exited', async () => {+ const kill = vi.spyOn(process, 'kill').mockReturnValue(true);+ const proc = fakeChild({ kill: () => true });+ (proc as unknown as { exitCode: number | null }).exitCode = 0;+ await killGracefully(proc, 500);+ expect(kill).toHaveBeenCalledWith(-424_242, 'SIGKILL');+ });+});diff --git a/packages/app/tests/stress/_helpers/fixtures.ts b/packages/app/tests/stress/_helpers/fixtures.tsindex a057e360c..ace6ea7a6 100644--- a/packages/app/tests/stress/_helpers/fixtures.ts+++ b/packages/app/tests/stress/_helpers/fixtures.ts@@ -476,8 +476,9 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({} catch (err) {const reason = err instanceof Error ? err.message : String(err);// Same guard shape as the teardown below: cleanup must run even if- // the kill throws a non-ESRCH error. The log file is deliberately- // NOT removed here — the thrown message cites its path for triage.+ // the kill throws. Which errnos it absorbs is `tolerateDuringTeardown`+ // in server-process.ts, not a set restated here. The log file is+ // deliberately NOT removed — the thrown message cites its path.try {await killGracefully(proc);} finally {@@ -492,9 +493,10 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({await use({ port, baseURL, contentDir });- // Cleanup must run even if killGracefully throws a non-ESRCH error —- // a leaked viteCacheDir under node_modules accumulates prebundled- // chunks with no reaper. Mirrors the per-test fixtures' teardown shape.+ // Cleanup must run even if killGracefully throws — a leaked+ // viteCacheDir under node_modules accumulates prebundled chunks with no+ // reaper. Which errnos it absorbs is `tolerateDuringTeardown` in+ // server-process.ts. Mirrors the per-test fixtures' teardown shape.try {await killGracefully(proc);} finally {diff --git a/packages/app/tests/stress/_helpers/global-warm-cache.ts b/packages/app/tests/stress/_helpers/global-warm-cache.tsindex 04c648cf3..7a59dc9c7 100644--- a/packages/app/tests/stress/_helpers/global-warm-cache.ts+++ b/packages/app/tests/stress/_helpers/global-warm-cache.ts@@ -159,8 +159,9 @@ async function buildSeedOnce(key: string): Promise<void> {// Kill BEFORE any buildDir cleanup: the server writes into buildDir (its// Vite cacheDir) until it exits, and the optimizer recreates a deleted// dir mid-write — orphaning it under node_modules/ with nothing to reap it.- // Cleanup still runs if the kill throws a non-ESRCH error (same guard- // shape as the worker-fixture teardown in fixtures.ts).+ // Cleanup still runs if the kill throws; which errnos it absorbs is+ // `tolerateDuringTeardown` in server-process.ts (same guard shape as the+ // worker-fixture teardown in fixtures.ts).try {await killGracefully(proc);} finally {diff --git a/packages/app/tests/stress/_helpers/server-process.ts b/packages/app/tests/stress/_helpers/server-process.tsindex f8873d7c6..8fab91e43 100644--- a/packages/app/tests/stress/_helpers/server-process.ts+++ b/packages/app/tests/stress/_helpers/server-process.ts@@ -3,11 +3,14 @@* file-scoped e2e fixtures.** Replaces previously file-private copies that drifted slightly:- * - `killGracefully` here is the ESRCH-safe variant. `proc.kill()` can+ * - `killGracefully` here is the errno-safe variant. `proc.kill()` can* race a process exit between the `exitCode` check and the actual* kill syscall — the unwrapped variant in earlier copies would throw* `ESRCH` from cleanup teardown and replace the real test failure- * with a misleading post-test error.+ * with a misleading post-test error. `EPERM` is the same race one step+ * further along (the pgid was released and re-taken) and gets the same+ * treatment; see `tolerateDuringTeardown` for why that is safe HERE and+ * nowhere a supervisor lives.* - `waitForHttpReady` requires an explicit `timeoutMs` so each fixture* names its tolerance at the call site (worker-scoped fixtures pick* ~30s for shared cached server; per-test fixtures pick ~60s for@@ -190,18 +193,59 @@ export async function waitForHttpReady(baseURL: string, timeoutMs: number): Prom}/**- * Group-kill half of the tree contract: `kill(-pid)` + ESRCH-swallow, the one- * place that discipline lives. While any member lives, POSIX reserves the- * pgid, so this cannot hit a recycled pid; with no members it reports ESRCH- * and does nothing. Returns false when the group no longer exists.+ * The two signal errnos teardown reads as "nothing of ours is left to+ * signal". Anything else rethrows.+ *+ * ESRCH — no process matched. The tree is already gone; the common case.+ * EPERM — a process matched but is not ours to signal. `kill(2)` reports+ * this for a whole group when no member could be signalled, so during+ * teardown it means the pid being held is no longer the group that was+ * spawned: the leader exited, the kernel released the pgid, and an+ * unrelated process — on macOS often a hardened system one — now holds+ * it. Either way nothing of ours survives, and throwing turns a finished,+ * fully-passing run into a failed one.+ *+ * The tolerance is scoped to teardown, and the scoping is structural rather+ * than a convention someone has to remember: `killGracefully` is the only+ * caller of `killGroup`/`signalTree`, and every caller of `killGracefully` is+ * a fixture cleanup path. No live-supervision caller exists whose "is my child+ * still running?" answer this could corrupt. (`killGroup` and `signalTree` are+ * exported only so `tests/integration/kill-gracefully-errno.test.ts` can drive+ * this policy directly; a production caller appearing in that list is the+ * signal that this reasoning needs redoing.) Code that has to KEEP a child+ * alive must not copy this — there EPERM is real news (the pid was recycled+ * out from under it) and swallowing it would hide the bug.+ *+ * EPERM warns rather than passing silently. The residual risk is a tree that+ * really is still ours, still alive, and unsignalable, which leaks a dev+ * server; this line is the only trace such a leak would leave.*/-function killGroup(pid: number, signal: NodeJS.Signals): boolean {+function tolerateDuringTeardown(err: unknown, attempt: string): false {+ const code = (err as NodeJS.ErrnoException).code;+ if (code === 'ESRCH') return false;+ if (code === 'EPERM') {+ console.warn(`[e2e teardown] ${attempt} reported EPERM; treating the group as already gone`);+ return false;+ }+ throw err;+}++/**+ * Group-kill half of the tree contract: `kill(-pid)`, the one place that+ * discipline lives. Returns false when nothing of ours was signalled.+ */+export function killGroup(pid: number, signal: NodeJS.Signals): boolean {+ // `kill(-pid)` only addresses a process group for a genuine pgid. `-0`+ // collapses to `kill(0, …)`, which signals OUR OWN group — the whole+ // Playwright run — and `-1` broadcasts to every process this user owns.+ // Node never yields pid 0 or 1 for a child, but the blast radius if it ever+ // did is the entire run, so refuse rather than trust the caller.+ if (!Number.isInteger(pid) || pid <= 1) return false;try {process.kill(-pid, signal);return true;} catch (err) {- if ((err as NodeJS.ErrnoException).code !== 'ESRCH') throw err;- return false;+ return tolerateDuringTeardown(err, `kill(-${pid}, ${signal})`);}}@@ -218,17 +262,46 @@ function killGroup(pid: number, signal: NodeJS.Signals): boolean {** Returns false when nothing was signalled (tree already gone).*/-function signalTree(proc: ChildProcess, signal: NodeJS.Signals): boolean {+export function signalTree(proc: ChildProcess, signal: NodeJS.Signals): boolean {const pid = proc.pid;if (pid === undefined) return false;if (killGroup(pid, signal)) return true;++ // `ChildProcess.kill()` does not report failure the way `process.kill()`+ // does. It returns false on ESRCH without throwing, throws only for+ // EINVAL/ENOSYS (a bad signal name — a programming error, so let it out),+ // and routes every other errno, in practice EPERM, to an 'error' EVENT on+ // `proc`.+ //+ // Every spawn site does register an 'error' listener, but not one of them+ // routes the event through the policy above, and each fails differently:+ // - `fixtures.ts` / `global-warm-cache.ts` hold a permanent `on('error')`+ // that only logs, so a teardown EPERM prints as a `spawn error:` line+ // blaming a spawn that in fact succeeded.+ // - the two `.private.e2e.ts` fixtures arm `once('error', reject)` inside+ // their readiness race. Reached from teardown that listener is either+ // already spent (a pre-readiness spawn failure fired it, so the+ // catch-path kill has no listener at all and an 'error' event with no+ // listener is an uncaught throw), or still armed but pointed at a+ // settled promise, where `reject` is a silent no-op.+ // So: crash on one path, misattributed log on the second, total silence on+ // the third. Capturing the event for the duration of this call is what+ // makes all three land on one policy. The boolean result is the honest+ // answer about whether a signal landed; the earlier shape discarded it and+ // reported success unconditionally.+ let emitted: Error | undefined;+ const capture = (err: Error) => {+ emitted = err;+ };+ proc.on('error', capture);+ let signalled: boolean;try {- proc.kill(signal);- return true;- } catch (err) {- if ((err as NodeJS.ErrnoException).code !== 'ESRCH') throw err;- return false;+ signalled = proc.kill(signal);+ } finally {+ proc.off('error', capture);}+ if (emitted !== undefined) return tolerateDuringTeardown(emitted, `child.kill(${signal})`);+ return signalled;}export async function killGracefully(proc: ChildProcess, timeoutMs = 5000): Promise<void> {@@ -239,9 +312,10 @@ export async function killGracefully(proc: ChildProcess, timeoutMs = 5000): Promreturn;}const exited = new Promise<void>((resolve) => proc.once('exit', () => resolve()));- // ESRCH races: the process can exit between the exitCode check above and- // either kill() call. Swallow ESRCH so cleanup teardown does not replace- // the real test result (and the post-use rmSync still runs).+ // Exit races: the process can exit between the exitCode check above and+ // either kill() call. `signalTree` absorbs the errnos that mean the tree is+ // already gone, so cleanup teardown does not replace the real test result+ // (and the post-use rmSync still runs).if (!signalTree(proc, 'SIGTERM')) return;await Promise.race([exited, wait(timeoutMs)]);if (proc.exitCode === null && proc.signalCode === null) {