Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Make a stress boot failure a distinct verdict from an assertion (#4874)

Nick Gomeza2c38a198505
diff --git a/packages/app/playwright.a11y.config.ts b/packages/app/playwright.a11y.config.ts
index 9926e046a..a05e765c2 100644
--- a/packages/app/playwright.a11y.config.ts
+++ b/packages/app/playwright.a11y.config.ts
@@ -1,6 +1,7 @@
import { availableParallelism } from 'node:os';
import { defineConfig } from '@playwright/test';
import { EXPECT_TIMEOUT_MS, resolveWorkerCount } from './playwright.config.ts';
+import { SETUP_NON_RESULT_REPORTER } from './tests/stress/_helpers/setup-non-result-reporter.ts';
/**
* A11y Playwright config — per-worker fixture isolation (same shape as
@@ -52,6 +53,7 @@ export default defineConfig({
reporter: [
['html', { open: 'never' }],
['list'],
+ [SETUP_NON_RESULT_REPORTER],
...(isCI
? [['github'] as const, ['json', { outputFile: 'test-results/a11y-results.json' }] as const]
: []),
diff --git a/packages/app/playwright.config.ts b/packages/app/playwright.config.ts
index a1e7633f3..7584b4d1e 100644
--- a/packages/app/playwright.config.ts
+++ b/packages/app/playwright.config.ts
@@ -1,5 +1,6 @@
import { availableParallelism } from 'node:os';
import { defineConfig } from '@playwright/test';
+import { SETUP_NON_RESULT_REPORTER } from './tests/stress/_helpers/setup-non-result-reporter.ts';
/**
* Per-worker server isolation: there is no `webServer` block — instead a
@@ -125,6 +126,7 @@ export default defineConfig({
reporter: [
['html', { open: 'never' }],
['list'],
+ [SETUP_NON_RESULT_REPORTER],
...(isCI
? [
['github'] as const,
diff --git a/packages/app/playwright.visual.config.ts b/packages/app/playwright.visual.config.ts
index 37a3fe9cd..a8c306afe 100644
--- a/packages/app/playwright.visual.config.ts
+++ b/packages/app/playwright.visual.config.ts
@@ -1,6 +1,7 @@
import { availableParallelism } from 'node:os';
import { defineConfig } from '@playwright/test';
import { EXPECT_TIMEOUT_MS, resolveWorkerCount } from './playwright.config.ts';
+import { SETUP_NON_RESULT_REPORTER } from './tests/stress/_helpers/setup-non-result-reporter.ts';
/**
* Visual Playwright config — per-worker fixture isolation (same shape as
@@ -48,6 +49,7 @@ export default defineConfig({
updateSnapshots: 'none',
fullyParallel: true,
workers: isCI ? 4 : resolveWorkerCount(availableParallelism()),
+ reporter: [['list'], [SETUP_NON_RESULT_REPORTER]],
use: {
// `baseURL` is populated by the worker-scoped fixture in
// `tests/stress/_helpers/fixtures.ts`. Leave unset so the fixture's
diff --git a/packages/app/tests/meta/playwright-worker-density.test.ts b/packages/app/tests/meta/playwright-worker-density.test.ts
index dabcf5f8f..f72de852e 100644
--- a/packages/app/tests/meta/playwright-worker-density.test.ts
+++ b/packages/app/tests/meta/playwright-worker-density.test.ts
@@ -1,9 +1,14 @@
import { availableParallelism } from 'node:os';
+import { fileURLToPath } from 'node:url';
import { describe, expect, test, vi } from 'vitest';
import { LOGICAL_CPUS_PER_WORKER, resolveWorkerCount } from '../../playwright.config';
const CI_RUNNER_LOGICAL_CPUS = 16;
+const SETUP_NON_RESULT_REPORTER_PATH = fileURLToPath(
+ new URL('../stress/_helpers/setup-non-result-reporter.ts', import.meta.url),
+);
+
const FIXTURE_SHARING_CONFIGS = {
e2e: () => import('../../playwright.config'),
a11y: () => import('../../playwright.a11y.config'),
@@ -73,3 +78,18 @@ describe('playwright worker density', () => {
},
);
});
+
+describe('setup non-result rendering', () => {
+ test.each(
+ CONFIG_NAMES.flatMap((which) => [
+ ['under CI', which, 'true'] as const,
+ ['off-CI', which, undefined] as const,
+ ]),
+ )('%s the %s config registers the setup non-result reporter', async (_setting, which, ci) => {
+ const { reporter } = await loadConfigWithCI(ci, which);
+ expect(
+ Array.isArray(reporter) ? reporter.map(([name]) => name) : [reporter],
+ `the ${which} config runs specs on the shared stress fixture, which declares a worker-server setup non-result on the test that triggered it, but it does not register ${SETUP_NON_RESULT_REPORTER_PATH}, so that tier prints the non-result as an undifferentiated failure`,
+ ).toContain(SETUP_NON_RESULT_REPORTER_PATH);
+ });
+});
diff --git a/packages/app/tests/meta/worker-server-boot-verdict.test.ts b/packages/app/tests/meta/worker-server-boot-verdict.test.ts
new file mode 100644
index 000000000..a4bbf1b9a
--- /dev/null
+++ b/packages/app/tests/meta/worker-server-boot-verdict.test.ts
@@ -0,0 +1,511 @@
+import { type ChildProcess, spawn } from 'node:child_process';
+import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import type {
+ JSONReport,
+ JSONReportSpec,
+ JSONReportSuite,
+ JSONReportTest,
+ JSONReportTestResult,
+} from '@playwright/test/reporter';
+import { afterAll, beforeAll, describe, expect, test } from 'vitest';
+
+const APP_ROOT = fileURLToPath(new URL('../..', import.meta.url));
+const CASES_DIR = fileURLToPath(new URL('../stress/_helpers/boot-verdict/', import.meta.url));
+const VERDICT_RUN_CONFIG = join(CASES_DIR, 'verdict-run.config.ts');
+const REFUSE_PID_SIGNALS = pathToFileURL(join(CASES_DIR, 'refuse-pid-signals.ts')).href;
+const PLAYWRIGHT_CLI = createRequire(join(APP_ROOT, 'package.json')).resolve(
+ '@playwright/test/cli',
+);
+
+const PLAYWRIGHT_RUN_LIVENESS_BOUND_MS = 60_000;
+const HELD_HANDLE_GRACE_MS = 10_000;
+const STDIO_CLOSE_GRACE_MS = 5_000;
+const HOOK_GRACE_MS = 5_000;
+const VERDICT_RUNS_HOOK_TIMEOUT_MS =
+ PLAYWRIGHT_RUN_LIVENESS_BOUND_MS + HELD_HANDLE_GRACE_MS + STDIO_CLOSE_GRACE_MS + HOOK_GRACE_MS;
+
+const BOOT_CASE = 'boot-prerequisite';
+const ASSERTION_CASE = 'assertion-failure';
+const CONTROL_CASE = 'passing-control';
+const RECOVERED_CASE = 'recovered-setup';
+
+interface VerdictRun {
+ cases: string[];
+ exitCode: number | null;
+ signal: NodeJS.Signals | null;
+ heldHandleBoundFired: boolean;
+ stdioClosed: boolean;
+ transcript: string;
+ report: JSONReport | undefined;
+ bodiesRan: string[];
+ bootFailureInjected: boolean;
+ workersRefusingGroupSignals: string[];
+}
+
+interface DeclaredResult {
+ status: JSONReportTestResult['status'];
+ expectedStatus: JSONReportTest['expectedStatus'];
+ outcome: JSONReportTest['status'];
+ ok: boolean;
+ annotationTypes: string[];
+}
+
+const runDirs: string[] = [];
+const liveChildren = new Set<ChildProcess>();
+
+function linesOf(path: string): string[] {
+ return existsSync(path)
+ ? readFileSync(path, 'utf8')
+ .split('\n')
+ .filter((line) => line !== '')
+ : [];
+}
+
+async function runVerdictCases(cases: string[], retries: number): Promise<VerdictRun> {
+ const runDir = mkdtempSync(join(tmpdir(), 'ok-boot-verdict-'));
+ runDirs.push(runDir);
+ const childTmp = join(runDir, 'tmp');
+ mkdirSync(childTmp);
+ const env: NodeJS.ProcessEnv = {
+ ...process.env,
+ OK_BOOT_VERDICT_RUN_DIR: runDir,
+ OK_BOOT_VERDICT_LIVENESS_BOUND_MS: String(PLAYWRIGHT_RUN_LIVENESS_BOUND_MS),
+ NODE_OPTIONS: `--import=${REFUSE_PID_SIGNALS}`,
+ TMPDIR: childTmp,
+ PLAYWRIGHT_HTML_OUTPUT_DIR: join(runDir, 'html-report'),
+ PLAYWRIGHT_HTML_OPEN: 'never',
+ };
+ delete env.CI;
+ delete env.GITHUB_ACTIONS;
+
+ const child = spawn(
+ process.execPath,
+ [
+ PLAYWRIGHT_CLI,
+ 'test',
+ '--config',
+ VERDICT_RUN_CONFIG,
+ '--workers=1',
+ `--retries=${retries}`,
+ ...cases.map((name) => `${name}.verdict-case.ts`),
+ ],
+ { cwd: APP_ROOT, env, stdio: ['ignore', 'pipe', 'pipe'] },
+ );
+ liveChildren.add(child);
+ let transcript = '';
+ child.stdout.on('data', (chunk: Buffer) => {
+ transcript += chunk.toString('utf8');
+ });
+ child.stderr.on('data', (chunk: Buffer) => {
+ transcript += chunk.toString('utf8');
+ });
+ const closed = new Promise<true>((resolve) => {
+ child.once('close', () => resolve(true));
+ });
+ let heldHandleBoundFired = false;
+ const heldHandleBound = setTimeout(() => {
+ heldHandleBoundFired = true;
+ child.kill('SIGKILL');
+ }, PLAYWRIGHT_RUN_LIVENESS_BOUND_MS + HELD_HANDLE_GRACE_MS);
+ const { exitCode, signal } = await new Promise<{
+ exitCode: number | null;
+ signal: NodeJS.Signals | null;
+ }>((resolve, reject) => {
+ child.once('error', reject);
+ child.once('exit', (code, exitSignal) => resolve({ exitCode: code, signal: exitSignal }));
+ }).finally(() => {
+ clearTimeout(heldHandleBound);
+ liveChildren.delete(child);
+ });
+ let stdioCloseBound: ReturnType<typeof setTimeout> | undefined;
+ const stdioClosed = await Promise.race([
+ closed,
+ new Promise<false>((resolve) => {
+ stdioCloseBound = setTimeout(() => resolve(false), STDIO_CLOSE_GRACE_MS);
+ }),
+ ]).finally(() => clearTimeout(stdioCloseBound));
+ if (!stdioClosed) {
+ child.stdout.destroy();
+ child.stderr.destroy();
+ }
+
+ const reportPath = join(runDir, 'results.json');
+ return {
+ cases,
+ exitCode,
+ signal,
+ heldHandleBoundFired,
+ stdioClosed,
+ transcript,
+ report: existsSync(reportPath)
+ ? (JSON.parse(readFileSync(reportPath, 'utf8')) as JSONReport)
+ : undefined,
+ bodiesRan: readdirSync(runDir)
+ .filter((entry) => entry.startsWith('body-ran.'))
+ .map((entry) => entry.slice('body-ran.'.length))
+ .sort(),
+ bootFailureInjected: existsSync(join(runDir, 'boot-failure-injected')),
+ workersRefusingGroupSignals: linesOf(join(runDir, 'refused-signals.jsonl')).flatMap((line) => {
+ const refusal = JSON.parse(line) as { pid: number; worker?: string };
+ return refusal.pid < 0 && refusal.worker !== undefined ? [refusal.worker] : [];
+ }),
+ };
+}
+
+function specsOf(suites: JSONReportSuite[]): JSONReportSpec[] {
+ return suites.flatMap((suite) => [...suite.specs, ...specsOf(suite.suites ?? [])]);
+}
+
+function soleTestOf(
+ run: VerdictRun,
+ caseName: string,
+): { spec: JSONReportSpec; test: JSONReportTest } {
+ const specs = specsOf(run.report?.suites ?? []).filter(
+ (spec) => spec.file === `${caseName}.verdict-case.ts`,
+ );
+ expect(specs, `${caseName} appears exactly once in the JSON report`).toHaveLength(1);
+ const [spec] = specs as [JSONReportSpec];
+ expect(spec.tests, `${caseName} ran in exactly one project`).toHaveLength(1);
+ const [jsonTest] = spec.tests as [JSONReportTest];
+ return { spec, test: jsonTest };
+}
+
+function declaredResult(run: VerdictRun, caseName: string): DeclaredResult {
+ const { spec, test: jsonTest } = soleTestOf(run, caseName);
+ expect(jsonTest.results, `${caseName} has exactly one attempt`).toHaveLength(1);
+ const [result] = jsonTest.results as [JSONReportTestResult];
+ const annotationTypes = new Set(
+ [...jsonTest.annotations, ...result.annotations].map((annotation) => annotation.type),
+ );
+ return {
+ status: result.status,
+ expectedStatus: jsonTest.expectedStatus,
+ outcome: jsonTest.status,
+ ok: spec.ok,
+ annotationTypes: [...annotationTypes].sort(),
+ };
+}
+
+function declaredMarkers(result: DeclaredResult): Set<string> {
+ return new Set([
+ `status:${result.status}`,
+ ...result.annotationTypes.map((type) => `annotation:${type}`),
+ ]);
+}
+
+function requireCompletedRun(run: VerdictRun): void {
+ const context = `verdict run [${run.cases.join(', ')}] transcript:\n${run.transcript}`;
+ expect(run.heldHandleBoundFired, `the held-handle bound preempted the run; ${context}`).toBe(
+ false,
+ );
+ expect(run.signal, `the run ended on a signal; ${context}`).toBeNull();
+ expect(
+ run.stdioClosed,
+ `the run exited, but its stdout and stderr had not both closed ${STDIO_CLOSE_GRACE_MS} ms later, so this transcript may be incomplete and a process the run started may still hold one of them open; ${context}`,
+ ).toBe(true);
+ expect(run.report, `the run wrote no JSON report; ${context}`).toBeDefined();
+ expect(
+ run.report?.errors ?? [],
+ `the run reported errors outside any test, such as its liveness bound; ${context}`,
+ ).toEqual([]);
+}
+
+function requireRefusedDevServerReaps(run: VerdictRun): void {
+ const bootAttemptWorkers = soleTestOf(run, BOOT_CASE).test.results.map((result) =>
+ String(result.workerIndex),
+ );
+ expect(
+ bootAttemptWorkers,
+ `verdict run [${run.cases.join(', ')}] recorded no boot attempt`,
+ ).not.toEqual([]);
+ expect(
+ run.workersRefusingGroupSignals,
+ `verdict run [${run.cases.join(', ')}] ran its boot attempts in Playwright workers ${bootAttemptWorkers.join(', ')}, and each of them reaps its exited dev server by signalling that server's process group, so each must have recorded that signal as refused; a worker missing here reaped outside the refusal preload and could have signalled a real process group`,
+ ).toEqual(expect.arrayContaining(bootAttemptWorkers));
+}
+
+const REPORTER_ERROR_MARKER = 'Error in reporter';
+
+function nonResultAnnotationType(): string {
+ const boot = declaredResult(bootRun, BOOT_CASE);
+ const executedTypes = new Set([
+ ...declaredResult(assertionRun, ASSERTION_CASE).annotationTypes,
+ ...declaredResult(bootRun, CONTROL_CASE).annotationTypes,
+ ]);
+ const nonResultTypes = boot.annotationTypes.filter((type) => !executedTypes.has(type));
+ expect(
+ nonResultTypes,
+ `the boot non-result declared the annotation types ${JSON.stringify(boot.annotationTypes)}; exactly one of them must be a type no executed result declares, and that type is the tag its rendering is read under`,
+ ).toHaveLength(1);
+ return nonResultTypes[0] as string;
+}
+
+function declaredReasons(result: JSONReportTestResult, type: string): Array<string | undefined> {
+ return result.annotations
+ .filter((annotation) => annotation.type === type)
+ .map((annotation) => annotation.description);
+}
+
+function linesTaggedWith(run: VerdictRun, type: string): string[] {
+ return run.transcript.split(/\r?\n/).filter((line) => line.startsWith(`[${type}]`));
+}
+
+function renderedNonResultLines(type: string): string[] {
+ const rendered = linesTaggedWith(bootRun, type);
+ expect(
+ rendered,
+ `the run holding the boot non-result rendered no line tagged [${type}], so the stress config's reporters never read the declaration; transcript:\n${bootRun.transcript}`,
+ ).not.toEqual([]);
+ return rendered;
+}
+
+let bootRun: VerdictRun;
+let assertionRun: VerdictRun;
+let retriedBootRun: VerdictRun;
+let recoveredRun: VerdictRun;
+
+function settledRun(settled: PromiseSettledResult<VerdictRun>): VerdictRun {
+ if (settled.status === 'rejected') throw settled.reason;
+ return settled.value;
+}
+
+beforeAll(async () => {
+ const [boot, assertion, retriedBoot, recovered] = await Promise.allSettled([
+ runVerdictCases([BOOT_CASE, CONTROL_CASE], 0),
+ runVerdictCases([ASSERTION_CASE], 0),
+ runVerdictCases([BOOT_CASE], 1),
+ runVerdictCases([RECOVERED_CASE], 1),
+ ]);
+ bootRun = settledRun(boot);
+ assertionRun = settledRun(assertion);
+ retriedBootRun = settledRun(retriedBoot);
+ recoveredRun = settledRun(recovered);
+ for (const run of [bootRun, assertionRun, retriedBootRun, recoveredRun]) {
+ requireCompletedRun(run);
+ }
+ expect(
+ bootRun.bootFailureInjected,
+ `the dev-server child never reached the injected failure; transcript:\n${bootRun.transcript}`,
+ ).toBe(true);
+ requireRefusedDevServerReaps(bootRun);
+ expect(bootRun.bodiesRan, 'only the control body ran beside the boot failure').toEqual([
+ CONTROL_CASE,
+ ]);
+ expect(assertionRun.bodiesRan, 'the assertion-failure body ran').toEqual([ASSERTION_CASE]);
+ expect(assertionRun.bootFailureInjected, 'the assertion-failure run booted no server').toBe(
+ false,
+ );
+ expect(
+ retriedBootRun.bootFailureInjected,
+ `the retried dev-server child never reached the injected failure; transcript:\n${retriedBootRun.transcript}`,
+ ).toBe(true);
+ requireRefusedDevServerReaps(retriedBootRun);
+ expect(retriedBootRun.bodiesRan, 'no attempt of the retried boot case ran its body').toEqual([]);
+ expect(
+ recoveredRun.bodiesRan,
+ 'the recovered case ran its body once its setup completed',
+ ).toEqual([RECOVERED_CASE]);
+}, VERDICT_RUNS_HOOK_TIMEOUT_MS);
+
+afterAll(() => {
+ for (const child of liveChildren) {
+ if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL');
+ }
+ for (const runDir of runDirs.splice(0)) rmSync(runDir, { recursive: true, force: true });
+});
+
+describe('a worker-server boot failure is declared as a non-result, not as an executed assertion failure', () => {
+ test('the boot non-result carries a declared marker that neither an executed assertion failure nor a passing test carries', () => {
+ const boot = declaredResult(bootRun, BOOT_CASE);
+ const assertionFailure = declaredResult(assertionRun, ASSERTION_CASE);
+ const control = declaredResult(bootRun, CONTROL_CASE);
+ const executedMarkers = new Set([
+ ...declaredMarkers(assertionFailure),
+ ...declaredMarkers(control),
+ ]);
+ const nonResultMarkers = [...declaredMarkers(boot)].filter(
+ (marker) => !executedMarkers.has(marker),
+ );
+
+ expect(
+ nonResultMarkers,
+ `boot declared ${JSON.stringify(boot)}; the executed assertion failure declared ${JSON.stringify(assertionFailure)}`,
+ ).not.toEqual([]);
+ });
+
+ test('the boot non-result still fails closed: never passed, never skipped, and its run exits non-zero', () => {
+ const boot = declaredResult(bootRun, BOOT_CASE);
+
+ expect(boot.status).not.toBe('passed');
+ expect(boot.status).not.toBe('skipped');
+ expect(boot.ok).toBe(false);
+ expect(declaredResult(bootRun, CONTROL_CASE).status).toBe('passed');
+ expect(bootRun.exitCode).not.toBe(0);
+ });
+
+ test('a declared non-result whose retry passes leaves its run passing: the declaration replaces no run status', () => {
+ const { test: recoveredTest } = soleTestOf(recoveredRun, RECOVERED_CASE);
+
+ expect(
+ recoveredTest.status,
+ "the recovered case is its run's only test, and its passing retry makes that test flaky",
+ ).toBe('flaky');
+ expect(
+ recoveredRun.exitCode,
+ 'the stock config sets failOnFlakyTests: false, so Playwright passes a run whose only test is flaky; a non-zero exit means a reporter replaced the run status because the run held a declared setup non-result',
+ ).toBe(0);
+ });
+
+ test('an executed assertion failure keeps its declared result and acquires no marker', () => {
+ expect(declaredResult(assertionRun, ASSERTION_CASE)).toEqual({
+ status: 'failed',
+ expectedStatus: 'passed',
+ outcome: 'unexpected',
+ ok: false,
+ annotationTypes: [],
+ });
+ expect(assertionRun.exitCode).not.toBe(0);
+ });
+
+ test('a passing test beside the boot non-result keeps its declared result', () => {
+ expect(declaredResult(bootRun, CONTROL_CASE)).toEqual({
+ status: 'passed',
+ expectedStatus: 'passed',
+ outcome: 'expected',
+ ok: true,
+ annotationTypes: [],
+ });
+ });
+});
+
+describe('the stress reporters render a declared setup non-result under its declared type, and nothing else', () => {
+ test('the run holding the boot non-result renders it, tagged, with its test title and declared reason, and does not render the passing test beside it', () => {
+ const type = nonResultAnnotationType();
+ const rendered = renderedNonResultLines(type);
+ const { spec: bootSpec, test: bootTest } = soleTestOf(bootRun, BOOT_CASE);
+ const { spec: controlSpec } = soleTestOf(bootRun, CONTROL_CASE);
+ const reasons = bootTest.results.flatMap((result) => declaredReasons(result, type));
+ expect(
+ reasons,
+ 'the boot attempt declares its non-result once, carrying the reason its setup did not complete',
+ ).toEqual([expect.stringMatching(/\S/)]);
+ const reasonFirstLine = (reasons[0] as string).split('\n')[0] as string;
+
+ expect(
+ rendered.filter((line) => line.includes(bootSpec.title)),
+ `no line tagged [${type}] names the boot test "${bootSpec.title}"; tagged lines:\n${rendered.join('\n')}`,
+ ).not.toEqual([]);
+ expect(
+ rendered.filter((line) => line.includes(reasonFirstLine)),
+ `no line tagged [${type}] carries the declared reason "${reasonFirstLine}"; tagged lines:\n${rendered.join('\n')}`,
+ ).not.toEqual([]);
+ expect(
+ rendered.filter((line) => line.includes(controlSpec.title)),
+ `a line tagged [${type}] names the passing test "${controlSpec.title}", which declared nothing`,
+ ).toEqual([]);
+ });
+
+ test('a run whose only failure is an executed assertion failure renders no line under the non-result type', () => {
+ const type = nonResultAnnotationType();
+ renderedNonResultLines(type);
+
+ expect(
+ linesTaggedWith(assertionRun, type),
+ `the executed assertion failure declared nothing, yet its run rendered lines tagged [${type}]; transcript:\n${assertionRun.transcript}`,
+ ).toEqual([]);
+ });
+
+ test('rendering the non-result raises no reporter error in any run', () => {
+ renderedNonResultLines(nonResultAnnotationType());
+
+ for (const run of [bootRun, assertionRun, retriedBootRun, recoveredRun]) {
+ expect(
+ run.transcript,
+ `verdict run [${run.cases.join(', ')}] reported a reporter error`,
+ ).not.toContain(REPORTER_ERROR_MARKER);
+ }
+ });
+
+ test('each attempt of a retried boot non-result declares its own non-result and is rendered as its own entry, the retry marked', () => {
+ const type = nonResultAnnotationType();
+ const { spec, test: retriedTest } = soleTestOf(retriedBootRun, BOOT_CASE);
+
+ expect(
+ retriedTest.results.map((result) => declaredReasons(result, type).length > 0),
+ `each of the boot case's attempts under --retries=1 must declare [${type}] on its own result`,
+ ).toEqual([true, true]);
+ expect(retriedTest.status, 'every attempt failed closed, so the test is unexpected').toBe(
+ 'unexpected',
+ );
+ expect(retriedBootRun.exitCode).not.toBe(0);
+ expect(
+ linesTaggedWith(retriedBootRun, type).filter((line) => line.includes(spec.title)),
+ `the retried run must render one tagged entry per attempt, the second marked as retry #1; transcript:\n${retriedBootRun.transcript}`,
+ ).toEqual([expect.not.stringContaining('(retry #'), expect.stringContaining('(retry #1)')]);
+ });
+
+ test('a declared attempt whose retry passes is never rendered as a counted failure', () => {
+ const type = nonResultAnnotationType();
+ const { test: recoveredTest } = soleTestOf(recoveredRun, RECOVERED_CASE);
+
+ expect(
+ recoveredTest.results.map((result) => declaredReasons(result, type).length > 0),
+ `the recovered case must declare [${type}] on its first attempt only and complete its setup on the retry`,
+ ).toEqual([true, false]);
+ expect(
+ recoveredTest.status,
+ 'the passing retry makes the test flaky, which the run does not count as failed',
+ ).toBe('flaky');
+ const rendered = linesTaggedWith(recoveredRun, type);
+ expect(
+ rendered,
+ `the recovered run rendered no line tagged [${type}] for its declared attempt; transcript:\n${recoveredRun.transcript}`,
+ ).not.toEqual([]);
+ expect(
+ rendered.filter((line) => /\bfailed\b/.test(line)),
+ `the recovered run holds no failed test, yet a line tagged [${type}] says failed; transcript:\n${recoveredRun.transcript}`,
+ ).toEqual([]);
+ });
+
+ test('every rendered entry ends with the final outcome results.json records for its test', () => {
+ const type = nonResultAnnotationType();
+
+ for (const [run, caseName] of [
+ [bootRun, BOOT_CASE],
+ [retriedBootRun, BOOT_CASE],
+ [recoveredRun, RECOVERED_CASE],
+ ] as const) {
+ const { spec, test: jsonTest } = soleTestOf(run, caseName);
+ const entries = linesTaggedWith(run, type).filter((line) => line.includes(spec.title));
+ expect(
+ entries,
+ `verdict run [${run.cases.join(', ')}] rendered no entry for "${spec.title}"; transcript:\n${run.transcript}`,
+ ).not.toEqual([]);
+ expect(
+ entries,
+ `each entry for "${spec.title}" in verdict run [${run.cases.join(', ')}] must end with "${jsonTest.status}", the outcome results.json records for that test`,
+ ).toEqual(entries.map(() => expect.stringMatching(new RegExp(`\\b${jsonTest.status}$`))));
+ }
+ });
+
+ test('the whole rendering is one block, so it cannot interleave with the per-test output beside it', () => {
+ const type = nonResultAnnotationType();
+ const lines = retriedBootRun.transcript.split(/\r?\n/);
+ const tagged = lines.flatMap((line, index) => (line.startsWith(`[${type}]`) ? [index] : []));
+
+ expect(
+ tagged.length,
+ `the retried run declares one non-result per attempt, so its rendering must span more than one line; transcript:\n${retriedBootRun.transcript}`,
+ ).toBeGreaterThan(1);
+ expect(
+ lines
+ .slice(tagged[0], tagged[tagged.length - 1] + 1)
+ .filter((line) => !line.startsWith(`[${type}]`)),
+ `every line between the first and last line tagged [${type}] must carry that tag; an untagged line among them means the rendering was emitted per test rather than once at the end, which is what lets it interleave with the terminal reporter; transcript:\n${retriedBootRun.transcript}`,
+ ).toEqual([]);
+ });
+});
diff --git a/packages/app/tests/meta/worker-server-budget.test.ts b/packages/app/tests/meta/worker-server-budget.test.ts
index 750099dba..0cdaf5e84 100644
--- a/packages/app/tests/meta/worker-server-budget.test.ts
+++ b/packages/app/tests/meta/worker-server-budget.test.ts
@@ -1659,6 +1659,40 @@ function drainGuardFindings(source: string): string[] {
return findings;
}
+const SETUP_NON_RESULT_DECLARER = 'declareSetupNonResult';
+const SETUP_NON_RESULT_DECLARATION = ' declareSetupNonResult(base.info(), reason);\n';
+const DRAIN_BLOCK_OPENER = ' let drainFailure: string | undefined;\n';
+
+function setupNonResultDeclarationFindings(source: string): string[] {
+ const sourceFile = parseBudgetSource(source);
+ const body = workerServerFixtureBody(sourceFile);
+ if (body === undefined) return ['scope-missing'];
+ const drains = budgetCallsWithin(body, DRAIN_CALLEE);
+ if (drains.length !== 1) return ['drain-call-count'];
+ const drain = drains[0] as CallExpression;
+ const thrown = body
+ .getDescendantsOfKind(SyntaxKind.ThrowStatement)
+ .find((statement) => statement.getStart() > drain.getStart());
+ if (thrown === undefined) return ['drain-rethrow-missing'];
+
+ const declarations = budgetCallsWithin(body, SETUP_NON_RESULT_DECLARER);
+ if (declarations.length === 0) return ['declaration-missing'];
+ const findings = declarations.length === 1 ? [] : ['declaration-count'];
+ for (const declaration of declarations) {
+ if (declaration.getStart() < drain.getEnd()) {
+ findings.push('declaration-before-drain');
+ } else if (declaration.getEnd() > thrown.getStart()) {
+ findings.push('declaration-after-throw');
+ } else if (
+ declaration.getParentIfKind(SyntaxKind.ExpressionStatement)?.getParent() !==
+ thrown.getParent()
+ ) {
+ findings.push('declaration-off-the-rethrow-path');
+ }
+ }
+ return findings;
+}
+
function checkBudgetWiringSite(
sourceFile: SourceFile,
spec: BudgetWiringSite,
@@ -2161,6 +2195,37 @@ describe('worker-server fixture budget wiring', () => {
).toEqual(expect.arrayContaining(['drain-drops-reason', 'drain-drops-tail']));
});
+ test('the setup non-result is declared once, after the drain and unconditionally on the rethrow path', () => {
+ expect(
+ setupNonResultDeclarationFindings(fixtureSource()),
+ 'the shipped fixture must declare the setup non-result exactly once, after the drain and as a statement beside the rethrow, or the controls below prove nothing',
+ ).toEqual([]);
+
+ const undeclared = mutatedFixtureSource(SETUP_NON_RESULT_DECLARATION, '');
+ const declaredBeforeDrain = undeclared.replace(
+ DRAIN_BLOCK_OPENER,
+ `${SETUP_NON_RESULT_DECLARATION}${DRAIN_BLOCK_OPENER}`,
+ );
+ expect(
+ declaredBeforeDrain,
+ `this control's drain anchor no longer matches ${FIXTURE_SOURCE_PATH}, so the assertion below would report the ordering it guards as broken when what actually went stale is the anchor`,
+ ).not.toBe(undeclared);
+ expect(
+ setupNonResultDeclarationFindings(declaredBeforeDrain),
+ 'declaring before the drain lets base.info() throw where no TestInfo is current, once Playwright has already timed out the fixture slot, and skip the reap of the detached dev server and the removal of its dirs',
+ ).toEqual(['declaration-before-drain']);
+
+ expect(
+ setupNonResultDeclarationFindings(
+ mutatedFixtureSource(
+ SETUP_NON_RESULT_DECLARATION,
+ ' if (openedServerLog !== undefined) declareSetupNonResult(base.info(), reason);\n',
+ ),
+ ),
+ 'a declaration some setup branches skip leaves those failures declared as executed assertion failures while the readiness branch the runtime check drives stays green',
+ ).toEqual(['declaration-off-the-rethrow-path']);
+ });
+
test('the seed-copy catch calls the rollback and still re-raises the copy failure itself', () => {
expect(
viteRollbackWiringFindings(serverProcessSource()),
diff --git a/packages/app/tests/stress/_helpers/boot-verdict/assertion-failure.verdict-case.ts b/packages/app/tests/stress/_helpers/boot-verdict/assertion-failure.verdict-case.ts
new file mode 100644
index 000000000..8898773a8
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/boot-verdict/assertion-failure.verdict-case.ts
@@ -0,0 +1,9 @@
+import { expect } from '@playwright/test';
+import { recordBodyRan, bootedTest as test } from './verdict-fixture.ts';
+
+test('a test whose worker server booted and whose body fails an assertion', async ({
+ workerServer,
+}) => {
+ recordBodyRan('assertion-failure', workerServer.baseURL);
+ expect('observed value').toBe('required value');
+});
diff --git a/packages/app/tests/stress/_helpers/boot-verdict/boot-prerequisite.verdict-case.ts b/packages/app/tests/stress/_helpers/boot-verdict/boot-prerequisite.verdict-case.ts
new file mode 100644
index 000000000..a2e38e5ef
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/boot-verdict/boot-prerequisite.verdict-case.ts
@@ -0,0 +1,5 @@
+import { recordBodyRan, bootFailureTest as test } from './verdict-fixture.ts';
+
+test('a test whose worker server never finishes booting', async ({ workerServer }) => {
+ recordBodyRan('boot-prerequisite', workerServer.baseURL);
+});
diff --git a/packages/app/tests/stress/_helpers/boot-verdict/inject-boot-failure.ts b/packages/app/tests/stress/_helpers/boot-verdict/inject-boot-failure.ts
new file mode 100644
index 000000000..274feecd2
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/boot-verdict/inject-boot-failure.ts
@@ -0,0 +1,9 @@
+import { appendFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+const runDir = process.env.OK_BOOT_VERDICT_RUN_DIR;
+if (runDir !== undefined) {
+ appendFileSync(join(runDir, 'boot-failure-injected'), `${process.pid}\n`);
+}
+
+throw new Error('OK_BOOT_VERDICT: injected dev-server boot failure before the server could bind');
diff --git a/packages/app/tests/stress/_helpers/boot-verdict/passing-control.verdict-case.ts b/packages/app/tests/stress/_helpers/boot-verdict/passing-control.verdict-case.ts
new file mode 100644
index 000000000..566d21944
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/boot-verdict/passing-control.verdict-case.ts
@@ -0,0 +1,7 @@
+import { expect } from '@playwright/test';
+import { recordBodyRan, bootedTest as test } from './verdict-fixture.ts';
+
+test('a test whose worker server booted and whose body passes', async ({ workerServer }) => {
+ recordBodyRan('passing-control', workerServer.baseURL);
+ expect('required value').toBe('required value');
+});
diff --git a/packages/app/tests/stress/_helpers/boot-verdict/recovered-setup.verdict-case.ts b/packages/app/tests/stress/_helpers/boot-verdict/recovered-setup.verdict-case.ts
new file mode 100644
index 000000000..1bd80f432
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/boot-verdict/recovered-setup.verdict-case.ts
@@ -0,0 +1,5 @@
+import { recordBodyRan, setupRecoveredOnRetryTest as test } from './verdict-fixture.ts';
+
+test('a test whose setup completes only on its retry', async ({ workerServer }) => {
+ recordBodyRan('recovered-setup', workerServer.baseURL);
+});
diff --git a/packages/app/tests/stress/_helpers/boot-verdict/refuse-pid-signals.ts b/packages/app/tests/stress/_helpers/boot-verdict/refuse-pid-signals.ts
new file mode 100644
index 000000000..6572c3c3d
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/boot-verdict/refuse-pid-signals.ts
@@ -0,0 +1,16 @@
+import { appendFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+const runDir = process.env.OK_BOOT_VERDICT_RUN_DIR;
+
+process.kill = (pid: number, signal?: string | number): true => {
+ if (runDir !== undefined) {
+ appendFileSync(
+ join(runDir, 'refused-signals.jsonl'),
+ `${JSON.stringify({ caller: process.pid, worker: process.env.TEST_WORKER_INDEX, pid, signal })}\n`,
+ );
+ }
+ throw Object.assign(new Error('the boot-verdict run refuses every pid-addressed signal'), {
+ code: 'ESRCH',
+ });
+};
diff --git a/packages/app/tests/stress/_helpers/boot-verdict/verdict-fixture.ts b/packages/app/tests/stress/_helpers/boot-verdict/verdict-fixture.ts
new file mode 100644
index 000000000..d2ea8b9e0
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/boot-verdict/verdict-fixture.ts
@@ -0,0 +1,78 @@
+import { appendFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import type { Browser } from '@playwright/test';
+import { test as stressTest, type WorkerServer } from '../fixtures.ts';
+import { declareSetupNonResult } from '../setup-non-result.ts';
+
+const INJECT_BOOT_FAILURE = pathToFileURL(join(import.meta.dirname, 'inject-boot-failure.ts')).href;
+
+function requireRunDir(): string {
+ const runDir = process.env.OK_BOOT_VERDICT_RUN_DIR;
+ if (runDir === undefined) {
+ throw new Error(
+ 'OK_BOOT_VERDICT_RUN_DIR is unset; run these cases through verdict-run.config.ts',
+ );
+ }
+ return runDir;
+}
+
+function unlaunchedBrowser(): Browser {
+ return new Proxy(Object.create(null) as Browser, {
+ get(_target, member) {
+ throw new Error(`the boot-verdict cases launch no browser, yet ${String(member)} was read`);
+ },
+ });
+}
+
+function undialledWorkerServer(): WorkerServer {
+ return { port: 0, baseURL: 'http://127.0.0.1:0', contentDir: requireRunDir() };
+}
+
+const verdictTest = stressTest.extend({
+ browser: [
+ // biome-ignore lint/correctness/noEmptyPattern: Playwright fixture contract
+ async ({}, use) => {
+ await use(unlaunchedBrowser());
+ },
+ { scope: 'worker' },
+ ],
+});
+
+export const bootFailureTest = verdictTest.extend({
+ workerServerEnv: [
+ { NODE_OPTIONS: `${process.env.NODE_OPTIONS ?? ''} --import=${INJECT_BOOT_FAILURE}` },
+ { scope: 'worker' },
+ ],
+});
+
+export const bootedTest = verdictTest.extend({
+ workerServer: [
+ // biome-ignore lint/correctness/noEmptyPattern: Playwright fixture contract
+ async ({}, use) => {
+ await use(undialledWorkerServer());
+ },
+ { scope: 'worker' },
+ ],
+});
+
+export const setupRecoveredOnRetryTest = verdictTest.extend({
+ workerServer: [
+ // biome-ignore lint/correctness/noEmptyPattern: Playwright fixture contract
+ async ({}, use) => {
+ const testInfo = verdictTest.info();
+ if (testInfo.retry === 0) {
+ const reason =
+ 'OK_BOOT_VERDICT: the first attempt declares its setup incomplete before dialling any server';
+ declareSetupNonResult(testInfo, reason);
+ throw new Error(reason);
+ }
+ await use(undialledWorkerServer());
+ },
+ { scope: 'worker' },
+ ],
+});
+
+export function recordBodyRan(caseName: string, detail = ''): void {
+ appendFileSync(join(requireRunDir(), `body-ran.${caseName}`), `${process.pid} ${detail}\n`);
+}
diff --git a/packages/app/tests/stress/_helpers/boot-verdict/verdict-run.config.ts b/packages/app/tests/stress/_helpers/boot-verdict/verdict-run.config.ts
new file mode 100644
index 000000000..908e66b52
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/boot-verdict/verdict-run.config.ts
@@ -0,0 +1,30 @@
+import { join } from 'node:path';
+import { defineConfig, type ReporterDescription } from '@playwright/test';
+import stock from '../../../../playwright.config.ts';
+
+const runDir = process.env.OK_BOOT_VERDICT_RUN_DIR;
+const livenessBoundMs = Number(process.env.OK_BOOT_VERDICT_LIVENESS_BOUND_MS);
+if (runDir === undefined || !Number.isFinite(livenessBoundMs) || livenessBoundMs <= 0) {
+ throw new Error(
+ 'verdict-run.config.ts needs OK_BOOT_VERDICT_RUN_DIR and a positive OK_BOOT_VERDICT_LIVENESS_BOUND_MS',
+ );
+}
+
+function stockReporters(): ReporterDescription[] {
+ if (!Array.isArray(stock.reporter)) {
+ throw new Error(
+ `the stock playwright config declares its reporters as ${JSON.stringify(stock.reporter)}, not as a list this run can extend`,
+ );
+ }
+ return stock.reporter;
+}
+
+export default defineConfig({
+ ...stock,
+ testDir: import.meta.dirname,
+ testMatch: /\.verdict-case\.ts$/,
+ globalSetup: [],
+ globalTimeout: livenessBoundMs,
+ outputDir: join(runDir, 'test-results'),
+ reporter: [...stockReporters(), ['json', { outputFile: join(runDir, 'results.json') }]],
+});
diff --git a/packages/app/tests/stress/_helpers/fixtures.ts b/packages/app/tests/stress/_helpers/fixtures.ts
index 0a98ac20d..7dd4ccb13 100644
--- a/packages/app/tests/stress/_helpers/fixtures.ts
+++ b/packages/app/tests/stress/_helpers/fixtures.ts
@@ -18,6 +18,7 @@ import {
tailServerLog,
waitForHttpReady,
} from './server-process.ts';
+import { declareSetupNonResult } from './setup-non-result.ts';
import { removeAllDuringTeardown, runTeardownPhases } from './teardown-fs.ts';
interface ProblemError extends Error {
@@ -436,6 +437,7 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
openedServerLog === undefined
? '--- no dev server log was open when this failure ended the setup ---'
: `--- dev server log tail (${openedServerLog.path}) ---\n${tailServerLog(openedServerLog)}`;
+ declareSetupNonResult(base.info(), reason);
throw new Error(
`${reason}${borrowed === undefined ? '' : `\n${borrowed}`}${drainFailure === undefined ? '' : `\n${drainFailure}`}\n${tail}`,
);
diff --git a/packages/app/tests/stress/_helpers/server-process.ts b/packages/app/tests/stress/_helpers/server-process.ts
index 4ace9b9a4..77f69815b 100644
--- a/packages/app/tests/stress/_helpers/server-process.ts
+++ b/packages/app/tests/stress/_helpers/server-process.ts
@@ -101,7 +101,6 @@ export async function checkCollabSync(
url: `ws://${loopbackHost === '::1' ? '[::1]' : '127.0.0.1'}:${port}/collab`,
name: SYSTEM_DOC_NAME,
document: doc,
- connect: false,
});
try {
await new Promise<void>((resolve, reject) => {
diff --git a/packages/app/tests/stress/_helpers/setup-non-result-reporter.ts b/packages/app/tests/stress/_helpers/setup-non-result-reporter.ts
new file mode 100644
index 000000000..99a826c17
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/setup-non-result-reporter.ts
@@ -0,0 +1,50 @@
+import { relative } from 'node:path';
+import { stdout } from 'node:process';
+import { fileURLToPath } from 'node:url';
+import type { FullConfig, Reporter, TestCase, TestResult } from '@playwright/test/reporter';
+import { declaredSetupNonResultReasons, SETUP_NON_RESULT_ANNOTATION } from './setup-non-result.ts';
+
+export const SETUP_NON_RESULT_REPORTER = fileURLToPath(import.meta.url);
+
+interface DeclaredSetupNonResult {
+ test: TestCase;
+ retry: number;
+ reasons: Array<string | undefined>;
+}
+
+export default class SetupNonResultReporter implements Reporter {
+ private rootDir = '';
+ private readonly declared: DeclaredSetupNonResult[] = [];
+
+ printsToStdio(): boolean {
+ return false;
+ }
+
+ onBegin(config: FullConfig): void {
+ this.rootDir = config.rootDir;
+ }
+
+ onTestEnd(test: TestCase, result: TestResult): void {
+ const reasons = declaredSetupNonResultReasons(result);
+ if (reasons.length > 0) this.declared.push({ test, retry: result.retry, reasons });
+ }
+
+ onEnd(): void {
+ if (this.declared.length === 0) return;
+ const tag = `[${SETUP_NON_RESULT_ANNOTATION}]`;
+ const lines = [
+ `${tag} ${this.declared.length} result(s) in this run are declared setup non-results: the test body never ran because its setup did not complete, so they are not assertion failures. Only declared non-results are listed, so a result's absence from this block does not show that its test body ran. Each entry ends with its test's final outcome, which this declaration does not change.`,
+ ];
+ for (const { test, retry, reasons } of this.declared) {
+ const where = `${relative(this.rootDir, test.location.file)}:${test.location.line}:${test.location.column}`;
+ lines.push(
+ `${tag} ${where} › ${test.title}${retry > 0 ? ` (retry #${retry})` : ''}; test outcome: ${test.outcome()}`,
+ );
+ for (const reason of reasons) {
+ if (reason === undefined) continue;
+ for (const line of reason.split('\n')) lines.push(`${tag} ${line}`);
+ }
+ }
+ stdout.write(`\n${lines.join('\n')}\n`);
+ }
+}
diff --git a/packages/app/tests/stress/_helpers/setup-non-result.ts b/packages/app/tests/stress/_helpers/setup-non-result.ts
new file mode 100644
index 000000000..43a12e763
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/setup-non-result.ts
@@ -0,0 +1,14 @@
+import type { TestInfo } from '@playwright/test';
+import type { TestResult } from '@playwright/test/reporter';
+
+export const SETUP_NON_RESULT_ANNOTATION = 'ok:setup-non-result';
+
+export function declareSetupNonResult(testInfo: TestInfo, reason: string): void {
+ testInfo.annotations.push({ type: SETUP_NON_RESULT_ANNOTATION, description: reason });
+}
+
+export function declaredSetupNonResultReasons(result: TestResult): Array<string | undefined> {
+ return result.annotations
+ .filter((annotation) => annotation.type === SETUP_NON_RESULT_ANNOTATION)
+ .map((annotation) => annotation.description);
+}
diff --git a/packages/app/tsconfig.json b/packages/app/tsconfig.json
index 5938e4114..87c76b29a 100644
--- a/packages/app/tsconfig.json
+++ b/packages/app/tsconfig.json
@@ -88,6 +88,7 @@
"tests/half-open-forced-close.test-helper.ts",
"tests/integration/audit-barrier.test-helper.ts",
"tests/stress/_helpers/acp-catalog.ts",
+ "tests/stress/_helpers/boot-verdict",
"tests/stress/_helpers/editor-state.ts",
"tests/stress/_helpers/image.ts",
"tests/stress/_helpers/theme-fade.ts"