Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Class Playwright setup non-results as not-run (#7807)

Nick Gomez4792ffee6033
diff --git a/packages/app/tests/meta/setup-non-result-reporter.test.ts b/packages/app/tests/meta/setup-non-result-reporter.test.ts
new file mode 100644
index 000000000..6e10b1b6a
--- /dev/null
+++ b/packages/app/tests/meta/setup-non-result-reporter.test.ts
@@ -0,0 +1,198 @@
+import { Buffer } from 'node:buffer';
+import { stdout } from 'node:process';
+import type { FullConfig, TestCase, TestResult } from '@playwright/test/reporter';
+import { describe, expect, test, vi } from 'vitest';
+import { SETUP_NON_RESULT_ANNOTATION } from '../stress/_helpers/setup-non-result.ts';
+import SetupNonResultReporter from '../stress/_helpers/setup-non-result-reporter.ts';
+
+const ROOT_DIR = '/synthetic-project';
+const SUMMARY_PREFIX = `[${SETUP_NON_RESULT_ANNOTATION}] summary `;
+
+interface Attempt {
+ status: TestResult['status'];
+ declared?: string;
+}
+
+interface ScenarioTest {
+ title: string;
+ attempts: readonly Attempt[];
+}
+
+interface Scenario {
+ tests: readonly ScenarioTest[];
+ runErrors?: number;
+}
+
+function playwrightOutcome(results: readonly TestResult[]): ReturnType<TestCase['outcome']> {
+ const failed = results.filter(({ status }) => status !== 'passed' && status !== 'skipped');
+ const passed = results.filter(({ status }) => status === 'passed');
+ if (failed.length === 0) return passed.length === 0 ? 'skipped' : 'expected';
+ return passed.length === 0 ? 'unexpected' : 'flaky';
+}
+
+function syntheticTestCase(title: string, line: number, results: TestResult[]): TestCase {
+ const testCase: Pick<TestCase, 'title' | 'location' | 'results' | 'outcome'> = {
+ title,
+ location: { file: `${ROOT_DIR}/tests/stress/synthetic.e2e.ts`, line, column: 1 },
+ results,
+ outcome: () => playwrightOutcome(results),
+ };
+ return testCase as TestCase;
+}
+
+function syntheticResult({ status, declared }: Attempt, retry: number): TestResult {
+ const result: Pick<TestResult, 'status' | 'retry' | 'annotations'> = {
+ status,
+ retry,
+ annotations:
+ declared === undefined ? [] : [{ type: SETUP_NON_RESULT_ANNOTATION, description: declared }],
+ };
+ return result as TestResult;
+}
+
+function reporterOutput({ tests, runErrors = 0 }: Scenario): string[] {
+ const reporter = new SetupNonResultReporter();
+ const config: Pick<FullConfig, 'rootDir'> = { rootDir: ROOT_DIR };
+ reporter.onBegin(config as FullConfig);
+ tests.forEach(({ title, attempts }, index) => {
+ const results: TestResult[] = [];
+ const testCase = syntheticTestCase(title, index + 1, results);
+ attempts.forEach((attempt, retry) => {
+ const result = syntheticResult(attempt, retry);
+ results.push(result);
+ reporter.onTestEnd(testCase, result);
+ });
+ });
+ for (let error = 0; error < runErrors; error += 1) reporter.onError();
+ const writes: string[] = [];
+ const write = vi.spyOn(stdout, 'write').mockImplementation((chunk: string | Uint8Array) => {
+ writes.push(typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8'));
+ return true;
+ });
+ try {
+ reporter.onEnd();
+ } finally {
+ write.mockRestore();
+ }
+ return writes.join('').split('\n');
+}
+
+function summaryLines(lines: readonly string[]): string[] {
+ return lines.filter((line) => line.startsWith(SUMMARY_PREFIX));
+}
+
+function summary(scenario: Scenario): unknown {
+ const lines = summaryLines(reporterOutput(scenario));
+ expect(lines).toHaveLength(1);
+ return JSON.parse(lines[0].slice(SUMMARY_PREFIX.length));
+}
+
+const BOOT_REASON = 'worker server did not report ready before its boot bound';
+const RETRY_BOOT_REASON = 'worker server exited before it reported ready';
+
+describe('SetupNonResultReporter summary', () => {
+ test('a declared boot failure alone is the only failure the summary counts', () => {
+ expect(
+ summary({
+ tests: [
+ { title: 'boot', attempts: [{ status: 'failed', declared: BOOT_REASON }] },
+ { title: 'passing neighbour', attempts: [{ status: 'passed' }] },
+ ],
+ }),
+ ).toEqual({ declaredUnexpected: 1, unexpected: 1, runErrors: 0, reasons: [BOOT_REASON] });
+ });
+
+ test('a declared boot beside an undeclared assertion failure counts both failures', () => {
+ expect(
+ summary({
+ tests: [
+ { title: 'boot', attempts: [{ status: 'failed', declared: BOOT_REASON }] },
+ { title: 'assertion', attempts: [{ status: 'failed' }] },
+ ],
+ }),
+ ).toEqual({ declaredUnexpected: 1, unexpected: 2, runErrors: 0, reasons: [BOOT_REASON] });
+ });
+
+ test('a boot retried with every attempt declared counts each attempt as declared', () => {
+ expect(
+ summary({
+ tests: [
+ {
+ title: 'boot',
+ attempts: [
+ { status: 'timedOut', declared: BOOT_REASON },
+ { status: 'failed', declared: RETRY_BOOT_REASON },
+ ],
+ },
+ ],
+ }),
+ ).toEqual({
+ declaredUnexpected: 2,
+ unexpected: 2,
+ runErrors: 0,
+ reasons: [BOOT_REASON, RETRY_BOOT_REASON],
+ });
+ });
+
+ test('a declared first attempt followed by a failed attempt whose body ran is not all declared', () => {
+ expect(
+ summary({
+ tests: [
+ {
+ title: 'boot then assertion',
+ attempts: [{ status: 'failed', declared: BOOT_REASON }, { status: 'failed' }],
+ },
+ ],
+ }),
+ ).toEqual({ declaredUnexpected: 1, unexpected: 2, runErrors: 0, reasons: [BOOT_REASON] });
+ });
+
+ test('a declared attempt that recovers on retry counts only the declared failed attempt', () => {
+ expect(
+ summary({
+ tests: [
+ {
+ title: 'boot then pass',
+ attempts: [{ status: 'failed', declared: BOOT_REASON }, { status: 'passed' }],
+ },
+ ],
+ }),
+ ).toEqual({ declaredUnexpected: 1, unexpected: 1, runErrors: 0, reasons: [BOOT_REASON] });
+ });
+
+ test('a run-level error is counted beside a declared boot failure', () => {
+ expect(
+ summary({
+ tests: [{ title: 'boot', attempts: [{ status: 'failed', declared: BOOT_REASON }] }],
+ runErrors: 1,
+ }),
+ ).toEqual({ declaredUnexpected: 1, unexpected: 1, runErrors: 1, reasons: [BOOT_REASON] });
+ });
+
+ test("each declared reason line follows its own test's entry, not only the summary", () => {
+ const lines = reporterOutput({
+ tests: [
+ { title: 'passing neighbour', attempts: [{ status: 'passed' }] },
+ {
+ title: 'boot',
+ attempts: [{ status: 'failed', declared: `${BOOT_REASON}\nsecond line` }],
+ },
+ ],
+ });
+ const tag = `[${SETUP_NON_RESULT_ANNOTATION}]`;
+ const entry = lines.findIndex((line) => line.startsWith(tag) && line.includes(' › boot;'));
+ expect(entry, `no entry line names the boot test; lines:\n${lines.join('\n')}`).not.toBe(-1);
+ expect(lines.slice(entry + 1, entry + 3)).toEqual([
+ `${tag} ${BOOT_REASON}`,
+ `${tag} second line`,
+ ]);
+ });
+
+ test('a run with no declared non-result writes no summary', () => {
+ expect(
+ summaryLines(
+ reporterOutput({ tests: [{ title: 'assertion', attempts: [{ status: 'failed' }] }] }),
+ ),
+ ).toEqual([]);
+ });
+});
diff --git a/packages/app/tests/meta/worker-server-boot-verdict.test.ts b/packages/app/tests/meta/worker-server-boot-verdict.test.ts
index 64a59fa68..64d99b20d 100644
--- a/packages/app/tests/meta/worker-server-boot-verdict.test.ts
+++ b/packages/app/tests/meta/worker-server-boot-verdict.test.ts
@@ -269,6 +269,16 @@ function linesTaggedWith(run: VerdictRun, type: string): string[] {
return run.transcript.split(/\r?\n/).filter((line) => line.startsWith(`[${type}]`));
}
+function renderedSummary(run: VerdictRun, type: string): unknown {
+ const prefix = `[${type}] summary `;
+ const summaries = linesTaggedWith(run, type).filter((line) => line.startsWith(prefix));
+ expect(
+ summaries,
+ `verdict run [${run.cases.join(', ')}] must render exactly one summary line tagged [${type}]; transcript:\n${run.transcript}`,
+ ).toHaveLength(1);
+ return JSON.parse((summaries[0] as string).slice(prefix.length));
+}
+
function renderedNonResultLines(type: string): string[] {
const rendered = linesTaggedWith(bootRun, type);
expect(
@@ -442,6 +452,40 @@ describe('the stress reporters render a declared setup non-result under its decl
).toEqual([]);
});
+ test('each run summarises its declared non-results with every failed result accounted for and no run error', () => {
+ const type = nonResultAnnotationType();
+ const reasonsOf = (run: VerdictRun, caseName: string) =>
+ soleTestOf(run, caseName).test.results.flatMap((result) => declaredReasons(result, type));
+
+ expect(
+ renderedSummary(bootRun, type),
+ 'the boot attempt is the only failed result in its run and is declared; the passing control is not counted',
+ ).toEqual({
+ declaredUnexpected: 1,
+ unexpected: 1,
+ runErrors: 0,
+ reasons: reasonsOf(bootRun, BOOT_CASE),
+ });
+ expect(
+ renderedSummary(retriedBootRun, type),
+ 'both failed attempts of the retried boot case are declared',
+ ).toEqual({
+ declaredUnexpected: 2,
+ unexpected: 2,
+ runErrors: 0,
+ reasons: reasonsOf(retriedBootRun, BOOT_CASE),
+ });
+ expect(
+ renderedSummary(recoveredRun, type),
+ 'only the declared first attempt of the recovered case failed',
+ ).toEqual({
+ declaredUnexpected: 1,
+ unexpected: 1,
+ runErrors: 0,
+ reasons: reasonsOf(recoveredRun, RECOVERED_CASE),
+ });
+ });
+
test('a run whose only failure is an executed assertion failure renders no line under the non-result type', () => {
const type = nonResultAnnotationType();
renderedNonResultLines(type);
diff --git a/packages/app/tests/stress/_helpers/setup-non-result-reporter.ts b/packages/app/tests/stress/_helpers/setup-non-result-reporter.ts
index 99a826c17..59991fcc8 100644
--- a/packages/app/tests/stress/_helpers/setup-non-result-reporter.ts
+++ b/packages/app/tests/stress/_helpers/setup-non-result-reporter.ts
@@ -10,11 +10,14 @@ interface DeclaredSetupNonResult {
test: TestCase;
retry: number;
reasons: Array<string | undefined>;
+ failed: boolean;
}
export default class SetupNonResultReporter implements Reporter {
private rootDir = '';
private readonly declared: DeclaredSetupNonResult[] = [];
+ private failedResults = 0;
+ private runErrors = 0;
printsToStdio(): boolean {
return false;
@@ -25,15 +28,30 @@ export default class SetupNonResultReporter implements Reporter {
}
onTestEnd(test: TestCase, result: TestResult): void {
+ const failed = result.status !== 'passed' && result.status !== 'skipped';
+ if (failed) this.failedResults += 1;
const reasons = declaredSetupNonResultReasons(result);
- if (reasons.length > 0) this.declared.push({ test, retry: result.retry, reasons });
+ if (reasons.length > 0) this.declared.push({ test, retry: result.retry, reasons, failed });
+ }
+
+ onError(): void {
+ this.runErrors += 1;
}
onEnd(): void {
if (this.declared.length === 0) return;
const tag = `[${SETUP_NON_RESULT_ANNOTATION}]`;
+ const declaredUnexpected = this.declared.filter(({ failed }) => failed);
const lines = [
`${tag} ${this.declared.length} result(s) in this run are declared setup non-results: the test body never ran because its setup did not complete, so they are not assertion failures. Only declared non-results are listed, so a result's absence from this block does not show that its test body ran. Each entry ends with its test's final outcome, which this declaration does not change.`,
+ `${tag} summary ${JSON.stringify({
+ declaredUnexpected: declaredUnexpected.length,
+ unexpected: this.failedResults,
+ runErrors: this.runErrors,
+ reasons: declaredUnexpected.flatMap(({ reasons }) =>
+ reasons.filter((reason): reason is string => reason !== undefined),
+ ),
+ })}`,
];
for (const { test, retry, reasons } of this.declared) {
const where = `${relative(this.rootDir, test.location.file)}:${test.location.line}:${test.location.column}`;