Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Class Playwright setup non-results as not-run (#7807)
Nick Gomez4792ffee6033
diff --git a/packages/app/tests/meta/setup-non-result-reporter.test.ts b/packages/app/tests/meta/setup-non-result-reporter.test.tsnew file mode 100644index 000000000..6e10b1b6a--- /dev/null+++ b/packages/app/tests/meta/setup-non-result-reporter.test.ts@@ -0,0 +1,198 @@+import { Buffer } from 'node:buffer';+import { stdout } from 'node:process';+import type { FullConfig, TestCase, TestResult } from '@playwright/test/reporter';+import { describe, expect, test, vi } from 'vitest';+import { SETUP_NON_RESULT_ANNOTATION } from '../stress/_helpers/setup-non-result.ts';+import SetupNonResultReporter from '../stress/_helpers/setup-non-result-reporter.ts';++const ROOT_DIR = '/synthetic-project';+const SUMMARY_PREFIX = `[${SETUP_NON_RESULT_ANNOTATION}] summary `;++interface Attempt {+ status: TestResult['status'];+ declared?: string;+}++interface ScenarioTest {+ title: string;+ attempts: readonly Attempt[];+}++interface Scenario {+ tests: readonly ScenarioTest[];+ runErrors?: number;+}++function playwrightOutcome(results: readonly TestResult[]): ReturnType<TestCase['outcome']> {+ const failed = results.filter(({ status }) => status !== 'passed' && status !== 'skipped');+ const passed = results.filter(({ status }) => status === 'passed');+ if (failed.length === 0) return passed.length === 0 ? 'skipped' : 'expected';+ return passed.length === 0 ? 'unexpected' : 'flaky';+}++function syntheticTestCase(title: string, line: number, results: TestResult[]): TestCase {+ const testCase: Pick<TestCase, 'title' | 'location' | 'results' | 'outcome'> = {+ title,+ location: { file: `${ROOT_DIR}/tests/stress/synthetic.e2e.ts`, line, column: 1 },+ results,+ outcome: () => playwrightOutcome(results),+ };+ return testCase as TestCase;+}++function syntheticResult({ status, declared }: Attempt, retry: number): TestResult {+ const result: Pick<TestResult, 'status' | 'retry' | 'annotations'> = {+ status,+ retry,+ annotations:+ declared === undefined ? [] : [{ type: SETUP_NON_RESULT_ANNOTATION, description: declared }],+ };+ return result as TestResult;+}++function reporterOutput({ tests, runErrors = 0 }: Scenario): string[] {+ const reporter = new SetupNonResultReporter();+ const config: Pick<FullConfig, 'rootDir'> = { rootDir: ROOT_DIR };+ reporter.onBegin(config as FullConfig);+ tests.forEach(({ title, attempts }, index) => {+ const results: TestResult[] = [];+ const testCase = syntheticTestCase(title, index + 1, results);+ attempts.forEach((attempt, retry) => {+ const result = syntheticResult(attempt, retry);+ results.push(result);+ reporter.onTestEnd(testCase, result);+ });+ });+ for (let error = 0; error < runErrors; error += 1) reporter.onError();+ const writes: string[] = [];+ const write = vi.spyOn(stdout, 'write').mockImplementation((chunk: string | Uint8Array) => {+ writes.push(typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8'));+ return true;+ });+ try {+ reporter.onEnd();+ } finally {+ write.mockRestore();+ }+ return writes.join('').split('\n');+}++function summaryLines(lines: readonly string[]): string[] {+ return lines.filter((line) => line.startsWith(SUMMARY_PREFIX));+}++function summary(scenario: Scenario): unknown {+ const lines = summaryLines(reporterOutput(scenario));+ expect(lines).toHaveLength(1);+ return JSON.parse(lines[0].slice(SUMMARY_PREFIX.length));+}++const BOOT_REASON = 'worker server did not report ready before its boot bound';+const RETRY_BOOT_REASON = 'worker server exited before it reported ready';++describe('SetupNonResultReporter summary', () => {+ test('a declared boot failure alone is the only failure the summary counts', () => {+ expect(+ summary({+ tests: [+ { title: 'boot', attempts: [{ status: 'failed', declared: BOOT_REASON }] },+ { title: 'passing neighbour', attempts: [{ status: 'passed' }] },+ ],+ }),+ ).toEqual({ declaredUnexpected: 1, unexpected: 1, runErrors: 0, reasons: [BOOT_REASON] });+ });++ test('a declared boot beside an undeclared assertion failure counts both failures', () => {+ expect(+ summary({+ tests: [+ { title: 'boot', attempts: [{ status: 'failed', declared: BOOT_REASON }] },+ { title: 'assertion', attempts: [{ status: 'failed' }] },+ ],+ }),+ ).toEqual({ declaredUnexpected: 1, unexpected: 2, runErrors: 0, reasons: [BOOT_REASON] });+ });++ test('a boot retried with every attempt declared counts each attempt as declared', () => {+ expect(+ summary({+ tests: [+ {+ title: 'boot',+ attempts: [+ { status: 'timedOut', declared: BOOT_REASON },+ { status: 'failed', declared: RETRY_BOOT_REASON },+ ],+ },+ ],+ }),+ ).toEqual({+ declaredUnexpected: 2,+ unexpected: 2,+ runErrors: 0,+ reasons: [BOOT_REASON, RETRY_BOOT_REASON],+ });+ });++ test('a declared first attempt followed by a failed attempt whose body ran is not all declared', () => {+ expect(+ summary({+ tests: [+ {+ title: 'boot then assertion',+ attempts: [{ status: 'failed', declared: BOOT_REASON }, { status: 'failed' }],+ },+ ],+ }),+ ).toEqual({ declaredUnexpected: 1, unexpected: 2, runErrors: 0, reasons: [BOOT_REASON] });+ });++ test('a declared attempt that recovers on retry counts only the declared failed attempt', () => {+ expect(+ summary({+ tests: [+ {+ title: 'boot then pass',+ attempts: [{ status: 'failed', declared: BOOT_REASON }, { status: 'passed' }],+ },+ ],+ }),+ ).toEqual({ declaredUnexpected: 1, unexpected: 1, runErrors: 0, reasons: [BOOT_REASON] });+ });++ test('a run-level error is counted beside a declared boot failure', () => {+ expect(+ summary({+ tests: [{ title: 'boot', attempts: [{ status: 'failed', declared: BOOT_REASON }] }],+ runErrors: 1,+ }),+ ).toEqual({ declaredUnexpected: 1, unexpected: 1, runErrors: 1, reasons: [BOOT_REASON] });+ });++ test("each declared reason line follows its own test's entry, not only the summary", () => {+ const lines = reporterOutput({+ tests: [+ { title: 'passing neighbour', attempts: [{ status: 'passed' }] },+ {+ title: 'boot',+ attempts: [{ status: 'failed', declared: `${BOOT_REASON}\nsecond line` }],+ },+ ],+ });+ const tag = `[${SETUP_NON_RESULT_ANNOTATION}]`;+ const entry = lines.findIndex((line) => line.startsWith(tag) && line.includes(' › boot;'));+ expect(entry, `no entry line names the boot test; lines:\n${lines.join('\n')}`).not.toBe(-1);+ expect(lines.slice(entry + 1, entry + 3)).toEqual([+ `${tag} ${BOOT_REASON}`,+ `${tag} second line`,+ ]);+ });++ test('a run with no declared non-result writes no summary', () => {+ expect(+ summaryLines(+ reporterOutput({ tests: [{ title: 'assertion', attempts: [{ status: 'failed' }] }] }),+ ),+ ).toEqual([]);+ });+});diff --git a/packages/app/tests/meta/worker-server-boot-verdict.test.ts b/packages/app/tests/meta/worker-server-boot-verdict.test.tsindex 64a59fa68..64d99b20d 100644--- a/packages/app/tests/meta/worker-server-boot-verdict.test.ts+++ b/packages/app/tests/meta/worker-server-boot-verdict.test.ts@@ -269,6 +269,16 @@ function linesTaggedWith(run: VerdictRun, type: string): string[] {return run.transcript.split(/\r?\n/).filter((line) => line.startsWith(`[${type}]`));}+function renderedSummary(run: VerdictRun, type: string): unknown {+ const prefix = `[${type}] summary `;+ const summaries = linesTaggedWith(run, type).filter((line) => line.startsWith(prefix));+ expect(+ summaries,+ `verdict run [${run.cases.join(', ')}] must render exactly one summary line tagged [${type}]; transcript:\n${run.transcript}`,+ ).toHaveLength(1);+ return JSON.parse((summaries[0] as string).slice(prefix.length));+}+function renderedNonResultLines(type: string): string[] {const rendered = linesTaggedWith(bootRun, type);expect(@@ -442,6 +452,40 @@ describe('the stress reporters render a declared setup non-result under its decl).toEqual([]);});+ test('each run summarises its declared non-results with every failed result accounted for and no run error', () => {+ const type = nonResultAnnotationType();+ const reasonsOf = (run: VerdictRun, caseName: string) =>+ soleTestOf(run, caseName).test.results.flatMap((result) => declaredReasons(result, type));++ expect(+ renderedSummary(bootRun, type),+ 'the boot attempt is the only failed result in its run and is declared; the passing control is not counted',+ ).toEqual({+ declaredUnexpected: 1,+ unexpected: 1,+ runErrors: 0,+ reasons: reasonsOf(bootRun, BOOT_CASE),+ });+ expect(+ renderedSummary(retriedBootRun, type),+ 'both failed attempts of the retried boot case are declared',+ ).toEqual({+ declaredUnexpected: 2,+ unexpected: 2,+ runErrors: 0,+ reasons: reasonsOf(retriedBootRun, BOOT_CASE),+ });+ expect(+ renderedSummary(recoveredRun, type),+ 'only the declared first attempt of the recovered case failed',+ ).toEqual({+ declaredUnexpected: 1,+ unexpected: 1,+ runErrors: 0,+ reasons: reasonsOf(recoveredRun, RECOVERED_CASE),+ });+ });+test('a run whose only failure is an executed assertion failure renders no line under the non-result type', () => {const type = nonResultAnnotationType();renderedNonResultLines(type);diff --git a/packages/app/tests/stress/_helpers/setup-non-result-reporter.ts b/packages/app/tests/stress/_helpers/setup-non-result-reporter.tsindex 99a826c17..59991fcc8 100644--- a/packages/app/tests/stress/_helpers/setup-non-result-reporter.ts+++ b/packages/app/tests/stress/_helpers/setup-non-result-reporter.ts@@ -10,11 +10,14 @@ interface DeclaredSetupNonResult {test: TestCase;retry: number;reasons: Array<string | undefined>;+ failed: boolean;}export default class SetupNonResultReporter implements Reporter {private rootDir = '';private readonly declared: DeclaredSetupNonResult[] = [];+ private failedResults = 0;+ private runErrors = 0;printsToStdio(): boolean {return false;@@ -25,15 +28,30 @@ export default class SetupNonResultReporter implements Reporter {}onTestEnd(test: TestCase, result: TestResult): void {+ const failed = result.status !== 'passed' && result.status !== 'skipped';+ if (failed) this.failedResults += 1;const reasons = declaredSetupNonResultReasons(result);- if (reasons.length > 0) this.declared.push({ test, retry: result.retry, reasons });+ if (reasons.length > 0) this.declared.push({ test, retry: result.retry, reasons, failed });+ }++ onError(): void {+ this.runErrors += 1;}onEnd(): void {if (this.declared.length === 0) return;const tag = `[${SETUP_NON_RESULT_ANNOTATION}]`;+ const declaredUnexpected = this.declared.filter(({ failed }) => failed);const lines = [`${tag} ${this.declared.length} result(s) in this run are declared setup non-results: the test body never ran because its setup did not complete, so they are not assertion failures. Only declared non-results are listed, so a result's absence from this block does not show that its test body ran. Each entry ends with its test's final outcome, which this declaration does not change.`,+ `${tag} summary ${JSON.stringify({+ declaredUnexpected: declaredUnexpected.length,+ unexpected: this.failedResults,+ runErrors: this.runErrors,+ reasons: declaredUnexpected.flatMap(({ reasons }) =>+ reasons.filter((reason): reason is string => reason !== undefined),+ ),+ })}`,];for (const { test, retry, reasons } of this.declared) {const where = `${relative(this.rootDir, test.location.file)}:${test.location.line}:${test.location.column}`;