Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
fix(reporter): surface reporter errors and fail the run (#41307)
Dmitry Gozman583e5fb9d40b
diff --git a/packages/playwright/src/cli/reportActions.ts b/packages/playwright/src/cli/reportActions.tsindex 7d1e7b94d3..3b8943ebc5 100644--- a/packages/playwright/src/cli/reportActions.ts+++ b/packages/playwright/src/cli/reportActions.ts@@ -43,8 +43,8 @@ export async function mergeReports(reportDir: string | undefined, opts: { [key:if (!reporterDescriptions)reporterDescriptions = [[commonConfig.defaultReporter]];const rootDirOverride = configFile ? config.config.rootDir : undefined;- await merge.createMergedReport(config, dir, reporterDescriptions!, rootDirOverride);- gracefullyProcessExitDoNotHang(0);+ const result = await merge.createMergedReport(config, dir, reporterDescriptions!, rootDirOverride);+ gracefullyProcessExitDoNotHang(result === 'failed' ? 1 : 0);}function resolveReporterOption(reporter?: string): ReporterDescription[] | undefined {diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.tsindex 23429ffbee..31d5df1ed1 100644--- a/packages/playwright/src/reporters/internalReporter.ts+++ b/packages/playwright/src/reporters/internalReporter.ts@@ -29,7 +29,7 @@ import type { FullConfig, FullResult, TestCase, TestError, TestResult, TestStep,export class InternalReporter implements ReporterV2 {- private _reporter: ReporterV2;+ private _reporter: Multiplexer;private _didBegin = false;private _config!: FullConfig;private _startTime: Date | undefined;@@ -43,6 +43,10 @@ export class InternalReporter implements ReporterV2 {return 'v2';}+ hasReporterErrors(): boolean {+ return this._reporter.hasReporterErrors();+ }+onConfigure(config: FullConfig) {this._config = config;this._startTime = new Date();diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.tsindex f9b964164e..03d89f6478 100644--- a/packages/playwright/src/reporters/merge.ts+++ b/packages/playwright/src/reporters/merge.ts@@ -44,7 +44,9 @@ type ReportData = {fullResult: JsonFullResult;};-export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[], rootDirOverride: string | undefined) {+export type MergeResult = 'passed' | 'failed';++export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[], rootDirOverride: string | undefined): Promise<MergeResult> {const reporters = await createReporters(config, 'merge', reporterDescriptions);const multiplexer = new Multiplexer(reporters);const stringPool = new StringInternPool();@@ -110,6 +112,9 @@ export async function createMergedReport(config: FullConfigInternal, dir: string});}await dispatchEvents(eventData.epilogue);+ // The merged report status is intentionally not surfaced as a failure - the+ // merge command only fails when a reporter itself errors.+ return multiplexer.hasReporterErrors() ? 'failed' : 'passed';}const commonEventNames = ['onBlobReportMetadata', 'onConfigure', 'onProject', 'onBegin', 'onEnd'];diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.tsindex 6596a2bd66..e3c5122f83 100644--- a/packages/playwright/src/reporters/multiplexer.ts+++ b/packages/playwright/src/reporters/multiplexer.ts@@ -14,12 +14,15 @@* limitations under the License.*/+import { serializeError } from '../util';+import type { ReportConfigureParams, ReportEndParams, ReporterV2 } from './reporterV2';import type { FullConfig, FullResult, TestCase, TestError, TestResult, TestStep, WorkerInfo } from '../../types/testReporter';import type { test } from '../common';export class Multiplexer implements ReporterV2 {private _reporters: ReporterV2[];+ private _hasReporterErrors = false;constructor(reporters: ReporterV2[]) {this._reporters = reporters;@@ -29,54 +32,58 @@ export class Multiplexer implements ReporterV2 {return 'v2';}+ hasReporterErrors(): boolean {+ return this._hasReporterErrors;+ }+onConfigure(config: FullConfig) {for (const reporter of this._reporters)- wrap(() => reporter.onConfigure?.(config));+ this._wrap(() => reporter.onConfigure?.(config));}onBegin(suite: test.Suite) {for (const reporter of this._reporters)- wrap(() => reporter.onBegin?.(suite));+ this._wrap(() => reporter.onBegin?.(suite));}onTestBegin(test: TestCase, result: TestResult) {for (const reporter of this._reporters)- wrap(() => reporter.onTestBegin?.(test, result));+ this._wrap(() => reporter.onTestBegin?.(test, result));}onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {for (const reporter of this._reporters)- wrap(() => reporter.onStdOut?.(chunk, test, result));+ this._wrap(() => reporter.onStdOut?.(chunk, test, result));}onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) {for (const reporter of this._reporters)- wrap(() => reporter.onStdErr?.(chunk, test, result));+ this._wrap(() => reporter.onStdErr?.(chunk, test, result));}async onTestPaused(test: TestCase, result: TestResult) {for (const reporter of this._reporters)- await wrapAsync(() => reporter.onTestPaused?.(test, result));+ await this._wrapAsync(() => reporter.onTestPaused?.(test, result));}onTestEnd(test: TestCase, result: TestResult) {for (const reporter of this._reporters)- wrap(() => reporter.onTestEnd?.(test, result));+ this._wrap(() => reporter.onTestEnd?.(test, result));}onReportConfigure(params: ReportConfigureParams): void {for (const reporter of this._reporters)- wrap(() => reporter.onReportConfigure?.(params));+ this._wrap(() => reporter.onReportConfigure?.(params));}onReportEnd(params: ReportEndParams): void {for (const reporter of this._reporters)- wrap(() => reporter.onReportEnd?.(params));+ this._wrap(() => reporter.onReportEnd?.(params));}async onEnd(result: FullResult) {for (const reporter of this._reporters) {- const outResult = await wrapAsync(() => reporter.onEnd?.(result));+ const outResult = await this._wrapAsync(() => reporter.onEnd?.(result));if (outResult?.status)result.status = outResult.status;}@@ -85,47 +92,48 @@ export class Multiplexer implements ReporterV2 {async onExit() {for (const reporter of this._reporters)- await wrapAsync(() => reporter.onExit?.());+ await this._wrapAsync(() => reporter.onExit?.());}onError(error: TestError, workerInfo?: WorkerInfo) {for (const reporter of this._reporters)- wrap(() => reporter.onError?.(error, workerInfo));+ this._wrap(() => reporter.onError?.(error, workerInfo), false);}onStepBegin(test: TestCase, result: TestResult, step: TestStep) {for (const reporter of this._reporters)- wrap(() => reporter.onStepBegin?.(test, result, step));+ this._wrap(() => reporter.onStepBegin?.(test, result, step));}onStepEnd(test: TestCase, result: TestResult, step: TestStep) {for (const reporter of this._reporters)- wrap(() => reporter.onStepEnd?.(test, result, step));+ this._wrap(() => reporter.onStepEnd?.(test, result, step));}printsToStdio(): boolean {return this._reporters.some(r => {let prints = false;- wrap(() => prints = r.printsToStdio ? r.printsToStdio() : true);+ this._wrap(() => prints = r.printsToStdio ? r.printsToStdio() : true, false);return prints;});}-}-async function wrapAsync<T>(callback: () => T | Promise<T>) {- try {- return await callback();- } catch (e) {- // eslint-disable-next-line no-console- console.error('Error in reporter', e);+ private _wrap(callback: () => void, redispatch: boolean = true) {+ try {+ callback();+ } catch (e) {+ this._hasReporterErrors = true;+ if (redispatch)+ this.onError(serializeError(e));+ }}-}-function wrap(callback: () => void) {- try {- callback();- } catch (e) {- // eslint-disable-next-line no-console- console.error('Error in reporter', e);+ private async _wrapAsync<T>(callback: () => T | Promise<T>): Promise<T | undefined> {+ try {+ return await callback();+ } catch (e) {+ this._hasReporterErrors = true;+ this.onError(serializeError(e));+ }}}diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.tsindex 1c53cb908c..1a829fdb98 100644--- a/packages/playwright/src/runner/tasks.ts+++ b/packages/playwright/src/runner/tasks.ts@@ -116,7 +116,7 @@ export class TestRun {result(): 'failed' | 'passed' {const hasFailedTests = this.rootSuite?.allTests().some(test => !test.ok());const hasFlakyTests = this.rootSuite?.allTests().some(test => test.outcome() === 'flaky');- return this.hasWorkerErrors || this.hasReachedMaxFailures() || hasFailedTests || (this.config.config.failOnFlakyTests && hasFlakyTests) ? 'failed' : 'passed';+ return this.hasWorkerErrors || this.reporter.hasReporterErrors() || this.hasReachedMaxFailures() || hasFailedTests || (this.config.config.failOnFlakyTests && hasFlakyTests) ? 'failed' : 'passed';}}@@ -146,6 +146,10 @@ async function finishTaskRun(testRun: TestRun, status: FullResult['status']) {if (modifiedResult && modifiedResult.status)status = modifiedResult.status;await testRun.reporter.onExit();+ // A reporter may have thrown during onEnd/onExit (or earlier), which is not+ // reflected in testRun.result() computed above. Fail the run in that case.+ if (status === 'passed' && testRun.reporter.hasReporterErrors())+ status = 'failed';return status;}diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.tsindex 9ded1e17cc..0f3590b0e0 100644--- a/tests/playwright-test/reporter-blob.spec.ts+++ b/tests/playwright-test/reporter-blob.spec.ts@@ -150,6 +150,39 @@ test('should call methods in right order', async ({ runInlineTest, mergeReportsexpect(lines.filter(l => l === 'onExit').length).toBe(1);});+test('should fail merge and report error when reporter throws in onEnd', async ({ runInlineTest, mergeReports }) => {+ const reportDir = test.info().outputPath('blob-report');+ const files = {+ 'throwing-reporter.js': `+ class ThrowingReporter {+ onEnd() {+ throw new Error('Error in merge onEnd!');+ }+ }+ module.exports = ThrowingReporter;+ `,+ 'playwright.config.ts': `+ module.exports = {+ reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]]+ };+ `,+ 'a.test.js': `+ import { test, expect } from '@playwright/test';+ test('math 1', async ({}) => {+ expect(1 + 1).toBe(2);+ });+ `,+ };+ await runInlineTest(files, { shard: `1/2` });+ await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });++ // Merge through the dot reporter so the reporter error is surfaced, and the+ // throwing reporter to fail the merge command.+ const { exitCode, output } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'dot,' + test.info().outputPath('throwing-reporter.js')] });+ expect(exitCode).toBe(1);+ expect(output).toContain('Error in merge onEnd!');+});+test('should merge into html with dependencies', async ({ runInlineTest, mergeReports, showReport, page }) => {const reportDir = test.info().outputPath('blob-report');const files = {diff --git a/tests/playwright-test/reporter-onend.spec.ts b/tests/playwright-test/reporter-onend.spec.tsindex 38665ea2e6..9442318809 100644--- a/tests/playwright-test/reporter-onend.spec.ts+++ b/tests/playwright-test/reporter-onend.spec.ts@@ -38,3 +38,43 @@ test('should override exit code', async ({ runInlineTest }) => {});expect(result.exitCode).toBe(0);});++test('should fail run and report error when reporter throws in onEnd', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'reporter.ts': `+ export default class Reporter {+ async onEnd() {+ throw new Error('Error in onEnd!');+ }+ }+ `,+ 'playwright.config.ts': `module.exports = { reporter: [['dot'], ['./reporter.ts']] };`,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('passing', async () => {});+ `,+ }, { reporter: '' });+ expect(result.exitCode).toBe(1);+ expect(result.passed).toBe(1);+ expect(result.output).toContain('Error in onEnd!');+});++test('should fail run and report error when reporter throws in onTestEnd', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'reporter.ts': `+ export default class Reporter {+ onTestEnd() {+ throw new Error('Error in onTestEnd!');+ }+ }+ `,+ 'playwright.config.ts': `module.exports = { reporter: [['dot'], ['./reporter.ts']] };`,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('passing', async () => {});+ `,+ }, { reporter: '' });+ expect(result.exitCode).toBe(1);+ expect(result.passed).toBe(1);+ expect(result.output).toContain('Error in onTestEnd!');+});