Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix(reporter): surface reporter errors and fail the run (#41307)

Dmitry Gozman583e5fb9d40b
diff --git a/packages/playwright/src/cli/reportActions.ts b/packages/playwright/src/cli/reportActions.ts
index 7d1e7b94d3..3b8943ebc5 100644
--- a/packages/playwright/src/cli/reportActions.ts
+++ b/packages/playwright/src/cli/reportActions.ts
@@ -43,8 +43,8 @@ export async function mergeReports(reportDir: string | undefined, opts: { [key:
if (!reporterDescriptions)
reporterDescriptions = [[commonConfig.defaultReporter]];
const rootDirOverride = configFile ? config.config.rootDir : undefined;
- await merge.createMergedReport(config, dir, reporterDescriptions!, rootDirOverride);
- gracefullyProcessExitDoNotHang(0);
+ const result = await merge.createMergedReport(config, dir, reporterDescriptions!, rootDirOverride);
+ gracefullyProcessExitDoNotHang(result === 'failed' ? 1 : 0);
}
function resolveReporterOption(reporter?: string): ReporterDescription[] | undefined {
diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts
index 23429ffbee..31d5df1ed1 100644
--- a/packages/playwright/src/reporters/internalReporter.ts
+++ b/packages/playwright/src/reporters/internalReporter.ts
@@ -29,7 +29,7 @@ import type { FullConfig, FullResult, TestCase, TestError, TestResult, TestStep,
export class InternalReporter implements ReporterV2 {
- private _reporter: ReporterV2;
+ private _reporter: Multiplexer;
private _didBegin = false;
private _config!: FullConfig;
private _startTime: Date | undefined;
@@ -43,6 +43,10 @@ export class InternalReporter implements ReporterV2 {
return 'v2';
}
+ hasReporterErrors(): boolean {
+ return this._reporter.hasReporterErrors();
+ }
+
onConfigure(config: FullConfig) {
this._config = config;
this._startTime = new Date();
diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts
index f9b964164e..03d89f6478 100644
--- a/packages/playwright/src/reporters/merge.ts
+++ b/packages/playwright/src/reporters/merge.ts
@@ -44,7 +44,9 @@ type ReportData = {
fullResult: JsonFullResult;
};
-export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[], rootDirOverride: string | undefined) {
+export type MergeResult = 'passed' | 'failed';
+
+export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[], rootDirOverride: string | undefined): Promise<MergeResult> {
const reporters = await createReporters(config, 'merge', reporterDescriptions);
const multiplexer = new Multiplexer(reporters);
const stringPool = new StringInternPool();
@@ -110,6 +112,9 @@ export async function createMergedReport(config: FullConfigInternal, dir: string
});
}
await dispatchEvents(eventData.epilogue);
+ // The merged report status is intentionally not surfaced as a failure - the
+ // merge command only fails when a reporter itself errors.
+ return multiplexer.hasReporterErrors() ? 'failed' : 'passed';
}
const commonEventNames = ['onBlobReportMetadata', 'onConfigure', 'onProject', 'onBegin', 'onEnd'];
diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.ts
index 6596a2bd66..e3c5122f83 100644
--- a/packages/playwright/src/reporters/multiplexer.ts
+++ b/packages/playwright/src/reporters/multiplexer.ts
@@ -14,12 +14,15 @@
* limitations under the License.
*/
+import { serializeError } from '../util';
+
import type { ReportConfigureParams, ReportEndParams, ReporterV2 } from './reporterV2';
import type { FullConfig, FullResult, TestCase, TestError, TestResult, TestStep, WorkerInfo } from '../../types/testReporter';
import type { test } from '../common';
export class Multiplexer implements ReporterV2 {
private _reporters: ReporterV2[];
+ private _hasReporterErrors = false;
constructor(reporters: ReporterV2[]) {
this._reporters = reporters;
@@ -29,54 +32,58 @@ export class Multiplexer implements ReporterV2 {
return 'v2';
}
+ hasReporterErrors(): boolean {
+ return this._hasReporterErrors;
+ }
+
onConfigure(config: FullConfig) {
for (const reporter of this._reporters)
- wrap(() => reporter.onConfigure?.(config));
+ this._wrap(() => reporter.onConfigure?.(config));
}
onBegin(suite: test.Suite) {
for (const reporter of this._reporters)
- wrap(() => reporter.onBegin?.(suite));
+ this._wrap(() => reporter.onBegin?.(suite));
}
onTestBegin(test: TestCase, result: TestResult) {
for (const reporter of this._reporters)
- wrap(() => reporter.onTestBegin?.(test, result));
+ this._wrap(() => reporter.onTestBegin?.(test, result));
}
onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
for (const reporter of this._reporters)
- wrap(() => reporter.onStdOut?.(chunk, test, result));
+ this._wrap(() => reporter.onStdOut?.(chunk, test, result));
}
onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
for (const reporter of this._reporters)
- wrap(() => reporter.onStdErr?.(chunk, test, result));
+ this._wrap(() => reporter.onStdErr?.(chunk, test, result));
}
async onTestPaused(test: TestCase, result: TestResult) {
for (const reporter of this._reporters)
- await wrapAsync(() => reporter.onTestPaused?.(test, result));
+ await this._wrapAsync(() => reporter.onTestPaused?.(test, result));
}
onTestEnd(test: TestCase, result: TestResult) {
for (const reporter of this._reporters)
- wrap(() => reporter.onTestEnd?.(test, result));
+ this._wrap(() => reporter.onTestEnd?.(test, result));
}
onReportConfigure(params: ReportConfigureParams): void {
for (const reporter of this._reporters)
- wrap(() => reporter.onReportConfigure?.(params));
+ this._wrap(() => reporter.onReportConfigure?.(params));
}
onReportEnd(params: ReportEndParams): void {
for (const reporter of this._reporters)
- wrap(() => reporter.onReportEnd?.(params));
+ this._wrap(() => reporter.onReportEnd?.(params));
}
async onEnd(result: FullResult) {
for (const reporter of this._reporters) {
- const outResult = await wrapAsync(() => reporter.onEnd?.(result));
+ const outResult = await this._wrapAsync(() => reporter.onEnd?.(result));
if (outResult?.status)
result.status = outResult.status;
}
@@ -85,47 +92,48 @@ export class Multiplexer implements ReporterV2 {
async onExit() {
for (const reporter of this._reporters)
- await wrapAsync(() => reporter.onExit?.());
+ await this._wrapAsync(() => reporter.onExit?.());
}
onError(error: TestError, workerInfo?: WorkerInfo) {
for (const reporter of this._reporters)
- wrap(() => reporter.onError?.(error, workerInfo));
+ this._wrap(() => reporter.onError?.(error, workerInfo), false);
}
onStepBegin(test: TestCase, result: TestResult, step: TestStep) {
for (const reporter of this._reporters)
- wrap(() => reporter.onStepBegin?.(test, result, step));
+ this._wrap(() => reporter.onStepBegin?.(test, result, step));
}
onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
for (const reporter of this._reporters)
- wrap(() => reporter.onStepEnd?.(test, result, step));
+ this._wrap(() => reporter.onStepEnd?.(test, result, step));
}
printsToStdio(): boolean {
return this._reporters.some(r => {
let prints = false;
- wrap(() => prints = r.printsToStdio ? r.printsToStdio() : true);
+ this._wrap(() => prints = r.printsToStdio ? r.printsToStdio() : true, false);
return prints;
});
}
-}
-async function wrapAsync<T>(callback: () => T | Promise<T>) {
- try {
- return await callback();
- } catch (e) {
- // eslint-disable-next-line no-console
- console.error('Error in reporter', e);
+ private _wrap(callback: () => void, redispatch: boolean = true) {
+ try {
+ callback();
+ } catch (e) {
+ this._hasReporterErrors = true;
+ if (redispatch)
+ this.onError(serializeError(e));
+ }
}
-}
-function wrap(callback: () => void) {
- try {
- callback();
- } catch (e) {
- // eslint-disable-next-line no-console
- console.error('Error in reporter', e);
+ private async _wrapAsync<T>(callback: () => T | Promise<T>): Promise<T | undefined> {
+ try {
+ return await callback();
+ } catch (e) {
+ this._hasReporterErrors = true;
+ this.onError(serializeError(e));
+ }
}
}
diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts
index 1c53cb908c..1a829fdb98 100644
--- a/packages/playwright/src/runner/tasks.ts
+++ b/packages/playwright/src/runner/tasks.ts
@@ -116,7 +116,7 @@ export class TestRun {
result(): 'failed' | 'passed' {
const hasFailedTests = this.rootSuite?.allTests().some(test => !test.ok());
const hasFlakyTests = this.rootSuite?.allTests().some(test => test.outcome() === 'flaky');
- return this.hasWorkerErrors || this.hasReachedMaxFailures() || hasFailedTests || (this.config.config.failOnFlakyTests && hasFlakyTests) ? 'failed' : 'passed';
+ return this.hasWorkerErrors || this.reporter.hasReporterErrors() || this.hasReachedMaxFailures() || hasFailedTests || (this.config.config.failOnFlakyTests && hasFlakyTests) ? 'failed' : 'passed';
}
}
@@ -146,6 +146,10 @@ async function finishTaskRun(testRun: TestRun, status: FullResult['status']) {
if (modifiedResult && modifiedResult.status)
status = modifiedResult.status;
await testRun.reporter.onExit();
+ // A reporter may have thrown during onEnd/onExit (or earlier), which is not
+ // reflected in testRun.result() computed above. Fail the run in that case.
+ if (status === 'passed' && testRun.reporter.hasReporterErrors())
+ status = 'failed';
return status;
}
diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts
index 9ded1e17cc..0f3590b0e0 100644
--- a/tests/playwright-test/reporter-blob.spec.ts
+++ b/tests/playwright-test/reporter-blob.spec.ts
@@ -150,6 +150,39 @@ test('should call methods in right order', async ({ runInlineTest, mergeReports
expect(lines.filter(l => l === 'onExit').length).toBe(1);
});
+test('should fail merge and report error when reporter throws in onEnd', async ({ runInlineTest, mergeReports }) => {
+ const reportDir = test.info().outputPath('blob-report');
+ const files = {
+ 'throwing-reporter.js': `
+ class ThrowingReporter {
+ onEnd() {
+ throw new Error('Error in merge onEnd!');
+ }
+ }
+ module.exports = ThrowingReporter;
+ `,
+ 'playwright.config.ts': `
+ module.exports = {
+ reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]]
+ };
+ `,
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('math 1', async ({}) => {
+ expect(1 + 1).toBe(2);
+ });
+ `,
+ };
+ await runInlineTest(files, { shard: `1/2` });
+ await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+
+ // Merge through the dot reporter so the reporter error is surfaced, and the
+ // throwing reporter to fail the merge command.
+ const { exitCode, output } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'dot,' + test.info().outputPath('throwing-reporter.js')] });
+ expect(exitCode).toBe(1);
+ expect(output).toContain('Error in merge onEnd!');
+});
+
test('should merge into html with dependencies', async ({ runInlineTest, mergeReports, showReport, page }) => {
const reportDir = test.info().outputPath('blob-report');
const files = {
diff --git a/tests/playwright-test/reporter-onend.spec.ts b/tests/playwright-test/reporter-onend.spec.ts
index 38665ea2e6..9442318809 100644
--- a/tests/playwright-test/reporter-onend.spec.ts
+++ b/tests/playwright-test/reporter-onend.spec.ts
@@ -38,3 +38,43 @@ test('should override exit code', async ({ runInlineTest }) => {
});
expect(result.exitCode).toBe(0);
});
+
+test('should fail run and report error when reporter throws in onEnd', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'reporter.ts': `
+ export default class Reporter {
+ async onEnd() {
+ throw new Error('Error in onEnd!');
+ }
+ }
+ `,
+ 'playwright.config.ts': `module.exports = { reporter: [['dot'], ['./reporter.ts']] };`,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('passing', async () => {});
+ `,
+ }, { reporter: '' });
+ expect(result.exitCode).toBe(1);
+ expect(result.passed).toBe(1);
+ expect(result.output).toContain('Error in onEnd!');
+});
+
+test('should fail run and report error when reporter throws in onTestEnd', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'reporter.ts': `
+ export default class Reporter {
+ onTestEnd() {
+ throw new Error('Error in onTestEnd!');
+ }
+ }
+ `,
+ 'playwright.config.ts': `module.exports = { reporter: [['dot'], ['./reporter.ts']] };`,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('passing', async () => {});
+ `,
+ }, { reporter: '' });
+ expect(result.exitCode).toBe(1);
+ expect(result.passed).toBe(1);
+ expect(result.output).toContain('Error in onTestEnd!');
+});