Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix(test runner): unhandled rejection in test.fail should not skip other tests (#37341)

Dmitry Gozman40bf2f05a0cd
diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts
index 806aed0532..9031be7eea 100644
--- a/packages/playwright/src/common/ipc.ts
+++ b/packages/playwright/src/common/ipc.ts
@@ -128,6 +128,7 @@ export type DonePayload = {
fatalErrors: TestInfoErrorImpl[];
skipTestsDueToSetupFailure: string[]; // test ids
fatalUnknownTestIds?: string[];
+ stoppedDueToUnhandledErrorInTestFail?: boolean;
};
export type TestOutputPayload = {
diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts
index 22293b2a9d..388586fe96 100644
--- a/packages/playwright/src/runner/dispatcher.ts
+++ b/packages/playwright/src/runner/dispatcher.ts
@@ -459,7 +459,7 @@ class JobDispatcher {
// - there are no remaining
// - we are here not because something failed
// - no unrecoverable worker error
- if (!this._remainingByTestId.size && !this._failedTests.size && !params.fatalErrors.length && !params.skipTestsDueToSetupFailure.length && !params.fatalUnknownTestIds && !params.unexpectedExitError) {
+ if (!this._remainingByTestId.size && !this._failedTests.size && !params.fatalErrors.length && !params.skipTestsDueToSetupFailure.length && !params.fatalUnknownTestIds && !params.unexpectedExitError && !params.stoppedDueToUnhandledErrorInTestFail) {
this._finished({ didFail: false });
return;
}
diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts
index e4eb1b82ad..29f421dcc9 100644
--- a/packages/playwright/src/worker/workerMain.ts
+++ b/packages/playwright/src/worker/workerMain.ts
@@ -51,6 +51,9 @@ export class WorkerMain extends ProcessRunner {
private _skipRemainingTestsInSuite: Suite | undefined;
// The stage of the full cleanup. Once "finished", we can safely stop running anything.
private _didRunFullCleanup = false;
+ // Whether the worker was stopped due to an unhandled error in a test marked with test.fail().
+ // This should force dispatcher to use a new worker instead.
+ private _stoppedDueToUnhandledErrorInTestFail = false;
// Whether the worker was requested to stop.
private _isStopped = false;
// This promise resolves once the single "run test group" call finishes.
@@ -191,8 +194,10 @@ export class WorkerMain extends ProcessRunner {
// an expect() error which we know does not mess things up.
const isExpectError = (error instanceof Error) && !!(error as any).matcherResult;
const shouldContinueInThisWorker = this._currentTest.expectedStatus === 'failed' && isExpectError;
- if (!shouldContinueInThisWorker)
+ if (!shouldContinueInThisWorker) {
+ this._stoppedDueToUnhandledErrorInTestFail = true;
void this._stop();
+ }
}
private async _loadIfNeeded() {
@@ -248,7 +253,8 @@ export class WorkerMain extends ProcessRunner {
const donePayload: DonePayload = {
fatalErrors: this._fatalErrors,
skipTestsDueToSetupFailure: [],
- fatalUnknownTestIds
+ fatalUnknownTestIds,
+ stoppedDueToUnhandledErrorInTestFail: this._stoppedDueToUnhandledErrorInTestFail,
};
for (const test of this._skipRemainingTestsInSuite?.allTests() || []) {
if (entries.has(test.id))
diff --git a/tests/playwright-test/runner.spec.ts b/tests/playwright-test/runner.spec.ts
index fb57fe16c4..3a2b9d7bac 100644
--- a/tests/playwright-test/runner.spec.ts
+++ b/tests/playwright-test/runner.spec.ts
@@ -751,9 +751,16 @@ test('slow double SIGINT should be respected in reporter.onExit', async ({ inter
});
test('unhandled exception in test.fail should restart worker and continue', async ({ runInlineTest }) => {
- const result = await runInlineTest({
+ const files = {
'a.spec.ts': `
- import { test, expect } from '@playwright/test';
+ import { test as baseTest, expect } from '@playwright/test';
+
+ const test = baseTest.extend({
+ worker: [async ({}, use, info) => {
+ await use();
+ console.log('\\n%%worker teardown=' + info.workerIndex);
+ }, { scope: 'worker', auto: true }],
+ });
test('bad', async () => {
test.fail();
@@ -767,12 +774,20 @@ test('unhandled exception in test.fail should restart worker and continue', asyn
test('good', () => {
console.log('\\n%%good running worker=' + test.info().workerIndex);
});
- `
- }, { retries: 1, reporter: 'list' });
- expect(result.exitCode).toBe(0);
- expect(result.passed).toBe(2);
- expect(result.failed).toBe(0);
- expect(result.outputLines).toEqual(['bad running worker=0', 'good running worker=1']);
+ `,
+ };
+ for (const parallel of [true, false]) {
+ await test.step(`parallel=${parallel}`, async () => {
+ const options = { retries: 1, reporter: 'list', workers: 1 };
+ if (parallel)
+ options['fully-parallel'] = true;
+ const result = await runInlineTest(files, options);
+ expect(result.exitCode).toBe(0);
+ expect(result.passed).toBe(2);
+ expect(result.failed).toBe(0);
+ expect(result.outputLines).toEqual(['bad running worker=0', 'worker teardown=0', 'good running worker=1', 'worker teardown=1']);
+ });
+ }
});
test('wait for workers to finish before reporter.onEnd', async ({ runInlineTest }) => {