Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix(test-runner): run teardown projects when maxFailures is reached (#41737)

Yury Semikhatsky686830af6839
diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts
index 87261df5df..bec64ec409 100644
--- a/packages/playwright/src/runner/dispatcher.ts
+++ b/packages/playwright/src/runner/dispatcher.ts
@@ -41,14 +41,17 @@ export class Dispatcher {
private _queuedOrRunningHashCount = new Map<string, number>();
private _finished = new ManualPromise<void>();
private _isStopped = true;
+ // Teardown phases keep running after maxFailures, so that cleanup is not skipped.
+ private _ignoreMaxFailures: boolean;
private _testRun: TestRun;
private _extraEnvByProjectId: EnvByProjectId = new Map();
private _producedEnvByProjectId: EnvByProjectId = new Map();
- constructor(testRun: TestRun) {
+ constructor(testRun: TestRun, options: { ignoreMaxFailures?: boolean } = {}) {
this._testRun = testRun;
+ this._ignoreMaxFailures = !!options.ignoreMaxFailures;
for (const project of testRun.config.projects) {
if (project.workers)
this._workerLimitPerProjectId.set(project.id, project.workers);
@@ -96,7 +99,7 @@ export class Dispatcher {
// 3. Claim both the job and the worker slot.
this._queue.splice(jobIndex, 1);
- const jobDispatcher = new JobDispatcher(job, this._testRun, () => this.stop().catch(() => {}));
+ const jobDispatcher = new JobDispatcher(job, this._testRun, this._ignoreMaxFailures ? undefined : () => this.stop().catch(() => {}));
this._workerSlots[workerIndex].jobDispatcher = jobDispatcher;
// 4. Run the job. This is the only async operation.
@@ -204,7 +207,7 @@ export class Dispatcher {
this._isStopped = false;
this._workerSlots = [];
// 0. Stop right away if we have reached max failures.
- if (this._testRun.hasReachedMaxFailures())
+ if (!this._ignoreMaxFailures && this._testRun.hasReachedMaxFailures())
void this.stop();
// 1. Allocate workers.
for (let i = 0; i < this._testRun.config.config.workers; i++)
@@ -292,7 +295,8 @@ class JobDispatcher {
readonly job: TestGroup;
private _testRun: TestRun;
- private _stopCallback: () => void;
+ // Missing callback means that maxFailures should be ignored, e.g. in teardown phases.
+ private _onMaxFailuresReached?: () => void;
private _listeners: RegisteredListener[] = [];
private _failedTests = new Set<testNs.TestCase>();
private _failedWithNonRetriableError = new Set<testNs.TestCase|testNs.Suite>();
@@ -302,13 +306,17 @@ class JobDispatcher {
private _workerIndex = 0;
private _currentlyRunning: { test: testNs.TestCase, result: TestResult } | undefined;
- constructor(job: TestGroup, testRun: TestRun, stopCallback: () => void) {
+ constructor(job: TestGroup, testRun: TestRun, onMaxFailuresReached?: () => void) {
this.job = job;
this._testRun = testRun;
- this._stopCallback = stopCallback;
+ this._onMaxFailuresReached = onMaxFailuresReached;
this._remainingByTestId = new Map(this.job.tests.map(e => [e.id, e]));
}
+ private _isStoppedByMaxFailures() {
+ return !!this._onMaxFailuresReached && this._testRun.hasReachedMaxFailures();
+ }
+
private _onTestBegin(params: ipc.TestBeginPayload) {
const test = this._remainingByTestId.get(params.testId);
if (!test) {
@@ -325,7 +333,7 @@ class JobDispatcher {
}
private _onTestEnd(params: ipc.TestEndPayload) {
- if (this._testRun.hasReachedMaxFailures()) {
+ if (this._isStoppedByMaxFailures()) {
// Do not show more than one error to avoid confusion, but report
// as interrupted to indicate that we did actually start the test.
params.status = 'interrupted';
@@ -457,7 +465,7 @@ class JobDispatcher {
for (const test of this._remainingByTestId.values()) {
if (!testIds.has(test.id))
continue;
- if (!this._testRun.hasReachedMaxFailures()) {
+ if (!this._isStoppedByMaxFailures()) {
this._failTestWithErrors(test, errors);
errors = []; // Only report errors for the first test.
}
@@ -639,7 +647,7 @@ class JobDispatcher {
// with skipped tests mixed in-between non-skipped. This makes
// for a better reporter experience.
const allTestsSkipped = this.job.tests.every(test => test.expectedStatus === 'skipped');
- if (allTestsSkipped && !this._testRun.hasReachedMaxFailures()) {
+ if (allTestsSkipped && !this._isStoppedByMaxFailures()) {
for (const test of this.job.tests) {
const result = test._appendTestResult();
this._testRun.reporter.onTestBegin?.(test, result);
@@ -659,12 +667,12 @@ class JobDispatcher {
private _reportTestEnd(test: testNs.TestCase, result: TestResult) {
this._testRun.reporter.onTestEnd?.(test, result);
- const hadMaxFailures = this._testRun.hasReachedMaxFailures();
+ const hadMaxFailures = this._isStoppedByMaxFailures();
// Test is considered failing after the last retry.
if (test.outcome() === 'unexpected' && test.results.length > test.retries)
++this._testRun.failedTestCount;
- if (!hadMaxFailures && this._testRun.hasReachedMaxFailures()) {
- this._stopCallback();
+ if (!hadMaxFailures && this._isStoppedByMaxFailures()) {
+ this._onMaxFailuresReached?.();
this._testRun.reporter.onError?.({ message: colors.red(`Testing stopped early after ${this._testRun.config.config.maxFailures} maximum allowed failures.`) });
}
}
diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts
index a2b2d328d8..7890ed51fc 100644
--- a/packages/playwright/src/runner/tasks.ts
+++ b/packages/playwright/src/runner/tasks.ts
@@ -395,6 +395,9 @@ function createPhasesTask(): Task<TestRun> {
const projectToSuite = new Map(testRun.rootSuite!.suites.map(suite => [suite._fullProject!, suite]));
const allProjects = [...projectToSuite.keys()];
const teardownToSetups = buildTeardownToSetupsMap(allProjects);
+ // Teardown projects keep running after maxFailures is reached, so that cleanup
+ // is not skipped. Nothing to ignore when maxFailures cannot stop the run.
+ const ignoreMaxFailuresProjectIds = new Set(testRun.config.config.maxFailures > 0 ? [...teardownToSetups.keys()].map(project => project.id) : []);
const teardownToSetupsDependents = new Map<commonConfig.FullProjectInternal, commonConfig.FullProjectInternal[]>();
for (const [teardown, setups] of teardownToSetups) {
const closure = buildDependentProjects(setups, allProjects);
@@ -414,14 +417,17 @@ function createPhasesTask(): Task<TestRun> {
phaseProjects.push(project);
}
- // Create a new phase.
for (const project of phaseProjects)
processed.add(project);
- if (phaseProjects.length) {
+ // Projects that ignore maxFailures run in their own phase.
+ for (const ignoreMaxFailures of [false, true]) {
+ const projects = phaseProjects.filter(project => ignoreMaxFailuresProjectIds.has(project.id) === ignoreMaxFailures);
+ if (!projects.length)
+ continue;
let testGroupsInPhase = 0;
- const phase: Phase = { dispatcher: new Dispatcher(testRun), projects: [] };
+ const phase: Phase = { dispatcher: new Dispatcher(testRun, { ignoreMaxFailures }), projects: [] };
testRun.phases.push(phase);
- for (const project of phaseProjects) {
+ for (const project of projects) {
const projectSuite = projectToSuite.get(project)!;
const testGroups = createTestGroups(projectSuite, testRun.config.config.workers);
phase.projects.push({ project, projectSuite, testGroups });
diff --git a/tests/playwright-test/max-failures.spec.ts b/tests/playwright-test/max-failures.spec.ts
index 56ad9f466f..099f60bece 100644
--- a/tests/playwright-test/max-failures.spec.ts
+++ b/tests/playwright-test/max-failures.spec.ts
@@ -182,6 +182,56 @@ test('max-failures should work across phases', async ({ runInlineTest }) => {
expect(result.output).not.toContain('running d');
});
+test('max-failures should still run teardown project', async ({ runInlineTest }) => {
+ test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/41713' });
+
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ module.exports = {
+ maxFailures: 1,
+ projects: [
+ { name: 'setup', testMatch: '**/setup.ts', teardown: 'teardown' },
+ { name: 'teardown', testMatch: '**/teardown.ts' },
+ { name: 'a', dependencies: ['setup'], testMatch: '**/a.spec.ts' },
+ // Unrelated chain that becomes ready to run together with 'teardown'.
+ { name: 'x', testMatch: '**/x.spec.ts' },
+ { name: 'y', dependencies: ['x'], testMatch: '**/y.spec.ts' },
+ { name: 'z', dependencies: ['y'], testMatch: '**/z.spec.ts' },
+ ],
+ };`,
+ 'setup.ts': `
+ import { test } from '@playwright/test';
+ test('setup', async ({}) => { console.log('\\n%%setup'); });
+ `,
+ 'teardown.ts': `
+ import { test } from '@playwright/test';
+ test('teardown', async ({}) => { console.log('\\n%%teardown'); });
+ `,
+ 'a.spec.ts': `
+ import { test, expect } from '@playwright/test';
+ test('failing', async ({}) => { expect(1).toBe(2); });
+ `,
+ 'x.spec.ts': `
+ import { test } from '@playwright/test';
+ test('x', async ({}) => {});
+ `,
+ 'y.spec.ts': `
+ import { test } from '@playwright/test';
+ test('y', async ({}) => {});
+ `,
+ 'z.spec.ts': `
+ import { test } from '@playwright/test';
+ test('z', async ({}) => { console.log('\\n%%z'); });
+ `,
+ }, { workers: 1 });
+ expect(result.exitCode).toBe(1);
+ expect(result.failed).toBe(1);
+ expect(result.output).toContain('%%setup');
+ expect(result.output).toContain('%%teardown');
+ // Regular project 'z' must not run after maxFailures.
+ expect(result.output).not.toContain('%%z');
+});
+
test('max-failures should not consider retries as failures', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `