Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
fix(test-runner): run teardown projects when maxFailures is reached (#41737)
Yury Semikhatsky686830af6839
diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.tsindex 87261df5df..bec64ec409 100644--- a/packages/playwright/src/runner/dispatcher.ts+++ b/packages/playwright/src/runner/dispatcher.ts@@ -41,14 +41,17 @@ export class Dispatcher {private _queuedOrRunningHashCount = new Map<string, number>();private _finished = new ManualPromise<void>();private _isStopped = true;+ // Teardown phases keep running after maxFailures, so that cleanup is not skipped.+ private _ignoreMaxFailures: boolean;private _testRun: TestRun;private _extraEnvByProjectId: EnvByProjectId = new Map();private _producedEnvByProjectId: EnvByProjectId = new Map();- constructor(testRun: TestRun) {+ constructor(testRun: TestRun, options: { ignoreMaxFailures?: boolean } = {}) {this._testRun = testRun;+ this._ignoreMaxFailures = !!options.ignoreMaxFailures;for (const project of testRun.config.projects) {if (project.workers)this._workerLimitPerProjectId.set(project.id, project.workers);@@ -96,7 +99,7 @@ export class Dispatcher {// 3. Claim both the job and the worker slot.this._queue.splice(jobIndex, 1);- const jobDispatcher = new JobDispatcher(job, this._testRun, () => this.stop().catch(() => {}));+ const jobDispatcher = new JobDispatcher(job, this._testRun, this._ignoreMaxFailures ? undefined : () => this.stop().catch(() => {}));this._workerSlots[workerIndex].jobDispatcher = jobDispatcher;// 4. Run the job. This is the only async operation.@@ -204,7 +207,7 @@ export class Dispatcher {this._isStopped = false;this._workerSlots = [];// 0. Stop right away if we have reached max failures.- if (this._testRun.hasReachedMaxFailures())+ if (!this._ignoreMaxFailures && this._testRun.hasReachedMaxFailures())void this.stop();// 1. Allocate workers.for (let i = 0; i < this._testRun.config.config.workers; i++)@@ -292,7 +295,8 @@ class JobDispatcher {readonly job: TestGroup;private _testRun: TestRun;- private _stopCallback: () => void;+ // Missing callback means that maxFailures should be ignored, e.g. in teardown phases.+ private _onMaxFailuresReached?: () => void;private _listeners: RegisteredListener[] = [];private _failedTests = new Set<testNs.TestCase>();private _failedWithNonRetriableError = new Set<testNs.TestCase|testNs.Suite>();@@ -302,13 +306,17 @@ class JobDispatcher {private _workerIndex = 0;private _currentlyRunning: { test: testNs.TestCase, result: TestResult } | undefined;- constructor(job: TestGroup, testRun: TestRun, stopCallback: () => void) {+ constructor(job: TestGroup, testRun: TestRun, onMaxFailuresReached?: () => void) {this.job = job;this._testRun = testRun;- this._stopCallback = stopCallback;+ this._onMaxFailuresReached = onMaxFailuresReached;this._remainingByTestId = new Map(this.job.tests.map(e => [e.id, e]));}+ private _isStoppedByMaxFailures() {+ return !!this._onMaxFailuresReached && this._testRun.hasReachedMaxFailures();+ }+private _onTestBegin(params: ipc.TestBeginPayload) {const test = this._remainingByTestId.get(params.testId);if (!test) {@@ -325,7 +333,7 @@ class JobDispatcher {}private _onTestEnd(params: ipc.TestEndPayload) {- if (this._testRun.hasReachedMaxFailures()) {+ if (this._isStoppedByMaxFailures()) {// Do not show more than one error to avoid confusion, but report// as interrupted to indicate that we did actually start the test.params.status = 'interrupted';@@ -457,7 +465,7 @@ class JobDispatcher {for (const test of this._remainingByTestId.values()) {if (!testIds.has(test.id))continue;- if (!this._testRun.hasReachedMaxFailures()) {+ if (!this._isStoppedByMaxFailures()) {this._failTestWithErrors(test, errors);errors = []; // Only report errors for the first test.}@@ -639,7 +647,7 @@ class JobDispatcher {// with skipped tests mixed in-between non-skipped. This makes// for a better reporter experience.const allTestsSkipped = this.job.tests.every(test => test.expectedStatus === 'skipped');- if (allTestsSkipped && !this._testRun.hasReachedMaxFailures()) {+ if (allTestsSkipped && !this._isStoppedByMaxFailures()) {for (const test of this.job.tests) {const result = test._appendTestResult();this._testRun.reporter.onTestBegin?.(test, result);@@ -659,12 +667,12 @@ class JobDispatcher {private _reportTestEnd(test: testNs.TestCase, result: TestResult) {this._testRun.reporter.onTestEnd?.(test, result);- const hadMaxFailures = this._testRun.hasReachedMaxFailures();+ const hadMaxFailures = this._isStoppedByMaxFailures();// Test is considered failing after the last retry.if (test.outcome() === 'unexpected' && test.results.length > test.retries)++this._testRun.failedTestCount;- if (!hadMaxFailures && this._testRun.hasReachedMaxFailures()) {- this._stopCallback();+ if (!hadMaxFailures && this._isStoppedByMaxFailures()) {+ this._onMaxFailuresReached?.();this._testRun.reporter.onError?.({ message: colors.red(`Testing stopped early after ${this._testRun.config.config.maxFailures} maximum allowed failures.`) });}}diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.tsindex a2b2d328d8..7890ed51fc 100644--- a/packages/playwright/src/runner/tasks.ts+++ b/packages/playwright/src/runner/tasks.ts@@ -395,6 +395,9 @@ function createPhasesTask(): Task<TestRun> {const projectToSuite = new Map(testRun.rootSuite!.suites.map(suite => [suite._fullProject!, suite]));const allProjects = [...projectToSuite.keys()];const teardownToSetups = buildTeardownToSetupsMap(allProjects);+ // Teardown projects keep running after maxFailures is reached, so that cleanup+ // is not skipped. Nothing to ignore when maxFailures cannot stop the run.+ const ignoreMaxFailuresProjectIds = new Set(testRun.config.config.maxFailures > 0 ? [...teardownToSetups.keys()].map(project => project.id) : []);const teardownToSetupsDependents = new Map<commonConfig.FullProjectInternal, commonConfig.FullProjectInternal[]>();for (const [teardown, setups] of teardownToSetups) {const closure = buildDependentProjects(setups, allProjects);@@ -414,14 +417,17 @@ function createPhasesTask(): Task<TestRun> {phaseProjects.push(project);}- // Create a new phase.for (const project of phaseProjects)processed.add(project);- if (phaseProjects.length) {+ // Projects that ignore maxFailures run in their own phase.+ for (const ignoreMaxFailures of [false, true]) {+ const projects = phaseProjects.filter(project => ignoreMaxFailuresProjectIds.has(project.id) === ignoreMaxFailures);+ if (!projects.length)+ continue;let testGroupsInPhase = 0;- const phase: Phase = { dispatcher: new Dispatcher(testRun), projects: [] };+ const phase: Phase = { dispatcher: new Dispatcher(testRun, { ignoreMaxFailures }), projects: [] };testRun.phases.push(phase);- for (const project of phaseProjects) {+ for (const project of projects) {const projectSuite = projectToSuite.get(project)!;const testGroups = createTestGroups(projectSuite, testRun.config.config.workers);phase.projects.push({ project, projectSuite, testGroups });diff --git a/tests/playwright-test/max-failures.spec.ts b/tests/playwright-test/max-failures.spec.tsindex 56ad9f466f..099f60bece 100644--- a/tests/playwright-test/max-failures.spec.ts+++ b/tests/playwright-test/max-failures.spec.ts@@ -182,6 +182,56 @@ test('max-failures should work across phases', async ({ runInlineTest }) => {expect(result.output).not.toContain('running d');});+test('max-failures should still run teardown project', async ({ runInlineTest }) => {+ test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/41713' });++ const result = await runInlineTest({+ 'playwright.config.ts': `+ module.exports = {+ maxFailures: 1,+ projects: [+ { name: 'setup', testMatch: '**/setup.ts', teardown: 'teardown' },+ { name: 'teardown', testMatch: '**/teardown.ts' },+ { name: 'a', dependencies: ['setup'], testMatch: '**/a.spec.ts' },+ // Unrelated chain that becomes ready to run together with 'teardown'.+ { name: 'x', testMatch: '**/x.spec.ts' },+ { name: 'y', dependencies: ['x'], testMatch: '**/y.spec.ts' },+ { name: 'z', dependencies: ['y'], testMatch: '**/z.spec.ts' },+ ],+ };`,+ 'setup.ts': `+ import { test } from '@playwright/test';+ test('setup', async ({}) => { console.log('\\n%%setup'); });+ `,+ 'teardown.ts': `+ import { test } from '@playwright/test';+ test('teardown', async ({}) => { console.log('\\n%%teardown'); });+ `,+ 'a.spec.ts': `+ import { test, expect } from '@playwright/test';+ test('failing', async ({}) => { expect(1).toBe(2); });+ `,+ 'x.spec.ts': `+ import { test } from '@playwright/test';+ test('x', async ({}) => {});+ `,+ 'y.spec.ts': `+ import { test } from '@playwright/test';+ test('y', async ({}) => {});+ `,+ 'z.spec.ts': `+ import { test } from '@playwright/test';+ test('z', async ({}) => { console.log('\\n%%z'); });+ `,+ }, { workers: 1 });+ expect(result.exitCode).toBe(1);+ expect(result.failed).toBe(1);+ expect(result.output).toContain('%%setup');+ expect(result.output).toContain('%%teardown');+ // Regular project 'z' must not run after maxFailures.+ expect(result.output).not.toContain('%%z');+});+test('max-failures should not consider retries as failures', async ({ runInlineTest }) => {const result = await runInlineTest({'playwright.config.ts': `