Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
fix(reporter): expose setup/teardown projects to preprocessSuite (#41731)
Simon Knott7b02a7a6d85a
diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.mdindex 26f8d619e2..67619c914e 100644--- a/docs/src/test-reporter-api/class-reporter.md+++ b/docs/src/test-reporter-api/class-reporter.md@@ -317,6 +317,6 @@ Resolved configuration.The root suite that contains the projects, files and test cases that will run.-The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. It contains only the top-level projects being run — setup and dependency projects are not included and cannot be excluded from here.+The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here.The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its built-in sharding after [`method: Reporter.preprocessSuite`] returns, unless the returned `implementsSharding` is `true`.diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.tsindex 991ac8304e..598a041d8a 100644--- a/packages/playwright/src/common/test.ts+++ b/packages/playwright/src/common/test.ts@@ -58,7 +58,7 @@ export class Suite extends Base {_parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none';_fullProject: FullProjectInternal | undefined;_fileId: string | undefined;- _preprocessing = false;+ _preprocessMode: 'editable' | 'readonly' | undefined = undefined;readonly _type: 'root' | 'project' | 'file' | 'describe';skip: (reason?: string) => void;@@ -270,22 +270,28 @@ export class Suite extends Base {}private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void {- if (!this._rootSuite()._preprocessing)+ const mode = this._resolvePreprocessMode();+ if (!mode)throw new Error(`Suite.${type}() can only be called from Reporter.preprocessSuite().`);+ if (mode === 'readonly')+ throw new Error(`Suite.${type}() cannot be called on a setup or teardown project; these always run in full.`);for (const test of this.allTests())test._applyPlanAnnotation({ type, description: reason, location });}exclude(): void {- if (!this._rootSuite()._preprocessing)+ const mode = this._resolvePreprocessMode();+ if (!mode)throw new Error(`Suite.exclude() can only be called from Reporter.preprocessSuite().`);if (!this.parent)throw new Error(`Suite.exclude() cannot be called on the root suite.`);+ if (mode === 'readonly')+ throw new Error(`Suite.exclude() cannot be called on a setup or teardown project; these always run in full.`);this.parent._detach(this);}- _rootSuite(): Suite {- return this.parent?._rootSuite() ?? this;+ _resolvePreprocessMode(): 'editable' | 'readonly' | undefined {+ return this._preprocessMode ?? this.parent?._resolvePreprocessMode();}}@@ -353,8 +359,11 @@ export class TestCase extends Base implements reporterTypes.TestCase {}private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void {- if (!this._rootSuite()._preprocessing)+ const mode = this.parent._resolvePreprocessMode();+ if (!mode)throw new Error(`TestCase.${type}() can only be called from Reporter.preprocessSuite().`);+ if (mode === 'readonly')+ throw new Error(`TestCase.${type}() cannot be called on a setup or teardown project test; these always run in full.`);this._applyPlanAnnotation({ type, description: reason, location });}@@ -368,15 +377,14 @@ export class TestCase extends Base implements reporterTypes.TestCase {}exclude(): void {- if (!this._rootSuite()._preprocessing)+ const mode = this.parent._resolvePreprocessMode();+ if (!mode)throw new Error(`TestCase.exclude() can only be called from Reporter.preprocessSuite().`);+ if (mode === 'readonly')+ throw new Error(`TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.`);this.parent._detach(this);}- _rootSuite(): Suite {- return this.parent._rootSuite();- }-_serialize(): any {return {kind: 'test',diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.tsindex 7766ec11f6..f4f092cd5d 100644--- a/packages/playwright/src/reporters/internalReporter.ts+++ b/packages/playwright/src/reporters/internalReporter.ts@@ -55,12 +55,7 @@ export class InternalReporter implements ReporterV2 {}async preprocessSuite(config: FullConfig, suite: testNs.Suite) {- suite._preprocessing = true;- try {- return await this._reporter.preprocessSuite?.(config, suite);- } finally {- suite._preprocessing = false;- }+ return await this._reporter.preprocessSuite?.(config, suite);}onBegin(suite: testNs.Suite) {diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.tsindex 9485a923ba..9878b14702 100644--- a/packages/playwright/src/runner/loadUtils.ts+++ b/packages/playwright/src/runner/loadUtils.ts@@ -165,7 +165,30 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho}}- const preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite);+ // Temporarily prepend unfiltered dependency projects for preprocessing.+ const dependencySuites = new Map<commonConfig.FullProjectInternal, testNs.Suite>();+ for (const [project, type] of projectClosure) {+ if (type !== 'dependency')+ continue;+ const dependencySuite = buildProjectSuite(project, projectSuites.get(project)!);+ dependencySuite._preprocessMode = 'readonly';+ dependencySuites.set(project, dependencySuite);+ rootSuite._prependSuite(dependencySuite);+ }++ rootSuite._preprocessMode = 'editable';+ let preprocessResult: Awaited<ReturnType<typeof testRun.reporter.preprocessSuite>>;+ try {+ preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite);+ } finally {+ // Continue the existing sharding and filtering pipeline with top-level projects only.+ rootSuite._preprocessMode = undefined;+ for (const dependencySuite of dependencySuites.values()) {+ dependencySuite._preprocessMode = undefined;+ rootSuite._detach(dependencySuite);+ }+ }+// Shard only the top-level projects.if (config.config.shard && !preprocessResult?.implementsSharding) {// Create test groups for top-level projects.@@ -193,16 +216,13 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], shosuiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => testRun.postShardTestFilters.every(filter => filter(test)));const topLevelProjects = [];- // Now prepend dependency projects without filtration.{// Filtering 'only' and sharding might have reduced the number of top-level projects.// Build the project closure to only include dependencies that are still needed.- const projectClosure = new Map(buildProjectsClosure(rootSuite.suites.map(suite => suite._fullProject!)));-- // Clone file suites for dependency projects.- for (const [project, level] of projectClosure.entries()) {- if (level === 'dependency')- rootSuite._prependSuite(buildProjectSuite(project, projectSuites.get(project)!));+ const finalProjectClosure = buildProjectsClosure(rootSuite.suites.map(suite => suite._fullProject!));+ for (const [project, type] of finalProjectClosure) {+ if (type === 'dependency')+ rootSuite._prependSuite(dependencySuites.get(project)!);elsetopLevelProjects.push(project);}diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.tsindex 47e406bcd1..f7809c29c6 100644--- a/packages/playwright/types/testReporter.d.ts+++ b/packages/playwright/types/testReporter.d.ts@@ -153,8 +153,7 @@ export interface Reporter {* @param suite The root suite that contains the projects, files and test cases that will run.** The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that- * match the current invocation. It contains only the top-level projects being run — setup and dependency projects are- * not included and cannot be excluded from here.+ * match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here.** The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its* built-in sharding afterdiff --git a/tests/playwright-test/reporter-preprocess-suite.spec.ts b/tests/playwright-test/reporter-preprocess-suite.spec.tsindex 7e2f163428..d9139a34c5 100644--- a/tests/playwright-test/reporter-preprocess-suite.spec.ts+++ b/tests/playwright-test/reporter-preprocess-suite.spec.ts@@ -392,15 +392,40 @@ test('multiple reporters declaring implementsSharding throws', async ({ runInlinexpect(result.outputLines.join('\n')).toContain(`Multiple reporters declare 'implementsSharding'`);});-test('plan.suite contains only top-level projects, not dependency/setup projects', async ({ runInlineTest }) => {+test('plan.suite exposes setup/teardown dependency projects but they are read-only', async ({ runInlineTest }) => {const result = await runInlineTest({'reporter.ts': `class Reporter {async preprocessSuite(config, suite) {- // The suite only exposes top-level projects, so a reporter has no handle on- // setup/dependency project tests and therefore cannot exclude them.console.log('%% plan projects: ' + suite.suites.map(s => s.title).join(','));console.log('%% plan tests: ' + suite.allTests().map(t => t.title).join(','));+ this.preprocessedTests = new Set(suite.allTests());+ const setupTest = suite.allTests().find(t => t.title === 'setup-test');+ for (const method of ['skip', 'fixme', 'fail', 'exclude']) {+ try {+ setupTest[method]();+ console.log('%% dep-' + method + ': no-throw');+ } catch (e) {+ console.log('%% dep-' + method + ': ' + e.message);+ }+ }+ const setupProject = suite.suites.find(s => s.title === 'setup');+ try {+ setupProject.exclude();+ console.log('%% dep-suite-exclude: no-throw');+ } catch (e) {+ console.log('%% dep-suite-exclude: ' + e.message);+ }+ }+ onBegin(config, suite) {+ console.log('%% same test objects: ' + suite.allTests().every(test => this.preprocessedTests.has(test)));+ const setupTest = suite.allTests().find(t => t.title === 'setup-test');+ try {+ setupTest.skip();+ console.log('%% dep-after-preprocess: no-throw');+ } catch (e) {+ console.log('%% dep-after-preprocess: ' + e.message);+ }}onTestEnd(test, result) {console.log('%% ran ' + test.parent.project().name + '/' + test.title);@@ -412,7 +437,8 @@ test('plan.suite contains only top-level projects, not dependency/setup projectsmodule.exports = {reporter: './reporter.ts',projects: [- { name: 'setup', testMatch: /a\\.setup\\.ts/ },+ { name: 'setup', testMatch: /a\\.setup\\.ts/, teardown: 'teardown' },+ { name: 'teardown', testMatch: /a\\.teardown\\.ts/ },{ name: 'main', testMatch: /a\\.test\\.ts/, dependencies: ['setup'] },],};@@ -421,6 +447,10 @@ test('plan.suite contains only top-level projects, not dependency/setup projectsimport { test } from '@playwright/test';test('setup-test', async () => {});`,+ 'a.teardown.ts': `+ import { test } from '@playwright/test';+ test('teardown-test', async () => {});+ `,'a.test.ts': `import { test } from '@playwright/test';test('main-test', async () => {});@@ -428,11 +458,65 @@ test('plan.suite contains only top-level projects, not dependency/setup projects}, { reporter: '', workers: 1 }, undefined, { additionalArgs: ['--project=main'] });expect(result.exitCode).toBe(0);- // plan only sees the top-level 'main' project; the 'setup' dependency is prepended afterwards.- expect(result.outputLines).toContain('plan projects: main');- // 'setup-test' is absent from the plan suite, proving setup/dependency tests are not exposed.- expect(result.outputLines).toContain('plan tests: main-test');- // Both the dependency and the main project still run.- expect(result.outputLines).toContain('ran setup/setup-test');- expect(result.outputLines).toContain('ran main/main-test');+ expect(result.outputLines).toEqual([+ 'plan projects: teardown,setup,main',+ 'plan tests: teardown-test,setup-test,main-test',+ 'dep-skip: TestCase.skip() cannot be called on a setup or teardown project test; these always run in full.',+ 'dep-fixme: TestCase.fixme() cannot be called on a setup or teardown project test; these always run in full.',+ 'dep-fail: TestCase.fail() cannot be called on a setup or teardown project test; these always run in full.',+ 'dep-exclude: TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.',+ 'dep-suite-exclude: Suite.exclude() cannot be called on a setup or teardown project; these always run in full.',+ 'same test objects: true',+ 'dep-after-preprocess: TestCase.skip() can only be called from Reporter.preprocessSuite().',+ 'ran setup/setup-test',+ 'ran main/main-test',+ 'ran teardown/teardown-test',+ ]);+});++test('plan.suite temporarily exposes dependencies without changing final project selection', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'reporter.ts': `+ class Reporter {+ async preprocessSuite(config, suite) {+ console.log('%% plan projects: ' + suite.suites.map(suite => suite.title).join(','));+ suite.suites.find(suite => suite.title === 'main').exclude();+ }+ onTestEnd(test, result) {+ console.log('%% ran ' + test.parent.project().name + '/' + test.title);+ }+ }+ module.exports = Reporter;+ `,+ 'playwright.config.ts': `+ module.exports = {+ reporter: './reporter.ts',+ projects: [+ { name: 'setup', testMatch: /setup\\.spec\\.ts/ },+ { name: 'main', testMatch: /main\\.spec\\.ts/, dependencies: ['setup'] },+ { name: 'keep', testMatch: /keep\\.spec\\.ts/ },+ ],+ };+ `,+ 'setup.spec.ts': `+ import { test, expect } from '@playwright/test';+ test('setup-test', async () => {+ expect(1).toBe(2);+ });+ `,+ 'main.spec.ts': `+ import { test } from '@playwright/test';+ test('main-test', async () => {});+ `,+ 'keep.spec.ts': `+ import { test } from '@playwright/test';+ test('keep-test', async () => {});+ `,+ }, { reporter: '', workers: 1 }, undefined, { additionalArgs: ['setup.spec.ts', 'main.spec.ts', 'keep.spec.ts'] });++ expect(result.exitCode).toBe(0);+ expect(result.outputLines).toEqual([+ 'plan projects: setup,main,keep',+ 'ran keep/keep-test',+ ]);});