Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix(reporter): expose setup/teardown projects to preprocessSuite (#41731)

Simon Knott7b02a7a6d85a
diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md
index 26f8d619e2..67619c914e 100644
--- a/docs/src/test-reporter-api/class-reporter.md
+++ b/docs/src/test-reporter-api/class-reporter.md
@@ -317,6 +317,6 @@ Resolved configuration.
The root suite that contains the projects, files and test cases that will run.
-The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. It contains only the top-level projects being run — setup and dependency projects are not included and cannot be excluded from here.
+The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here.
The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its built-in sharding after [`method: Reporter.preprocessSuite`] returns, unless the returned `implementsSharding` is `true`.
diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts
index 991ac8304e..598a041d8a 100644
--- a/packages/playwright/src/common/test.ts
+++ b/packages/playwright/src/common/test.ts
@@ -58,7 +58,7 @@ export class Suite extends Base {
_parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none';
_fullProject: FullProjectInternal | undefined;
_fileId: string | undefined;
- _preprocessing = false;
+ _preprocessMode: 'editable' | 'readonly' | undefined = undefined;
readonly _type: 'root' | 'project' | 'file' | 'describe';
skip: (reason?: string) => void;
@@ -270,22 +270,28 @@ export class Suite extends Base {
}
private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void {
- if (!this._rootSuite()._preprocessing)
+ const mode = this._resolvePreprocessMode();
+ if (!mode)
throw new Error(`Suite.${type}() can only be called from Reporter.preprocessSuite().`);
+ if (mode === 'readonly')
+ throw new Error(`Suite.${type}() cannot be called on a setup or teardown project; these always run in full.`);
for (const test of this.allTests())
test._applyPlanAnnotation({ type, description: reason, location });
}
exclude(): void {
- if (!this._rootSuite()._preprocessing)
+ const mode = this._resolvePreprocessMode();
+ if (!mode)
throw new Error(`Suite.exclude() can only be called from Reporter.preprocessSuite().`);
if (!this.parent)
throw new Error(`Suite.exclude() cannot be called on the root suite.`);
+ if (mode === 'readonly')
+ throw new Error(`Suite.exclude() cannot be called on a setup or teardown project; these always run in full.`);
this.parent._detach(this);
}
- _rootSuite(): Suite {
- return this.parent?._rootSuite() ?? this;
+ _resolvePreprocessMode(): 'editable' | 'readonly' | undefined {
+ return this._preprocessMode ?? this.parent?._resolvePreprocessMode();
}
}
@@ -353,8 +359,11 @@ export class TestCase extends Base implements reporterTypes.TestCase {
}
private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void {
- if (!this._rootSuite()._preprocessing)
+ const mode = this.parent._resolvePreprocessMode();
+ if (!mode)
throw new Error(`TestCase.${type}() can only be called from Reporter.preprocessSuite().`);
+ if (mode === 'readonly')
+ throw new Error(`TestCase.${type}() cannot be called on a setup or teardown project test; these always run in full.`);
this._applyPlanAnnotation({ type, description: reason, location });
}
@@ -368,15 +377,14 @@ export class TestCase extends Base implements reporterTypes.TestCase {
}
exclude(): void {
- if (!this._rootSuite()._preprocessing)
+ const mode = this.parent._resolvePreprocessMode();
+ if (!mode)
throw new Error(`TestCase.exclude() can only be called from Reporter.preprocessSuite().`);
+ if (mode === 'readonly')
+ throw new Error(`TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.`);
this.parent._detach(this);
}
- _rootSuite(): Suite {
- return this.parent._rootSuite();
- }
-
_serialize(): any {
return {
kind: 'test',
diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts
index 7766ec11f6..f4f092cd5d 100644
--- a/packages/playwright/src/reporters/internalReporter.ts
+++ b/packages/playwright/src/reporters/internalReporter.ts
@@ -55,12 +55,7 @@ export class InternalReporter implements ReporterV2 {
}
async preprocessSuite(config: FullConfig, suite: testNs.Suite) {
- suite._preprocessing = true;
- try {
- return await this._reporter.preprocessSuite?.(config, suite);
- } finally {
- suite._preprocessing = false;
- }
+ return await this._reporter.preprocessSuite?.(config, suite);
}
onBegin(suite: testNs.Suite) {
diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts
index 9485a923ba..9878b14702 100644
--- a/packages/playwright/src/runner/loadUtils.ts
+++ b/packages/playwright/src/runner/loadUtils.ts
@@ -165,7 +165,30 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
}
}
- const preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite);
+ // Temporarily prepend unfiltered dependency projects for preprocessing.
+ const dependencySuites = new Map<commonConfig.FullProjectInternal, testNs.Suite>();
+ for (const [project, type] of projectClosure) {
+ if (type !== 'dependency')
+ continue;
+ const dependencySuite = buildProjectSuite(project, projectSuites.get(project)!);
+ dependencySuite._preprocessMode = 'readonly';
+ dependencySuites.set(project, dependencySuite);
+ rootSuite._prependSuite(dependencySuite);
+ }
+
+ rootSuite._preprocessMode = 'editable';
+ let preprocessResult: Awaited<ReturnType<typeof testRun.reporter.preprocessSuite>>;
+ try {
+ preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite);
+ } finally {
+ // Continue the existing sharding and filtering pipeline with top-level projects only.
+ rootSuite._preprocessMode = undefined;
+ for (const dependencySuite of dependencySuites.values()) {
+ dependencySuite._preprocessMode = undefined;
+ rootSuite._detach(dependencySuite);
+ }
+ }
+
// Shard only the top-level projects.
if (config.config.shard && !preprocessResult?.implementsSharding) {
// Create test groups for top-level projects.
@@ -193,16 +216,13 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => testRun.postShardTestFilters.every(filter => filter(test)));
const topLevelProjects = [];
- // Now prepend dependency projects without filtration.
{
// Filtering 'only' and sharding might have reduced the number of top-level projects.
// Build the project closure to only include dependencies that are still needed.
- const projectClosure = new Map(buildProjectsClosure(rootSuite.suites.map(suite => suite._fullProject!)));
-
- // Clone file suites for dependency projects.
- for (const [project, level] of projectClosure.entries()) {
- if (level === 'dependency')
- rootSuite._prependSuite(buildProjectSuite(project, projectSuites.get(project)!));
+ const finalProjectClosure = buildProjectsClosure(rootSuite.suites.map(suite => suite._fullProject!));
+ for (const [project, type] of finalProjectClosure) {
+ if (type === 'dependency')
+ rootSuite._prependSuite(dependencySuites.get(project)!);
else
topLevelProjects.push(project);
}
diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts
index 47e406bcd1..f7809c29c6 100644
--- a/packages/playwright/types/testReporter.d.ts
+++ b/packages/playwright/types/testReporter.d.ts
@@ -153,8 +153,7 @@ export interface Reporter {
* @param suite The root suite that contains the projects, files and test cases that will run.
*
* The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that
- * match the current invocation. It contains only the top-level projects being run — setup and dependency projects are
- * not included and cannot be excluded from here.
+ * match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here.
*
* The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its
* built-in sharding after
diff --git a/tests/playwright-test/reporter-preprocess-suite.spec.ts b/tests/playwright-test/reporter-preprocess-suite.spec.ts
index 7e2f163428..d9139a34c5 100644
--- a/tests/playwright-test/reporter-preprocess-suite.spec.ts
+++ b/tests/playwright-test/reporter-preprocess-suite.spec.ts
@@ -392,15 +392,40 @@ test('multiple reporters declaring implementsSharding throws', async ({ runInlin
expect(result.outputLines.join('\n')).toContain(`Multiple reporters declare 'implementsSharding'`);
});
-test('plan.suite contains only top-level projects, not dependency/setup projects', async ({ runInlineTest }) => {
+test('plan.suite exposes setup/teardown dependency projects but they are read-only', async ({ runInlineTest }) => {
const result = await runInlineTest({
'reporter.ts': `
class Reporter {
async preprocessSuite(config, suite) {
- // The suite only exposes top-level projects, so a reporter has no handle on
- // setup/dependency project tests and therefore cannot exclude them.
console.log('%% plan projects: ' + suite.suites.map(s => s.title).join(','));
console.log('%% plan tests: ' + suite.allTests().map(t => t.title).join(','));
+ this.preprocessedTests = new Set(suite.allTests());
+ const setupTest = suite.allTests().find(t => t.title === 'setup-test');
+ for (const method of ['skip', 'fixme', 'fail', 'exclude']) {
+ try {
+ setupTest[method]();
+ console.log('%% dep-' + method + ': no-throw');
+ } catch (e) {
+ console.log('%% dep-' + method + ': ' + e.message);
+ }
+ }
+ const setupProject = suite.suites.find(s => s.title === 'setup');
+ try {
+ setupProject.exclude();
+ console.log('%% dep-suite-exclude: no-throw');
+ } catch (e) {
+ console.log('%% dep-suite-exclude: ' + e.message);
+ }
+ }
+ onBegin(config, suite) {
+ console.log('%% same test objects: ' + suite.allTests().every(test => this.preprocessedTests.has(test)));
+ const setupTest = suite.allTests().find(t => t.title === 'setup-test');
+ try {
+ setupTest.skip();
+ console.log('%% dep-after-preprocess: no-throw');
+ } catch (e) {
+ console.log('%% dep-after-preprocess: ' + e.message);
+ }
}
onTestEnd(test, result) {
console.log('%% ran ' + test.parent.project().name + '/' + test.title);
@@ -412,7 +437,8 @@ test('plan.suite contains only top-level projects, not dependency/setup projects
module.exports = {
reporter: './reporter.ts',
projects: [
- { name: 'setup', testMatch: /a\\.setup\\.ts/ },
+ { name: 'setup', testMatch: /a\\.setup\\.ts/, teardown: 'teardown' },
+ { name: 'teardown', testMatch: /a\\.teardown\\.ts/ },
{ name: 'main', testMatch: /a\\.test\\.ts/, dependencies: ['setup'] },
],
};
@@ -421,6 +447,10 @@ test('plan.suite contains only top-level projects, not dependency/setup projects
import { test } from '@playwright/test';
test('setup-test', async () => {});
`,
+ 'a.teardown.ts': `
+ import { test } from '@playwright/test';
+ test('teardown-test', async () => {});
+ `,
'a.test.ts': `
import { test } from '@playwright/test';
test('main-test', async () => {});
@@ -428,11 +458,65 @@ test('plan.suite contains only top-level projects, not dependency/setup projects
}, { reporter: '', workers: 1 }, undefined, { additionalArgs: ['--project=main'] });
expect(result.exitCode).toBe(0);
- // plan only sees the top-level 'main' project; the 'setup' dependency is prepended afterwards.
- expect(result.outputLines).toContain('plan projects: main');
- // 'setup-test' is absent from the plan suite, proving setup/dependency tests are not exposed.
- expect(result.outputLines).toContain('plan tests: main-test');
- // Both the dependency and the main project still run.
- expect(result.outputLines).toContain('ran setup/setup-test');
- expect(result.outputLines).toContain('ran main/main-test');
+ expect(result.outputLines).toEqual([
+ 'plan projects: teardown,setup,main',
+ 'plan tests: teardown-test,setup-test,main-test',
+ 'dep-skip: TestCase.skip() cannot be called on a setup or teardown project test; these always run in full.',
+ 'dep-fixme: TestCase.fixme() cannot be called on a setup or teardown project test; these always run in full.',
+ 'dep-fail: TestCase.fail() cannot be called on a setup or teardown project test; these always run in full.',
+ 'dep-exclude: TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.',
+ 'dep-suite-exclude: Suite.exclude() cannot be called on a setup or teardown project; these always run in full.',
+ 'same test objects: true',
+ 'dep-after-preprocess: TestCase.skip() can only be called from Reporter.preprocessSuite().',
+ 'ran setup/setup-test',
+ 'ran main/main-test',
+ 'ran teardown/teardown-test',
+ ]);
+});
+
+test('plan.suite temporarily exposes dependencies without changing final project selection', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'reporter.ts': `
+ class Reporter {
+ async preprocessSuite(config, suite) {
+ console.log('%% plan projects: ' + suite.suites.map(suite => suite.title).join(','));
+ suite.suites.find(suite => suite.title === 'main').exclude();
+ }
+ onTestEnd(test, result) {
+ console.log('%% ran ' + test.parent.project().name + '/' + test.title);
+ }
+ }
+ module.exports = Reporter;
+ `,
+ 'playwright.config.ts': `
+ module.exports = {
+ reporter: './reporter.ts',
+ projects: [
+ { name: 'setup', testMatch: /setup\\.spec\\.ts/ },
+ { name: 'main', testMatch: /main\\.spec\\.ts/, dependencies: ['setup'] },
+ { name: 'keep', testMatch: /keep\\.spec\\.ts/ },
+ ],
+ };
+ `,
+ 'setup.spec.ts': `
+ import { test, expect } from '@playwright/test';
+ test('setup-test', async () => {
+ expect(1).toBe(2);
+ });
+ `,
+ 'main.spec.ts': `
+ import { test } from '@playwright/test';
+ test('main-test', async () => {});
+ `,
+ 'keep.spec.ts': `
+ import { test } from '@playwright/test';
+ test('keep-test', async () => {});
+ `,
+ }, { reporter: '', workers: 1 }, undefined, { additionalArgs: ['setup.spec.ts', 'main.spec.ts', 'keep.spec.ts'] });
+
+ expect(result.exitCode).toBe(0);
+ expect(result.outputLines).toEqual([
+ 'plan projects: setup,main,keep',
+ 'ran keep/keep-test',
+ ]);
});