Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(test-runner): annotate serial suites for custom sharding (#42164)

Simon Knott53344b3c83a2
diff --git a/packages/playwright/src/common/suiteUtils.ts b/packages/playwright/src/common/suiteUtils.ts
index 5748da9cf1..1248aedc92 100644
--- a/packages/playwright/src/common/suiteUtils.ts
+++ b/packages/playwright/src/common/suiteUtils.ts
@@ -57,6 +57,8 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit
for (let parentSuite: Suite | undefined = suite; parentSuite; parentSuite = parentSuite.parent) {
if (parentSuite._staticAnnotations.length)
test.annotations.unshift(...parentSuite._staticAnnotations);
+ if (parentSuite._parallelMode === 'serial')
+ test.annotations.unshift({ type: 'serial', location: parentSuite.location });
if (parentSuite._locks.length)
test._locks.push(...parentSuite._locks);
if (inheritedRetries === undefined && parentSuite._retries !== undefined)
diff --git a/tests/playwright-test/reporter-preprocess.spec.ts b/tests/playwright-test/reporter-preprocess.spec.ts
index 12930c32f5..66a84ba4fb 100644
--- a/tests/playwright-test/reporter-preprocess.spec.ts
+++ b/tests/playwright-test/reporter-preprocess.spec.ts
@@ -522,3 +522,41 @@ test('plan.suite temporarily exposes dependencies without changing final project
'ran keep/keep-test',
]);
});
+
+test('serial suites expose a serial annotation for custom sharding', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'reporter.ts': `
+ class Reporter {
+ async preprocess({ suite }) {
+ for (const t of suite.allTests())
+ console.log('%% ' + t.title + ':' + t.annotations.filter(a => a.type === 'serial').length);
+ }
+ }
+ module.exports = Reporter;
+ `,
+ 'playwright.config.ts': `module.exports = { reporter: './reporter.ts' };`,
+ 'a.test.ts': `
+ import { test } from '@playwright/test';
+ test('plain', async () => {});
+ test.describe.serial('s', () => {
+ test('serial', async () => {});
+ });
+ test.describe('c', () => {
+ test.describe.configure({ mode: 'serial' });
+ test('configure', async () => {});
+ });
+ test.describe.serial('outer', () => {
+ test.describe.serial('inner', () => {
+ test('nested', async () => {});
+ });
+ });
+ `,
+ }, { reporter: '', workers: 1 });
+ expect(result.exitCode).toBe(0);
+ expect(result.outputLines).toEqual([
+ 'plain:0',
+ 'serial:1',
+ 'configure:1',
+ 'nested:2',
+ ]);
+});
diff --git a/tests/playwright-test/test-modifiers.spec.ts b/tests/playwright-test/test-modifiers.spec.ts
index abe3f1682c..9a31b5ae72 100644
--- a/tests/playwright-test/test-modifiers.spec.ts
+++ b/tests/playwright-test/test-modifiers.spec.ts
@@ -695,10 +695,11 @@ test('static modifiers should be added in serial mode', async ({ runInlineTest }
expect(result.passed).toBe(0);
expect(result.skipped).toBe(2);
expect(result.didNotRun).toBe(1);
- expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'slow', location: { file: expect.any(String), line: 6, column: 14 } }]);
- expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'fixme', location: { file: expect.any(String), line: 9, column: 12 } }]);
- expect(result.report.suites[0].specs[2].tests[0].annotations).toEqual([{ type: 'skip', location: { file: expect.any(String), line: 11, column: 12 } }]);
- expect(result.report.suites[0].specs[3].tests[0].annotations).toEqual([]);
+ const serial = { type: 'serial', location: { file: expect.any(String), line: 0, column: 0 } };
+ expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([serial, { type: 'slow', location: { file: expect.any(String), line: 6, column: 14 } }]);
+ expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([serial, { type: 'fixme', location: { file: expect.any(String), line: 9, column: 12 } }]);
+ expect(result.report.suites[0].specs[2].tests[0].annotations).toEqual([serial, { type: 'skip', location: { file: expect.any(String), line: 11, column: 12 } }]);
+ expect(result.report.suites[0].specs[3].tests[0].annotations).toEqual([serial]);
});
test('should contain only one slow modifier', async ({ runInlineTest }) => {