Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
fix(test runner): do not count statically skipped tests when sharding (#42916)
Yury Semikhatsky5ea3ac5c1ea4
diff --git a/docs/src/test-sharding-js.md b/docs/src/test-sharding-js.mdindex b1c0292c8a..67b8ef623a 100644--- a/docs/src/test-sharding-js.md+++ b/docs/src/test-sharding-js.md@@ -44,6 +44,7 @@ Without the fullyParallel setting, Playwright Test defaults to file-level granul- **With** `fullyParallel: true`: Tests are split at the individual test level, leading to more balanced shard execution.- **Without** `fullyParallel`: Tests are split at the file level, so to balance the shards, it's important to keep your test files small and evenly sized.- To ensure the most effective use of sharding, especially in CI environments, it is recommended to use `fullyParallel: true` when aiming for balanced distribution across shards. Otherwise, you may need to manually organize your test files to avoid imbalances.+- Tests that are statically skipped, for example with [`method: Test.skip`] or [`method: Test.fixme`], are not counted when balancing shards, because they do not run.## Merging reports from multiple shardsdiff --git a/packages/playwright/src/runner/testGroups.ts b/packages/playwright/src/runner/testGroups.tsindex 937ad8ac56..e7db48224b 100644--- a/packages/playwright/src/runner/testGroups.ts+++ b/packages/playwright/src/runner/testGroups.ts@@ -160,10 +160,12 @@ export function filterForShard(shard: { total: number, current: number }, weight//// Shards are still balanced by the number of tests, not files,// even in the case of non-paralleled files.-- let shardableTotal = 0;- for (const group of testGroups)- shardableTotal += group.tests.length;+ //+ // Statically skipped tests take no time, so they are not counted.+ const activeSizes = testGroups.map(group => group.tests.filter(test => test.expectedStatus !== 'skipped').length);+ // If all tests are skipped, balance them as usual.+ const groupSizes = activeSizes.some(Boolean) ? activeSizes : testGroups.map(group => group.tests.length);+ const shardableTotal = groupSizes.reduce((a, b) => a + b, 0);// Each shard gets some tests.const shardSizes = weights.map(w => Math.floor(w * shardableTotal / totalWeight));@@ -180,12 +182,14 @@ export function filterForShard(shard: { total: number, current: number }, weightlet current = 0;const result = new Set<TestGroup>();- for (const group of testGroups) {+ testGroups.forEach((group, index) => {// Any test group goes to the shard that contains the first test of this group.- // So, this shard gets any group that starts at [from; to)- if (current >= from && current < to)+ // So, this shard gets any group that starts at [from; to).+ // Fully skipped groups go along with the preceding group.+ const position = groupSizes[index] ? current : Math.max(current - 1, 0);+ if (position >= from && position < to)result.add(group);- current += group.tests.length;- }+ current += groupSizes[index];+ });return result;}diff --git a/tests/playwright-test/shard.spec.ts b/tests/playwright-test/shard.spec.tsindex 16ef26c251..15774e8e37 100644--- a/tests/playwright-test/shard.spec.ts+++ b/tests/playwright-test/shard.spec.ts@@ -357,3 +357,59 @@ test('should respect custom shard weights', async ({ runInlineTest }) => {]);});});++test('should not count statically skipped tests when sharding', async ({ runInlineTest }) => {+ test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/42875' });+ const tests = {+ 'a.spec.ts': `+ import { test } from '@playwright/test';+ test.describe.configure({ mode: 'parallel' });+ test.skip('skip1', async () => {});+ test.fixme('skip2', async () => {});+ test.describe.skip('suite', () => {+ test('skip3', async () => {});+ test('skip4', async () => {});+ });+ `,+ 'b.spec.ts': `+ import { test } from '@playwright/test';+ test.describe.configure({ mode: 'parallel' });+ for (let i = 1; i <= 4; i++) {+ test('test' + i, async () => {+ console.log('\\n%%b-test' + i + '-done');+ });+ }+ `,+ };++ await test.step('shard 1', async () => {+ const result = await runInlineTest(tests, { shard: '1/2', workers: 1 });+ expect(result.exitCode).toBe(0);+ expect(result.passed).toBe(2);+ expect(result.skipped).toBe(4);+ expect(result.outputLines).toEqual(['b-test1-done', 'b-test2-done']);+ });+ await test.step('shard 2', async () => {+ const result = await runInlineTest(tests, { shard: '2/2', workers: 1 });+ expect(result.exitCode).toBe(0);+ expect(result.passed).toBe(2);+ expect(result.skipped).toBe(0);+ expect(result.outputLines).toEqual(['b-test3-done', 'b-test4-done']);+ });+});++test('should shard when all tests are skipped', async ({ runInlineTest }) => {+ const tests = {+ 'a.spec.ts': `+ import { test } from '@playwright/test';+ test.describe.configure({ mode: 'parallel' });+ test.skip('skip1', async () => {});+ test.skip('skip2', async () => {});+ `,+ };+ for (const shard of ['1/2', '2/2']) {+ const result = await runInlineTest(tests, { shard, workers: 1 });+ expect(result.exitCode).toBe(0);+ expect(result.skipped).toBe(1);+ }+});