Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat: custom sharding weights (#38624)
Simon Knottcb93c448a482
diff --git a/docs/src/test-sharding-js.md b/docs/src/test-sharding-js.mdindex f0722a9bd4..44b583f0f0 100644--- a/docs/src/test-sharding-js.md+++ b/docs/src/test-sharding-js.md@@ -45,6 +45,12 @@ Without the fullyParallel setting, Playwright Test defaults to file-level granul- **Without** `fullyParallel`: Tests are split at the file level, so to balance the shards, it's important to keep your test files small and evenly sized.- To ensure the most effective use of sharding, especially in CI environments, it is recommended to use `fullyParallel: true` when aiming for balanced distribution across shards. Otherwise, you may need to manually organize your test files to avoid imbalances.+## Rebalancing Shards++```bash+npx playwright test --shard=1/4 --shard-weights=26:24:25:25+```+## Merging reports from multiple shardsIn the previous example, each test shard has its own test report. If you want to have a combined report showing all the test results from all the shards, you can merge them.diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.tsindex ffe6a5e438..ad2de5304f 100644--- a/packages/playwright/src/common/ipc.ts+++ b/packages/playwright/src/common/ipc.ts@@ -37,6 +37,7 @@ export type ConfigCLIOverrides = {reporter?: ReporterDescription[];additionalReporters?: ReporterDescription[];shard?: { current: number, total: number };+ shardWeights?: number[];timeout?: number;tsconfig?: string;ignoreSnapshots?: boolean;diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.tsindex 3695c6de15..47c4adb633 100644--- a/packages/playwright/src/program.ts+++ b/packages/playwright/src/program.ts@@ -300,6 +300,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverridretries: options.retries ? parseInt(options.retries, 10) : undefined,reporter: resolveReporterOption(options.reporter),shard: resolveShardOption(options.shard),+ shardWeights: resolveShardWeightsOption(options.shardWeights),timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,tsconfig: options.tsconfig ? path.resolve(process.cwd(), options.tsconfig) : undefined,ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined,@@ -372,6 +373,18 @@ function resolveShardOption(shard?: string): ConfigCLIOverrides['shard'] {return { current, total };}+function resolveShardWeightsOption(shardWeights?: string): ConfigCLIOverrides['shardWeights'] {+ if (!shardWeights)+ return undefined;++ return shardWeights.split(':').map(w => {+ const weight = parseInt(w, 10);+ if (isNaN(weight) || weight < 0)+ throw new Error(`--shard-weights "${shardWeights}" weights must be non-negative numbers`);+ return weight;+ });+}+function resolveReporter(id: string) {if (builtInReporters.includes(id as any))return id;@@ -411,6 +424,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?:['--retries <retries>', { description: `Maximum retry count for flaky tests, zero for no retries (default: no retries)` }],['--run-agents', { description: `Run agents to generate the code for page.perform` }],['--shard <shard>', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }],+ ['--shard-weights <weights>', { description: `Weights for each shard, colon-separated, for example "2:1:1" for 3 shards where the first shard should be allocated half of the work` }],['--test-list <file>', { description: `Path to a file containing a list of tests to run. See https://playwright.dev/docs/test-cli for more details.` }],['--test-list-invert <file>', { description: `Path to a file containing a list of tests to skip. See https://playwright.dev/docs/test-cli for more details.` }],['--timeout <timeout>', { description: `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})` }],diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.tsindex 192bdc9978..19b0393bbc 100644--- a/packages/playwright/src/runner/loadUtils.ts+++ b/packages/playwright/src/runner/loadUtils.ts@@ -189,7 +189,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho}// Shard test groups.- const testGroupsInThisShard = filterForShard(config.config.shard, testGroups);+ const testGroupsInThisShard = filterForShard(config.config.shard, config.configCLIOverrides.shardWeights, testGroups);const testsInThisShard = new Set<TestCase>();for (const group of testGroupsInThisShard) {for (const test of group.tests)diff --git a/packages/playwright/src/runner/testGroups.ts b/packages/playwright/src/runner/testGroups.tsindex 5743c7044b..c70ebd7586 100644--- a/packages/playwright/src/runner/testGroups.ts+++ b/packages/playwright/src/runner/testGroups.ts@@ -130,7 +130,12 @@ export function createTestGroups(projectSuite: Suite, expectedParallelism: numbereturn result;}-export function filterForShard(shard: { total: number, current: number }, testGroups: TestGroup[]): Set<TestGroup> {+export function filterForShard(shard: { total: number, current: number }, weights: number[] | undefined, testGroups: TestGroup[]): Set<TestGroup> {+ weights ??= Array.from({ length: shard.total }, () => 1);+ if (weights.length !== shard.total)+ throw new Error(`--shard-weights number of weights must match the shard total of ${shard.total}`);++ const totalWeight = weights.reduce((a, b) => a + b, 0);// Note that sharding works based on test groups.// This means parallel files will be sharded by single tests,// while non-parallel files will be sharded by the whole file.@@ -143,13 +148,17 @@ export function filterForShard(shard: { total: number, current: number }, testGrshardableTotal += group.tests.length;// Each shard gets some tests.- const shardSize = Math.floor(shardableTotal / shard.total);- // First few shards get one more test each.- const extraOne = shardableTotal - shardSize * shard.total;+ const shardSizes = weights.map(w => Math.floor(w * shardableTotal / totalWeight));+ const remainder = shardableTotal - shardSizes.reduce((a, b) => a + b, 0);+ for (let i = 0; i < remainder; i++) {+ // First few shards get one more test each.+ shardSizes[i % shardSizes.length]++;+ }- const currentShard = shard.current - 1; // Make it zero-based for calculations.- const from = shardSize * currentShard + Math.min(extraOne, currentShard);- const to = from + shardSize + (currentShard < extraOne ? 1 : 0);+ let from = 0;+ for (let i = 0; i < shard.current - 1; i++)+ from += shardSizes[i];+ const to = from + shardSizes[shard.current - 1];let current = 0;const result = new Set<TestGroup>();diff --git a/tests/playwright-test/shard.spec.ts b/tests/playwright-test/shard.spec.tsindex dcd75a9d23..bc3eb2f434 100644--- a/tests/playwright-test/shard.spec.ts+++ b/tests/playwright-test/shard.spec.ts@@ -16,6 +16,8 @@import { test, expect } from './playwright-test-fixtures';+test.describe.configure({ mode: 'parallel' });+const tests = {'a1.spec.ts': `import { test } from '@playwright/test';@@ -330,3 +332,28 @@ test('should shard tests with beforeAll based on shards total instead of workersexpect(result.outputLines).toEqual(['beforeAll', 'test7']);}});++test('should respect custom shard weights', async ({ runInlineTest }) => {+ await test.step('shard 1', async () => {+ const result = await runInlineTest(tests, { 'shard': '1/2', 'shard-weights': '40:60', 'workers': 1 });+ expect.soft(result.exitCode).toBe(0);+ expect.soft(result.outputLines).toEqual([+ 'a1-test1-done',+ 'a1-test2-done',+ 'a1-test3-done',+ 'a1-test4-done',+ ]);+ });+ await test.step('shard 2', async () => {+ const result = await runInlineTest(tests, { 'shard': '2/2', 'shard-weights': '40:60', 'workers': 1 });+ expect.soft(result.exitCode).toBe(0);+ expect.soft(result.outputLines).toEqual([+ 'a2-test1-done',+ 'a2-test2-done',+ 'a3-test1-done',+ 'a3-test2-done',+ 'a4-test1-done',+ 'a4-test2-done',+ ]);+ });+});