Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(test-runner): add --shuffle to run tests in random order (#42877)
Pavel Feldman9234738fbcb7
diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.mdindex 5a2bb5ca52..a38210032b 100644--- a/docs/src/test-cli-js.md+++ b/docs/src/test-cli-js.md@@ -105,6 +105,7 @@ npx playwright test --ui| `--reporter <reporter>` | Reporter to use, comma-separated, can be "dot", "line", "list", or others (default: "list" locally and "dot" on CI). You can also pass a path to a custom reporter file. || `--retries <retries>` | Maximum retry count for flaky tests, zero for no retries (default: no retries). || `--shard <shard>` | Shard tests and execute only the selected shard, specified in the form "current/all", 1-based, e.g., "3/5". |+| `--shuffle [seed]` | Schedule tests in a random order: test files are shuffled, and so are individual tests in [parallel mode](./test-parallel.md). Tests that must run together, for example in a serial suite, keep their order. The seed is printed at the start of the run, pass it to reproduce the same order. || `--test-list <file>` | Path to a file containing a list of tests to run. See [test list](#test-list) for details. || `--test-list-invert <file>` | Path to a file containing a list of tests to skip. See [test list](#test-list) for details. || `--timeout <timeout>` | Specify test timeout threshold in milliseconds, zero for unlimited (default: 30 seconds). |diff --git a/packages/playwright/src/cli/testActions.ts b/packages/playwright/src/cli/testActions.tsindex f3cd08f907..c03abcff80 100644--- a/packages/playwright/src/cli/testActions.ts+++ b/packages/playwright/src/cli/testActions.ts@@ -128,6 +128,7 @@ function overridesFromOptions(options: { [key: string]: any }): ipc.ConfigCLIOvereporter: resolveReporterOption(options.reporter),additionalReporters: resolveReporterOption(options.addReporter),shard: resolveShardOption(options.shard),+ shuffle: resolveShuffleOption(options.shuffle),timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,tsconfig: options.tsconfig ? path.resolve(process.cwd(), options.tsconfig) : undefined,ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined,@@ -170,6 +171,14 @@ function resolveReporterOption(reporter?: string): ReporterDescription[] | undefreturn reporter.split(',').map((r: string) => [resolveReporter(r)]);}+function resolveShuffleOption(shuffle?: string | boolean): string | undefined {+ if (!shuffle)+ return undefined;+ if (shuffle === true)+ return String(Math.floor(Math.random() * 1e9));+ return shuffle;+}+function resolveShardOption(shard?: string): ipc.ConfigCLIOverrides['shard'] {if (!shard)return undefined;diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.tsindex 975fa56d0b..872e06b30e 100644--- a/packages/playwright/src/common/ipc.ts+++ b/packages/playwright/src/common/ipc.ts@@ -38,6 +38,7 @@ export type ConfigCLIOverrides = {reporter?: ReporterDescription[];additionalReporters?: ReporterDescription[];shard?: { current: number, total: number };+ shuffle?: string;timeout?: number;tsconfig?: string;ignoreSnapshots?: boolean;diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.tsindex 2bedd91609..9042f96522 100644--- a/packages/playwright/src/program.ts+++ b/packages/playwright/src/program.ts@@ -235,6 +235,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?:['--shard <shard>', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }],['--test-list <file>', { description: `Path to a file containing a list of tests to run. See https://playwright.dev/docs/test-cli for more details.` }],['--test-list-invert <file>', { description: `Path to a file containing a list of tests to skip. See https://playwright.dev/docs/test-cli for more details.` }],+ ['--shuffle [seed]', { description: `Schedule test files, and tests in parallel mode, in a random order; pass a seed to reproduce a previous order` }],['--timeout <timeout>', { description: `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${config.defaultTimeout})` }],['--trace <mode>', { description: `Force tracing mode`, choices: kTraceModes as string[] }],['--tsconfig <path>', { description: `Path to a single tsconfig applicable to all imported files (default: look up tsconfig for each imported file separately)` }],diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.tsindex 062584f54d..00bbb0172b 100644--- a/packages/playwright/src/reporters/base.ts+++ b/packages/playwright/src/reporters/base.ts@@ -242,9 +242,10 @@ export class TerminalReporter implements ReporterV2 {protected generateStartingMessage() {const jobs = this.config.metadata.actualWorkers ?? this.config.workers;const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : '';+ const shuffleDetails = this.config.metadata.shuffleSeed ? `, shuffle seed ${this.config.metadata.shuffleSeed}` : '';if (!this.totalTestCount)return '';- return '\n' + this.screen.colors.dim('Running ') + this.totalTestCount + this.screen.colors.dim(` test${this.totalTestCount !== 1 ? 's' : ''} using `) + jobs + this.screen.colors.dim(` worker${jobs !== 1 ? 's' : ''}${shardDetails}`);+ return '\n' + this.screen.colors.dim('Running ') + this.totalTestCount + this.screen.colors.dim(` test${this.totalTestCount !== 1 ? 's' : ''} using `) + jobs + this.screen.colors.dim(` worker${jobs !== 1 ? 's' : ''}${shardDetails}${shuffleDetails}`);}protected getSlowTests(): [string, number][] {diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.tsindex 17ea0785a4..1361860eab 100644--- a/packages/playwright/src/runner/tasks.ts+++ b/packages/playwright/src/runner/tasks.ts@@ -30,7 +30,7 @@ import { applySuggestedRebaselines, clearSuggestedRebaselines } from './rebase';import { TaskRunner } from './taskRunner';import { detectChangedTestFiles } from './vcs';import { cc, config as commonConfig, FullConfigInternal, suiteUtils, test as testNs } from '../common';-import { createTestGroups } from '../runner/testGroups';+import { createTestGroups, shuffleTestGroups } from '../runner/testGroups';import { createTitleMatcher, forceRegExp, removeDirAndLogToConsole } from '../util';import type { TestGroup } from '../runner/testGroups';@@ -418,6 +418,8 @@ function createPhasesTask(): Task<TestRun> {}testRun.config.config.metadata.actualWorkers = Math.min(testRun.config.config.workers, maxConcurrentTestGroups);+ if (testRun.config.configCLIOverrides.shuffle)+ testRun.config.config.metadata.shuffleSeed = testRun.config.configCLIOverrides.shuffle;},};}@@ -449,6 +451,9 @@ function createRunTestsTask(): Task<TestRun> {phaseTestGroups.push(...testGroups);}+ if (testRun.config.configCLIOverrides.shuffle)+ shuffleTestGroups(phaseTestGroups, testRun.config.configCLIOverrides.shuffle);+if (phaseTestGroups.length) {await dispatcher!.run(phaseTestGroups, extraEnvByProjectId);await dispatcher.stop();diff --git a/packages/playwright/src/runner/testGroups.ts b/packages/playwright/src/runner/testGroups.tsindex 5c7bebc340..937ad8ac56 100644--- a/packages/playwright/src/runner/testGroups.ts+++ b/packages/playwright/src/runner/testGroups.ts@@ -14,6 +14,8 @@* limitations under the License.*/+import { calculateSha1 } from '@utils/crypto';+import type { test } from '../common';export type TestGroup = {@@ -141,6 +143,11 @@ export function createTestGroups(projectSuite: test.Suite, expectedParallelism:return result;}+export function shuffleTestGroups(testGroups: TestGroup[], seed: string) {+ const keys = new Map(testGroups.map(group => [group, calculateSha1(seed + '\x1e' + group.tests[0].id)]));+ testGroups.sort((a, b) => keys.get(a)!.localeCompare(keys.get(b)!));+}+export function filterForShard(shard: { total: number, current: number }, weights: number[] | undefined, testGroups: TestGroup[]): Set<TestGroup> {weights ??= Array.from({ length: shard.total }, () => 1);if (weights.length !== shard.total)diff --git a/tests/playwright-test/shuffle.spec.ts b/tests/playwright-test/shuffle.spec.tsnew file mode 100644index 0000000000..2f9af40c74--- /dev/null+++ b/tests/playwright-test/shuffle.spec.ts@@ -0,0 +1,84 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { test, expect } from './playwright-test-fixtures';++const count = 10;++function testFile(name: string) {+ return `+ import { test } from '@playwright/test';+ ${Array.from({ length: count }, (_, i) => `test('test${i}', () => console.log('\\n%%${name}-test${i}'));`).join('\n')}+ `;+}++const fileNames = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'];+const files = Object.fromEntries(fileNames.map(name => [`${name}.spec.ts`, testFile(name)]));+const testsInFile = (name: string) => Array.from({ length: count }, (_, i) => `${name}-test${i}`);+const naturalOrder = fileNames.flatMap(testsInFile);++test('should shuffle files and keep tests within a file in order', async ({ runInlineTest }) => {+ const result = await runInlineTest(files, { workers: 1, shuffle: '42' });+ expect(result.exitCode).toBe(0);+ expect(result.passed).toBe(naturalOrder.length);+ expect(result.output).toContain('shuffle seed 42');+ expect(result.outputLines).not.toEqual(naturalOrder);+ const fileOrder = [...new Set(result.outputLines.map(line => line.split('-')[0]))];+ expect(result.outputLines).toEqual(fileOrder.flatMap(testsInFile));++ const result2 = await runInlineTest(files, { workers: 1, shuffle: '42' });+ expect(result2.outputLines).toEqual(result.outputLines);++ const result3 = await runInlineTest(files, { workers: 1, shuffle: '43' });+ expect(result3.outputLines).not.toEqual(result.outputLines);+});++test('should shuffle individual tests in fully parallel mode', async ({ runInlineTest }) => {+ const result = await runInlineTest(files, { 'workers': 1, 'shuffle': '42', 'fully-parallel': true });+ expect(result.exitCode).toBe(0);+ expect(result.passed).toBe(naturalOrder.length);+ const aTests = result.outputLines.filter(line => line.startsWith('a-'));+ expect(aTests).not.toEqual(testsInFile('a'));+ expect([...aTests].sort()).toEqual([...testsInFile('a')].sort());+});++test('should pick and print a random seed', async ({ runInlineTest }) => {+ const result = await runInlineTest(files, { workers: 1, shuffle: true });+ expect(result.exitCode).toBe(0);+ const seed = result.output.match(/shuffle seed (\d+)/)![1];+ const result2 = await runInlineTest(files, { workers: 1, shuffle: seed });+ expect(result2.outputLines).toEqual(result.outputLines);+});++test('should keep serial suites together in parallel mode', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'a.spec.ts': `+ import { test, expect } from '@playwright/test';+ test.describe.configure({ mode: 'parallel' });+ ${Array.from({ length: count }, (_, i) => `test('test${i}', () => console.log('\\n%%test${i}'));`).join('\n')}+ test.describe.serial('serial', () => {+ let counter = 0;+ ${Array.from({ length: count }, (_, i) => `test('serial${i}', () => { expect(counter++).toBe(${i}); console.log('\\n%%serial${i}'); });`).join('\n')}+ });+ `,+ }, { workers: 1, shuffle: '7' });+ expect(result.exitCode).toBe(0);+ expect(result.passed).toBe(2 * count);+ const lines = result.outputLines;+ const serialStart = lines.indexOf('serial0');+ expect(lines.slice(serialStart, serialStart + count)).toEqual(Array.from({ length: count }, (_, i) => `serial${i}`));+ expect(lines.filter(line => line.startsWith('test'))).not.toEqual(Array.from({ length: count }, (_, i) => `test${i}`));+});