Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(test-runner): add --shuffle to run tests in random order (#42877)

Pavel Feldman9234738fbcb7
diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md
index 5a2bb5ca52..a38210032b 100644
--- a/docs/src/test-cli-js.md
+++ b/docs/src/test-cli-js.md
@@ -105,6 +105,7 @@ npx playwright test --ui
| `--reporter <reporter>` | Reporter to use, comma-separated, can be "dot", "line", "list", or others (default: "list" locally and "dot" on CI). You can also pass a path to a custom reporter file. |
| `--retries <retries>` | Maximum retry count for flaky tests, zero for no retries (default: no retries). |
| `--shard <shard>` | Shard tests and execute only the selected shard, specified in the form "current/all", 1-based, e.g., "3/5". |
+| `--shuffle [seed]` | Schedule tests in a random order: test files are shuffled, and so are individual tests in [parallel mode](./test-parallel.md). Tests that must run together, for example in a serial suite, keep their order. The seed is printed at the start of the run, pass it to reproduce the same order. |
| `--test-list <file>` | Path to a file containing a list of tests to run. See [test list](#test-list) for details. |
| `--test-list-invert <file>` | Path to a file containing a list of tests to skip. See [test list](#test-list) for details. |
| `--timeout <timeout>` | Specify test timeout threshold in milliseconds, zero for unlimited (default: 30 seconds). |
diff --git a/packages/playwright/src/cli/testActions.ts b/packages/playwright/src/cli/testActions.ts
index f3cd08f907..c03abcff80 100644
--- a/packages/playwright/src/cli/testActions.ts
+++ b/packages/playwright/src/cli/testActions.ts
@@ -128,6 +128,7 @@ function overridesFromOptions(options: { [key: string]: any }): ipc.ConfigCLIOve
reporter: resolveReporterOption(options.reporter),
additionalReporters: resolveReporterOption(options.addReporter),
shard: resolveShardOption(options.shard),
+ shuffle: resolveShuffleOption(options.shuffle),
timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,
tsconfig: options.tsconfig ? path.resolve(process.cwd(), options.tsconfig) : undefined,
ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined,
@@ -170,6 +171,14 @@ function resolveReporterOption(reporter?: string): ReporterDescription[] | undef
return reporter.split(',').map((r: string) => [resolveReporter(r)]);
}
+function resolveShuffleOption(shuffle?: string | boolean): string | undefined {
+ if (!shuffle)
+ return undefined;
+ if (shuffle === true)
+ return String(Math.floor(Math.random() * 1e9));
+ return shuffle;
+}
+
function resolveShardOption(shard?: string): ipc.ConfigCLIOverrides['shard'] {
if (!shard)
return undefined;
diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts
index 975fa56d0b..872e06b30e 100644
--- a/packages/playwright/src/common/ipc.ts
+++ b/packages/playwright/src/common/ipc.ts
@@ -38,6 +38,7 @@ export type ConfigCLIOverrides = {
reporter?: ReporterDescription[];
additionalReporters?: ReporterDescription[];
shard?: { current: number, total: number };
+ shuffle?: string;
timeout?: number;
tsconfig?: string;
ignoreSnapshots?: boolean;
diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts
index 2bedd91609..9042f96522 100644
--- a/packages/playwright/src/program.ts
+++ b/packages/playwright/src/program.ts
@@ -235,6 +235,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?:
['--shard <shard>', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }],
['--test-list <file>', { description: `Path to a file containing a list of tests to run. See https://playwright.dev/docs/test-cli for more details.` }],
['--test-list-invert <file>', { description: `Path to a file containing a list of tests to skip. See https://playwright.dev/docs/test-cli for more details.` }],
+ ['--shuffle [seed]', { description: `Schedule test files, and tests in parallel mode, in a random order; pass a seed to reproduce a previous order` }],
['--timeout <timeout>', { description: `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${config.defaultTimeout})` }],
['--trace <mode>', { description: `Force tracing mode`, choices: kTraceModes as string[] }],
['--tsconfig <path>', { description: `Path to a single tsconfig applicable to all imported files (default: look up tsconfig for each imported file separately)` }],
diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts
index 062584f54d..00bbb0172b 100644
--- a/packages/playwright/src/reporters/base.ts
+++ b/packages/playwright/src/reporters/base.ts
@@ -242,9 +242,10 @@ export class TerminalReporter implements ReporterV2 {
protected generateStartingMessage() {
const jobs = this.config.metadata.actualWorkers ?? this.config.workers;
const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : '';
+ const shuffleDetails = this.config.metadata.shuffleSeed ? `, shuffle seed ${this.config.metadata.shuffleSeed}` : '';
if (!this.totalTestCount)
return '';
- return '\n' + this.screen.colors.dim('Running ') + this.totalTestCount + this.screen.colors.dim(` test${this.totalTestCount !== 1 ? 's' : ''} using `) + jobs + this.screen.colors.dim(` worker${jobs !== 1 ? 's' : ''}${shardDetails}`);
+ return '\n' + this.screen.colors.dim('Running ') + this.totalTestCount + this.screen.colors.dim(` test${this.totalTestCount !== 1 ? 's' : ''} using `) + jobs + this.screen.colors.dim(` worker${jobs !== 1 ? 's' : ''}${shardDetails}${shuffleDetails}`);
}
protected getSlowTests(): [string, number][] {
diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts
index 17ea0785a4..1361860eab 100644
--- a/packages/playwright/src/runner/tasks.ts
+++ b/packages/playwright/src/runner/tasks.ts
@@ -30,7 +30,7 @@ import { applySuggestedRebaselines, clearSuggestedRebaselines } from './rebase';
import { TaskRunner } from './taskRunner';
import { detectChangedTestFiles } from './vcs';
import { cc, config as commonConfig, FullConfigInternal, suiteUtils, test as testNs } from '../common';
-import { createTestGroups } from '../runner/testGroups';
+import { createTestGroups, shuffleTestGroups } from '../runner/testGroups';
import { createTitleMatcher, forceRegExp, removeDirAndLogToConsole } from '../util';
import type { TestGroup } from '../runner/testGroups';
@@ -418,6 +418,8 @@ function createPhasesTask(): Task<TestRun> {
}
testRun.config.config.metadata.actualWorkers = Math.min(testRun.config.config.workers, maxConcurrentTestGroups);
+ if (testRun.config.configCLIOverrides.shuffle)
+ testRun.config.config.metadata.shuffleSeed = testRun.config.configCLIOverrides.shuffle;
},
};
}
@@ -449,6 +451,9 @@ function createRunTestsTask(): Task<TestRun> {
phaseTestGroups.push(...testGroups);
}
+ if (testRun.config.configCLIOverrides.shuffle)
+ shuffleTestGroups(phaseTestGroups, testRun.config.configCLIOverrides.shuffle);
+
if (phaseTestGroups.length) {
await dispatcher!.run(phaseTestGroups, extraEnvByProjectId);
await dispatcher.stop();
diff --git a/packages/playwright/src/runner/testGroups.ts b/packages/playwright/src/runner/testGroups.ts
index 5c7bebc340..937ad8ac56 100644
--- a/packages/playwright/src/runner/testGroups.ts
+++ b/packages/playwright/src/runner/testGroups.ts
@@ -14,6 +14,8 @@
* limitations under the License.
*/
+import { calculateSha1 } from '@utils/crypto';
+
import type { test } from '../common';
export type TestGroup = {
@@ -141,6 +143,11 @@ export function createTestGroups(projectSuite: test.Suite, expectedParallelism:
return result;
}
+export function shuffleTestGroups(testGroups: TestGroup[], seed: string) {
+ const keys = new Map(testGroups.map(group => [group, calculateSha1(seed + '\x1e' + group.tests[0].id)]));
+ testGroups.sort((a, b) => keys.get(a)!.localeCompare(keys.get(b)!));
+}
+
export function filterForShard(shard: { total: number, current: number }, weights: number[] | undefined, testGroups: TestGroup[]): Set<TestGroup> {
weights ??= Array.from({ length: shard.total }, () => 1);
if (weights.length !== shard.total)
diff --git a/tests/playwright-test/shuffle.spec.ts b/tests/playwright-test/shuffle.spec.ts
new file mode 100644
index 0000000000..2f9af40c74
--- /dev/null
+++ b/tests/playwright-test/shuffle.spec.ts
@@ -0,0 +1,84 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+const count = 10;
+
+function testFile(name: string) {
+ return `
+ import { test } from '@playwright/test';
+ ${Array.from({ length: count }, (_, i) => `test('test${i}', () => console.log('\\n%%${name}-test${i}'));`).join('\n')}
+ `;
+}
+
+const fileNames = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'];
+const files = Object.fromEntries(fileNames.map(name => [`${name}.spec.ts`, testFile(name)]));
+const testsInFile = (name: string) => Array.from({ length: count }, (_, i) => `${name}-test${i}`);
+const naturalOrder = fileNames.flatMap(testsInFile);
+
+test('should shuffle files and keep tests within a file in order', async ({ runInlineTest }) => {
+ const result = await runInlineTest(files, { workers: 1, shuffle: '42' });
+ expect(result.exitCode).toBe(0);
+ expect(result.passed).toBe(naturalOrder.length);
+ expect(result.output).toContain('shuffle seed 42');
+ expect(result.outputLines).not.toEqual(naturalOrder);
+ const fileOrder = [...new Set(result.outputLines.map(line => line.split('-')[0]))];
+ expect(result.outputLines).toEqual(fileOrder.flatMap(testsInFile));
+
+ const result2 = await runInlineTest(files, { workers: 1, shuffle: '42' });
+ expect(result2.outputLines).toEqual(result.outputLines);
+
+ const result3 = await runInlineTest(files, { workers: 1, shuffle: '43' });
+ expect(result3.outputLines).not.toEqual(result.outputLines);
+});
+
+test('should shuffle individual tests in fully parallel mode', async ({ runInlineTest }) => {
+ const result = await runInlineTest(files, { 'workers': 1, 'shuffle': '42', 'fully-parallel': true });
+ expect(result.exitCode).toBe(0);
+ expect(result.passed).toBe(naturalOrder.length);
+ const aTests = result.outputLines.filter(line => line.startsWith('a-'));
+ expect(aTests).not.toEqual(testsInFile('a'));
+ expect([...aTests].sort()).toEqual([...testsInFile('a')].sort());
+});
+
+test('should pick and print a random seed', async ({ runInlineTest }) => {
+ const result = await runInlineTest(files, { workers: 1, shuffle: true });
+ expect(result.exitCode).toBe(0);
+ const seed = result.output.match(/shuffle seed (\d+)/)![1];
+ const result2 = await runInlineTest(files, { workers: 1, shuffle: seed });
+ expect(result2.outputLines).toEqual(result.outputLines);
+});
+
+test('should keep serial suites together in parallel mode', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'a.spec.ts': `
+ import { test, expect } from '@playwright/test';
+ test.describe.configure({ mode: 'parallel' });
+ ${Array.from({ length: count }, (_, i) => `test('test${i}', () => console.log('\\n%%test${i}'));`).join('\n')}
+ test.describe.serial('serial', () => {
+ let counter = 0;
+ ${Array.from({ length: count }, (_, i) => `test('serial${i}', () => { expect(counter++).toBe(${i}); console.log('\\n%%serial${i}'); });`).join('\n')}
+ });
+ `,
+ }, { workers: 1, shuffle: '7' });
+ expect(result.exitCode).toBe(0);
+ expect(result.passed).toBe(2 * count);
+ const lines = result.outputLines;
+ const serialStart = lines.indexOf('serial0');
+ expect(lines.slice(serialStart, serialStart + count)).toEqual(Array.from({ length: count }, (_, i) => `serial${i}`));
+ expect(lines.filter(line => line.startsWith('test'))).not.toEqual(Array.from({ length: count }, (_, i) => `test${i}`));
+});