Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(test runner): --test-list and --test-list-invert (#37552)
Dmitry Gozmand5b4c6535b61
diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.mdindex 0688e328d7..d45ee1d6a3 100644--- a/docs/src/test-cli-js.md+++ b/docs/src/test-cli-js.md@@ -103,6 +103,8 @@ npx playwright test --ui| `--reporter <reporter>` | Reporter to use, comma-separated, can be "dot", "line", "list", or others (default: "list"). You can also pass a path to a custom reporter file. || `--retries <retries>` | Maximum retry count for flaky tests, zero for no retries (default: no retries). || `--shard <shard>` | Shard tests and execute only the selected shard, specified in the form "current/all", 1-based, e.g., "3/5". |+| `--test-list <file>` | Path to a file containing a list of tests to run. See [test list](#test-list) for details. |+| `--test-list-invert <file>` | Path to a file containing a list of tests to skip. See [test list](#test-list) for details. || `--timeout <timeout>` | Specify test timeout threshold in milliseconds, zero for unlimited (default: 30 seconds). || `--trace <mode>` | Force tracing mode, can be `on`, `off`, `on-first-retry`, `on-all-retries`, `retain-on-failure`, `retain-on-first-failure`. || `--tsconfig <path>` | Path to a single tsconfig applicable to all imported files (default: look up tsconfig for each imported file separately). |@@ -113,6 +115,29 @@ npx playwright test --ui| `--update-source-method [mode]` | Update snapshots with actual results. Possible values are "patch" (default), "3way" and "overwrite". "Patch" creates a unified diff file that can be used to update the source code later. "3way" generates merge conflict markers in source code. "Overwrite" overwrites the source code with the new snapshot values.|| `-x` | Stop after the first failure. |+#### Test list++Options `--test-list` and `--test-list-invert` accept a path to a test list file. This file should list tests in the format similar to the output produced in `--list` mode.++```txt+# This is a test list file.+# It can include comments and empty lines.++# Fully qualified test with a project:+[chromium] › path/to/example.spec.ts:3:9 › suite › nested suite › example test++# This test is included for all projects:+path/to/example.spec.ts:3:9 › example test++# Use "›" or ">" as a separator:+[firefox] > example.spec.ts > suite > nested suite > example test++# Line/column numbers are completely ignored, you can omit them.+# Three entries below refer to the same test:+example.spec.ts › example test+example.spec.ts:15 › example test+example.spec.ts:42:42 › example test+```### Show Reportdiff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.tsindex d6ef36ba35..50eb2ac8a7 100644--- a/packages/playwright/src/common/config.ts+++ b/packages/playwright/src/common/config.ts@@ -57,6 +57,8 @@ export class FullConfigInternal {cliListOnly = false;cliPassWithNoTests?: boolean;cliLastFailed?: boolean;+ cliTestList?: string;+ cliTestListInvert?: string;preOnlyTestFilters: TestCaseFilter[] = [];postShardTestFilters: TestCaseFilter[] = [];defineConfigWasUsed = false;diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.tsindex 81f285fd4b..0ed4fd76d4 100644--- a/packages/playwright/src/program.ts+++ b/packages/playwright/src/program.ts@@ -203,6 +203,8 @@ async function runTests(args: string[], opts: { [key: string]: any }) {config.cliProjectFilter = opts.project || undefined;config.cliPassWithNoTests = !!opts.passWithNoTests;config.cliLastFailed = !!opts.lastFailed;+ config.cliTestList = opts.testList ? path.resolve(process.cwd(), opts.testList) : undefined;+ config.cliTestListInvert = opts.testListInvert ? path.resolve(process.cwd(), opts.testListInvert) : undefined;// Evaluate project filters against config before starting execution. This enables a consistent error message across run modesfilterProjects(config.projects, config.cliProjectFilter);@@ -400,6 +402,8 @@ const testOptions: [string, { description: string, choices?: string[], preset?:['--reporter <reporter>', { description: `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")` }],['--retries <retries>', { description: `Maximum retry count for flaky tests, zero for no retries (default: no retries)` }],['--shard <shard>', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }],+ ['--test-list <file>', { description: `Path to a file containing a list of tests to run. See https://playwright.dev/docs/test-cli for more details.` }],+ ['--test-list-invert <file>', { description: `Path to a file containing a list of tests to skip. See https://playwright.dev/docs/test-cli for more details.` }],['--timeout <timeout>', { description: `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})` }],['--trace <mode>', { description: `Force tracing mode`, choices: kTraceModes as string[] }],['--tsconfig <path>', { description: `Path to a single tsconfig applicable to all imported files (default: look up tsconfig for each imported file separately)` }],diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.tsindex 065cd0cc4a..e61a2cca86 100644--- a/packages/playwright/src/runner/loadUtils.ts+++ b/packages/playwright/src/runner/loadUtils.ts@@ -15,9 +15,11 @@*/import path from 'path';+import fs from 'fs';+import { toPosixPath } from 'playwright-core/lib/utils';import { InProcessLoaderHost, OutOfProcessLoaderHost } from './loaderHost';-import { createFileFiltersFromArguments, createFileMatcherFromArguments, createTitleMatcher, errorWithFile, forceRegExp } from '../util';+import { createFileFiltersFromArguments, createFileMatcherFromArguments, createTitleMatcher, errorWithFile, forceRegExp, parseLocationArg } from '../util';import { buildProjectsClosure, collectFilesForProject, filterProjects } from './projectUtils';import { createTestGroups, filterForShard } from './testGroups';import { applyRepeatEachIndex, bindFileSuiteToProject, filterByFocusedLine, filterOnly, filterTestsRemoveEmptySuites } from '../common/suiteUtils';@@ -346,3 +348,34 @@ function sourceMapSources(file: string, cache: Map<string, string[]>): string[]return sources;}}++export async function loadTestList(config: FullConfigInternal, filePath: string): Promise<TestCaseFilter> {+ try {+ const content = await fs.promises.readFile(filePath, 'utf-8');+ const lines = content.split('\n').map(line => line.trim()).filter(line => line && !line.startsWith('#'));+ const descriptions = lines.map(line => {+ const delimiter = line.includes('›') ? '›' : '>';+ const tokens = line.split(delimiter).map(token => token.trim());+ let project: string | undefined;+ if (tokens[0].startsWith('[')) {+ if (!tokens[0].endsWith(']'))+ throw new Error(`Malformed test description: ${line}`);+ project = tokens[0].substring(1, tokens[0].length - 1);+ tokens.shift();+ }+ return { project, file: toPosixPath(parseLocationArg(tokens[0]).file), titlePath: tokens.slice(1) };+ });+ return (test: TestCase) => descriptions.some(d => {+ // Note: there is no root yet at the time of filtering.+ const [projectName, , ...titles] = test.titlePath();+ if (d.project !== undefined && d.project !== projectName)+ return false;+ const relativeFile = toPosixPath(path.relative(config.config.rootDir, test.location.file));+ if (relativeFile !== d.file)+ return false;+ return d.titlePath.length === titles.length && d.titlePath.every((_, index) => titles[index] === d.titlePath[index]);+ });+ } catch (e) {+ throw errorWithFile(filePath, 'Cannot read test list file: ' + e.message);+ }+}diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.tsindex 7d6245ad87..96cadb02d9 100644--- a/packages/playwright/src/runner/tasks.ts+++ b/packages/playwright/src/runner/tasks.ts@@ -23,7 +23,7 @@ import { debug } from 'playwright-core/lib/utilsBundle';import { Dispatcher } from './dispatcher';import { FailureTracker } from './failureTracker';-import { collectProjectsAndTestFiles, createRootSuite, loadFileSuites, loadGlobalHook } from './loadUtils';+import { collectProjectsAndTestFiles, createRootSuite, loadFileSuites, loadGlobalHook, loadTestList } from './loadUtils';import { buildDependentProjects, buildTeardownToSetupsMap, filterProjects } from './projectUtils';import { applySuggestedRebaselines, clearSuggestedRebaselines } from './rebase';import { TaskRunner } from './taskRunner';@@ -271,11 +271,24 @@ export function createLoadTask(mode: 'out-of-process' | 'in-process', options: {testRun.config.preOnlyTestFilters.push(test => changedFiles.has(test.location.file));}+ if (testRun.config.cliTestList) {+ const testListFilter = await loadTestList(testRun.config, testRun.config.cliTestList);+ testRun.config.preOnlyTestFilters.push(testListFilter);+ }++ if (testRun.config.cliTestListInvert) {+ const testListInvertFilter = await loadTestList(testRun.config, testRun.config.cliTestListInvert);+ testRun.config.preOnlyTestFilters.push(test => !testListInvertFilter(test));+ }+const { rootSuite, topLevelProjects } = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);testRun.rootSuite = rootSuite;testRun.failureTracker.onRootSuite(rootSuite, topLevelProjects);// Fail when no tests.- if (options.failOnLoadErrors && !testRun.rootSuite.allTests().length && !testRun.config.cliPassWithNoTests && !testRun.config.config.shard && !testRun.config.cliOnlyChanged) {+ if (options.failOnLoadErrors && !testRun.rootSuite.allTests().length+ && !testRun.config.cliPassWithNoTests+ && !testRun.config.config.shard && !testRun.config.cliOnlyChanged+ && !testRun.config.cliTestList && !testRun.config.cliTestListInvert) {if (testRun.config.cliArgs.length) {throw new Error([`No tests found.`,diff --git a/packages/playwright/src/util.ts b/packages/playwright/src/util.tsindex 65040aea91..6ab72eb1db 100644--- a/packages/playwright/src/util.ts+++ b/packages/playwright/src/util.ts@@ -85,14 +85,19 @@ export type TestFileFilter = {export type TestCaseFilter = (test: TestCase) => boolean;+export function parseLocationArg(arg: string): { file: string, line: number | null, column: number | null } {+ const match = /^(.*?):(\d+):?(\d+)?$/.exec(arg);+ return {+ file: match ? match[1] : arg,+ line: match ? parseInt(match[2], 10) : null,+ column: match?.[3] ? parseInt(match[3], 10) : null,+ };+}+export function createFileFiltersFromArguments(args: string[]): TestFileFilter[] {return args.map(arg => {- const match = /^(.*?):(\d+):?(\d+)?$/.exec(arg);- return {- re: forceRegExp(match ? match[1] : arg),- line: match ? parseInt(match[2], 10) : null,- column: match?.[3] ? parseInt(match[3], 10) : null,- };+ const parsed = parseLocationArg(arg);+ return { re: forceRegExp(parsed.file), line: parsed.line, column: parsed.column };});}diff --git a/tests/playwright-test/test-list.spec.ts b/tests/playwright-test/test-list.spec.tsnew file mode 100644index 0000000000..c307d46153--- /dev/null+++ b/tests/playwright-test/test-list.spec.ts@@ -0,0 +1,132 @@+/**+ * Copyright Microsoft Corporation. All rights reserved.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import path from 'path';+import fs from 'fs';+import { test, expect } from './playwright-test-fixtures';++const varietyWorkspace = {+ 'playwright.config.ts': `+ module.exports = { projects: [{ name: 'p1' }, { name: 'p2' }], testDir: 'tests', };+ `,+ 'dir/test.list': `+ # this is a multiline comment++ # with an empty line in between+ [p1] > dir1/a.test.ts > test1+ [p2] › dir2${path.sep}b.spec.ts › suite › test2++ # more comments+ dir3/c.spec.ts > test2+ `,+ 'empty.list': `+ # nothing to see here+ `,+ 'ignore.list': `+ [p1] > dir1/a.test.ts > test1+ `,+ 'tests/dir1/a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('test1', async () => { console.log('\\n%%a-test1-' + test.info().project.name); });+ test('test2', async () => { console.log('\\n%%a-test2-' + test.info().project.name); });+ `,+ 'tests/dir2/b.spec.ts': `+ import { test, expect } from '@playwright/test';+ test.describe('suite', () => {+ test('test1', async () => { console.log('\\n%%b-test1-' + test.info().project.name); });+ test.describe(() => {+ test('test2', async () => { console.log('\\n%%b-test2-' + test.info().project.name); });+ });+ });+ `,+ 'tests/dir3/c.spec.ts': `+ import { test, expect } from '@playwright/test';+ test('test1', async () => { console.log('\\n%%c-test1-' + test.info().project.name); });+ test('test2', async () => { console.log('\\n%%c-test2-' + test.info().project.name); });+ `,+};++test('--test-list should work', async ({ runInlineTest }) => {+ const result = await runInlineTest(varietyWorkspace, { 'workers': 1, 'test-list': 'dir/test.list' });+ expect(result.exitCode).toBe(0);+ expect(result.passed).toBe(4);+ expect(result.outputLines).toEqual([+ 'a-test1-p1',+ 'c-test2-p1',+ 'b-test2-p2',+ 'c-test2-p2',+ ]);+});++test('--test-list-invert should work', async ({ runInlineTest }) => {+ const result = await runInlineTest(varietyWorkspace, { 'workers': 1, 'test-list-invert': 'dir/test.list' });+ expect(result.exitCode).toBe(0);+ expect(result.passed).toBe(8);+ expect(result.outputLines).toEqual([+ 'a-test2-p1',+ 'b-test1-p1',+ 'b-test2-p1',+ 'c-test1-p1',+ 'a-test1-p2',+ 'a-test2-p2',+ 'b-test1-p2',+ 'c-test1-p2',+ ]);+});++test('--test-list applies before --test-list-invert', async ({ runInlineTest }) => {+ const result = await runInlineTest(varietyWorkspace, { 'workers': 1, 'test-list': 'dir/test.list', 'test-list-invert': 'ignore.list' });+ expect(result.exitCode).toBe(0);+ expect(result.passed).toBe(3);+ expect(result.outputLines).toEqual([+ 'c-test2-p1',+ 'b-test2-p2',+ 'c-test2-p2',+ ]);+});++test('empty --test-list should work', async ({ runInlineTest }) => {+ const result = await runInlineTest(varietyWorkspace, { 'workers': 1, 'test-list': 'empty.list' });+ expect(result.exitCode).toBe(0);+ expect(result.passed).toBe(0);+ expect(result.outputLines).toEqual([+ ]);+});++test('--list output should work for --test-list', async ({ runInlineTest }) => {+ const listResult = await runInlineTest(varietyWorkspace, { 'list': true });+ expect(listResult.exitCode).toBe(0);+ const lines = listResult.output.split('\n').filter(line => !line.includes('Listing tests') && !line.includes('Total:'));+ await fs.promises.writeFile(test.info().outputPath('generated.list'), lines.join('\n'), 'utf-8');++ const result = await runInlineTest(varietyWorkspace, { 'workers': 1, 'test-list': 'generated.list' });+ expect(result.exitCode).toBe(0);+ expect(result.passed).toBe(12);+ expect(result.outputLines).toEqual([+ 'a-test1-p1',+ 'a-test2-p1',+ 'b-test1-p1',+ 'b-test2-p1',+ 'c-test1-p1',+ 'c-test2-p1',+ 'a-test1-p2',+ 'a-test2-p2',+ 'b-test1-p2',+ 'b-test2-p2',+ 'c-test1-p2',+ 'c-test2-p2',+ ]);+});diff --git a/utils/doclint/cli.js b/utils/doclint/cli.jsindex 958a13e181..33f2846ee0 100755--- a/utils/doclint/cli.js+++ b/utils/doclint/cli.js@@ -218,7 +218,7 @@ async function run() {'Dockerfile',]);if (!allowedCodeLangs.has(node.codeLang.split(' ')[0]))- throw new Error(`${path.relative(PROJECT_DIR, filePath)} contains code block with invalid code block language ${node.codeLang}`);+ throw new Error(`${path.relative(PROJECT_DIR, filePath)} contains code block with invalid code block language "${node.codeLang}"`);}if (node.type.startsWith('h')) {const hash = mdSectionHash(node.text || '');