Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat: `--filter` that accepts a json filter file (#36944)
Dmitry Gozmaneea3f4f4a649
diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.mdindex 693522e0ac..0082a3f9c3 100644--- a/docs/src/test-cli-js.md+++ b/docs/src/test-cli-js.md@@ -82,6 +82,7 @@ npx playwright test --ui| `-c <file>` or `--config <file>` | Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}". Defaults to `playwright.config.ts` or `playwright.config.js` in the current directory. || `--debug` | Run tests with Playwright Inspector. Shortcut for `PWDEBUG=1` environment variable and `--timeout=0 --max-failures=1 --headed --workers=1` options. || `--fail-on-flaky-tests` | Fail if any test is flagged as flaky (default: false). |+| `--filter` | Path to a [test filter file](#test-filter-file). || `--forbid-only` | Fail if `test.only` is called (default: false). Useful on CI. || `--fully-parallel` | Run all tests in parallel (default: false). || `--global-timeout <timeout>` | Maximum time this test suite can run in milliseconds (default: unlimited). |@@ -113,6 +114,19 @@ npx playwright test --ui| `--update-source-method [mode]` | Update snapshots with actual results. Possible values are "patch" (default), "3way" and "overwrite". "Patch" creates a unified diff file that can be used to update the source code later. "3way" generates merge conflict markers in source code. "Overwrite" overwrites the source code with the new snapshot values.|| `-x` | Stop after the first failure. |+#### Test filter file++Option `--filter` allows to specify a json file to filter tests. It must contain an array property `titlePath`. For each test that should be run, `titlePath` contains a project name, relative path to the test file, all suite titles and the test title, similar to [`method: TestCase.titlePath`].++```json title="filter.json"+{+ "titlePath": [+ ["chromium", "path/to/example.spec.ts", "describe title", "test title"],+ ["firefox", "path/to/another.spec.ts", "top-level test title"]+ ]+}+```+### Show ReportDisplay HTML report from previous test run. [Read more about the HTML reporter](./test-reporters#html-reporter).diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.tsindex d7c1854211..5876889ffb 100644--- a/packages/playwright/src/common/config.ts+++ b/packages/playwright/src/common/config.ts@@ -22,7 +22,7 @@ import { getPackageJsonPath, mergeObjects } from '../util';import type { Config, Fixtures, Metadata, Project, ReporterDescription } from '../../types/test';import type { TestRunnerPluginRegistration } from '../plugins';-import type { Matcher } from '../util';+import type { TestCaseFilter } from '../util';import type { ConfigCLIOverrides } from './ipc';import type { Location } from '../../types/testReporter';import type { FullConfig, FullProject } from '../../types/testReporter';@@ -57,8 +57,9 @@ export class FullConfigInternal {cliListOnly = false;cliPassWithNoTests?: boolean;cliLastFailed?: boolean;- testIdMatcher?: Matcher;- lastFailedTestIdMatcher?: Matcher;+ cliFilterFile?: string;+ preOnlyTestFilters: TestCaseFilter[] = [];+ postShardTestFilters: TestCaseFilter[] = [];defineConfigWasUsed = false;globalSetups: string[] = [];diff --git a/packages/playwright/src/common/suiteUtils.ts b/packages/playwright/src/common/suiteUtils.tsindex 20c46e3157..4108a9696e 100644--- a/packages/playwright/src/common/suiteUtils.ts+++ b/packages/playwright/src/common/suiteUtils.ts@@ -22,10 +22,10 @@ import { createFileMatcher } from '../util';import type { FullProjectInternal } from './config';import type { Suite, TestCase } from './test';-import type { Matcher, TestFileFilter } from '../util';+import type { TestCaseFilter, TestFileFilter } from '../util';-export function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) {+export function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: TestCaseFilter) {for (const child of suite.suites) {if (!suiteFilter(child))filterSuite(child, suiteFilter, testFilter);@@ -35,7 +35,7 @@ export function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => booleasuite._entries = suite._entries.filter(e => entries.has(e)); // Preserve the order.}-export function filterTestsRemoveEmptySuites(suite: Suite, filter: (test: TestCase) => boolean): boolean {+export function filterTestsRemoveEmptySuites(suite: Suite, filter: TestCaseFilter): boolean {const filteredSuites = suite.suites.filter(child => filterTestsRemoveEmptySuites(child, filter));const filteredTests = suite.tests.filter(filter);const entries = new Set([...filteredSuites, ...filteredTests]);@@ -111,7 +111,7 @@ export function filterOnly(suite: Suite) {return filterSuiteWithOnlySemantics(suite, suiteFilter, testFilter);}-export function filterSuiteWithOnlySemantics(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) {+function filterSuiteWithOnlySemantics(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: TestCaseFilter) {const onlySuites = suite.suites.filter(child => filterSuiteWithOnlySemantics(child, suiteFilter, testFilter) || suiteFilter(child));const onlyTests = suite.tests.filter(testFilter);const onlyEntries = new Set([...onlySuites, ...onlyTests]);@@ -132,12 +132,6 @@ export function filterByFocusedLine(suite: Suite, focusedTestFileLines: TestFilereturn filterSuite(suite, suiteFilter, testFilter);}-export function filterByTestIds(suite: Suite, testIdMatcher: Matcher | undefined) {- if (!testIdMatcher)- return;- filterTestsRemoveEmptySuites(suite, test => testIdMatcher(test.id));-}-function createFileMatcherFromFilter(filter: TestFileFilter) {const fileMatcher = createFileMatcher(filter.re || filter.exact || '');return (testFileName: string, testLine: number, testColumn: number) =>diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.tsindex c2f5f4540c..bc402b10f8 100644--- a/packages/playwright/src/program.ts+++ b/packages/playwright/src/program.ts@@ -176,6 +176,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) {config.cliProjectFilter = opts.project || undefined;config.cliPassWithNoTests = !!opts.passWithNoTests;config.cliLastFailed = !!opts.lastFailed;+ config.cliFilterFile = opts.filter ? path.resolve(process.cwd(), opts.filter) : undefined;// Evaluate project filters against config before starting execution. This enables a consistent error message across run modesfilterProjects(config.projects, config.cliProjectFilter);@@ -380,6 +381,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?:['-c, --config <file>', { description: `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"` }],['--debug', { description: `Run tests with Playwright Inspector. Shortcut for "PWDEBUG=1" environment variable and "--timeout=0 --max-failures=1 --headed --workers=1" options` }],['--fail-on-flaky-tests', { description: `Fail if any test is flagged as flaky (default: false)` }],+ ['--filter <file>', { description: `Path to a test filter file. See documentation for details.` }],['--forbid-only', { description: `Fail if test.only is called (default: false)` }],['--fully-parallel', { description: `Run all tests in parallel (default: false)` }],['--global-timeout <timeout>', { description: `Maximum time this test suite can run in milliseconds (default: unlimited)` }],diff --git a/packages/playwright/src/runner/lastRun.ts b/packages/playwright/src/runner/lastRun.tsindex 931bca3972..93ffe9205f 100644--- a/packages/playwright/src/runner/lastRun.ts+++ b/packages/playwright/src/runner/lastRun.ts@@ -45,7 +45,9 @@ export class LastRunReporter implements ReporterV2 {return;try {const lastRunInfo = JSON.parse(await fs.promises.readFile(this._lastRunFile, 'utf8')) as LastRunInfo;- this._config.lastFailedTestIdMatcher = id => lastRunInfo.failedTests.includes(id);+ const failedTestIds = new Set(lastRunInfo.failedTests);+ // Explicitly apply --last-failed filter after sharding.+ this._config.postShardTestFilters.push(test => failedTestIds.has(test.id));} catch {}}@@ -65,9 +67,11 @@ export class LastRunReporter implements ReporterV2 {async onEnd(result: FullResult) {if (!this._lastRunFile || this._config.cliListOnly)return;+ const lastRunInfo: LastRunInfo = {+ status: result.status,+ failedTests: this._suite?.allTests().filter(t => !t.ok()).map(t => t.id) || [],+ };await fs.promises.mkdir(path.dirname(this._lastRunFile), { recursive: true });- const failedTests = this._suite?.allTests().filter(t => !t.ok()).map(t => t.id);- const lastRunReport = JSON.stringify({ status: result.status, failedTests }, undefined, 2);- await fs.promises.writeFile(this._lastRunFile, lastRunReport);+ await fs.promises.writeFile(this._lastRunFile, JSON.stringify(lastRunInfo, undefined, 2));}}diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.tsindex f4963c478f..05c5bba718 100644--- a/packages/playwright/src/runner/loadUtils.ts+++ b/packages/playwright/src/runner/loadUtils.ts@@ -20,7 +20,7 @@ import { InProcessLoaderHost, OutOfProcessLoaderHost } from './loaderHost';import { createFileFiltersFromArguments, createFileMatcherFromArguments, createTitleMatcher, errorWithFile, forceRegExp } from '../util';import { buildProjectsClosure, collectFilesForProject, filterProjects } from './projectUtils';import { createTestGroups, filterForShard } from './testGroups';-import { applyRepeatEachIndex, bindFileSuiteToProject, filterByFocusedLine, filterByTestIds, filterOnly, filterTestsRemoveEmptySuites } from '../common/suiteUtils';+import { applyRepeatEachIndex, bindFileSuiteToProject, filterByFocusedLine, filterOnly, filterTestsRemoveEmptySuites } from '../common/suiteUtils';import { Suite } from '../common/test';import { dependenciesForTestFile } from '../transform/compilationCache';import { requireOrImport } from '../transform/transform';@@ -32,7 +32,7 @@ import type { FullConfig, Reporter, TestError } from '../../types/testReporter';import type { FullProjectInternal } from '../common/config';import type { FullConfigInternal } from '../common/config';import type { TestCase } from '../common/test';-import type { Matcher, TestFileFilter } from '../util';+import type { Matcher, TestCaseFilter, TestFileFilter } from '../util';import type { RawSourceMap } from '../utilsBundle';@@ -120,7 +120,7 @@ export async function loadFileSuites(testRun: TestRun, mode: 'out-of-process' |}}-export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean, additionalFileMatcher?: Matcher): Promise<Suite> {+export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean): Promise<Suite> {const config = testRun.config;// Create root suite, where each child will be a project suite with cloned file suites inside it.const rootSuite = new Suite('', 'root');@@ -140,7 +140,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], shoconst projectSuite = createProjectSuite(project, fileSuites);projectSuites.set(project, projectSuite);- const filteredProjectSuite = filterProjectSuite(projectSuite, { cliFileFilters, cliTitleMatcher, testIdMatcher: config.testIdMatcher, additionalFileMatcher });+ const filteredProjectSuite = filterProjectSuite(projectSuite, { cliFileFilters, cliTitleMatcher, testFilters: config.preOnlyTestFilters });filteredProjectSuites.set(project, filteredProjectSuite);}}@@ -197,9 +197,8 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], shofilterTestsRemoveEmptySuites(rootSuite, test => testsInThisShard.has(test));}- // Explicitly apply --last-failed filter after sharding.- if (config.lastFailedTestIdMatcher)- filterByTestIds(rootSuite, config.lastFailedTestIdMatcher);+ if (config.postShardTestFilters.length)+ filterTestsRemoveEmptySuites(rootSuite, test => config.postShardTestFilters.every(filter => filter(test)));// Now prepend dependency projects without filtration.{@@ -233,20 +232,18 @@ function createProjectSuite(project: FullProjectInternal, fileSuites: Suite[]):return projectSuite;}-function filterProjectSuite(projectSuite: Suite, options: { cliFileFilters: TestFileFilter[], cliTitleMatcher?: Matcher, testIdMatcher?: Matcher, additionalFileMatcher?: Matcher }): Suite {+function filterProjectSuite(projectSuite: Suite, options: { cliFileFilters: TestFileFilter[], cliTitleMatcher?: Matcher, testFilters: TestCaseFilter[] }): Suite {// Fast path.- if (!options.cliFileFilters.length && !options.cliTitleMatcher && !options.testIdMatcher && !options.additionalFileMatcher)+ if (!options.cliFileFilters.length && !options.cliTitleMatcher && !options.testFilters.length)return projectSuite;const result = projectSuite._deepClone();if (options.cliFileFilters.length)filterByFocusedLine(result, options.cliFileFilters);- if (options.testIdMatcher)- filterByTestIds(result, options.testIdMatcher);filterTestsRemoveEmptySuites(result, (test: TestCase) => {- if (options.cliTitleMatcher && !options.cliTitleMatcher(test._grepTitleWithTags()))+ if (!options.testFilters.every(filter => filter(test)))return false;- if (options.additionalFileMatcher && !options.additionalFileMatcher(test.location.file))+ if (options.cliTitleMatcher && !options.cliTitleMatcher(test._grepTitleWithTags()))return false;return true;});diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.tsindex 66d4e0ce6d..c7113c204b 100644--- a/packages/playwright/src/runner/tasks.ts+++ b/packages/playwright/src/runner/tasks.ts@@ -31,10 +31,9 @@ import { detectChangedTestFiles } from './vcs';import { Suite } from '../common/test';import { createTestGroups } from '../runner/testGroups';import { cacheDir } from '../transform/compilationCache';-import { removeDirAndLogToConsole } from '../util';+import { loadTestFilterFile, removeDirAndLogToConsole } from '../util';import type { TestGroup } from '../runner/testGroups';-import type { Matcher } from '../util';import type { EnvByProjectId } from './dispatcher';import type { TestRunnerPluginRegistration } from '../plugins';import type { Task } from './taskRunner';@@ -265,13 +264,22 @@ export function createLoadTask(mode: 'out-of-process' | 'in-process', options: {await plugin.instance?.populateDependencies?.();}- let cliOnlyChangedMatcher: Matcher | undefined = undefined;if (testRun.config.cliOnlyChanged) {const changedFiles = await detectChangedTestFiles(testRun.config.cliOnlyChanged, testRun.config.configDir);- cliOnlyChangedMatcher = file => changedFiles.has(file);+ testRun.config.preOnlyTestFilters.push(test => changedFiles.has(test.location.file));}- testRun.rootSuite = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly, cliOnlyChangedMatcher);+ if (testRun.config.cliFilterFile) {+ try {+ testRun.config.preOnlyTestFilters.push(await loadTestFilterFile(testRun.config.cliFilterFile));+ } catch (error) {+ if (options.failOnLoadErrors)+ throw error;+ softErrors.push(error);+ }+ }++ testRun.rootSuite = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);testRun.failureTracker.onRootSuite(testRun.rootSuite);// Fail when no tests.if (options.failOnLoadErrors && !testRun.rootSuite.allTests().length && !testRun.config.cliPassWithNoTests && !testRun.config.config.shard && !testRun.config.cliOnlyChanged) {diff --git a/packages/playwright/src/runner/testServer.ts b/packages/playwright/src/runner/testServer.tsindex e31e380ea9..7a6129d5e1 100644--- a/packages/playwright/src/runner/testServer.ts+++ b/packages/playwright/src/runner/testServer.ts@@ -331,14 +331,17 @@ export class TestServerDispatcher implements TestServerInterface {if (!config)return { status: 'failed' };- const testIdSet = params.testIds ? new Set<string>(params.testIds) : null;config.cliListOnly = false;config.cliPassWithNoTests = true;config.cliArgs = params.locations || [];config.cliGrep = params.grep;config.cliGrepInvert = params.grepInvert;config.cliProjectFilter = params.projects?.length ? params.projects : undefined;- config.testIdMatcher = testIdSet ? id => testIdSet.has(id) : undefined;+ config.preOnlyTestFilters = [];+ if (params.testIds) {+ const testIdSet = new Set<string>(params.testIds);+ config.preOnlyTestFilters.push(test => testIdSet.has(test.id));+ }const configReporters = await createReporters(config, 'test', true);const reporter = new InternalReporter([...configReporters, wireReporter]);diff --git a/packages/playwright/src/util.ts b/packages/playwright/src/util.tsindex a7d120770f..bc06ac02f8 100644--- a/packages/playwright/src/util.ts+++ b/packages/playwright/src/util.ts@@ -19,13 +19,14 @@ import path from 'path';import url from 'url';import util from 'util';-import { parseStackFrame, sanitizeForFilePath, calculateSha1, isRegExp, isString, stringifyStackFrames, escapeWithQuotes } from 'playwright-core/lib/utils';+import { rewriteErrorMessage, toPosixPath, parseStackFrame, sanitizeForFilePath, calculateSha1, isRegExp, isString, stringifyStackFrames, escapeWithQuotes } from 'playwright-core/lib/utils';import { colors, debug, mime, minimatch } from 'playwright-core/lib/utilsBundle';import type { Location } from './../types/testReporter';import type { TestInfoErrorImpl } from './common/ipc';import type { StackFrame } from '@protocol/channels';import type { RawStack } from 'playwright-core/lib/utils';+import type { TestCase } from './common/test';const PLAYWRIGHT_TEST_PATH = path.join(__dirname, '..');const PLAYWRIGHT_CORE_PATH = path.dirname(require.resolve('playwright-core/package.json'));@@ -82,6 +83,8 @@ export type TestFileFilter = {column: number | null;};+export type TestCaseFilter = (test: TestCase) => boolean;+export function createFileFiltersFromArguments(args: string[]): TestFileFilter[] {return args.map(arg => {const match = /^(.*?):(\d+):?(\d+)?$/.exec(arg);@@ -439,3 +442,24 @@ export function stepTitle(category: TestStepCategory, title: string): string {return `[${category}] ${title}`;}}++type TestFileFilterData = {+ titlePath: string[][],+};++export async function loadTestFilterFile(filePath: string): Promise<TestCaseFilter> {+ try {+ const data = JSON.parse(await fs.promises.readFile(filePath, 'utf8')) as TestFileFilterData;+ if (!data || typeof data !== 'object' || !Array.isArray(data.titlePath) || !data.titlePath.every(path => Array.isArray(path)))+ throw new Error(`Wrong test filter file format`);+ const toId = (titlePath: string[]) => {+ const [project, file, ...titles] = titlePath; // At the time of filtering, there is no root yet.+ return `${project}\x1e${toPosixPath(file)}\x1e${titles.join('\x1e')}`;+ };+ const ids = new Set(data.titlePath.map(titlePath => toId(titlePath)));+ return test => ids.has(toId(test.titlePath()));+ } catch (error) {+ rewriteErrorMessage(error, `Failed to read test filter file "${filePath}": ${error.message}`);+ throw error;+ }+}diff --git a/tests/playwright-test/test-filter.spec.ts b/tests/playwright-test/test-filter.spec.tsnew file mode 100644index 0000000000..acb792aebf--- /dev/null+++ b/tests/playwright-test/test-filter.spec.ts@@ -0,0 +1,95 @@+/**+ * Copyright Microsoft Corporation. All rights reserved.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { test, expect } from './playwright-test-fixtures';++test('--filter should work', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'playwright.config.ts': `+ module.exports = { projects: [{}, { name: 'p1' }, { name: 'p2' }] };+ `,+ 'filter.json': `+ {+ "titlePath": [+ ["p2", "b.test.ts", "test2"],+ ["p1", "a.test.ts", "suite2", "test2"],+ ["p1", "b.test.ts", "suite2", "test1"],+ ["", "a.test.ts", "suite1", "test1"]+ ]+ }+ `,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test.describe('suite1', () => {+ test('test1', async () => { console.log('\\n%%a-' + test.info().project.name + '-suite1-test1'); });+ test('test2', async () => { console.log('\\n%%a-' + test.info().project.name + '-suite1-test2'); });+ });+ test.describe('suite2', () => {+ test('test1', async () => { console.log('\\n%%a-' + test.info().project.name + '-suite2-test1'); });+ test('test2', async () => { console.log('\\n%%a-' + test.info().project.name + '-suite2-test2'); });+ });+ test('test1', async () => { console.log('\\n%%a-' + test.info().project.name + '-test1'); });+ test('test2', async () => { console.log('\\n%%a-' + test.info().project.name + '-test2'); });+ `,+ 'b.test.ts': `+ import { test, expect } from '@playwright/test';+ test.describe('suite1', () => {+ test('test1', async () => { console.log('\\n%%b-' + test.info().project.name + '-suite1-test1'); });+ test('test2', async () => { console.log('\\n%%b-' + test.info().project.name + '-suite1-test2'); });+ });+ test.describe('suite2', () => {+ test('test1', async () => { console.log('\\n%%b-' + test.info().project.name + '-suite2-test1'); });+ test('test2', async () => { console.log('\\n%%b-' + test.info().project.name + '-suite2-test2'); });+ });+ test('test1', async () => { console.log('\\n%%b-' + test.info().project.name + '-test1'); });+ test('test2', async () => { console.log('\\n%%b-' + test.info().project.name + '-test2'); });+ `,+ }, { workers: 1, filter: 'filter.json' });+ expect(result.exitCode).toBe(0);+ expect(result.passed).toBe(4);+ expect(result.failed).toBe(0);+ expect(result.outputLines).toEqual([+ 'a--suite1-test1',+ 'a-p1-suite2-test2',+ 'b-p1-suite2-test1',+ 'b-p2-test2',+ ]);+});++test('--filter should complain about missing file', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('test1', () => {});+ `,+ }, { workers: 1, filter: 'filter.json' });+ expect(result.exitCode).toBe(1);+ expect(result.output).toContain(`Failed to read test filter file "${test.info().outputPath('filter.json')}"`);+});++test('--filter should complain about malformed file', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'filter.json': `+ { "titlePath": [[], "not an array"] }+ `,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('test1', () => {});+ `,+ }, { workers: 1, filter: 'filter.json' });+ expect(result.exitCode).toBe(1);+ expect(result.output).toContain(`Failed to read test filter file "${test.info().outputPath('filter.json')}": Wrong test filter file format`);+});