Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat: --last-run-file (#37209)
Dmitry Gozman4fdeec7b7c46
diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.mdindex 0688e328d7..a47f1a60da 100644--- a/docs/src/test-cli-js.md+++ b/docs/src/test-cli-js.md@@ -91,6 +91,7 @@ npx playwright test --ui| `--ignore-snapshots` | Ignore screenshot and snapshot expectations. || `-j <workers>` or `--workers <workers>` | Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 50%). || `--last-failed` | Only re-run the failures. |+| `--last-run-file` | Path to the last-run file (default: "test-results/.last-run.json"). This json file is used for the `--last-failed` option. You can also apply a custom filter by providing an array of [`property: TestCase.id`] in the `filterTests` property. || `--list` | Collect all the tests and report them, but do not run. || `--max-failures <N>` or `-x` | Stop after the first `N` failures. Passing `-x` stops after the first failure. || `--no-deps` | Do not run project dependencies. |diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.tsindex 660aa77add..98d079abab 100644--- a/packages/playwright/src/common/config.ts+++ b/packages/playwright/src/common/config.ts@@ -57,6 +57,7 @@ export class FullConfigInternal {cliListOnly = false;cliPassWithNoTests?: boolean;cliLastFailed?: boolean;+ cliLastRunFile?: string;preOnlyTestFilters: TestCaseFilter[] = [];postShardTestFilters: TestCaseFilter[] = [];defineConfigWasUsed = false;diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.tsindex 5bef1dd87b..b23c389ca2 100644--- a/packages/playwright/src/program.ts+++ b/packages/playwright/src/program.ts@@ -176,6 +176,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) {config.cliProjectFilter = opts.project || undefined;config.cliPassWithNoTests = !!opts.passWithNoTests;config.cliLastFailed = !!opts.lastFailed;+ config.cliLastRunFile = opts.lastRunFile ? path.resolve(process.cwd(), opts.lastRunFile) : undefined;// Evaluate project filters against config before starting execution. This enables a consistent error message across run modesfilterProjects(config.projects, config.cliProjectFilter);@@ -361,6 +362,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?:['--headed', { description: `Run tests in headed browsers (default: headless)` }],['--ignore-snapshots', { description: `Ignore screenshot and snapshot expectations` }],['--last-failed', { description: `Only re-run the failures` }],+ ['--last-run-file <file>', { description: `Path to the last-run file (default: "test-results/.last-run.json")` }],['--list', { description: `Collect all the tests and report them, but do not run` }],['--max-failures <N>', { description: `Stop after the first N failures` }],['--no-deps', { description: `Do not run project dependencies` }],diff --git a/packages/playwright/src/runner/lastRun.ts b/packages/playwright/src/runner/lastRun.tsindex 93ffe9205f..d6721fd54e 100644--- a/packages/playwright/src/runner/lastRun.ts+++ b/packages/playwright/src/runner/lastRun.ts@@ -24,8 +24,9 @@ import type { FullConfigInternal } from '../common/config';import type { ReporterV2 } from '../reporters/reporterV2';type LastRunInfo = {- status: FullResult['status'];- failedTests: string[];+ status?: FullResult['status'];+ failedTests?: string[];+ filterTests?: string[];};export class LastRunReporter implements ReporterV2 {@@ -35,19 +36,28 @@ export class LastRunReporter implements ReporterV2 {constructor(config: FullConfigInternal) {this._config = config;- const [project] = filterProjects(config.projects, config.cliProjectFilter);- if (project)- this._lastRunFile = path.join(project.project.outputDir, '.last-run.json');+ if (config.cliLastRunFile) {+ this._lastRunFile = config.cliLastRunFile;+ } else {+ const [project] = filterProjects(config.projects, config.cliProjectFilter);+ if (project)+ this._lastRunFile = path.join(project.project.outputDir, '.last-run.json');+ }}- async filterLastFailed() {+ async applyFilter() {if (!this._lastRunFile)return;try {const lastRunInfo = JSON.parse(await fs.promises.readFile(this._lastRunFile, 'utf8')) as LastRunInfo;- const failedTestIds = new Set(lastRunInfo.failedTests);- // Explicitly apply --last-failed filter after sharding.- this._config.postShardTestFilters.push(test => failedTestIds.has(test.id));+ if (lastRunInfo.filterTests) {+ const filterTestIds = new Set(lastRunInfo.filterTests);+ this._config.preOnlyTestFilters.push(test => filterTestIds.has(test.id));+ }+ if (this._config.cliLastFailed) {+ const failedTestIds = new Set(lastRunInfo.failedTests ?? []);+ this._config.postShardTestFilters.push(test => failedTestIds.has(test.id));+ }} catch {}}diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.tsindex faf2e50d7f..b2612e7aa0 100644--- a/packages/playwright/src/runner/testRunner.ts+++ b/packages/playwright/src/runner/testRunner.ts@@ -493,8 +493,7 @@ export async function runAllTestsWithConfig(config: FullConfigInternal): Promiseconst reporters = await createReporters(config, listOnly ? 'list' : 'test', false);const lastRun = new LastRunReporter(config);- if (config.cliLastFailed)- await lastRun.filterLastFailed();+ await lastRun.applyFilter();const reporter = new InternalReporter([...reporters, lastRun]);const tasks = listOnly ? [diff --git a/tests/playwright-test/runner.spec.ts b/tests/playwright-test/runner.spec.tsindex dece006a06..fb57fe16c4 100644--- a/tests/playwright-test/runner.spec.ts+++ b/tests/playwright-test/runner.spec.ts@@ -842,7 +842,7 @@ test('should run last failed tests', async ({ runInlineTest }) => {expect(result2.failed).toBe(1);});-test('should run last failed tests in a shard', async ({ runInlineTest }) => {+test('should run last failed tests in a shard with a custom path', async ({ runInlineTest }) => {const workspace = {'a.spec.js': `import { test, expect } from '@playwright/test';@@ -859,17 +859,62 @@ test('should run last failed tests in a shard', async ({ runInlineTest }) => {});`,};- const result1 = await runInlineTest(workspace, { shard: '2/2' });+ const result1 = await runInlineTest(workspace, { 'shard': '2/2', 'last-run-file': 'custom.json' });expect(result1.exitCode).toBe(1);expect(result1.passed).toBe(1);expect(result1.failed).toBe(1);expect(result1.output).toContain('b.spec.js:3:11 › pass-b');expect(result1.output).toContain('b.spec.js:4:11 › fail-b');+ expect(fs.existsSync(test.info().outputPath('custom.json'))).toBe(true);- const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: ['--last-failed'] });+ const result2 = await runInlineTest(workspace, { 'shard': '2/2', 'last-failed': true, 'last-run-file': 'custom.json' });expect(result2.exitCode).toBe(1);expect(result2.passed).toBe(0);expect(result2.failed).toBe(1);expect(result2.output).not.toContain('b.spec.js:3:11 › pass-b');expect(result2.output).toContain('b.spec.js:4:11 › fail-b');});++test('should apply filterTests from last-run.json', async ({ runInlineTest }) => {+ const workspace = {+ 'reporter.ts': `+ import fs from 'fs';+ import path from 'path';+ export default class MyReporter {+ onBegin(config, suite) {+ const filterTests = suite.allTests().filter(t => t.title !== 'test2').map(t => t.id);+ const json = JSON.stringify({ filterTests });+ fs.writeFileSync(path.join(__dirname, 'filter.json'), json);+ }+ }+ `,+ 'a.spec.js': `+ import { test, expect } from '@playwright/test';+ test('test1', async () => {+ console.log('\\n%%test1');+ });+ test('test2', async () => {+ console.log('\\n%%test2');+ });+ test('test3', async () => {+ console.log('\\n%%test3');+ expect(1).toBe(2);+ });+ `+ };+ const result1 = await runInlineTest(workspace, { reporter: './reporter.ts', list: true });+ expect(result1.exitCode).toBe(0);+ expect(fs.existsSync(test.info().outputPath('filter.json'))).toBe(true);++ const result2 = await runInlineTest(workspace, { 'last-run-file': 'filter.json' });+ expect(result2.exitCode).toBe(1);+ expect(result2.passed).toBe(1);+ expect(result2.failed).toBe(1);+ expect(result2.outputLines).toEqual(['test1', 'test3']);++ const result3 = await runInlineTest(workspace, { 'last-run-file': 'filter.json', 'last-failed': true });+ expect(result3.exitCode).toBe(1);+ expect(result3.passed).toBe(0);+ expect(result3.failed).toBe(1);+ expect(result3.outputLines).toEqual(['test3']);+});