Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(test-runner): read last failed tests from --last-failed=<file> (#42995)
Pavel Feldman3456aa7d080d
diff --git a/docs/src/running-tests-js.md b/docs/src/running-tests-js.mdindex 0925ef7db8..1c563a568f 100644--- a/docs/src/running-tests-js.md+++ b/docs/src/running-tests-js.md@@ -91,12 +91,20 @@ To run only the tests that failed in the last test run, first run your tests andnpx playwright test --last-failed```-Playwright stores the list of failed tests from the previous run in `<outputDir>/.last-run.json` (see [`property: TestConfig.outputDir`](./test-configuration.md)). To use a different file path, pass `--last-failed-file=<path>` or set `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE`.+Playwright stores the list of failed tests from the previous run in `<outputDir>/.last-run.json` (see [`property: TestConfig.outputDir`](./test-configuration.md)). To write it to a different file, pass `--last-run-output-file=<path>` or set `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE`.```bash-npx playwright test --last-failed --last-failed-file=.cache/last-run-shard-1.json+npx playwright test --last-run-output-file=.cache/last-run-shard-1.json```+To re-run the failures recorded in a specific file, pass it to `--last-failed=<path>`. The run fails if the file does not exist. The file is only read, the results of the re-run are written to the default location or to the `--last-run-output-file` path. Without a path, `--last-failed` always reads `<outputDir>/.last-run.json`.++```bash+npx playwright test --last-failed=.cache/last-run-shard-1.json+```++Relative paths are resolved against the current working directory.+### Run tests in VS CodeTests can be run right from VS Code using the [VS Code extension](https://marketplace.visualstudio.com/items?itemName=ms-playwright.playwright). Once installed you can simply click the green triangle next to the test you want to run or run all tests from the testing sidebar. Check out our [Getting Started with VS Code](./getting-started-vscode.md) guide for more details.diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.mdindex 02fa76f030..23b34edfad 100644--- a/docs/src/test-cli-js.md+++ b/docs/src/test-cli-js.md@@ -91,8 +91,8 @@ npx playwright test --ui| `--headed` | Run tests in headed browsers (default: headless). || `--ignore-snapshots` | Ignore screenshot and snapshot expectations. || `-j <workers>` or `--workers <workers>` | Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 50%). |-| `--last-failed` | Only re-run the failures. |-| `--last-failed-file <file>` | Override the default last-run JSON path for `--last-failed` (default: `<outputDir>/.last-run.json`). Same as `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE` environment variable. |+| `--last-failed [file]` | Only re-run the failures. Optionally takes a path to the last run file to read the failures from, use `--last-failed=<file>` form (default: `<outputDir>/.last-run.json`). |+| `--last-run-output-file <file>` | Path to write the last run file to (default: `<outputDir>/.last-run.json`). Same as `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE` environment variable. || `--list` | Collect all the tests and report them, but do not run. || `--max-failures <N>` or `-x` | Stop after the first `N` failures. Passing `-x` stops after the first failure. || `--no-deps` | Do not run project dependencies. |diff --git a/packages/playwright/src/cli/testActions.ts b/packages/playwright/src/cli/testActions.tsindex c03abcff80..f3aed24751 100644--- a/packages/playwright/src/cli/testActions.ts+++ b/packages/playwright/src/cli/testActions.ts@@ -40,7 +40,8 @@ export async function runTests(args: string[], opts: { [key: string]: any }) {projectFilter: opts.project || undefined,passWithNoTests: !!opts.passWithNoTests,lastFailed: !!opts.lastFailed,- lastFailedFile: opts.lastFailedFile,+ lastRunInputFile: typeof opts.lastFailed === 'string' ? opts.lastFailed : undefined,+ lastRunOutputFile: opts.lastRunOutputFile,testList: opts.testList ? path.resolve(process.cwd(), opts.testList) : undefined,testListInvert: opts.testListInvert ? path.resolve(process.cwd(), opts.testListInvert) : undefined,shardWeights: resolveShardWeightsOption(),diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.tsindex 9042f96522..505c05b668 100644--- a/packages/playwright/src/program.ts+++ b/packages/playwright/src/program.ts@@ -218,8 +218,8 @@ const testOptions: [string, { description: string, choices?: string[], preset?:['-G, --grep-invert <grep>', { description: `Only run tests that do not match this regular expression` }],['--headed', { description: `Run tests in headed browsers (default: headless)` }],['--ignore-snapshots', { description: `Ignore screenshot and snapshot expectations` }],- ['--last-failed', { description: `Only re-run the failures` }],- ['--last-failed-file <file>', { description: `Override the default path for the last-run JSON file used with --last-failed (default: <outputDir>/.last-run.json). Same as PLAYWRIGHT_LAST_RUN_OUTPUT_FILE environment variable.` }],+ ['--last-failed [file]', { description: `Only re-run the failures. Optionally takes a path to the last run file to read the failures from, use --last-failed=<file> form (default: <outputDir>/.last-run.json)` }],+ ['--last-run-output-file <file>', { description: `Path to write the last run file to (default: <outputDir>/.last-run.json). Same as PLAYWRIGHT_LAST_RUN_OUTPUT_FILE environment variable.` }],['--list', { description: `Collect all the tests and report them, but do not run` }],['--max-failures <N>', { description: `Stop after the first N failures` }],['--no-deps', { description: `Do not run project dependencies` }],diff --git a/packages/playwright/src/runner/lastRun.ts b/packages/playwright/src/runner/lastRun.tsindex 8312da4471..97bf5c8e4b 100644--- a/packages/playwright/src/runner/lastRun.ts+++ b/packages/playwright/src/runner/lastRun.ts@@ -17,6 +17,8 @@import fs from 'fs';import path from 'path';+import { errorWithFile } from '../util';+import type { FullResult, Suite, TestCase } from '../../types/testReporter';import type { config as commonConfig } from '../common';import type { ReporterV2 } from '../reporters/reporterV2';@@ -35,30 +37,31 @@ function didNotRun(test: TestCase): boolean {}export class LastRunReporter implements ReporterV2 {- private _lastRunFile: string | undefined;+ private _inputFile: string | undefined;+ private _isDefaultInputFile: boolean;+ private _outputFile: string | undefined;private _suite: Suite | undefined;private _listMode: boolean;- constructor(filteredProjects: commonConfig.FullProjectInternal[], listMode?: boolean, lastFailedFileOverride?: string) {- this._listMode = !!listMode;- const override = lastFailedFileOverride ?? process.env.PLAYWRIGHT_LAST_RUN_OUTPUT_FILE;- if (override) {- this._lastRunFile = path.resolve(process.cwd(), override);- } else {- const [project] = filteredProjects;- if (project)- this._lastRunFile = path.join(project.project.outputDir, '.last-run.json');- }+ constructor(filteredProjects: commonConfig.FullProjectInternal[], options: { listMode?: boolean, inputFile?: string, outputFile?: string }) {+ this._listMode = !!options.listMode;+ const [project] = filteredProjects;+ const defaultFile = project ? path.join(project.project.outputDir, '.last-run.json') : undefined;+ const outputFile = options.outputFile ?? process.env.PLAYWRIGHT_LAST_RUN_OUTPUT_FILE;+ this._outputFile = outputFile ? path.resolve(process.cwd(), outputFile) : defaultFile;+ this._inputFile = options.inputFile ? path.resolve(process.cwd(), options.inputFile) : defaultFile;+ this._isDefaultInputFile = !options.inputFile;}async filterLastFailed(): Promise<string[] | undefined> {- if (!this._lastRunFile)+ if (!this._inputFile)return undefined;try {- const lastRunInfo = JSON.parse(await fs.promises.readFile(this._lastRunFile, 'utf8')) as LastRunInfo;- return lastRunInfo.failedTests;- } catch {- return undefined;+ return await readFailedTests(this._inputFile);+ } catch (e) {+ if (this._isDefaultInputFile)+ return undefined;+ throw e;}}@@ -75,13 +78,25 @@ export class LastRunReporter implements ReporterV2 {}async onEnd(result: FullResult) {- if (!this._lastRunFile || this._listMode)+ if (!this._outputFile || this._listMode)return;const lastRunInfo: LastRunInfo = {status: result.status,failedTests: this._suite?.allTests().filter(t => !t.ok() || didNotRun(t)).map(t => t.id) || [],};- await fs.promises.mkdir(path.dirname(this._lastRunFile), { recursive: true });- await fs.promises.writeFile(this._lastRunFile, JSON.stringify(lastRunInfo, undefined, 2));+ await fs.promises.mkdir(path.dirname(this._outputFile), { recursive: true });+ await fs.promises.writeFile(this._outputFile, JSON.stringify(lastRunInfo, undefined, 2));+ }+}++async function readFailedTests(file: string): Promise<string[]> {+ let lastRunInfo: LastRunInfo;+ try {+ lastRunInfo = JSON.parse(await fs.promises.readFile(file, 'utf8'));+ } catch (e) {+ throw errorWithFile(file, 'Cannot read last run file: ' + e.message);}+ if (!Array.isArray(lastRunInfo?.failedTests))+ throw errorWithFile(file, 'Cannot read last run file: "failedTests" list is missing');+ return lastRunInfo.failedTests;}diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.tsindex 1361860eab..c9426b0c73 100644--- a/packages/playwright/src/runner/tasks.ts+++ b/packages/playwright/src/runner/tasks.ts@@ -63,7 +63,8 @@ export type TestRunOptions = {listMode?: boolean;passWithNoTests?: boolean;lastFailed?: boolean;- lastFailedFile?: string;+ lastRunInputFile?: string;+ lastRunOutputFile?: string;testList?: string;testListInvert?: string;lastFailedTestIds?: string[];diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.tsindex df44df98a1..e472b9c4df 100644--- a/packages/playwright/src/runner/testRunner.ts+++ b/packages/playwright/src/runner/testRunner.ts@@ -429,7 +429,7 @@ export async function runAllTestsWithConfig(config: FullConfigInternal, options:const filteredProjects = filterProjects(config.projects, options.projectFilter);const reporters = await createReporters(config, options.listMode ? 'list' : 'test', undefined, options);- const lastRun = new LastRunReporter(filteredProjects, options.listMode, options.lastFailedFile);+ const lastRun = new LastRunReporter(filteredProjects, { listMode: options.listMode, inputFile: options.lastRunInputFile, outputFile: options.lastRunOutputFile });if (options.lastFailed) {const lastFailedTestIds = await lastRun.filterLastFailed();if (lastFailedTestIds)diff --git a/tests/playwright-test/runner.spec.ts b/tests/playwright-test/runner.spec.tsindex 812f6cade2..5914e2094a 100644--- a/tests/playwright-test/runner.spec.ts+++ b/tests/playwright-test/runner.spec.ts@@ -977,7 +977,7 @@ test('should run last failed tests in a shard with PLAYWRIGHT_LAST_RUN_OUTPUT_FIexpect(result1.output).toContain('b.spec.js:3:11 › pass-b');expect(result1.output).toContain('b.spec.js:4:11 › fail-b');- const result2 = await runInlineTest(workspace, { shard: '2/2' }, env, { additionalArgs: ['--last-failed'] });+ const result2 = await runInlineTest(workspace, { shard: '2/2' }, env, { additionalArgs: [`--last-failed=${customRel}`] });expect(result2.exitCode).toBe(1);expect(result2.passed).toBe(0);expect(result2.failed).toBe(1);@@ -985,11 +985,11 @@ test('should run last failed tests in a shard with PLAYWRIGHT_LAST_RUN_OUTPUT_FIexpect(result2.output).toContain('b.spec.js:4:11 › fail-b');});-test('should run last failed tests in a shard with --last-failed-file', async ({ runInlineTest }, testInfo) => {+test('should run last failed tests in a shard with --last-run-output-file', async ({ runInlineTest }, testInfo) => {const customRel = '.cache/shard-2-cli-last-run.json';const customAbs = path.join(testInfo.outputPath(), customRel);const defaultLastRun = path.join(testInfo.outputPath(), 'test-results', '.last-run.json');- const lastRunArgs = ['--last-failed', `--last-failed-file=${customRel}`];+ const lastRunArgs = [`--last-run-output-file=${customRel}`];const workspace = {'a.spec.js': `import { test, expect } from '@playwright/test';@@ -1015,10 +1015,92 @@ test('should run last failed tests in a shard with --last-failed-file', async ({expect(result1.output).toContain('b.spec.js:3:11 › pass-b');expect(result1.output).toContain('b.spec.js:4:11 › fail-b');- const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: lastRunArgs });+ const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: [`--last-failed=${customRel}`, ...lastRunArgs] });expect(result2.exitCode).toBe(1);expect(result2.passed).toBe(0);expect(result2.failed).toBe(1);expect(result2.output).not.toContain('b.spec.js:3:11 › pass-b');expect(result2.output).toContain('b.spec.js:4:11 › fail-b');});++test('should read --last-failed=<file> without overwriting it', async ({ runInlineTest }, testInfo) => {+ const inputFile = testInfo.outputPath('.cache', 'input.json');+ const outputFile = testInfo.outputPath('.cache', 'output.json');+ const defaultLastRun = testInfo.outputPath('test-results', '.last-run.json');+ const spec = (expected: number) => `+ import { test, expect } from '@playwright/test';+ test('pass-a', async () => {});+ test('fail-a', async () => {+ expect(1).toBe(${expected});+ });+ `;++ const result1 = await runInlineTest({ 'a.spec.js': spec(2) }, {}, {}, { additionalArgs: ['--last-run-output-file=.cache/input.json'] });+ expect(result1.exitCode).toBe(1);+ expect(result1.passed).toBe(1);+ expect(result1.failed).toBe(1);+ expect(fs.existsSync(defaultLastRun)).toBe(false);+ const input = fs.readFileSync(inputFile, 'utf8');+ expect(JSON.parse(input).failedTests).toHaveLength(1);++ const result2 = await runInlineTest({ 'a.spec.js': spec(1) }, {}, {}, { additionalArgs: ['--last-failed=.cache/input.json'] });+ expect(result2.exitCode).toBe(0);+ expect(result2.passed).toBe(1);+ expect(result2.output).not.toContain('pass-a');+ expect(result2.output).toContain('fail-a');+ expect(fs.readFileSync(inputFile, 'utf8')).toBe(input);+ expect(JSON.parse(fs.readFileSync(defaultLastRun, 'utf8'))).toEqual({ status: 'passed', failedTests: [] });++ const result3 = await runInlineTest({ 'a.spec.js': spec(1) }, {}, {}, { additionalArgs: ['--last-failed=.cache/input.json', '--last-run-output-file=.cache/output.json'] });+ expect(result3.exitCode).toBe(0);+ expect(result3.passed).toBe(1);+ expect(result3.output).not.toContain('pass-a');+ expect(fs.readFileSync(inputFile, 'utf8')).toBe(input);+ expect(JSON.parse(fs.readFileSync(outputFile, 'utf8'))).toEqual({ status: 'passed', failedTests: [] });+ expect(fs.existsSync(defaultLastRun)).toBe(false);+});++test('should fail when --last-failed=<file> is missing or malformed', async ({ runInlineTest }) => {+ const workspace = {+ 'a.spec.js': `+ import { test, expect } from '@playwright/test';+ test('pass-a', async () => {});+ `,+ 'malformed.json': `{ "status": "failed" }`,+ };++ const result1 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed=missing.json'] });+ expect(result1.exitCode).toBe(1);+ expect(result1.passed).toBe(0);+ expect(result1.output).toContain('missing.json: Cannot read last run file: ENOENT');++ const result2 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed=malformed.json'] });+ expect(result2.exitCode).toBe(1);+ expect(result2.passed).toBe(0);+ expect(result2.output).toContain('malformed.json: Cannot read last run file: "failedTests" list is missing');+});++test('should resolve --last-failed and --last-run-output-file against cwd', async ({ runInlineTest }, testInfo) => {+ const workspace = {+ 'dir/playwright.config.ts': `module.exports = {};`,+ 'dir/a.spec.js': `+ import { test, expect } from '@playwright/test';+ test('pass-a', async () => {});+ test('fail-a', async () => {+ expect(1).toBe(2);+ });+ `,+ };++ const result1 = await runInlineTest(workspace, { config: 'dir/playwright.config.ts' }, {}, { additionalArgs: ['--last-run-output-file=last-run.json'] });+ expect(result1.exitCode).toBe(1);+ expect(result1.passed).toBe(1);+ expect(result1.failed).toBe(1);+ expect(fs.existsSync(testInfo.outputPath('last-run.json'))).toBe(true);+ expect(fs.existsSync(testInfo.outputPath('dir', 'last-run.json'))).toBe(false);++ const result2 = await runInlineTest(workspace, { config: 'dir/playwright.config.ts' }, {}, { additionalArgs: ['--last-failed=last-run.json'] });+ expect(result2.exitCode).toBe(1);+ expect(result2.passed).toBe(0);+ expect(result2.failed).toBe(1);+});