Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(test-runner): read last failed tests from --last-failed=<file> (#42995)

Pavel Feldman3456aa7d080d
diff --git a/docs/src/running-tests-js.md b/docs/src/running-tests-js.md
index 0925ef7db8..1c563a568f 100644
--- a/docs/src/running-tests-js.md
+++ b/docs/src/running-tests-js.md
@@ -91,12 +91,20 @@ To run only the tests that failed in the last test run, first run your tests and
npx playwright test --last-failed
```
-Playwright stores the list of failed tests from the previous run in `<outputDir>/.last-run.json` (see [`property: TestConfig.outputDir`](./test-configuration.md)). To use a different file path, pass `--last-failed-file=<path>` or set `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE`.
+Playwright stores the list of failed tests from the previous run in `<outputDir>/.last-run.json` (see [`property: TestConfig.outputDir`](./test-configuration.md)). To write it to a different file, pass `--last-run-output-file=<path>` or set `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE`.
```bash
-npx playwright test --last-failed --last-failed-file=.cache/last-run-shard-1.json
+npx playwright test --last-run-output-file=.cache/last-run-shard-1.json
```
+To re-run the failures recorded in a specific file, pass it to `--last-failed=<path>`. The run fails if the file does not exist. The file is only read, the results of the re-run are written to the default location or to the `--last-run-output-file` path. Without a path, `--last-failed` always reads `<outputDir>/.last-run.json`.
+
+```bash
+npx playwright test --last-failed=.cache/last-run-shard-1.json
+```
+
+Relative paths are resolved against the current working directory.
+
### Run tests in VS Code
Tests can be run right from VS Code using the [VS Code extension](https://marketplace.visualstudio.com/items?itemName=ms-playwright.playwright). Once installed you can simply click the green triangle next to the test you want to run or run all tests from the testing sidebar. Check out our [Getting Started with VS Code](./getting-started-vscode.md) guide for more details.
diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md
index 02fa76f030..23b34edfad 100644
--- a/docs/src/test-cli-js.md
+++ b/docs/src/test-cli-js.md
@@ -91,8 +91,8 @@ npx playwright test --ui
| `--headed` | Run tests in headed browsers (default: headless). |
| `--ignore-snapshots` | Ignore screenshot and snapshot expectations. |
| `-j <workers>` or `--workers <workers>` | Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 50%). |
-| `--last-failed` | Only re-run the failures. |
-| `--last-failed-file <file>` | Override the default last-run JSON path for `--last-failed` (default: `<outputDir>/.last-run.json`). Same as `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE` environment variable. |
+| `--last-failed [file]` | Only re-run the failures. Optionally takes a path to the last run file to read the failures from, use `--last-failed=<file>` form (default: `<outputDir>/.last-run.json`). |
+| `--last-run-output-file <file>` | Path to write the last run file to (default: `<outputDir>/.last-run.json`). Same as `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE` environment variable. |
| `--list` | Collect all the tests and report them, but do not run. |
| `--max-failures <N>` or `-x` | Stop after the first `N` failures. Passing `-x` stops after the first failure. |
| `--no-deps` | Do not run project dependencies. |
diff --git a/packages/playwright/src/cli/testActions.ts b/packages/playwright/src/cli/testActions.ts
index c03abcff80..f3aed24751 100644
--- a/packages/playwright/src/cli/testActions.ts
+++ b/packages/playwright/src/cli/testActions.ts
@@ -40,7 +40,8 @@ export async function runTests(args: string[], opts: { [key: string]: any }) {
projectFilter: opts.project || undefined,
passWithNoTests: !!opts.passWithNoTests,
lastFailed: !!opts.lastFailed,
- lastFailedFile: opts.lastFailedFile,
+ lastRunInputFile: typeof opts.lastFailed === 'string' ? opts.lastFailed : undefined,
+ lastRunOutputFile: opts.lastRunOutputFile,
testList: opts.testList ? path.resolve(process.cwd(), opts.testList) : undefined,
testListInvert: opts.testListInvert ? path.resolve(process.cwd(), opts.testListInvert) : undefined,
shardWeights: resolveShardWeightsOption(),
diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts
index 9042f96522..505c05b668 100644
--- a/packages/playwright/src/program.ts
+++ b/packages/playwright/src/program.ts
@@ -218,8 +218,8 @@ const testOptions: [string, { description: string, choices?: string[], preset?:
['-G, --grep-invert <grep>', { description: `Only run tests that do not match this regular expression` }],
['--headed', { description: `Run tests in headed browsers (default: headless)` }],
['--ignore-snapshots', { description: `Ignore screenshot and snapshot expectations` }],
- ['--last-failed', { description: `Only re-run the failures` }],
- ['--last-failed-file <file>', { description: `Override the default path for the last-run JSON file used with --last-failed (default: <outputDir>/.last-run.json). Same as PLAYWRIGHT_LAST_RUN_OUTPUT_FILE environment variable.` }],
+ ['--last-failed [file]', { description: `Only re-run the failures. Optionally takes a path to the last run file to read the failures from, use --last-failed=<file> form (default: <outputDir>/.last-run.json)` }],
+ ['--last-run-output-file <file>', { description: `Path to write the last run file to (default: <outputDir>/.last-run.json). Same as PLAYWRIGHT_LAST_RUN_OUTPUT_FILE environment variable.` }],
['--list', { description: `Collect all the tests and report them, but do not run` }],
['--max-failures <N>', { description: `Stop after the first N failures` }],
['--no-deps', { description: `Do not run project dependencies` }],
diff --git a/packages/playwright/src/runner/lastRun.ts b/packages/playwright/src/runner/lastRun.ts
index 8312da4471..97bf5c8e4b 100644
--- a/packages/playwright/src/runner/lastRun.ts
+++ b/packages/playwright/src/runner/lastRun.ts
@@ -17,6 +17,8 @@
import fs from 'fs';
import path from 'path';
+import { errorWithFile } from '../util';
+
import type { FullResult, Suite, TestCase } from '../../types/testReporter';
import type { config as commonConfig } from '../common';
import type { ReporterV2 } from '../reporters/reporterV2';
@@ -35,30 +37,31 @@ function didNotRun(test: TestCase): boolean {
}
export class LastRunReporter implements ReporterV2 {
- private _lastRunFile: string | undefined;
+ private _inputFile: string | undefined;
+ private _isDefaultInputFile: boolean;
+ private _outputFile: string | undefined;
private _suite: Suite | undefined;
private _listMode: boolean;
- constructor(filteredProjects: commonConfig.FullProjectInternal[], listMode?: boolean, lastFailedFileOverride?: string) {
- this._listMode = !!listMode;
- const override = lastFailedFileOverride ?? process.env.PLAYWRIGHT_LAST_RUN_OUTPUT_FILE;
- if (override) {
- this._lastRunFile = path.resolve(process.cwd(), override);
- } else {
- const [project] = filteredProjects;
- if (project)
- this._lastRunFile = path.join(project.project.outputDir, '.last-run.json');
- }
+ constructor(filteredProjects: commonConfig.FullProjectInternal[], options: { listMode?: boolean, inputFile?: string, outputFile?: string }) {
+ this._listMode = !!options.listMode;
+ const [project] = filteredProjects;
+ const defaultFile = project ? path.join(project.project.outputDir, '.last-run.json') : undefined;
+ const outputFile = options.outputFile ?? process.env.PLAYWRIGHT_LAST_RUN_OUTPUT_FILE;
+ this._outputFile = outputFile ? path.resolve(process.cwd(), outputFile) : defaultFile;
+ this._inputFile = options.inputFile ? path.resolve(process.cwd(), options.inputFile) : defaultFile;
+ this._isDefaultInputFile = !options.inputFile;
}
async filterLastFailed(): Promise<string[] | undefined> {
- if (!this._lastRunFile)
+ if (!this._inputFile)
return undefined;
try {
- const lastRunInfo = JSON.parse(await fs.promises.readFile(this._lastRunFile, 'utf8')) as LastRunInfo;
- return lastRunInfo.failedTests;
- } catch {
- return undefined;
+ return await readFailedTests(this._inputFile);
+ } catch (e) {
+ if (this._isDefaultInputFile)
+ return undefined;
+ throw e;
}
}
@@ -75,13 +78,25 @@ export class LastRunReporter implements ReporterV2 {
}
async onEnd(result: FullResult) {
- if (!this._lastRunFile || this._listMode)
+ if (!this._outputFile || this._listMode)
return;
const lastRunInfo: LastRunInfo = {
status: result.status,
failedTests: this._suite?.allTests().filter(t => !t.ok() || didNotRun(t)).map(t => t.id) || [],
};
- await fs.promises.mkdir(path.dirname(this._lastRunFile), { recursive: true });
- await fs.promises.writeFile(this._lastRunFile, JSON.stringify(lastRunInfo, undefined, 2));
+ await fs.promises.mkdir(path.dirname(this._outputFile), { recursive: true });
+ await fs.promises.writeFile(this._outputFile, JSON.stringify(lastRunInfo, undefined, 2));
+ }
+}
+
+async function readFailedTests(file: string): Promise<string[]> {
+ let lastRunInfo: LastRunInfo;
+ try {
+ lastRunInfo = JSON.parse(await fs.promises.readFile(file, 'utf8'));
+ } catch (e) {
+ throw errorWithFile(file, 'Cannot read last run file: ' + e.message);
}
+ if (!Array.isArray(lastRunInfo?.failedTests))
+ throw errorWithFile(file, 'Cannot read last run file: "failedTests" list is missing');
+ return lastRunInfo.failedTests;
}
diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts
index 1361860eab..c9426b0c73 100644
--- a/packages/playwright/src/runner/tasks.ts
+++ b/packages/playwright/src/runner/tasks.ts
@@ -63,7 +63,8 @@ export type TestRunOptions = {
listMode?: boolean;
passWithNoTests?: boolean;
lastFailed?: boolean;
- lastFailedFile?: string;
+ lastRunInputFile?: string;
+ lastRunOutputFile?: string;
testList?: string;
testListInvert?: string;
lastFailedTestIds?: string[];
diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts
index df44df98a1..e472b9c4df 100644
--- a/packages/playwright/src/runner/testRunner.ts
+++ b/packages/playwright/src/runner/testRunner.ts
@@ -429,7 +429,7 @@ export async function runAllTestsWithConfig(config: FullConfigInternal, options:
const filteredProjects = filterProjects(config.projects, options.projectFilter);
const reporters = await createReporters(config, options.listMode ? 'list' : 'test', undefined, options);
- const lastRun = new LastRunReporter(filteredProjects, options.listMode, options.lastFailedFile);
+ const lastRun = new LastRunReporter(filteredProjects, { listMode: options.listMode, inputFile: options.lastRunInputFile, outputFile: options.lastRunOutputFile });
if (options.lastFailed) {
const lastFailedTestIds = await lastRun.filterLastFailed();
if (lastFailedTestIds)
diff --git a/tests/playwright-test/runner.spec.ts b/tests/playwright-test/runner.spec.ts
index 812f6cade2..5914e2094a 100644
--- a/tests/playwright-test/runner.spec.ts
+++ b/tests/playwright-test/runner.spec.ts
@@ -977,7 +977,7 @@ test('should run last failed tests in a shard with PLAYWRIGHT_LAST_RUN_OUTPUT_FI
expect(result1.output).toContain('b.spec.js:3:11 › pass-b');
expect(result1.output).toContain('b.spec.js:4:11 › fail-b');
- const result2 = await runInlineTest(workspace, { shard: '2/2' }, env, { additionalArgs: ['--last-failed'] });
+ const result2 = await runInlineTest(workspace, { shard: '2/2' }, env, { additionalArgs: [`--last-failed=${customRel}`] });
expect(result2.exitCode).toBe(1);
expect(result2.passed).toBe(0);
expect(result2.failed).toBe(1);
@@ -985,11 +985,11 @@ test('should run last failed tests in a shard with PLAYWRIGHT_LAST_RUN_OUTPUT_FI
expect(result2.output).toContain('b.spec.js:4:11 › fail-b');
});
-test('should run last failed tests in a shard with --last-failed-file', async ({ runInlineTest }, testInfo) => {
+test('should run last failed tests in a shard with --last-run-output-file', async ({ runInlineTest }, testInfo) => {
const customRel = '.cache/shard-2-cli-last-run.json';
const customAbs = path.join(testInfo.outputPath(), customRel);
const defaultLastRun = path.join(testInfo.outputPath(), 'test-results', '.last-run.json');
- const lastRunArgs = ['--last-failed', `--last-failed-file=${customRel}`];
+ const lastRunArgs = [`--last-run-output-file=${customRel}`];
const workspace = {
'a.spec.js': `
import { test, expect } from '@playwright/test';
@@ -1015,10 +1015,92 @@ test('should run last failed tests in a shard with --last-failed-file', async ({
expect(result1.output).toContain('b.spec.js:3:11 › pass-b');
expect(result1.output).toContain('b.spec.js:4:11 › fail-b');
- const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: lastRunArgs });
+ const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: [`--last-failed=${customRel}`, ...lastRunArgs] });
expect(result2.exitCode).toBe(1);
expect(result2.passed).toBe(0);
expect(result2.failed).toBe(1);
expect(result2.output).not.toContain('b.spec.js:3:11 › pass-b');
expect(result2.output).toContain('b.spec.js:4:11 › fail-b');
});
+
+test('should read --last-failed=<file> without overwriting it', async ({ runInlineTest }, testInfo) => {
+ const inputFile = testInfo.outputPath('.cache', 'input.json');
+ const outputFile = testInfo.outputPath('.cache', 'output.json');
+ const defaultLastRun = testInfo.outputPath('test-results', '.last-run.json');
+ const spec = (expected: number) => `
+ import { test, expect } from '@playwright/test';
+ test('pass-a', async () => {});
+ test('fail-a', async () => {
+ expect(1).toBe(${expected});
+ });
+ `;
+
+ const result1 = await runInlineTest({ 'a.spec.js': spec(2) }, {}, {}, { additionalArgs: ['--last-run-output-file=.cache/input.json'] });
+ expect(result1.exitCode).toBe(1);
+ expect(result1.passed).toBe(1);
+ expect(result1.failed).toBe(1);
+ expect(fs.existsSync(defaultLastRun)).toBe(false);
+ const input = fs.readFileSync(inputFile, 'utf8');
+ expect(JSON.parse(input).failedTests).toHaveLength(1);
+
+ const result2 = await runInlineTest({ 'a.spec.js': spec(1) }, {}, {}, { additionalArgs: ['--last-failed=.cache/input.json'] });
+ expect(result2.exitCode).toBe(0);
+ expect(result2.passed).toBe(1);
+ expect(result2.output).not.toContain('pass-a');
+ expect(result2.output).toContain('fail-a');
+ expect(fs.readFileSync(inputFile, 'utf8')).toBe(input);
+ expect(JSON.parse(fs.readFileSync(defaultLastRun, 'utf8'))).toEqual({ status: 'passed', failedTests: [] });
+
+ const result3 = await runInlineTest({ 'a.spec.js': spec(1) }, {}, {}, { additionalArgs: ['--last-failed=.cache/input.json', '--last-run-output-file=.cache/output.json'] });
+ expect(result3.exitCode).toBe(0);
+ expect(result3.passed).toBe(1);
+ expect(result3.output).not.toContain('pass-a');
+ expect(fs.readFileSync(inputFile, 'utf8')).toBe(input);
+ expect(JSON.parse(fs.readFileSync(outputFile, 'utf8'))).toEqual({ status: 'passed', failedTests: [] });
+ expect(fs.existsSync(defaultLastRun)).toBe(false);
+});
+
+test('should fail when --last-failed=<file> is missing or malformed', async ({ runInlineTest }) => {
+ const workspace = {
+ 'a.spec.js': `
+ import { test, expect } from '@playwright/test';
+ test('pass-a', async () => {});
+ `,
+ 'malformed.json': `{ "status": "failed" }`,
+ };
+
+ const result1 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed=missing.json'] });
+ expect(result1.exitCode).toBe(1);
+ expect(result1.passed).toBe(0);
+ expect(result1.output).toContain('missing.json: Cannot read last run file: ENOENT');
+
+ const result2 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed=malformed.json'] });
+ expect(result2.exitCode).toBe(1);
+ expect(result2.passed).toBe(0);
+ expect(result2.output).toContain('malformed.json: Cannot read last run file: "failedTests" list is missing');
+});
+
+test('should resolve --last-failed and --last-run-output-file against cwd', async ({ runInlineTest }, testInfo) => {
+ const workspace = {
+ 'dir/playwright.config.ts': `module.exports = {};`,
+ 'dir/a.spec.js': `
+ import { test, expect } from '@playwright/test';
+ test('pass-a', async () => {});
+ test('fail-a', async () => {
+ expect(1).toBe(2);
+ });
+ `,
+ };
+
+ const result1 = await runInlineTest(workspace, { config: 'dir/playwright.config.ts' }, {}, { additionalArgs: ['--last-run-output-file=last-run.json'] });
+ expect(result1.exitCode).toBe(1);
+ expect(result1.passed).toBe(1);
+ expect(result1.failed).toBe(1);
+ expect(fs.existsSync(testInfo.outputPath('last-run.json'))).toBe(true);
+ expect(fs.existsSync(testInfo.outputPath('dir', 'last-run.json'))).toBe(false);
+
+ const result2 = await runInlineTest(workspace, { config: 'dir/playwright.config.ts' }, {}, { additionalArgs: ['--last-failed=last-run.json'] });
+ expect(result2.exitCode).toBe(1);
+ expect(result2.passed).toBe(0);
+ expect(result2.failed).toBe(1);
+});