Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

Revert "feat(test-runner): read last failed tests from --last-failed=<file>" (#43053)

Pavel Feldman45b43ff3581d
diff --git a/docs/src/running-tests-js.md b/docs/src/running-tests-js.md
index 1c563a568f..0925ef7db8 100644
--- a/docs/src/running-tests-js.md
+++ b/docs/src/running-tests-js.md
@@ -91,20 +91,12 @@ To run only the tests that failed in the last test run, first run your tests and
npx playwright test --last-failed
```
-Playwright stores the list of failed tests from the previous run in `<outputDir>/.last-run.json` (see [`property: TestConfig.outputDir`](./test-configuration.md)). To write it to a different file, pass `--last-run-output-file=<path>` or set `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE`.
+Playwright stores the list of failed tests from the previous run in `<outputDir>/.last-run.json` (see [`property: TestConfig.outputDir`](./test-configuration.md)). To use a different file path, pass `--last-failed-file=<path>` or set `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE`.
```bash
-npx playwright test --last-run-output-file=.cache/last-run-shard-1.json
+npx playwright test --last-failed --last-failed-file=.cache/last-run-shard-1.json
```
-To re-run the failures recorded in a specific file, pass it to `--last-failed=<path>`. The run fails if the file does not exist. The file is only read, the results of the re-run are written to the default location or to the `--last-run-output-file` path. Without a path, `--last-failed` always reads `<outputDir>/.last-run.json`.
-
-```bash
-npx playwright test --last-failed=.cache/last-run-shard-1.json
-```
-
-Relative paths are resolved against the current working directory.
-
### Run tests in VS Code
Tests can be run right from VS Code using the [VS Code extension](https://marketplace.visualstudio.com/items?itemName=ms-playwright.playwright). Once installed you can simply click the green triangle next to the test you want to run or run all tests from the testing sidebar. Check out our [Getting Started with VS Code](./getting-started-vscode.md) guide for more details.
diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md
index 67c8dbb5d5..fadedf6b74 100644
--- a/docs/src/test-cli-js.md
+++ b/docs/src/test-cli-js.md
@@ -91,8 +91,8 @@ npx playwright test --ui
| `--headed` | Run tests in headed browsers (default: headless). |
| `--ignore-snapshots` | Ignore screenshot and snapshot expectations. |
| `-j <workers>` or `--workers <workers>` | Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 50%). |
-| `--last-failed [file]` | Only re-run the failures. Optionally takes a path to the last run file to read the failures from, use `--last-failed=<file>` form (default: `<outputDir>/.last-run.json`). |
-| `--last-run-output-file <file>` | Path to write the last run file to (default: `<outputDir>/.last-run.json`). Same as `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE` environment variable. |
+| `--last-failed` | Only re-run the failures. |
+| `--last-failed-file <file>` | Override the default last-run JSON path for `--last-failed` (default: `<outputDir>/.last-run.json`). Same as `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE` environment variable. |
| `--list` | Collect all the tests and report them, but do not run. |
| `--max-failures <N>` or `-x` | Stop after the first `N` failures. Passing `-x` stops after the first failure. |
| `--no-deps` | Do not run project dependencies. |
diff --git a/packages/playwright/src/cli/testActions.ts b/packages/playwright/src/cli/testActions.ts
index 904d2a76ac..b6fd25facb 100644
--- a/packages/playwright/src/cli/testActions.ts
+++ b/packages/playwright/src/cli/testActions.ts
@@ -38,8 +38,7 @@ export async function runTests(args: string[], opts: { [key: string]: any }) {
projectFilter: opts.project || undefined,
passWithNoTests: !!opts.passWithNoTests,
lastFailed: !!opts.lastFailed,
- lastRunInputFile: typeof opts.lastFailed === 'string' ? opts.lastFailed : undefined,
- lastRunOutputFile: opts.lastRunOutputFile,
+ lastFailedFile: opts.lastFailedFile,
testList: opts.testList ? path.resolve(process.cwd(), opts.testList) : undefined,
testListInvert: opts.testListInvert ? path.resolve(process.cwd(), opts.testListInvert) : undefined,
shardWeights: resolveShardWeightsOption(),
diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts
index 505c05b668..9042f96522 100644
--- a/packages/playwright/src/program.ts
+++ b/packages/playwright/src/program.ts
@@ -218,8 +218,8 @@ const testOptions: [string, { description: string, choices?: string[], preset?:
['-G, --grep-invert <grep>', { description: `Only run tests that do not match this regular expression` }],
['--headed', { description: `Run tests in headed browsers (default: headless)` }],
['--ignore-snapshots', { description: `Ignore screenshot and snapshot expectations` }],
- ['--last-failed [file]', { description: `Only re-run the failures. Optionally takes a path to the last run file to read the failures from, use --last-failed=<file> form (default: <outputDir>/.last-run.json)` }],
- ['--last-run-output-file <file>', { description: `Path to write the last run file to (default: <outputDir>/.last-run.json). Same as PLAYWRIGHT_LAST_RUN_OUTPUT_FILE environment variable.` }],
+ ['--last-failed', { description: `Only re-run the failures` }],
+ ['--last-failed-file <file>', { description: `Override the default path for the last-run JSON file used with --last-failed (default: <outputDir>/.last-run.json). Same as PLAYWRIGHT_LAST_RUN_OUTPUT_FILE environment variable.` }],
['--list', { description: `Collect all the tests and report them, but do not run` }],
['--max-failures <N>', { description: `Stop after the first N failures` }],
['--no-deps', { description: `Do not run project dependencies` }],
diff --git a/packages/playwright/src/runner/lastRun.ts b/packages/playwright/src/runner/lastRun.ts
index 97bf5c8e4b..8312da4471 100644
--- a/packages/playwright/src/runner/lastRun.ts
+++ b/packages/playwright/src/runner/lastRun.ts
@@ -17,8 +17,6 @@
import fs from 'fs';
import path from 'path';
-import { errorWithFile } from '../util';
-
import type { FullResult, Suite, TestCase } from '../../types/testReporter';
import type { config as commonConfig } from '../common';
import type { ReporterV2 } from '../reporters/reporterV2';
@@ -37,31 +35,30 @@ function didNotRun(test: TestCase): boolean {
}
export class LastRunReporter implements ReporterV2 {
- private _inputFile: string | undefined;
- private _isDefaultInputFile: boolean;
- private _outputFile: string | undefined;
+ private _lastRunFile: string | undefined;
private _suite: Suite | undefined;
private _listMode: boolean;
- constructor(filteredProjects: commonConfig.FullProjectInternal[], options: { listMode?: boolean, inputFile?: string, outputFile?: string }) {
- this._listMode = !!options.listMode;
- const [project] = filteredProjects;
- const defaultFile = project ? path.join(project.project.outputDir, '.last-run.json') : undefined;
- const outputFile = options.outputFile ?? process.env.PLAYWRIGHT_LAST_RUN_OUTPUT_FILE;
- this._outputFile = outputFile ? path.resolve(process.cwd(), outputFile) : defaultFile;
- this._inputFile = options.inputFile ? path.resolve(process.cwd(), options.inputFile) : defaultFile;
- this._isDefaultInputFile = !options.inputFile;
+ constructor(filteredProjects: commonConfig.FullProjectInternal[], listMode?: boolean, lastFailedFileOverride?: string) {
+ this._listMode = !!listMode;
+ const override = lastFailedFileOverride ?? process.env.PLAYWRIGHT_LAST_RUN_OUTPUT_FILE;
+ if (override) {
+ this._lastRunFile = path.resolve(process.cwd(), override);
+ } else {
+ const [project] = filteredProjects;
+ if (project)
+ this._lastRunFile = path.join(project.project.outputDir, '.last-run.json');
+ }
}
async filterLastFailed(): Promise<string[] | undefined> {
- if (!this._inputFile)
+ if (!this._lastRunFile)
return undefined;
try {
- return await readFailedTests(this._inputFile);
- } catch (e) {
- if (this._isDefaultInputFile)
- return undefined;
- throw e;
+ const lastRunInfo = JSON.parse(await fs.promises.readFile(this._lastRunFile, 'utf8')) as LastRunInfo;
+ return lastRunInfo.failedTests;
+ } catch {
+ return undefined;
}
}
@@ -78,25 +75,13 @@ export class LastRunReporter implements ReporterV2 {
}
async onEnd(result: FullResult) {
- if (!this._outputFile || this._listMode)
+ if (!this._lastRunFile || this._listMode)
return;
const lastRunInfo: LastRunInfo = {
status: result.status,
failedTests: this._suite?.allTests().filter(t => !t.ok() || didNotRun(t)).map(t => t.id) || [],
};
- await fs.promises.mkdir(path.dirname(this._outputFile), { recursive: true });
- await fs.promises.writeFile(this._outputFile, JSON.stringify(lastRunInfo, undefined, 2));
- }
-}
-
-async function readFailedTests(file: string): Promise<string[]> {
- let lastRunInfo: LastRunInfo;
- try {
- lastRunInfo = JSON.parse(await fs.promises.readFile(file, 'utf8'));
- } catch (e) {
- throw errorWithFile(file, 'Cannot read last run file: ' + e.message);
+ await fs.promises.mkdir(path.dirname(this._lastRunFile), { recursive: true });
+ await fs.promises.writeFile(this._lastRunFile, JSON.stringify(lastRunInfo, undefined, 2));
}
- if (!Array.isArray(lastRunInfo?.failedTests))
- throw errorWithFile(file, 'Cannot read last run file: "failedTests" list is missing');
- return lastRunInfo.failedTests;
}
diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts
index c9426b0c73..1361860eab 100644
--- a/packages/playwright/src/runner/tasks.ts
+++ b/packages/playwright/src/runner/tasks.ts
@@ -63,8 +63,7 @@ export type TestRunOptions = {
listMode?: boolean;
passWithNoTests?: boolean;
lastFailed?: boolean;
- lastRunInputFile?: string;
- lastRunOutputFile?: string;
+ lastFailedFile?: string;
testList?: string;
testListInvert?: string;
lastFailedTestIds?: string[];
diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts
index e472b9c4df..df44df98a1 100644
--- a/packages/playwright/src/runner/testRunner.ts
+++ b/packages/playwright/src/runner/testRunner.ts
@@ -429,7 +429,7 @@ export async function runAllTestsWithConfig(config: FullConfigInternal, options:
const filteredProjects = filterProjects(config.projects, options.projectFilter);
const reporters = await createReporters(config, options.listMode ? 'list' : 'test', undefined, options);
- const lastRun = new LastRunReporter(filteredProjects, { listMode: options.listMode, inputFile: options.lastRunInputFile, outputFile: options.lastRunOutputFile });
+ const lastRun = new LastRunReporter(filteredProjects, options.listMode, options.lastFailedFile);
if (options.lastFailed) {
const lastFailedTestIds = await lastRun.filterLastFailed();
if (lastFailedTestIds)
diff --git a/tests/playwright-test/runner.spec.ts b/tests/playwright-test/runner.spec.ts
index 5914e2094a..812f6cade2 100644
--- a/tests/playwright-test/runner.spec.ts
+++ b/tests/playwright-test/runner.spec.ts
@@ -977,7 +977,7 @@ test('should run last failed tests in a shard with PLAYWRIGHT_LAST_RUN_OUTPUT_FI
expect(result1.output).toContain('b.spec.js:3:11 › pass-b');
expect(result1.output).toContain('b.spec.js:4:11 › fail-b');
- const result2 = await runInlineTest(workspace, { shard: '2/2' }, env, { additionalArgs: [`--last-failed=${customRel}`] });
+ const result2 = await runInlineTest(workspace, { shard: '2/2' }, env, { additionalArgs: ['--last-failed'] });
expect(result2.exitCode).toBe(1);
expect(result2.passed).toBe(0);
expect(result2.failed).toBe(1);
@@ -985,11 +985,11 @@ test('should run last failed tests in a shard with PLAYWRIGHT_LAST_RUN_OUTPUT_FI
expect(result2.output).toContain('b.spec.js:4:11 › fail-b');
});
-test('should run last failed tests in a shard with --last-run-output-file', async ({ runInlineTest }, testInfo) => {
+test('should run last failed tests in a shard with --last-failed-file', async ({ runInlineTest }, testInfo) => {
const customRel = '.cache/shard-2-cli-last-run.json';
const customAbs = path.join(testInfo.outputPath(), customRel);
const defaultLastRun = path.join(testInfo.outputPath(), 'test-results', '.last-run.json');
- const lastRunArgs = [`--last-run-output-file=${customRel}`];
+ const lastRunArgs = ['--last-failed', `--last-failed-file=${customRel}`];
const workspace = {
'a.spec.js': `
import { test, expect } from '@playwright/test';
@@ -1015,92 +1015,10 @@ test('should run last failed tests in a shard with --last-run-output-file', asyn
expect(result1.output).toContain('b.spec.js:3:11 › pass-b');
expect(result1.output).toContain('b.spec.js:4:11 › fail-b');
- const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: [`--last-failed=${customRel}`, ...lastRunArgs] });
+ const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: lastRunArgs });
expect(result2.exitCode).toBe(1);
expect(result2.passed).toBe(0);
expect(result2.failed).toBe(1);
expect(result2.output).not.toContain('b.spec.js:3:11 › pass-b');
expect(result2.output).toContain('b.spec.js:4:11 › fail-b');
});
-
-test('should read --last-failed=<file> without overwriting it', async ({ runInlineTest }, testInfo) => {
- const inputFile = testInfo.outputPath('.cache', 'input.json');
- const outputFile = testInfo.outputPath('.cache', 'output.json');
- const defaultLastRun = testInfo.outputPath('test-results', '.last-run.json');
- const spec = (expected: number) => `
- import { test, expect } from '@playwright/test';
- test('pass-a', async () => {});
- test('fail-a', async () => {
- expect(1).toBe(${expected});
- });
- `;
-
- const result1 = await runInlineTest({ 'a.spec.js': spec(2) }, {}, {}, { additionalArgs: ['--last-run-output-file=.cache/input.json'] });
- expect(result1.exitCode).toBe(1);
- expect(result1.passed).toBe(1);
- expect(result1.failed).toBe(1);
- expect(fs.existsSync(defaultLastRun)).toBe(false);
- const input = fs.readFileSync(inputFile, 'utf8');
- expect(JSON.parse(input).failedTests).toHaveLength(1);
-
- const result2 = await runInlineTest({ 'a.spec.js': spec(1) }, {}, {}, { additionalArgs: ['--last-failed=.cache/input.json'] });
- expect(result2.exitCode).toBe(0);
- expect(result2.passed).toBe(1);
- expect(result2.output).not.toContain('pass-a');
- expect(result2.output).toContain('fail-a');
- expect(fs.readFileSync(inputFile, 'utf8')).toBe(input);
- expect(JSON.parse(fs.readFileSync(defaultLastRun, 'utf8'))).toEqual({ status: 'passed', failedTests: [] });
-
- const result3 = await runInlineTest({ 'a.spec.js': spec(1) }, {}, {}, { additionalArgs: ['--last-failed=.cache/input.json', '--last-run-output-file=.cache/output.json'] });
- expect(result3.exitCode).toBe(0);
- expect(result3.passed).toBe(1);
- expect(result3.output).not.toContain('pass-a');
- expect(fs.readFileSync(inputFile, 'utf8')).toBe(input);
- expect(JSON.parse(fs.readFileSync(outputFile, 'utf8'))).toEqual({ status: 'passed', failedTests: [] });
- expect(fs.existsSync(defaultLastRun)).toBe(false);
-});
-
-test('should fail when --last-failed=<file> is missing or malformed', async ({ runInlineTest }) => {
- const workspace = {
- 'a.spec.js': `
- import { test, expect } from '@playwright/test';
- test('pass-a', async () => {});
- `,
- 'malformed.json': `{ "status": "failed" }`,
- };
-
- const result1 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed=missing.json'] });
- expect(result1.exitCode).toBe(1);
- expect(result1.passed).toBe(0);
- expect(result1.output).toContain('missing.json: Cannot read last run file: ENOENT');
-
- const result2 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed=malformed.json'] });
- expect(result2.exitCode).toBe(1);
- expect(result2.passed).toBe(0);
- expect(result2.output).toContain('malformed.json: Cannot read last run file: "failedTests" list is missing');
-});
-
-test('should resolve --last-failed and --last-run-output-file against cwd', async ({ runInlineTest }, testInfo) => {
- const workspace = {
- 'dir/playwright.config.ts': `module.exports = {};`,
- 'dir/a.spec.js': `
- import { test, expect } from '@playwright/test';
- test('pass-a', async () => {});
- test('fail-a', async () => {
- expect(1).toBe(2);
- });
- `,
- };
-
- const result1 = await runInlineTest(workspace, { config: 'dir/playwright.config.ts' }, {}, { additionalArgs: ['--last-run-output-file=last-run.json'] });
- expect(result1.exitCode).toBe(1);
- expect(result1.passed).toBe(1);
- expect(result1.failed).toBe(1);
- expect(fs.existsSync(testInfo.outputPath('last-run.json'))).toBe(true);
- expect(fs.existsSync(testInfo.outputPath('dir', 'last-run.json'))).toBe(false);
-
- const result2 = await runInlineTest(workspace, { config: 'dir/playwright.config.ts' }, {}, { additionalArgs: ['--last-failed=last-run.json'] });
- expect(result2.exitCode).toBe(1);
- expect(result2.passed).toBe(0);
- expect(result2.failed).toBe(1);
-});