Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
Revert "feat(test-runner): read last failed tests from --last-failed=<file>" (#43053)
Pavel Feldman45b43ff3581d
diff --git a/docs/src/running-tests-js.md b/docs/src/running-tests-js.mdindex 1c563a568f..0925ef7db8 100644--- a/docs/src/running-tests-js.md+++ b/docs/src/running-tests-js.md@@ -91,20 +91,12 @@ To run only the tests that failed in the last test run, first run your tests andnpx playwright test --last-failed```-Playwright stores the list of failed tests from the previous run in `<outputDir>/.last-run.json` (see [`property: TestConfig.outputDir`](./test-configuration.md)). To write it to a different file, pass `--last-run-output-file=<path>` or set `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE`.+Playwright stores the list of failed tests from the previous run in `<outputDir>/.last-run.json` (see [`property: TestConfig.outputDir`](./test-configuration.md)). To use a different file path, pass `--last-failed-file=<path>` or set `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE`.```bash-npx playwright test --last-run-output-file=.cache/last-run-shard-1.json+npx playwright test --last-failed --last-failed-file=.cache/last-run-shard-1.json```-To re-run the failures recorded in a specific file, pass it to `--last-failed=<path>`. The run fails if the file does not exist. The file is only read, the results of the re-run are written to the default location or to the `--last-run-output-file` path. Without a path, `--last-failed` always reads `<outputDir>/.last-run.json`.--```bash-npx playwright test --last-failed=.cache/last-run-shard-1.json-```--Relative paths are resolved against the current working directory.-### Run tests in VS CodeTests can be run right from VS Code using the [VS Code extension](https://marketplace.visualstudio.com/items?itemName=ms-playwright.playwright). Once installed you can simply click the green triangle next to the test you want to run or run all tests from the testing sidebar. Check out our [Getting Started with VS Code](./getting-started-vscode.md) guide for more details.diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.mdindex 67c8dbb5d5..fadedf6b74 100644--- a/docs/src/test-cli-js.md+++ b/docs/src/test-cli-js.md@@ -91,8 +91,8 @@ npx playwright test --ui| `--headed` | Run tests in headed browsers (default: headless). || `--ignore-snapshots` | Ignore screenshot and snapshot expectations. || `-j <workers>` or `--workers <workers>` | Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 50%). |-| `--last-failed [file]` | Only re-run the failures. Optionally takes a path to the last run file to read the failures from, use `--last-failed=<file>` form (default: `<outputDir>/.last-run.json`). |-| `--last-run-output-file <file>` | Path to write the last run file to (default: `<outputDir>/.last-run.json`). Same as `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE` environment variable. |+| `--last-failed` | Only re-run the failures. |+| `--last-failed-file <file>` | Override the default last-run JSON path for `--last-failed` (default: `<outputDir>/.last-run.json`). Same as `PLAYWRIGHT_LAST_RUN_OUTPUT_FILE` environment variable. || `--list` | Collect all the tests and report them, but do not run. || `--max-failures <N>` or `-x` | Stop after the first `N` failures. Passing `-x` stops after the first failure. || `--no-deps` | Do not run project dependencies. |diff --git a/packages/playwright/src/cli/testActions.ts b/packages/playwright/src/cli/testActions.tsindex 904d2a76ac..b6fd25facb 100644--- a/packages/playwright/src/cli/testActions.ts+++ b/packages/playwright/src/cli/testActions.ts@@ -38,8 +38,7 @@ export async function runTests(args: string[], opts: { [key: string]: any }) {projectFilter: opts.project || undefined,passWithNoTests: !!opts.passWithNoTests,lastFailed: !!opts.lastFailed,- lastRunInputFile: typeof opts.lastFailed === 'string' ? opts.lastFailed : undefined,- lastRunOutputFile: opts.lastRunOutputFile,+ lastFailedFile: opts.lastFailedFile,testList: opts.testList ? path.resolve(process.cwd(), opts.testList) : undefined,testListInvert: opts.testListInvert ? path.resolve(process.cwd(), opts.testListInvert) : undefined,shardWeights: resolveShardWeightsOption(),diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.tsindex 505c05b668..9042f96522 100644--- a/packages/playwright/src/program.ts+++ b/packages/playwright/src/program.ts@@ -218,8 +218,8 @@ const testOptions: [string, { description: string, choices?: string[], preset?:['-G, --grep-invert <grep>', { description: `Only run tests that do not match this regular expression` }],['--headed', { description: `Run tests in headed browsers (default: headless)` }],['--ignore-snapshots', { description: `Ignore screenshot and snapshot expectations` }],- ['--last-failed [file]', { description: `Only re-run the failures. Optionally takes a path to the last run file to read the failures from, use --last-failed=<file> form (default: <outputDir>/.last-run.json)` }],- ['--last-run-output-file <file>', { description: `Path to write the last run file to (default: <outputDir>/.last-run.json). Same as PLAYWRIGHT_LAST_RUN_OUTPUT_FILE environment variable.` }],+ ['--last-failed', { description: `Only re-run the failures` }],+ ['--last-failed-file <file>', { description: `Override the default path for the last-run JSON file used with --last-failed (default: <outputDir>/.last-run.json). Same as PLAYWRIGHT_LAST_RUN_OUTPUT_FILE environment variable.` }],['--list', { description: `Collect all the tests and report them, but do not run` }],['--max-failures <N>', { description: `Stop after the first N failures` }],['--no-deps', { description: `Do not run project dependencies` }],diff --git a/packages/playwright/src/runner/lastRun.ts b/packages/playwright/src/runner/lastRun.tsindex 97bf5c8e4b..8312da4471 100644--- a/packages/playwright/src/runner/lastRun.ts+++ b/packages/playwright/src/runner/lastRun.ts@@ -17,8 +17,6 @@import fs from 'fs';import path from 'path';-import { errorWithFile } from '../util';-import type { FullResult, Suite, TestCase } from '../../types/testReporter';import type { config as commonConfig } from '../common';import type { ReporterV2 } from '../reporters/reporterV2';@@ -37,31 +35,30 @@ function didNotRun(test: TestCase): boolean {}export class LastRunReporter implements ReporterV2 {- private _inputFile: string | undefined;- private _isDefaultInputFile: boolean;- private _outputFile: string | undefined;+ private _lastRunFile: string | undefined;private _suite: Suite | undefined;private _listMode: boolean;- constructor(filteredProjects: commonConfig.FullProjectInternal[], options: { listMode?: boolean, inputFile?: string, outputFile?: string }) {- this._listMode = !!options.listMode;- const [project] = filteredProjects;- const defaultFile = project ? path.join(project.project.outputDir, '.last-run.json') : undefined;- const outputFile = options.outputFile ?? process.env.PLAYWRIGHT_LAST_RUN_OUTPUT_FILE;- this._outputFile = outputFile ? path.resolve(process.cwd(), outputFile) : defaultFile;- this._inputFile = options.inputFile ? path.resolve(process.cwd(), options.inputFile) : defaultFile;- this._isDefaultInputFile = !options.inputFile;+ constructor(filteredProjects: commonConfig.FullProjectInternal[], listMode?: boolean, lastFailedFileOverride?: string) {+ this._listMode = !!listMode;+ const override = lastFailedFileOverride ?? process.env.PLAYWRIGHT_LAST_RUN_OUTPUT_FILE;+ if (override) {+ this._lastRunFile = path.resolve(process.cwd(), override);+ } else {+ const [project] = filteredProjects;+ if (project)+ this._lastRunFile = path.join(project.project.outputDir, '.last-run.json');+ }}async filterLastFailed(): Promise<string[] | undefined> {- if (!this._inputFile)+ if (!this._lastRunFile)return undefined;try {- return await readFailedTests(this._inputFile);- } catch (e) {- if (this._isDefaultInputFile)- return undefined;- throw e;+ const lastRunInfo = JSON.parse(await fs.promises.readFile(this._lastRunFile, 'utf8')) as LastRunInfo;+ return lastRunInfo.failedTests;+ } catch {+ return undefined;}}@@ -78,25 +75,13 @@ export class LastRunReporter implements ReporterV2 {}async onEnd(result: FullResult) {- if (!this._outputFile || this._listMode)+ if (!this._lastRunFile || this._listMode)return;const lastRunInfo: LastRunInfo = {status: result.status,failedTests: this._suite?.allTests().filter(t => !t.ok() || didNotRun(t)).map(t => t.id) || [],};- await fs.promises.mkdir(path.dirname(this._outputFile), { recursive: true });- await fs.promises.writeFile(this._outputFile, JSON.stringify(lastRunInfo, undefined, 2));- }-}--async function readFailedTests(file: string): Promise<string[]> {- let lastRunInfo: LastRunInfo;- try {- lastRunInfo = JSON.parse(await fs.promises.readFile(file, 'utf8'));- } catch (e) {- throw errorWithFile(file, 'Cannot read last run file: ' + e.message);+ await fs.promises.mkdir(path.dirname(this._lastRunFile), { recursive: true });+ await fs.promises.writeFile(this._lastRunFile, JSON.stringify(lastRunInfo, undefined, 2));}- if (!Array.isArray(lastRunInfo?.failedTests))- throw errorWithFile(file, 'Cannot read last run file: "failedTests" list is missing');- return lastRunInfo.failedTests;}diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.tsindex c9426b0c73..1361860eab 100644--- a/packages/playwright/src/runner/tasks.ts+++ b/packages/playwright/src/runner/tasks.ts@@ -63,8 +63,7 @@ export type TestRunOptions = {listMode?: boolean;passWithNoTests?: boolean;lastFailed?: boolean;- lastRunInputFile?: string;- lastRunOutputFile?: string;+ lastFailedFile?: string;testList?: string;testListInvert?: string;lastFailedTestIds?: string[];diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.tsindex e472b9c4df..df44df98a1 100644--- a/packages/playwright/src/runner/testRunner.ts+++ b/packages/playwright/src/runner/testRunner.ts@@ -429,7 +429,7 @@ export async function runAllTestsWithConfig(config: FullConfigInternal, options:const filteredProjects = filterProjects(config.projects, options.projectFilter);const reporters = await createReporters(config, options.listMode ? 'list' : 'test', undefined, options);- const lastRun = new LastRunReporter(filteredProjects, { listMode: options.listMode, inputFile: options.lastRunInputFile, outputFile: options.lastRunOutputFile });+ const lastRun = new LastRunReporter(filteredProjects, options.listMode, options.lastFailedFile);if (options.lastFailed) {const lastFailedTestIds = await lastRun.filterLastFailed();if (lastFailedTestIds)diff --git a/tests/playwright-test/runner.spec.ts b/tests/playwright-test/runner.spec.tsindex 5914e2094a..812f6cade2 100644--- a/tests/playwright-test/runner.spec.ts+++ b/tests/playwright-test/runner.spec.ts@@ -977,7 +977,7 @@ test('should run last failed tests in a shard with PLAYWRIGHT_LAST_RUN_OUTPUT_FIexpect(result1.output).toContain('b.spec.js:3:11 › pass-b');expect(result1.output).toContain('b.spec.js:4:11 › fail-b');- const result2 = await runInlineTest(workspace, { shard: '2/2' }, env, { additionalArgs: [`--last-failed=${customRel}`] });+ const result2 = await runInlineTest(workspace, { shard: '2/2' }, env, { additionalArgs: ['--last-failed'] });expect(result2.exitCode).toBe(1);expect(result2.passed).toBe(0);expect(result2.failed).toBe(1);@@ -985,11 +985,11 @@ test('should run last failed tests in a shard with PLAYWRIGHT_LAST_RUN_OUTPUT_FIexpect(result2.output).toContain('b.spec.js:4:11 › fail-b');});-test('should run last failed tests in a shard with --last-run-output-file', async ({ runInlineTest }, testInfo) => {+test('should run last failed tests in a shard with --last-failed-file', async ({ runInlineTest }, testInfo) => {const customRel = '.cache/shard-2-cli-last-run.json';const customAbs = path.join(testInfo.outputPath(), customRel);const defaultLastRun = path.join(testInfo.outputPath(), 'test-results', '.last-run.json');- const lastRunArgs = [`--last-run-output-file=${customRel}`];+ const lastRunArgs = ['--last-failed', `--last-failed-file=${customRel}`];const workspace = {'a.spec.js': `import { test, expect } from '@playwright/test';@@ -1015,92 +1015,10 @@ test('should run last failed tests in a shard with --last-run-output-file', asynexpect(result1.output).toContain('b.spec.js:3:11 › pass-b');expect(result1.output).toContain('b.spec.js:4:11 › fail-b');- const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: [`--last-failed=${customRel}`, ...lastRunArgs] });+ const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: lastRunArgs });expect(result2.exitCode).toBe(1);expect(result2.passed).toBe(0);expect(result2.failed).toBe(1);expect(result2.output).not.toContain('b.spec.js:3:11 › pass-b');expect(result2.output).toContain('b.spec.js:4:11 › fail-b');});--test('should read --last-failed=<file> without overwriting it', async ({ runInlineTest }, testInfo) => {- const inputFile = testInfo.outputPath('.cache', 'input.json');- const outputFile = testInfo.outputPath('.cache', 'output.json');- const defaultLastRun = testInfo.outputPath('test-results', '.last-run.json');- const spec = (expected: number) => `- import { test, expect } from '@playwright/test';- test('pass-a', async () => {});- test('fail-a', async () => {- expect(1).toBe(${expected});- });- `;-- const result1 = await runInlineTest({ 'a.spec.js': spec(2) }, {}, {}, { additionalArgs: ['--last-run-output-file=.cache/input.json'] });- expect(result1.exitCode).toBe(1);- expect(result1.passed).toBe(1);- expect(result1.failed).toBe(1);- expect(fs.existsSync(defaultLastRun)).toBe(false);- const input = fs.readFileSync(inputFile, 'utf8');- expect(JSON.parse(input).failedTests).toHaveLength(1);-- const result2 = await runInlineTest({ 'a.spec.js': spec(1) }, {}, {}, { additionalArgs: ['--last-failed=.cache/input.json'] });- expect(result2.exitCode).toBe(0);- expect(result2.passed).toBe(1);- expect(result2.output).not.toContain('pass-a');- expect(result2.output).toContain('fail-a');- expect(fs.readFileSync(inputFile, 'utf8')).toBe(input);- expect(JSON.parse(fs.readFileSync(defaultLastRun, 'utf8'))).toEqual({ status: 'passed', failedTests: [] });-- const result3 = await runInlineTest({ 'a.spec.js': spec(1) }, {}, {}, { additionalArgs: ['--last-failed=.cache/input.json', '--last-run-output-file=.cache/output.json'] });- expect(result3.exitCode).toBe(0);- expect(result3.passed).toBe(1);- expect(result3.output).not.toContain('pass-a');- expect(fs.readFileSync(inputFile, 'utf8')).toBe(input);- expect(JSON.parse(fs.readFileSync(outputFile, 'utf8'))).toEqual({ status: 'passed', failedTests: [] });- expect(fs.existsSync(defaultLastRun)).toBe(false);-});--test('should fail when --last-failed=<file> is missing or malformed', async ({ runInlineTest }) => {- const workspace = {- 'a.spec.js': `- import { test, expect } from '@playwright/test';- test('pass-a', async () => {});- `,- 'malformed.json': `{ "status": "failed" }`,- };-- const result1 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed=missing.json'] });- expect(result1.exitCode).toBe(1);- expect(result1.passed).toBe(0);- expect(result1.output).toContain('missing.json: Cannot read last run file: ENOENT');-- const result2 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed=malformed.json'] });- expect(result2.exitCode).toBe(1);- expect(result2.passed).toBe(0);- expect(result2.output).toContain('malformed.json: Cannot read last run file: "failedTests" list is missing');-});--test('should resolve --last-failed and --last-run-output-file against cwd', async ({ runInlineTest }, testInfo) => {- const workspace = {- 'dir/playwright.config.ts': `module.exports = {};`,- 'dir/a.spec.js': `- import { test, expect } from '@playwright/test';- test('pass-a', async () => {});- test('fail-a', async () => {- expect(1).toBe(2);- });- `,- };-- const result1 = await runInlineTest(workspace, { config: 'dir/playwright.config.ts' }, {}, { additionalArgs: ['--last-run-output-file=last-run.json'] });- expect(result1.exitCode).toBe(1);- expect(result1.passed).toBe(1);- expect(result1.failed).toBe(1);- expect(fs.existsSync(testInfo.outputPath('last-run.json'))).toBe(true);- expect(fs.existsSync(testInfo.outputPath('dir', 'last-run.json'))).toBe(false);-- const result2 = await runInlineTest(workspace, { config: 'dir/playwright.config.ts' }, {}, { additionalArgs: ['--last-failed=last-run.json'] });- expect(result2.exitCode).toBe(1);- expect(result2.passed).toBe(0);- expect(result2.failed).toBe(1);-});