Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat: --last-run-file (#37209)

Dmitry Gozman4fdeec7b7c46
diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md
index 0688e328d7..a47f1a60da 100644
--- a/docs/src/test-cli-js.md
+++ b/docs/src/test-cli-js.md
@@ -91,6 +91,7 @@ npx playwright test --ui
| `--ignore-snapshots` | Ignore screenshot and snapshot expectations. |
| `-j <workers>` or `--workers <workers>` | Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 50%). |
| `--last-failed` | Only re-run the failures. |
+| `--last-run-file` | Path to the last-run file (default: "test-results/.last-run.json"). This json file is used for the `--last-failed` option. You can also apply a custom filter by providing an array of [`property: TestCase.id`] in the `filterTests` property. |
| `--list` | Collect all the tests and report them, but do not run. |
| `--max-failures <N>` or `-x` | Stop after the first `N` failures. Passing `-x` stops after the first failure. |
| `--no-deps` | Do not run project dependencies. |
diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts
index 660aa77add..98d079abab 100644
--- a/packages/playwright/src/common/config.ts
+++ b/packages/playwright/src/common/config.ts
@@ -57,6 +57,7 @@ export class FullConfigInternal {
cliListOnly = false;
cliPassWithNoTests?: boolean;
cliLastFailed?: boolean;
+ cliLastRunFile?: string;
preOnlyTestFilters: TestCaseFilter[] = [];
postShardTestFilters: TestCaseFilter[] = [];
defineConfigWasUsed = false;
diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts
index 5bef1dd87b..b23c389ca2 100644
--- a/packages/playwright/src/program.ts
+++ b/packages/playwright/src/program.ts
@@ -176,6 +176,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
config.cliProjectFilter = opts.project || undefined;
config.cliPassWithNoTests = !!opts.passWithNoTests;
config.cliLastFailed = !!opts.lastFailed;
+ config.cliLastRunFile = opts.lastRunFile ? path.resolve(process.cwd(), opts.lastRunFile) : undefined;
// Evaluate project filters against config before starting execution. This enables a consistent error message across run modes
filterProjects(config.projects, config.cliProjectFilter);
@@ -361,6 +362,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?:
['--headed', { description: `Run tests in headed browsers (default: headless)` }],
['--ignore-snapshots', { description: `Ignore screenshot and snapshot expectations` }],
['--last-failed', { description: `Only re-run the failures` }],
+ ['--last-run-file <file>', { description: `Path to the last-run file (default: "test-results/.last-run.json")` }],
['--list', { description: `Collect all the tests and report them, but do not run` }],
['--max-failures <N>', { description: `Stop after the first N failures` }],
['--no-deps', { description: `Do not run project dependencies` }],
diff --git a/packages/playwright/src/runner/lastRun.ts b/packages/playwright/src/runner/lastRun.ts
index 93ffe9205f..d6721fd54e 100644
--- a/packages/playwright/src/runner/lastRun.ts
+++ b/packages/playwright/src/runner/lastRun.ts
@@ -24,8 +24,9 @@ import type { FullConfigInternal } from '../common/config';
import type { ReporterV2 } from '../reporters/reporterV2';
type LastRunInfo = {
- status: FullResult['status'];
- failedTests: string[];
+ status?: FullResult['status'];
+ failedTests?: string[];
+ filterTests?: string[];
};
export class LastRunReporter implements ReporterV2 {
@@ -35,19 +36,28 @@ export class LastRunReporter implements ReporterV2 {
constructor(config: FullConfigInternal) {
this._config = config;
- const [project] = filterProjects(config.projects, config.cliProjectFilter);
- if (project)
- this._lastRunFile = path.join(project.project.outputDir, '.last-run.json');
+ if (config.cliLastRunFile) {
+ this._lastRunFile = config.cliLastRunFile;
+ } else {
+ const [project] = filterProjects(config.projects, config.cliProjectFilter);
+ if (project)
+ this._lastRunFile = path.join(project.project.outputDir, '.last-run.json');
+ }
}
- async filterLastFailed() {
+ async applyFilter() {
if (!this._lastRunFile)
return;
try {
const lastRunInfo = JSON.parse(await fs.promises.readFile(this._lastRunFile, 'utf8')) as LastRunInfo;
- const failedTestIds = new Set(lastRunInfo.failedTests);
- // Explicitly apply --last-failed filter after sharding.
- this._config.postShardTestFilters.push(test => failedTestIds.has(test.id));
+ if (lastRunInfo.filterTests) {
+ const filterTestIds = new Set(lastRunInfo.filterTests);
+ this._config.preOnlyTestFilters.push(test => filterTestIds.has(test.id));
+ }
+ if (this._config.cliLastFailed) {
+ const failedTestIds = new Set(lastRunInfo.failedTests ?? []);
+ this._config.postShardTestFilters.push(test => failedTestIds.has(test.id));
+ }
} catch {
}
}
diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts
index faf2e50d7f..b2612e7aa0 100644
--- a/packages/playwright/src/runner/testRunner.ts
+++ b/packages/playwright/src/runner/testRunner.ts
@@ -493,8 +493,7 @@ export async function runAllTestsWithConfig(config: FullConfigInternal): Promise
const reporters = await createReporters(config, listOnly ? 'list' : 'test', false);
const lastRun = new LastRunReporter(config);
- if (config.cliLastFailed)
- await lastRun.filterLastFailed();
+ await lastRun.applyFilter();
const reporter = new InternalReporter([...reporters, lastRun]);
const tasks = listOnly ? [
diff --git a/tests/playwright-test/runner.spec.ts b/tests/playwright-test/runner.spec.ts
index dece006a06..fb57fe16c4 100644
--- a/tests/playwright-test/runner.spec.ts
+++ b/tests/playwright-test/runner.spec.ts
@@ -842,7 +842,7 @@ test('should run last failed tests', async ({ runInlineTest }) => {
expect(result2.failed).toBe(1);
});
-test('should run last failed tests in a shard', async ({ runInlineTest }) => {
+test('should run last failed tests in a shard with a custom path', async ({ runInlineTest }) => {
const workspace = {
'a.spec.js': `
import { test, expect } from '@playwright/test';
@@ -859,17 +859,62 @@ test('should run last failed tests in a shard', async ({ runInlineTest }) => {
});
`,
};
- const result1 = await runInlineTest(workspace, { shard: '2/2' });
+ const result1 = await runInlineTest(workspace, { 'shard': '2/2', 'last-run-file': 'custom.json' });
expect(result1.exitCode).toBe(1);
expect(result1.passed).toBe(1);
expect(result1.failed).toBe(1);
expect(result1.output).toContain('b.spec.js:3:11 › pass-b');
expect(result1.output).toContain('b.spec.js:4:11 › fail-b');
+ expect(fs.existsSync(test.info().outputPath('custom.json'))).toBe(true);
- const result2 = await runInlineTest(workspace, { shard: '2/2' }, {}, { additionalArgs: ['--last-failed'] });
+ const result2 = await runInlineTest(workspace, { 'shard': '2/2', 'last-failed': true, 'last-run-file': 'custom.json' });
expect(result2.exitCode).toBe(1);
expect(result2.passed).toBe(0);
expect(result2.failed).toBe(1);
expect(result2.output).not.toContain('b.spec.js:3:11 › pass-b');
expect(result2.output).toContain('b.spec.js:4:11 › fail-b');
});
+
+test('should apply filterTests from last-run.json', async ({ runInlineTest }) => {
+ const workspace = {
+ 'reporter.ts': `
+ import fs from 'fs';
+ import path from 'path';
+ export default class MyReporter {
+ onBegin(config, suite) {
+ const filterTests = suite.allTests().filter(t => t.title !== 'test2').map(t => t.id);
+ const json = JSON.stringify({ filterTests });
+ fs.writeFileSync(path.join(__dirname, 'filter.json'), json);
+ }
+ }
+ `,
+ 'a.spec.js': `
+ import { test, expect } from '@playwright/test';
+ test('test1', async () => {
+ console.log('\\n%%test1');
+ });
+ test('test2', async () => {
+ console.log('\\n%%test2');
+ });
+ test('test3', async () => {
+ console.log('\\n%%test3');
+ expect(1).toBe(2);
+ });
+ `
+ };
+ const result1 = await runInlineTest(workspace, { reporter: './reporter.ts', list: true });
+ expect(result1.exitCode).toBe(0);
+ expect(fs.existsSync(test.info().outputPath('filter.json'))).toBe(true);
+
+ const result2 = await runInlineTest(workspace, { 'last-run-file': 'filter.json' });
+ expect(result2.exitCode).toBe(1);
+ expect(result2.passed).toBe(1);
+ expect(result2.failed).toBe(1);
+ expect(result2.outputLines).toEqual(['test1', 'test3']);
+
+ const result3 = await runInlineTest(workspace, { 'last-run-file': 'filter.json', 'last-failed': true });
+ expect(result3.exitCode).toBe(1);
+ expect(result3.passed).toBe(0);
+ expect(result3.failed).toBe(1);
+ expect(result3.outputLines).toEqual(['test3']);
+});