Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix(test-runner): recover beforeAll-skipped tests for --last-failed (#42056)

Piotr Kozłowski0484712ca531
diff --git a/packages/playwright/src/runner/lastRun.ts b/packages/playwright/src/runner/lastRun.ts
index 57a073f541..8312da4471 100644
--- a/packages/playwright/src/runner/lastRun.ts
+++ b/packages/playwright/src/runner/lastRun.ts
@@ -17,7 +17,7 @@
import fs from 'fs';
import path from 'path';
-import type { FullResult, Suite } from '../../types/testReporter';
+import type { FullResult, Suite, TestCase } from '../../types/testReporter';
import type { config as commonConfig } from '../common';
import type { ReporterV2 } from '../reporters/reporterV2';
@@ -26,6 +26,14 @@ type LastRunInfo = {
failedTests: string[];
};
+function didNotRun(test: TestCase): boolean {
+ if (test.outcome() !== 'skipped')
+ return false;
+ if (test.results.some(result => result.status === 'interrupted'))
+ return false;
+ return !test.results.length || test.expectedStatus !== 'skipped';
+}
+
export class LastRunReporter implements ReporterV2 {
private _lastRunFile: string | undefined;
private _suite: Suite | undefined;
@@ -71,7 +79,7 @@ export class LastRunReporter implements ReporterV2 {
return;
const lastRunInfo: LastRunInfo = {
status: result.status,
- failedTests: this._suite?.allTests().filter(t => !t.ok()).map(t => t.id) || [],
+ failedTests: this._suite?.allTests().filter(t => !t.ok() || didNotRun(t)).map(t => t.id) || [],
};
await fs.promises.mkdir(path.dirname(this._lastRunFile), { recursive: true });
await fs.promises.writeFile(this._lastRunFile, JSON.stringify(lastRunInfo, undefined, 2));
diff --git a/tests/playwright-test/runner.spec.ts b/tests/playwright-test/runner.spec.ts
index 4973103484..812f6cade2 100644
--- a/tests/playwright-test/runner.spec.ts
+++ b/tests/playwright-test/runner.spec.ts
@@ -866,6 +866,55 @@ test('should run nothing with --last-failed when previous run had no failures',
expect(result2.didNotRun).toBe(0);
});
+test('should run last failed tests that did not run because a beforeAll hook failed', async ({ runInlineTest }) => {
+ const workspace = {
+ 'a.spec.js': `
+ import fs from 'fs';
+ import { test, expect } from '@playwright/test';
+ test.beforeAll(() => {
+ if (!fs.existsSync('marker.txt')) {
+ fs.writeFileSync('marker.txt', '');
+ throw new Error('from beforeAll');
+ }
+ });
+ test('one', () => {});
+ test('two', () => {});
+ `
+ };
+ const result1 = await runInlineTest(workspace);
+ expect(result1.exitCode).toBe(1);
+ expect(result1.failed).toBe(1);
+ expect(result1.didNotRun).toBe(1);
+
+ const result2 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed'] });
+ expect(result2.exitCode).toBe(0);
+ expect(result2.passed).toBe(2);
+ expect(result2.didNotRun).toBe(0);
+});
+
+test('should not run intentionally skipped tests with --last-failed', async ({ runInlineTest }) => {
+ const workspace = {
+ 'a.spec.js': `
+ import { test, expect } from '@playwright/test';
+ test('fail', () => {
+ expect(1).toBe(2);
+ });
+ test('skipped', () => {
+ test.skip();
+ });
+ `
+ };
+ const result1 = await runInlineTest(workspace);
+ expect(result1.exitCode).toBe(1);
+ expect(result1.failed).toBe(1);
+ expect(result1.skipped).toBe(1);
+
+ const result2 = await runInlineTest(workspace, {}, {}, { additionalArgs: ['--last-failed'] });
+ expect(result2.exitCode).toBe(1);
+ expect(result2.failed).toBe(1);
+ expect(result2.skipped).toBe(0);
+});
+
test('should run last failed tests in a shard', async ({ runInlineTest }) => {
const workspace = {
'a.spec.js': `