Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat: retain-on-failure-and-retries trace option (#39387)
Yuvalmoc0554cbe6a87
diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.mdindex 4c60a5c896..1785e86ee5 100644--- a/docs/src/test-api/class-testoptions.md+++ b/docs/src/test-api/class-testoptions.md@@ -588,8 +588,8 @@ export default defineConfig({## property: TestOptions.trace* since: v1.10-- type: <[Object]|[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"retain-on-first-failure">>- - `mode` <[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries"|"retain-on-first-failure">> Trace recording mode.+- type: <[Object]|[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"retain-on-first-failure"|"retain-on-failure-and-retries">>+ - `mode` <[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries"|"retain-on-first-failure"|"retain-on-failure-and-retries">> Trace recording mode.- `attachments` ?<[boolean]> Whether to include test attachments. Defaults to true. Optional.- `screenshots` ?<[boolean]> Whether to capture screenshots during tracing. Screenshots are used to build a timeline preview. Defaults to true. Optional.- `snapshots` ?<[boolean]> Whether to capture DOM snapshot on every action. Defaults to true. Optional.@@ -602,6 +602,7 @@ Whether to record trace for each test. Defaults to `'off'`.* `'on-all-retries'`: Record trace only when retrying a test.* `'retain-on-failure'`: Record trace for each test. When test run passes, remove the recorded trace.* `'retain-on-first-failure'`: Record trace for the first run of each test, but not for retries. When test run passes, remove the recorded trace.+* `'retain-on-failure-and-retries'`: Record trace for each test run. Retains all traces when an attempt fails.For more control, pass an object that specifies `mode` and trace features to enable.diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.mdindex 15c6517e18..abf499a3b6 100644--- a/docs/src/test-cli-js.md+++ b/docs/src/test-cli-js.md@@ -106,7 +106,7 @@ npx playwright test --ui| `--test-list <file>` | Path to a file containing a list of tests to run. See [test list](#test-list) for details. || `--test-list-invert <file>` | Path to a file containing a list of tests to skip. See [test list](#test-list) for details. || `--timeout <timeout>` | Specify test timeout threshold in milliseconds, zero for unlimited (default: 30 seconds). |-| `--trace <mode>` | Force tracing mode, can be `on`, `off`, `on-first-retry`, `on-all-retries`, `retain-on-failure`, `retain-on-first-failure`. |+| `--trace <mode>` | Force tracing mode, can be `on`, `off`, `on-first-retry`, `on-all-retries`, `retain-on-failure`, `retain-on-first-failure`, `retain-on-failure-and-retries`. || `--tsconfig <path>` | Path to a single tsconfig applicable to all imported files (default: look up tsconfig for each imported file separately). || `--ui` | Run tests in interactive UI mode. || `--ui-host <host>` | Host to serve UI on; specifying this option opens UI in a browser tab. |diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.tsindex 6d44f5c48d..e5af5baa4c 100644--- a/packages/playwright/src/program.ts+++ b/packages/playwright/src/program.ts@@ -395,7 +395,7 @@ function resolveReporter(id: string) {return require.resolve(id, { paths: [process.cwd()] });}-const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries', 'retain-on-failure', 'retain-on-first-failure'];+const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries', 'retain-on-failure', 'retain-on-first-failure', 'retain-on-failure-and-retries'];// Note: update docs/src/test-cli-js.md when you update this, program is the source of truth.diff --git a/packages/playwright/src/worker/testTracing.ts b/packages/playwright/src/worker/testTracing.tsindex 2935415f7f..fa99c112a9 100644--- a/packages/playwright/src/worker/testTracing.ts+++ b/packages/playwright/src/worker/testTracing.ts@@ -83,6 +83,9 @@ export class TestTracing {if (this._options?.mode === 'retain-on-first-failure' && this._testInfo.retry === 0)return true;+ if (this._options?.mode === 'retain-on-failure-and-retries')+ return true;+return false;}@@ -159,6 +162,8 @@ export class TestTracing {if (!this._options)return true;const testFailed = this._testInfo.status !== this._testInfo.expectedStatus;+ if (this._options.mode === 'retain-on-failure-and-retries')+ return !testFailed && this._testInfo.retry === 0;return !testFailed && (this._options.mode === 'retain-on-failure' || this._options.mode === 'retain-on-first-failure');}diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.tsindex 7660c2931c..0553609ebd 100644--- a/packages/playwright/types/test.d.ts+++ b/packages/playwright/types/test.d.ts@@ -6903,6 +6903,7 @@ export interface PlaywrightWorkerOptions {* - `'retain-on-failure'`: Record trace for each test. When test run passes, remove the recorded trace.* - `'retain-on-first-failure'`: Record trace for the first run of each test, but not for retries. When test run* passes, remove the recorded trace.+ * - `'retain-on-failure-and-retries'`: Record trace for each test run. Retains all traces when an attempt fails.** For more control, pass an object that specifies `mode` and trace features to enable.*@@ -6953,7 +6954,7 @@ export interface PlaywrightWorkerOptions {}export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure';-export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure';+export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries';export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';export type AgentOptions = {provider?: {diff --git a/tests/playwright-test/playwright.artifacts.spec.ts b/tests/playwright-test/playwright.artifacts.spec.tsindex 5f19139e46..cbdff1d396 100644--- a/tests/playwright-test/playwright.artifacts.spec.ts+++ b/tests/playwright-test/playwright.artifacts.spec.ts@@ -426,6 +426,52 @@ test('should work with trace: retain-on-first-failure', async ({ runInlineTest }]);});+test('should work with trace: retain-on-failure-and-retries', async ({ runInlineTest }, testInfo) => {+ const result = await runInlineTest({+ ...testFiles,+ 'playwright.config.ts': `+ module.exports = { use: { trace: 'retain-on-failure-and-retries' } };+ `,+ }, { workers: 1, retries: 2 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });++ expect(result.exitCode).toBe(1);+ expect(result.passed).toBe(5);+ expect(result.failed).toBe(5);+ expect(listFiles(testInfo.outputPath('test-results'))).toEqual([+ '.last-run.json',+ 'artifacts-failing',+ ' trace.zip',+ 'artifacts-failing-retry1',+ ' trace.zip',+ 'artifacts-failing-retry2',+ ' trace.zip',+ 'artifacts-own-context-failing',+ ' trace.zip',+ 'artifacts-own-context-failing-retry1',+ ' trace.zip',+ 'artifacts-own-context-failing-retry2',+ ' trace.zip',+ 'artifacts-persistent-failing',+ ' trace.zip',+ 'artifacts-persistent-failing-retry1',+ ' trace.zip',+ 'artifacts-persistent-failing-retry2',+ ' trace.zip',+ 'artifacts-shared-shared-failing',+ ' trace.zip',+ 'artifacts-shared-shared-failing-retry1',+ ' trace.zip',+ 'artifacts-shared-shared-failing-retry2',+ ' trace.zip',+ 'artifacts-two-contexts-failing',+ ' trace.zip',+ 'artifacts-two-contexts-failing-retry1',+ ' trace.zip',+ 'artifacts-two-contexts-failing-retry2',+ ' trace.zip',+ ]);+});+test('should take screenshot when page is closed in afterEach', async ({ runInlineTest }, testInfo) => {const result = await runInlineTest({'playwright.config.ts': `diff --git a/tests/playwright-test/playwright.trace.spec.ts b/tests/playwright-test/playwright.trace.spec.tsindex 17a0bc609f..a39a0e8d87 100644--- a/tests/playwright-test/playwright.trace.spec.ts+++ b/tests/playwright-test/playwright.trace.spec.ts@@ -385,7 +385,7 @@ test('should respect --trace', async ({ runInlineTest }, testInfo) => {expect(fs.existsSync(testInfo.outputPath('test-results', 'a-test-1', 'trace.zip'))).toBeTruthy();});-for (const mode of ['off', 'retain-on-failure', 'on-first-retry', 'on-all-retries', 'retain-on-first-failure']) {+for (const mode of ['off', 'retain-on-failure', 'on-first-retry', 'on-all-retries', 'retain-on-first-failure', 'retain-on-failure-and-retries']) {test(`trace:${mode} should not create trace zip artifact if page test passed`, async ({ runInlineTest }) => {const result = await runInlineTest({'a.spec.ts': `@@ -1152,6 +1152,46 @@ test('trace:retain-on-first-failure should create trace if request context is diexpect(result.failed).toBe(1);});+test('trace:retain-on-failure-and-retries should keep all traces when test is flaky', async ({ runInlineTest }, testInfo) => {+ const result = await runInlineTest({+ 'a.spec.ts': `+ import { test, expect } from '@playwright/test';+ test('flaky', async ({ page }) => {+ await page.goto('about:blank');+ expect(test.info().retry).toBe(1);+ });+ `,+ }, { trace: 'retain-on-failure-and-retries', retries: 1 });++ expect(result.exitCode).toBe(0);+ expect(result.flaky).toBe(1);++ const firstRunTracePath = testInfo.outputPath('test-results', 'a-flaky', 'trace.zip');+ expect(fs.existsSync(firstRunTracePath)).toBeTruthy();+ const retryTracePath = testInfo.outputPath('test-results', 'a-flaky-retry1', 'trace.zip');+ expect(fs.existsSync(retryTracePath)).toBeTruthy();+});++test('trace:retain-on-failure-and-retries should keep all traces when test fails on retries', async ({ runInlineTest }, testInfo) => {+ const result = await runInlineTest({+ 'a.spec.ts': `+ import { test, expect } from '@playwright/test';+ test('fail', async ({ page }) => {+ await page.goto('about:blank');+ expect(true).toBe(false);+ });+ `,+ }, { trace: 'retain-on-failure-and-retries', retries: 1 });++ expect(result.exitCode).toBe(1);+ expect(result.failed).toBe(1);++ const firstRunTracePath = testInfo.outputPath('test-results', 'a-fail', 'trace.zip');+ expect(fs.existsSync(firstRunTracePath)).toBeTruthy();+ const retryTracePath = testInfo.outputPath('test-results', 'a-fail-retry1', 'trace.zip');+ expect(fs.existsSync(retryTracePath)).toBeTruthy();+});+test('should not corrupt actions when no library trace is present', async ({ runInlineTest }) => {const result = await runInlineTest({'a.spec.ts': `diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.tsindex 4d0f95de9f..e2529457cc 100644--- a/utils/generate_types/overrides-test.d.ts+++ b/utils/generate_types/overrides-test.d.ts@@ -263,7 +263,7 @@ export interface PlaywrightWorkerOptions {}export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure';-export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure';+export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries';export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';export type AgentOptions = {provider?: {