Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(test): support more video recording modes (#41085)
Yury Semikhatsky2ff4137d0f52
diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.mdindex 62425a56c0..fdf956d6a6 100644--- a/docs/src/test-api/class-testoptions.md+++ b/docs/src/test-api/class-testoptions.md@@ -634,8 +634,8 @@ export default defineConfig({## property: TestOptions.video* since: v1.10-- type: <[Object]|[VideoMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">>- - `mode` <[VideoMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">> Video recording mode.+- type: <[Object]|[VideoMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries"|"retain-on-first-failure"|"retain-on-failure-and-retries"|"retain-all-failures">>+ - `mode` <[VideoMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries"|"retain-on-first-failure"|"retain-on-failure-and-retries"|"retain-all-failures">> Video recording mode.- `size` ?<[Object]> Size of the recorded video. Optional.- `width` <[int]>- `height` <[int]>@@ -652,8 +652,12 @@ export default defineConfig({Whether to record video for each test. Defaults to `'off'`.* `'off'`: Do not record video.* `'on'`: Record video for each test.-* `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs.* `'on-first-retry'`: Record video only when retrying a test for the first time.+* `'on-all-retries'`: Record video only when retrying a test.+* `'retain-on-failure'`: Record video for each test. When test run passes, remove the recorded video.+* `'retain-on-first-failure'`: Record video for the first run of each test, but not for retries. When test run passes, remove the recorded video.+* `'retain-on-failure-and-retries'`: Record video for each test run. Retains all videos when an attempt fails.+* `'retain-all-failures'`: Record video for each test run. Retains the video only for attempts that failed, regardless of the final test outcome.To control video size, pass an object with `mode` and `size` properties. If video size is not specified, it will be equal to [`property: TestOptions.viewport`] scaled down to fit into 800x800. If `viewport` is not configured explicitly the video size defaults to 800x450. Actual picture of each page will be scaled down if necessary to fit the specified size.diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.tsindex b820f11690..0c393abf8e 100644--- a/packages/playwright/src/index.ts+++ b/packages/playwright/src/index.ts@@ -436,8 +436,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures, UtilityTestFixtclosed = true;const closeReason = testInfo.status === 'timedOut' ? 'Test timeout of ' + testInfo.timeout + 'ms exceeded.' : 'Test ended.';await context.close({ reason: closeReason });- const testFailed = testInfo.status !== testInfo.expectedStatus;- const preserveVideo = captureVideo && (videoMode === 'on' || (testFailed && videoMode === 'retain-on-failure') || (videoMode === 'on-first-retry' && testInfo.retry === 1));+ const preserveVideo = captureVideo && shouldPreserveVideo(videoMode, testInfo);if (preserveVideo) {const { pagesWithVideo: pagesForVideo } = contexts.get(context)!;const videos = pagesForVideo.map(p => p.video()).filter(video => !!video);@@ -521,7 +520,31 @@ function normalizeVideoMode(video: VideoMode | 'retry-with-video' | { mode: Vide}function shouldCaptureVideo(videoMode: VideoMode, testInfo: TestInfo) {- return (videoMode === 'on' || videoMode === 'retain-on-failure' || (videoMode === 'on-first-retry' && testInfo.retry === 1));+ return videoMode === 'on'+ || videoMode === 'retain-on-failure'+ || videoMode === 'retain-on-failure-and-retries'+ || videoMode === 'retain-all-failures'+ || (videoMode === 'on-first-retry' && testInfo.retry === 1)+ || (videoMode === 'on-all-retries' && testInfo.retry > 0)+ || (videoMode === 'retain-on-first-failure' && testInfo.retry === 0);+}++function shouldPreserveVideo(videoMode: VideoMode, testInfo: TestInfo) {+ const testFailed = testInfo.status !== testInfo.expectedStatus;+ switch (videoMode) {+ case 'on':+ case 'on-first-retry':+ case 'on-all-retries':+ return true;+ case 'retain-on-failure':+ case 'retain-on-first-failure':+ case 'retain-all-failures':+ return testFailed;+ case 'retain-on-failure-and-retries':+ return testFailed || testInfo.retry > 0;+ default:+ return false;+ }}function normalizeScreenshotMode(screenshot: ScreenshotOption): ScreenshotMode {diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.tsindex f916e685b4..b42fc1477a 100644--- a/packages/playwright/types/test.d.ts+++ b/packages/playwright/types/test.d.ts@@ -7024,8 +7024,14 @@ export interface PlaywrightWorkerOptions {* Whether to record video for each test. Defaults to `'off'`.* - `'off'`: Do not record video.* - `'on'`: Record video for each test.- * - `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs.* - `'on-first-retry'`: Record video only when retrying a test for the first time.+ * - `'on-all-retries'`: Record video only when retrying a test.+ * - `'retain-on-failure'`: Record video for each test. When test run passes, remove the recorded video.+ * - `'retain-on-first-failure'`: Record video for the first run of each test, but not for retries. When test run+ * passes, remove the recorded video.+ * - `'retain-on-failure-and-retries'`: Record video for each test run. Retains all videos when an attempt fails.+ * - `'retain-all-failures'`: Record video for each test run. Retains the video only for attempts that failed,+ * regardless of the final test outcome.** To control video size, pass an object with `mode` and `size` properties. If video size is not specified, it will be* equal to [testOptions.viewport](https://playwright.dev/docs/api/class-testoptions#test-options-viewport) scaled@@ -7056,7 +7062,7 @@ export interface PlaywrightWorkerOptions {export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure';export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries' | 'retain-all-failures';-export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';+export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries' | 'retain-all-failures';/*** Playwright Test provides many options to configure test environment,* [Browser](https://playwright.dev/docs/api/class-browser),diff --git a/tests/playwright-test/playwright.spec.ts b/tests/playwright-test/playwright.spec.tsindex 89339d93f4..fa3b4e37e5 100644--- a/tests/playwright-test/playwright.spec.ts+++ b/tests/playwright-test/playwright.spec.ts@@ -525,6 +525,126 @@ test('should work with video: on-first-retry', async ({ runInlineTest }) => {}, errorPrompt]);});+test('should work with video: on-all-retries', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'playwright.config.ts': `+ module.exports = { use: { video: 'on-all-retries' }, retries: 2, name: 'chromium' };+ `,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('fail', async ({ page }) => {+ await page.setContent('<div>FAIL</div>');+ await page.waitForTimeout(1000);+ test.expect(1 + 1).toBe(1);+ });+ `,+ }, { workers: 1 });++ expect(result.exitCode).toBe(1);+ expect(result.failed).toBe(1);++ const dirFail = test.info().outputPath('test-results', 'a-fail-chromium');+ expect(fs.readdirSync(dirFail).find(file => file.endsWith('webm'))).toBeFalsy();++ const dirRetry1 = test.info().outputPath('test-results', 'a-fail-chromium-retry1');+ expect(fs.readdirSync(dirRetry1).find(file => file.endsWith('webm'))).toBeTruthy();++ const dirRetry2 = test.info().outputPath('test-results', 'a-fail-chromium-retry2');+ expect(fs.readdirSync(dirRetry2).find(file => file.endsWith('webm'))).toBeTruthy();+});++test('should work with video: retain-on-first-failure', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'playwright.config.ts': `+ module.exports = { use: { video: 'retain-on-first-failure' }, retries: 1, name: 'chromium' };+ `,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('pass', async ({ page }) => {+ await page.setContent('<div>PASS</div>');+ await page.waitForTimeout(1000);+ test.expect(1 + 1).toBe(2);+ });+ test('fail', async ({ page }) => {+ await page.setContent('<div>FAIL</div>');+ await page.waitForTimeout(1000);+ test.expect(1 + 1).toBe(1);+ });+ `,+ }, { workers: 1 });++ expect(result.exitCode).toBe(1);+ expect(result.passed).toBe(1);+ expect(result.failed).toBe(1);++ const dirPass = test.info().outputPath('test-results', 'a-pass-chromium');+ const videoPass = fs.existsSync(dirPass) ? fs.readdirSync(dirPass).find(file => file.endsWith('webm')) : undefined;+ expect(videoPass).toBeFalsy();++ // First run failed, so the video is retained.+ const dirFail = test.info().outputPath('test-results', 'a-fail-chromium');+ expect(fs.readdirSync(dirFail).find(file => file.endsWith('webm'))).toBeTruthy();++ // No video is captured on retries.+ const dirRetry = test.info().outputPath('test-results', 'a-fail-chromium-retry1');+ expect(fs.readdirSync(dirRetry).find(file => file.endsWith('webm'))).toBeFalsy();+});++test('should work with video: retain-on-failure-and-retries', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'playwright.config.ts': `+ module.exports = { use: { video: 'retain-on-failure-and-retries' }, retries: 1, name: 'chromium' };+ `,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('flaky', async ({ page }) => {+ await page.setContent('<div>FLAKY</div>');+ await page.waitForTimeout(1000);+ test.expect(test.info().retry).toBe(1);+ });+ `,+ }, { workers: 1 });++ expect(result.exitCode).toBe(0);+ expect(result.flaky).toBe(1);++ // First attempt failed, video retained.+ const dirFail = test.info().outputPath('test-results', 'a-flaky-chromium');+ expect(fs.readdirSync(dirFail).find(file => file.endsWith('webm'))).toBeTruthy();++ // Retry passed, but all videos are retained once the test was retried.+ const dirRetry = test.info().outputPath('test-results', 'a-flaky-chromium-retry1');+ expect(fs.readdirSync(dirRetry).find(file => file.endsWith('webm'))).toBeTruthy();+});++test('should work with video: retain-all-failures', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'playwright.config.ts': `+ module.exports = { use: { video: 'retain-all-failures' }, retries: 1, name: 'chromium' };+ `,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('flaky', async ({ page }) => {+ await page.setContent('<div>FLAKY</div>');+ await page.waitForTimeout(1000);+ test.expect(test.info().retry).toBe(1);+ });+ `,+ }, { workers: 1 });++ expect(result.exitCode).toBe(0);+ expect(result.flaky).toBe(1);++ // First attempt failed, video retained.+ const dirFail = test.info().outputPath('test-results', 'a-flaky-chromium');+ expect(fs.readdirSync(dirFail).find(file => file.endsWith('webm'))).toBeTruthy();++ // Retry passed, so its video is removed.+ const dirRetry = test.info().outputPath('test-results', 'a-flaky-chromium-retry1');+ const videoRetry = fs.existsSync(dirRetry) ? fs.readdirSync(dirRetry).find(file => file.endsWith('webm')) : undefined;+ expect(videoRetry).toBeFalsy();+});+test('should work with video size', async ({ runInlineTest }) => {const result = await runInlineTest({'playwright.config.js': `diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.tsindex 09c36aa081..0a6c79c187 100644--- a/utils/generate_types/overrides-test.d.ts+++ b/utils/generate_types/overrides-test.d.ts@@ -268,7 +268,7 @@ export interface PlaywrightWorkerOptions {export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure';export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries' | 'retain-all-failures';-export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';+export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries' | 'retain-all-failures';export interface PlaywrightTestOptions {acceptDownloads: boolean;bypassCSP: boolean;