Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(test): support more video recording modes (#41085)

Yury Semikhatsky2ff4137d0f52
diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md
index 62425a56c0..fdf956d6a6 100644
--- a/docs/src/test-api/class-testoptions.md
+++ b/docs/src/test-api/class-testoptions.md
@@ -634,8 +634,8 @@ export default defineConfig({
## property: TestOptions.video
* since: v1.10
-- type: <[Object]|[VideoMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">>
- - `mode` <[VideoMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">> Video recording mode.
+- type: <[Object]|[VideoMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries"|"retain-on-first-failure"|"retain-on-failure-and-retries"|"retain-all-failures">>
+ - `mode` <[VideoMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries"|"retain-on-first-failure"|"retain-on-failure-and-retries"|"retain-all-failures">> Video recording mode.
- `size` ?<[Object]> Size of the recorded video. Optional.
- `width` <[int]>
- `height` <[int]>
@@ -652,8 +652,12 @@ export default defineConfig({
Whether to record video for each test. Defaults to `'off'`.
* `'off'`: Do not record video.
* `'on'`: Record video for each test.
-* `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs.
* `'on-first-retry'`: Record video only when retrying a test for the first time.
+* `'on-all-retries'`: Record video only when retrying a test.
+* `'retain-on-failure'`: Record video for each test. When test run passes, remove the recorded video.
+* `'retain-on-first-failure'`: Record video for the first run of each test, but not for retries. When test run passes, remove the recorded video.
+* `'retain-on-failure-and-retries'`: Record video for each test run. Retains all videos when an attempt fails.
+* `'retain-all-failures'`: Record video for each test run. Retains the video only for attempts that failed, regardless of the final test outcome.
To control video size, pass an object with `mode` and `size` properties. If video size is not specified, it will be equal to [`property: TestOptions.viewport`] scaled down to fit into 800x800. If `viewport` is not configured explicitly the video size defaults to 800x450. Actual picture of each page will be scaled down if necessary to fit the specified size.
diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts
index b820f11690..0c393abf8e 100644
--- a/packages/playwright/src/index.ts
+++ b/packages/playwright/src/index.ts
@@ -436,8 +436,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures, UtilityTestFixt
closed = true;
const closeReason = testInfo.status === 'timedOut' ? 'Test timeout of ' + testInfo.timeout + 'ms exceeded.' : 'Test ended.';
await context.close({ reason: closeReason });
- const testFailed = testInfo.status !== testInfo.expectedStatus;
- const preserveVideo = captureVideo && (videoMode === 'on' || (testFailed && videoMode === 'retain-on-failure') || (videoMode === 'on-first-retry' && testInfo.retry === 1));
+ const preserveVideo = captureVideo && shouldPreserveVideo(videoMode, testInfo);
if (preserveVideo) {
const { pagesWithVideo: pagesForVideo } = contexts.get(context)!;
const videos = pagesForVideo.map(p => p.video()).filter(video => !!video);
@@ -521,7 +520,31 @@ function normalizeVideoMode(video: VideoMode | 'retry-with-video' | { mode: Vide
}
function shouldCaptureVideo(videoMode: VideoMode, testInfo: TestInfo) {
- return (videoMode === 'on' || videoMode === 'retain-on-failure' || (videoMode === 'on-first-retry' && testInfo.retry === 1));
+ return videoMode === 'on'
+ || videoMode === 'retain-on-failure'
+ || videoMode === 'retain-on-failure-and-retries'
+ || videoMode === 'retain-all-failures'
+ || (videoMode === 'on-first-retry' && testInfo.retry === 1)
+ || (videoMode === 'on-all-retries' && testInfo.retry > 0)
+ || (videoMode === 'retain-on-first-failure' && testInfo.retry === 0);
+}
+
+function shouldPreserveVideo(videoMode: VideoMode, testInfo: TestInfo) {
+ const testFailed = testInfo.status !== testInfo.expectedStatus;
+ switch (videoMode) {
+ case 'on':
+ case 'on-first-retry':
+ case 'on-all-retries':
+ return true;
+ case 'retain-on-failure':
+ case 'retain-on-first-failure':
+ case 'retain-all-failures':
+ return testFailed;
+ case 'retain-on-failure-and-retries':
+ return testFailed || testInfo.retry > 0;
+ default:
+ return false;
+ }
}
function normalizeScreenshotMode(screenshot: ScreenshotOption): ScreenshotMode {
diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts
index f916e685b4..b42fc1477a 100644
--- a/packages/playwright/types/test.d.ts
+++ b/packages/playwright/types/test.d.ts
@@ -7024,8 +7024,14 @@ export interface PlaywrightWorkerOptions {
* Whether to record video for each test. Defaults to `'off'`.
* - `'off'`: Do not record video.
* - `'on'`: Record video for each test.
- * - `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs.
* - `'on-first-retry'`: Record video only when retrying a test for the first time.
+ * - `'on-all-retries'`: Record video only when retrying a test.
+ * - `'retain-on-failure'`: Record video for each test. When test run passes, remove the recorded video.
+ * - `'retain-on-first-failure'`: Record video for the first run of each test, but not for retries. When test run
+ * passes, remove the recorded video.
+ * - `'retain-on-failure-and-retries'`: Record video for each test run. Retains all videos when an attempt fails.
+ * - `'retain-all-failures'`: Record video for each test run. Retains the video only for attempts that failed,
+ * regardless of the final test outcome.
*
* To control video size, pass an object with `mode` and `size` properties. If video size is not specified, it will be
* equal to [testOptions.viewport](https://playwright.dev/docs/api/class-testoptions#test-options-viewport) scaled
@@ -7056,7 +7062,7 @@ export interface PlaywrightWorkerOptions {
export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure';
export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries' | 'retain-all-failures';
-export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';
+export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries' | 'retain-all-failures';
/**
* Playwright Test provides many options to configure test environment,
* [Browser](https://playwright.dev/docs/api/class-browser),
diff --git a/tests/playwright-test/playwright.spec.ts b/tests/playwright-test/playwright.spec.ts
index 89339d93f4..fa3b4e37e5 100644
--- a/tests/playwright-test/playwright.spec.ts
+++ b/tests/playwright-test/playwright.spec.ts
@@ -525,6 +525,126 @@ test('should work with video: on-first-retry', async ({ runInlineTest }) => {
}, errorPrompt]);
});
+test('should work with video: on-all-retries', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ module.exports = { use: { video: 'on-all-retries' }, retries: 2, name: 'chromium' };
+ `,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('fail', async ({ page }) => {
+ await page.setContent('<div>FAIL</div>');
+ await page.waitForTimeout(1000);
+ test.expect(1 + 1).toBe(1);
+ });
+ `,
+ }, { workers: 1 });
+
+ expect(result.exitCode).toBe(1);
+ expect(result.failed).toBe(1);
+
+ const dirFail = test.info().outputPath('test-results', 'a-fail-chromium');
+ expect(fs.readdirSync(dirFail).find(file => file.endsWith('webm'))).toBeFalsy();
+
+ const dirRetry1 = test.info().outputPath('test-results', 'a-fail-chromium-retry1');
+ expect(fs.readdirSync(dirRetry1).find(file => file.endsWith('webm'))).toBeTruthy();
+
+ const dirRetry2 = test.info().outputPath('test-results', 'a-fail-chromium-retry2');
+ expect(fs.readdirSync(dirRetry2).find(file => file.endsWith('webm'))).toBeTruthy();
+});
+
+test('should work with video: retain-on-first-failure', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ module.exports = { use: { video: 'retain-on-first-failure' }, retries: 1, name: 'chromium' };
+ `,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('pass', async ({ page }) => {
+ await page.setContent('<div>PASS</div>');
+ await page.waitForTimeout(1000);
+ test.expect(1 + 1).toBe(2);
+ });
+ test('fail', async ({ page }) => {
+ await page.setContent('<div>FAIL</div>');
+ await page.waitForTimeout(1000);
+ test.expect(1 + 1).toBe(1);
+ });
+ `,
+ }, { workers: 1 });
+
+ expect(result.exitCode).toBe(1);
+ expect(result.passed).toBe(1);
+ expect(result.failed).toBe(1);
+
+ const dirPass = test.info().outputPath('test-results', 'a-pass-chromium');
+ const videoPass = fs.existsSync(dirPass) ? fs.readdirSync(dirPass).find(file => file.endsWith('webm')) : undefined;
+ expect(videoPass).toBeFalsy();
+
+ // First run failed, so the video is retained.
+ const dirFail = test.info().outputPath('test-results', 'a-fail-chromium');
+ expect(fs.readdirSync(dirFail).find(file => file.endsWith('webm'))).toBeTruthy();
+
+ // No video is captured on retries.
+ const dirRetry = test.info().outputPath('test-results', 'a-fail-chromium-retry1');
+ expect(fs.readdirSync(dirRetry).find(file => file.endsWith('webm'))).toBeFalsy();
+});
+
+test('should work with video: retain-on-failure-and-retries', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ module.exports = { use: { video: 'retain-on-failure-and-retries' }, retries: 1, name: 'chromium' };
+ `,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('flaky', async ({ page }) => {
+ await page.setContent('<div>FLAKY</div>');
+ await page.waitForTimeout(1000);
+ test.expect(test.info().retry).toBe(1);
+ });
+ `,
+ }, { workers: 1 });
+
+ expect(result.exitCode).toBe(0);
+ expect(result.flaky).toBe(1);
+
+ // First attempt failed, video retained.
+ const dirFail = test.info().outputPath('test-results', 'a-flaky-chromium');
+ expect(fs.readdirSync(dirFail).find(file => file.endsWith('webm'))).toBeTruthy();
+
+ // Retry passed, but all videos are retained once the test was retried.
+ const dirRetry = test.info().outputPath('test-results', 'a-flaky-chromium-retry1');
+ expect(fs.readdirSync(dirRetry).find(file => file.endsWith('webm'))).toBeTruthy();
+});
+
+test('should work with video: retain-all-failures', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ module.exports = { use: { video: 'retain-all-failures' }, retries: 1, name: 'chromium' };
+ `,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('flaky', async ({ page }) => {
+ await page.setContent('<div>FLAKY</div>');
+ await page.waitForTimeout(1000);
+ test.expect(test.info().retry).toBe(1);
+ });
+ `,
+ }, { workers: 1 });
+
+ expect(result.exitCode).toBe(0);
+ expect(result.flaky).toBe(1);
+
+ // First attempt failed, video retained.
+ const dirFail = test.info().outputPath('test-results', 'a-flaky-chromium');
+ expect(fs.readdirSync(dirFail).find(file => file.endsWith('webm'))).toBeTruthy();
+
+ // Retry passed, so its video is removed.
+ const dirRetry = test.info().outputPath('test-results', 'a-flaky-chromium-retry1');
+ const videoRetry = fs.existsSync(dirRetry) ? fs.readdirSync(dirRetry).find(file => file.endsWith('webm')) : undefined;
+ expect(videoRetry).toBeFalsy();
+});
+
test('should work with video size', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.js': `
diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts
index 09c36aa081..0a6c79c187 100644
--- a/utils/generate_types/overrides-test.d.ts
+++ b/utils/generate_types/overrides-test.d.ts
@@ -268,7 +268,7 @@ export interface PlaywrightWorkerOptions {
export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure';
export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries' | 'retain-all-failures';
-export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';
+export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries' | 'retain-all-failures';
export interface PlaywrightTestOptions {
acceptDownloads: boolean;
bypassCSP: boolean;