Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore(bidi): implement screencasting using `browsingContext.captureScreenshot` (#41569)
Holger Benlfdd0bc913dfe
diff --git a/packages/playwright-core/src/server/bidi/bidiPage.ts b/packages/playwright-core/src/server/bidi/bidiPage.tsindex aec8cc1249..908a16c24e 100644--- a/packages/playwright-core/src/server/bidi/bidiPage.ts+++ b/packages/playwright-core/src/server/bidi/bidiPage.ts@@ -27,6 +27,7 @@ import { BidiNetworkManager } from './bidiNetworkManager';import { BidiPDF } from './bidiPdf';import * as bidi from './third_party/bidiProtocol';import { nullProgress } from '../progress';+import { startAutomaticVideoRecording } from '../videoRecorder';import * as frames from '../frames';import * as network from '../network';@@ -56,6 +57,8 @@ export class BidiPage implements PageDelegate {private _initScriptIds = new Map<InitScript, string>();private readonly _fragmentNavigations = new Set<string>();private readonly _failedNavigations = new Map<string, string>();+ private _screencastTimer: NodeJS.Timeout | undefined;+ private _waitingForScreenshot = false;constructor(browserContext: BidiBrowserContext, bidiSession: BidiSession, opener: BidiPage | null) {this._session = bidiSession;@@ -103,6 +106,7 @@ export class BidiPage implements PageDelegate {// If the page is created by the Playwright client's call, some initialization// may be pending. Wait for it to complete before reporting the page as new.]);+ startAutomaticVideoRecording(this._page);}didClose() {@@ -576,9 +580,53 @@ export class BidiPage implements PageDelegate {}startScreencast(options: { width: number, height: number, quality: number }) {+ if (this._screencastTimer)+ return;++ this._waitingForScreenshot = false;+ this._screencastTimer = setInterval(async () => {+ if (this._waitingForScreenshot)+ return;+ if (this._session.isDisposed()) {+ this.stopScreencast();+ return;+ }++ this._waitingForScreenshot = true;+ const payload = await this._session.sendMayFail('browsingContext.captureScreenshot', {+ context: this._session.sessionId,+ format: {+ type: 'image/jpeg',+ quality: options.quality / 100+ }+ });+ if (payload) {+ const buffer = Buffer.from(payload.data, 'base64');+ const { width, height } = jpegDimensions(buffer);+ this._page.screencast.onScreencastFrame({+ buffer,+ frameSwapWallTime: Date.now(),+ viewportWidth: width,+ viewportHeight: height,+ });+ }+ this._waitingForScreenshot = false;+ }, 40);}stopScreencast() {+ if (this._screencastTimer) {+ clearInterval(this._screencastTimer);+ this._screencastTimer = undefined;+ }+ }++ getFFmpegVideoFilterArgs({ width, height }: { width: number, height: number }) {+ // We use "scale" and "pad" video filters (-vf option) to resize incoming frames+ // that might be of a different size to the desired video size.+ // https://ffmpeg.org/ffmpeg-filters.html#scale+ // https://ffmpeg.org/ffmpeg-filters.html#pad-1+ return `scale=w='min(iw,${width})':h='min(ih,${height})':force_original_aspect_ratio=decrease:eval=frame,pad=${width}:${height}:0:0:gray`;}rafCountForStablePosition(): number {@@ -679,3 +727,22 @@ export class BidiPage implements PageDelegate {function toBidiExecutionContext(executionContext: dom.FrameExecutionContext): BidiExecutionContext {return executionContext.delegate as BidiExecutionContext;}++function jpegDimensions(buffer: Buffer): { width: number, height: number } {+ let i = 2; // skip SOI marker (FF D8)+ while (i < buffer.length - 8) {+ if (buffer[i] !== 0xFF)+ break;+ const marker = buffer[i + 1];+ const segmentLength = buffer.readUInt16BE(i + 2);+ // SOF markers: C0 (baseline), C2 (progressive), C1, C3, C5-C7, C9-CB, CD-CF+ if ((marker >= 0xC0 && marker <= 0xC3) || (marker >= 0xC5 && marker <= 0xC7) ||+ (marker >= 0xC9 && marker <= 0xCB) || (marker >= 0xCD && marker <= 0xCF)) {+ const height = buffer.readUInt16BE(i + 5);+ const width = buffer.readUInt16BE(i + 7);+ return { width, height };+ }+ i += 2 + segmentLength;+ }+ throw new Error('Could not parse JPEG dimensions');+}diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.tsindex 8b19b238bd..92193c60f1 100644--- a/packages/playwright-core/src/server/page.ts+++ b/packages/playwright-core/src/server/page.ts@@ -105,6 +105,8 @@ export interface PageDelegate {// WebKit hack.shouldToggleStyleSheetToSyncAnimations(): boolean;setDockTile(image: Buffer): Promise<void>;+ // Allow Bidi to set different ffmpeg video filter args.+ getFFmpegVideoFilterArgs?: (options: { width: number, height: number }) => string;}type EmulatedSize = { screen: types.Size, viewport: types.Size };diff --git a/packages/playwright-core/src/server/videoRecorder.ts b/packages/playwright-core/src/server/videoRecorder.tsindex d1608e45e1..292d6d84be 100644--- a/packages/playwright-core/src/server/videoRecorder.ts+++ b/packages/playwright-core/src/server/videoRecorder.ts@@ -30,7 +30,7 @@ import { registry } from './registry';import type * as types from './types';import type { ChildProcess } from 'child_process';import type { Screencast, ScreencastClient } from './screencast';-import type { Page } from './page';+import type { Page, PageDelegate } from './page';const fps = 25;@@ -60,7 +60,7 @@ export class VideoRecorder {const { size } = this._screencast.addClient(this._client);// For video files only, prioritize encoding into the given size, regardless of the actual pixel data.const videoSize = options.size ?? size;- this._videoRecorder = new FfmpegVideoRecorder(ffmpegPath, videoSize, outputFile);+ this._videoRecorder = new FfmpegVideoRecorder(ffmpegPath, videoSize, outputFile, this._screencast.page.delegate);this._artifact = new Artifact(this._screencast.page.browserContext, outputFile);return this._artifact;}@@ -108,16 +108,16 @@ class FfmpegVideoRecorder {private _launchPromise: Promise<Error | null>;private _outputFile: string;- constructor(ffmpegPath: string, size: types.Size, outputFile: string) {+ constructor(ffmpegPath: string, size: types.Size, outputFile: string, page: PageDelegate) {if (!outputFile.endsWith('.webm'))throw new Error('File must have .webm extension');this._outputFile = outputFile;this._ffmpegPath = ffmpegPath;this._size = size;- this._launchPromise = this._launch().catch(e => e);+ this._launchPromise = this._launch(page).catch(e => e);}- private async _launch() {+ private async _launch(page: PageDelegate) {await mkdirIfNeeded(this._outputFile);// How to tune the codec:// 1. Read vp8 documentation to figure out the options.@@ -163,7 +163,8 @@ class FfmpegVideoRecorder {const w = this._size.width;const h = this._size.height;- const args = `-loglevel error -f matroska -fpsprobesize 0 -probesize 32 -analyzeduration 0 -i pipe:0 -y -an -r ${fps} -c:v vp8 -qmin 0 -qmax 50 -crf 8 -deadline realtime -speed 8 -b:v 1M -threads 1 -vf pad=${w}:${h}:0:0:gray,crop=${w}:${h}:0:0`.split(' ');+ const videoFilterArgs = page.getFFmpegVideoFilterArgs?.({ width: w, height: h }) ?? `pad=${w}:${h}:0:0:gray,crop=${w}:${h}:0:0`;+ const args = `-loglevel error -f matroska -fpsprobesize 0 -probesize 32 -analyzeduration 0 -i pipe:0 -y -an -r ${fps} -c:v vp8 -qmin 0 -qmax 50 -crf 8 -deadline realtime -speed 8 -b:v 1M -threads 1 -vf ${videoFilterArgs}`.split(' ');args.push(this._outputFile);const { launchedProcess, gracefullyClose } = await launchProcess({diff --git a/tests/bidi/expectations/moz-firefox-nightly-library.txt b/tests/bidi/expectations/moz-firefox-nightly-library.txtindex 768a2632f2..e1fab0a7e2 100644--- a/tests/bidi/expectations/moz-firefox-nightly-library.txt+++ b/tests/bidi/expectations/moz-firefox-nightly-library.txt@@ -56,16 +56,10 @@ library/browsercontext-viewport-mobile.spec.ts › mobile viewport › should sulibrary/browsercontext-viewport.spec.ts › should drag with high dpi [fail]library/browsercontext-viewport.spec.ts › should support touch with null viewport [fail]library/browsertype-connect.spec.ts › launchServer › should be able to connect 20 times to a single server without warnings [timeout]-library/browsertype-connect.spec.ts › launchServer › should name video file after page guid when connecting to remote browser with artifactsDir [fail]library/browsertype-connect.spec.ts › launchServer › should save download [fail]-library/browsertype-connect.spec.ts › launchServer › should save videos to artifactsDir [fail]-library/browsertype-connect.spec.ts › launchServer › should saveAs videos from remote browser [timeout]library/browsertype-connect.spec.ts › launchServer › should upload a folder [fail]library/browsertype-connect.spec.ts › run-server › should be able to connect 20 times to a single server without warnings [timeout]-library/browsertype-connect.spec.ts › run-server › should name video file after page guid when connecting to remote browser with artifactsDir [fail]library/browsertype-connect.spec.ts › run-server › should save download [fail]-library/browsertype-connect.spec.ts › run-server › should save videos to artifactsDir [fail]-library/browsertype-connect.spec.ts › run-server › should saveAs videos from remote browser [timeout]library/browsertype-connect.spec.ts › run-server › should upload a folder [fail]library/browsertype-launch.spec.ts › should reject if launched browser fails immediately [fail]library/capabilities.spec.ts › SharedArrayBuffer should work @smoke [timeout]@@ -155,11 +149,11 @@ library/inspector/cli-codegen-3.spec.ts › cli codegen › should generate framlibrary/launcher.spec.ts › should throw a friendly error if its headed and there is no xserver on linux running [fail]library/locator-dispatchevent-touch.spec.ts › should support touch points in touch event arguments [fail]library/multiclient.spec.ts › last emulateMedia wins [fail]-library/multiclient.spec.ts › screencast should deliver cached last frame to a new client [fail]library/multiclient.spec.ts › should chain routes [timeout]library/multiclient.spec.ts › should unroute websockets [timeout]library/page-clock.spec.ts › popup › should tick after popup [flaky]library/page-event-crash.spec.ts › should be able to close context when page crashes [timeout]+library/page-event-crash.spec.ts › should be able to close page after crash [timeout]library/page-event-crash.spec.ts › should cancel navigation when page crashes [timeout]library/page-event-crash.spec.ts › should cancel waitForEvent when page crashes [timeout]library/page-event-crash.spec.ts › should emit crash event when page crashes [timeout]@@ -181,13 +175,7 @@ library/role-utils.spec.ts › axe-core accessible-text [timeout]library/role-utils.spec.ts › wpt accname #2 [timeout]library/role-utils.spec.ts › wpt accname #3 [timeout]library/route-web-socket.spec.ts › should emit close upon frame detach [timeout]-library/screencast.spec.ts › onFrame receives viewport size [fail]library/screencast.spec.ts › screencast.start delivers frames via onFrame callback [fail]-library/screencast.spec.ts › start dispose stops recording [fail]-library/screencast.spec.ts › start returns a disposable that stops screencast [fail]-library/screencast.spec.ts › start reuses existing screencast when video recording is running [fail]-library/screencast.spec.ts › start should work when recordVideo is set [fail]-library/screencast.spec.ts › supports multiple onFrame listeners [fail]library/screenshot.spec.ts › page screenshot › should work with device scale factor and scale:css [fail]library/screenshot.spec.ts › page screenshot › should work with device scale factor, clip and scale:css [fail]library/selector-generator.spec.ts › selector generator › should work in dynamic iframes without navigation [fail]@@ -201,9 +189,6 @@ library/tap.spec.ts › should send well formed touch points [timeout]library/tap.spec.ts › should wait until an element is visible to tap it [fail]library/tap.spec.ts › should work with modifiers [timeout]library/tap.spec.ts › trial run should not tap [fail]-library/trace-viewer.spec.ts › canvas clipping [timeout]-library/trace-viewer.spec.ts › canvas clipping in iframe [timeout]-library/trace-viewer.spec.ts › canvas disabled title [fail]library/trace-viewer.spec.ts › should filter network requests by multiple resource types [flaky]library/trace-viewer.spec.ts › should filter network requests by url [fail]library/trace-viewer.spec.ts › should filter network requests by websocket type [fail]@@ -213,41 +198,9 @@ library/trace-viewer.spec.ts › should pick locator in iframe [fail]library/trace-viewer.spec.ts › should render console [fail]library/trace-viewer.spec.ts › should show only one pointer with multilevel iframes [fail]library/trace-viewer.spec.ts › should show websocket messages [fail]-library/tracing.spec.ts › should collect trace with resources, but no js [fail]-library/tracing.spec.ts › should produce screencast frames crop [fail]library/tracing.spec.ts › should produce screencast frames fit [fail]library/tracing.spec.ts › should produce screencast frames scale [fail]-library/video.spec.ts › screencast › should be 800x450 by default [timeout]-library/video.spec.ts › screencast › should be 800x600 with null viewport [timeout]-library/video.spec.ts › screencast › should capture css transformation [timeout]-library/video.spec.ts › screencast › should capture full viewport [fail]-library/video.spec.ts › screencast › should capture full viewport on hidpi [fail]-library/video.spec.ts › screencast › should capture navigation [timeout]-library/video.spec.ts › screencast › should capture static page [timeout]-library/video.spec.ts › screencast › should capture static page in persistent context @smoke [fail]-library/video.spec.ts › screencast › should close ffmpeg even if there were no frames [fail]-library/video.spec.ts › screencast › should continue recording main page after popup closes [timeout]-library/video.spec.ts › screencast › should delete video [timeout]-library/video.spec.ts › screencast › should emulate an iphone [timeout]-library/video.spec.ts › screencast › should expose video path [timeout]-library/video.spec.ts › screencast › should expose video path blank page [timeout]-library/video.spec.ts › screencast › should expose video path blank popup [timeout]-library/video.spec.ts › screencast › should fail when recordVideo is set [fail]-library/video.spec.ts › screencast › should not create video for internal pages [fail]-library/video.spec.ts › screencast › should scale frames down to the requested size [timeout]-library/video.spec.ts › screencast › should throw if browser dies [fail]-library/video.spec.ts › screencast › should throw on browser close [fail]-library/video.spec.ts › screencast › should use viewport scaled down to fit into 800x800 as default size [timeout]-library/video.spec.ts › screencast › should wait for video to finish if page was closed [fail]-library/video.spec.ts › screencast › should work for popups [timeout]-library/video.spec.ts › screencast › should work with old options [timeout]-library/video.spec.ts › screencast › should work with relative path for recordVideo.dir [timeout]-library/video.spec.ts › screencast › should work with video+trace [timeout]-library/video.spec.ts › screencast › should work with weird screen resolution [timeout]-library/video.spec.ts › screencast › video.start dispose stops recording [fail]-library/video.spec.ts › screencast › video.start should fail when recordVideo is set, but stop should work [fail]-library/video.spec.ts › screencast › video.start/stop twice [timeout]-library/video.spec.ts › should saveAs video [timeout]+library/tracing.spec.ts › should save trace while a WebSocket keeps streaming frames [fail]library/web-socket.spec.ts › should emit binary frame events [timeout]library/web-socket.spec.ts › should emit close events [timeout]library/web-socket.spec.ts › should emit error [timeout]diff --git a/tests/bidi/expectations/moz-firefox-nightly-page.txt b/tests/bidi/expectations/moz-firefox-nightly-page.txtindex 487d2f75c0..1841a75ac3 100644--- a/tests/bidi/expectations/moz-firefox-nightly-page.txt+++ b/tests/bidi/expectations/moz-firefox-nightly-page.txt@@ -106,6 +106,8 @@ page/page-keyboard.spec.ts › should press audio and media control keys [fail]page/page-keyboard.spec.ts › should send a character with insertText [fail]page/page-mouse.spec.ts › should always round down [fail]page/page-mouse.spec.ts › should dblclick the div [timeout]+page/page-network-idle.spec.ts › should not wait for an open EventSource connection [timeout]+page/page-network-idle.spec.ts › should not wait for an open EventSource connection in setContent [timeout]page/page-network-request.spec.ts › should not allow to access frame on popup main request [fail]page/page-network-request.spec.ts › should not get preflight CORS requests when intercepting [fail]page/page-network-request.spec.ts › should parse the data if content-type is application/x-www-form-urlencoded [fail]@@ -121,6 +123,7 @@ page/page-network-response.spec.ts › should report all headers [fail]page/page-network-response.spec.ts › should report if request was fromServiceWorker [fail]page/page-network-response.spec.ts › should report multiple set-cookie headers [fail]page/page-network-response.spec.ts › should wait until response completes [fail]+page/page-request-continue.spec.ts › continue with headers should send fresh cookie from the browser cookie store [fail]page/page-request-continue.spec.ts › should not throw if request was cancelled by the page [timeout]page/page-request-fulfill.spec.ts › should fulfill with gzip and readback [timeout]page/page-request-fulfill.spec.ts › should not throw if request was cancelled by the page [timeout]diff --git a/tests/library/video.spec.ts b/tests/library/video.spec.tsindex 848e633297..3c1cff83e4 100644--- a/tests/library/video.spec.ts+++ b/tests/library/video.spec.ts@@ -334,8 +334,8 @@ it.describe('screencast', () => {}});- it('should work for popups', async ({ browser, server, browserName, trace }, testInfo) => {- it.fixme(browserName === 'firefox', 'https://github.com/microsoft/playwright/issues/14557');+ it('should work for popups', async ({ browser, server, browserName, isBidi }, testInfo) => {+ it.fixme(browserName === 'firefox' && !isBidi, 'https://github.com/microsoft/playwright/issues/14557');const videosPath = testInfo.outputPath('');const size = { width: 600, height: 400 };const context = await browser.newContext({@@ -453,8 +453,8 @@ it.describe('screencast', () => {expect(videoPlayer.videoHeight).toBe(450);});- it('should be 800x600 with null viewport', async ({ browser, headless, browserName }, testInfo) => {- it.fixme(browserName === 'firefox' && headless, 'Fails in headless on bots');+ it('should be 800x600 with null viewport', async ({ browser, headless, browserName, isBidi }, testInfo) => {+ it.fixme(browserName === 'firefox' && headless && !isBidi, 'Fails in headless on bots');const context = await browser.newContext({recordVideo: {