Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore(bidi): implement screencasting using `browsingContext.captureScreenshot` (#41569)

Holger Benlfdd0bc913dfe
diff --git a/packages/playwright-core/src/server/bidi/bidiPage.ts b/packages/playwright-core/src/server/bidi/bidiPage.ts
index aec8cc1249..908a16c24e 100644
--- a/packages/playwright-core/src/server/bidi/bidiPage.ts
+++ b/packages/playwright-core/src/server/bidi/bidiPage.ts
@@ -27,6 +27,7 @@ import { BidiNetworkManager } from './bidiNetworkManager';
import { BidiPDF } from './bidiPdf';
import * as bidi from './third_party/bidiProtocol';
import { nullProgress } from '../progress';
+import { startAutomaticVideoRecording } from '../videoRecorder';
import * as frames from '../frames';
import * as network from '../network';
@@ -56,6 +57,8 @@ export class BidiPage implements PageDelegate {
private _initScriptIds = new Map<InitScript, string>();
private readonly _fragmentNavigations = new Set<string>();
private readonly _failedNavigations = new Map<string, string>();
+ private _screencastTimer: NodeJS.Timeout | undefined;
+ private _waitingForScreenshot = false;
constructor(browserContext: BidiBrowserContext, bidiSession: BidiSession, opener: BidiPage | null) {
this._session = bidiSession;
@@ -103,6 +106,7 @@ export class BidiPage implements PageDelegate {
// If the page is created by the Playwright client's call, some initialization
// may be pending. Wait for it to complete before reporting the page as new.
]);
+ startAutomaticVideoRecording(this._page);
}
didClose() {
@@ -576,9 +580,53 @@ export class BidiPage implements PageDelegate {
}
startScreencast(options: { width: number, height: number, quality: number }) {
+ if (this._screencastTimer)
+ return;
+
+ this._waitingForScreenshot = false;
+ this._screencastTimer = setInterval(async () => {
+ if (this._waitingForScreenshot)
+ return;
+ if (this._session.isDisposed()) {
+ this.stopScreencast();
+ return;
+ }
+
+ this._waitingForScreenshot = true;
+ const payload = await this._session.sendMayFail('browsingContext.captureScreenshot', {
+ context: this._session.sessionId,
+ format: {
+ type: 'image/jpeg',
+ quality: options.quality / 100
+ }
+ });
+ if (payload) {
+ const buffer = Buffer.from(payload.data, 'base64');
+ const { width, height } = jpegDimensions(buffer);
+ this._page.screencast.onScreencastFrame({
+ buffer,
+ frameSwapWallTime: Date.now(),
+ viewportWidth: width,
+ viewportHeight: height,
+ });
+ }
+ this._waitingForScreenshot = false;
+ }, 40);
}
stopScreencast() {
+ if (this._screencastTimer) {
+ clearInterval(this._screencastTimer);
+ this._screencastTimer = undefined;
+ }
+ }
+
+ getFFmpegVideoFilterArgs({ width, height }: { width: number, height: number }) {
+ // We use "scale" and "pad" video filters (-vf option) to resize incoming frames
+ // that might be of a different size to the desired video size.
+ // https://ffmpeg.org/ffmpeg-filters.html#scale
+ // https://ffmpeg.org/ffmpeg-filters.html#pad-1
+ return `scale=w='min(iw,${width})':h='min(ih,${height})':force_original_aspect_ratio=decrease:eval=frame,pad=${width}:${height}:0:0:gray`;
}
rafCountForStablePosition(): number {
@@ -679,3 +727,22 @@ export class BidiPage implements PageDelegate {
function toBidiExecutionContext(executionContext: dom.FrameExecutionContext): BidiExecutionContext {
return executionContext.delegate as BidiExecutionContext;
}
+
+function jpegDimensions(buffer: Buffer): { width: number, height: number } {
+ let i = 2; // skip SOI marker (FF D8)
+ while (i < buffer.length - 8) {
+ if (buffer[i] !== 0xFF)
+ break;
+ const marker = buffer[i + 1];
+ const segmentLength = buffer.readUInt16BE(i + 2);
+ // SOF markers: C0 (baseline), C2 (progressive), C1, C3, C5-C7, C9-CB, CD-CF
+ if ((marker >= 0xC0 && marker <= 0xC3) || (marker >= 0xC5 && marker <= 0xC7) ||
+ (marker >= 0xC9 && marker <= 0xCB) || (marker >= 0xCD && marker <= 0xCF)) {
+ const height = buffer.readUInt16BE(i + 5);
+ const width = buffer.readUInt16BE(i + 7);
+ return { width, height };
+ }
+ i += 2 + segmentLength;
+ }
+ throw new Error('Could not parse JPEG dimensions');
+}
diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts
index 8b19b238bd..92193c60f1 100644
--- a/packages/playwright-core/src/server/page.ts
+++ b/packages/playwright-core/src/server/page.ts
@@ -105,6 +105,8 @@ export interface PageDelegate {
// WebKit hack.
shouldToggleStyleSheetToSyncAnimations(): boolean;
setDockTile(image: Buffer): Promise<void>;
+ // Allow Bidi to set different ffmpeg video filter args.
+ getFFmpegVideoFilterArgs?: (options: { width: number, height: number }) => string;
}
type EmulatedSize = { screen: types.Size, viewport: types.Size };
diff --git a/packages/playwright-core/src/server/videoRecorder.ts b/packages/playwright-core/src/server/videoRecorder.ts
index d1608e45e1..292d6d84be 100644
--- a/packages/playwright-core/src/server/videoRecorder.ts
+++ b/packages/playwright-core/src/server/videoRecorder.ts
@@ -30,7 +30,7 @@ import { registry } from './registry';
import type * as types from './types';
import type { ChildProcess } from 'child_process';
import type { Screencast, ScreencastClient } from './screencast';
-import type { Page } from './page';
+import type { Page, PageDelegate } from './page';
const fps = 25;
@@ -60,7 +60,7 @@ export class VideoRecorder {
const { size } = this._screencast.addClient(this._client);
// For video files only, prioritize encoding into the given size, regardless of the actual pixel data.
const videoSize = options.size ?? size;
- this._videoRecorder = new FfmpegVideoRecorder(ffmpegPath, videoSize, outputFile);
+ this._videoRecorder = new FfmpegVideoRecorder(ffmpegPath, videoSize, outputFile, this._screencast.page.delegate);
this._artifact = new Artifact(this._screencast.page.browserContext, outputFile);
return this._artifact;
}
@@ -108,16 +108,16 @@ class FfmpegVideoRecorder {
private _launchPromise: Promise<Error | null>;
private _outputFile: string;
- constructor(ffmpegPath: string, size: types.Size, outputFile: string) {
+ constructor(ffmpegPath: string, size: types.Size, outputFile: string, page: PageDelegate) {
if (!outputFile.endsWith('.webm'))
throw new Error('File must have .webm extension');
this._outputFile = outputFile;
this._ffmpegPath = ffmpegPath;
this._size = size;
- this._launchPromise = this._launch().catch(e => e);
+ this._launchPromise = this._launch(page).catch(e => e);
}
- private async _launch() {
+ private async _launch(page: PageDelegate) {
await mkdirIfNeeded(this._outputFile);
// How to tune the codec:
// 1. Read vp8 documentation to figure out the options.
@@ -163,7 +163,8 @@ class FfmpegVideoRecorder {
const w = this._size.width;
const h = this._size.height;
- const args = `-loglevel error -f matroska -fpsprobesize 0 -probesize 32 -analyzeduration 0 -i pipe:0 -y -an -r ${fps} -c:v vp8 -qmin 0 -qmax 50 -crf 8 -deadline realtime -speed 8 -b:v 1M -threads 1 -vf pad=${w}:${h}:0:0:gray,crop=${w}:${h}:0:0`.split(' ');
+ const videoFilterArgs = page.getFFmpegVideoFilterArgs?.({ width: w, height: h }) ?? `pad=${w}:${h}:0:0:gray,crop=${w}:${h}:0:0`;
+ const args = `-loglevel error -f matroska -fpsprobesize 0 -probesize 32 -analyzeduration 0 -i pipe:0 -y -an -r ${fps} -c:v vp8 -qmin 0 -qmax 50 -crf 8 -deadline realtime -speed 8 -b:v 1M -threads 1 -vf ${videoFilterArgs}`.split(' ');
args.push(this._outputFile);
const { launchedProcess, gracefullyClose } = await launchProcess({
diff --git a/tests/bidi/expectations/moz-firefox-nightly-library.txt b/tests/bidi/expectations/moz-firefox-nightly-library.txt
index 768a2632f2..e1fab0a7e2 100644
--- a/tests/bidi/expectations/moz-firefox-nightly-library.txt
+++ b/tests/bidi/expectations/moz-firefox-nightly-library.txt
@@ -56,16 +56,10 @@ library/browsercontext-viewport-mobile.spec.ts › mobile viewport › should su
library/browsercontext-viewport.spec.ts › should drag with high dpi [fail]
library/browsercontext-viewport.spec.ts › should support touch with null viewport [fail]
library/browsertype-connect.spec.ts › launchServer › should be able to connect 20 times to a single server without warnings [timeout]
-library/browsertype-connect.spec.ts › launchServer › should name video file after page guid when connecting to remote browser with artifactsDir [fail]
library/browsertype-connect.spec.ts › launchServer › should save download [fail]
-library/browsertype-connect.spec.ts › launchServer › should save videos to artifactsDir [fail]
-library/browsertype-connect.spec.ts › launchServer › should saveAs videos from remote browser [timeout]
library/browsertype-connect.spec.ts › launchServer › should upload a folder [fail]
library/browsertype-connect.spec.ts › run-server › should be able to connect 20 times to a single server without warnings [timeout]
-library/browsertype-connect.spec.ts › run-server › should name video file after page guid when connecting to remote browser with artifactsDir [fail]
library/browsertype-connect.spec.ts › run-server › should save download [fail]
-library/browsertype-connect.spec.ts › run-server › should save videos to artifactsDir [fail]
-library/browsertype-connect.spec.ts › run-server › should saveAs videos from remote browser [timeout]
library/browsertype-connect.spec.ts › run-server › should upload a folder [fail]
library/browsertype-launch.spec.ts › should reject if launched browser fails immediately [fail]
library/capabilities.spec.ts › SharedArrayBuffer should work @smoke [timeout]
@@ -155,11 +149,11 @@ library/inspector/cli-codegen-3.spec.ts › cli codegen › should generate fram
library/launcher.spec.ts › should throw a friendly error if its headed and there is no xserver on linux running [fail]
library/locator-dispatchevent-touch.spec.ts › should support touch points in touch event arguments [fail]
library/multiclient.spec.ts › last emulateMedia wins [fail]
-library/multiclient.spec.ts › screencast should deliver cached last frame to a new client [fail]
library/multiclient.spec.ts › should chain routes [timeout]
library/multiclient.spec.ts › should unroute websockets [timeout]
library/page-clock.spec.ts › popup › should tick after popup [flaky]
library/page-event-crash.spec.ts › should be able to close context when page crashes [timeout]
+library/page-event-crash.spec.ts › should be able to close page after crash [timeout]
library/page-event-crash.spec.ts › should cancel navigation when page crashes [timeout]
library/page-event-crash.spec.ts › should cancel waitForEvent when page crashes [timeout]
library/page-event-crash.spec.ts › should emit crash event when page crashes [timeout]
@@ -181,13 +175,7 @@ library/role-utils.spec.ts › axe-core accessible-text [timeout]
library/role-utils.spec.ts › wpt accname #2 [timeout]
library/role-utils.spec.ts › wpt accname #3 [timeout]
library/route-web-socket.spec.ts › should emit close upon frame detach [timeout]
-library/screencast.spec.ts › onFrame receives viewport size [fail]
library/screencast.spec.ts › screencast.start delivers frames via onFrame callback [fail]
-library/screencast.spec.ts › start dispose stops recording [fail]
-library/screencast.spec.ts › start returns a disposable that stops screencast [fail]
-library/screencast.spec.ts › start reuses existing screencast when video recording is running [fail]
-library/screencast.spec.ts › start should work when recordVideo is set [fail]
-library/screencast.spec.ts › supports multiple onFrame listeners [fail]
library/screenshot.spec.ts › page screenshot › should work with device scale factor and scale:css [fail]
library/screenshot.spec.ts › page screenshot › should work with device scale factor, clip and scale:css [fail]
library/selector-generator.spec.ts › selector generator › should work in dynamic iframes without navigation [fail]
@@ -201,9 +189,6 @@ library/tap.spec.ts › should send well formed touch points [timeout]
library/tap.spec.ts › should wait until an element is visible to tap it [fail]
library/tap.spec.ts › should work with modifiers [timeout]
library/tap.spec.ts › trial run should not tap [fail]
-library/trace-viewer.spec.ts › canvas clipping [timeout]
-library/trace-viewer.spec.ts › canvas clipping in iframe [timeout]
-library/trace-viewer.spec.ts › canvas disabled title [fail]
library/trace-viewer.spec.ts › should filter network requests by multiple resource types [flaky]
library/trace-viewer.spec.ts › should filter network requests by url [fail]
library/trace-viewer.spec.ts › should filter network requests by websocket type [fail]
@@ -213,41 +198,9 @@ library/trace-viewer.spec.ts › should pick locator in iframe [fail]
library/trace-viewer.spec.ts › should render console [fail]
library/trace-viewer.spec.ts › should show only one pointer with multilevel iframes [fail]
library/trace-viewer.spec.ts › should show websocket messages [fail]
-library/tracing.spec.ts › should collect trace with resources, but no js [fail]
-library/tracing.spec.ts › should produce screencast frames crop [fail]
library/tracing.spec.ts › should produce screencast frames fit [fail]
library/tracing.spec.ts › should produce screencast frames scale [fail]
-library/video.spec.ts › screencast › should be 800x450 by default [timeout]
-library/video.spec.ts › screencast › should be 800x600 with null viewport [timeout]
-library/video.spec.ts › screencast › should capture css transformation [timeout]
-library/video.spec.ts › screencast › should capture full viewport [fail]
-library/video.spec.ts › screencast › should capture full viewport on hidpi [fail]
-library/video.spec.ts › screencast › should capture navigation [timeout]
-library/video.spec.ts › screencast › should capture static page [timeout]
-library/video.spec.ts › screencast › should capture static page in persistent context @smoke [fail]
-library/video.spec.ts › screencast › should close ffmpeg even if there were no frames [fail]
-library/video.spec.ts › screencast › should continue recording main page after popup closes [timeout]
-library/video.spec.ts › screencast › should delete video [timeout]
-library/video.spec.ts › screencast › should emulate an iphone [timeout]
-library/video.spec.ts › screencast › should expose video path [timeout]
-library/video.spec.ts › screencast › should expose video path blank page [timeout]
-library/video.spec.ts › screencast › should expose video path blank popup [timeout]
-library/video.spec.ts › screencast › should fail when recordVideo is set [fail]
-library/video.spec.ts › screencast › should not create video for internal pages [fail]
-library/video.spec.ts › screencast › should scale frames down to the requested size [timeout]
-library/video.spec.ts › screencast › should throw if browser dies [fail]
-library/video.spec.ts › screencast › should throw on browser close [fail]
-library/video.spec.ts › screencast › should use viewport scaled down to fit into 800x800 as default size [timeout]
-library/video.spec.ts › screencast › should wait for video to finish if page was closed [fail]
-library/video.spec.ts › screencast › should work for popups [timeout]
-library/video.spec.ts › screencast › should work with old options [timeout]
-library/video.spec.ts › screencast › should work with relative path for recordVideo.dir [timeout]
-library/video.spec.ts › screencast › should work with video+trace [timeout]
-library/video.spec.ts › screencast › should work with weird screen resolution [timeout]
-library/video.spec.ts › screencast › video.start dispose stops recording [fail]
-library/video.spec.ts › screencast › video.start should fail when recordVideo is set, but stop should work [fail]
-library/video.spec.ts › screencast › video.start/stop twice [timeout]
-library/video.spec.ts › should saveAs video [timeout]
+library/tracing.spec.ts › should save trace while a WebSocket keeps streaming frames [fail]
library/web-socket.spec.ts › should emit binary frame events [timeout]
library/web-socket.spec.ts › should emit close events [timeout]
library/web-socket.spec.ts › should emit error [timeout]
diff --git a/tests/bidi/expectations/moz-firefox-nightly-page.txt b/tests/bidi/expectations/moz-firefox-nightly-page.txt
index 487d2f75c0..1841a75ac3 100644
--- a/tests/bidi/expectations/moz-firefox-nightly-page.txt
+++ b/tests/bidi/expectations/moz-firefox-nightly-page.txt
@@ -106,6 +106,8 @@ page/page-keyboard.spec.ts › should press audio and media control keys [fail]
page/page-keyboard.spec.ts › should send a character with insertText [fail]
page/page-mouse.spec.ts › should always round down [fail]
page/page-mouse.spec.ts › should dblclick the div [timeout]
+page/page-network-idle.spec.ts › should not wait for an open EventSource connection [timeout]
+page/page-network-idle.spec.ts › should not wait for an open EventSource connection in setContent [timeout]
page/page-network-request.spec.ts › should not allow to access frame on popup main request [fail]
page/page-network-request.spec.ts › should not get preflight CORS requests when intercepting [fail]
page/page-network-request.spec.ts › should parse the data if content-type is application/x-www-form-urlencoded [fail]
@@ -121,6 +123,7 @@ page/page-network-response.spec.ts › should report all headers [fail]
page/page-network-response.spec.ts › should report if request was fromServiceWorker [fail]
page/page-network-response.spec.ts › should report multiple set-cookie headers [fail]
page/page-network-response.spec.ts › should wait until response completes [fail]
+page/page-request-continue.spec.ts › continue with headers should send fresh cookie from the browser cookie store [fail]
page/page-request-continue.spec.ts › should not throw if request was cancelled by the page [timeout]
page/page-request-fulfill.spec.ts › should fulfill with gzip and readback [timeout]
page/page-request-fulfill.spec.ts › should not throw if request was cancelled by the page [timeout]
diff --git a/tests/library/video.spec.ts b/tests/library/video.spec.ts
index 848e633297..3c1cff83e4 100644
--- a/tests/library/video.spec.ts
+++ b/tests/library/video.spec.ts
@@ -334,8 +334,8 @@ it.describe('screencast', () => {
}
});
- it('should work for popups', async ({ browser, server, browserName, trace }, testInfo) => {
- it.fixme(browserName === 'firefox', 'https://github.com/microsoft/playwright/issues/14557');
+ it('should work for popups', async ({ browser, server, browserName, isBidi }, testInfo) => {
+ it.fixme(browserName === 'firefox' && !isBidi, 'https://github.com/microsoft/playwright/issues/14557');
const videosPath = testInfo.outputPath('');
const size = { width: 600, height: 400 };
const context = await browser.newContext({
@@ -453,8 +453,8 @@ it.describe('screencast', () => {
expect(videoPlayer.videoHeight).toBe(450);
});
- it('should be 800x600 with null viewport', async ({ browser, headless, browserName }, testInfo) => {
- it.fixme(browserName === 'firefox' && headless, 'Fails in headless on bots');
+ it('should be 800x600 with null viewport', async ({ browser, headless, browserName, isBidi }, testInfo) => {
+ it.fixme(browserName === 'firefox' && headless && !isBidi, 'Fails in headless on bots');
const context = await browser.newContext({
recordVideo: {