🎥 Make videos programmatically with React
Reduce CDP overhead during shared-memory capture
Jonny Burger9f1adfec0fee
diff --git a/packages/browser-studio/src/BrowserStudio.tsx b/packages/browser-studio/src/BrowserStudio.tsxindex 7176c24714..a5e54b9fe0 100644--- a/packages/browser-studio/src/BrowserStudio.tsx+++ b/packages/browser-studio/src/BrowserStudio.tsx@@ -145,6 +145,7 @@ const getBrowserRenderDefaults = (): RenderDefaults => {repro: false,sampleRate: 48_000,scale: 1,+ sharedMemoryCapture: null,stillImageFormat: 'png',userAgent: null,videoBitrate: null,diff --git a/packages/renderer/src/browser/Connection.ts b/packages/renderer/src/browser/Connection.tsindex 3ad55f713a..a70bebb9e5 100644--- a/packages/renderer/src/browser/Connection.ts+++ b/packages/renderer/src/browser/Connection.ts@@ -24,9 +24,7 @@ interface ConnectionCallback {resolve: (value: {value: any; size: number}) => void;reject: Function;method: string;- returnSize: boolean;- stack: string;- fn: string;+ params: object | null;}const ConnectionEmittedEvents = {@@ -74,9 +72,7 @@ export class Connection extends EventEmitter {resolve,reject,method,- returnSize: true,- stack: new Error().stack ?? '',- fn: method + JSON.stringify(params),+ params: params ?? null,});},);@@ -130,10 +126,8 @@ export class Connection extends EventEmitter {if (object.error) {callback.reject(createProtocolError(callback.method, object));- } else if (callback.returnSize) {- callback.resolve({value: object.result, size: message.length});} else {- callback.resolve(object.result);+ callback.resolve({value: object.result, size: message.length});}}} else {@@ -245,7 +239,10 @@ export class CDPSession extends EventEmitter {(resolve, reject) => {if (this.#callbacks.size > 100) {for (const callback of this.#callbacks.values()) {- Log.info({indent: false, logLevel: 'info'}, callback.fn);+ Log.info(+ {indent: false, logLevel: 'info'},+ callback.method + JSON.stringify(callback.params),+ );}throw new Error('Leak detected: Too many callbacks');@@ -255,9 +252,7 @@ export class CDPSession extends EventEmitter {resolve,reject,method,- returnSize: true,- stack: new Error().stack ?? '',- fn: method + JSON.stringify(params),+ params: params ?? null,});},);@@ -269,10 +264,8 @@ export class CDPSession extends EventEmitter {this.#callbacks.delete(object.id);if (object.error) {callback.reject(createProtocolError(callback.method, object));- } else if (callback.returnSize) {- callback.resolve({value: object.result, size});} else {- callback.resolve(object.result);+ callback.resolve({value: object.result, size});}} else {this.emit(object.method, object.params);diff --git a/packages/renderer/src/browser/DOMWorld.ts b/packages/renderer/src/browser/DOMWorld.tsindex 51f81cc985..d760d9d8dd 100644--- a/packages/renderer/src/browser/DOMWorld.ts+++ b/packages/renderer/src/browser/DOMWorld.ts@@ -28,7 +28,7 @@ import type {import type {ExecutionContext} from './ExecutionContext';import type {Frame} from './FrameManager';import type {JSHandle} from './JSHandle';-import {isString} from './util';+import {isString, valueFromRemoteObject} from './util';export class DOMWorld {#frame: Frame;@@ -227,11 +227,8 @@ class WaitTask {const runCount = ++this.#runCount;let success: JSHandle | null = null;let error: Error | null = null;- const context = await this.#domWorld.executionContext();- if (this.#terminated || runCount !== this.#runCount) {- return;- }-+ const contextPromise = this.#domWorld.executionContext();+ const context = await contextPromise;if (this.#terminated || runCount !== this.#runCount) {return;}@@ -256,17 +253,21 @@ class WaitTask {}// Ignore timeouts in pageScript - we track timeouts ourselves.- // If the frame's execution context has already changed, `frame.evaluate` will- // throw an error - ignore this predicate run altogether.+ // Primitive results need no further CDP evaluation, but must still belong+ // to the current execution context. Object handles are checked in the page.if (!error &&- (await this.#domWorld- .evaluate((s) => {- return !s;- }, success)- .catch(() => {- return true;- }))+ (success && !success._remoteObject.objectId+ ? !valueFromRemoteObject(success._remoteObject) ||+ !this.#domWorld._hasContext() ||+ this.#domWorld.executionContext() !== contextPromise+ : await this.#domWorld+ .evaluate((s) => {+ return !s;+ }, success)+ .catch(() => {+ return true;+ }))) {if (!success) {throw new Error('Assertion: result handle is not available');@@ -276,6 +277,11 @@ class WaitTask {return;}+ if (this.#terminated || runCount !== this.#runCount) {+ await success?.dispose();+ return;+ }+if (error) {if (error.message.includes('TypeError: binding is not a function')) {return this.rerun();diff --git a/packages/renderer/src/remotion-shared-memory.ts b/packages/renderer/src/remotion-shared-memory.tsindex 47c2a84a2f..79bec23425 100644--- a/packages/renderer/src/remotion-shared-memory.ts+++ b/packages/renderer/src/remotion-shared-memory.ts@@ -11,6 +11,7 @@ import {import type {LogLevel} from './log-level';import {Log} from './logger';import type {PixelFormat} from './pixel-format';+import {setScreenshotBackground} from './screenshot-background';const encoderPadding = 64;const slotsPerPage = 1;@@ -324,6 +325,14 @@ export class RemotionSharedMemoryCapture {await new Promise<void>((resolve) => pool.drainWaiters.push(resolve));}+ try {+ await setScreenshotBackground(page, false);+ } catch (error) {+ if (!bestEffort && !isTargetClosedErr(error as Error) && !cleanupError) {+ cleanupError = error;+ }+ }+try {await page._client().send('Page.remotionDestroyFramePool');} catch (error) {@@ -461,6 +470,7 @@ export class RemotionSharedMemoryCapture {}async #destroyPools() {+ let cleanupError: unknown = null;for (const pool of this.#pools.values()) {if (pool.ownedSlots.size > 0) {throw new Error(@@ -468,16 +478,27 @@ export class RemotionSharedMemoryCapture {);}+ try {+ await setScreenshotBackground(pool.page, false);+ } catch (error) {+ if (!isTargetClosedErr(error as Error) && !cleanupError) {+ cleanupError = error;+ }+ }+try {await pool.page._client().send('Page.remotionDestroyFramePool');} catch (error) {- if (!isTargetClosedErr(error as Error)) {- throw error;+ if (!isTargetClosedErr(error as Error) && !cleanupError) {+ cleanupError = error;}}}this.#pools.clear();+ if (cleanupError) {+ throw cleanupError;+ }}}diff --git a/packages/renderer/src/screenshot-background.ts b/packages/renderer/src/screenshot-background.tsnew file mode 100644index 0000000000..be8e618177--- /dev/null+++ b/packages/renderer/src/screenshot-background.ts@@ -0,0 +1,22 @@+import type {Page} from './browser/BrowserPage';++const transparentBackgrounds = new WeakSet<Page>();++export const setScreenshotBackground = async (+ page: Page,+ omitBackground: boolean,+) => {+ if (transparentBackgrounds.has(page) === omitBackground) {+ return;+ }++ if (omitBackground) {+ await page._client().send('Emulation.setDefaultBackgroundColorOverride', {+ color: {r: 0, g: 0, b: 0, a: 0},+ });+ transparentBackgrounds.add(page);+ } else {+ await page._client().send('Emulation.setDefaultBackgroundColorOverride');+ transparentBackgrounds.delete(page);+ }+};diff --git a/packages/renderer/src/screenshot-task.ts b/packages/renderer/src/screenshot-task.tsindex b79afc9ea2..eb16f1e722 100644--- a/packages/renderer/src/screenshot-task.ts+++ b/packages/renderer/src/screenshot-task.ts@@ -6,6 +6,7 @@ import type {CapturedFrame,RemotionSharedMemoryCapture,} from './remotion-shared-memory';+import {setScreenshotBackground} from './screenshot-background';export const screenshotTask = async ({format,@@ -35,11 +36,7 @@ export const screenshotTask = async ({targetId: target._targetId,});- if (omitBackground) {- await client.send('Emulation.setDefaultBackgroundColorOverride', {- color: {r: 0, g: 0, b: 0, a: 0},- });- }+ await setScreenshotBackground(page, omitBackground);const cap = startPerfMeasure('capture');try {@@ -104,20 +101,9 @@ export const screenshotTask = async ({const rawFrame = await reservation.publish(value.remotionFrame);stopPerfMeasure(cap);- if (omitBackground) {- try {- await client.send('Emulation.setDefaultBackgroundColorOverride');- } catch (error) {- try {- await rawFrame.release();- } catch {- // Preserve the background reset error that made the capture fail.- }-- throw error;- }- }+ // Keep the background configured until pool cleanup. Encoded+ // screenshots restore it below, including thumbnails on this page.return rawFrame;}@@ -126,7 +112,7 @@ export const screenshotTask = async ({stopPerfMeasure(cap);if (omitBackground) {- await client.send('Emulation.setDefaultBackgroundColorOverride');+ await setScreenshotBackground(page, false);}const buffer = Buffer.from(result.data, 'base64');diff --git a/packages/renderer/src/seek-to-frame.ts b/packages/renderer/src/seek-to-frame.tsindex 98fb3314c1..b9f173bcca 100644--- a/packages/renderer/src/seek-to-frame.ts+++ b/packages/renderer/src/seek-to-frame.ts@@ -211,5 +211,5 @@ export const seekToFrame = async ({timeoutInMilliseconds,});await waitForReady({page, timeoutInMilliseconds, frame, indent, logLevel});- await page.evaluateHandle('document.fonts.ready');+ await page.evaluate('document.fonts.ready.then(() => true)');};diff --git a/packages/renderer/src/test/remotion-shared-memory.test.ts b/packages/renderer/src/test/remotion-shared-memory.test.tsindex 72c4e853b5..33610423c6 100644--- a/packages/renderer/src/test/remotion-shared-memory.test.ts+++ b/packages/renderer/src/test/remotion-shared-memory.test.ts@@ -320,9 +320,10 @@ test('signals pool retirement before waiting for an unhanded frame to drain', asexpect(calls).toContain('Page.remotionDestroyFramePool');});-test('releases a published frame if transparent background reset fails', async () => {+test('keeps transparent backgrounds between frames and destroys the pool if reset fails', async () => {let backgroundCalls = 0;let releaseCalls = 0;+ let destroyCalls = 0;const client = {send: (method: string,@@ -370,6 +371,10 @@ test('releases a published frame if transparent background reset fails', async (releaseCalls++;}+ if (method === 'Page.remotionDestroyFramePool') {+ destroyCalls++;+ }+return Promise.resolve({value: undefined});},};@@ -386,8 +391,8 @@ test('releases a published frame if transparent background reset fails', async (});await capture.ensurePage(page);- await expect(- screenshotTask({+ for (let i = 0; i < 2; i++) {+ const frame = await screenshotTask({format: 'png',height: 16,width: 16,@@ -396,10 +401,17 @@ test('releases a published frame if transparent background reset fails', async (jpegQuality: undefined,scale: 1,remotionSharedMemory: capture,- }),- ).rejects.toThrow('background reset failed');- expect(releaseCalls).toBe(1);- await capture.destroy();+ });+ expect(Buffer.isBuffer(frame)).toBe(false);+ if (!Buffer.isBuffer(frame)) {+ await frame.release();+ }+ }++ expect(backgroundCalls).toBe(1);+ expect(releaseCalls).toBe(2);+ await expect(capture.destroy()).rejects.toThrow('background reset failed');+ expect(destroyCalls).toBe(1);});test('uses raw viewport capture for every video image format and falls back on vanilla Chrome', async () => {diff --git a/packages/studio-shared/src/render-defaults.ts b/packages/studio-shared/src/render-defaults.tsindex 03bc1bf174..f48963434d 100644--- a/packages/studio-shared/src/render-defaults.ts+++ b/packages/studio-shared/src/render-defaults.ts@@ -71,11 +71,10 @@ export type RenderDefaults = {allowHtmlInCanvas: boolean;sampleRate: number;configFileRenderDefaults: ConfigFileRenderDefaults | null;- // Optional to preserve compatibility with custom Studio integrations.- sharedMemoryCapture?: {+ sharedMemoryCapture: {browserExecutable: string | null;disabled: boolean;- };+ } | null;};declare global {diff --git a/packages/studio/src/components/RenderModal/ServerRenderModal.tsx b/packages/studio/src/components/RenderModal/ServerRenderModal.tsxindex ac7315bd91..a777ba595c 100644--- a/packages/studio/src/components/RenderModal/ServerRenderModal.tsx+++ b/packages/studio/src/components/RenderModal/ServerRenderModal.tsx@@ -485,15 +485,13 @@ const RenderModal: React.FC<const sharedMemoryCaptureRequest = useMemo(() => ({browserExecutable:- renderDefaults.sharedMemoryCapture?.browserExecutable ?? null,+ renderDefaults.sharedMemoryCapture === null+ ? null+ : renderDefaults.sharedMemoryCapture.browserExecutable,chromeMode,chromiumOptions,}),- [- renderDefaults.sharedMemoryCapture?.browserExecutable,- chromeMode,- chromiumOptions,- ],+ [renderDefaults.sharedMemoryCapture, chromeMode, chromiumOptions],);const sharedMemoryCaptureRequestKey = JSON.stringify(sharedMemoryCaptureRequest,@@ -504,7 +502,8 @@ const RenderModal: React.FC<} | null>(null);const canUseSharedMemoryCapture =!readOnlyStudio &&- renderDefaults.sharedMemoryCapture?.disabled === false &&+ renderDefaults.sharedMemoryCapture !== null &&+ !renderDefaults.sharedMemoryCapture.disabled &&renderMode === 'video' &&!disallowParallelEncoding &&BrowserSafeApis.canUseParallelEncoding(codec);diff --git a/packages/studio/src/test/make-render-command.test.ts b/packages/studio/src/test/make-render-command.test.tsindex f1ab99fdbc..e9e47fe68c 100644--- a/packages/studio/src/test/make-render-command.test.ts+++ b/packages/studio/src/test/make-render-command.test.ts@@ -54,6 +54,7 @@ const getDefaults = (): RenderDefaults => {allowHtmlInCanvas: false,sampleRate: 48000,configFileRenderDefaults: null,+ sharedMemoryCapture: null,};};