Skip to content

🎥 Make videos programmatically with React

Reduce CDP overhead during shared-memory capture

Jonny Burger9f1adfec0fee
diff --git a/packages/browser-studio/src/BrowserStudio.tsx b/packages/browser-studio/src/BrowserStudio.tsx
index 7176c24714..a5e54b9fe0 100644
--- a/packages/browser-studio/src/BrowserStudio.tsx
+++ b/packages/browser-studio/src/BrowserStudio.tsx
@@ -145,6 +145,7 @@ const getBrowserRenderDefaults = (): RenderDefaults => {
repro: false,
sampleRate: 48_000,
scale: 1,
+ sharedMemoryCapture: null,
stillImageFormat: 'png',
userAgent: null,
videoBitrate: null,
diff --git a/packages/renderer/src/browser/Connection.ts b/packages/renderer/src/browser/Connection.ts
index 3ad55f713a..a70bebb9e5 100644
--- a/packages/renderer/src/browser/Connection.ts
+++ b/packages/renderer/src/browser/Connection.ts
@@ -24,9 +24,7 @@ interface ConnectionCallback {
resolve: (value: {value: any; size: number}) => void;
reject: Function;
method: string;
- returnSize: boolean;
- stack: string;
- fn: string;
+ params: object | null;
}
const ConnectionEmittedEvents = {
@@ -74,9 +72,7 @@ export class Connection extends EventEmitter {
resolve,
reject,
method,
- returnSize: true,
- stack: new Error().stack ?? '',
- fn: method + JSON.stringify(params),
+ params: params ?? null,
});
},
);
@@ -130,10 +126,8 @@ export class Connection extends EventEmitter {
if (object.error) {
callback.reject(createProtocolError(callback.method, object));
- } else if (callback.returnSize) {
- callback.resolve({value: object.result, size: message.length});
} else {
- callback.resolve(object.result);
+ callback.resolve({value: object.result, size: message.length});
}
}
} else {
@@ -245,7 +239,10 @@ export class CDPSession extends EventEmitter {
(resolve, reject) => {
if (this.#callbacks.size > 100) {
for (const callback of this.#callbacks.values()) {
- Log.info({indent: false, logLevel: 'info'}, callback.fn);
+ Log.info(
+ {indent: false, logLevel: 'info'},
+ callback.method + JSON.stringify(callback.params),
+ );
}
throw new Error('Leak detected: Too many callbacks');
@@ -255,9 +252,7 @@ export class CDPSession extends EventEmitter {
resolve,
reject,
method,
- returnSize: true,
- stack: new Error().stack ?? '',
- fn: method + JSON.stringify(params),
+ params: params ?? null,
});
},
);
@@ -269,10 +264,8 @@ export class CDPSession extends EventEmitter {
this.#callbacks.delete(object.id);
if (object.error) {
callback.reject(createProtocolError(callback.method, object));
- } else if (callback.returnSize) {
- callback.resolve({value: object.result, size});
} else {
- callback.resolve(object.result);
+ callback.resolve({value: object.result, size});
}
} else {
this.emit(object.method, object.params);
diff --git a/packages/renderer/src/browser/DOMWorld.ts b/packages/renderer/src/browser/DOMWorld.ts
index 51f81cc985..d760d9d8dd 100644
--- a/packages/renderer/src/browser/DOMWorld.ts
+++ b/packages/renderer/src/browser/DOMWorld.ts
@@ -28,7 +28,7 @@ import type {
import type {ExecutionContext} from './ExecutionContext';
import type {Frame} from './FrameManager';
import type {JSHandle} from './JSHandle';
-import {isString} from './util';
+import {isString, valueFromRemoteObject} from './util';
export class DOMWorld {
#frame: Frame;
@@ -227,11 +227,8 @@ class WaitTask {
const runCount = ++this.#runCount;
let success: JSHandle | null = null;
let error: Error | null = null;
- const context = await this.#domWorld.executionContext();
- if (this.#terminated || runCount !== this.#runCount) {
- return;
- }
-
+ const contextPromise = this.#domWorld.executionContext();
+ const context = await contextPromise;
if (this.#terminated || runCount !== this.#runCount) {
return;
}
@@ -256,17 +253,21 @@ class WaitTask {
}
// Ignore timeouts in pageScript - we track timeouts ourselves.
- // If the frame's execution context has already changed, `frame.evaluate` will
- // throw an error - ignore this predicate run altogether.
+ // Primitive results need no further CDP evaluation, but must still belong
+ // to the current execution context. Object handles are checked in the page.
if (
!error &&
- (await this.#domWorld
- .evaluate((s) => {
- return !s;
- }, success)
- .catch(() => {
- return true;
- }))
+ (success && !success._remoteObject.objectId
+ ? !valueFromRemoteObject(success._remoteObject) ||
+ !this.#domWorld._hasContext() ||
+ this.#domWorld.executionContext() !== contextPromise
+ : await this.#domWorld
+ .evaluate((s) => {
+ return !s;
+ }, success)
+ .catch(() => {
+ return true;
+ }))
) {
if (!success) {
throw new Error('Assertion: result handle is not available');
@@ -276,6 +277,11 @@ class WaitTask {
return;
}
+ if (this.#terminated || runCount !== this.#runCount) {
+ await success?.dispose();
+ return;
+ }
+
if (error) {
if (error.message.includes('TypeError: binding is not a function')) {
return this.rerun();
diff --git a/packages/renderer/src/remotion-shared-memory.ts b/packages/renderer/src/remotion-shared-memory.ts
index 47c2a84a2f..79bec23425 100644
--- a/packages/renderer/src/remotion-shared-memory.ts
+++ b/packages/renderer/src/remotion-shared-memory.ts
@@ -11,6 +11,7 @@ import {
import type {LogLevel} from './log-level';
import {Log} from './logger';
import type {PixelFormat} from './pixel-format';
+import {setScreenshotBackground} from './screenshot-background';
const encoderPadding = 64;
const slotsPerPage = 1;
@@ -324,6 +325,14 @@ export class RemotionSharedMemoryCapture {
await new Promise<void>((resolve) => pool.drainWaiters.push(resolve));
}
+ try {
+ await setScreenshotBackground(page, false);
+ } catch (error) {
+ if (!bestEffort && !isTargetClosedErr(error as Error) && !cleanupError) {
+ cleanupError = error;
+ }
+ }
+
try {
await page._client().send('Page.remotionDestroyFramePool');
} catch (error) {
@@ -461,6 +470,7 @@ export class RemotionSharedMemoryCapture {
}
async #destroyPools() {
+ let cleanupError: unknown = null;
for (const pool of this.#pools.values()) {
if (pool.ownedSlots.size > 0) {
throw new Error(
@@ -468,16 +478,27 @@ export class RemotionSharedMemoryCapture {
);
}
+ try {
+ await setScreenshotBackground(pool.page, false);
+ } catch (error) {
+ if (!isTargetClosedErr(error as Error) && !cleanupError) {
+ cleanupError = error;
+ }
+ }
+
try {
await pool.page._client().send('Page.remotionDestroyFramePool');
} catch (error) {
- if (!isTargetClosedErr(error as Error)) {
- throw error;
+ if (!isTargetClosedErr(error as Error) && !cleanupError) {
+ cleanupError = error;
}
}
}
this.#pools.clear();
+ if (cleanupError) {
+ throw cleanupError;
+ }
}
}
diff --git a/packages/renderer/src/screenshot-background.ts b/packages/renderer/src/screenshot-background.ts
new file mode 100644
index 0000000000..be8e618177
--- /dev/null
+++ b/packages/renderer/src/screenshot-background.ts
@@ -0,0 +1,22 @@
+import type {Page} from './browser/BrowserPage';
+
+const transparentBackgrounds = new WeakSet<Page>();
+
+export const setScreenshotBackground = async (
+ page: Page,
+ omitBackground: boolean,
+) => {
+ if (transparentBackgrounds.has(page) === omitBackground) {
+ return;
+ }
+
+ if (omitBackground) {
+ await page._client().send('Emulation.setDefaultBackgroundColorOverride', {
+ color: {r: 0, g: 0, b: 0, a: 0},
+ });
+ transparentBackgrounds.add(page);
+ } else {
+ await page._client().send('Emulation.setDefaultBackgroundColorOverride');
+ transparentBackgrounds.delete(page);
+ }
+};
diff --git a/packages/renderer/src/screenshot-task.ts b/packages/renderer/src/screenshot-task.ts
index b79afc9ea2..eb16f1e722 100644
--- a/packages/renderer/src/screenshot-task.ts
+++ b/packages/renderer/src/screenshot-task.ts
@@ -6,6 +6,7 @@ import type {
CapturedFrame,
RemotionSharedMemoryCapture,
} from './remotion-shared-memory';
+import {setScreenshotBackground} from './screenshot-background';
export const screenshotTask = async ({
format,
@@ -35,11 +36,7 @@ export const screenshotTask = async ({
targetId: target._targetId,
});
- if (omitBackground) {
- await client.send('Emulation.setDefaultBackgroundColorOverride', {
- color: {r: 0, g: 0, b: 0, a: 0},
- });
- }
+ await setScreenshotBackground(page, omitBackground);
const cap = startPerfMeasure('capture');
try {
@@ -104,20 +101,9 @@ export const screenshotTask = async ({
const rawFrame = await reservation.publish(value.remotionFrame);
stopPerfMeasure(cap);
- if (omitBackground) {
- try {
- await client.send('Emulation.setDefaultBackgroundColorOverride');
- } catch (error) {
- try {
- await rawFrame.release();
- } catch {
- // Preserve the background reset error that made the capture fail.
- }
-
- throw error;
- }
- }
+ // Keep the background configured until pool cleanup. Encoded
+ // screenshots restore it below, including thumbnails on this page.
return rawFrame;
}
@@ -126,7 +112,7 @@ export const screenshotTask = async ({
stopPerfMeasure(cap);
if (omitBackground) {
- await client.send('Emulation.setDefaultBackgroundColorOverride');
+ await setScreenshotBackground(page, false);
}
const buffer = Buffer.from(result.data, 'base64');
diff --git a/packages/renderer/src/seek-to-frame.ts b/packages/renderer/src/seek-to-frame.ts
index 98fb3314c1..b9f173bcca 100644
--- a/packages/renderer/src/seek-to-frame.ts
+++ b/packages/renderer/src/seek-to-frame.ts
@@ -211,5 +211,5 @@ export const seekToFrame = async ({
timeoutInMilliseconds,
});
await waitForReady({page, timeoutInMilliseconds, frame, indent, logLevel});
- await page.evaluateHandle('document.fonts.ready');
+ await page.evaluate('document.fonts.ready.then(() => true)');
};
diff --git a/packages/renderer/src/test/remotion-shared-memory.test.ts b/packages/renderer/src/test/remotion-shared-memory.test.ts
index 72c4e853b5..33610423c6 100644
--- a/packages/renderer/src/test/remotion-shared-memory.test.ts
+++ b/packages/renderer/src/test/remotion-shared-memory.test.ts
@@ -320,9 +320,10 @@ test('signals pool retirement before waiting for an unhanded frame to drain', as
expect(calls).toContain('Page.remotionDestroyFramePool');
});
-test('releases a published frame if transparent background reset fails', async () => {
+test('keeps transparent backgrounds between frames and destroys the pool if reset fails', async () => {
let backgroundCalls = 0;
let releaseCalls = 0;
+ let destroyCalls = 0;
const client = {
send: (
method: string,
@@ -370,6 +371,10 @@ test('releases a published frame if transparent background reset fails', async (
releaseCalls++;
}
+ if (method === 'Page.remotionDestroyFramePool') {
+ destroyCalls++;
+ }
+
return Promise.resolve({value: undefined});
},
};
@@ -386,8 +391,8 @@ test('releases a published frame if transparent background reset fails', async (
});
await capture.ensurePage(page);
- await expect(
- screenshotTask({
+ for (let i = 0; i < 2; i++) {
+ const frame = await screenshotTask({
format: 'png',
height: 16,
width: 16,
@@ -396,10 +401,17 @@ test('releases a published frame if transparent background reset fails', async (
jpegQuality: undefined,
scale: 1,
remotionSharedMemory: capture,
- }),
- ).rejects.toThrow('background reset failed');
- expect(releaseCalls).toBe(1);
- await capture.destroy();
+ });
+ expect(Buffer.isBuffer(frame)).toBe(false);
+ if (!Buffer.isBuffer(frame)) {
+ await frame.release();
+ }
+ }
+
+ expect(backgroundCalls).toBe(1);
+ expect(releaseCalls).toBe(2);
+ await expect(capture.destroy()).rejects.toThrow('background reset failed');
+ expect(destroyCalls).toBe(1);
});
test('uses raw viewport capture for every video image format and falls back on vanilla Chrome', async () => {
diff --git a/packages/studio-shared/src/render-defaults.ts b/packages/studio-shared/src/render-defaults.ts
index 03bc1bf174..f48963434d 100644
--- a/packages/studio-shared/src/render-defaults.ts
+++ b/packages/studio-shared/src/render-defaults.ts
@@ -71,11 +71,10 @@ export type RenderDefaults = {
allowHtmlInCanvas: boolean;
sampleRate: number;
configFileRenderDefaults: ConfigFileRenderDefaults | null;
- // Optional to preserve compatibility with custom Studio integrations.
- sharedMemoryCapture?: {
+ sharedMemoryCapture: {
browserExecutable: string | null;
disabled: boolean;
- };
+ } | null;
};
declare global {
diff --git a/packages/studio/src/components/RenderModal/ServerRenderModal.tsx b/packages/studio/src/components/RenderModal/ServerRenderModal.tsx
index ac7315bd91..a777ba595c 100644
--- a/packages/studio/src/components/RenderModal/ServerRenderModal.tsx
+++ b/packages/studio/src/components/RenderModal/ServerRenderModal.tsx
@@ -485,15 +485,13 @@ const RenderModal: React.FC<
const sharedMemoryCaptureRequest = useMemo(
() => ({
browserExecutable:
- renderDefaults.sharedMemoryCapture?.browserExecutable ?? null,
+ renderDefaults.sharedMemoryCapture === null
+ ? null
+ : renderDefaults.sharedMemoryCapture.browserExecutable,
chromeMode,
chromiumOptions,
}),
- [
- renderDefaults.sharedMemoryCapture?.browserExecutable,
- chromeMode,
- chromiumOptions,
- ],
+ [renderDefaults.sharedMemoryCapture, chromeMode, chromiumOptions],
);
const sharedMemoryCaptureRequestKey = JSON.stringify(
sharedMemoryCaptureRequest,
@@ -504,7 +502,8 @@ const RenderModal: React.FC<
} | null>(null);
const canUseSharedMemoryCapture =
!readOnlyStudio &&
- renderDefaults.sharedMemoryCapture?.disabled === false &&
+ renderDefaults.sharedMemoryCapture !== null &&
+ !renderDefaults.sharedMemoryCapture.disabled &&
renderMode === 'video' &&
!disallowParallelEncoding &&
BrowserSafeApis.canUseParallelEncoding(codec);
diff --git a/packages/studio/src/test/make-render-command.test.ts b/packages/studio/src/test/make-render-command.test.ts
index f1ab99fdbc..e9e47fe68c 100644
--- a/packages/studio/src/test/make-render-command.test.ts
+++ b/packages/studio/src/test/make-render-command.test.ts
@@ -54,6 +54,7 @@ const getDefaults = (): RenderDefaults => {
allowHtmlInCanvas: false,
sampleRate: 48000,
configFileRenderDefaults: null,
+ sharedMemoryCapture: null,
};
};