Skip to content

🎥 Make videos programmatically with React

Use shared-memory frame capture on Apple Silicon

Jonny Burger80ec1aad2276
diff --git a/packages/cli/src/render-flows/render.ts b/packages/cli/src/render-flows/render.ts
index 58f851a846..be547afb75 100644
--- a/packages/cli/src/render-flows/render.ts
+++ b/packages/cli/src/render-flows/render.ts
@@ -658,6 +658,8 @@ export const renderVideoFlow = async ({
sampleRate: resolvedSampleRate,
onBrowserLog: null,
onFrameBuffer: null,
+ onFrame: null,
+ remotionSharedMemory: null,
logLevel,
serializedResolvedPropsWithCustomSchema:
NoReactInternals.serializeJSONWithSpecialTypes({
diff --git a/packages/compositor-darwin-arm64/ffmpeg b/packages/compositor-darwin-arm64/ffmpeg
index 998f5cecf5..a4ffbe1b9b 100755
Binary files a/packages/compositor-darwin-arm64/ffmpeg and b/packages/compositor-darwin-arm64/ffmpeg differ
diff --git a/packages/compositor-darwin-arm64/ffprobe b/packages/compositor-darwin-arm64/ffprobe
index 6dde0b0eff..9031c32cdb 100755
Binary files a/packages/compositor-darwin-arm64/ffprobe and b/packages/compositor-darwin-arm64/ffprobe differ
diff --git a/packages/compositor-darwin-arm64/libavcodec.dylib b/packages/compositor-darwin-arm64/libavcodec.dylib
index 4fc2093e35..f2dedf4896 100755
Binary files a/packages/compositor-darwin-arm64/libavcodec.dylib and b/packages/compositor-darwin-arm64/libavcodec.dylib differ
diff --git a/packages/compositor-darwin-arm64/libavdevice.dylib b/packages/compositor-darwin-arm64/libavdevice.dylib
index ac3838c4b2..eb0fc3a21f 100755
Binary files a/packages/compositor-darwin-arm64/libavdevice.dylib and b/packages/compositor-darwin-arm64/libavdevice.dylib differ
diff --git a/packages/compositor-darwin-arm64/libavfilter.dylib b/packages/compositor-darwin-arm64/libavfilter.dylib
index 46503ae772..c8c0f6914c 100755
Binary files a/packages/compositor-darwin-arm64/libavfilter.dylib and b/packages/compositor-darwin-arm64/libavfilter.dylib differ
diff --git a/packages/compositor-darwin-arm64/libavformat.dylib b/packages/compositor-darwin-arm64/libavformat.dylib
index e5fcffa15c..18691f2c2b 100755
Binary files a/packages/compositor-darwin-arm64/libavformat.dylib and b/packages/compositor-darwin-arm64/libavformat.dylib differ
diff --git a/packages/compositor-darwin-arm64/libavutil.dylib b/packages/compositor-darwin-arm64/libavutil.dylib
index 28e7e247b9..3a3754fdd5 100755
Binary files a/packages/compositor-darwin-arm64/libavutil.dylib and b/packages/compositor-darwin-arm64/libavutil.dylib differ
diff --git a/packages/compositor-darwin-arm64/libswresample.dylib b/packages/compositor-darwin-arm64/libswresample.dylib
index 1e91290642..d923e601c7 100755
Binary files a/packages/compositor-darwin-arm64/libswresample.dylib and b/packages/compositor-darwin-arm64/libswresample.dylib differ
diff --git a/packages/compositor-darwin-arm64/libswscale.dylib b/packages/compositor-darwin-arm64/libswscale.dylib
index c708e3df63..d8a99807fc 100755
Binary files a/packages/compositor-darwin-arm64/libswscale.dylib and b/packages/compositor-darwin-arm64/libswscale.dylib differ
diff --git a/packages/renderer/src/browser/BrowserFetcher.ts b/packages/renderer/src/browser/BrowserFetcher.ts
index 35760665e0..41935b33c4 100644
--- a/packages/renderer/src/browser/BrowserFetcher.ts
+++ b/packages/renderer/src/browser/BrowserFetcher.ts
@@ -54,7 +54,7 @@ interface BrowserFetcherRevisionInfo {
local: boolean;
}
-const getPlatform = (): Platform => {
+export const getPlatform = (): Platform => {
const platform = os.platform();
switch (platform) {
case 'darwin':
@@ -82,13 +82,23 @@ const getVersionFilePath = (chromeMode: ChromeMode): string => {
return path.join(downloadsFolder, 'VERSION');
};
-const getExpectedVersion = (
- version: string | null,
- _chromeMode: ChromeMode,
-): string => {
+export const getExpectedVersion = ({
+ version,
+ chromeMode,
+ platform,
+}: {
+ version: string | null;
+ chromeMode: ChromeMode;
+ platform: Platform;
+}): string => {
if (version) {
return version;
}
+
+ if (chromeMode === 'headless-shell' && platform === 'mac-arm64') {
+ return `${TESTED_VERSION}-remotion-v2`;
+ }
+
return TESTED_VERSION;
};
@@ -129,7 +139,7 @@ export const downloadBrowser = async ({
const downloadsFolder = getDownloadsFolder(chromeMode);
const archivePath = path.join(downloadsFolder, fileName);
const outputPath = getFolderPath(downloadsFolder, platform);
- const expectedVersion = getExpectedVersion(version, chromeMode);
+ const expectedVersion = getExpectedVersion({version, chromeMode, platform});
if (await existsAsync(outputPath)) {
const installedVersion = readVersionFile(chromeMode);
diff --git a/packages/renderer/src/browser/devtools-commands.ts b/packages/renderer/src/browser/devtools-commands.ts
index f468648d5d..63fc8e37af 100644
--- a/packages/renderer/src/browser/devtools-commands.ts
+++ b/packages/renderer/src/browser/devtools-commands.ts
@@ -19,6 +19,9 @@ import type {
NavigateResponse,
PrintPDFRequest,
PrintPDFResponse,
+ RemotionCreateFramePoolRequest,
+ RemotionCreateFramePoolResponse,
+ RemotionReleaseFrameRequest,
ReleaseObjectRequest,
SetAutoAttachRequest,
SetDefaultBackgroundColorOverrideRequest,
@@ -243,6 +246,18 @@ export interface Commands {
paramsType: [CaptureScreenshotRequest?];
returnType: CaptureScreenshotResponse;
};
+ 'Page.remotionCreateFramePool': {
+ paramsType: [RemotionCreateFramePoolRequest];
+ returnType: RemotionCreateFramePoolResponse;
+ };
+ 'Page.remotionReleaseFrame': {
+ paramsType: [RemotionReleaseFrameRequest];
+ returnType: void;
+ };
+ 'Page.remotionDestroyFramePool': {
+ paramsType: [];
+ returnType: void;
+ };
'Page.printToPDF': {
paramsType: [PrintPDFRequest];
returnType: PrintPDFResponse;
diff --git a/packages/renderer/src/browser/devtools-types.ts b/packages/renderer/src/browser/devtools-types.ts
index 06837d5056..d3653a15ac 100644
--- a/packages/renderer/src/browser/devtools-types.ts
+++ b/packages/renderer/src/browser/devtools-types.ts
@@ -733,6 +733,21 @@ export interface CaptureScreenshotRequest {
* Optimize image encoding for speed, not for resulting size (defaults to false) EXPERIMENTAL
*/
optimizeForSpeed?: boolean;
+ /**
+ * Writes the captured pixels into this slot of a Remotion shared-memory
+ * frame pool instead of encoding an image. Remotion Chromium only.
+ */
+ remotionFrameSlot?: number;
+ /**
+ * Ownership token for the shared-memory slot. Must be supplied together
+ * with `remotionFrameSlot`. Remotion Chromium only.
+ */
+ remotionFrameId?: string;
+ /**
+ * Skips the temporary viewport resize used by the regular CDP screenshot
+ * path. Remotion Chromium only.
+ */
+ remotionFastViewport?: boolean;
}
export interface CaptureScreenshotResponse {
@@ -740,6 +755,36 @@ export interface CaptureScreenshotResponse {
* Base64-encoded image data. (Encoded as a base64 string when passed over JSON)
*/
data: string;
+ remotionFrame?: RemotionRawFrame;
+}
+
+export interface RemotionRawFrame {
+ slot: number;
+ frameId: string;
+ width: number;
+ height: number;
+ stride: number;
+ byteLength: number;
+ pixelFormat: 'bgra';
+ alphaType: 'opaque' | 'straight';
+ colorSpace: 'srgb';
+ capturePath: string;
+}
+
+export interface RemotionCreateFramePoolRequest {
+ slotCount: number;
+ slotCapacity: number;
+}
+
+export interface RemotionCreateFramePoolResponse {
+ sharedMemoryName: string;
+ slotCount: number;
+ slotCapacity: number;
+}
+
+export interface RemotionReleaseFrameRequest {
+ slot: number;
+ frameId: string;
}
export interface PrintPDFRequest {
diff --git a/packages/renderer/src/browser/get-chrome-download-url.ts b/packages/renderer/src/browser/get-chrome-download-url.ts
index bd7b816685..b80ec92a50 100644
--- a/packages/renderer/src/browser/get-chrome-download-url.ts
+++ b/packages/renderer/src/browser/get-chrome-download-url.ts
@@ -129,6 +129,10 @@ export function getChromeDownloadUrl({
}
if (chromeMode === 'headless-shell') {
+ if (platform === 'mac-arm64' && version === null) {
+ return `https://remotion.media/chromium-headless-shell-mac-arm64-${TESTED_VERSION}.zip?clear`;
+ }
+
// Amazon Linux 2023 needs a special build.
// This binary is compatible with older glibc (no 2.35 requirement).
if (isAmazonLinux2023() && platform === 'linux64' && !version) {
diff --git a/packages/renderer/src/ensure-browser.ts b/packages/renderer/src/ensure-browser.ts
index 1202b79d91..bd1c054a43 100644
--- a/packages/renderer/src/ensure-browser.ts
+++ b/packages/renderer/src/ensure-browser.ts
@@ -3,6 +3,8 @@ import type {BrowserExecutable} from './browser-executable';
import {defaultBrowserDownloadProgress} from './browser/browser-download-progress-bar';
import {
downloadBrowser,
+ getExpectedVersion,
+ getPlatform,
getRevisionInfo,
readVersionFile,
TESTED_VERSION,
@@ -50,7 +52,26 @@ const internalEnsureBrowserUncapped = async ({
onBrowserDownload,
chromeMode,
}: InternalEnsureBrowserOptions): Promise<BrowserStatus> => {
- const status = getBrowserStatus({browserExecutable, chromeMode});
+ const defaultStatus = getBrowserStatus({
+ browserExecutable,
+ chromeMode,
+ version: null,
+ });
+ if (
+ defaultStatus.type === 'local-puppeteer-browser' ||
+ defaultStatus.type === 'user-defined-path'
+ ) {
+ return defaultStatus;
+ }
+
+ const {onProgress, version} = onBrowserDownload({chromeMode});
+ const status = version
+ ? getBrowserStatus({browserExecutable, chromeMode, version})
+ : defaultStatus;
+ if (status.type === 'local-puppeteer-browser') {
+ return status;
+ }
+
if (status.type === 'version-mismatch') {
const versionInfo = status.actualVersion
? ` (installed: ${status.actualVersion})`
@@ -62,12 +83,10 @@ const internalEnsureBrowserUncapped = async ({
}
if (status.type === 'no-browser' || status.type === 'version-mismatch') {
- const {onProgress, version} = onBrowserDownload({chromeMode});
-
await downloadBrowser({indent, logLevel, onProgress, version, chromeMode});
}
- const newStatus = getBrowserStatus({browserExecutable, chromeMode});
+ const newStatus = getBrowserStatus({browserExecutable, chromeMode, version});
return newStatus;
};
@@ -83,9 +102,11 @@ export const internalEnsureBrowser = (
const getBrowserStatus = ({
browserExecutable,
chromeMode,
+ version,
}: {
browserExecutable: BrowserExecutable;
chromeMode: ChromeMode;
+ version: string | null;
}): BrowserStatus => {
if (browserExecutable) {
if (!fs.existsSync(browserExecutable)) {
@@ -100,7 +121,14 @@ const getBrowserStatus = ({
const revision = getRevisionInfo(chromeMode);
if (revision.local && fs.existsSync(revision.executablePath)) {
const actualVersion = readVersionFile(chromeMode);
- if (actualVersion === TESTED_VERSION) {
+ if (
+ actualVersion ===
+ getExpectedVersion({
+ version,
+ chromeMode,
+ platform: getPlatform(),
+ })
+ ) {
return {path: revision.executablePath, type: 'local-puppeteer-browser'};
}
diff --git a/packages/renderer/src/open-browser.ts b/packages/renderer/src/open-browser.ts
index 0816a6990f..c7137ec61e 100644
--- a/packages/renderer/src/open-browser.ts
+++ b/packages/renderer/src/open-browser.ts
@@ -271,6 +271,7 @@ export const internalOpenBrowser = async ({
chromiumOptions?.userAgent
? `--user-agent="${chromiumOptions.userAgent}"`
: null,
+ '--remotion-shared-memory-capture',
'--remote-debugging-port=0',
`--user-data-dir=${userDataDir}`,
].filter(Boolean) as string[],
diff --git a/packages/renderer/src/prespawn-ffmpeg.ts b/packages/renderer/src/prespawn-ffmpeg.ts
index a91dbddd4a..8f8977c1a3 100644
--- a/packages/renderer/src/prespawn-ffmpeg.ts
+++ b/packages/renderer/src/prespawn-ffmpeg.ts
@@ -1,3 +1,4 @@
+import type {Readable, Writable} from 'node:stream';
import type {_InternalTypes} from 'remotion';
import type {OnLog} from './browser/BrowserPage';
import {callFf} from './call-ffmpeg';
@@ -20,8 +21,11 @@ import {
validateSelectedPixelFormatAndCodecCombination,
} from './pixel-format';
import {resolveHardwareAcceleration} from './probe-encoder';
+import type {RemotionRawFrame} from './remotion-shared-memory';
+import {createRemotionSharedMemoryFfmpegBridge} from './remotion-shared-memory-ffmpeg';
import {validateDimension, validateFps} from './validate';
import {validateEvenDimensionsWithCodec} from './validate-even-dimensions-with-codec';
+import {writeWithBackpressure} from './write-with-backpressure';
type RunningStatus =
| {
@@ -62,6 +66,63 @@ type PreStitcherOptions = {
binariesDirectory: string | null;
hardwareAcceleration: HardwareAccelerationOption;
onLog: OnLog;
+ inputMode: 'encoded-image' | 'remotion-shared-memory';
+};
+
+const remotionSharedMemorySupport = new Map<string, Promise<boolean>>();
+
+export const ffmpegSupportsRemotionSharedMemory = ({
+ binariesDirectory,
+ indent,
+ logLevel,
+}: {
+ binariesDirectory: string | null;
+ indent: boolean;
+ logLevel: LogLevel;
+}) => {
+ const key = binariesDirectory ?? '<bundled>';
+ const cached = remotionSharedMemorySupport.get(key);
+ if (cached) {
+ return cached;
+ }
+
+ const result = (async () => {
+ try {
+ const task = callFf({
+ bin: 'ffmpeg',
+ args: ['-hide_banner', '-devices'],
+ indent,
+ logLevel,
+ binariesDirectory,
+ cancelSignal: undefined,
+ });
+ const output = await task;
+ return /(?:^|\n)\s*D\s{2}remotionshm(?:\s|$)/.test(
+ `${output.stdout}\n${output.stderr}`,
+ );
+ } catch {
+ return false;
+ }
+ })();
+ remotionSharedMemorySupport.set(key, result);
+ return result;
+};
+
+const fpsAsFraction = (fps: number) => {
+ const denominator = 1_000_000;
+ let numerator = Math.round(fps * denominator);
+ let divisor = denominator;
+ let left = numerator;
+ let right = divisor;
+ while (right !== 0) {
+ const remainder = left % right;
+ left = right;
+ right = remainder;
+ }
+
+ numerator /= left;
+ divisor /= left;
+ return `${numerator}/${divisor}`;
};
export const prespawnFfmpeg = (options: PreStitcherOptions) => {
@@ -104,32 +165,58 @@ export const prespawnFfmpeg = (options: PreStitcherOptions) => {
onLog: options.onLog,
});
+ const encodingArgs = generateFfmpegArgs({
+ hasPreencoded: false,
+ proResProfileName,
+ pixelFormat,
+ x264Preset: options.x264Preset,
+ gopSize: options.gopSize,
+ codec,
+ crf: options.crf,
+ videoBitrate: options.videoBitrate,
+ encodingMaxRate: options.encodingMaxRate,
+ encodingBufferSize: options.encodingBufferSize,
+ colorSpace: options.colorSpace,
+ hardwareAcceleration: resolvedHardwareAcceleration,
+ indent: options.indent,
+ logLevel: options.logLevel,
+ });
+ const encodingArgsWithOwnedPixels =
+ options.inputMode === 'remotion-shared-memory'
+ ? encodingArgs.some(([flag]) => flag === '-vf')
+ ? encodingArgs.map((args) =>
+ args[0] === '-vf' ? ['-vf', `copy,${args[1]}`] : args,
+ )
+ : [...encodingArgs, ['-vf', 'copy']]
+ : encodingArgs;
+
const ffmpegArgs = [
- ['-r', options.fps],
- ...[
- ['-f', 'image2pipe'],
- ['-s', `${options.width}x${options.height}`],
- // If scale is very small (like 0.1), FFMPEG cannot figure out the image
- // format on it's own and we need to hint the format
- ['-vcodec', options.imageFormat === 'jpeg' ? 'mjpeg' : 'png'],
- ['-i', '-'],
- ],
- ...generateFfmpegArgs({
- hasPreencoded: false,
- proResProfileName,
- pixelFormat,
- x264Preset: options.x264Preset,
- gopSize: options.gopSize,
- codec,
- crf: options.crf,
- videoBitrate: options.videoBitrate,
- encodingMaxRate: options.encodingMaxRate,
- encodingBufferSize: options.encodingBufferSize,
- colorSpace: options.colorSpace,
- hardwareAcceleration: resolvedHardwareAcceleration,
- indent: options.indent,
- logLevel: options.logLevel,
- }),
+ ...(options.inputMode === 'remotion-shared-memory'
+ ? [
+ ['-nostdin'],
+ ['-xerror'],
+ ['-nofind_stream_info'],
+ ['-threads:v', '1'],
+ ['-f', 'remotionshm'],
+ ['-video_size', `${options.width}x${options.height}`],
+ ['-framerate', fpsAsFraction(options.fps)],
+ ['-control_fd', '3'],
+ ['-ack_fd', '4'],
+ ['-i', 'remotion'],
+ ]
+ : [
+ ['-r', options.fps],
+ ['-f', 'image2pipe'],
+ ['-s', `${options.width}x${options.height}`],
+ // If scale is very small (like 0.1), FFMPEG cannot figure out the image
+ // format on its own and we need to hint the format.
+ ['-vcodec', options.imageFormat === 'jpeg' ? 'mjpeg' : 'png'],
+ ['-i', '-'],
+ ]),
+ ...encodingArgsWithOwnedPixels,
+ options.inputMode === 'remotion-shared-memory'
+ ? ['-fps_mode', 'passthrough']
+ : null,
'-y',
options.outputLocation,
@@ -164,7 +251,18 @@ export const prespawnFfmpeg = (options: PreStitcherOptions) => {
logLevel: options.logLevel,
binariesDirectory: options.binariesDirectory,
cancelSignal: options.signal,
+ options:
+ options.inputMode === 'remotion-shared-memory'
+ ? {stdio: ['ignore', 'pipe', 'pipe', 'pipe', 'pipe']}
+ : undefined,
});
+ const remotionSharedMemory =
+ options.inputMode === 'remotion-shared-memory'
+ ? createRemotionSharedMemoryFfmpegBridge({
+ control: task.stdio[3] as Writable,
+ acknowledgements: task.stdio[4] as Readable,
+ })
+ : null;
let ffmpegOutput = '';
task.stderr?.on('data', (data: Buffer) => {
@@ -198,5 +296,46 @@ export const prespawnFfmpeg = (options: PreStitcherOptions) => {
}
});
- return {task, getLogs: () => ffmpegOutput, getExitStatus: () => exitCode};
+ return {
+ task,
+ getLogs: () => ffmpegOutput,
+ getExitStatus: () => exitCode,
+ writeFrame: async ({
+ frame,
+ pts,
+ }: {
+ frame: Buffer | RemotionRawFrame;
+ pts: number;
+ }) => {
+ if (remotionSharedMemory) {
+ if (Buffer.isBuffer(frame)) {
+ throw new Error('Expected a Remotion shared-memory frame.');
+ }
+
+ return remotionSharedMemory.writeFrame({frame, pts});
+ }
+
+ if (!Buffer.isBuffer(frame)) {
+ throw new Error('Expected an encoded image frame.');
+ }
+
+ if (!task.stdin) {
+ throw new Error('FFmpeg stdin is not available.');
+ }
+
+ await writeWithBackpressure({data: frame, writable: task.stdin});
+ return {waitForAck: Promise.resolve()};
+ },
+ finishInput: async () => {
+ if (remotionSharedMemory) {
+ await remotionSharedMemory.finish();
+ return;
+ }
+
+ task.stdin?.end();
+ },
+ abortInput: (error: Error) => {
+ remotionSharedMemory?.fail(error);
+ },
+ };
};
diff --git a/packages/renderer/src/puppeteer-screenshot.ts b/packages/renderer/src/puppeteer-screenshot.ts
index 4128c900d7..6ee956848b 100644
--- a/packages/renderer/src/puppeteer-screenshot.ts
+++ b/packages/renderer/src/puppeteer-screenshot.ts
@@ -1,6 +1,10 @@
import * as assert from 'node:assert';
import type {Page} from './browser/BrowserPage';
import type {StillImageFormat} from './image-format';
+import type {
+ CapturedFrame,
+ RemotionSharedMemoryCapture,
+} from './remotion-shared-memory';
import {screenshotTask} from './screenshot-task';
export const screenshot = (options: {
@@ -12,7 +16,8 @@ export const screenshot = (options: {
width: number;
height: number;
scale: number;
-}): Promise<Buffer> => {
+ remotionSharedMemory: RemotionSharedMemoryCapture | null;
+}): Promise<CapturedFrame> => {
if (options.jpegQuality) {
assert.ok(
typeof options.jpegQuality === 'number',
@@ -40,6 +45,7 @@ export const screenshot = (options: {
path: options.path,
jpegQuality: options.type === 'jpeg' ? options.jpegQuality : undefined,
scale: options.scale,
+ remotionSharedMemory: options.remotionSharedMemory,
}),
);
};
diff --git a/packages/renderer/src/remotion-shared-memory-ffmpeg.ts b/packages/renderer/src/remotion-shared-memory-ffmpeg.ts
new file mode 100644
index 0000000000..caee71e5ee
--- /dev/null
+++ b/packages/renderer/src/remotion-shared-memory-ffmpeg.ts
@@ -0,0 +1,421 @@
+import {createInterface} from 'node:readline';
+import type {Readable, Writable} from 'node:stream';
+import type {
+ RemotionRawFrame,
+ RemotionSharedMemoryPoolLifetime,
+} from './remotion-shared-memory';
+import {writeWithBackpressure} from './write-with-backpressure';
+
+type RegisteredPool = {
+ poolId: number;
+ sharedMemoryName: string;
+ poolLifetime: RemotionSharedMemoryPoolLifetime;
+ registered: boolean;
+ registrationStarted: boolean;
+ registrationSettled: boolean;
+ registration: Promise<boolean>;
+ resolveRegistration: (registered: boolean) => void;
+ retired: boolean;
+ pendingCount: number;
+ drainWaiters: Array<() => void>;
+ unregistering: Promise<void> | null;
+ unsubscribe: (() => void) | null;
+};
+
+type PendingFrame = {
+ frame: RemotionRawFrame;
+ pool: RegisteredPool;
+ resolve: () => void;
+ reject: (error: Error) => void;
+};
+
+const pendingKey = (poolId: number, slot: number) => `${poolId}:${slot}`;
+
+export const serializeRemotionSharedMemoryPool = (frame: RemotionRawFrame) =>
+ `P\t${frame.poolId}\t${frame.sharedMemoryName}\t${frame.slotCount}\t${frame.slotCapacity}\n`;
+
+export const serializeRemotionSharedMemoryFrame = ({
+ frame,
+ pts,
+}: {
+ frame: RemotionRawFrame;
+ pts: number;
+}) =>
+ `F\t${frame.poolId}\t${frame.slot}\t${frame.frameId}\t${frame.width}\t${frame.height}\t${frame.stride}\t${frame.byteLength}\t${pts}\n`;
+
+export const serializeRemotionSharedMemoryPoolRetirement = (poolId: number) =>
+ `U\t${poolId}\n`;
+
+export const parseRemotionSharedMemoryAck = (line: string) => {
+ const match = /^(\d+)\t(\d+)\t(\d+)$/.exec(line);
+ if (!match) {
+ throw new Error(`Invalid Remotion shared-memory ACK: ${line}`);
+ }
+
+ const poolId = Number(match[1]);
+ const slot = Number(match[2]);
+ if (
+ !Number.isSafeInteger(poolId) ||
+ poolId < 1 ||
+ !Number.isSafeInteger(slot) ||
+ slot < 0
+ ) {
+ throw new Error(`Invalid Remotion shared-memory ACK: ${line}`);
+ }
+
+ return {poolId, slot, frameId: match[3]};
+};
+
+export const createRemotionSharedMemoryFfmpegBridge = ({
+ control,
+ acknowledgements,
+}: {
+ control: Writable;
+ acknowledgements: Readable;
+}) => {
+ const registeredPools = new Map<number, RegisteredPool>();
+ const pending = new Map<string, PendingFrame>();
+ let fatalError: Error | null = null;
+ let finishing = false;
+ let controlEnded = false;
+ let controlWrites = Promise.resolve();
+ let finishPromise: Promise<void> | null = null;
+
+ const toError = (error: unknown) =>
+ error instanceof Error ? error : new Error(String(error));
+
+ const releaseIgnoringErrors = async (frame: RemotionRawFrame) => {
+ try {
+ await frame.release();
+ } catch {
+ // The original bridge error is more useful. release() still abandons the
+ // local slot in a finally block if the CDP session is unavailable.
+ }
+ };
+
+ const settleRegistration = (pool: RegisteredPool, registered: boolean) => {
+ if (pool.registrationSettled) {
+ return;
+ }
+
+ pool.registrationSettled = true;
+ pool.registered = registered;
+ pool.resolveRegistration(registered);
+ };
+
+ const signalPoolDrained = (pool: RegisteredPool) => {
+ if (pool.pendingCount === 0) {
+ pool.drainWaiters.splice(0).forEach((waiter) => waiter());
+ }
+ };
+
+ const removePending = (key: string, value: PendingFrame) => {
+ if (pending.get(key) !== value) {
+ return false;
+ }
+
+ pending.delete(key);
+ value.pool.pendingCount--;
+ signalPoolDrained(value.pool);
+ return true;
+ };
+
+ const fail = (error: Error) => {
+ if (fatalError) {
+ return;
+ }
+
+ fatalError = error;
+ for (const pool of registeredPools.values()) {
+ settleRegistration(pool, false);
+ }
+
+ for (const [key, value] of [...pending.entries()]) {
+ if (removePending(key, value)) {
+ value.reject(error);
+ releaseIgnoringErrors(value.frame).catch(() => undefined);
+ }
+ }
+ };
+
+ const enqueueControlWrite = (data: string) => {
+ if (finishing || controlEnded) {
+ return Promise.reject(
+ new Error('FFmpeg shared-memory input has already been finished.'),
+ );
+ }
+
+ const operation = controlWrites.then(async () => {
+ if (fatalError) {
+ throw fatalError;
+ }
+
+ await writeWithBackpressure({
+ data: Buffer.from(data),
+ writable: control,
+ });
+ });
+ controlWrites = operation;
+ return operation;
+ };
+
+ const unregisterPool = (pool: RegisteredPool) => {
+ pool.retired = true;
+ if (pool.unregistering) {
+ return pool.unregistering;
+ }
+
+ pool.unregistering = (async () => {
+ try {
+ const wasRegistered = await pool.registration;
+ if (!wasRegistered || fatalError || finishing || controlEnded) {
+ return;
+ }
+
+ if (pool.pendingCount > 0) {
+ await new Promise<void>((resolve) => pool.drainWaiters.push(resolve));
+ }
+
+ if (fatalError || finishing || controlEnded) {
+ return;
+ }
+
+ try {
+ await enqueueControlWrite(
+ serializeRemotionSharedMemoryPoolRetirement(pool.poolId),
+ );
+ } catch (error) {
+ const normalizedError = toError(error);
+ fail(normalizedError);
+ throw normalizedError;
+ }
+ } finally {
+ pool.unsubscribe?.();
+ pool.unsubscribe = null;
+ if (registeredPools.get(pool.poolId) === pool) {
+ registeredPools.delete(pool.poolId);
+ }
+ }
+ })();
+ return pool.unregistering;
+ };
+
+ const createRegisteredPool = (frame: RemotionRawFrame) => {
+ let resolveRegistration!: (registered: boolean) => void;
+ const registration = new Promise<boolean>((resolve) => {
+ resolveRegistration = resolve;
+ });
+ const pool: RegisteredPool = {
+ poolId: frame.poolId,
+ sharedMemoryName: frame.sharedMemoryName,
+ poolLifetime: frame.poolLifetime,
+ registered: false,
+ registrationStarted: false,
+ registrationSettled: false,
+ registration,
+ resolveRegistration,
+ retired: false,
+ pendingCount: 0,
+ drainWaiters: [],
+ unregistering: null,
+ unsubscribe: null,
+ };
+ const unsubscribe = frame.poolLifetime.onRetired(() =>
+ unregisterPool(pool),
+ );
+ if (!unsubscribe) {
+ return null;
+ }
+
+ pool.unsubscribe = unsubscribe;
+ registeredPools.set(frame.poolId, pool);
+ return pool;
+ };
+
+ const lines = createInterface({input: acknowledgements});
+ lines.on('line', (line) => {
+ (async () => {
+ const ack = parseRemotionSharedMemoryAck(line);
+ const key = pendingKey(ack.poolId, ack.slot);
+ const value = pending.get(key);
+ if (!value || value.frame.frameId !== ack.frameId) {
+ throw new Error(
+ `Unexpected Remotion shared-memory ACK for ${ack.poolId}:${ack.slot}:${ack.frameId}.`,
+ );
+ }
+
+ try {
+ await value.frame.release();
+ } catch (error) {
+ const normalizedError = toError(error);
+ if (removePending(key, value)) {
+ value.reject(normalizedError);
+ fail(normalizedError);
+ }
+
+ return;
+ }
+
+ if (removePending(key, value)) {
+ value.resolve();
+ }
+ })().catch((error) => fail(toError(error)));
+ });
+ lines.on('error', (error) => fail(toError(error)));
+ lines.on('close', () => {
+ if (pending.size > 0 || !finishing) {
+ fail(
+ new Error(
+ `FFmpeg closed its Remotion shared-memory ACK pipe${pending.size > 0 ? ` with ${pending.size} frame(s) pending` : ' before input finished'}.`,
+ ),
+ );
+ }
+ });
+ acknowledgements.on('error', (error) => fail(toError(error)));
+ acknowledgements.on('close', () => {
+ if (pending.size > 0 || !finishing) {
+ fail(
+ new Error(
+ `FFmpeg closed its Remotion shared-memory ACK pipe${pending.size > 0 ? ` with ${pending.size} frame(s) pending` : ' before input finished'}.`,
+ ),
+ );
+ }
+ });
+ control.on('error', (error) => fail(toError(error)));
+
+ return {
+ writeFrame: async ({
+ frame,
+ pts,
+ }: {
+ frame: RemotionRawFrame;
+ pts: number;
+ }) => {
+ if (fatalError) {
+ await releaseIgnoringErrors(frame);
+ throw fatalError;
+ }
+
+ if (finishing || controlEnded || frame.poolLifetime.isRetired()) {
+ await releaseIgnoringErrors(frame);
+ throw new Error(
+ `Target closed: Remotion shared-memory pool ${frame.poolId} has already been retired.`,
+ );
+ }
+
+ let pool = registeredPools.get(frame.poolId);
+ if (!pool) {
+ const createdPool = createRegisteredPool(frame);
+ if (!createdPool) {
+ await releaseIgnoringErrors(frame);
+ throw new Error(
+ `Target closed: Remotion shared-memory pool ${frame.poolId} has already been retired.`,
+ );
+ }
+
+ pool = createdPool;
+ } else if (
+ pool.sharedMemoryName !== frame.sharedMemoryName ||
+ pool.poolLifetime !== frame.poolLifetime
+ ) {
+ const error = new Error(
+ `Remotion shared-memory pool ${frame.poolId} changed its identity.`,
+ );
+ await releaseIgnoringErrors(frame);
+ fail(error);
+ throw error;
+ }
+
+ if (pool.retired || frame.poolLifetime.isRetired()) {
+ await releaseIgnoringErrors(frame);
+ throw new Error(
+ `Target closed: Remotion shared-memory pool ${frame.poolId} has already been retired.`,
+ );
+ }
+
+ const key = pendingKey(frame.poolId, frame.slot);
+ if (pending.has(key)) {
+ const error = new Error(
+ `Remotion shared-memory slot ${frame.poolId}:${frame.slot} was reused before its ACK.`,
+ );
+ await releaseIgnoringErrors(frame);
+ fail(error);
+ throw error;
+ }
+
+ let resolveAck!: () => void;
+ let rejectAck!: (error: Error) => void;
+ const ack = new Promise<void>((resolve, reject) => {
+ resolveAck = resolve;
+ rejectAck = reject;
+ });
+ ack.catch(() => undefined);
+ const pendingFrame: PendingFrame = {
+ frame,
+ pool,
+ resolve: resolveAck,
+ reject: rejectAck,
+ };
+ pending.set(key, pendingFrame);
+ pool.pendingCount++;
+
+ const registersPool = !pool.registrationStarted;
+ if (registersPool) {
+ pool.registrationStarted = true;
+ }
+
+ const operation = enqueueControlWrite(
+ (registersPool ? serializeRemotionSharedMemoryPool(frame) : '') +
+ serializeRemotionSharedMemoryFrame({frame, pts}),
+ );
+ if (registersPool) {
+ operation.then(
+ () => settleRegistration(pool, true),
+ () => settleRegistration(pool, false),
+ );
+ }
+
+ try {
+ await operation;
+ } catch (error) {
+ const normalizedError = toError(error);
+ fail(normalizedError);
+ throw normalizedError;
+ }
+
+ return {waitForAck: ack};
+ },
+ finish: () => {
+ if (finishPromise) {
+ return finishPromise;
+ }
+
+ finishing = true;
+ finishPromise = (async () => {
+ try {
+ await controlWrites;
+ if (fatalError) {
+ throw fatalError;
+ }
+
+ await new Promise<void>((resolve, reject) => {
+ control.end((error?: Error | null) => {
+ if (error) {
+ reject(error);
+ } else {
+ resolve();
+ }
+ });
+ });
+ controlEnded = true;
+ } catch (error) {
+ const normalizedError = toError(error);
+ fail(normalizedError);
+ throw normalizedError;
+ }
+ })();
+ return finishPromise;
+ },
+ fail,
+ };
+};
diff --git a/packages/renderer/src/remotion-shared-memory.ts b/packages/renderer/src/remotion-shared-memory.ts
new file mode 100644
index 0000000000..90b6402dd0
--- /dev/null
+++ b/packages/renderer/src/remotion-shared-memory.ts
@@ -0,0 +1,408 @@
+import type {Page} from './browser/BrowserPage';
+import type {RemotionRawFrame as CdpRemotionRawFrame} from './browser/devtools-types';
+import {isTargetClosedErr} from './browser/flaky-errors';
+import type {LogLevel} from './log-level';
+import {Log} from './logger';
+
+const encoderPadding = 64;
+const slotsPerPage = 1;
+
+type FramePool = {
+ page: Page;
+ poolId: number;
+ sharedMemoryName: string;
+ slotCount: number;
+ slotCapacity: number;
+ availableSlots: number[];
+ waiters: Array<() => void>;
+ drainWaiters: Array<() => void>;
+ ownedSlots: Map<number, string>;
+ forgotten: boolean;
+ poolLifetime: ManagedRemotionSharedMemoryPoolLifetime;
+};
+
+export type RemotionSharedMemoryPoolLifetime = {
+ isRetired: () => boolean;
+ waitForRetirement: () => Promise<void>;
+ onRetired: (listener: () => Promise<void>) => (() => void) | null;
+};
+
+type ManagedRemotionSharedMemoryPoolLifetime =
+ RemotionSharedMemoryPoolLifetime & {
+ retire: () => Promise<void>;
+ };
+
+export type RemotionRawFrame = CdpRemotionRawFrame & {
+ type: 'remotion-shared-memory';
+ poolId: number;
+ sharedMemoryName: string;
+ slotCount: number;
+ slotCapacity: number;
+ poolLifetime: RemotionSharedMemoryPoolLifetime;
+ release: () => Promise<void>;
+};
+
+export type CapturedFrame = Buffer | RemotionRawFrame;
+
+const isExpectedCapabilityError = (error: unknown) => {
+ const message = error instanceof Error ? error.message : String(error);
+ return (
+ message.includes("'Page.remotionCreateFramePool' wasn't found") ||
+ message.includes('Method not found') ||
+ message.includes('Raw frames require') ||
+ message.includes('Raw frames are only supported') ||
+ message.includes('Could not create Remotion shared memory') ||
+ message.includes('Could not size Remotion shared memory') ||
+ message.includes('Could not map Remotion shared memory') ||
+ message.includes('Invalid Remotion frame pool dimensions')
+ );
+};
+
+export class RemotionSharedMemoryCapture {
+ #pools = new Map<Page, FramePool>();
+ #nextPoolId = 1;
+ #nextFrameId = 1;
+ #supported: boolean | null = null;
+ #initialization: Promise<void> | null = null;
+ readonly #options: {
+ width: number;
+ height: number;
+ indent: boolean;
+ logLevel: LogLevel;
+ };
+
+ constructor(options: {
+ width: number;
+ height: number;
+ indent: boolean;
+ logLevel: LogLevel;
+ }) {
+ this.#options = options;
+ }
+
+ isSupported() {
+ return this.#supported === true;
+ }
+
+ async initializePages(pages: Page[]) {
+ if (this.#initialization) {
+ await this.#initialization;
+ if (this.#supported) {
+ for (const page of pages) {
+ await this.ensurePage(page);
+ }
+ }
+
+ return;
+ }
+
+ this.#initialization = (async () => {
+ try {
+ for (const page of pages) {
+ await this.#createPool(page);
+ }
+
+ this.#supported = true;
+ } catch (error) {
+ await this.#destroyPools();
+ if (!isExpectedCapabilityError(error)) {
+ throw error;
+ }
+
+ this.#supported = false;
+ Log.verbose(
+ {indent: this.#options.indent, logLevel: this.#options.logLevel},
+ 'Remotion shared-memory capture is unavailable. Falling back to encoded screenshots.',
+ );
+ }
+ })();
+
+ await this.#initialization;
+ }
+
+ async ensurePage(page: Page) {
+ if (this.#pools.has(page) || this.#supported === false) {
+ return;
+ }
+
+ if (this.#supported === null) {
+ await this.initializePages([page]);
+ return;
+ }
+
+ await this.#createPool(page);
+ }
+
+ async acquire(page: Page) {
+ if (!this.#supported) {
+ return null;
+ }
+
+ const pool = this.#pools.get(page);
+ if (!pool) {
+ throw new Error('No Remotion shared-memory pool exists for this page.');
+ }
+
+ while (pool.availableSlots.length === 0) {
+ if (pool.forgotten) {
+ throw new Error('The page owning this shared-memory pool was closed.');
+ }
+
+ await new Promise<void>((resolve) => pool.waiters.push(resolve));
+ }
+
+ const slot = pool.availableSlots.shift();
+ if (slot === undefined) {
+ throw new Error('Expected an available Remotion shared-memory slot.');
+ }
+
+ if (!Number.isSafeInteger(this.#nextFrameId)) {
+ throw new Error(
+ 'Remotion shared-memory frame ID exceeded the safe range.',
+ );
+ }
+
+ const frameId = String(this.#nextFrameId++);
+ pool.ownedSlots.set(slot, frameId);
+ let published = false;
+ let releasePromise: Promise<void> | null = null;
+ const releasePublished = () => {
+ releasePromise ??= (async () => {
+ try {
+ await page._client().send('Page.remotionReleaseFrame', {
+ slot,
+ frameId,
+ });
+ } catch (error) {
+ if (!isTargetClosedErr(error as Error)) {
+ throw error;
+ }
+ } finally {
+ this.#returnSlot(pool, slot, frameId);
+ }
+ })();
+ return releasePromise;
+ };
+
+ return {
+ slot,
+ frameId,
+ fail: () => {
+ if (!published) {
+ this.#returnSlot(pool, slot, frameId);
+ }
+ },
+ discardPublished: async () => {
+ published = true;
+ await releasePublished();
+ },
+ publish: async (
+ metadata: CdpRemotionRawFrame,
+ ): Promise<RemotionRawFrame> => {
+ published = true;
+ if (
+ metadata.slot !== slot ||
+ metadata.frameId !== frameId ||
+ metadata.width !== this.#options.width ||
+ metadata.height !== this.#options.height ||
+ metadata.stride < metadata.width * 4 ||
+ metadata.byteLength > pool.slotCapacity - encoderPadding ||
+ metadata.pixelFormat !== 'bgra' ||
+ metadata.colorSpace !== 'srgb'
+ ) {
+ await releasePublished();
+ throw new Error(
+ `Chromium returned invalid Remotion shared-memory metadata: ${JSON.stringify(metadata)}`,
+ );
+ }
+
+ return {
+ ...metadata,
+ type: 'remotion-shared-memory',
+ poolId: pool.poolId,
+ sharedMemoryName: pool.sharedMemoryName,
+ slotCount: pool.slotCount,
+ slotCapacity: pool.slotCapacity,
+ poolLifetime: pool.poolLifetime,
+ release: releasePublished,
+ };
+ },
+ };
+ }
+
+ async destroy() {
+ await this.#destroyPools();
+ }
+
+ async forgetPage(page: Page) {
+ await this.#forgetPage(page, false);
+ }
+
+ async forgetAllPages() {
+ await Promise.all(
+ [...this.#pools.keys()].map((page) => this.#forgetPage(page, false)),
+ );
+ }
+
+ async dispose() {
+ await Promise.all(
+ [...this.#pools.keys()].map((page) => this.#forgetPage(page, true)),
+ );
+
+ this.#supported = false;
+ }
+
+ async #forgetPage(page: Page, bestEffort: boolean) {
+ const pool = this.#pools.get(page);
+ if (!pool) {
+ return;
+ }
+
+ this.#pools.delete(page);
+ pool.forgotten = true;
+ pool.waiters.splice(0).forEach((waiter) => waiter());
+
+ let cleanupError: unknown = null;
+ try {
+ await pool.poolLifetime.retire();
+ } catch (error) {
+ cleanupError = error;
+ }
+
+ if (pool.ownedSlots.size > 0) {
+ await new Promise<void>((resolve) => pool.drainWaiters.push(resolve));
+ }
+
+ try {
+ await page._client().send('Page.remotionDestroyFramePool');
+ } catch (error) {
+ if (!bestEffort && !isTargetClosedErr(error as Error) && !cleanupError) {
+ cleanupError = error;
+ }
+ }
+
+ if (cleanupError && !bestEffort) {
+ throw cleanupError;
+ }
+ }
+
+ async #createPool(page: Page) {
+ if (this.#pools.has(page)) {
+ return;
+ }
+
+ const slotCapacity =
+ this.#options.width * this.#options.height * 4 + encoderPadding;
+ if (!Number.isSafeInteger(slotCapacity)) {
+ throw new Error(
+ 'Remotion shared-memory frame pool size is not a safe integer.',
+ );
+ }
+
+ const {value} = await page._client().send('Page.remotionCreateFramePool', {
+ slotCount: slotsPerPage,
+ slotCapacity,
+ });
+ if (
+ !value.sharedMemoryName ||
+ value.slotCount !== slotsPerPage ||
+ value.slotCapacity !== slotCapacity
+ ) {
+ throw new Error(
+ 'Chromium returned invalid Remotion frame pool metadata.',
+ );
+ }
+
+ let retired = false;
+ let retirement: Promise<void> | null = null;
+ let notifyRetirement!: () => void;
+ const retirementSignal = new Promise<void>((resolve) => {
+ notifyRetirement = resolve;
+ });
+ const retirementListeners = new Set<() => Promise<void>>();
+ const poolLifetime: ManagedRemotionSharedMemoryPoolLifetime = {
+ isRetired: () => retired,
+ waitForRetirement: () => retirementSignal,
+ onRetired: (listener) => {
+ if (retired) {
+ return null;
+ }
+
+ retirementListeners.add(listener);
+ return () => retirementListeners.delete(listener);
+ },
+ retire: () => {
+ if (!retirement) {
+ retired = true;
+ notifyRetirement();
+ retirement = (async () => {
+ await Promise.all(
+ [...retirementListeners].map((listener) => listener()),
+ );
+ })();
+ }
+
+ return retirement;
+ },
+ };
+
+ this.#pools.set(page, {
+ page,
+ poolId: this.#nextPoolId++,
+ sharedMemoryName: value.sharedMemoryName,
+ slotCount: value.slotCount,
+ slotCapacity: value.slotCapacity,
+ availableSlots: Array.from(
+ {length: value.slotCount},
+ (_, index) => index,
+ ),
+ waiters: [],
+ drainWaiters: [],
+ ownedSlots: new Map(),
+ forgotten: false,
+ poolLifetime,
+ });
+ }
+
+ #returnSlot(pool: FramePool, slot: number, frameId: string) {
+ if (pool.ownedSlots.get(slot) !== frameId) {
+ throw new Error(
+ `Remotion shared-memory slot ${pool.poolId}:${slot} is not owned by frame ${frameId}.`,
+ );
+ }
+
+ pool.ownedSlots.delete(slot);
+ if (pool.forgotten) {
+ if (pool.ownedSlots.size === 0) {
+ pool.drainWaiters.splice(0).forEach((waiter) => waiter());
+ }
+ } else {
+ pool.availableSlots.push(slot);
+ pool.waiters.shift()?.();
+ }
+ }
+
+ async #destroyPools() {
+ for (const pool of this.#pools.values()) {
+ if (pool.ownedSlots.size > 0) {
+ throw new Error(
+ `Cannot destroy Remotion shared-memory pool ${pool.poolId} while it owns ${pool.ownedSlots.size} frame(s).`,
+ );
+ }
+
+ try {
+ await pool.page._client().send('Page.remotionDestroyFramePool');
+ } catch (error) {
+ if (!isTargetClosedErr(error as Error)) {
+ throw error;
+ }
+ }
+ }
+
+ this.#pools.clear();
+ }
+}
+
+export const isRemotionRawFrame = (
+ frame: CapturedFrame,
+): frame is RemotionRawFrame => !Buffer.isBuffer(frame);
diff --git a/packages/renderer/src/render-frame-and-retry-target-close.ts b/packages/renderer/src/render-frame-and-retry-target-close.ts
index 98fa9b5267..22a04a002a 100644
--- a/packages/renderer/src/render-frame-and-retry-target-close.ts
+++ b/packages/renderer/src/render-frame-and-retry-target-close.ts
@@ -14,6 +14,10 @@ import type {CancelSignal} from './make-cancel-signal';
import {cancelErrorMessages, isUserCancelledRender} from './make-cancel-signal';
import type {NextFrameToRender} from './next-frame-to-render';
import type {Pool} from './pool';
+import type {
+ CapturedFrame,
+ RemotionSharedMemoryCapture,
+} from './remotion-shared-memory';
import {renderFrame} from './render-frame';
import type {FrameAndAssets, OnArtifact} from './render-frames';
import type {BrowserReplacer} from './replace-browser';
@@ -48,6 +52,8 @@ export const renderFrameAndRetryTargetClose = async ({
framesRenderedObj,
lastFrame,
onFrameBuffer,
+ onFrame,
+ remotionSharedMemory,
onFrameUpdate,
nextFrameToRender,
imageSequencePattern,
@@ -87,6 +93,10 @@ export const renderFrameAndRetryTargetClose = async ({
| null
| ((buffer: Buffer, frame: number) => void | Promise<void>)
| undefined;
+ onFrame:
+ | null
+ | ((frame: CapturedFrame, frameNumber: number) => void | Promise<void>);
+ remotionSharedMemory: RemotionSharedMemoryCapture | null;
onFrameUpdate:
| null
| ((
@@ -136,6 +146,8 @@ export const renderFrameAndRetryTargetClose = async ({
lastFrame,
onError,
onFrameBuffer,
+ onFrame,
+ remotionSharedMemory,
onFrameUpdate,
outputDir,
stoppedSignal,
@@ -185,6 +197,7 @@ export const renderFrameAndRetryTargetClose = async ({
if (shouldRetryError) {
const pool = await poolPromise;
// Replace the closed page
+ await remotionSharedMemory?.forgetPage(freePage);
const newPage = await makeNewPage(frame, freePage.pageIndex);
pool.release(newPage);
Log.warn(
@@ -224,6 +237,8 @@ export const renderFrameAndRetryTargetClose = async ({
framesRenderedObj,
lastFrame,
onFrameBuffer,
+ onFrame,
+ remotionSharedMemory,
onFrameUpdate,
nextFrameToRender,
imageSequencePattern,
@@ -238,6 +253,7 @@ export const renderFrameAndRetryTargetClose = async ({
`The browser crashed while rendering frame ${frame}, retrying ${retriesLeft} more times. Learn more about this error under https://www.remotion.dev/docs/target-closed`,
);
// Replace the entire browser
+ await remotionSharedMemory?.forgetAllPages();
await browserReplacer.replaceBrowser(makeBrowser, async () => {
const pages = new Array(concurrencyOrFramesToRender)
.fill(true)
@@ -281,6 +297,8 @@ export const renderFrameAndRetryTargetClose = async ({
framesRenderedObj,
lastFrame,
onFrameBuffer,
+ onFrame,
+ remotionSharedMemory,
onFrameUpdate,
nextFrameToRender,
imageSequencePattern,
diff --git a/packages/renderer/src/render-frame-with-option-to-reject.ts b/packages/renderer/src/render-frame-with-option-to-reject.ts
index 63de6dc1f3..bec487431e 100644
--- a/packages/renderer/src/render-frame-with-option-to-reject.ts
+++ b/packages/renderer/src/render-frame-with-option-to-reject.ts
@@ -17,6 +17,11 @@ import type {VideoImageFormat} from './image-format';
import type {LogLevel} from './log-level';
import {Log} from './logger';
import type {CancelSignal} from './make-cancel-signal';
+import type {
+ CapturedFrame,
+ RemotionSharedMemoryCapture,
+} from './remotion-shared-memory';
+import {isRemotionRawFrame} from './remotion-shared-memory';
import type {FrameAndAssets, OnArtifact} from './render-frames';
import {seekToFrame} from './seek-to-frame';
import {takeFrame} from './take-frame';
@@ -34,6 +39,8 @@ export const renderFrameWithOptionToReject = async ({
timeoutInMilliseconds,
outputDir,
onFrameBuffer,
+ onFrame,
+ remotionSharedMemory,
imageFormat,
onError,
lastFrame,
@@ -72,6 +79,10 @@ export const renderFrameWithOptionToReject = async ({
| null
| ((buffer: Buffer, frame: number) => void | Promise<void>)
| undefined;
+ onFrame:
+ | null
+ | ((frame: CapturedFrame, frameNumber: number) => void | Promise<void>);
+ remotionSharedMemory: RemotionSharedMemoryCapture | null;
imageFormat: VideoImageFormat;
onError: (err: Error) => void;
lastFrame: number;
@@ -111,7 +122,9 @@ export const renderFrameWithOptionToReject = async ({
return Promise.reject(new Error('Render was stopped'));
}
+ let pageError: Error | null = null;
const errorCallbackOnFrame = (err: Error) => {
+ pageError ??= err;
reject(err);
};
@@ -121,171 +134,268 @@ export const renderFrameWithOptionToReject = async ({
frame,
});
page.on('error', errorCallbackOnFrame);
+ let pageListenersCleaned = false;
+ const cleanupPageListeners = () => {
+ if (pageListenersCleaned) {
+ return;
+ }
- const startSeeking = Date.now();
+ pageListenersCleaned = true;
+ cleanupPageError();
+ page.off('error', errorCallbackOnFrame);
+ };
- await seekToFrame({
- frame,
- page,
- composition: compId,
- timeoutInMilliseconds,
- indent,
- logLevel,
- attempt,
- });
+ try {
+ const startSeeking = Date.now();
- const timeToSeek = Date.now() - startSeeking;
- if (timeToSeek > 1000) {
- Log.verbose(
- {indent, logLevel},
- `Seeking to frame ${frame} took ${timeToSeek}ms`,
- );
- }
+ await seekToFrame({
+ frame,
+ page,
+ composition: compId,
+ timeoutInMilliseconds,
+ indent,
+ logLevel,
+ attempt,
+ });
- if (!outputDir && !onFrameBuffer && imageFormat !== 'none') {
- throw new Error(
- 'Called renderFrames() without specifying either `outputDir` or `onFrameBuffer`',
- );
- }
+ const timeToSeek = Date.now() - startSeeking;
+ if (timeToSeek > 1000) {
+ Log.verbose(
+ {indent, logLevel},
+ `Seeking to frame ${frame} took ${timeToSeek}ms`,
+ );
+ }
- if (outputDir && onFrameBuffer && imageFormat !== 'none') {
- throw new Error(
- 'Pass either `outputDir` or `onFrameBuffer` to renderFrames(), not both.',
- );
- }
+ if (!outputDir && !onFrameBuffer && !onFrame && imageFormat !== 'none') {
+ throw new Error(
+ 'Called renderFrames() without specifying either `outputDir` or `onFrameBuffer`',
+ );
+ }
- const [buffer, collectedAssets] = await Promise.all([
- takeFrame({
- freePage: page,
- height,
- imageFormat: assetsOnly ? 'none' : imageFormat,
- output:
- index === null
- ? null
- : path.join(
- frameDir,
- getFrameOutputFileName({
- frame,
- imageFormat,
- index,
- countType,
- lastFrame,
- totalFrames: framesToRender.length,
- imageSequencePattern,
- }),
- ),
- jpegQuality,
- width,
- scale,
- wantsBuffer: Boolean(onFrameBuffer),
- timeoutInMilliseconds,
- }),
- collectAssets({
- frame,
- freePage: page,
- timeoutInMilliseconds,
- }),
- ]);
- if (onFrameBuffer && !assetsOnly) {
- if (!buffer) {
- throw new Error('unexpected null buffer');
+ if (outputDir && (onFrameBuffer || onFrame) && imageFormat !== 'none') {
+ throw new Error(
+ 'Pass either `outputDir` or `onFrameBuffer` to renderFrames(), not both.',
+ );
}
- await onFrameBuffer(buffer, frame);
- }
+ const [captureResult, assetsResult] = await Promise.allSettled([
+ takeFrame({
+ freePage: page,
+ height,
+ imageFormat: assetsOnly ? 'none' : imageFormat,
+ output:
+ index === null
+ ? null
+ : path.join(
+ frameDir,
+ getFrameOutputFileName({
+ frame,
+ imageFormat,
+ index,
+ countType,
+ lastFrame,
+ totalFrames: framesToRender.length,
+ imageSequencePattern,
+ }),
+ ),
+ jpegQuality,
+ width,
+ scale,
+ wantsBuffer: Boolean(onFrameBuffer || onFrame),
+ timeoutInMilliseconds,
+ remotionSharedMemory,
+ }),
+ collectAssets({
+ frame,
+ freePage: page,
+ timeoutInMilliseconds,
+ }),
+ ]);
+ if (captureResult.status === 'rejected') {
+ throw captureResult.reason;
+ }
- const onlyAvailableAssets = assets.filter(truthy);
+ const buffer = captureResult.value;
+ if (assetsResult.status === 'rejected') {
+ if (buffer !== null && isRemotionRawFrame(buffer)) {
+ try {
+ await buffer.release();
+ } catch {
+ // Preserve the asset collection error so the frame can be retried.
+ }
+ }
- const previousAudioRenderAssets = onlyAvailableAssets
- .map((a) => a.audioAndVideoAssets)
- .flat(2);
+ throw assetsResult.reason;
+ }
- const previousArtifactAssets = onlyAvailableAssets
- .map((a) => a.artifactAssets)
- .flat(2);
+ const collectedAssets = assetsResult.value;
+ let frameBufferForArtifacts = Buffer.isBuffer(buffer) ? buffer : null;
+ if (
+ buffer !== null &&
+ isRemotionRawFrame(buffer) &&
+ collectedAssets.some(
+ (asset) =>
+ asset.type === 'artifact' && asset.contentType === 'thumbnail',
+ )
+ ) {
+ try {
+ const thumbnail = await takeFrame({
+ freePage: page,
+ height,
+ imageFormat,
+ output: null,
+ jpegQuality,
+ width,
+ scale,
+ wantsBuffer: true,
+ timeoutInMilliseconds,
+ remotionSharedMemory: null,
+ });
+ if (!thumbnail) {
+ throw new Error('Could not capture a thumbnail artifact.');
+ }
- const audioAndVideoAssets = onlyAudioAndVideoAssets(collectedAssets);
- const artifactAssets = onlyArtifact({
- assets: collectedAssets,
- frameBuffer: buffer,
- });
+ frameBufferForArtifacts = thumbnail;
+ } catch (error) {
+ try {
+ await buffer.release();
+ } catch {
+ // Preserve the thumbnail capture error so the frame can be retried.
+ }
- for (const artifact of artifactAssets) {
- for (const previousArtifact of previousArtifactAssets) {
- if (artifact.filename === previousArtifact.filename) {
- return Promise.reject(
- new Error(
- `An artifact with output "${artifact.filename}" was already registered at frame ${previousArtifact.frame}, but now registered again at frame ${artifact.frame}. Artifacts must have unique names. https://remotion.dev/docs/artifacts`,
- ),
- );
+ throw error;
}
}
- onArtifact?.(artifact);
- }
+ cleanupPageListeners();
+ if (pageError) {
+ if (buffer !== null && isRemotionRawFrame(buffer)) {
+ try {
+ await buffer.release();
+ } catch {
+ // Preserve the page error so the frame can be retried.
+ }
+ }
- const compressedAssets = audioAndVideoAssets.map((asset) => {
- return compressAsset(previousAudioRenderAssets, asset);
- });
+ throw pageError;
+ }
- const inlineAudioAssets = onlyInlineAudio(collectedAssets);
- const outputFrame =
- allFramesAndExtraFrames[0] + allFramesAndExtraFrames.indexOf(frame);
+ if ((onFrameBuffer || onFrame) && !assetsOnly) {
+ if (!buffer) {
+ throw new Error('unexpected null buffer');
+ }
- assets.push({
- audioAndVideoAssets: compressedAssets,
- frame,
- artifactAssets: artifactAssets.map((a) => {
- return {
- frame: a.frame,
- filename: a.filename,
- };
- }),
- inlineAudioAssets,
- });
+ if (onFrame) {
+ await onFrame(buffer, frame);
+ } else {
+ if (isRemotionRawFrame(buffer)) {
+ try {
+ await buffer.release();
+ } catch {
+ // The type mismatch is the primary error.
+ }
- for (const renderAsset of compressedAssets) {
- downloadAndMapAssetsToFileUrl({
- renderAsset,
- onDownload,
- downloadMap,
- indent,
- logLevel,
- binariesDirectory,
- cancelSignalForAudioAnalysis: cancelSignal,
- shouldAnalyzeAudioImmediately: true,
- }).catch((err) => {
- const truncateWithEllipsis =
- renderAsset.src.substring(0, 1000) +
- (renderAsset.src.length > 1000 ? '...' : '');
- onError(
- new Error(
- `Error while downloading ${truncateWithEllipsis}: ${(err as Error).stack}`,
- ),
- );
+ throw new Error(
+ 'Raw shared-memory frames cannot be passed to renderFrames().',
+ );
+ }
+
+ await onFrameBuffer?.(buffer, frame);
+ }
+ }
+
+ const onlyAvailableAssets = assets.filter(truthy);
+
+ const previousAudioRenderAssets = onlyAvailableAssets
+ .map((a) => a.audioAndVideoAssets)
+ .flat(2);
+
+ const previousArtifactAssets = onlyAvailableAssets
+ .map((a) => a.artifactAssets)
+ .flat(2);
+
+ const audioAndVideoAssets = onlyAudioAndVideoAssets(collectedAssets);
+ const artifactAssets = onlyArtifact({
+ assets: collectedAssets,
+ frameBuffer: frameBufferForArtifacts,
});
- }
- for (const renderAsset of inlineAudioAssets) {
- downloadMap.inlineAudioMixing.addAsset({
- asset: {...renderAsset, frame: outputFrame},
- fps,
- totalNumberOfFrames: allFramesAndExtraFrames.length,
- firstFrame: allFramesAndExtraFrames[0],
- trimLeftOffset,
- trimRightOffset,
+ for (const artifact of artifactAssets) {
+ for (const previousArtifact of previousArtifactAssets) {
+ if (artifact.filename === previousArtifact.filename) {
+ return Promise.reject(
+ new Error(
+ `An artifact with output "${artifact.filename}" was already registered at frame ${previousArtifact.frame}, but now registered again at frame ${artifact.frame}. Artifacts must have unique names. https://remotion.dev/docs/artifacts`,
+ ),
+ );
+ }
+ }
+
+ onArtifact?.(artifact);
+ }
+
+ const compressedAssets = audioAndVideoAssets.map((asset) => {
+ return compressAsset(previousAudioRenderAssets, asset);
});
- }
- cleanupPageError();
- page.off('error', errorCallbackOnFrame);
+ const inlineAudioAssets = onlyInlineAudio(collectedAssets);
+ const outputFrame =
+ allFramesAndExtraFrames[0] + allFramesAndExtraFrames.indexOf(frame);
- if (!assetsOnly) {
- framesRenderedObj.count++;
- onFrameUpdate?.(
- framesRenderedObj.count,
+ assets.push({
+ audioAndVideoAssets: compressedAssets,
frame,
- performance.now() - startTime,
- );
+ artifactAssets: artifactAssets.map((a) => {
+ return {
+ frame: a.frame,
+ filename: a.filename,
+ };
+ }),
+ inlineAudioAssets,
+ });
+
+ for (const renderAsset of compressedAssets) {
+ downloadAndMapAssetsToFileUrl({
+ renderAsset,
+ onDownload,
+ downloadMap,
+ indent,
+ logLevel,
+ binariesDirectory,
+ cancelSignalForAudioAnalysis: cancelSignal,
+ shouldAnalyzeAudioImmediately: true,
+ }).catch((err) => {
+ const truncateWithEllipsis =
+ renderAsset.src.substring(0, 1000) +
+ (renderAsset.src.length > 1000 ? '...' : '');
+ onError(
+ new Error(
+ `Error while downloading ${truncateWithEllipsis}: ${(err as Error).stack}`,
+ ),
+ );
+ });
+ }
+
+ for (const renderAsset of inlineAudioAssets) {
+ downloadMap.inlineAudioMixing.addAsset({
+ asset: {...renderAsset, frame: outputFrame},
+ fps,
+ totalNumberOfFrames: allFramesAndExtraFrames.length,
+ firstFrame: allFramesAndExtraFrames[0],
+ trimLeftOffset,
+ trimRightOffset,
+ });
+ }
+
+ if (!assetsOnly) {
+ framesRenderedObj.count++;
+ onFrameUpdate?.(
+ framesRenderedObj.count,
+ frame,
+ performance.now() - startTime,
+ );
+ }
+ } finally {
+ cleanupPageListeners();
}
};
diff --git a/packages/renderer/src/render-frame.ts b/packages/renderer/src/render-frame.ts
index 1ec4021bbd..215d8f70df 100644
--- a/packages/renderer/src/render-frame.ts
+++ b/packages/renderer/src/render-frame.ts
@@ -7,6 +7,10 @@ import type {VideoImageFormat} from './image-format';
import type {LogLevel} from './log-level';
import type {CancelSignal} from './make-cancel-signal';
import type {NextFrameToRender} from './next-frame-to-render';
+import type {
+ CapturedFrame,
+ RemotionSharedMemoryCapture,
+} from './remotion-shared-memory';
import {renderFrameWithOptionToReject} from './render-frame-with-option-to-reject';
import type {FrameAndAssets, OnArtifact} from './render-frames';
@@ -33,6 +37,8 @@ export const renderFrame = ({
timeoutInMilliseconds,
lastFrame,
onFrameBuffer,
+ onFrame,
+ remotionSharedMemory,
onFrameUpdate,
framesRenderedObj,
frame,
@@ -66,6 +72,10 @@ export const renderFrame = ({
| null
| ((buffer: Buffer, frame: number) => void | Promise<void>)
| undefined;
+ onFrame:
+ | null
+ | ((frame: CapturedFrame, frameNumber: number) => void | Promise<void>);
+ remotionSharedMemory: RemotionSharedMemoryCapture | null;
lastFrame: number;
onFrameUpdate:
| null
@@ -96,6 +106,8 @@ export const renderFrame = ({
timeoutInMilliseconds,
imageFormat,
onFrameBuffer,
+ onFrame,
+ remotionSharedMemory,
outputDir,
assets,
binariesDirectory,
diff --git a/packages/renderer/src/render-frames.ts b/packages/renderer/src/render-frames.ts
index a13b653d97..ae3cf63523 100644
--- a/packages/renderer/src/render-frames.ts
+++ b/packages/renderer/src/render-frames.ts
@@ -47,6 +47,10 @@ import type {optionsMap} from './options/options-map';
import {Pool} from './pool';
import type {RemotionServer} from './prepare-server';
import {makeOrReuseServer} from './prepare-server';
+import type {
+ CapturedFrame,
+ RemotionSharedMemoryCapture,
+} from './remotion-shared-memory';
import {renderFrameAndRetryTargetClose} from './render-frame-and-retry-target-close';
import type {BrowserReplacer} from './replace-browser';
import {handleBrowserCrash} from './replace-browser';
@@ -88,6 +92,10 @@ type InternalRenderFramesOptions = {
onFrameBuffer:
| null
| ((buffer: Buffer, frame: number) => void | Promise<void>);
+ onFrame:
+ | null
+ | ((frame: CapturedFrame, frameNumber: number) => void | Promise<void>);
+ remotionSharedMemory: RemotionSharedMemoryCapture | null;
onDownload: RenderMediaOnDownload | null;
chromiumOptions: ChromiumOptions;
scale: number;
@@ -127,6 +135,10 @@ type InnerRenderFramesOptions = {
onFrameBuffer:
| null
| ((buffer: Buffer, frame: number) => void | Promise<void>);
+ onFrame:
+ | null
+ | ((frame: CapturedFrame, frameNumber: number) => void | Promise<void>);
+ remotionSharedMemory: RemotionSharedMemoryCapture | null;
onArtifact: OnArtifact | null;
onDownload: RenderMediaOnDownload | null;
timeoutInMilliseconds: number;
@@ -229,6 +241,8 @@ const innerRenderFrames = async ({
envVariables,
onBrowserLog,
onFrameBuffer,
+ onFrame,
+ remotionSharedMemory,
onDownload,
pagesArray,
serveUrl,
@@ -308,8 +322,8 @@ const innerRenderFrames = async ({
resolvedConcurrency,
);
- const makeNewPage = (frame: number, pageIndex: number) => {
- return makePage({
+ const makeNewPage = async (frame: number, pageIndex: number) => {
+ const page = await makePage({
context: sourceMapGetter,
initialFrame: frame,
browserReplacer,
@@ -334,6 +348,8 @@ const innerRenderFrames = async ({
darkMode,
sampleRate,
});
+ await remotionSharedMemory?.ensurePage(page);
+ return page;
};
const getPool = async () => {
@@ -447,6 +463,8 @@ const innerRenderFrames = async ({
lastFrame,
makeNewPage,
onFrameBuffer,
+ onFrame,
+ remotionSharedMemory,
onFrameUpdate,
nextFrameToRender,
imageSequencePattern: pattern,
@@ -498,6 +516,8 @@ const internalRenderFramesRaw = ({
onBrowserLog,
onDownload,
onFrameBuffer,
+ onFrame,
+ remotionSharedMemory,
onFrameUpdate,
onStart,
outputDir,
@@ -641,6 +661,8 @@ const internalRenderFramesRaw = ({
muted,
onBrowserLog,
onFrameBuffer,
+ onFrame,
+ remotionSharedMemory,
onFrameUpdate,
onStart,
outputDir,
@@ -848,6 +870,8 @@ export const renderFrames = (
muted: muted ?? false,
onBrowserLog: onBrowserLog ?? null,
onFrameBuffer: onFrameBuffer ?? null,
+ onFrame: null,
+ remotionSharedMemory: null,
onFrameUpdate,
onStart,
outputDir,
diff --git a/packages/renderer/src/render-media.ts b/packages/renderer/src/render-media.ts
index 403eac384d..06f13dd048 100644
--- a/packages/renderer/src/render-media.ts
+++ b/packages/renderer/src/render-media.ts
@@ -57,9 +57,16 @@ import {
} from './pixel-format';
import type {RemotionServer} from './prepare-server';
import {makeOrReuseServer} from './prepare-server';
-import {prespawnFfmpeg} from './prespawn-ffmpeg';
+import {
+ ffmpegSupportsRemotionSharedMemory,
+ prespawnFfmpeg,
+} from './prespawn-ffmpeg';
import {shouldUseParallelEncoding} from './prestitcher-memory-usage';
import {validateSelectedCodecAndProResCombination} from './prores-profile';
+import {
+ isRemotionRawFrame,
+ RemotionSharedMemoryCapture,
+} from './remotion-shared-memory';
import type {OnArtifact} from './render-frames';
import {internalRenderFrames} from './render-frames';
import {
@@ -80,7 +87,6 @@ import {validateOutputFilename} from './validate-output-filename';
import {validateScale} from './validate-scale';
import {validateBitrate} from './validate-videobitrate';
import {wrapWithErrorHandling} from './wrap-with-error-handling';
-import {writeWithBackpressure} from './write-with-backpressure';
export type StitchingState = 'encoding' | 'muxing';
@@ -228,7 +234,7 @@ type RenderMediaResult = {
contentType: string;
};
-const internalRenderMediaRaw = ({
+const internalRenderMediaRaw = async ({
proResProfile,
x264Preset,
gopSize,
@@ -489,6 +495,21 @@ const internalRenderMediaRaw = ({
});
const actualWidth = widthEvenDimensions * scale;
const actualHeight = heightEvenDimensions * scale;
+ const canUseRemotionSharedMemory =
+ preEncodedFileLocation !== null &&
+ (await ffmpegSupportsRemotionSharedMemory({
+ binariesDirectory,
+ indent,
+ logLevel,
+ }));
+ const remotionSharedMemory = canUseRemotionSharedMemory
+ ? new RemotionSharedMemoryCapture({
+ width: Math.round(actualWidth),
+ height: Math.round(actualHeight),
+ indent,
+ logLevel,
+ })
+ : null;
const composition = {
...compositionWithPossibleUnevenDimensions,
@@ -539,8 +560,16 @@ const internalRenderMediaRaw = ({
validateFps(fps, 'in "renderMedia()"', codec === 'gif');
- const createPrestitcherIfNecessary = () => {
- if (preEncodedFileLocation) {
+ let preStitcherInputMode: 'encoded-image' | 'remotion-shared-memory' | null =
+ null;
+ let notifyPreStitcherStarted: () => void;
+ const preStitcherStarted = new Promise<void>((resolve) => {
+ notifyPreStitcherStarted = resolve;
+ });
+ const createPrestitcherIfNecessary = (
+ inputMode: 'encoded-image' | 'remotion-shared-memory',
+ ) => {
+ if (preEncodedFileLocation && !preStitcher) {
preStitcher = prespawnFfmpeg({
width: actualWidth,
height: actualHeight,
@@ -568,17 +597,30 @@ const internalRenderMediaRaw = ({
binariesDirectory,
hardwareAcceleration,
onLog,
+ inputMode,
});
+ preStitcherInputMode = inputMode;
stitcherFfmpeg = preStitcher.task;
+ notifyPreStitcherStarted!();
+ }
+
+ if (preStitcherInputMode !== inputMode) {
+ throw new Error(
+ `Cannot mix ${inputMode} frames with ${preStitcherInputMode} FFmpeg input.`,
+ );
}
};
const waitForPrestitcherIfNecessary = async (): Promise<{
usesParallelEncoding: boolean;
}> => {
+ if (preEncodedFileLocation) {
+ await preStitcherStarted;
+ }
+
if (stitcherFfmpeg) {
await waitForFinish();
- stitcherFfmpeg?.stdin?.end();
+ await preStitcher?.finishInput();
try {
await stitcherFfmpeg;
} catch {
@@ -626,7 +668,7 @@ const internalRenderMediaRaw = ({
Promise.resolve(undefined);
const happyPath = new Promise<RenderMediaResult>((resolve, reject) => {
- Promise.resolve(createPrestitcherIfNecessary())
+ Promise.resolve()
.then(() => {
return makeOrReuseServer(
reusedServer,
@@ -699,43 +741,101 @@ const internalRenderMediaRaw = ({
outputFramesInSequence: true,
puppeteerInstance,
everyNthFrame,
- onFrameBuffer: parallelEncoding
- ? async (buffer, frame) => {
- await waitForRightTimeOfFrameToBeInserted(frame);
- if (cancelled) {
- return;
- }
-
- const id = startPerfMeasure('piping');
- const exitStatus = preStitcher?.getExitStatus();
- if (exitStatus?.type === 'quit-successfully') {
- throw new Error(
- `FFmpeg already quit while trying to pipe frame ${frame} to it. Stderr: ${exitStatus.stderr}`,
+ onFrameBuffer: null,
+ onFrame: parallelEncoding
+ ? async (capturedFrame, frame) => {
+ let handedToFfmpeg = false;
+ try {
+ if (isRemotionRawFrame(capturedFrame)) {
+ let notifyRetirement!: () => void;
+ const retirement = new Promise<boolean>(
+ (resolveRetirement) => {
+ notifyRetirement = () => resolveRetirement(true);
+ },
+ );
+ const unsubscribeFromRetirement =
+ capturedFrame.poolLifetime.onRetired(() => {
+ notifyRetirement();
+ return Promise.resolve();
+ });
+ if (!unsubscribeFromRetirement) {
+ notifyRetirement();
+ }
+
+ let retiredBeforeItsTurn: boolean;
+ try {
+ retiredBeforeItsTurn = await Promise.race([
+ waitForRightTimeOfFrameToBeInserted(frame).then(
+ () => false,
+ ),
+ retirement,
+ ]);
+ } finally {
+ unsubscribeFromRetirement?.();
+ }
+
+ if (retiredBeforeItsTurn) {
+ throw new Error(
+ 'Target closed while a raw frame was waiting for its FFmpeg turn.',
+ );
+ }
+
+ if (capturedFrame.poolLifetime.isRetired()) {
+ throw new Error(
+ 'Target closed before a raw frame was sent to FFmpeg.',
+ );
+ }
+ } else {
+ await waitForRightTimeOfFrameToBeInserted(frame);
+ }
+
+ if (cancelled) {
+ return;
+ }
+
+ const id = startPerfMeasure('piping');
+ const exitStatus = preStitcher?.getExitStatus();
+ if (exitStatus?.type === 'quit-successfully') {
+ throw new Error(
+ `FFmpeg already quit while trying to pipe frame ${frame} to it. Stderr: ${exitStatus.stderr}`,
+ );
+ }
+
+ if (exitStatus?.type === 'quit-with-error') {
+ throw new Error(
+ `FFmpeg quit with code ${exitStatus.exitCode}${exitStatus.signal ? ` (signal ${exitStatus.signal})` : ''} while piping frame ${frame}. Stderr: ${exitStatus.stderr}`,
+ );
+ }
+
+ const inputMode = isRemotionRawFrame(capturedFrame)
+ ? 'remotion-shared-memory'
+ : 'encoded-image';
+ createPrestitcherIfNecessary(inputMode);
+ const frameIndex = framesToRender.indexOf(frame);
+ const writeResult = await preStitcher?.writeFrame({
+ frame: capturedFrame,
+ pts: frameIndex,
+ });
+ if (!writeResult) {
+ throw new Error(
+ `FFmpeg is not available while trying to pipe frame ${frame} to it.`,
+ );
+ }
+
+ handedToFfmpeg = true;
+ stopPerfMeasure(id);
+ setFrameToStitch(
+ framesToRender[frameIndex + 1] ?? lastFrameToRender + 1,
);
+ await writeResult.waitForAck;
+ } finally {
+ if (!handedToFfmpeg && isRemotionRawFrame(capturedFrame)) {
+ await capturedFrame.release();
+ }
}
-
- if (exitStatus?.type === 'quit-with-error') {
- throw new Error(
- `FFmpeg quit with code ${exitStatus.exitCode}${exitStatus.signal ? ` (signal ${exitStatus.signal})` : ''} while piping frame ${frame}. Stderr: ${exitStatus.stderr}`,
- );
- }
-
- const stdin = stitcherFfmpeg?.stdin;
- if (!stdin) {
- throw new Error(
- `FFmpeg stdin is not available while trying to pipe frame ${frame} to it.`,
- );
- }
-
- await writeWithBackpressure({data: buffer, writable: stdin});
- stopPerfMeasure(id);
-
- const frameIndex = framesToRender.indexOf(frame);
- setFrameToStitch(
- framesToRender[frameIndex + 1] ?? lastFrameToRender + 1,
- );
}
: null,
+ remotionSharedMemory,
webpackBundleOrServeUrl: serveUrl,
onBrowserLog,
onDownload,
@@ -773,7 +873,8 @@ const internalRenderMediaRaw = ({
waitForPrestitcherIfNecessary(),
]);
})
- .then(([{assetsInfo}]) => {
+ .then(async ([{assetsInfo}]) => {
+ await remotionSharedMemory?.destroy();
renderedDoneIn = Date.now() - renderStart;
Log.verbose(
@@ -908,6 +1009,7 @@ const internalRenderMediaRaw = ({
* Therefore we first kill the FFMPEG process before deleting the file
*/
cancelled = true;
+ preStitcher?.abortInput(err as Error);
cancelRenderFrames.cancel();
cancelStitcher.cancel();
cancelPrestitcher.cancel();
@@ -934,7 +1036,8 @@ const internalRenderMediaRaw = ({
reject(err);
})
- .finally(() => {
+ .finally(async () => {
+ await remotionSharedMemory?.dispose();
if (
preEncodedFileLocation !== null &&
fs.existsSync(preEncodedFileLocation)
diff --git a/packages/renderer/src/render-still.ts b/packages/renderer/src/render-still.ts
index b7c8755ff6..3910663f10 100644
--- a/packages/renderer/src/render-still.ts
+++ b/packages/renderer/src/render-still.ts
@@ -375,6 +375,7 @@ const innerRenderStill = async ({
jpegQuality,
wantsBuffer: !output,
timeoutInMilliseconds,
+ remotionSharedMemory: null,
}),
collectAssets({
frame,
diff --git a/packages/renderer/src/screenshot-task.ts b/packages/renderer/src/screenshot-task.ts
index 234b7e3f08..b79afc9ea2 100644
--- a/packages/renderer/src/screenshot-task.ts
+++ b/packages/renderer/src/screenshot-task.ts
@@ -2,6 +2,10 @@ import fs from 'node:fs';
import type {Page} from './browser/BrowserPage';
import type {StillImageFormat} from './image-format';
import {startPerfMeasure, stopPerfMeasure} from './perf';
+import type {
+ CapturedFrame,
+ RemotionSharedMemoryCapture,
+} from './remotion-shared-memory';
export const screenshotTask = async ({
format,
@@ -12,6 +16,7 @@ export const screenshotTask = async ({
path,
jpegQuality,
scale,
+ remotionSharedMemory,
}: {
page: Page;
format: StillImageFormat;
@@ -21,7 +26,8 @@ export const screenshotTask = async ({
width: number;
height: number;
scale: number;
-}): Promise<Buffer> => {
+ remotionSharedMemory: RemotionSharedMemoryCapture | null;
+}): Promise<CapturedFrame> => {
const client = page._client();
const target = page.target();
@@ -63,20 +69,58 @@ export const screenshotTask = async ({
!process.env.DISABLE_FROM_SURFACE || height > 8192 || width > 8192;
const scaleFactor = fromSurface ? 1 : scale;
- const {value} = await client.send('Page.captureScreenshot', {
- format,
- quality: jpegQuality,
- clip: {
- x: 0,
- y: 0,
- height: height * scaleFactor,
- scale: 1,
- width: width * scaleFactor,
- },
- captureBeyondViewport: true,
- optimizeForSpeed: true,
- fromSurface,
- });
+ const reservation = await remotionSharedMemory?.acquire(page);
+ let value;
+ try {
+ ({value} = await client.send('Page.captureScreenshot', {
+ format,
+ quality: jpegQuality,
+ clip: {
+ x: 0,
+ y: 0,
+ height: height * scaleFactor,
+ scale: 1,
+ width: width * scaleFactor,
+ },
+ captureBeyondViewport: true,
+ optimizeForSpeed: true,
+ fromSurface,
+ remotionFrameSlot: reservation?.slot,
+ remotionFrameId: reservation?.frameId,
+ remotionFastViewport: reservation ? true : undefined,
+ }));
+ } catch (error) {
+ reservation?.fail();
+ throw error;
+ }
+
+ if (reservation) {
+ if (value.data !== '' || !value.remotionFrame) {
+ await reservation.discardPublished();
+ throw new Error(
+ 'Chromium did not return a Remotion shared-memory frame.',
+ );
+ }
+
+ const rawFrame = await reservation.publish(value.remotionFrame);
+ stopPerfMeasure(cap);
+ if (omitBackground) {
+ try {
+ await client.send('Emulation.setDefaultBackgroundColorOverride');
+ } catch (error) {
+ try {
+ await rawFrame.release();
+ } catch {
+ // Preserve the background reset error that made the capture fail.
+ }
+
+ throw error;
+ }
+ }
+
+ return rawFrame;
+ }
+
result = value;
}
diff --git a/packages/renderer/src/take-frame.ts b/packages/renderer/src/take-frame.ts
index 88a9575b45..1eb7b1872d 100644
--- a/packages/renderer/src/take-frame.ts
+++ b/packages/renderer/src/take-frame.ts
@@ -2,18 +2,12 @@ import type {Page} from './browser/BrowserPage';
import type {StillImageFormat, VideoImageFormat} from './image-format';
import {puppeteerEvaluateWithCatch} from './puppeteer-evaluate';
import {screenshot} from './puppeteer-screenshot';
+import type {
+ CapturedFrame,
+ RemotionSharedMemoryCapture,
+} from './remotion-shared-memory';
-export const takeFrame = async ({
- freePage,
- imageFormat,
- jpegQuality,
- width,
- height,
- output,
- scale,
- wantsBuffer,
- timeoutInMilliseconds,
-}: {
+type TakeFrameOptions = {
freePage: Page;
imageFormat: VideoImageFormat | StillImageFormat;
jpegQuality: number | undefined;
@@ -23,9 +17,28 @@ export const takeFrame = async ({
scale: number;
wantsBuffer: boolean;
timeoutInMilliseconds: number;
-}): Promise<Buffer | null> => {
+ remotionSharedMemory: RemotionSharedMemoryCapture | null;
+};
+
+type TakeFrameResult<T extends RemotionSharedMemoryCapture | null> =
+ T extends null ? Buffer | null : CapturedFrame | null;
+
+export const takeFrame = async <T extends RemotionSharedMemoryCapture | null>({
+ freePage,
+ imageFormat,
+ jpegQuality,
+ width,
+ height,
+ output,
+ scale,
+ wantsBuffer,
+ timeoutInMilliseconds,
+ remotionSharedMemory,
+}: Omit<TakeFrameOptions, 'remotionSharedMemory'> & {
+ remotionSharedMemory: T;
+}): Promise<TakeFrameResult<T>> => {
if (imageFormat === 'none') {
- return null;
+ return null as TakeFrameResult<T>;
}
if (
@@ -63,7 +76,8 @@ export const takeFrame = async ({
width,
height,
scale,
+ remotionSharedMemory,
});
- return buf;
+ return buf as TakeFrameResult<T>;
};
diff --git a/packages/renderer/src/test/get-chrome-download-url.test.ts b/packages/renderer/src/test/get-chrome-download-url.test.ts
new file mode 100644
index 0000000000..116bbffe9c
--- /dev/null
+++ b/packages/renderer/src/test/get-chrome-download-url.test.ts
@@ -0,0 +1,66 @@
+import {expect, test} from 'bun:test';
+import {getExpectedVersion} from '../browser/BrowserFetcher';
+import {
+ getChromeDownloadUrl,
+ TESTED_VERSION,
+} from '../browser/get-chrome-download-url';
+
+test('downloads the Remotion Chromium fork by default on Apple Silicon', () => {
+ expect(
+ getChromeDownloadUrl({
+ platform: 'mac-arm64',
+ version: null,
+ chromeMode: 'headless-shell',
+ }),
+ ).toBe(
+ `https://remotion.media/chromium-headless-shell-mac-arm64-${TESTED_VERSION}.zip?clear`,
+ );
+});
+
+test('keeps explicit Apple Silicon Chromium versions on the official CDN', () => {
+ expect(
+ getChromeDownloadUrl({
+ platform: 'mac-arm64',
+ version: '148.0.0.0',
+ chromeMode: 'headless-shell',
+ }),
+ ).toBe(
+ 'https://storage.googleapis.com/chrome-for-testing-public/148.0.0.0/mac-arm64/chrome-headless-shell-mac-arm64.zip',
+ );
+});
+
+test('keeps Chrome for Testing on the official CDN', () => {
+ expect(
+ getChromeDownloadUrl({
+ platform: 'mac-arm64',
+ version: null,
+ chromeMode: 'chrome-for-testing',
+ }),
+ ).toBe(
+ `https://storage.googleapis.com/chrome-for-testing-public/${TESTED_VERSION}/mac-arm64/chrome-mac-arm64.zip`,
+ );
+});
+
+test('uses a distinct cache marker for the Remotion Apple Silicon build', () => {
+ expect(
+ getExpectedVersion({
+ platform: 'mac-arm64',
+ version: null,
+ chromeMode: 'headless-shell',
+ }),
+ ).toBe(`${TESTED_VERSION}-remotion-v2`);
+ expect(
+ getExpectedVersion({
+ platform: 'mac-arm64',
+ version: TESTED_VERSION,
+ chromeMode: 'headless-shell',
+ }),
+ ).toBe(TESTED_VERSION);
+ expect(
+ getExpectedVersion({
+ platform: 'linux64',
+ version: null,
+ chromeMode: 'headless-shell',
+ }),
+ ).toBe(TESTED_VERSION);
+});
diff --git a/packages/renderer/src/test/remotion-shared-memory.test.ts b/packages/renderer/src/test/remotion-shared-memory.test.ts
new file mode 100644
index 0000000000..c295743ee2
--- /dev/null
+++ b/packages/renderer/src/test/remotion-shared-memory.test.ts
@@ -0,0 +1,534 @@
+import {expect, test} from 'bun:test';
+import {PassThrough, Writable} from 'node:stream';
+import type {Page} from '../browser/BrowserPage';
+import type {CaptureScreenshotRequest} from '../browser/devtools-types';
+import type {
+ RemotionRawFrame,
+ RemotionSharedMemoryPoolLifetime,
+} from '../remotion-shared-memory';
+import {RemotionSharedMemoryCapture} from '../remotion-shared-memory';
+import {
+ createRemotionSharedMemoryFfmpegBridge,
+ parseRemotionSharedMemoryAck,
+ serializeRemotionSharedMemoryFrame,
+ serializeRemotionSharedMemoryPool,
+ serializeRemotionSharedMemoryPoolRetirement,
+} from '../remotion-shared-memory-ffmpeg';
+import {screenshotTask} from '../screenshot-task';
+
+const makePoolLifetime = (): {
+ poolLifetime: RemotionSharedMemoryPoolLifetime;
+ retire: () => Promise<void>;
+} => {
+ let retired = false;
+ let retirement: Promise<void> | null = null;
+ let notifyRetirement!: () => void;
+ const retirementSignal = new Promise<void>((resolve) => {
+ notifyRetirement = resolve;
+ });
+ const listeners = new Set<() => Promise<void>>();
+ const poolLifetime: RemotionSharedMemoryPoolLifetime = {
+ isRetired: () => retired,
+ waitForRetirement: () => retirementSignal,
+ onRetired: (listener) => {
+ if (retired) {
+ return null;
+ }
+
+ listeners.add(listener);
+ return () => listeners.delete(listener);
+ },
+ };
+
+ return {
+ poolLifetime,
+ retire: () => {
+ if (!retirement) {
+ retired = true;
+ notifyRetirement();
+ retirement = Promise.all(
+ [...listeners].map((listener) => listener()),
+ ).then(() => undefined);
+ }
+
+ return retirement;
+ },
+ };
+};
+
+const makeRawFrame = (
+ release: () => Promise<void>,
+ poolLifetime = makePoolLifetime().poolLifetime,
+): RemotionRawFrame => ({
+ type: 'remotion-shared-memory',
+ poolId: 4,
+ sharedMemoryName: '/rmshm-123',
+ slotCount: 2,
+ slotCapacity: 1088,
+ slot: 1,
+ frameId: '9007199254740993',
+ width: 16,
+ height: 16,
+ stride: 64,
+ byteLength: 1024,
+ pixelFormat: 'bgra',
+ alphaType: 'straight',
+ colorSpace: 'srgb',
+ capturePath: 'viewport-redraw',
+ poolLifetime,
+ release,
+});
+
+test('bridges frame descriptors and releases Chromium slots on matching ACKs', async () => {
+ const control = new PassThrough();
+ const acknowledgements = new PassThrough();
+ let controlOutput = '';
+ control.on('data', (data) => {
+ controlOutput += data.toString();
+ });
+ const lifetime = makePoolLifetime();
+ let releases = 0;
+ const frame = makeRawFrame(() => {
+ releases++;
+ return Promise.resolve();
+ }, lifetime.poolLifetime);
+ const bridge = createRemotionSharedMemoryFfmpegBridge({
+ control,
+ acknowledgements,
+ });
+
+ const {waitForAck} = await bridge.writeFrame({frame, pts: 7});
+ expect(controlOutput).toBe(
+ serializeRemotionSharedMemoryPool(frame) +
+ serializeRemotionSharedMemoryFrame({frame, pts: 7}),
+ );
+ expect(releases).toBe(0);
+
+ acknowledgements.write('4\t1\t9007199254740993\n');
+ await waitForAck;
+ expect(releases).toBe(1);
+ expect(parseRemotionSharedMemoryAck('4\t1\t9007199254740993')).toEqual({
+ poolId: 4,
+ slot: 1,
+ frameId: '9007199254740993',
+ });
+
+ await bridge.finish();
+ await lifetime.retire();
+ expect(controlOutput).not.toContain(
+ serializeRemotionSharedMemoryPoolRetirement(frame.poolId),
+ );
+ expect(() => parseRemotionSharedMemoryAck('4\t1\tunsafe\n')).toThrow(
+ 'Invalid Remotion shared-memory ACK',
+ );
+});
+
+test('unregisters a retired pool only after its final frame ACK', async () => {
+ const control = new PassThrough();
+ const acknowledgements = new PassThrough();
+ let controlOutput = '';
+ control.on('data', (data) => {
+ controlOutput += data.toString();
+ });
+ const lifetime = makePoolLifetime();
+ let releases = 0;
+ const frame = makeRawFrame(() => {
+ releases++;
+ return Promise.resolve();
+ }, lifetime.poolLifetime);
+ const bridge = createRemotionSharedMemoryFfmpegBridge({
+ control,
+ acknowledgements,
+ });
+
+ const {waitForAck} = await bridge.writeFrame({frame, pts: 3});
+ let retirementFinished = false;
+ const retirement = lifetime.retire().then(() => {
+ retirementFinished = true;
+ });
+ await Promise.resolve();
+ expect(retirementFinished).toBe(false);
+ expect(controlOutput).not.toContain(
+ serializeRemotionSharedMemoryPoolRetirement(frame.poolId),
+ );
+
+ acknowledgements.write('4\t1\t9007199254740993\n');
+ await waitForAck;
+ await retirement;
+ expect(releases).toBe(1);
+ expect(controlOutput).toBe(
+ serializeRemotionSharedMemoryPool(frame) +
+ serializeRemotionSharedMemoryFrame({frame, pts: 3}) +
+ serializeRemotionSharedMemoryPoolRetirement(frame.poolId),
+ );
+ await bridge.finish();
+});
+
+test('does not register or unregister a pool retired before bridge ownership', async () => {
+ const control = new PassThrough();
+ const acknowledgements = new PassThrough();
+ let controlOutput = '';
+ control.on('data', (data) => {
+ controlOutput += data.toString();
+ });
+ const lifetime = makePoolLifetime();
+ await lifetime.retire();
+ let releases = 0;
+ const frame = makeRawFrame(() => {
+ releases++;
+ return Promise.resolve();
+ }, lifetime.poolLifetime);
+ const bridge = createRemotionSharedMemoryFfmpegBridge({
+ control,
+ acknowledgements,
+ });
+
+ await expect(bridge.writeFrame({frame, pts: 0})).rejects.toThrow(
+ 'Target closed',
+ );
+ expect(releases).toBe(1);
+ expect(controlOutput).toBe('');
+ await bridge.finish();
+});
+
+test('settles and releases frames when the ACK or bridge fails', async () => {
+ const control = new PassThrough();
+ const acknowledgements = new PassThrough();
+ let failedReleaseCalls = 0;
+ const failedReleaseFrame = makeRawFrame(() => {
+ failedReleaseCalls++;
+ return Promise.reject(new Error('CDP release failed'));
+ });
+ const bridge = createRemotionSharedMemoryFfmpegBridge({
+ control,
+ acknowledgements,
+ });
+ const {waitForAck} = await bridge.writeFrame({
+ frame: failedReleaseFrame,
+ pts: 0,
+ });
+ acknowledgements.write('4\t1\t9007199254740993\n');
+ await expect(waitForAck).rejects.toThrow('CDP release failed');
+ expect(failedReleaseCalls).toBe(1);
+
+ const writeError = new Error('control pipe failed');
+ const failedControl = new Writable({
+ highWaterMark: 1,
+ write: (_chunk, _encoding, callback) => callback(writeError),
+ });
+ const unusedAcknowledgements = new PassThrough();
+ const writeFailureLifetime = makePoolLifetime();
+ let writeFailureReleases = 0;
+ const writeFailureFrame = makeRawFrame(() => {
+ writeFailureReleases++;
+ return Promise.resolve();
+ }, writeFailureLifetime.poolLifetime);
+ const failedBridge = createRemotionSharedMemoryFfmpegBridge({
+ control: failedControl,
+ acknowledgements: unusedAcknowledgements,
+ });
+ await expect(
+ failedBridge.writeFrame({frame: writeFailureFrame, pts: 0}),
+ ).rejects.toThrow('control pipe failed');
+ await writeFailureLifetime.retire();
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ expect(writeFailureReleases).toBe(1);
+});
+
+test('treats an ACK pipe EOF before finish as fatal', async () => {
+ const control = new PassThrough();
+ const acknowledgements = new PassThrough();
+ const bridge = createRemotionSharedMemoryFfmpegBridge({
+ control,
+ acknowledgements,
+ });
+ acknowledgements.end();
+ await new Promise((resolve) => acknowledgements.once('end', resolve));
+ await Promise.resolve();
+
+ let releases = 0;
+ const frame = makeRawFrame(() => {
+ releases++;
+ return Promise.resolve();
+ });
+ await expect(bridge.writeFrame({frame, pts: 0})).rejects.toThrow(
+ 'before input finished',
+ );
+ expect(releases).toBe(1);
+ control.end();
+});
+
+test('signals pool retirement before waiting for an unhanded frame to drain', async () => {
+ const calls: string[] = [];
+ const client = {
+ send: (method: string) => {
+ calls.push(method);
+ if (method === 'Page.remotionCreateFramePool') {
+ return Promise.resolve({
+ value: {
+ sharedMemoryName: '/rmshm-retirement',
+ slotCount: 1,
+ slotCapacity: 1088,
+ },
+ });
+ }
+
+ return Promise.resolve({value: undefined});
+ },
+ };
+ const page = {
+ _client: () => client,
+ target: () => ({_targetId: 'target'}),
+ } as unknown as Page;
+ const capture = new RemotionSharedMemoryCapture({
+ width: 16,
+ height: 16,
+ indent: false,
+ logLevel: 'error',
+ });
+ await capture.ensurePage(page);
+ const reservation = await capture.acquire(page);
+ if (!reservation) {
+ throw new Error('Expected a shared-memory reservation.');
+ }
+
+ const frame = await reservation.publish({
+ slot: reservation.slot,
+ frameId: reservation.frameId,
+ width: 16,
+ height: 16,
+ stride: 64,
+ byteLength: 1024,
+ pixelFormat: 'bgra',
+ alphaType: 'straight',
+ colorSpace: 'srgb',
+ capturePath: 'viewport-redraw',
+ });
+ let forgotten = false;
+ const forget = capture.forgetPage(page).then(() => {
+ forgotten = true;
+ });
+ await frame.poolLifetime.waitForRetirement();
+ await Promise.resolve();
+ expect(forgotten).toBe(false);
+ expect(calls).not.toContain('Page.remotionDestroyFramePool');
+
+ await frame.release();
+ await forget;
+ expect(calls).toContain('Page.remotionDestroyFramePool');
+});
+
+test('releases a published frame if transparent background reset fails', async () => {
+ let backgroundCalls = 0;
+ let releaseCalls = 0;
+ const client = {
+ send: (
+ method: string,
+ params?: Record<string, unknown>,
+ ): Promise<{value: unknown}> => {
+ if (method === 'Page.remotionCreateFramePool') {
+ return Promise.resolve({
+ value: {
+ sharedMemoryName: '/rmshm-reset',
+ slotCount: 1,
+ slotCapacity: 1088,
+ },
+ });
+ }
+
+ if (method === 'Page.captureScreenshot') {
+ const request = params as CaptureScreenshotRequest;
+ return Promise.resolve({
+ value: {
+ data: '',
+ remotionFrame: {
+ slot: request.remotionFrameSlot,
+ frameId: request.remotionFrameId,
+ width: 16,
+ height: 16,
+ stride: 64,
+ byteLength: 1024,
+ pixelFormat: 'bgra',
+ alphaType: 'straight',
+ colorSpace: 'srgb',
+ capturePath: 'viewport-redraw',
+ },
+ },
+ });
+ }
+
+ if (method === 'Emulation.setDefaultBackgroundColorOverride') {
+ backgroundCalls++;
+ if (backgroundCalls === 2) {
+ return Promise.reject(new Error('background reset failed'));
+ }
+ }
+
+ if (method === 'Page.remotionReleaseFrame') {
+ releaseCalls++;
+ }
+
+ return Promise.resolve({value: undefined});
+ },
+ };
+ const page = {
+ _client: () => client,
+ target: () => ({_targetId: 'target'}),
+ } as unknown as Page;
+ const capture = new RemotionSharedMemoryCapture({
+ width: 16,
+ height: 16,
+ indent: false,
+ logLevel: 'error',
+ });
+ await capture.ensurePage(page);
+
+ await expect(
+ screenshotTask({
+ format: 'png',
+ height: 16,
+ width: 16,
+ omitBackground: true,
+ page,
+ jpegQuality: undefined,
+ scale: 1,
+ remotionSharedMemory: capture,
+ }),
+ ).rejects.toThrow('background reset failed');
+ expect(releaseCalls).toBe(1);
+ await capture.destroy();
+});
+
+test('uses raw viewport capture for every video image format and falls back on vanilla Chrome', async () => {
+ const rawCaptureRequests: CaptureScreenshotRequest[] = [];
+ const released: Array<{slot: number; frameId: string}> = [];
+ const rawClient = {
+ send: (method: string, params?: Record<string, unknown>) => {
+ if (method === 'Page.remotionCreateFramePool') {
+ return Promise.resolve({
+ value: {
+ sharedMemoryName: '/rmshm-test',
+ slotCount: 1,
+ slotCapacity: 1088,
+ },
+ });
+ }
+
+ if (method === 'Page.captureScreenshot') {
+ const request = params as CaptureScreenshotRequest;
+ rawCaptureRequests.push(request);
+ return Promise.resolve({
+ value: {
+ data: '',
+ remotionFrame: {
+ slot: request.remotionFrameSlot,
+ frameId: request.remotionFrameId,
+ width: 16,
+ height: 16,
+ stride: 64,
+ byteLength: 1024,
+ pixelFormat: 'bgra',
+ alphaType: request.format === 'jpeg' ? 'opaque' : 'straight',
+ colorSpace: 'srgb',
+ capturePath: 'viewport-redraw',
+ },
+ },
+ });
+ }
+
+ if (method === 'Page.remotionReleaseFrame') {
+ released.push(params as {slot: number; frameId: string});
+ }
+
+ return Promise.resolve({value: undefined});
+ },
+ };
+ const rawPage = {
+ _client: () => rawClient,
+ target: () => ({_targetId: 'target'}),
+ } as unknown as Page;
+ const rawCapture = new RemotionSharedMemoryCapture({
+ width: 16,
+ height: 16,
+ indent: false,
+ logLevel: 'error',
+ });
+ await rawCapture.ensurePage(rawPage);
+
+ for (const format of ['jpeg', 'png', 'webp'] as const) {
+ const frame = await screenshotTask({
+ format,
+ height: 16,
+ width: 16,
+ omitBackground: format !== 'jpeg',
+ page: rawPage,
+ jpegQuality: format === 'jpeg' ? 80 : undefined,
+ scale: 1,
+ remotionSharedMemory: rawCapture,
+ });
+ expect(Buffer.isBuffer(frame)).toBe(false);
+ if (!Buffer.isBuffer(frame)) {
+ await frame.release();
+ }
+ }
+
+ expect(
+ rawCaptureRequests.map((request) => ({
+ format: request.format,
+ fast: request.remotionFastViewport,
+ hasSlot: request.remotionFrameSlot !== undefined,
+ })),
+ ).toEqual([
+ {format: 'jpeg', fast: true, hasSlot: true},
+ {format: 'png', fast: true, hasSlot: true},
+ {format: 'webp', fast: true, hasSlot: true},
+ ]);
+ expect(released).toHaveLength(3);
+ await rawCapture.destroy();
+
+ const fallbackRequests: CaptureScreenshotRequest[] = [];
+ const vanillaClient = {
+ send: (method: string, params?: Record<string, unknown>) => {
+ if (method === 'Page.remotionCreateFramePool') {
+ return Promise.reject(
+ new Error(
+ "Protocol error (Page.remotionCreateFramePool): 'Page.remotionCreateFramePool' wasn't found",
+ ),
+ );
+ }
+
+ if (method === 'Page.captureScreenshot') {
+ fallbackRequests.push(params as CaptureScreenshotRequest);
+ return Promise.resolve({
+ value: {data: Buffer.from('jpeg').toString('base64')},
+ });
+ }
+
+ return Promise.resolve({value: undefined});
+ },
+ };
+ const vanillaPage = {
+ _client: () => vanillaClient,
+ target: () => ({_targetId: 'target'}),
+ } as unknown as Page;
+ const fallbackCapture = new RemotionSharedMemoryCapture({
+ width: 16,
+ height: 16,
+ indent: false,
+ logLevel: 'error',
+ });
+ await fallbackCapture.ensurePage(vanillaPage);
+ const encoded = await screenshotTask({
+ format: 'jpeg',
+ height: 16,
+ width: 16,
+ omitBackground: false,
+ page: vanillaPage,
+ jpegQuality: 80,
+ scale: 1,
+ remotionSharedMemory: fallbackCapture,
+ });
+ expect(encoded).toEqual(Buffer.from('jpeg'));
+ expect(fallbackRequests[0]?.remotionFastViewport).toBeUndefined();
+ expect(fallbackRequests[0]?.remotionFrameSlot).toBeUndefined();
+});