🎥 Make videos programmatically with React
Use shared-memory frame capture on Apple Silicon
Jonny Burger80ec1aad2276
diff --git a/packages/cli/src/render-flows/render.ts b/packages/cli/src/render-flows/render.tsindex 58f851a846..be547afb75 100644--- a/packages/cli/src/render-flows/render.ts+++ b/packages/cli/src/render-flows/render.ts@@ -658,6 +658,8 @@ export const renderVideoFlow = async ({sampleRate: resolvedSampleRate,onBrowserLog: null,onFrameBuffer: null,+ onFrame: null,+ remotionSharedMemory: null,logLevel,serializedResolvedPropsWithCustomSchema:NoReactInternals.serializeJSONWithSpecialTypes({diff --git a/packages/compositor-darwin-arm64/ffmpeg b/packages/compositor-darwin-arm64/ffmpegindex 998f5cecf5..a4ffbe1b9b 100755Binary files a/packages/compositor-darwin-arm64/ffmpeg and b/packages/compositor-darwin-arm64/ffmpeg differdiff --git a/packages/compositor-darwin-arm64/ffprobe b/packages/compositor-darwin-arm64/ffprobeindex 6dde0b0eff..9031c32cdb 100755Binary files a/packages/compositor-darwin-arm64/ffprobe and b/packages/compositor-darwin-arm64/ffprobe differdiff --git a/packages/compositor-darwin-arm64/libavcodec.dylib b/packages/compositor-darwin-arm64/libavcodec.dylibindex 4fc2093e35..f2dedf4896 100755Binary files a/packages/compositor-darwin-arm64/libavcodec.dylib and b/packages/compositor-darwin-arm64/libavcodec.dylib differdiff --git a/packages/compositor-darwin-arm64/libavdevice.dylib b/packages/compositor-darwin-arm64/libavdevice.dylibindex ac3838c4b2..eb0fc3a21f 100755Binary files a/packages/compositor-darwin-arm64/libavdevice.dylib and b/packages/compositor-darwin-arm64/libavdevice.dylib differdiff --git a/packages/compositor-darwin-arm64/libavfilter.dylib b/packages/compositor-darwin-arm64/libavfilter.dylibindex 46503ae772..c8c0f6914c 100755Binary files a/packages/compositor-darwin-arm64/libavfilter.dylib and b/packages/compositor-darwin-arm64/libavfilter.dylib differdiff --git a/packages/compositor-darwin-arm64/libavformat.dylib b/packages/compositor-darwin-arm64/libavformat.dylibindex e5fcffa15c..18691f2c2b 100755Binary files a/packages/compositor-darwin-arm64/libavformat.dylib and b/packages/compositor-darwin-arm64/libavformat.dylib differdiff --git a/packages/compositor-darwin-arm64/libavutil.dylib b/packages/compositor-darwin-arm64/libavutil.dylibindex 28e7e247b9..3a3754fdd5 100755Binary files a/packages/compositor-darwin-arm64/libavutil.dylib and b/packages/compositor-darwin-arm64/libavutil.dylib differdiff --git a/packages/compositor-darwin-arm64/libswresample.dylib b/packages/compositor-darwin-arm64/libswresample.dylibindex 1e91290642..d923e601c7 100755Binary files a/packages/compositor-darwin-arm64/libswresample.dylib and b/packages/compositor-darwin-arm64/libswresample.dylib differdiff --git a/packages/compositor-darwin-arm64/libswscale.dylib b/packages/compositor-darwin-arm64/libswscale.dylibindex c708e3df63..d8a99807fc 100755Binary files a/packages/compositor-darwin-arm64/libswscale.dylib and b/packages/compositor-darwin-arm64/libswscale.dylib differdiff --git a/packages/renderer/src/browser/BrowserFetcher.ts b/packages/renderer/src/browser/BrowserFetcher.tsindex 35760665e0..41935b33c4 100644--- a/packages/renderer/src/browser/BrowserFetcher.ts+++ b/packages/renderer/src/browser/BrowserFetcher.ts@@ -54,7 +54,7 @@ interface BrowserFetcherRevisionInfo {local: boolean;}-const getPlatform = (): Platform => {+export const getPlatform = (): Platform => {const platform = os.platform();switch (platform) {case 'darwin':@@ -82,13 +82,23 @@ const getVersionFilePath = (chromeMode: ChromeMode): string => {return path.join(downloadsFolder, 'VERSION');};-const getExpectedVersion = (- version: string | null,- _chromeMode: ChromeMode,-): string => {+export const getExpectedVersion = ({+ version,+ chromeMode,+ platform,+}: {+ version: string | null;+ chromeMode: ChromeMode;+ platform: Platform;+}): string => {if (version) {return version;}++ if (chromeMode === 'headless-shell' && platform === 'mac-arm64') {+ return `${TESTED_VERSION}-remotion-v2`;+ }+return TESTED_VERSION;};@@ -129,7 +139,7 @@ export const downloadBrowser = async ({const downloadsFolder = getDownloadsFolder(chromeMode);const archivePath = path.join(downloadsFolder, fileName);const outputPath = getFolderPath(downloadsFolder, platform);- const expectedVersion = getExpectedVersion(version, chromeMode);+ const expectedVersion = getExpectedVersion({version, chromeMode, platform});if (await existsAsync(outputPath)) {const installedVersion = readVersionFile(chromeMode);diff --git a/packages/renderer/src/browser/devtools-commands.ts b/packages/renderer/src/browser/devtools-commands.tsindex f468648d5d..63fc8e37af 100644--- a/packages/renderer/src/browser/devtools-commands.ts+++ b/packages/renderer/src/browser/devtools-commands.ts@@ -19,6 +19,9 @@ import type {NavigateResponse,PrintPDFRequest,PrintPDFResponse,+ RemotionCreateFramePoolRequest,+ RemotionCreateFramePoolResponse,+ RemotionReleaseFrameRequest,ReleaseObjectRequest,SetAutoAttachRequest,SetDefaultBackgroundColorOverrideRequest,@@ -243,6 +246,18 @@ export interface Commands {paramsType: [CaptureScreenshotRequest?];returnType: CaptureScreenshotResponse;};+ 'Page.remotionCreateFramePool': {+ paramsType: [RemotionCreateFramePoolRequest];+ returnType: RemotionCreateFramePoolResponse;+ };+ 'Page.remotionReleaseFrame': {+ paramsType: [RemotionReleaseFrameRequest];+ returnType: void;+ };+ 'Page.remotionDestroyFramePool': {+ paramsType: [];+ returnType: void;+ };'Page.printToPDF': {paramsType: [PrintPDFRequest];returnType: PrintPDFResponse;diff --git a/packages/renderer/src/browser/devtools-types.ts b/packages/renderer/src/browser/devtools-types.tsindex 06837d5056..d3653a15ac 100644--- a/packages/renderer/src/browser/devtools-types.ts+++ b/packages/renderer/src/browser/devtools-types.ts@@ -733,6 +733,21 @@ export interface CaptureScreenshotRequest {* Optimize image encoding for speed, not for resulting size (defaults to false) EXPERIMENTAL*/optimizeForSpeed?: boolean;+ /**+ * Writes the captured pixels into this slot of a Remotion shared-memory+ * frame pool instead of encoding an image. Remotion Chromium only.+ */+ remotionFrameSlot?: number;+ /**+ * Ownership token for the shared-memory slot. Must be supplied together+ * with `remotionFrameSlot`. Remotion Chromium only.+ */+ remotionFrameId?: string;+ /**+ * Skips the temporary viewport resize used by the regular CDP screenshot+ * path. Remotion Chromium only.+ */+ remotionFastViewport?: boolean;}export interface CaptureScreenshotResponse {@@ -740,6 +755,36 @@ export interface CaptureScreenshotResponse {* Base64-encoded image data. (Encoded as a base64 string when passed over JSON)*/data: string;+ remotionFrame?: RemotionRawFrame;+}++export interface RemotionRawFrame {+ slot: number;+ frameId: string;+ width: number;+ height: number;+ stride: number;+ byteLength: number;+ pixelFormat: 'bgra';+ alphaType: 'opaque' | 'straight';+ colorSpace: 'srgb';+ capturePath: string;+}++export interface RemotionCreateFramePoolRequest {+ slotCount: number;+ slotCapacity: number;+}++export interface RemotionCreateFramePoolResponse {+ sharedMemoryName: string;+ slotCount: number;+ slotCapacity: number;+}++export interface RemotionReleaseFrameRequest {+ slot: number;+ frameId: string;}export interface PrintPDFRequest {diff --git a/packages/renderer/src/browser/get-chrome-download-url.ts b/packages/renderer/src/browser/get-chrome-download-url.tsindex bd7b816685..b80ec92a50 100644--- a/packages/renderer/src/browser/get-chrome-download-url.ts+++ b/packages/renderer/src/browser/get-chrome-download-url.ts@@ -129,6 +129,10 @@ export function getChromeDownloadUrl({}if (chromeMode === 'headless-shell') {+ if (platform === 'mac-arm64' && version === null) {+ return `https://remotion.media/chromium-headless-shell-mac-arm64-${TESTED_VERSION}.zip?clear`;+ }+// Amazon Linux 2023 needs a special build.// This binary is compatible with older glibc (no 2.35 requirement).if (isAmazonLinux2023() && platform === 'linux64' && !version) {diff --git a/packages/renderer/src/ensure-browser.ts b/packages/renderer/src/ensure-browser.tsindex 1202b79d91..bd1c054a43 100644--- a/packages/renderer/src/ensure-browser.ts+++ b/packages/renderer/src/ensure-browser.ts@@ -3,6 +3,8 @@ import type {BrowserExecutable} from './browser-executable';import {defaultBrowserDownloadProgress} from './browser/browser-download-progress-bar';import {downloadBrowser,+ getExpectedVersion,+ getPlatform,getRevisionInfo,readVersionFile,TESTED_VERSION,@@ -50,7 +52,26 @@ const internalEnsureBrowserUncapped = async ({onBrowserDownload,chromeMode,}: InternalEnsureBrowserOptions): Promise<BrowserStatus> => {- const status = getBrowserStatus({browserExecutable, chromeMode});+ const defaultStatus = getBrowserStatus({+ browserExecutable,+ chromeMode,+ version: null,+ });+ if (+ defaultStatus.type === 'local-puppeteer-browser' ||+ defaultStatus.type === 'user-defined-path'+ ) {+ return defaultStatus;+ }++ const {onProgress, version} = onBrowserDownload({chromeMode});+ const status = version+ ? getBrowserStatus({browserExecutable, chromeMode, version})+ : defaultStatus;+ if (status.type === 'local-puppeteer-browser') {+ return status;+ }+if (status.type === 'version-mismatch') {const versionInfo = status.actualVersion? ` (installed: ${status.actualVersion})`@@ -62,12 +83,10 @@ const internalEnsureBrowserUncapped = async ({}if (status.type === 'no-browser' || status.type === 'version-mismatch') {- const {onProgress, version} = onBrowserDownload({chromeMode});-await downloadBrowser({indent, logLevel, onProgress, version, chromeMode});}- const newStatus = getBrowserStatus({browserExecutable, chromeMode});+ const newStatus = getBrowserStatus({browserExecutable, chromeMode, version});return newStatus;};@@ -83,9 +102,11 @@ export const internalEnsureBrowser = (const getBrowserStatus = ({browserExecutable,chromeMode,+ version,}: {browserExecutable: BrowserExecutable;chromeMode: ChromeMode;+ version: string | null;}): BrowserStatus => {if (browserExecutable) {if (!fs.existsSync(browserExecutable)) {@@ -100,7 +121,14 @@ const getBrowserStatus = ({const revision = getRevisionInfo(chromeMode);if (revision.local && fs.existsSync(revision.executablePath)) {const actualVersion = readVersionFile(chromeMode);- if (actualVersion === TESTED_VERSION) {+ if (+ actualVersion ===+ getExpectedVersion({+ version,+ chromeMode,+ platform: getPlatform(),+ })+ ) {return {path: revision.executablePath, type: 'local-puppeteer-browser'};}diff --git a/packages/renderer/src/open-browser.ts b/packages/renderer/src/open-browser.tsindex 0816a6990f..c7137ec61e 100644--- a/packages/renderer/src/open-browser.ts+++ b/packages/renderer/src/open-browser.ts@@ -271,6 +271,7 @@ export const internalOpenBrowser = async ({chromiumOptions?.userAgent? `--user-agent="${chromiumOptions.userAgent}"`: null,+ '--remotion-shared-memory-capture','--remote-debugging-port=0',`--user-data-dir=${userDataDir}`,].filter(Boolean) as string[],diff --git a/packages/renderer/src/prespawn-ffmpeg.ts b/packages/renderer/src/prespawn-ffmpeg.tsindex a91dbddd4a..8f8977c1a3 100644--- a/packages/renderer/src/prespawn-ffmpeg.ts+++ b/packages/renderer/src/prespawn-ffmpeg.ts@@ -1,3 +1,4 @@+import type {Readable, Writable} from 'node:stream';import type {_InternalTypes} from 'remotion';import type {OnLog} from './browser/BrowserPage';import {callFf} from './call-ffmpeg';@@ -20,8 +21,11 @@ import {validateSelectedPixelFormatAndCodecCombination,} from './pixel-format';import {resolveHardwareAcceleration} from './probe-encoder';+import type {RemotionRawFrame} from './remotion-shared-memory';+import {createRemotionSharedMemoryFfmpegBridge} from './remotion-shared-memory-ffmpeg';import {validateDimension, validateFps} from './validate';import {validateEvenDimensionsWithCodec} from './validate-even-dimensions-with-codec';+import {writeWithBackpressure} from './write-with-backpressure';type RunningStatus =| {@@ -62,6 +66,63 @@ type PreStitcherOptions = {binariesDirectory: string | null;hardwareAcceleration: HardwareAccelerationOption;onLog: OnLog;+ inputMode: 'encoded-image' | 'remotion-shared-memory';+};++const remotionSharedMemorySupport = new Map<string, Promise<boolean>>();++export const ffmpegSupportsRemotionSharedMemory = ({+ binariesDirectory,+ indent,+ logLevel,+}: {+ binariesDirectory: string | null;+ indent: boolean;+ logLevel: LogLevel;+}) => {+ const key = binariesDirectory ?? '<bundled>';+ const cached = remotionSharedMemorySupport.get(key);+ if (cached) {+ return cached;+ }++ const result = (async () => {+ try {+ const task = callFf({+ bin: 'ffmpeg',+ args: ['-hide_banner', '-devices'],+ indent,+ logLevel,+ binariesDirectory,+ cancelSignal: undefined,+ });+ const output = await task;+ return /(?:^|\n)\s*D\s{2}remotionshm(?:\s|$)/.test(+ `${output.stdout}\n${output.stderr}`,+ );+ } catch {+ return false;+ }+ })();+ remotionSharedMemorySupport.set(key, result);+ return result;+};++const fpsAsFraction = (fps: number) => {+ const denominator = 1_000_000;+ let numerator = Math.round(fps * denominator);+ let divisor = denominator;+ let left = numerator;+ let right = divisor;+ while (right !== 0) {+ const remainder = left % right;+ left = right;+ right = remainder;+ }++ numerator /= left;+ divisor /= left;+ return `${numerator}/${divisor}`;};export const prespawnFfmpeg = (options: PreStitcherOptions) => {@@ -104,32 +165,58 @@ export const prespawnFfmpeg = (options: PreStitcherOptions) => {onLog: options.onLog,});+ const encodingArgs = generateFfmpegArgs({+ hasPreencoded: false,+ proResProfileName,+ pixelFormat,+ x264Preset: options.x264Preset,+ gopSize: options.gopSize,+ codec,+ crf: options.crf,+ videoBitrate: options.videoBitrate,+ encodingMaxRate: options.encodingMaxRate,+ encodingBufferSize: options.encodingBufferSize,+ colorSpace: options.colorSpace,+ hardwareAcceleration: resolvedHardwareAcceleration,+ indent: options.indent,+ logLevel: options.logLevel,+ });+ const encodingArgsWithOwnedPixels =+ options.inputMode === 'remotion-shared-memory'+ ? encodingArgs.some(([flag]) => flag === '-vf')+ ? encodingArgs.map((args) =>+ args[0] === '-vf' ? ['-vf', `copy,${args[1]}`] : args,+ )+ : [...encodingArgs, ['-vf', 'copy']]+ : encodingArgs;+const ffmpegArgs = [- ['-r', options.fps],- ...[- ['-f', 'image2pipe'],- ['-s', `${options.width}x${options.height}`],- // If scale is very small (like 0.1), FFMPEG cannot figure out the image- // format on it's own and we need to hint the format- ['-vcodec', options.imageFormat === 'jpeg' ? 'mjpeg' : 'png'],- ['-i', '-'],- ],- ...generateFfmpegArgs({- hasPreencoded: false,- proResProfileName,- pixelFormat,- x264Preset: options.x264Preset,- gopSize: options.gopSize,- codec,- crf: options.crf,- videoBitrate: options.videoBitrate,- encodingMaxRate: options.encodingMaxRate,- encodingBufferSize: options.encodingBufferSize,- colorSpace: options.colorSpace,- hardwareAcceleration: resolvedHardwareAcceleration,- indent: options.indent,- logLevel: options.logLevel,- }),+ ...(options.inputMode === 'remotion-shared-memory'+ ? [+ ['-nostdin'],+ ['-xerror'],+ ['-nofind_stream_info'],+ ['-threads:v', '1'],+ ['-f', 'remotionshm'],+ ['-video_size', `${options.width}x${options.height}`],+ ['-framerate', fpsAsFraction(options.fps)],+ ['-control_fd', '3'],+ ['-ack_fd', '4'],+ ['-i', 'remotion'],+ ]+ : [+ ['-r', options.fps],+ ['-f', 'image2pipe'],+ ['-s', `${options.width}x${options.height}`],+ // If scale is very small (like 0.1), FFMPEG cannot figure out the image+ // format on its own and we need to hint the format.+ ['-vcodec', options.imageFormat === 'jpeg' ? 'mjpeg' : 'png'],+ ['-i', '-'],+ ]),+ ...encodingArgsWithOwnedPixels,+ options.inputMode === 'remotion-shared-memory'+ ? ['-fps_mode', 'passthrough']+ : null,'-y',options.outputLocation,@@ -164,7 +251,18 @@ export const prespawnFfmpeg = (options: PreStitcherOptions) => {logLevel: options.logLevel,binariesDirectory: options.binariesDirectory,cancelSignal: options.signal,+ options:+ options.inputMode === 'remotion-shared-memory'+ ? {stdio: ['ignore', 'pipe', 'pipe', 'pipe', 'pipe']}+ : undefined,});+ const remotionSharedMemory =+ options.inputMode === 'remotion-shared-memory'+ ? createRemotionSharedMemoryFfmpegBridge({+ control: task.stdio[3] as Writable,+ acknowledgements: task.stdio[4] as Readable,+ })+ : null;let ffmpegOutput = '';task.stderr?.on('data', (data: Buffer) => {@@ -198,5 +296,46 @@ export const prespawnFfmpeg = (options: PreStitcherOptions) => {}});- return {task, getLogs: () => ffmpegOutput, getExitStatus: () => exitCode};+ return {+ task,+ getLogs: () => ffmpegOutput,+ getExitStatus: () => exitCode,+ writeFrame: async ({+ frame,+ pts,+ }: {+ frame: Buffer | RemotionRawFrame;+ pts: number;+ }) => {+ if (remotionSharedMemory) {+ if (Buffer.isBuffer(frame)) {+ throw new Error('Expected a Remotion shared-memory frame.');+ }++ return remotionSharedMemory.writeFrame({frame, pts});+ }++ if (!Buffer.isBuffer(frame)) {+ throw new Error('Expected an encoded image frame.');+ }++ if (!task.stdin) {+ throw new Error('FFmpeg stdin is not available.');+ }++ await writeWithBackpressure({data: frame, writable: task.stdin});+ return {waitForAck: Promise.resolve()};+ },+ finishInput: async () => {+ if (remotionSharedMemory) {+ await remotionSharedMemory.finish();+ return;+ }++ task.stdin?.end();+ },+ abortInput: (error: Error) => {+ remotionSharedMemory?.fail(error);+ },+ };};diff --git a/packages/renderer/src/puppeteer-screenshot.ts b/packages/renderer/src/puppeteer-screenshot.tsindex 4128c900d7..6ee956848b 100644--- a/packages/renderer/src/puppeteer-screenshot.ts+++ b/packages/renderer/src/puppeteer-screenshot.ts@@ -1,6 +1,10 @@import * as assert from 'node:assert';import type {Page} from './browser/BrowserPage';import type {StillImageFormat} from './image-format';+import type {+ CapturedFrame,+ RemotionSharedMemoryCapture,+} from './remotion-shared-memory';import {screenshotTask} from './screenshot-task';export const screenshot = (options: {@@ -12,7 +16,8 @@ export const screenshot = (options: {width: number;height: number;scale: number;-}): Promise<Buffer> => {+ remotionSharedMemory: RemotionSharedMemoryCapture | null;+}): Promise<CapturedFrame> => {if (options.jpegQuality) {assert.ok(typeof options.jpegQuality === 'number',@@ -40,6 +45,7 @@ export const screenshot = (options: {path: options.path,jpegQuality: options.type === 'jpeg' ? options.jpegQuality : undefined,scale: options.scale,+ remotionSharedMemory: options.remotionSharedMemory,}),);};diff --git a/packages/renderer/src/remotion-shared-memory-ffmpeg.ts b/packages/renderer/src/remotion-shared-memory-ffmpeg.tsnew file mode 100644index 0000000000..caee71e5ee--- /dev/null+++ b/packages/renderer/src/remotion-shared-memory-ffmpeg.ts@@ -0,0 +1,421 @@+import {createInterface} from 'node:readline';+import type {Readable, Writable} from 'node:stream';+import type {+ RemotionRawFrame,+ RemotionSharedMemoryPoolLifetime,+} from './remotion-shared-memory';+import {writeWithBackpressure} from './write-with-backpressure';++type RegisteredPool = {+ poolId: number;+ sharedMemoryName: string;+ poolLifetime: RemotionSharedMemoryPoolLifetime;+ registered: boolean;+ registrationStarted: boolean;+ registrationSettled: boolean;+ registration: Promise<boolean>;+ resolveRegistration: (registered: boolean) => void;+ retired: boolean;+ pendingCount: number;+ drainWaiters: Array<() => void>;+ unregistering: Promise<void> | null;+ unsubscribe: (() => void) | null;+};++type PendingFrame = {+ frame: RemotionRawFrame;+ pool: RegisteredPool;+ resolve: () => void;+ reject: (error: Error) => void;+};++const pendingKey = (poolId: number, slot: number) => `${poolId}:${slot}`;++export const serializeRemotionSharedMemoryPool = (frame: RemotionRawFrame) =>+ `P\t${frame.poolId}\t${frame.sharedMemoryName}\t${frame.slotCount}\t${frame.slotCapacity}\n`;++export const serializeRemotionSharedMemoryFrame = ({+ frame,+ pts,+}: {+ frame: RemotionRawFrame;+ pts: number;+}) =>+ `F\t${frame.poolId}\t${frame.slot}\t${frame.frameId}\t${frame.width}\t${frame.height}\t${frame.stride}\t${frame.byteLength}\t${pts}\n`;++export const serializeRemotionSharedMemoryPoolRetirement = (poolId: number) =>+ `U\t${poolId}\n`;++export const parseRemotionSharedMemoryAck = (line: string) => {+ const match = /^(\d+)\t(\d+)\t(\d+)$/.exec(line);+ if (!match) {+ throw new Error(`Invalid Remotion shared-memory ACK: ${line}`);+ }++ const poolId = Number(match[1]);+ const slot = Number(match[2]);+ if (+ !Number.isSafeInteger(poolId) ||+ poolId < 1 ||+ !Number.isSafeInteger(slot) ||+ slot < 0+ ) {+ throw new Error(`Invalid Remotion shared-memory ACK: ${line}`);+ }++ return {poolId, slot, frameId: match[3]};+};++export const createRemotionSharedMemoryFfmpegBridge = ({+ control,+ acknowledgements,+}: {+ control: Writable;+ acknowledgements: Readable;+}) => {+ const registeredPools = new Map<number, RegisteredPool>();+ const pending = new Map<string, PendingFrame>();+ let fatalError: Error | null = null;+ let finishing = false;+ let controlEnded = false;+ let controlWrites = Promise.resolve();+ let finishPromise: Promise<void> | null = null;++ const toError = (error: unknown) =>+ error instanceof Error ? error : new Error(String(error));++ const releaseIgnoringErrors = async (frame: RemotionRawFrame) => {+ try {+ await frame.release();+ } catch {+ // The original bridge error is more useful. release() still abandons the+ // local slot in a finally block if the CDP session is unavailable.+ }+ };++ const settleRegistration = (pool: RegisteredPool, registered: boolean) => {+ if (pool.registrationSettled) {+ return;+ }++ pool.registrationSettled = true;+ pool.registered = registered;+ pool.resolveRegistration(registered);+ };++ const signalPoolDrained = (pool: RegisteredPool) => {+ if (pool.pendingCount === 0) {+ pool.drainWaiters.splice(0).forEach((waiter) => waiter());+ }+ };++ const removePending = (key: string, value: PendingFrame) => {+ if (pending.get(key) !== value) {+ return false;+ }++ pending.delete(key);+ value.pool.pendingCount--;+ signalPoolDrained(value.pool);+ return true;+ };++ const fail = (error: Error) => {+ if (fatalError) {+ return;+ }++ fatalError = error;+ for (const pool of registeredPools.values()) {+ settleRegistration(pool, false);+ }++ for (const [key, value] of [...pending.entries()]) {+ if (removePending(key, value)) {+ value.reject(error);+ releaseIgnoringErrors(value.frame).catch(() => undefined);+ }+ }+ };++ const enqueueControlWrite = (data: string) => {+ if (finishing || controlEnded) {+ return Promise.reject(+ new Error('FFmpeg shared-memory input has already been finished.'),+ );+ }++ const operation = controlWrites.then(async () => {+ if (fatalError) {+ throw fatalError;+ }++ await writeWithBackpressure({+ data: Buffer.from(data),+ writable: control,+ });+ });+ controlWrites = operation;+ return operation;+ };++ const unregisterPool = (pool: RegisteredPool) => {+ pool.retired = true;+ if (pool.unregistering) {+ return pool.unregistering;+ }++ pool.unregistering = (async () => {+ try {+ const wasRegistered = await pool.registration;+ if (!wasRegistered || fatalError || finishing || controlEnded) {+ return;+ }++ if (pool.pendingCount > 0) {+ await new Promise<void>((resolve) => pool.drainWaiters.push(resolve));+ }++ if (fatalError || finishing || controlEnded) {+ return;+ }++ try {+ await enqueueControlWrite(+ serializeRemotionSharedMemoryPoolRetirement(pool.poolId),+ );+ } catch (error) {+ const normalizedError = toError(error);+ fail(normalizedError);+ throw normalizedError;+ }+ } finally {+ pool.unsubscribe?.();+ pool.unsubscribe = null;+ if (registeredPools.get(pool.poolId) === pool) {+ registeredPools.delete(pool.poolId);+ }+ }+ })();+ return pool.unregistering;+ };++ const createRegisteredPool = (frame: RemotionRawFrame) => {+ let resolveRegistration!: (registered: boolean) => void;+ const registration = new Promise<boolean>((resolve) => {+ resolveRegistration = resolve;+ });+ const pool: RegisteredPool = {+ poolId: frame.poolId,+ sharedMemoryName: frame.sharedMemoryName,+ poolLifetime: frame.poolLifetime,+ registered: false,+ registrationStarted: false,+ registrationSettled: false,+ registration,+ resolveRegistration,+ retired: false,+ pendingCount: 0,+ drainWaiters: [],+ unregistering: null,+ unsubscribe: null,+ };+ const unsubscribe = frame.poolLifetime.onRetired(() =>+ unregisterPool(pool),+ );+ if (!unsubscribe) {+ return null;+ }++ pool.unsubscribe = unsubscribe;+ registeredPools.set(frame.poolId, pool);+ return pool;+ };++ const lines = createInterface({input: acknowledgements});+ lines.on('line', (line) => {+ (async () => {+ const ack = parseRemotionSharedMemoryAck(line);+ const key = pendingKey(ack.poolId, ack.slot);+ const value = pending.get(key);+ if (!value || value.frame.frameId !== ack.frameId) {+ throw new Error(+ `Unexpected Remotion shared-memory ACK for ${ack.poolId}:${ack.slot}:${ack.frameId}.`,+ );+ }++ try {+ await value.frame.release();+ } catch (error) {+ const normalizedError = toError(error);+ if (removePending(key, value)) {+ value.reject(normalizedError);+ fail(normalizedError);+ }++ return;+ }++ if (removePending(key, value)) {+ value.resolve();+ }+ })().catch((error) => fail(toError(error)));+ });+ lines.on('error', (error) => fail(toError(error)));+ lines.on('close', () => {+ if (pending.size > 0 || !finishing) {+ fail(+ new Error(+ `FFmpeg closed its Remotion shared-memory ACK pipe${pending.size > 0 ? ` with ${pending.size} frame(s) pending` : ' before input finished'}.`,+ ),+ );+ }+ });+ acknowledgements.on('error', (error) => fail(toError(error)));+ acknowledgements.on('close', () => {+ if (pending.size > 0 || !finishing) {+ fail(+ new Error(+ `FFmpeg closed its Remotion shared-memory ACK pipe${pending.size > 0 ? ` with ${pending.size} frame(s) pending` : ' before input finished'}.`,+ ),+ );+ }+ });+ control.on('error', (error) => fail(toError(error)));++ return {+ writeFrame: async ({+ frame,+ pts,+ }: {+ frame: RemotionRawFrame;+ pts: number;+ }) => {+ if (fatalError) {+ await releaseIgnoringErrors(frame);+ throw fatalError;+ }++ if (finishing || controlEnded || frame.poolLifetime.isRetired()) {+ await releaseIgnoringErrors(frame);+ throw new Error(+ `Target closed: Remotion shared-memory pool ${frame.poolId} has already been retired.`,+ );+ }++ let pool = registeredPools.get(frame.poolId);+ if (!pool) {+ const createdPool = createRegisteredPool(frame);+ if (!createdPool) {+ await releaseIgnoringErrors(frame);+ throw new Error(+ `Target closed: Remotion shared-memory pool ${frame.poolId} has already been retired.`,+ );+ }++ pool = createdPool;+ } else if (+ pool.sharedMemoryName !== frame.sharedMemoryName ||+ pool.poolLifetime !== frame.poolLifetime+ ) {+ const error = new Error(+ `Remotion shared-memory pool ${frame.poolId} changed its identity.`,+ );+ await releaseIgnoringErrors(frame);+ fail(error);+ throw error;+ }++ if (pool.retired || frame.poolLifetime.isRetired()) {+ await releaseIgnoringErrors(frame);+ throw new Error(+ `Target closed: Remotion shared-memory pool ${frame.poolId} has already been retired.`,+ );+ }++ const key = pendingKey(frame.poolId, frame.slot);+ if (pending.has(key)) {+ const error = new Error(+ `Remotion shared-memory slot ${frame.poolId}:${frame.slot} was reused before its ACK.`,+ );+ await releaseIgnoringErrors(frame);+ fail(error);+ throw error;+ }++ let resolveAck!: () => void;+ let rejectAck!: (error: Error) => void;+ const ack = new Promise<void>((resolve, reject) => {+ resolveAck = resolve;+ rejectAck = reject;+ });+ ack.catch(() => undefined);+ const pendingFrame: PendingFrame = {+ frame,+ pool,+ resolve: resolveAck,+ reject: rejectAck,+ };+ pending.set(key, pendingFrame);+ pool.pendingCount++;++ const registersPool = !pool.registrationStarted;+ if (registersPool) {+ pool.registrationStarted = true;+ }++ const operation = enqueueControlWrite(+ (registersPool ? serializeRemotionSharedMemoryPool(frame) : '') ++ serializeRemotionSharedMemoryFrame({frame, pts}),+ );+ if (registersPool) {+ operation.then(+ () => settleRegistration(pool, true),+ () => settleRegistration(pool, false),+ );+ }++ try {+ await operation;+ } catch (error) {+ const normalizedError = toError(error);+ fail(normalizedError);+ throw normalizedError;+ }++ return {waitForAck: ack};+ },+ finish: () => {+ if (finishPromise) {+ return finishPromise;+ }++ finishing = true;+ finishPromise = (async () => {+ try {+ await controlWrites;+ if (fatalError) {+ throw fatalError;+ }++ await new Promise<void>((resolve, reject) => {+ control.end((error?: Error | null) => {+ if (error) {+ reject(error);+ } else {+ resolve();+ }+ });+ });+ controlEnded = true;+ } catch (error) {+ const normalizedError = toError(error);+ fail(normalizedError);+ throw normalizedError;+ }+ })();+ return finishPromise;+ },+ fail,+ };+};diff --git a/packages/renderer/src/remotion-shared-memory.ts b/packages/renderer/src/remotion-shared-memory.tsnew file mode 100644index 0000000000..90b6402dd0--- /dev/null+++ b/packages/renderer/src/remotion-shared-memory.ts@@ -0,0 +1,408 @@+import type {Page} from './browser/BrowserPage';+import type {RemotionRawFrame as CdpRemotionRawFrame} from './browser/devtools-types';+import {isTargetClosedErr} from './browser/flaky-errors';+import type {LogLevel} from './log-level';+import {Log} from './logger';++const encoderPadding = 64;+const slotsPerPage = 1;++type FramePool = {+ page: Page;+ poolId: number;+ sharedMemoryName: string;+ slotCount: number;+ slotCapacity: number;+ availableSlots: number[];+ waiters: Array<() => void>;+ drainWaiters: Array<() => void>;+ ownedSlots: Map<number, string>;+ forgotten: boolean;+ poolLifetime: ManagedRemotionSharedMemoryPoolLifetime;+};++export type RemotionSharedMemoryPoolLifetime = {+ isRetired: () => boolean;+ waitForRetirement: () => Promise<void>;+ onRetired: (listener: () => Promise<void>) => (() => void) | null;+};++type ManagedRemotionSharedMemoryPoolLifetime =+ RemotionSharedMemoryPoolLifetime & {+ retire: () => Promise<void>;+ };++export type RemotionRawFrame = CdpRemotionRawFrame & {+ type: 'remotion-shared-memory';+ poolId: number;+ sharedMemoryName: string;+ slotCount: number;+ slotCapacity: number;+ poolLifetime: RemotionSharedMemoryPoolLifetime;+ release: () => Promise<void>;+};++export type CapturedFrame = Buffer | RemotionRawFrame;++const isExpectedCapabilityError = (error: unknown) => {+ const message = error instanceof Error ? error.message : String(error);+ return (+ message.includes("'Page.remotionCreateFramePool' wasn't found") ||+ message.includes('Method not found') ||+ message.includes('Raw frames require') ||+ message.includes('Raw frames are only supported') ||+ message.includes('Could not create Remotion shared memory') ||+ message.includes('Could not size Remotion shared memory') ||+ message.includes('Could not map Remotion shared memory') ||+ message.includes('Invalid Remotion frame pool dimensions')+ );+};++export class RemotionSharedMemoryCapture {+ #pools = new Map<Page, FramePool>();+ #nextPoolId = 1;+ #nextFrameId = 1;+ #supported: boolean | null = null;+ #initialization: Promise<void> | null = null;+ readonly #options: {+ width: number;+ height: number;+ indent: boolean;+ logLevel: LogLevel;+ };++ constructor(options: {+ width: number;+ height: number;+ indent: boolean;+ logLevel: LogLevel;+ }) {+ this.#options = options;+ }++ isSupported() {+ return this.#supported === true;+ }++ async initializePages(pages: Page[]) {+ if (this.#initialization) {+ await this.#initialization;+ if (this.#supported) {+ for (const page of pages) {+ await this.ensurePage(page);+ }+ }++ return;+ }++ this.#initialization = (async () => {+ try {+ for (const page of pages) {+ await this.#createPool(page);+ }++ this.#supported = true;+ } catch (error) {+ await this.#destroyPools();+ if (!isExpectedCapabilityError(error)) {+ throw error;+ }++ this.#supported = false;+ Log.verbose(+ {indent: this.#options.indent, logLevel: this.#options.logLevel},+ 'Remotion shared-memory capture is unavailable. Falling back to encoded screenshots.',+ );+ }+ })();++ await this.#initialization;+ }++ async ensurePage(page: Page) {+ if (this.#pools.has(page) || this.#supported === false) {+ return;+ }++ if (this.#supported === null) {+ await this.initializePages([page]);+ return;+ }++ await this.#createPool(page);+ }++ async acquire(page: Page) {+ if (!this.#supported) {+ return null;+ }++ const pool = this.#pools.get(page);+ if (!pool) {+ throw new Error('No Remotion shared-memory pool exists for this page.');+ }++ while (pool.availableSlots.length === 0) {+ if (pool.forgotten) {+ throw new Error('The page owning this shared-memory pool was closed.');+ }++ await new Promise<void>((resolve) => pool.waiters.push(resolve));+ }++ const slot = pool.availableSlots.shift();+ if (slot === undefined) {+ throw new Error('Expected an available Remotion shared-memory slot.');+ }++ if (!Number.isSafeInteger(this.#nextFrameId)) {+ throw new Error(+ 'Remotion shared-memory frame ID exceeded the safe range.',+ );+ }++ const frameId = String(this.#nextFrameId++);+ pool.ownedSlots.set(slot, frameId);+ let published = false;+ let releasePromise: Promise<void> | null = null;+ const releasePublished = () => {+ releasePromise ??= (async () => {+ try {+ await page._client().send('Page.remotionReleaseFrame', {+ slot,+ frameId,+ });+ } catch (error) {+ if (!isTargetClosedErr(error as Error)) {+ throw error;+ }+ } finally {+ this.#returnSlot(pool, slot, frameId);+ }+ })();+ return releasePromise;+ };++ return {+ slot,+ frameId,+ fail: () => {+ if (!published) {+ this.#returnSlot(pool, slot, frameId);+ }+ },+ discardPublished: async () => {+ published = true;+ await releasePublished();+ },+ publish: async (+ metadata: CdpRemotionRawFrame,+ ): Promise<RemotionRawFrame> => {+ published = true;+ if (+ metadata.slot !== slot ||+ metadata.frameId !== frameId ||+ metadata.width !== this.#options.width ||+ metadata.height !== this.#options.height ||+ metadata.stride < metadata.width * 4 ||+ metadata.byteLength > pool.slotCapacity - encoderPadding ||+ metadata.pixelFormat !== 'bgra' ||+ metadata.colorSpace !== 'srgb'+ ) {+ await releasePublished();+ throw new Error(+ `Chromium returned invalid Remotion shared-memory metadata: ${JSON.stringify(metadata)}`,+ );+ }++ return {+ ...metadata,+ type: 'remotion-shared-memory',+ poolId: pool.poolId,+ sharedMemoryName: pool.sharedMemoryName,+ slotCount: pool.slotCount,+ slotCapacity: pool.slotCapacity,+ poolLifetime: pool.poolLifetime,+ release: releasePublished,+ };+ },+ };+ }++ async destroy() {+ await this.#destroyPools();+ }++ async forgetPage(page: Page) {+ await this.#forgetPage(page, false);+ }++ async forgetAllPages() {+ await Promise.all(+ [...this.#pools.keys()].map((page) => this.#forgetPage(page, false)),+ );+ }++ async dispose() {+ await Promise.all(+ [...this.#pools.keys()].map((page) => this.#forgetPage(page, true)),+ );++ this.#supported = false;+ }++ async #forgetPage(page: Page, bestEffort: boolean) {+ const pool = this.#pools.get(page);+ if (!pool) {+ return;+ }++ this.#pools.delete(page);+ pool.forgotten = true;+ pool.waiters.splice(0).forEach((waiter) => waiter());++ let cleanupError: unknown = null;+ try {+ await pool.poolLifetime.retire();+ } catch (error) {+ cleanupError = error;+ }++ if (pool.ownedSlots.size > 0) {+ await new Promise<void>((resolve) => pool.drainWaiters.push(resolve));+ }++ try {+ await page._client().send('Page.remotionDestroyFramePool');+ } catch (error) {+ if (!bestEffort && !isTargetClosedErr(error as Error) && !cleanupError) {+ cleanupError = error;+ }+ }++ if (cleanupError && !bestEffort) {+ throw cleanupError;+ }+ }++ async #createPool(page: Page) {+ if (this.#pools.has(page)) {+ return;+ }++ const slotCapacity =+ this.#options.width * this.#options.height * 4 + encoderPadding;+ if (!Number.isSafeInteger(slotCapacity)) {+ throw new Error(+ 'Remotion shared-memory frame pool size is not a safe integer.',+ );+ }++ const {value} = await page._client().send('Page.remotionCreateFramePool', {+ slotCount: slotsPerPage,+ slotCapacity,+ });+ if (+ !value.sharedMemoryName ||+ value.slotCount !== slotsPerPage ||+ value.slotCapacity !== slotCapacity+ ) {+ throw new Error(+ 'Chromium returned invalid Remotion frame pool metadata.',+ );+ }++ let retired = false;+ let retirement: Promise<void> | null = null;+ let notifyRetirement!: () => void;+ const retirementSignal = new Promise<void>((resolve) => {+ notifyRetirement = resolve;+ });+ const retirementListeners = new Set<() => Promise<void>>();+ const poolLifetime: ManagedRemotionSharedMemoryPoolLifetime = {+ isRetired: () => retired,+ waitForRetirement: () => retirementSignal,+ onRetired: (listener) => {+ if (retired) {+ return null;+ }++ retirementListeners.add(listener);+ return () => retirementListeners.delete(listener);+ },+ retire: () => {+ if (!retirement) {+ retired = true;+ notifyRetirement();+ retirement = (async () => {+ await Promise.all(+ [...retirementListeners].map((listener) => listener()),+ );+ })();+ }++ return retirement;+ },+ };++ this.#pools.set(page, {+ page,+ poolId: this.#nextPoolId++,+ sharedMemoryName: value.sharedMemoryName,+ slotCount: value.slotCount,+ slotCapacity: value.slotCapacity,+ availableSlots: Array.from(+ {length: value.slotCount},+ (_, index) => index,+ ),+ waiters: [],+ drainWaiters: [],+ ownedSlots: new Map(),+ forgotten: false,+ poolLifetime,+ });+ }++ #returnSlot(pool: FramePool, slot: number, frameId: string) {+ if (pool.ownedSlots.get(slot) !== frameId) {+ throw new Error(+ `Remotion shared-memory slot ${pool.poolId}:${slot} is not owned by frame ${frameId}.`,+ );+ }++ pool.ownedSlots.delete(slot);+ if (pool.forgotten) {+ if (pool.ownedSlots.size === 0) {+ pool.drainWaiters.splice(0).forEach((waiter) => waiter());+ }+ } else {+ pool.availableSlots.push(slot);+ pool.waiters.shift()?.();+ }+ }++ async #destroyPools() {+ for (const pool of this.#pools.values()) {+ if (pool.ownedSlots.size > 0) {+ throw new Error(+ `Cannot destroy Remotion shared-memory pool ${pool.poolId} while it owns ${pool.ownedSlots.size} frame(s).`,+ );+ }++ try {+ await pool.page._client().send('Page.remotionDestroyFramePool');+ } catch (error) {+ if (!isTargetClosedErr(error as Error)) {+ throw error;+ }+ }+ }++ this.#pools.clear();+ }+}++export const isRemotionRawFrame = (+ frame: CapturedFrame,+): frame is RemotionRawFrame => !Buffer.isBuffer(frame);diff --git a/packages/renderer/src/render-frame-and-retry-target-close.ts b/packages/renderer/src/render-frame-and-retry-target-close.tsindex 98fa9b5267..22a04a002a 100644--- a/packages/renderer/src/render-frame-and-retry-target-close.ts+++ b/packages/renderer/src/render-frame-and-retry-target-close.ts@@ -14,6 +14,10 @@ import type {CancelSignal} from './make-cancel-signal';import {cancelErrorMessages, isUserCancelledRender} from './make-cancel-signal';import type {NextFrameToRender} from './next-frame-to-render';import type {Pool} from './pool';+import type {+ CapturedFrame,+ RemotionSharedMemoryCapture,+} from './remotion-shared-memory';import {renderFrame} from './render-frame';import type {FrameAndAssets, OnArtifact} from './render-frames';import type {BrowserReplacer} from './replace-browser';@@ -48,6 +52,8 @@ export const renderFrameAndRetryTargetClose = async ({framesRenderedObj,lastFrame,onFrameBuffer,+ onFrame,+ remotionSharedMemory,onFrameUpdate,nextFrameToRender,imageSequencePattern,@@ -87,6 +93,10 @@ export const renderFrameAndRetryTargetClose = async ({| null| ((buffer: Buffer, frame: number) => void | Promise<void>)| undefined;+ onFrame:+ | null+ | ((frame: CapturedFrame, frameNumber: number) => void | Promise<void>);+ remotionSharedMemory: RemotionSharedMemoryCapture | null;onFrameUpdate:| null| ((@@ -136,6 +146,8 @@ export const renderFrameAndRetryTargetClose = async ({lastFrame,onError,onFrameBuffer,+ onFrame,+ remotionSharedMemory,onFrameUpdate,outputDir,stoppedSignal,@@ -185,6 +197,7 @@ export const renderFrameAndRetryTargetClose = async ({if (shouldRetryError) {const pool = await poolPromise;// Replace the closed page+ await remotionSharedMemory?.forgetPage(freePage);const newPage = await makeNewPage(frame, freePage.pageIndex);pool.release(newPage);Log.warn(@@ -224,6 +237,8 @@ export const renderFrameAndRetryTargetClose = async ({framesRenderedObj,lastFrame,onFrameBuffer,+ onFrame,+ remotionSharedMemory,onFrameUpdate,nextFrameToRender,imageSequencePattern,@@ -238,6 +253,7 @@ export const renderFrameAndRetryTargetClose = async ({`The browser crashed while rendering frame ${frame}, retrying ${retriesLeft} more times. Learn more about this error under https://www.remotion.dev/docs/target-closed`,);// Replace the entire browser+ await remotionSharedMemory?.forgetAllPages();await browserReplacer.replaceBrowser(makeBrowser, async () => {const pages = new Array(concurrencyOrFramesToRender).fill(true)@@ -281,6 +297,8 @@ export const renderFrameAndRetryTargetClose = async ({framesRenderedObj,lastFrame,onFrameBuffer,+ onFrame,+ remotionSharedMemory,onFrameUpdate,nextFrameToRender,imageSequencePattern,diff --git a/packages/renderer/src/render-frame-with-option-to-reject.ts b/packages/renderer/src/render-frame-with-option-to-reject.tsindex 63de6dc1f3..bec487431e 100644--- a/packages/renderer/src/render-frame-with-option-to-reject.ts+++ b/packages/renderer/src/render-frame-with-option-to-reject.ts@@ -17,6 +17,11 @@ import type {VideoImageFormat} from './image-format';import type {LogLevel} from './log-level';import {Log} from './logger';import type {CancelSignal} from './make-cancel-signal';+import type {+ CapturedFrame,+ RemotionSharedMemoryCapture,+} from './remotion-shared-memory';+import {isRemotionRawFrame} from './remotion-shared-memory';import type {FrameAndAssets, OnArtifact} from './render-frames';import {seekToFrame} from './seek-to-frame';import {takeFrame} from './take-frame';@@ -34,6 +39,8 @@ export const renderFrameWithOptionToReject = async ({timeoutInMilliseconds,outputDir,onFrameBuffer,+ onFrame,+ remotionSharedMemory,imageFormat,onError,lastFrame,@@ -72,6 +79,10 @@ export const renderFrameWithOptionToReject = async ({| null| ((buffer: Buffer, frame: number) => void | Promise<void>)| undefined;+ onFrame:+ | null+ | ((frame: CapturedFrame, frameNumber: number) => void | Promise<void>);+ remotionSharedMemory: RemotionSharedMemoryCapture | null;imageFormat: VideoImageFormat;onError: (err: Error) => void;lastFrame: number;@@ -111,7 +122,9 @@ export const renderFrameWithOptionToReject = async ({return Promise.reject(new Error('Render was stopped'));}+ let pageError: Error | null = null;const errorCallbackOnFrame = (err: Error) => {+ pageError ??= err;reject(err);};@@ -121,171 +134,268 @@ export const renderFrameWithOptionToReject = async ({frame,});page.on('error', errorCallbackOnFrame);+ let pageListenersCleaned = false;+ const cleanupPageListeners = () => {+ if (pageListenersCleaned) {+ return;+ }- const startSeeking = Date.now();+ pageListenersCleaned = true;+ cleanupPageError();+ page.off('error', errorCallbackOnFrame);+ };- await seekToFrame({- frame,- page,- composition: compId,- timeoutInMilliseconds,- indent,- logLevel,- attempt,- });+ try {+ const startSeeking = Date.now();- const timeToSeek = Date.now() - startSeeking;- if (timeToSeek > 1000) {- Log.verbose(- {indent, logLevel},- `Seeking to frame ${frame} took ${timeToSeek}ms`,- );- }+ await seekToFrame({+ frame,+ page,+ composition: compId,+ timeoutInMilliseconds,+ indent,+ logLevel,+ attempt,+ });- if (!outputDir && !onFrameBuffer && imageFormat !== 'none') {- throw new Error(- 'Called renderFrames() without specifying either `outputDir` or `onFrameBuffer`',- );- }+ const timeToSeek = Date.now() - startSeeking;+ if (timeToSeek > 1000) {+ Log.verbose(+ {indent, logLevel},+ `Seeking to frame ${frame} took ${timeToSeek}ms`,+ );+ }- if (outputDir && onFrameBuffer && imageFormat !== 'none') {- throw new Error(- 'Pass either `outputDir` or `onFrameBuffer` to renderFrames(), not both.',- );- }+ if (!outputDir && !onFrameBuffer && !onFrame && imageFormat !== 'none') {+ throw new Error(+ 'Called renderFrames() without specifying either `outputDir` or `onFrameBuffer`',+ );+ }- const [buffer, collectedAssets] = await Promise.all([- takeFrame({- freePage: page,- height,- imageFormat: assetsOnly ? 'none' : imageFormat,- output:- index === null- ? null- : path.join(- frameDir,- getFrameOutputFileName({- frame,- imageFormat,- index,- countType,- lastFrame,- totalFrames: framesToRender.length,- imageSequencePattern,- }),- ),- jpegQuality,- width,- scale,- wantsBuffer: Boolean(onFrameBuffer),- timeoutInMilliseconds,- }),- collectAssets({- frame,- freePage: page,- timeoutInMilliseconds,- }),- ]);- if (onFrameBuffer && !assetsOnly) {- if (!buffer) {- throw new Error('unexpected null buffer');+ if (outputDir && (onFrameBuffer || onFrame) && imageFormat !== 'none') {+ throw new Error(+ 'Pass either `outputDir` or `onFrameBuffer` to renderFrames(), not both.',+ );}- await onFrameBuffer(buffer, frame);- }+ const [captureResult, assetsResult] = await Promise.allSettled([+ takeFrame({+ freePage: page,+ height,+ imageFormat: assetsOnly ? 'none' : imageFormat,+ output:+ index === null+ ? null+ : path.join(+ frameDir,+ getFrameOutputFileName({+ frame,+ imageFormat,+ index,+ countType,+ lastFrame,+ totalFrames: framesToRender.length,+ imageSequencePattern,+ }),+ ),+ jpegQuality,+ width,+ scale,+ wantsBuffer: Boolean(onFrameBuffer || onFrame),+ timeoutInMilliseconds,+ remotionSharedMemory,+ }),+ collectAssets({+ frame,+ freePage: page,+ timeoutInMilliseconds,+ }),+ ]);+ if (captureResult.status === 'rejected') {+ throw captureResult.reason;+ }- const onlyAvailableAssets = assets.filter(truthy);+ const buffer = captureResult.value;+ if (assetsResult.status === 'rejected') {+ if (buffer !== null && isRemotionRawFrame(buffer)) {+ try {+ await buffer.release();+ } catch {+ // Preserve the asset collection error so the frame can be retried.+ }+ }- const previousAudioRenderAssets = onlyAvailableAssets- .map((a) => a.audioAndVideoAssets)- .flat(2);+ throw assetsResult.reason;+ }- const previousArtifactAssets = onlyAvailableAssets- .map((a) => a.artifactAssets)- .flat(2);+ const collectedAssets = assetsResult.value;+ let frameBufferForArtifacts = Buffer.isBuffer(buffer) ? buffer : null;+ if (+ buffer !== null &&+ isRemotionRawFrame(buffer) &&+ collectedAssets.some(+ (asset) =>+ asset.type === 'artifact' && asset.contentType === 'thumbnail',+ )+ ) {+ try {+ const thumbnail = await takeFrame({+ freePage: page,+ height,+ imageFormat,+ output: null,+ jpegQuality,+ width,+ scale,+ wantsBuffer: true,+ timeoutInMilliseconds,+ remotionSharedMemory: null,+ });+ if (!thumbnail) {+ throw new Error('Could not capture a thumbnail artifact.');+ }- const audioAndVideoAssets = onlyAudioAndVideoAssets(collectedAssets);- const artifactAssets = onlyArtifact({- assets: collectedAssets,- frameBuffer: buffer,- });+ frameBufferForArtifacts = thumbnail;+ } catch (error) {+ try {+ await buffer.release();+ } catch {+ // Preserve the thumbnail capture error so the frame can be retried.+ }- for (const artifact of artifactAssets) {- for (const previousArtifact of previousArtifactAssets) {- if (artifact.filename === previousArtifact.filename) {- return Promise.reject(- new Error(- `An artifact with output "${artifact.filename}" was already registered at frame ${previousArtifact.frame}, but now registered again at frame ${artifact.frame}. Artifacts must have unique names. https://remotion.dev/docs/artifacts`,- ),- );+ throw error;}}- onArtifact?.(artifact);- }+ cleanupPageListeners();+ if (pageError) {+ if (buffer !== null && isRemotionRawFrame(buffer)) {+ try {+ await buffer.release();+ } catch {+ // Preserve the page error so the frame can be retried.+ }+ }- const compressedAssets = audioAndVideoAssets.map((asset) => {- return compressAsset(previousAudioRenderAssets, asset);- });+ throw pageError;+ }- const inlineAudioAssets = onlyInlineAudio(collectedAssets);- const outputFrame =- allFramesAndExtraFrames[0] + allFramesAndExtraFrames.indexOf(frame);+ if ((onFrameBuffer || onFrame) && !assetsOnly) {+ if (!buffer) {+ throw new Error('unexpected null buffer');+ }- assets.push({- audioAndVideoAssets: compressedAssets,- frame,- artifactAssets: artifactAssets.map((a) => {- return {- frame: a.frame,- filename: a.filename,- };- }),- inlineAudioAssets,- });+ if (onFrame) {+ await onFrame(buffer, frame);+ } else {+ if (isRemotionRawFrame(buffer)) {+ try {+ await buffer.release();+ } catch {+ // The type mismatch is the primary error.+ }- for (const renderAsset of compressedAssets) {- downloadAndMapAssetsToFileUrl({- renderAsset,- onDownload,- downloadMap,- indent,- logLevel,- binariesDirectory,- cancelSignalForAudioAnalysis: cancelSignal,- shouldAnalyzeAudioImmediately: true,- }).catch((err) => {- const truncateWithEllipsis =- renderAsset.src.substring(0, 1000) +- (renderAsset.src.length > 1000 ? '...' : '');- onError(- new Error(- `Error while downloading ${truncateWithEllipsis}: ${(err as Error).stack}`,- ),- );+ throw new Error(+ 'Raw shared-memory frames cannot be passed to renderFrames().',+ );+ }++ await onFrameBuffer?.(buffer, frame);+ }+ }++ const onlyAvailableAssets = assets.filter(truthy);++ const previousAudioRenderAssets = onlyAvailableAssets+ .map((a) => a.audioAndVideoAssets)+ .flat(2);++ const previousArtifactAssets = onlyAvailableAssets+ .map((a) => a.artifactAssets)+ .flat(2);++ const audioAndVideoAssets = onlyAudioAndVideoAssets(collectedAssets);+ const artifactAssets = onlyArtifact({+ assets: collectedAssets,+ frameBuffer: frameBufferForArtifacts,});- }- for (const renderAsset of inlineAudioAssets) {- downloadMap.inlineAudioMixing.addAsset({- asset: {...renderAsset, frame: outputFrame},- fps,- totalNumberOfFrames: allFramesAndExtraFrames.length,- firstFrame: allFramesAndExtraFrames[0],- trimLeftOffset,- trimRightOffset,+ for (const artifact of artifactAssets) {+ for (const previousArtifact of previousArtifactAssets) {+ if (artifact.filename === previousArtifact.filename) {+ return Promise.reject(+ new Error(+ `An artifact with output "${artifact.filename}" was already registered at frame ${previousArtifact.frame}, but now registered again at frame ${artifact.frame}. Artifacts must have unique names. https://remotion.dev/docs/artifacts`,+ ),+ );+ }+ }++ onArtifact?.(artifact);+ }++ const compressedAssets = audioAndVideoAssets.map((asset) => {+ return compressAsset(previousAudioRenderAssets, asset);});- }- cleanupPageError();- page.off('error', errorCallbackOnFrame);+ const inlineAudioAssets = onlyInlineAudio(collectedAssets);+ const outputFrame =+ allFramesAndExtraFrames[0] + allFramesAndExtraFrames.indexOf(frame);- if (!assetsOnly) {- framesRenderedObj.count++;- onFrameUpdate?.(- framesRenderedObj.count,+ assets.push({+ audioAndVideoAssets: compressedAssets,frame,- performance.now() - startTime,- );+ artifactAssets: artifactAssets.map((a) => {+ return {+ frame: a.frame,+ filename: a.filename,+ };+ }),+ inlineAudioAssets,+ });++ for (const renderAsset of compressedAssets) {+ downloadAndMapAssetsToFileUrl({+ renderAsset,+ onDownload,+ downloadMap,+ indent,+ logLevel,+ binariesDirectory,+ cancelSignalForAudioAnalysis: cancelSignal,+ shouldAnalyzeAudioImmediately: true,+ }).catch((err) => {+ const truncateWithEllipsis =+ renderAsset.src.substring(0, 1000) ++ (renderAsset.src.length > 1000 ? '...' : '');+ onError(+ new Error(+ `Error while downloading ${truncateWithEllipsis}: ${(err as Error).stack}`,+ ),+ );+ });+ }++ for (const renderAsset of inlineAudioAssets) {+ downloadMap.inlineAudioMixing.addAsset({+ asset: {...renderAsset, frame: outputFrame},+ fps,+ totalNumberOfFrames: allFramesAndExtraFrames.length,+ firstFrame: allFramesAndExtraFrames[0],+ trimLeftOffset,+ trimRightOffset,+ });+ }++ if (!assetsOnly) {+ framesRenderedObj.count++;+ onFrameUpdate?.(+ framesRenderedObj.count,+ frame,+ performance.now() - startTime,+ );+ }+ } finally {+ cleanupPageListeners();}};diff --git a/packages/renderer/src/render-frame.ts b/packages/renderer/src/render-frame.tsindex 1ec4021bbd..215d8f70df 100644--- a/packages/renderer/src/render-frame.ts+++ b/packages/renderer/src/render-frame.ts@@ -7,6 +7,10 @@ import type {VideoImageFormat} from './image-format';import type {LogLevel} from './log-level';import type {CancelSignal} from './make-cancel-signal';import type {NextFrameToRender} from './next-frame-to-render';+import type {+ CapturedFrame,+ RemotionSharedMemoryCapture,+} from './remotion-shared-memory';import {renderFrameWithOptionToReject} from './render-frame-with-option-to-reject';import type {FrameAndAssets, OnArtifact} from './render-frames';@@ -33,6 +37,8 @@ export const renderFrame = ({timeoutInMilliseconds,lastFrame,onFrameBuffer,+ onFrame,+ remotionSharedMemory,onFrameUpdate,framesRenderedObj,frame,@@ -66,6 +72,10 @@ export const renderFrame = ({| null| ((buffer: Buffer, frame: number) => void | Promise<void>)| undefined;+ onFrame:+ | null+ | ((frame: CapturedFrame, frameNumber: number) => void | Promise<void>);+ remotionSharedMemory: RemotionSharedMemoryCapture | null;lastFrame: number;onFrameUpdate:| null@@ -96,6 +106,8 @@ export const renderFrame = ({timeoutInMilliseconds,imageFormat,onFrameBuffer,+ onFrame,+ remotionSharedMemory,outputDir,assets,binariesDirectory,diff --git a/packages/renderer/src/render-frames.ts b/packages/renderer/src/render-frames.tsindex a13b653d97..ae3cf63523 100644--- a/packages/renderer/src/render-frames.ts+++ b/packages/renderer/src/render-frames.ts@@ -47,6 +47,10 @@ import type {optionsMap} from './options/options-map';import {Pool} from './pool';import type {RemotionServer} from './prepare-server';import {makeOrReuseServer} from './prepare-server';+import type {+ CapturedFrame,+ RemotionSharedMemoryCapture,+} from './remotion-shared-memory';import {renderFrameAndRetryTargetClose} from './render-frame-and-retry-target-close';import type {BrowserReplacer} from './replace-browser';import {handleBrowserCrash} from './replace-browser';@@ -88,6 +92,10 @@ type InternalRenderFramesOptions = {onFrameBuffer:| null| ((buffer: Buffer, frame: number) => void | Promise<void>);+ onFrame:+ | null+ | ((frame: CapturedFrame, frameNumber: number) => void | Promise<void>);+ remotionSharedMemory: RemotionSharedMemoryCapture | null;onDownload: RenderMediaOnDownload | null;chromiumOptions: ChromiumOptions;scale: number;@@ -127,6 +135,10 @@ type InnerRenderFramesOptions = {onFrameBuffer:| null| ((buffer: Buffer, frame: number) => void | Promise<void>);+ onFrame:+ | null+ | ((frame: CapturedFrame, frameNumber: number) => void | Promise<void>);+ remotionSharedMemory: RemotionSharedMemoryCapture | null;onArtifact: OnArtifact | null;onDownload: RenderMediaOnDownload | null;timeoutInMilliseconds: number;@@ -229,6 +241,8 @@ const innerRenderFrames = async ({envVariables,onBrowserLog,onFrameBuffer,+ onFrame,+ remotionSharedMemory,onDownload,pagesArray,serveUrl,@@ -308,8 +322,8 @@ const innerRenderFrames = async ({resolvedConcurrency,);- const makeNewPage = (frame: number, pageIndex: number) => {- return makePage({+ const makeNewPage = async (frame: number, pageIndex: number) => {+ const page = await makePage({context: sourceMapGetter,initialFrame: frame,browserReplacer,@@ -334,6 +348,8 @@ const innerRenderFrames = async ({darkMode,sampleRate,});+ await remotionSharedMemory?.ensurePage(page);+ return page;};const getPool = async () => {@@ -447,6 +463,8 @@ const innerRenderFrames = async ({lastFrame,makeNewPage,onFrameBuffer,+ onFrame,+ remotionSharedMemory,onFrameUpdate,nextFrameToRender,imageSequencePattern: pattern,@@ -498,6 +516,8 @@ const internalRenderFramesRaw = ({onBrowserLog,onDownload,onFrameBuffer,+ onFrame,+ remotionSharedMemory,onFrameUpdate,onStart,outputDir,@@ -641,6 +661,8 @@ const internalRenderFramesRaw = ({muted,onBrowserLog,onFrameBuffer,+ onFrame,+ remotionSharedMemory,onFrameUpdate,onStart,outputDir,@@ -848,6 +870,8 @@ export const renderFrames = (muted: muted ?? false,onBrowserLog: onBrowserLog ?? null,onFrameBuffer: onFrameBuffer ?? null,+ onFrame: null,+ remotionSharedMemory: null,onFrameUpdate,onStart,outputDir,diff --git a/packages/renderer/src/render-media.ts b/packages/renderer/src/render-media.tsindex 403eac384d..06f13dd048 100644--- a/packages/renderer/src/render-media.ts+++ b/packages/renderer/src/render-media.ts@@ -57,9 +57,16 @@ import {} from './pixel-format';import type {RemotionServer} from './prepare-server';import {makeOrReuseServer} from './prepare-server';-import {prespawnFfmpeg} from './prespawn-ffmpeg';+import {+ ffmpegSupportsRemotionSharedMemory,+ prespawnFfmpeg,+} from './prespawn-ffmpeg';import {shouldUseParallelEncoding} from './prestitcher-memory-usage';import {validateSelectedCodecAndProResCombination} from './prores-profile';+import {+ isRemotionRawFrame,+ RemotionSharedMemoryCapture,+} from './remotion-shared-memory';import type {OnArtifact} from './render-frames';import {internalRenderFrames} from './render-frames';import {@@ -80,7 +87,6 @@ import {validateOutputFilename} from './validate-output-filename';import {validateScale} from './validate-scale';import {validateBitrate} from './validate-videobitrate';import {wrapWithErrorHandling} from './wrap-with-error-handling';-import {writeWithBackpressure} from './write-with-backpressure';export type StitchingState = 'encoding' | 'muxing';@@ -228,7 +234,7 @@ type RenderMediaResult = {contentType: string;};-const internalRenderMediaRaw = ({+const internalRenderMediaRaw = async ({proResProfile,x264Preset,gopSize,@@ -489,6 +495,21 @@ const internalRenderMediaRaw = ({});const actualWidth = widthEvenDimensions * scale;const actualHeight = heightEvenDimensions * scale;+ const canUseRemotionSharedMemory =+ preEncodedFileLocation !== null &&+ (await ffmpegSupportsRemotionSharedMemory({+ binariesDirectory,+ indent,+ logLevel,+ }));+ const remotionSharedMemory = canUseRemotionSharedMemory+ ? new RemotionSharedMemoryCapture({+ width: Math.round(actualWidth),+ height: Math.round(actualHeight),+ indent,+ logLevel,+ })+ : null;const composition = {...compositionWithPossibleUnevenDimensions,@@ -539,8 +560,16 @@ const internalRenderMediaRaw = ({validateFps(fps, 'in "renderMedia()"', codec === 'gif');- const createPrestitcherIfNecessary = () => {- if (preEncodedFileLocation) {+ let preStitcherInputMode: 'encoded-image' | 'remotion-shared-memory' | null =+ null;+ let notifyPreStitcherStarted: () => void;+ const preStitcherStarted = new Promise<void>((resolve) => {+ notifyPreStitcherStarted = resolve;+ });+ const createPrestitcherIfNecessary = (+ inputMode: 'encoded-image' | 'remotion-shared-memory',+ ) => {+ if (preEncodedFileLocation && !preStitcher) {preStitcher = prespawnFfmpeg({width: actualWidth,height: actualHeight,@@ -568,17 +597,30 @@ const internalRenderMediaRaw = ({binariesDirectory,hardwareAcceleration,onLog,+ inputMode,});+ preStitcherInputMode = inputMode;stitcherFfmpeg = preStitcher.task;+ notifyPreStitcherStarted!();+ }++ if (preStitcherInputMode !== inputMode) {+ throw new Error(+ `Cannot mix ${inputMode} frames with ${preStitcherInputMode} FFmpeg input.`,+ );}};const waitForPrestitcherIfNecessary = async (): Promise<{usesParallelEncoding: boolean;}> => {+ if (preEncodedFileLocation) {+ await preStitcherStarted;+ }+if (stitcherFfmpeg) {await waitForFinish();- stitcherFfmpeg?.stdin?.end();+ await preStitcher?.finishInput();try {await stitcherFfmpeg;} catch {@@ -626,7 +668,7 @@ const internalRenderMediaRaw = ({Promise.resolve(undefined);const happyPath = new Promise<RenderMediaResult>((resolve, reject) => {- Promise.resolve(createPrestitcherIfNecessary())+ Promise.resolve().then(() => {return makeOrReuseServer(reusedServer,@@ -699,43 +741,101 @@ const internalRenderMediaRaw = ({outputFramesInSequence: true,puppeteerInstance,everyNthFrame,- onFrameBuffer: parallelEncoding- ? async (buffer, frame) => {- await waitForRightTimeOfFrameToBeInserted(frame);- if (cancelled) {- return;- }-- const id = startPerfMeasure('piping');- const exitStatus = preStitcher?.getExitStatus();- if (exitStatus?.type === 'quit-successfully') {- throw new Error(- `FFmpeg already quit while trying to pipe frame ${frame} to it. Stderr: ${exitStatus.stderr}`,+ onFrameBuffer: null,+ onFrame: parallelEncoding+ ? async (capturedFrame, frame) => {+ let handedToFfmpeg = false;+ try {+ if (isRemotionRawFrame(capturedFrame)) {+ let notifyRetirement!: () => void;+ const retirement = new Promise<boolean>(+ (resolveRetirement) => {+ notifyRetirement = () => resolveRetirement(true);+ },+ );+ const unsubscribeFromRetirement =+ capturedFrame.poolLifetime.onRetired(() => {+ notifyRetirement();+ return Promise.resolve();+ });+ if (!unsubscribeFromRetirement) {+ notifyRetirement();+ }++ let retiredBeforeItsTurn: boolean;+ try {+ retiredBeforeItsTurn = await Promise.race([+ waitForRightTimeOfFrameToBeInserted(frame).then(+ () => false,+ ),+ retirement,+ ]);+ } finally {+ unsubscribeFromRetirement?.();+ }++ if (retiredBeforeItsTurn) {+ throw new Error(+ 'Target closed while a raw frame was waiting for its FFmpeg turn.',+ );+ }++ if (capturedFrame.poolLifetime.isRetired()) {+ throw new Error(+ 'Target closed before a raw frame was sent to FFmpeg.',+ );+ }+ } else {+ await waitForRightTimeOfFrameToBeInserted(frame);+ }++ if (cancelled) {+ return;+ }++ const id = startPerfMeasure('piping');+ const exitStatus = preStitcher?.getExitStatus();+ if (exitStatus?.type === 'quit-successfully') {+ throw new Error(+ `FFmpeg already quit while trying to pipe frame ${frame} to it. Stderr: ${exitStatus.stderr}`,+ );+ }++ if (exitStatus?.type === 'quit-with-error') {+ throw new Error(+ `FFmpeg quit with code ${exitStatus.exitCode}${exitStatus.signal ? ` (signal ${exitStatus.signal})` : ''} while piping frame ${frame}. Stderr: ${exitStatus.stderr}`,+ );+ }++ const inputMode = isRemotionRawFrame(capturedFrame)+ ? 'remotion-shared-memory'+ : 'encoded-image';+ createPrestitcherIfNecessary(inputMode);+ const frameIndex = framesToRender.indexOf(frame);+ const writeResult = await preStitcher?.writeFrame({+ frame: capturedFrame,+ pts: frameIndex,+ });+ if (!writeResult) {+ throw new Error(+ `FFmpeg is not available while trying to pipe frame ${frame} to it.`,+ );+ }++ handedToFfmpeg = true;+ stopPerfMeasure(id);+ setFrameToStitch(+ framesToRender[frameIndex + 1] ?? lastFrameToRender + 1,);+ await writeResult.waitForAck;+ } finally {+ if (!handedToFfmpeg && isRemotionRawFrame(capturedFrame)) {+ await capturedFrame.release();+ }}-- if (exitStatus?.type === 'quit-with-error') {- throw new Error(- `FFmpeg quit with code ${exitStatus.exitCode}${exitStatus.signal ? ` (signal ${exitStatus.signal})` : ''} while piping frame ${frame}. Stderr: ${exitStatus.stderr}`,- );- }-- const stdin = stitcherFfmpeg?.stdin;- if (!stdin) {- throw new Error(- `FFmpeg stdin is not available while trying to pipe frame ${frame} to it.`,- );- }-- await writeWithBackpressure({data: buffer, writable: stdin});- stopPerfMeasure(id);-- const frameIndex = framesToRender.indexOf(frame);- setFrameToStitch(- framesToRender[frameIndex + 1] ?? lastFrameToRender + 1,- );}: null,+ remotionSharedMemory,webpackBundleOrServeUrl: serveUrl,onBrowserLog,onDownload,@@ -773,7 +873,8 @@ const internalRenderMediaRaw = ({waitForPrestitcherIfNecessary(),]);})- .then(([{assetsInfo}]) => {+ .then(async ([{assetsInfo}]) => {+ await remotionSharedMemory?.destroy();renderedDoneIn = Date.now() - renderStart;Log.verbose(@@ -908,6 +1009,7 @@ const internalRenderMediaRaw = ({* Therefore we first kill the FFMPEG process before deleting the file*/cancelled = true;+ preStitcher?.abortInput(err as Error);cancelRenderFrames.cancel();cancelStitcher.cancel();cancelPrestitcher.cancel();@@ -934,7 +1036,8 @@ const internalRenderMediaRaw = ({reject(err);})- .finally(() => {+ .finally(async () => {+ await remotionSharedMemory?.dispose();if (preEncodedFileLocation !== null &&fs.existsSync(preEncodedFileLocation)diff --git a/packages/renderer/src/render-still.ts b/packages/renderer/src/render-still.tsindex b7c8755ff6..3910663f10 100644--- a/packages/renderer/src/render-still.ts+++ b/packages/renderer/src/render-still.ts@@ -375,6 +375,7 @@ const innerRenderStill = async ({jpegQuality,wantsBuffer: !output,timeoutInMilliseconds,+ remotionSharedMemory: null,}),collectAssets({frame,diff --git a/packages/renderer/src/screenshot-task.ts b/packages/renderer/src/screenshot-task.tsindex 234b7e3f08..b79afc9ea2 100644--- a/packages/renderer/src/screenshot-task.ts+++ b/packages/renderer/src/screenshot-task.ts@@ -2,6 +2,10 @@ import fs from 'node:fs';import type {Page} from './browser/BrowserPage';import type {StillImageFormat} from './image-format';import {startPerfMeasure, stopPerfMeasure} from './perf';+import type {+ CapturedFrame,+ RemotionSharedMemoryCapture,+} from './remotion-shared-memory';export const screenshotTask = async ({format,@@ -12,6 +16,7 @@ export const screenshotTask = async ({path,jpegQuality,scale,+ remotionSharedMemory,}: {page: Page;format: StillImageFormat;@@ -21,7 +26,8 @@ export const screenshotTask = async ({width: number;height: number;scale: number;-}): Promise<Buffer> => {+ remotionSharedMemory: RemotionSharedMemoryCapture | null;+}): Promise<CapturedFrame> => {const client = page._client();const target = page.target();@@ -63,20 +69,58 @@ export const screenshotTask = async ({!process.env.DISABLE_FROM_SURFACE || height > 8192 || width > 8192;const scaleFactor = fromSurface ? 1 : scale;- const {value} = await client.send('Page.captureScreenshot', {- format,- quality: jpegQuality,- clip: {- x: 0,- y: 0,- height: height * scaleFactor,- scale: 1,- width: width * scaleFactor,- },- captureBeyondViewport: true,- optimizeForSpeed: true,- fromSurface,- });+ const reservation = await remotionSharedMemory?.acquire(page);+ let value;+ try {+ ({value} = await client.send('Page.captureScreenshot', {+ format,+ quality: jpegQuality,+ clip: {+ x: 0,+ y: 0,+ height: height * scaleFactor,+ scale: 1,+ width: width * scaleFactor,+ },+ captureBeyondViewport: true,+ optimizeForSpeed: true,+ fromSurface,+ remotionFrameSlot: reservation?.slot,+ remotionFrameId: reservation?.frameId,+ remotionFastViewport: reservation ? true : undefined,+ }));+ } catch (error) {+ reservation?.fail();+ throw error;+ }++ if (reservation) {+ if (value.data !== '' || !value.remotionFrame) {+ await reservation.discardPublished();+ throw new Error(+ 'Chromium did not return a Remotion shared-memory frame.',+ );+ }++ const rawFrame = await reservation.publish(value.remotionFrame);+ stopPerfMeasure(cap);+ if (omitBackground) {+ try {+ await client.send('Emulation.setDefaultBackgroundColorOverride');+ } catch (error) {+ try {+ await rawFrame.release();+ } catch {+ // Preserve the background reset error that made the capture fail.+ }++ throw error;+ }+ }++ return rawFrame;+ }+result = value;}diff --git a/packages/renderer/src/take-frame.ts b/packages/renderer/src/take-frame.tsindex 88a9575b45..1eb7b1872d 100644--- a/packages/renderer/src/take-frame.ts+++ b/packages/renderer/src/take-frame.ts@@ -2,18 +2,12 @@ import type {Page} from './browser/BrowserPage';import type {StillImageFormat, VideoImageFormat} from './image-format';import {puppeteerEvaluateWithCatch} from './puppeteer-evaluate';import {screenshot} from './puppeteer-screenshot';+import type {+ CapturedFrame,+ RemotionSharedMemoryCapture,+} from './remotion-shared-memory';-export const takeFrame = async ({- freePage,- imageFormat,- jpegQuality,- width,- height,- output,- scale,- wantsBuffer,- timeoutInMilliseconds,-}: {+type TakeFrameOptions = {freePage: Page;imageFormat: VideoImageFormat | StillImageFormat;jpegQuality: number | undefined;@@ -23,9 +17,28 @@ export const takeFrame = async ({scale: number;wantsBuffer: boolean;timeoutInMilliseconds: number;-}): Promise<Buffer | null> => {+ remotionSharedMemory: RemotionSharedMemoryCapture | null;+};++type TakeFrameResult<T extends RemotionSharedMemoryCapture | null> =+ T extends null ? Buffer | null : CapturedFrame | null;++export const takeFrame = async <T extends RemotionSharedMemoryCapture | null>({+ freePage,+ imageFormat,+ jpegQuality,+ width,+ height,+ output,+ scale,+ wantsBuffer,+ timeoutInMilliseconds,+ remotionSharedMemory,+}: Omit<TakeFrameOptions, 'remotionSharedMemory'> & {+ remotionSharedMemory: T;+}): Promise<TakeFrameResult<T>> => {if (imageFormat === 'none') {- return null;+ return null as TakeFrameResult<T>;}if (@@ -63,7 +76,8 @@ export const takeFrame = async ({width,height,scale,+ remotionSharedMemory,});- return buf;+ return buf as TakeFrameResult<T>;};diff --git a/packages/renderer/src/test/get-chrome-download-url.test.ts b/packages/renderer/src/test/get-chrome-download-url.test.tsnew file mode 100644index 0000000000..116bbffe9c--- /dev/null+++ b/packages/renderer/src/test/get-chrome-download-url.test.ts@@ -0,0 +1,66 @@+import {expect, test} from 'bun:test';+import {getExpectedVersion} from '../browser/BrowserFetcher';+import {+ getChromeDownloadUrl,+ TESTED_VERSION,+} from '../browser/get-chrome-download-url';++test('downloads the Remotion Chromium fork by default on Apple Silicon', () => {+ expect(+ getChromeDownloadUrl({+ platform: 'mac-arm64',+ version: null,+ chromeMode: 'headless-shell',+ }),+ ).toBe(+ `https://remotion.media/chromium-headless-shell-mac-arm64-${TESTED_VERSION}.zip?clear`,+ );+});++test('keeps explicit Apple Silicon Chromium versions on the official CDN', () => {+ expect(+ getChromeDownloadUrl({+ platform: 'mac-arm64',+ version: '148.0.0.0',+ chromeMode: 'headless-shell',+ }),+ ).toBe(+ 'https://storage.googleapis.com/chrome-for-testing-public/148.0.0.0/mac-arm64/chrome-headless-shell-mac-arm64.zip',+ );+});++test('keeps Chrome for Testing on the official CDN', () => {+ expect(+ getChromeDownloadUrl({+ platform: 'mac-arm64',+ version: null,+ chromeMode: 'chrome-for-testing',+ }),+ ).toBe(+ `https://storage.googleapis.com/chrome-for-testing-public/${TESTED_VERSION}/mac-arm64/chrome-mac-arm64.zip`,+ );+});++test('uses a distinct cache marker for the Remotion Apple Silicon build', () => {+ expect(+ getExpectedVersion({+ platform: 'mac-arm64',+ version: null,+ chromeMode: 'headless-shell',+ }),+ ).toBe(`${TESTED_VERSION}-remotion-v2`);+ expect(+ getExpectedVersion({+ platform: 'mac-arm64',+ version: TESTED_VERSION,+ chromeMode: 'headless-shell',+ }),+ ).toBe(TESTED_VERSION);+ expect(+ getExpectedVersion({+ platform: 'linux64',+ version: null,+ chromeMode: 'headless-shell',+ }),+ ).toBe(TESTED_VERSION);+});diff --git a/packages/renderer/src/test/remotion-shared-memory.test.ts b/packages/renderer/src/test/remotion-shared-memory.test.tsnew file mode 100644index 0000000000..c295743ee2--- /dev/null+++ b/packages/renderer/src/test/remotion-shared-memory.test.ts@@ -0,0 +1,534 @@+import {expect, test} from 'bun:test';+import {PassThrough, Writable} from 'node:stream';+import type {Page} from '../browser/BrowserPage';+import type {CaptureScreenshotRequest} from '../browser/devtools-types';+import type {+ RemotionRawFrame,+ RemotionSharedMemoryPoolLifetime,+} from '../remotion-shared-memory';+import {RemotionSharedMemoryCapture} from '../remotion-shared-memory';+import {+ createRemotionSharedMemoryFfmpegBridge,+ parseRemotionSharedMemoryAck,+ serializeRemotionSharedMemoryFrame,+ serializeRemotionSharedMemoryPool,+ serializeRemotionSharedMemoryPoolRetirement,+} from '../remotion-shared-memory-ffmpeg';+import {screenshotTask} from '../screenshot-task';++const makePoolLifetime = (): {+ poolLifetime: RemotionSharedMemoryPoolLifetime;+ retire: () => Promise<void>;+} => {+ let retired = false;+ let retirement: Promise<void> | null = null;+ let notifyRetirement!: () => void;+ const retirementSignal = new Promise<void>((resolve) => {+ notifyRetirement = resolve;+ });+ const listeners = new Set<() => Promise<void>>();+ const poolLifetime: RemotionSharedMemoryPoolLifetime = {+ isRetired: () => retired,+ waitForRetirement: () => retirementSignal,+ onRetired: (listener) => {+ if (retired) {+ return null;+ }++ listeners.add(listener);+ return () => listeners.delete(listener);+ },+ };++ return {+ poolLifetime,+ retire: () => {+ if (!retirement) {+ retired = true;+ notifyRetirement();+ retirement = Promise.all(+ [...listeners].map((listener) => listener()),+ ).then(() => undefined);+ }++ return retirement;+ },+ };+};++const makeRawFrame = (+ release: () => Promise<void>,+ poolLifetime = makePoolLifetime().poolLifetime,+): RemotionRawFrame => ({+ type: 'remotion-shared-memory',+ poolId: 4,+ sharedMemoryName: '/rmshm-123',+ slotCount: 2,+ slotCapacity: 1088,+ slot: 1,+ frameId: '9007199254740993',+ width: 16,+ height: 16,+ stride: 64,+ byteLength: 1024,+ pixelFormat: 'bgra',+ alphaType: 'straight',+ colorSpace: 'srgb',+ capturePath: 'viewport-redraw',+ poolLifetime,+ release,+});++test('bridges frame descriptors and releases Chromium slots on matching ACKs', async () => {+ const control = new PassThrough();+ const acknowledgements = new PassThrough();+ let controlOutput = '';+ control.on('data', (data) => {+ controlOutput += data.toString();+ });+ const lifetime = makePoolLifetime();+ let releases = 0;+ const frame = makeRawFrame(() => {+ releases++;+ return Promise.resolve();+ }, lifetime.poolLifetime);+ const bridge = createRemotionSharedMemoryFfmpegBridge({+ control,+ acknowledgements,+ });++ const {waitForAck} = await bridge.writeFrame({frame, pts: 7});+ expect(controlOutput).toBe(+ serializeRemotionSharedMemoryPool(frame) ++ serializeRemotionSharedMemoryFrame({frame, pts: 7}),+ );+ expect(releases).toBe(0);++ acknowledgements.write('4\t1\t9007199254740993\n');+ await waitForAck;+ expect(releases).toBe(1);+ expect(parseRemotionSharedMemoryAck('4\t1\t9007199254740993')).toEqual({+ poolId: 4,+ slot: 1,+ frameId: '9007199254740993',+ });++ await bridge.finish();+ await lifetime.retire();+ expect(controlOutput).not.toContain(+ serializeRemotionSharedMemoryPoolRetirement(frame.poolId),+ );+ expect(() => parseRemotionSharedMemoryAck('4\t1\tunsafe\n')).toThrow(+ 'Invalid Remotion shared-memory ACK',+ );+});++test('unregisters a retired pool only after its final frame ACK', async () => {+ const control = new PassThrough();+ const acknowledgements = new PassThrough();+ let controlOutput = '';+ control.on('data', (data) => {+ controlOutput += data.toString();+ });+ const lifetime = makePoolLifetime();+ let releases = 0;+ const frame = makeRawFrame(() => {+ releases++;+ return Promise.resolve();+ }, lifetime.poolLifetime);+ const bridge = createRemotionSharedMemoryFfmpegBridge({+ control,+ acknowledgements,+ });++ const {waitForAck} = await bridge.writeFrame({frame, pts: 3});+ let retirementFinished = false;+ const retirement = lifetime.retire().then(() => {+ retirementFinished = true;+ });+ await Promise.resolve();+ expect(retirementFinished).toBe(false);+ expect(controlOutput).not.toContain(+ serializeRemotionSharedMemoryPoolRetirement(frame.poolId),+ );++ acknowledgements.write('4\t1\t9007199254740993\n');+ await waitForAck;+ await retirement;+ expect(releases).toBe(1);+ expect(controlOutput).toBe(+ serializeRemotionSharedMemoryPool(frame) ++ serializeRemotionSharedMemoryFrame({frame, pts: 3}) ++ serializeRemotionSharedMemoryPoolRetirement(frame.poolId),+ );+ await bridge.finish();+});++test('does not register or unregister a pool retired before bridge ownership', async () => {+ const control = new PassThrough();+ const acknowledgements = new PassThrough();+ let controlOutput = '';+ control.on('data', (data) => {+ controlOutput += data.toString();+ });+ const lifetime = makePoolLifetime();+ await lifetime.retire();+ let releases = 0;+ const frame = makeRawFrame(() => {+ releases++;+ return Promise.resolve();+ }, lifetime.poolLifetime);+ const bridge = createRemotionSharedMemoryFfmpegBridge({+ control,+ acknowledgements,+ });++ await expect(bridge.writeFrame({frame, pts: 0})).rejects.toThrow(+ 'Target closed',+ );+ expect(releases).toBe(1);+ expect(controlOutput).toBe('');+ await bridge.finish();+});++test('settles and releases frames when the ACK or bridge fails', async () => {+ const control = new PassThrough();+ const acknowledgements = new PassThrough();+ let failedReleaseCalls = 0;+ const failedReleaseFrame = makeRawFrame(() => {+ failedReleaseCalls++;+ return Promise.reject(new Error('CDP release failed'));+ });+ const bridge = createRemotionSharedMemoryFfmpegBridge({+ control,+ acknowledgements,+ });+ const {waitForAck} = await bridge.writeFrame({+ frame: failedReleaseFrame,+ pts: 0,+ });+ acknowledgements.write('4\t1\t9007199254740993\n');+ await expect(waitForAck).rejects.toThrow('CDP release failed');+ expect(failedReleaseCalls).toBe(1);++ const writeError = new Error('control pipe failed');+ const failedControl = new Writable({+ highWaterMark: 1,+ write: (_chunk, _encoding, callback) => callback(writeError),+ });+ const unusedAcknowledgements = new PassThrough();+ const writeFailureLifetime = makePoolLifetime();+ let writeFailureReleases = 0;+ const writeFailureFrame = makeRawFrame(() => {+ writeFailureReleases++;+ return Promise.resolve();+ }, writeFailureLifetime.poolLifetime);+ const failedBridge = createRemotionSharedMemoryFfmpegBridge({+ control: failedControl,+ acknowledgements: unusedAcknowledgements,+ });+ await expect(+ failedBridge.writeFrame({frame: writeFailureFrame, pts: 0}),+ ).rejects.toThrow('control pipe failed');+ await writeFailureLifetime.retire();+ await new Promise((resolve) => setTimeout(resolve, 0));+ expect(writeFailureReleases).toBe(1);+});++test('treats an ACK pipe EOF before finish as fatal', async () => {+ const control = new PassThrough();+ const acknowledgements = new PassThrough();+ const bridge = createRemotionSharedMemoryFfmpegBridge({+ control,+ acknowledgements,+ });+ acknowledgements.end();+ await new Promise((resolve) => acknowledgements.once('end', resolve));+ await Promise.resolve();++ let releases = 0;+ const frame = makeRawFrame(() => {+ releases++;+ return Promise.resolve();+ });+ await expect(bridge.writeFrame({frame, pts: 0})).rejects.toThrow(+ 'before input finished',+ );+ expect(releases).toBe(1);+ control.end();+});++test('signals pool retirement before waiting for an unhanded frame to drain', async () => {+ const calls: string[] = [];+ const client = {+ send: (method: string) => {+ calls.push(method);+ if (method === 'Page.remotionCreateFramePool') {+ return Promise.resolve({+ value: {+ sharedMemoryName: '/rmshm-retirement',+ slotCount: 1,+ slotCapacity: 1088,+ },+ });+ }++ return Promise.resolve({value: undefined});+ },+ };+ const page = {+ _client: () => client,+ target: () => ({_targetId: 'target'}),+ } as unknown as Page;+ const capture = new RemotionSharedMemoryCapture({+ width: 16,+ height: 16,+ indent: false,+ logLevel: 'error',+ });+ await capture.ensurePage(page);+ const reservation = await capture.acquire(page);+ if (!reservation) {+ throw new Error('Expected a shared-memory reservation.');+ }++ const frame = await reservation.publish({+ slot: reservation.slot,+ frameId: reservation.frameId,+ width: 16,+ height: 16,+ stride: 64,+ byteLength: 1024,+ pixelFormat: 'bgra',+ alphaType: 'straight',+ colorSpace: 'srgb',+ capturePath: 'viewport-redraw',+ });+ let forgotten = false;+ const forget = capture.forgetPage(page).then(() => {+ forgotten = true;+ });+ await frame.poolLifetime.waitForRetirement();+ await Promise.resolve();+ expect(forgotten).toBe(false);+ expect(calls).not.toContain('Page.remotionDestroyFramePool');++ await frame.release();+ await forget;+ expect(calls).toContain('Page.remotionDestroyFramePool');+});++test('releases a published frame if transparent background reset fails', async () => {+ let backgroundCalls = 0;+ let releaseCalls = 0;+ const client = {+ send: (+ method: string,+ params?: Record<string, unknown>,+ ): Promise<{value: unknown}> => {+ if (method === 'Page.remotionCreateFramePool') {+ return Promise.resolve({+ value: {+ sharedMemoryName: '/rmshm-reset',+ slotCount: 1,+ slotCapacity: 1088,+ },+ });+ }++ if (method === 'Page.captureScreenshot') {+ const request = params as CaptureScreenshotRequest;+ return Promise.resolve({+ value: {+ data: '',+ remotionFrame: {+ slot: request.remotionFrameSlot,+ frameId: request.remotionFrameId,+ width: 16,+ height: 16,+ stride: 64,+ byteLength: 1024,+ pixelFormat: 'bgra',+ alphaType: 'straight',+ colorSpace: 'srgb',+ capturePath: 'viewport-redraw',+ },+ },+ });+ }++ if (method === 'Emulation.setDefaultBackgroundColorOverride') {+ backgroundCalls++;+ if (backgroundCalls === 2) {+ return Promise.reject(new Error('background reset failed'));+ }+ }++ if (method === 'Page.remotionReleaseFrame') {+ releaseCalls++;+ }++ return Promise.resolve({value: undefined});+ },+ };+ const page = {+ _client: () => client,+ target: () => ({_targetId: 'target'}),+ } as unknown as Page;+ const capture = new RemotionSharedMemoryCapture({+ width: 16,+ height: 16,+ indent: false,+ logLevel: 'error',+ });+ await capture.ensurePage(page);++ await expect(+ screenshotTask({+ format: 'png',+ height: 16,+ width: 16,+ omitBackground: true,+ page,+ jpegQuality: undefined,+ scale: 1,+ remotionSharedMemory: capture,+ }),+ ).rejects.toThrow('background reset failed');+ expect(releaseCalls).toBe(1);+ await capture.destroy();+});++test('uses raw viewport capture for every video image format and falls back on vanilla Chrome', async () => {+ const rawCaptureRequests: CaptureScreenshotRequest[] = [];+ const released: Array<{slot: number; frameId: string}> = [];+ const rawClient = {+ send: (method: string, params?: Record<string, unknown>) => {+ if (method === 'Page.remotionCreateFramePool') {+ return Promise.resolve({+ value: {+ sharedMemoryName: '/rmshm-test',+ slotCount: 1,+ slotCapacity: 1088,+ },+ });+ }++ if (method === 'Page.captureScreenshot') {+ const request = params as CaptureScreenshotRequest;+ rawCaptureRequests.push(request);+ return Promise.resolve({+ value: {+ data: '',+ remotionFrame: {+ slot: request.remotionFrameSlot,+ frameId: request.remotionFrameId,+ width: 16,+ height: 16,+ stride: 64,+ byteLength: 1024,+ pixelFormat: 'bgra',+ alphaType: request.format === 'jpeg' ? 'opaque' : 'straight',+ colorSpace: 'srgb',+ capturePath: 'viewport-redraw',+ },+ },+ });+ }++ if (method === 'Page.remotionReleaseFrame') {+ released.push(params as {slot: number; frameId: string});+ }++ return Promise.resolve({value: undefined});+ },+ };+ const rawPage = {+ _client: () => rawClient,+ target: () => ({_targetId: 'target'}),+ } as unknown as Page;+ const rawCapture = new RemotionSharedMemoryCapture({+ width: 16,+ height: 16,+ indent: false,+ logLevel: 'error',+ });+ await rawCapture.ensurePage(rawPage);++ for (const format of ['jpeg', 'png', 'webp'] as const) {+ const frame = await screenshotTask({+ format,+ height: 16,+ width: 16,+ omitBackground: format !== 'jpeg',+ page: rawPage,+ jpegQuality: format === 'jpeg' ? 80 : undefined,+ scale: 1,+ remotionSharedMemory: rawCapture,+ });+ expect(Buffer.isBuffer(frame)).toBe(false);+ if (!Buffer.isBuffer(frame)) {+ await frame.release();+ }+ }++ expect(+ rawCaptureRequests.map((request) => ({+ format: request.format,+ fast: request.remotionFastViewport,+ hasSlot: request.remotionFrameSlot !== undefined,+ })),+ ).toEqual([+ {format: 'jpeg', fast: true, hasSlot: true},+ {format: 'png', fast: true, hasSlot: true},+ {format: 'webp', fast: true, hasSlot: true},+ ]);+ expect(released).toHaveLength(3);+ await rawCapture.destroy();++ const fallbackRequests: CaptureScreenshotRequest[] = [];+ const vanillaClient = {+ send: (method: string, params?: Record<string, unknown>) => {+ if (method === 'Page.remotionCreateFramePool') {+ return Promise.reject(+ new Error(+ "Protocol error (Page.remotionCreateFramePool): 'Page.remotionCreateFramePool' wasn't found",+ ),+ );+ }++ if (method === 'Page.captureScreenshot') {+ fallbackRequests.push(params as CaptureScreenshotRequest);+ return Promise.resolve({+ value: {data: Buffer.from('jpeg').toString('base64')},+ });+ }++ return Promise.resolve({value: undefined});+ },+ };+ const vanillaPage = {+ _client: () => vanillaClient,+ target: () => ({_targetId: 'target'}),+ } as unknown as Page;+ const fallbackCapture = new RemotionSharedMemoryCapture({+ width: 16,+ height: 16,+ indent: false,+ logLevel: 'error',+ });+ await fallbackCapture.ensurePage(vanillaPage);+ const encoded = await screenshotTask({+ format: 'jpeg',+ height: 16,+ width: 16,+ omitBackground: false,+ page: vanillaPage,+ jpegQuality: 80,+ scale: 1,+ remotionSharedMemory: fallbackCapture,+ });+ expect(encoded).toEqual(Buffer.from('jpeg'));+ expect(fallbackRequests[0]?.remotionFastViewport).toBeUndefined();+ expect(fallbackRequests[0]?.remotionFrameSlot).toBeUndefined();+});