🎥 Make videos programmatically with React
`@remotion/renderer`: Apply backpressure while encoding frames (#10656)
Jonny Burger3d623772b027
diff --git a/packages/docs/docs/renderer/render-frames.mdx b/packages/docs/docs/renderer/render-frames.mdxindex 01628d50b6..9e78d916f4 100644--- a/packages/docs/docs/renderer/render-frames.mdx+++ b/packages/docs/docs/renderer/render-frames.mdx@@ -251,6 +251,8 @@ A token that allows the render to be cancelled. See: [`makeCancelSignal()`](/docIf you passed `null` to `outputDir`, this method will be called passing a buffer of the current frame. This is mostly used internally by Remotion to implement [`renderMedia()`](/docs/renderer/render-media) and might have limited usefulness for end users.+From <AvailableFrom v="4.0.515" inline />, the callback may return a promise. Remotion will wait for it to resolve before reusing the renderer for another frame, allowing the callback to apply backpressure.+### `timeoutInMilliseconds?`<AvailableFrom v="2.6.3" />A number describing how long one frame may take to resolve all [`delayRender()`](/docs/delay-render) calls before the [render times out and fails](/docs/timeout). Default: `30000`diff --git a/packages/it-tests/src/ssr/render-frames.test.ts b/packages/it-tests/src/ssr/render-frames.test.tsindex eaf50222e1..a3c6f2fe4d 100644--- a/packages/it-tests/src/ssr/render-frames.test.ts+++ b/packages/it-tests/src/ssr/render-frames.test.ts@@ -111,3 +111,65 @@ test('renderFrames() should render selected frames', async () => {await puppeteerInstance.close({silent: false});}});++test('renderFrames() should await async frame buffer callbacks', async () => {+ const puppeteerInstance = await openBrowser('chrome');++ try {+ const scripts = await selectComposition({+ id: 'scripts',+ serveUrl: exampleBuild,+ puppeteerInstance,+ inputProps: {},+ });+ const completedFrames: number[] = [];+ let notifyFirstFrameReceived: () => void = () => {+ throw new Error('The first frame was not received.');+ };+ const firstFrameReceived = new Promise<void>((resolve) => {+ notifyFirstFrameReceived = resolve;+ });+ let releaseFirstFrame: () => void = () => {+ throw new Error('The first frame was not received.');+ };+ const firstFrameCanFinish = new Promise<void>((resolve) => {+ releaseFirstFrame = resolve;+ });++ const render = renderFrames({+ composition: scripts,+ frames: [0],+ imageFormat: 'jpeg',+ inputProps: {},+ onFrameUpdate: () => undefined,+ serveUrl: exampleBuild,+ concurrency: 1,+ outputDir: null,+ onStart: () => undefined,+ puppeteerInstance,+ onFrameBuffer: async (_buffer, frame) => {+ if (frame === 0) {+ notifyFirstFrameReceived();+ await firstFrameCanFinish;+ }++ completedFrames.push(frame);+ },+ });++ await firstFrameReceived;+ const stateWhileCallbackIsPending = await Promise.race([+ render.then(() => 'finished' as const),+ new Promise<'pending'>((resolve) => {+ setTimeout(() => resolve('pending'), 500);+ }),+ ]);+ releaseFirstFrame();+ await render;++ expect(stateWhileCallbackIsPending).toBe('pending');+ expect(completedFrames).toEqual([0]);+ } finally {+ await puppeteerInstance.close({silent: false});+ }+});diff --git a/packages/renderer/src/prestitcher-memory-usage.ts b/packages/renderer/src/prestitcher-memory-usage.tsindex 339edc2652..d20d79e8ad 100644--- a/packages/renderer/src/prestitcher-memory-usage.ts+++ b/packages/renderer/src/prestitcher-memory-usage.ts@@ -1,6 +1,18 @@import type {LogLevel} from './log-level';import {getAvailableMemory} from './memory/get-available-memory';+const MINIMUM_MEMORY_LEFT = 2_000_000_000;++export const hasEnoughMemoryForParallelEncoding = ({+ freeMemory,+ estimatedUsage,+}: {+ freeMemory: number;+ estimatedUsage: number;+}) => {+ return freeMemory - estimatedUsage > MINIMUM_MEMORY_LEFT;+};+const estimateMemoryUsageForPrestitcher = ({width,height,@@ -31,9 +43,10 @@ export const shouldUseParallelEncoding = ({width,});- const hasEnoughMemory =- freeMemory - estimatedUsage > 2_000_000_000 &&- estimatedUsage / freeMemory < 0.5;+ const hasEnoughMemory = hasEnoughMemoryForParallelEncoding({+ estimatedUsage,+ freeMemory,+ });return {hasEnoughMemory,diff --git a/packages/renderer/src/render-frame-and-retry-target-close.ts b/packages/renderer/src/render-frame-and-retry-target-close.tsindex e932b5be03..98fa9b5267 100644--- a/packages/renderer/src/render-frame-and-retry-target-close.ts+++ b/packages/renderer/src/render-frame-and-retry-target-close.ts@@ -83,7 +83,10 @@ export const renderFrameAndRetryTargetClose = async ({concurrencyOrFramesToRender: number;lastFrame: number;framesRenderedObj: {count: number};- onFrameBuffer: null | ((buffer: Buffer, frame: number) => void) | undefined;+ onFrameBuffer:+ | null+ | ((buffer: Buffer, frame: number) => void | Promise<void>)+ | undefined;onFrameUpdate:| null| ((diff --git a/packages/renderer/src/render-frame-with-option-to-reject.ts b/packages/renderer/src/render-frame-with-option-to-reject.tsindex a93b7a6173..63de6dc1f3 100644--- a/packages/renderer/src/render-frame-with-option-to-reject.ts+++ b/packages/renderer/src/render-frame-with-option-to-reject.ts@@ -68,7 +68,10 @@ export const renderFrameWithOptionToReject = async ({indent: boolean;logLevel: LogLevel;outputDir: string | null;- onFrameBuffer: null | ((buffer: Buffer, frame: number) => void) | undefined;+ onFrameBuffer:+ | null+ | ((buffer: Buffer, frame: number) => void | Promise<void>)+ | undefined;imageFormat: VideoImageFormat;onError: (err: Error) => void;lastFrame: number;@@ -188,7 +191,7 @@ export const renderFrameWithOptionToReject = async ({throw new Error('unexpected null buffer');}- onFrameBuffer(buffer, frame);+ await onFrameBuffer(buffer, frame);}const onlyAvailableAssets = assets.filter(truthy);diff --git a/packages/renderer/src/render-frame.ts b/packages/renderer/src/render-frame.tsindex 1427c6022e..1ec4021bbd 100644--- a/packages/renderer/src/render-frame.ts+++ b/packages/renderer/src/render-frame.ts@@ -62,7 +62,10 @@ export const renderFrame = ({stoppedSignal: {stopped: boolean};timeoutInMilliseconds: number;outputDir: string | null;- onFrameBuffer: null | ((buffer: Buffer, frame: number) => void) | undefined;+ onFrameBuffer:+ | null+ | ((buffer: Buffer, frame: number) => void | Promise<void>)+ | undefined;lastFrame: number;onFrameUpdate:| nulldiff --git a/packages/renderer/src/render-frames.ts b/packages/renderer/src/render-frames.tsindex 464bd60223..a13b653d97 100644--- a/packages/renderer/src/render-frames.ts+++ b/packages/renderer/src/render-frames.ts@@ -85,7 +85,9 @@ type InternalRenderFramesOptions = {puppeteerInstance: HeadlessBrowser | undefined;browserExecutable: BrowserExecutable | null;onBrowserLog: null | ((log: BrowserLog) => void);- onFrameBuffer: null | ((buffer: Buffer, frame: number) => void);+ onFrameBuffer:+ | null+ | ((buffer: Buffer, frame: number) => void | Promise<void>);onDownload: RenderMediaOnDownload | null;chromiumOptions: ChromiumOptions;scale: number;@@ -122,7 +124,9 @@ type InnerRenderFramesOptions = {outputFramesInSequence: boolean;everyNthFrame: number;onBrowserLog: null | ((log: BrowserLog) => void);- onFrameBuffer: null | ((buffer: Buffer, frame: number) => void);+ onFrameBuffer:+ | null+ | ((buffer: Buffer, frame: number) => void | Promise<void>);onArtifact: OnArtifact | null;onDownload: RenderMediaOnDownload | null;timeoutInMilliseconds: number;@@ -195,7 +199,7 @@ export type RenderFramesOptions = Prettify<puppeteerInstance?: HeadlessBrowser;browserExecutable?: BrowserExecutable;onBrowserLog?: (log: BrowserLog) => void;- onFrameBuffer?: (buffer: Buffer, frame: number) => void;+ onFrameBuffer?: (buffer: Buffer, frame: number) => void | Promise<void>;onDownload?: RenderMediaOnDownload;timeoutInMilliseconds?: number;chromiumOptions?: ChromiumOptions;diff --git a/packages/renderer/src/render-media.ts b/packages/renderer/src/render-media.tsindex 7c85773f05..d987bc98c4 100644--- a/packages/renderer/src/render-media.ts+++ b/packages/renderer/src/render-media.ts@@ -80,6 +80,7 @@ import {validateOutputFilename} from './validate-output-filename';import {validateScale} from './validate-scale';import {validateBitrate} from './validate-videobitrate';import {wrapWithErrorHandling} from './wrap-with-error-handling';+import {writeWithBackpressure} from './write-with-backpressure';export type StitchingState = 'encoding' | 'muxing';@@ -718,7 +719,14 @@ const internalRenderMediaRaw = ({);}- stitcherFfmpeg?.stdin?.write(buffer);+ const stdin = stitcherFfmpeg?.stdin;+ if (!stdin) {+ throw new Error(+ `FFmpeg stdin is not available while trying to pipe frame ${frame} to it.`,+ );+ }++ await writeWithBackpressure({data: buffer, writable: stdin});stopPerfMeasure(id);const frameIndex = framesToRender.indexOf(frame);diff --git a/packages/renderer/src/test/prestitcher-memory-usage.test.ts b/packages/renderer/src/test/prestitcher-memory-usage.test.tsnew file mode 100644index 0000000000..3df7e94b3f--- /dev/null+++ b/packages/renderer/src/test/prestitcher-memory-usage.test.ts@@ -0,0 +1,19 @@+import {expect, test} from 'bun:test';+import {hasEnoughMemoryForParallelEncoding} from '../prestitcher-memory-usage';++test('parallel encoding keeps a fixed 2GB memory reserve', () => {+ const estimatedUsage = 8_294_400_000;++ expect(+ hasEnoughMemoryForParallelEncoding({+ estimatedUsage,+ freeMemory: 11_000_000_000,+ }),+ ).toBe(true);+ expect(+ hasEnoughMemoryForParallelEncoding({+ estimatedUsage,+ freeMemory: 10_000_000_000,+ }),+ ).toBe(false);+});diff --git a/packages/renderer/src/test/write-with-backpressure.test.ts b/packages/renderer/src/test/write-with-backpressure.test.tsnew file mode 100644index 0000000000..3ada18166f--- /dev/null+++ b/packages/renderer/src/test/write-with-backpressure.test.ts@@ -0,0 +1,32 @@+import {expect, test} from 'bun:test';+import {Writable} from 'node:stream';+import {writeWithBackpressure} from '../write-with-backpressure';++test('waits for a writable stream to drain', async () => {+ let hasStartedWriting = false;+ let finishWrite: (error?: Error | null) => void = () => {+ throw new Error('The write did not start.');+ };++ const writable = new Writable({+ highWaterMark: 1,+ write: (_chunk, _encoding, callback) => {+ hasStartedWriting = true;+ finishWrite = callback;+ },+ });+ let settled = false;+ const write = writeWithBackpressure({+ data: Buffer.from('frame'),+ writable,+ }).then(() => {+ settled = true;+ });++ await Promise.resolve();+ expect(settled).toBe(false);+ expect(hasStartedWriting).toBe(true);+ finishWrite();+ await write;+ expect(settled).toBe(true);+});diff --git a/packages/renderer/src/write-with-backpressure.ts b/packages/renderer/src/write-with-backpressure.tsnew file mode 100644index 0000000000..676ec42178--- /dev/null+++ b/packages/renderer/src/write-with-backpressure.ts@@ -0,0 +1,46 @@+import type {Writable} from 'node:stream';++export const writeWithBackpressure = ({+ data,+ writable,+}: {+ data: Buffer;+ writable: Writable;+}): Promise<void> => {+ if (writable.write(data)) {+ return Promise.resolve();+ }++ return new Promise<void>((resolve, reject) => {+ const cleanup = () => {+ writable.off('close', onClose);+ writable.off('drain', onDrain);+ writable.off('error', onError);+ };++ const onClose = () => {+ cleanup();+ reject(+ new Error('The writable stream closed before it accepted all data.'),+ );+ };++ const onDrain = () => {+ cleanup();+ resolve();+ };++ const onError = (error: Error) => {+ cleanup();+ reject(error);+ };++ writable.once('close', onClose);+ writable.once('drain', onDrain);+ writable.once('error', onError);++ if (writable.destroyed) {+ onClose();+ }+ });+};