Skip to content

🎥 Make videos programmatically with React

`@remotion/renderer`: Apply backpressure while encoding frames (#10656)

Jonny Burger3d623772b027
diff --git a/packages/docs/docs/renderer/render-frames.mdx b/packages/docs/docs/renderer/render-frames.mdx
index 01628d50b6..9e78d916f4 100644
--- a/packages/docs/docs/renderer/render-frames.mdx
+++ b/packages/docs/docs/renderer/render-frames.mdx
@@ -251,6 +251,8 @@ A token that allows the render to be cancelled. See: [`makeCancelSignal()`](/doc
If you passed `null` to `outputDir`, this method will be called passing a buffer of the current frame. This is mostly used internally by Remotion to implement [`renderMedia()`](/docs/renderer/render-media) and might have limited usefulness for end users.
+From <AvailableFrom v="4.0.515" inline />, the callback may return a promise. Remotion will wait for it to resolve before reusing the renderer for another frame, allowing the callback to apply backpressure.
+
### `timeoutInMilliseconds?`<AvailableFrom v="2.6.3" />
A number describing how long one frame may take to resolve all [`delayRender()`](/docs/delay-render) calls before the [render times out and fails](/docs/timeout). Default: `30000`
diff --git a/packages/it-tests/src/ssr/render-frames.test.ts b/packages/it-tests/src/ssr/render-frames.test.ts
index eaf50222e1..a3c6f2fe4d 100644
--- a/packages/it-tests/src/ssr/render-frames.test.ts
+++ b/packages/it-tests/src/ssr/render-frames.test.ts
@@ -111,3 +111,65 @@ test('renderFrames() should render selected frames', async () => {
await puppeteerInstance.close({silent: false});
}
});
+
+test('renderFrames() should await async frame buffer callbacks', async () => {
+ const puppeteerInstance = await openBrowser('chrome');
+
+ try {
+ const scripts = await selectComposition({
+ id: 'scripts',
+ serveUrl: exampleBuild,
+ puppeteerInstance,
+ inputProps: {},
+ });
+ const completedFrames: number[] = [];
+ let notifyFirstFrameReceived: () => void = () => {
+ throw new Error('The first frame was not received.');
+ };
+ const firstFrameReceived = new Promise<void>((resolve) => {
+ notifyFirstFrameReceived = resolve;
+ });
+ let releaseFirstFrame: () => void = () => {
+ throw new Error('The first frame was not received.');
+ };
+ const firstFrameCanFinish = new Promise<void>((resolve) => {
+ releaseFirstFrame = resolve;
+ });
+
+ const render = renderFrames({
+ composition: scripts,
+ frames: [0],
+ imageFormat: 'jpeg',
+ inputProps: {},
+ onFrameUpdate: () => undefined,
+ serveUrl: exampleBuild,
+ concurrency: 1,
+ outputDir: null,
+ onStart: () => undefined,
+ puppeteerInstance,
+ onFrameBuffer: async (_buffer, frame) => {
+ if (frame === 0) {
+ notifyFirstFrameReceived();
+ await firstFrameCanFinish;
+ }
+
+ completedFrames.push(frame);
+ },
+ });
+
+ await firstFrameReceived;
+ const stateWhileCallbackIsPending = await Promise.race([
+ render.then(() => 'finished' as const),
+ new Promise<'pending'>((resolve) => {
+ setTimeout(() => resolve('pending'), 500);
+ }),
+ ]);
+ releaseFirstFrame();
+ await render;
+
+ expect(stateWhileCallbackIsPending).toBe('pending');
+ expect(completedFrames).toEqual([0]);
+ } finally {
+ await puppeteerInstance.close({silent: false});
+ }
+});
diff --git a/packages/renderer/src/prestitcher-memory-usage.ts b/packages/renderer/src/prestitcher-memory-usage.ts
index 339edc2652..d20d79e8ad 100644
--- a/packages/renderer/src/prestitcher-memory-usage.ts
+++ b/packages/renderer/src/prestitcher-memory-usage.ts
@@ -1,6 +1,18 @@
import type {LogLevel} from './log-level';
import {getAvailableMemory} from './memory/get-available-memory';
+const MINIMUM_MEMORY_LEFT = 2_000_000_000;
+
+export const hasEnoughMemoryForParallelEncoding = ({
+ freeMemory,
+ estimatedUsage,
+}: {
+ freeMemory: number;
+ estimatedUsage: number;
+}) => {
+ return freeMemory - estimatedUsage > MINIMUM_MEMORY_LEFT;
+};
+
const estimateMemoryUsageForPrestitcher = ({
width,
height,
@@ -31,9 +43,10 @@ export const shouldUseParallelEncoding = ({
width,
});
- const hasEnoughMemory =
- freeMemory - estimatedUsage > 2_000_000_000 &&
- estimatedUsage / freeMemory < 0.5;
+ const hasEnoughMemory = hasEnoughMemoryForParallelEncoding({
+ estimatedUsage,
+ freeMemory,
+ });
return {
hasEnoughMemory,
diff --git a/packages/renderer/src/render-frame-and-retry-target-close.ts b/packages/renderer/src/render-frame-and-retry-target-close.ts
index e932b5be03..98fa9b5267 100644
--- a/packages/renderer/src/render-frame-and-retry-target-close.ts
+++ b/packages/renderer/src/render-frame-and-retry-target-close.ts
@@ -83,7 +83,10 @@ export const renderFrameAndRetryTargetClose = async ({
concurrencyOrFramesToRender: number;
lastFrame: number;
framesRenderedObj: {count: number};
- onFrameBuffer: null | ((buffer: Buffer, frame: number) => void) | undefined;
+ onFrameBuffer:
+ | null
+ | ((buffer: Buffer, frame: number) => void | Promise<void>)
+ | undefined;
onFrameUpdate:
| null
| ((
diff --git a/packages/renderer/src/render-frame-with-option-to-reject.ts b/packages/renderer/src/render-frame-with-option-to-reject.ts
index a93b7a6173..63de6dc1f3 100644
--- a/packages/renderer/src/render-frame-with-option-to-reject.ts
+++ b/packages/renderer/src/render-frame-with-option-to-reject.ts
@@ -68,7 +68,10 @@ export const renderFrameWithOptionToReject = async ({
indent: boolean;
logLevel: LogLevel;
outputDir: string | null;
- onFrameBuffer: null | ((buffer: Buffer, frame: number) => void) | undefined;
+ onFrameBuffer:
+ | null
+ | ((buffer: Buffer, frame: number) => void | Promise<void>)
+ | undefined;
imageFormat: VideoImageFormat;
onError: (err: Error) => void;
lastFrame: number;
@@ -188,7 +191,7 @@ export const renderFrameWithOptionToReject = async ({
throw new Error('unexpected null buffer');
}
- onFrameBuffer(buffer, frame);
+ await onFrameBuffer(buffer, frame);
}
const onlyAvailableAssets = assets.filter(truthy);
diff --git a/packages/renderer/src/render-frame.ts b/packages/renderer/src/render-frame.ts
index 1427c6022e..1ec4021bbd 100644
--- a/packages/renderer/src/render-frame.ts
+++ b/packages/renderer/src/render-frame.ts
@@ -62,7 +62,10 @@ export const renderFrame = ({
stoppedSignal: {stopped: boolean};
timeoutInMilliseconds: number;
outputDir: string | null;
- onFrameBuffer: null | ((buffer: Buffer, frame: number) => void) | undefined;
+ onFrameBuffer:
+ | null
+ | ((buffer: Buffer, frame: number) => void | Promise<void>)
+ | undefined;
lastFrame: number;
onFrameUpdate:
| null
diff --git a/packages/renderer/src/render-frames.ts b/packages/renderer/src/render-frames.ts
index 464bd60223..a13b653d97 100644
--- a/packages/renderer/src/render-frames.ts
+++ b/packages/renderer/src/render-frames.ts
@@ -85,7 +85,9 @@ type InternalRenderFramesOptions = {
puppeteerInstance: HeadlessBrowser | undefined;
browserExecutable: BrowserExecutable | null;
onBrowserLog: null | ((log: BrowserLog) => void);
- onFrameBuffer: null | ((buffer: Buffer, frame: number) => void);
+ onFrameBuffer:
+ | null
+ | ((buffer: Buffer, frame: number) => void | Promise<void>);
onDownload: RenderMediaOnDownload | null;
chromiumOptions: ChromiumOptions;
scale: number;
@@ -122,7 +124,9 @@ type InnerRenderFramesOptions = {
outputFramesInSequence: boolean;
everyNthFrame: number;
onBrowserLog: null | ((log: BrowserLog) => void);
- onFrameBuffer: null | ((buffer: Buffer, frame: number) => void);
+ onFrameBuffer:
+ | null
+ | ((buffer: Buffer, frame: number) => void | Promise<void>);
onArtifact: OnArtifact | null;
onDownload: RenderMediaOnDownload | null;
timeoutInMilliseconds: number;
@@ -195,7 +199,7 @@ export type RenderFramesOptions = Prettify<
puppeteerInstance?: HeadlessBrowser;
browserExecutable?: BrowserExecutable;
onBrowserLog?: (log: BrowserLog) => void;
- onFrameBuffer?: (buffer: Buffer, frame: number) => void;
+ onFrameBuffer?: (buffer: Buffer, frame: number) => void | Promise<void>;
onDownload?: RenderMediaOnDownload;
timeoutInMilliseconds?: number;
chromiumOptions?: ChromiumOptions;
diff --git a/packages/renderer/src/render-media.ts b/packages/renderer/src/render-media.ts
index 7c85773f05..d987bc98c4 100644
--- a/packages/renderer/src/render-media.ts
+++ b/packages/renderer/src/render-media.ts
@@ -80,6 +80,7 @@ import {validateOutputFilename} from './validate-output-filename';
import {validateScale} from './validate-scale';
import {validateBitrate} from './validate-videobitrate';
import {wrapWithErrorHandling} from './wrap-with-error-handling';
+import {writeWithBackpressure} from './write-with-backpressure';
export type StitchingState = 'encoding' | 'muxing';
@@ -718,7 +719,14 @@ const internalRenderMediaRaw = ({
);
}
- stitcherFfmpeg?.stdin?.write(buffer);
+ const stdin = stitcherFfmpeg?.stdin;
+ if (!stdin) {
+ throw new Error(
+ `FFmpeg stdin is not available while trying to pipe frame ${frame} to it.`,
+ );
+ }
+
+ await writeWithBackpressure({data: buffer, writable: stdin});
stopPerfMeasure(id);
const frameIndex = framesToRender.indexOf(frame);
diff --git a/packages/renderer/src/test/prestitcher-memory-usage.test.ts b/packages/renderer/src/test/prestitcher-memory-usage.test.ts
new file mode 100644
index 0000000000..3df7e94b3f
--- /dev/null
+++ b/packages/renderer/src/test/prestitcher-memory-usage.test.ts
@@ -0,0 +1,19 @@
+import {expect, test} from 'bun:test';
+import {hasEnoughMemoryForParallelEncoding} from '../prestitcher-memory-usage';
+
+test('parallel encoding keeps a fixed 2GB memory reserve', () => {
+ const estimatedUsage = 8_294_400_000;
+
+ expect(
+ hasEnoughMemoryForParallelEncoding({
+ estimatedUsage,
+ freeMemory: 11_000_000_000,
+ }),
+ ).toBe(true);
+ expect(
+ hasEnoughMemoryForParallelEncoding({
+ estimatedUsage,
+ freeMemory: 10_000_000_000,
+ }),
+ ).toBe(false);
+});
diff --git a/packages/renderer/src/test/write-with-backpressure.test.ts b/packages/renderer/src/test/write-with-backpressure.test.ts
new file mode 100644
index 0000000000..3ada18166f
--- /dev/null
+++ b/packages/renderer/src/test/write-with-backpressure.test.ts
@@ -0,0 +1,32 @@
+import {expect, test} from 'bun:test';
+import {Writable} from 'node:stream';
+import {writeWithBackpressure} from '../write-with-backpressure';
+
+test('waits for a writable stream to drain', async () => {
+ let hasStartedWriting = false;
+ let finishWrite: (error?: Error | null) => void = () => {
+ throw new Error('The write did not start.');
+ };
+
+ const writable = new Writable({
+ highWaterMark: 1,
+ write: (_chunk, _encoding, callback) => {
+ hasStartedWriting = true;
+ finishWrite = callback;
+ },
+ });
+ let settled = false;
+ const write = writeWithBackpressure({
+ data: Buffer.from('frame'),
+ writable,
+ }).then(() => {
+ settled = true;
+ });
+
+ await Promise.resolve();
+ expect(settled).toBe(false);
+ expect(hasStartedWriting).toBe(true);
+ finishWrite();
+ await write;
+ expect(settled).toBe(true);
+});
diff --git a/packages/renderer/src/write-with-backpressure.ts b/packages/renderer/src/write-with-backpressure.ts
new file mode 100644
index 0000000000..676ec42178
--- /dev/null
+++ b/packages/renderer/src/write-with-backpressure.ts
@@ -0,0 +1,46 @@
+import type {Writable} from 'node:stream';
+
+export const writeWithBackpressure = ({
+ data,
+ writable,
+}: {
+ data: Buffer;
+ writable: Writable;
+}): Promise<void> => {
+ if (writable.write(data)) {
+ return Promise.resolve();
+ }
+
+ return new Promise<void>((resolve, reject) => {
+ const cleanup = () => {
+ writable.off('close', onClose);
+ writable.off('drain', onDrain);
+ writable.off('error', onError);
+ };
+
+ const onClose = () => {
+ cleanup();
+ reject(
+ new Error('The writable stream closed before it accepted all data.'),
+ );
+ };
+
+ const onDrain = () => {
+ cleanup();
+ resolve();
+ };
+
+ const onError = (error: Error) => {
+ cleanup();
+ reject(error);
+ };
+
+ writable.once('close', onClose);
+ writable.once('drain', onDrain);
+ writable.once('error', onError);
+
+ if (writable.destroyed) {
+ onClose();
+ }
+ });
+};