Skip to content

🎥 Make videos programmatically with React

`@remotion/media`: Add a `maxCanvasSinkFrameSize` prop to `<Video>` for smaller preview frames

Don Carignana128482ade7a
diff --git a/packages/docs/docs/media/video.mdx b/packages/docs/docs/media/video.mdx
index fd8337dfbd..b40f7930de 100644
--- a/packages/docs/docs/media/video.mdx
+++ b/packages/docs/docs/media/video.mdx
@@ -319,6 +319,41 @@ export const MyComposition = () => {
The CSS property `object-fit` is not supported.
:::
+### `maxCanvasSinkFrameSize?`<AvailableFrom v="4.0.530" />
+
+Limits the size of the frames that are drawn during the preview, in pixels. Pass a `width`, a `height` or both. By default, the frames have the size of the video.
+
+The frames keep the aspect ratio of the video, fit within the size and are never upscaled. They get scaled up to the size of the `<canvas>` when drawn, so the layout does not change.
+
+Smaller frames use less memory, which helps when a large video is shown small. The video is still decoded at full size, and rendering is not affected.
+
+For example, use [`useCurrentScale()`](/docs/use-current-scale) to pick a size:
+
+```tsx twoslash title="Smaller frames when the preview is zoomed out"
+import {useCurrentScale} from 'remotion';
+import {Video} from '@remotion/media';
+
+// ---cut---
+export const MyComposition = () => {
+ const scale = useCurrentScale();
+ const width = scale > 0.5 ? 1920 : scale > 0.25 ? 960 : 480;
+
+ return <Video src="https://remotion.media/video.mp4" maxCanvasSinkFrameSize={{width}} />;
+};
+```
+
+In the preview, [`onVideoFrame`](#onvideoframe) receives the smaller frames, and [`effects`](#effects) receive them scaled up to the size of the `<canvas>`.
+
+Changing the numbers re-initializes the video, so prefer a few fixed sizes over a value that changes on every resize. Passing a new object with the same numbers does not.
+
+#### `width?`
+
+The maximum width of the frames.
+
+#### `height?`
+
+The maximum height of the frames.
+
### `loop?`
Makes the video loop indefinitely.
diff --git a/packages/media/src/audio/audio-for-preview.tsx b/packages/media/src/audio/audio-for-preview.tsx
index cb40ec0e4e..01824c6790 100644
--- a/packages/media/src/audio/audio-for-preview.tsx
+++ b/packages/media/src/audio/audio-for-preview.tsx
@@ -242,6 +242,7 @@ const AudioForPreviewAssertedShowing: React.FC<NewAudioForPreviewProps> = ({
tagType: 'audio',
getEffects: () => [],
getEffectChainState: () => null,
+ maxCanvasSinkFrameSize: null,
onError: (error) => {
const [action, errorToUse] = callOnErrorAndResolve({
onError: onErrorRef.current,
diff --git a/packages/media/src/index.ts b/packages/media/src/index.ts
index 842da05b58..b36843f889 100644
--- a/packages/media/src/index.ts
+++ b/packages/media/src/index.ts
@@ -16,6 +16,7 @@ export {MediaErrorAction} from './on-error';
export type {MediaRequestInit} from './request-init';
export {
FallbackOffthreadVideoProps,
+ MaxCanvasSinkFrameSize,
VideoObjectFit,
VideoProps,
} from './video/props';
diff --git a/packages/media/src/media-player.ts b/packages/media/src/media-player.ts
index b998d958d9..1218095cff 100644
--- a/packages/media/src/media-player.ts
+++ b/packages/media/src/media-player.ts
@@ -30,6 +30,7 @@ import type {MediaRequestInit} from './request-init';
import type {SharedAudioContextForMediaPlayer} from './shared-audio-context-for-media-player';
import type {VideoIteratorManager} from './video-iterator-manager';
import {videoIteratorManager} from './video-iterator-manager';
+import type {MaxCanvasSinkFrameSize} from './video/props';
export type MediaPlayerInitResult =
| {type: 'success'; durationInSeconds: number}
@@ -94,6 +95,8 @@ export class MediaPlayer {
height: number,
) => EffectChainState | null;
+ private maxCanvasSinkFrameSize: MaxCanvasSinkFrameSize | null;
+
private initializationPromise: Promise<MediaPlayerInitResult> | null = null;
private premountAwareDelayPlayback: PremountAwareDelayPlayback;
@@ -127,6 +130,7 @@ export class MediaPlayer {
tagType,
getEffects,
getEffectChainState,
+ maxCanvasSinkFrameSize,
onError,
}: {
canvas: HTMLCanvasElement | OffscreenCanvas | null;
@@ -157,6 +161,7 @@ export class MediaPlayer {
width: number,
height: number,
) => EffectChainState | null;
+ maxCanvasSinkFrameSize: MaxCanvasSinkFrameSize | null;
onError: ((error: Error) => void) | null;
}) {
this.canvas = canvas ?? null;
@@ -197,6 +202,7 @@ export class MediaPlayer {
this.tagType = tagType;
this.getEffects = getEffects;
this.getEffectChainState = getEffectChainState;
+ this.maxCanvasSinkFrameSize = maxCanvasSinkFrameSize;
this.onError = onError;
if (canvas) {
@@ -386,6 +392,7 @@ export class MediaPlayer {
getIsLooping: () => this.loop,
getEffects: this.getEffects,
getEffectChainState: this.getEffectChainState,
+ maxCanvasSinkFrameSize: this.maxCanvasSinkFrameSize,
});
}
diff --git a/packages/media/src/test/media-player.test.ts b/packages/media/src/test/media-player.test.ts
index 1060f54d85..91724f2ff4 100644
--- a/packages/media/src/test/media-player.test.ts
+++ b/packages/media/src/test/media-player.test.ts
@@ -65,6 +65,7 @@ const makeAudioPlayer = (
tagType,
getEffects: () => [],
getEffectChainState: () => null,
+ maxCanvasSinkFrameSize: null,
onError,
});
};
@@ -195,6 +196,7 @@ test('dispose should immediately unblock a pending playback catch-up', async ()
tagType: 'video',
getEffects: () => [],
getEffectChainState: () => null,
+ maxCanvasSinkFrameSize: null,
onError: null,
});
diff --git a/packages/media/src/test/premount-arm-kicks-scheduler.test.ts b/packages/media/src/test/premount-arm-kicks-scheduler.test.ts
index 0fe259e9f0..ce078792f3 100644
--- a/packages/media/src/test/premount-arm-kicks-scheduler.test.ts
+++ b/packages/media/src/test/premount-arm-kicks-scheduler.test.ts
@@ -38,6 +38,7 @@ const makePlayer = ({
tagType: 'audio',
getEffects: () => [],
getEffectChainState: () => null,
+ maxCanvasSinkFrameSize: null,
onError: null,
});
};
diff --git a/packages/media/src/test/trim-change-seek.test.ts b/packages/media/src/test/trim-change-seek.test.ts
index 1f1822ba89..83bfc359d5 100644
--- a/packages/media/src/test/trim-change-seek.test.ts
+++ b/packages/media/src/test/trim-change-seek.test.ts
@@ -28,6 +28,7 @@ test('setTrimRange should atomically update trims when paused', async () => {
tagType: 'video',
getEffects: () => [],
getEffectChainState: () => null,
+ maxCanvasSinkFrameSize: null,
onError: null,
});
diff --git a/packages/media/src/test/video-iterator-manager.test.ts b/packages/media/src/test/video-iterator-manager.test.ts
index a333c792c1..7939873dd9 100644
--- a/packages/media/src/test/video-iterator-manager.test.ts
+++ b/packages/media/src/test/video-iterator-manager.test.ts
@@ -98,6 +98,7 @@ const makeManager = (
getIsLooping: () => false,
getEffects: () => [],
getEffectChainState: () => null,
+ maxCanvasSinkFrameSize: null,
});
test('plays at a high playback rate without restarting the iterator', async () => {
@@ -196,6 +197,7 @@ test('seek should not cause overlapping block/unblock cycles', async () => {
getIsLooping: () => false,
getEffects: () => [],
getEffectChainState: () => null,
+ maxCanvasSinkFrameSize: null,
});
const nonceManager = makeNonceManager();
@@ -270,6 +272,7 @@ test('rapid sequential seeks should not cause overlapping blocks', async () => {
getIsLooping: () => false,
getEffects: () => [],
getEffectChainState: () => null,
+ maxCanvasSinkFrameSize: null,
});
const nonceManager = makeNonceManager();
@@ -319,6 +322,7 @@ test('redrawCurrentFrame should not create a new video iterator', async () => {
getIsLooping: () => false,
getEffects: () => [],
getEffectChainState: () => null,
+ maxCanvasSinkFrameSize: null,
});
const nonceManager = makeNonceManager();
diff --git a/packages/media/src/test/video-with-no-frames-in-beginning.test.ts b/packages/media/src/test/video-with-no-frames-in-beginning.test.ts
index fad14b6788..40348a8ded 100644
--- a/packages/media/src/test/video-with-no-frames-in-beginning.test.ts
+++ b/packages/media/src/test/video-with-no-frames-in-beginning.test.ts
@@ -46,6 +46,7 @@ test('in preview, should properly buffer and draw frames', async (t) => {
getOnVideoFrameCallback: () => null,
getEffects: () => [],
getEffectChainState: () => null,
+ maxCanvasSinkFrameSize: null,
});
const nonceManager = makeNonceManager();
diff --git a/packages/media/src/video-iterator-manager.ts b/packages/media/src/video-iterator-manager.ts
index f01baac294..74732d8e0b 100644
--- a/packages/media/src/video-iterator-manager.ts
+++ b/packages/media/src/video-iterator-manager.ts
@@ -10,6 +10,7 @@ import type {DelayPlaybackIfNotPremounting} from './delay-playback-if-not-premou
import {roundTo4Digits} from './helpers/round-to-4-digits';
import type {Nonce} from './nonce-manager';
import {makePrewarmedVideoIteratorCache} from './prewarm-iterator-for-looping';
+import type {MaxCanvasSinkFrameSize} from './video/props';
import {
createVideoIterator,
type VideoIterator,
@@ -55,6 +56,7 @@ export const videoIteratorManager = async ({
getIsLooping,
getEffects,
getEffectChainState,
+ maxCanvasSinkFrameSize,
}: {
videoTrack: InputVideoTrack;
delayPlaybackHandleIfNotPremounting: () => DelayPlaybackIfNotPremounting;
@@ -71,6 +73,7 @@ export const videoIteratorManager = async ({
width: number,
height: number,
) => EffectChainState | null;
+ maxCanvasSinkFrameSize: MaxCanvasSinkFrameSize | null;
}) => {
let videoIteratorsCreated = 0;
let videoFrameIterator: VideoIterator | null = null;
@@ -84,15 +87,27 @@ export const videoIteratorManager = async ({
lastDrawnFrame = null;
};
+ const displayWidth = await videoTrack.getDisplayWidth();
+ const displayHeight = await videoTrack.getDisplayHeight();
if (canvas) {
- const displayWidth = await videoTrack.getDisplayWidth();
- const displayHeight = await videoTrack.getDisplayHeight();
if (canvas.width !== displayWidth || canvas.height !== displayHeight) {
canvas.width = displayWidth;
canvas.height = displayHeight;
}
}
+ const maxWidth = maxCanvasSinkFrameSize?.width ?? Infinity;
+ const maxHeight = maxCanvasSinkFrameSize?.height ?? Infinity;
+ // Only pass the side that limits the size, Mediabunny derives the other one
+ // from the aspect ratio. Never upscale. 'fill' stretches over the rounding of
+ // the derived side instead of leaving a semi-transparent edge.
+ const frameSize =
+ maxWidth / displayWidth < Math.min(1, maxHeight / displayHeight)
+ ? {width: maxWidth, fit: 'fill' as const}
+ : maxHeight < displayHeight
+ ? {height: maxHeight, fit: 'fill' as const}
+ : {};
+
const canvasSink = new CanvasSink(videoTrack, {
// Match the preview look-ahead buffer size. CanvasSink may reuse pooled
// canvas objects for later decoded frames, so Remotion copies pixels into
@@ -100,8 +115,12 @@ export const videoIteratorManager = async ({
poolSize: 3,
fit: 'contain',
alpha: true,
+ ...frameSize,
});
+ // Effects expect a source with the size of the output.
+ let upscaledFrame: OffscreenCanvas | null = null;
+
const prewarmedVideoIteratorCache =
makePrewarmedVideoIteratorCache(canvasSink);
@@ -114,9 +133,27 @@ export const videoIteratorManager = async ({
chainState &&
canvas instanceof HTMLCanvasElement
) {
+ let source: CanvasImageSource = frame.canvas;
+ if (
+ frame.canvas.width !== canvas.width ||
+ frame.canvas.height !== canvas.height
+ ) {
+ upscaledFrame ??= new OffscreenCanvas(canvas.width, canvas.height);
+ const upscaledContext = upscaledFrame.getContext('2d')!;
+ upscaledContext.clearRect(0, 0, canvas.width, canvas.height);
+ upscaledContext.drawImage(
+ frame.canvas,
+ 0,
+ 0,
+ canvas.width,
+ canvas.height,
+ );
+ source = upscaledFrame;
+ }
+
await runEffectChain({
state: chainState,
- source: frame.canvas,
+ source,
effects,
output: canvas,
width: canvas.width,
@@ -124,7 +161,7 @@ export const videoIteratorManager = async ({
});
} else {
context.clearRect(0, 0, canvas.width, canvas.height);
- context.drawImage(frame.canvas, 0, 0);
+ context.drawImage(frame.canvas, 0, 0, canvas.width, canvas.height);
}
}
};
diff --git a/packages/media/src/video/props.ts b/packages/media/src/video/props.ts
index a6bab99c55..140fef1d59 100644
--- a/packages/media/src/video/props.ts
+++ b/packages/media/src/video/props.ts
@@ -24,6 +24,10 @@ export type VideoObjectFit =
| 'none'
| 'scale-down';
+export type MaxCanvasSinkFrameSize =
+ | {width: number; height?: number}
+ | {width?: number; height: number};
+
export type FallbackOffthreadVideoProps = {
acceptableTimeShiftInSeconds?: number;
transparent?: boolean;
@@ -80,6 +84,7 @@ type OptionalVideoProps = {
credentials: RequestCredentials | undefined;
requestInit: MediaRequestInit | undefined;
objectFit: VideoObjectFit;
+ maxCanvasSinkFrameSize: MaxCanvasSinkFrameSize | null;
_experimentalInitiallyDrawCachedFrame: boolean;
effects: EffectsProp;
};
diff --git a/packages/media/src/video/video-for-preview.tsx b/packages/media/src/video/video-for-preview.tsx
index 6b7e9b7fda..365e19c66b 100644
--- a/packages/media/src/video/video-for-preview.tsx
+++ b/packages/media/src/video/video-for-preview.tsx
@@ -29,6 +29,7 @@ import type {MediaRequestInit} from '../request-init';
import {useCommonEffects} from '../use-common-effects';
import type {
FallbackOffthreadVideoProps,
+ MaxCanvasSinkFrameSize,
NativeVideoProps,
VideoObjectFit,
} from './props';
@@ -75,6 +76,7 @@ type VideoForPreviewProps = NativeVideoProps & {
readonly credentials: RequestCredentials | undefined;
readonly requestInit: MediaRequestInit | undefined;
readonly objectFit: VideoObjectFit;
+ readonly maxCanvasSinkFrameSize: MaxCanvasSinkFrameSize | null;
readonly setMediaDurationInSeconds: (durationInSeconds: number) => void;
readonly _experimentalInitiallyDrawCachedFrame: boolean;
readonly effects: EffectDefinitionAndStack<unknown>[];
@@ -109,6 +111,7 @@ const VideoForPreviewAssertedShowing: React.FC<
credentials,
requestInit,
objectFit: objectFitProp,
+ maxCanvasSinkFrameSize,
_experimentalInitiallyDrawCachedFrame,
effects,
setMediaDurationInSeconds,
@@ -181,6 +184,9 @@ const VideoForPreviewAssertedShowing: React.FC<
const effectChainStateRef = useRef(effectChainState);
effectChainStateRef.current = effectChainState;
+ const maxCanvasSinkFrameSizeRef = useRef(maxCanvasSinkFrameSize);
+ maxCanvasSinkFrameSizeRef.current = maxCanvasSinkFrameSize;
+
const parentSequence = useContext(SequenceContext);
const isPremounting = Boolean(parentSequence?.premounting);
const isPostmounting = Boolean(parentSequence?.postmounting);
@@ -306,6 +312,7 @@ const VideoForPreviewAssertedShowing: React.FC<
getEffects: () => effectsRef.current,
getEffectChainState: (width, height) =>
effectChainStateRef.current?.get(width, height)!,
+ maxCanvasSinkFrameSize: maxCanvasSinkFrameSizeRef.current,
onError: (error) => {
const [action, errorToUse] = callOnErrorAndResolve({
onError: onErrorRef.current,
@@ -475,6 +482,10 @@ const VideoForPreviewAssertedShowing: React.FC<
credentials,
initialRequestInit,
setMediaDurationInSeconds,
+ // Compare the numbers, so that a new object with the same size does not
+ // recreate the player.
+ maxCanvasSinkFrameSize?.width,
+ maxCanvasSinkFrameSize?.height,
]);
warnAboutObjectFitInStyleOrClassName({style, className, logLevel});
diff --git a/packages/media/src/video/video.tsx b/packages/media/src/video/video.tsx
index bb17a9ed1b..749dacff53 100644
--- a/packages/media/src/video/video.tsx
+++ b/packages/media/src/video/video.tsx
@@ -9,6 +9,7 @@ import {
type SequenceControls,
type InteractivitySchema,
} from 'remotion';
+import {NoReactInternals} from 'remotion/no-react';
import {useLoopedVolume} from '../looped-frame';
import {getLoopDisplay} from '../show-in-timeline';
import {validateToneFrequency} from '../validate-tone-frequency';
@@ -108,6 +109,7 @@ const InnerVideo: React.FC<
requestInit,
controls,
objectFit,
+ maxCanvasSinkFrameSize,
_experimentalInitiallyDrawCachedFrame,
effects,
setMediaDurationInSeconds,
@@ -177,6 +179,22 @@ const InnerVideo: React.FC<
);
}
+ if (maxCanvasSinkFrameSize?.width !== undefined) {
+ NoReactInternals.validateDimension(
+ maxCanvasSinkFrameSize.width,
+ 'maxCanvasSinkFrameSize.width',
+ 'of the <Video /> component',
+ );
+ }
+
+ if (maxCanvasSinkFrameSize?.height !== undefined) {
+ NoReactInternals.validateDimension(
+ maxCanvasSinkFrameSize.height,
+ 'maxCanvasSinkFrameSize.height',
+ 'of the <Video /> component',
+ );
+ }
+
return (
<VideoForPreview
{...props}
@@ -206,6 +224,7 @@ const InnerVideo: React.FC<
requestInit={requestInit}
controls={controls}
objectFit={objectFit}
+ maxCanvasSinkFrameSize={maxCanvasSinkFrameSize}
effects={effects}
_experimentalInitiallyDrawCachedFrame={
_experimentalInitiallyDrawCachedFrame
@@ -246,6 +265,7 @@ const VideoInner: React.FC<
requestInit,
controls,
objectFit,
+ maxCanvasSinkFrameSize,
_experimentalInitiallyDrawCachedFrame,
effects,
durationInFrames,
@@ -440,6 +460,7 @@ const VideoInner: React.FC<
requestInit={requestInit}
controls={controls}
objectFit={objectFit ?? 'contain'}
+ maxCanvasSinkFrameSize={maxCanvasSinkFrameSize ?? null}
_experimentalInitiallyDrawCachedFrame={
_experimentalInitiallyDrawCachedFrame ?? false
}