🎥 Make videos programmatically with React
`@remotion/media`: Add a `maxCanvasSinkFrameSize` prop to `<Video>` for smaller preview frames
Don Carignana128482ade7a
diff --git a/packages/docs/docs/media/video.mdx b/packages/docs/docs/media/video.mdxindex fd8337dfbd..b40f7930de 100644--- a/packages/docs/docs/media/video.mdx+++ b/packages/docs/docs/media/video.mdx@@ -319,6 +319,41 @@ export const MyComposition = () => {The CSS property `object-fit` is not supported.:::+### `maxCanvasSinkFrameSize?`<AvailableFrom v="4.0.530" />++Limits the size of the frames that are drawn during the preview, in pixels. Pass a `width`, a `height` or both. By default, the frames have the size of the video.++The frames keep the aspect ratio of the video, fit within the size and are never upscaled. They get scaled up to the size of the `<canvas>` when drawn, so the layout does not change.++Smaller frames use less memory, which helps when a large video is shown small. The video is still decoded at full size, and rendering is not affected.++For example, use [`useCurrentScale()`](/docs/use-current-scale) to pick a size:++```tsx twoslash title="Smaller frames when the preview is zoomed out"+import {useCurrentScale} from 'remotion';+import {Video} from '@remotion/media';++// ---cut---+export const MyComposition = () => {+ const scale = useCurrentScale();+ const width = scale > 0.5 ? 1920 : scale > 0.25 ? 960 : 480;++ return <Video src="https://remotion.media/video.mp4" maxCanvasSinkFrameSize={{width}} />;+};+```++In the preview, [`onVideoFrame`](#onvideoframe) receives the smaller frames, and [`effects`](#effects) receive them scaled up to the size of the `<canvas>`.++Changing the numbers re-initializes the video, so prefer a few fixed sizes over a value that changes on every resize. Passing a new object with the same numbers does not.++#### `width?`++The maximum width of the frames.++#### `height?`++The maximum height of the frames.+### `loop?`Makes the video loop indefinitely.diff --git a/packages/media/src/audio/audio-for-preview.tsx b/packages/media/src/audio/audio-for-preview.tsxindex cb40ec0e4e..01824c6790 100644--- a/packages/media/src/audio/audio-for-preview.tsx+++ b/packages/media/src/audio/audio-for-preview.tsx@@ -242,6 +242,7 @@ const AudioForPreviewAssertedShowing: React.FC<NewAudioForPreviewProps> = ({tagType: 'audio',getEffects: () => [],getEffectChainState: () => null,+ maxCanvasSinkFrameSize: null,onError: (error) => {const [action, errorToUse] = callOnErrorAndResolve({onError: onErrorRef.current,diff --git a/packages/media/src/index.ts b/packages/media/src/index.tsindex 842da05b58..b36843f889 100644--- a/packages/media/src/index.ts+++ b/packages/media/src/index.ts@@ -16,6 +16,7 @@ export {MediaErrorAction} from './on-error';export type {MediaRequestInit} from './request-init';export {FallbackOffthreadVideoProps,+ MaxCanvasSinkFrameSize,VideoObjectFit,VideoProps,} from './video/props';diff --git a/packages/media/src/media-player.ts b/packages/media/src/media-player.tsindex b998d958d9..1218095cff 100644--- a/packages/media/src/media-player.ts+++ b/packages/media/src/media-player.ts@@ -30,6 +30,7 @@ import type {MediaRequestInit} from './request-init';import type {SharedAudioContextForMediaPlayer} from './shared-audio-context-for-media-player';import type {VideoIteratorManager} from './video-iterator-manager';import {videoIteratorManager} from './video-iterator-manager';+import type {MaxCanvasSinkFrameSize} from './video/props';export type MediaPlayerInitResult =| {type: 'success'; durationInSeconds: number}@@ -94,6 +95,8 @@ export class MediaPlayer {height: number,) => EffectChainState | null;+ private maxCanvasSinkFrameSize: MaxCanvasSinkFrameSize | null;+private initializationPromise: Promise<MediaPlayerInitResult> | null = null;private premountAwareDelayPlayback: PremountAwareDelayPlayback;@@ -127,6 +130,7 @@ export class MediaPlayer {tagType,getEffects,getEffectChainState,+ maxCanvasSinkFrameSize,onError,}: {canvas: HTMLCanvasElement | OffscreenCanvas | null;@@ -157,6 +161,7 @@ export class MediaPlayer {width: number,height: number,) => EffectChainState | null;+ maxCanvasSinkFrameSize: MaxCanvasSinkFrameSize | null;onError: ((error: Error) => void) | null;}) {this.canvas = canvas ?? null;@@ -197,6 +202,7 @@ export class MediaPlayer {this.tagType = tagType;this.getEffects = getEffects;this.getEffectChainState = getEffectChainState;+ this.maxCanvasSinkFrameSize = maxCanvasSinkFrameSize;this.onError = onError;if (canvas) {@@ -386,6 +392,7 @@ export class MediaPlayer {getIsLooping: () => this.loop,getEffects: this.getEffects,getEffectChainState: this.getEffectChainState,+ maxCanvasSinkFrameSize: this.maxCanvasSinkFrameSize,});}diff --git a/packages/media/src/test/media-player.test.ts b/packages/media/src/test/media-player.test.tsindex 1060f54d85..91724f2ff4 100644--- a/packages/media/src/test/media-player.test.ts+++ b/packages/media/src/test/media-player.test.ts@@ -65,6 +65,7 @@ const makeAudioPlayer = (tagType,getEffects: () => [],getEffectChainState: () => null,+ maxCanvasSinkFrameSize: null,onError,});};@@ -195,6 +196,7 @@ test('dispose should immediately unblock a pending playback catch-up', async ()tagType: 'video',getEffects: () => [],getEffectChainState: () => null,+ maxCanvasSinkFrameSize: null,onError: null,});diff --git a/packages/media/src/test/premount-arm-kicks-scheduler.test.ts b/packages/media/src/test/premount-arm-kicks-scheduler.test.tsindex 0fe259e9f0..ce078792f3 100644--- a/packages/media/src/test/premount-arm-kicks-scheduler.test.ts+++ b/packages/media/src/test/premount-arm-kicks-scheduler.test.ts@@ -38,6 +38,7 @@ const makePlayer = ({tagType: 'audio',getEffects: () => [],getEffectChainState: () => null,+ maxCanvasSinkFrameSize: null,onError: null,});};diff --git a/packages/media/src/test/trim-change-seek.test.ts b/packages/media/src/test/trim-change-seek.test.tsindex 1f1822ba89..83bfc359d5 100644--- a/packages/media/src/test/trim-change-seek.test.ts+++ b/packages/media/src/test/trim-change-seek.test.ts@@ -28,6 +28,7 @@ test('setTrimRange should atomically update trims when paused', async () => {tagType: 'video',getEffects: () => [],getEffectChainState: () => null,+ maxCanvasSinkFrameSize: null,onError: null,});diff --git a/packages/media/src/test/video-iterator-manager.test.ts b/packages/media/src/test/video-iterator-manager.test.tsindex a333c792c1..7939873dd9 100644--- a/packages/media/src/test/video-iterator-manager.test.ts+++ b/packages/media/src/test/video-iterator-manager.test.ts@@ -98,6 +98,7 @@ const makeManager = (getIsLooping: () => false,getEffects: () => [],getEffectChainState: () => null,+ maxCanvasSinkFrameSize: null,});test('plays at a high playback rate without restarting the iterator', async () => {@@ -196,6 +197,7 @@ test('seek should not cause overlapping block/unblock cycles', async () => {getIsLooping: () => false,getEffects: () => [],getEffectChainState: () => null,+ maxCanvasSinkFrameSize: null,});const nonceManager = makeNonceManager();@@ -270,6 +272,7 @@ test('rapid sequential seeks should not cause overlapping blocks', async () => {getIsLooping: () => false,getEffects: () => [],getEffectChainState: () => null,+ maxCanvasSinkFrameSize: null,});const nonceManager = makeNonceManager();@@ -319,6 +322,7 @@ test('redrawCurrentFrame should not create a new video iterator', async () => {getIsLooping: () => false,getEffects: () => [],getEffectChainState: () => null,+ maxCanvasSinkFrameSize: null,});const nonceManager = makeNonceManager();diff --git a/packages/media/src/test/video-with-no-frames-in-beginning.test.ts b/packages/media/src/test/video-with-no-frames-in-beginning.test.tsindex fad14b6788..40348a8ded 100644--- a/packages/media/src/test/video-with-no-frames-in-beginning.test.ts+++ b/packages/media/src/test/video-with-no-frames-in-beginning.test.ts@@ -46,6 +46,7 @@ test('in preview, should properly buffer and draw frames', async (t) => {getOnVideoFrameCallback: () => null,getEffects: () => [],getEffectChainState: () => null,+ maxCanvasSinkFrameSize: null,});const nonceManager = makeNonceManager();diff --git a/packages/media/src/video-iterator-manager.ts b/packages/media/src/video-iterator-manager.tsindex f01baac294..74732d8e0b 100644--- a/packages/media/src/video-iterator-manager.ts+++ b/packages/media/src/video-iterator-manager.ts@@ -10,6 +10,7 @@ import type {DelayPlaybackIfNotPremounting} from './delay-playback-if-not-premouimport {roundTo4Digits} from './helpers/round-to-4-digits';import type {Nonce} from './nonce-manager';import {makePrewarmedVideoIteratorCache} from './prewarm-iterator-for-looping';+import type {MaxCanvasSinkFrameSize} from './video/props';import {createVideoIterator,type VideoIterator,@@ -55,6 +56,7 @@ export const videoIteratorManager = async ({getIsLooping,getEffects,getEffectChainState,+ maxCanvasSinkFrameSize,}: {videoTrack: InputVideoTrack;delayPlaybackHandleIfNotPremounting: () => DelayPlaybackIfNotPremounting;@@ -71,6 +73,7 @@ export const videoIteratorManager = async ({width: number,height: number,) => EffectChainState | null;+ maxCanvasSinkFrameSize: MaxCanvasSinkFrameSize | null;}) => {let videoIteratorsCreated = 0;let videoFrameIterator: VideoIterator | null = null;@@ -84,15 +87,27 @@ export const videoIteratorManager = async ({lastDrawnFrame = null;};+ const displayWidth = await videoTrack.getDisplayWidth();+ const displayHeight = await videoTrack.getDisplayHeight();if (canvas) {- const displayWidth = await videoTrack.getDisplayWidth();- const displayHeight = await videoTrack.getDisplayHeight();if (canvas.width !== displayWidth || canvas.height !== displayHeight) {canvas.width = displayWidth;canvas.height = displayHeight;}}+ const maxWidth = maxCanvasSinkFrameSize?.width ?? Infinity;+ const maxHeight = maxCanvasSinkFrameSize?.height ?? Infinity;+ // Only pass the side that limits the size, Mediabunny derives the other one+ // from the aspect ratio. Never upscale. 'fill' stretches over the rounding of+ // the derived side instead of leaving a semi-transparent edge.+ const frameSize =+ maxWidth / displayWidth < Math.min(1, maxHeight / displayHeight)+ ? {width: maxWidth, fit: 'fill' as const}+ : maxHeight < displayHeight+ ? {height: maxHeight, fit: 'fill' as const}+ : {};+const canvasSink = new CanvasSink(videoTrack, {// Match the preview look-ahead buffer size. CanvasSink may reuse pooled// canvas objects for later decoded frames, so Remotion copies pixels into@@ -100,8 +115,12 @@ export const videoIteratorManager = async ({poolSize: 3,fit: 'contain',alpha: true,+ ...frameSize,});+ // Effects expect a source with the size of the output.+ let upscaledFrame: OffscreenCanvas | null = null;+const prewarmedVideoIteratorCache =makePrewarmedVideoIteratorCache(canvasSink);@@ -114,9 +133,27 @@ export const videoIteratorManager = async ({chainState &&canvas instanceof HTMLCanvasElement) {+ let source: CanvasImageSource = frame.canvas;+ if (+ frame.canvas.width !== canvas.width ||+ frame.canvas.height !== canvas.height+ ) {+ upscaledFrame ??= new OffscreenCanvas(canvas.width, canvas.height);+ const upscaledContext = upscaledFrame.getContext('2d')!;+ upscaledContext.clearRect(0, 0, canvas.width, canvas.height);+ upscaledContext.drawImage(+ frame.canvas,+ 0,+ 0,+ canvas.width,+ canvas.height,+ );+ source = upscaledFrame;+ }+await runEffectChain({state: chainState,- source: frame.canvas,+ source,effects,output: canvas,width: canvas.width,@@ -124,7 +161,7 @@ export const videoIteratorManager = async ({});} else {context.clearRect(0, 0, canvas.width, canvas.height);- context.drawImage(frame.canvas, 0, 0);+ context.drawImage(frame.canvas, 0, 0, canvas.width, canvas.height);}}};diff --git a/packages/media/src/video/props.ts b/packages/media/src/video/props.tsindex a6bab99c55..140fef1d59 100644--- a/packages/media/src/video/props.ts+++ b/packages/media/src/video/props.ts@@ -24,6 +24,10 @@ export type VideoObjectFit =| 'none'| 'scale-down';+export type MaxCanvasSinkFrameSize =+ | {width: number; height?: number}+ | {width?: number; height: number};+export type FallbackOffthreadVideoProps = {acceptableTimeShiftInSeconds?: number;transparent?: boolean;@@ -80,6 +84,7 @@ type OptionalVideoProps = {credentials: RequestCredentials | undefined;requestInit: MediaRequestInit | undefined;objectFit: VideoObjectFit;+ maxCanvasSinkFrameSize: MaxCanvasSinkFrameSize | null;_experimentalInitiallyDrawCachedFrame: boolean;effects: EffectsProp;};diff --git a/packages/media/src/video/video-for-preview.tsx b/packages/media/src/video/video-for-preview.tsxindex 6b7e9b7fda..365e19c66b 100644--- a/packages/media/src/video/video-for-preview.tsx+++ b/packages/media/src/video/video-for-preview.tsx@@ -29,6 +29,7 @@ import type {MediaRequestInit} from '../request-init';import {useCommonEffects} from '../use-common-effects';import type {FallbackOffthreadVideoProps,+ MaxCanvasSinkFrameSize,NativeVideoProps,VideoObjectFit,} from './props';@@ -75,6 +76,7 @@ type VideoForPreviewProps = NativeVideoProps & {readonly credentials: RequestCredentials | undefined;readonly requestInit: MediaRequestInit | undefined;readonly objectFit: VideoObjectFit;+ readonly maxCanvasSinkFrameSize: MaxCanvasSinkFrameSize | null;readonly setMediaDurationInSeconds: (durationInSeconds: number) => void;readonly _experimentalInitiallyDrawCachedFrame: boolean;readonly effects: EffectDefinitionAndStack<unknown>[];@@ -109,6 +111,7 @@ const VideoForPreviewAssertedShowing: React.FC<credentials,requestInit,objectFit: objectFitProp,+ maxCanvasSinkFrameSize,_experimentalInitiallyDrawCachedFrame,effects,setMediaDurationInSeconds,@@ -181,6 +184,9 @@ const VideoForPreviewAssertedShowing: React.FC<const effectChainStateRef = useRef(effectChainState);effectChainStateRef.current = effectChainState;+ const maxCanvasSinkFrameSizeRef = useRef(maxCanvasSinkFrameSize);+ maxCanvasSinkFrameSizeRef.current = maxCanvasSinkFrameSize;+const parentSequence = useContext(SequenceContext);const isPremounting = Boolean(parentSequence?.premounting);const isPostmounting = Boolean(parentSequence?.postmounting);@@ -306,6 +312,7 @@ const VideoForPreviewAssertedShowing: React.FC<getEffects: () => effectsRef.current,getEffectChainState: (width, height) =>effectChainStateRef.current?.get(width, height)!,+ maxCanvasSinkFrameSize: maxCanvasSinkFrameSizeRef.current,onError: (error) => {const [action, errorToUse] = callOnErrorAndResolve({onError: onErrorRef.current,@@ -475,6 +482,10 @@ const VideoForPreviewAssertedShowing: React.FC<credentials,initialRequestInit,setMediaDurationInSeconds,+ // Compare the numbers, so that a new object with the same size does not+ // recreate the player.+ maxCanvasSinkFrameSize?.width,+ maxCanvasSinkFrameSize?.height,]);warnAboutObjectFitInStyleOrClassName({style, className, logLevel});diff --git a/packages/media/src/video/video.tsx b/packages/media/src/video/video.tsxindex bb17a9ed1b..749dacff53 100644--- a/packages/media/src/video/video.tsx+++ b/packages/media/src/video/video.tsx@@ -9,6 +9,7 @@ import {type SequenceControls,type InteractivitySchema,} from 'remotion';+import {NoReactInternals} from 'remotion/no-react';import {useLoopedVolume} from '../looped-frame';import {getLoopDisplay} from '../show-in-timeline';import {validateToneFrequency} from '../validate-tone-frequency';@@ -108,6 +109,7 @@ const InnerVideo: React.FC<requestInit,controls,objectFit,+ maxCanvasSinkFrameSize,_experimentalInitiallyDrawCachedFrame,effects,setMediaDurationInSeconds,@@ -177,6 +179,22 @@ const InnerVideo: React.FC<);}+ if (maxCanvasSinkFrameSize?.width !== undefined) {+ NoReactInternals.validateDimension(+ maxCanvasSinkFrameSize.width,+ 'maxCanvasSinkFrameSize.width',+ 'of the <Video /> component',+ );+ }++ if (maxCanvasSinkFrameSize?.height !== undefined) {+ NoReactInternals.validateDimension(+ maxCanvasSinkFrameSize.height,+ 'maxCanvasSinkFrameSize.height',+ 'of the <Video /> component',+ );+ }+return (<VideoForPreview{...props}@@ -206,6 +224,7 @@ const InnerVideo: React.FC<requestInit={requestInit}controls={controls}objectFit={objectFit}+ maxCanvasSinkFrameSize={maxCanvasSinkFrameSize}effects={effects}_experimentalInitiallyDrawCachedFrame={_experimentalInitiallyDrawCachedFrame@@ -246,6 +265,7 @@ const VideoInner: React.FC<requestInit,controls,objectFit,+ maxCanvasSinkFrameSize,_experimentalInitiallyDrawCachedFrame,effects,durationInFrames,@@ -440,6 +460,7 @@ const VideoInner: React.FC<requestInit={requestInit}controls={controls}objectFit={objectFit ?? 'contain'}+ maxCanvasSinkFrameSize={maxCanvasSinkFrameSize ?? null}_experimentalInitiallyDrawCachedFrame={_experimentalInitiallyDrawCachedFrame ?? false}