🎥 Make videos programmatically with React
`@remotion/media`: Add effect output size to `<Video>`
dukeeagle4f975ded0ac1
diff --git a/packages/docs/docs/media/video.mdx b/packages/docs/docs/media/video.mdxindex 8c562bc14b..838fb67c42 100644--- a/packages/docs/docs/media/video.mdx+++ b/packages/docs/docs/media/video.mdx@@ -67,6 +67,32 @@ The URL of the video to be rendered. Can be a remote URL or a local file referenApply [effects](/docs/effects/api) to the video frame after it has been drawn to the canvas.+### `effectsOutputSize?`<AvailableFrom v="4.0.513" />++Sets the width and height of the canvas on which [`effects`](#effects) are evaluated. By default, the encoded video dimensions are used.++Use this prop to keep pixel-based effects consistent when the same video is available at different resolutions, for example when switching between a proxy and an original file.++```tsx twoslash+import {pixelate} from '@remotion/effects/pixelate';+import {Video} from '@remotion/media';+import {staticFile} from 'remotion';++export const MyVideo = () => {+ return <Video src={staticFile('video.webm')} effects={[pixelate({blockSize: 32})]} effectsOutputSize={{width: 1920, height: 1080}} />;+};+```++The values are rounded to the nearest pixel and must be positive and finite. This prop has no effect if `effects` is empty.++Without `effectsOutputSize`, the same effect has a different scale on 512px and 2048px encodes:++<img src="/img/media/video-effects-output-size-before.png" alt="The same pixelate effect has different block sizes on two encodes of the same video" />++Setting `effectsOutputSize` to 1024 × 1024 gives both encodes the same effect scale:++<img src="/img/media/video-effects-output-size-after.png" alt="The same pixelate effect has matching block sizes after setting effectsOutputSize" />+### `cropLeft?`<AvailableFrom v="4.0.500" />Crops the video from the left by a ratio between `0` and `1`. See [`cropLeft`](/docs/sequence#cropleft).diff --git a/packages/docs/static/img/media/video-effects-output-size-after.png b/packages/docs/static/img/media/video-effects-output-size-after.pngnew file mode 100644index 0000000000..17053193b6Binary files /dev/null and b/packages/docs/static/img/media/video-effects-output-size-after.png differdiff --git a/packages/docs/static/img/media/video-effects-output-size-before.png b/packages/docs/static/img/media/video-effects-output-size-before.pngnew file mode 100644index 0000000000..e1b12f62fbBinary files /dev/null and b/packages/docs/static/img/media/video-effects-output-size-before.png differdiff --git a/packages/example/src/EffectsTestbed/VideoEffectsOutputSize.tsx b/packages/example/src/EffectsTestbed/VideoEffectsOutputSize.tsxnew file mode 100644index 0000000000..dea324bdc9--- /dev/null+++ b/packages/example/src/EffectsTestbed/VideoEffectsOutputSize.tsx@@ -0,0 +1,65 @@+import {pixelate} from '@remotion/effects/pixelate';+import {Video} from '@remotion/media';+import React from 'react';+import {AbsoluteFill, staticFile} from 'remotion';++const panelStyle: React.CSSProperties = {+ backgroundColor: '#09090b',+ border: '2px solid #3f3f46',+ borderRadius: 24,+ display: 'flex',+ flex: 1,+ flexDirection: 'column',+ overflow: 'hidden',+};++const labelStyle: React.CSSProperties = {+ color: '#fafafa',+ fontFamily: 'Arial, sans-serif',+ fontSize: 28,+ fontWeight: 600,+ padding: '22px 26px',+};++export const VideoEffectsOutputSize: React.FC<{+ readonly effectsOutputSize?: {width: number; height: number};+}> = ({effectsOutputSize}) => {+ const videos = [+ {label: '512 x 512 source', src: staticFile('blush-0.5x.webm')},+ {label: '2048 x 2048 source', src: staticFile('blush-2x.webm')},+ ];++ return (+ <AbsoluteFill+ style={{+ backgroundColor: '#18181b',+ color: '#fafafa',+ fontFamily: 'Arial, sans-serif',+ padding: 56,+ }}+ >+ <div style={{fontSize: 44, fontWeight: 700}}>+ Same video, same display size, pixelate({`{blockSize: 128}`})+ </div>+ <div style={{color: '#a1a1aa', fontSize: 26, marginTop: 12}}>+ {effectsOutputSize+ ? 'effectsOutputSize: 1024 x 1024'+ : 'Default: effect size follows the encoded source'}+ </div>+ <div style={{display: 'flex', flex: 1, gap: 32, marginTop: 36}}>+ {videos.map((video) => (+ <div key={video.src} style={panelStyle}>+ <div style={labelStyle}>{video.label}</div>+ <Video+ effects={[pixelate({blockSize: 128})]}+ effectsOutputSize={effectsOutputSize}+ muted+ src={video.src}+ style={{height: 650, width: '100%'}}+ />+ </div>+ ))}+ </div>+ </AbsoluteFill>+ );+};diff --git a/packages/example/src/Root.tsx b/packages/example/src/Root.tsxindex 1b5004db46..1d13912185 100644--- a/packages/example/src/Root.tsx+++ b/packages/example/src/Root.tsx@@ -242,6 +242,7 @@ import {PaletteMapEffect} from './EffectsTestbed/PaletteMapEffect';import {RadialProgressiveBlurTest} from './EffectsTestbed/RadialProgressiveBlur';import {TearTest} from './EffectsTestbed/Tear';import {VideoEffectsFastRefresh} from './EffectsTestbed/VideoEffectsFastRefresh';+import {VideoEffectsOutputSize} from './EffectsTestbed/VideoEffectsOutputSize';import {Empty} from './Empty';import {Issue8974IndependentVideosTimeline,@@ -2347,6 +2348,23 @@ export const Index: React.FC = () => {fps={30}durationInFrames={300}/>+ <Composition+ id="video-effects-output-size-before"+ component={VideoEffectsOutputSize}+ width={1920}+ height={1080}+ fps={30}+ durationInFrames={150}+ />+ <Composition+ id="video-effects-output-size-after"+ component={VideoEffectsOutputSize}+ width={1920}+ height={1080}+ fps={30}+ durationInFrames={150}+ defaultProps={{effectsOutputSize: {width: 1024, height: 1024}}}+ /></Folder><Folder name="Premount"><Compositiondiff --git a/packages/media/src/index.ts b/packages/media/src/index.tsindex 842da05b58..e7139631ce 100644--- a/packages/media/src/index.ts+++ b/packages/media/src/index.ts@@ -15,6 +15,7 @@ export {AudioProps, FallbackHtml5AudioProps} from './audio/props';export {MediaErrorAction} from './on-error';export type {MediaRequestInit} from './request-init';export {+ EffectsOutputSize,FallbackOffthreadVideoProps,VideoObjectFit,VideoProps,diff --git a/packages/media/src/media-player.ts b/packages/media/src/media-player.tsindex 615e2560bb..b17ff6a5a7 100644--- a/packages/media/src/media-player.ts+++ b/packages/media/src/media-player.ts@@ -30,6 +30,7 @@ import type {MediaRequestInit} from './request-init';import type {SharedAudioContextForMediaPlayer} from './shared-audio-context-for-media-player';import type {VideoIteratorManager} from './video-iterator-manager';import {videoIteratorManager} from './video-iterator-manager';+import type {EffectsOutputSize} from './video/props';export type MediaPlayerInitResult =| {type: 'success'; durationInSeconds: number}@@ -84,6 +85,8 @@ export class MediaPlayer {height: number,) => EffectChainState | null;+ private getEffectsOutputSize: () => EffectsOutputSize | undefined;+private initializationPromise: Promise<MediaPlayerInitResult> | null = null;private premountAwareDelayPlayback: PremountAwareDelayPlayback;@@ -115,6 +118,7 @@ export class MediaPlayer {tagType,getEffects,getEffectChainState,+ getEffectsOutputSize = () => undefined,}: {canvas: HTMLCanvasElement | OffscreenCanvas | null;src: string;@@ -144,6 +148,7 @@ export class MediaPlayer {width: number,height: number,) => EffectChainState | null;+ getEffectsOutputSize?: () => EffectsOutputSize | undefined;}) {this.canvas = canvas ?? null;this.src = src;@@ -183,6 +188,7 @@ export class MediaPlayer {this.tagType = tagType;this.getEffects = getEffects;this.getEffectChainState = getEffectChainState;+ this.getEffectsOutputSize = getEffectsOutputSize;if (canvas) {const context = canvas.getContext('2d', {@@ -359,6 +365,7 @@ export class MediaPlayer {getIsLooping: () => this.loop,getEffects: this.getEffects,getEffectChainState: this.getEffectChainState,+ getEffectsOutputSize: this.getEffectsOutputSize,});}diff --git a/packages/media/src/test/resolve-effects-output-size.test.ts b/packages/media/src/test/resolve-effects-output-size.test.tsnew file mode 100644index 0000000000..461a03948f--- /dev/null+++ b/packages/media/src/test/resolve-effects-output-size.test.ts@@ -0,0 +1,37 @@+import {expect, test} from 'vitest';+import {resolveEffectsOutputSize} from '../video/resolve-effects-output-size';++test('uses source dimensions by default', () => {+ expect(+ resolveEffectsOutputSize({+ sourceWidth: 1920,+ sourceHeight: 1080,+ effectsOutputSize: undefined,+ }),+ ).toEqual({width: 1920, height: 1080});+});++test('rounds the requested effect output size', () => {+ expect(+ resolveEffectsOutputSize({+ sourceWidth: 512,+ sourceHeight: 512,+ effectsOutputSize: {width: 1024.4, height: 1023.6},+ }),+ ).toEqual({width: 1024, height: 1024});+});++test.each([+ {width: 0, height: 100},+ {width: 100, height: -1},+ {width: Number.NaN, height: 100},+ {width: 100, height: Number.POSITIVE_INFINITY},+])('rejects invalid dimensions: %o', (effectsOutputSize) => {+ expect(() =>+ resolveEffectsOutputSize({+ sourceWidth: 512,+ sourceHeight: 512,+ effectsOutputSize,+ }),+ ).toThrow('width and height must be positive finite numbers');+});diff --git a/packages/media/src/test/video-effects-output-size.test.tsx b/packages/media/src/test/video-effects-output-size.test.tsxnew file mode 100644index 0000000000..8ee276cdf7--- /dev/null+++ b/packages/media/src/test/video-effects-output-size.test.tsx@@ -0,0 +1,94 @@+import {Player} from '@remotion/player';+import React from 'react';+import {createRoot} from 'react-dom/client';+import {Internals} from 'remotion';+import {expect, test} from 'vitest';+import {Video} from '../video/video';++const passthrough = Internals.createEffect<+ Record<string, never>,+ CanvasRenderingContext2D+>({+ type: 'dev.remotion.media.test.passthrough',+ label: 'passthrough()',+ documentationLink: null,+ backend: '2d',+ calculateKey: () => 'passthrough',+ setup: (target) => {+ const context = target.getContext('2d');+ if (!context) {+ throw new Error('Could not get 2D context');+ }++ return context;+ },+ apply: ({source, state, width, height}) => {+ state.clearRect(0, 0, width, height);+ state.drawImage(source, 0, 0, width, height);+ },+ cleanup: () => undefined,+ schema: {},+ validateParams: () => undefined,+});++const waitFor = async (predicate: () => boolean) => {+ const started = Date.now();+ while (Date.now() - started < 10000) {+ if (predicate()) {+ return;+ }++ await new Promise((resolve) => setTimeout(resolve, 10));+ }++ throw new Error('Timed out waiting for condition');+};++test('applies effects at effectsOutputSize in preview', async () => {+ const container = document.createElement('div');+ document.body.appendChild(container);++ const Composition: React.FC = () => {+ return (+ <Video+ data-testid="video"+ effects={[passthrough()]}+ effectsOutputSize={{width: 320, height: 180}}+ muted+ src="/bigbuckbunny.mp4"+ />+ );+ };++ const root = createRoot(container);+ root.render(+ <Player+ acknowledgeRemotionLicense+ component={Composition}+ compositionHeight={720}+ compositionWidth={1280}+ durationInFrames={100}+ fps={30}+ initiallyMuted+ inputProps={{}}+ />,+ );++ try {+ await waitFor(() => {+ const currentCanvas = container.querySelector(+ '[data-testid="video"]',+ ) as HTMLCanvasElement | null;+ return currentCanvas?.width === 320 && currentCanvas.height === 180;+ });++ const canvas = container.querySelector(+ '[data-testid="video"]',+ ) as HTMLCanvasElement;+ expect(canvas.width).toBe(320);+ expect(canvas.height).toBe(180);+ } finally {+ root.unmount();+ container.remove();+ }+});diff --git a/packages/media/src/video-iterator-manager.ts b/packages/media/src/video-iterator-manager.tsindex dd6a681bb0..fd175b1634 100644--- a/packages/media/src/video-iterator-manager.ts+++ b/packages/media/src/video-iterator-manager.ts@@ -10,6 +10,8 @@ import type {DelayPlaybackIfNotPremounting} from './delay-playback-if-not-premouimport {roundTo4Digits} from './helpers/round-to-4-digits';import type {Nonce} from './nonce-manager';import {makePrewarmedVideoIteratorCache} from './prewarm-iterator-for-looping';+import type {EffectsOutputSize} from './video/props';+import {resolveEffectsOutputSize} from './video/resolve-effects-output-size';import {createVideoIterator,type VideoIterator,@@ -54,6 +56,7 @@ export const videoIteratorManager = async ({getIsLooping,getEffects,getEffectChainState,+ getEffectsOutputSize = () => undefined,}: {videoTrack: InputVideoTrack;delayPlaybackHandleIfNotPremounting: () => DelayPlaybackIfNotPremounting;@@ -70,6 +73,7 @@ export const videoIteratorManager = async ({width: number,height: number,) => EffectChainState | null;+ getEffectsOutputSize?: () => EffectsOutputSize | undefined;}) => {let videoIteratorsCreated = 0;let videoFrameIterator: VideoIterator | null = null;@@ -106,7 +110,27 @@ export const videoIteratorManager = async ({const paintFrame = async (frame: WrappedCanvas): Promise<void> => {if (context && canvas) {const effects = getEffects();- const chainState = getEffectChainState(canvas.width, canvas.height);+ const outputSize =+ effects.length > 0+ ? resolveEffectsOutputSize({+ sourceWidth: frame.canvas.width,+ sourceHeight: frame.canvas.height,+ effectsOutputSize: getEffectsOutputSize(),+ })+ : {width: frame.canvas.width, height: frame.canvas.height};++ if (+ canvas.width !== outputSize.width ||+ canvas.height !== outputSize.height+ ) {+ canvas.width = outputSize.width;+ canvas.height = outputSize.height;+ }++ const chainState = getEffectChainState(+ outputSize.width,+ outputSize.height,+ );if (effects.length > 0 &&chainState &&@@ -117,12 +141,18 @@ export const videoIteratorManager = async ({source: frame.canvas,effects,output: canvas,- width: canvas.width,- height: canvas.height,+ width: outputSize.width,+ height: outputSize.height,});} else {context.clearRect(0, 0, canvas.width, canvas.height);- context.drawImage(frame.canvas, 0, 0);+ context.drawImage(+ frame.canvas,+ 0,+ 0,+ outputSize.width,+ outputSize.height,+ );}}};diff --git a/packages/media/src/video/props.ts b/packages/media/src/video/props.tsindex 2ce13c0ceb..bacc82f71b 100644--- a/packages/media/src/video/props.ts+++ b/packages/media/src/video/props.ts@@ -24,6 +24,11 @@ export type VideoObjectFit =| 'none'| 'scale-down';+export type EffectsOutputSize = {+ readonly width: number;+ readonly height: number;+};+export type FallbackOffthreadVideoProps = {acceptableTimeShiftInSeconds?: number;transparent?: boolean;@@ -76,6 +81,7 @@ type OptionalVideoProps = {objectFit: VideoObjectFit;_experimentalInitiallyDrawCachedFrame: boolean;effects: EffectsProp;+ effectsOutputSize: EffectsOutputSize | undefined;};export type NativeVideoProps = Omit<diff --git a/packages/media/src/video/resolve-effects-output-size.ts b/packages/media/src/video/resolve-effects-output-size.tsnew file mode 100644index 0000000000..a650552df0--- /dev/null+++ b/packages/media/src/video/resolve-effects-output-size.ts@@ -0,0 +1,31 @@+import type {EffectsOutputSize} from './props';++export const resolveEffectsOutputSize = ({+ sourceWidth,+ sourceHeight,+ effectsOutputSize,+}: {+ readonly sourceWidth: number;+ readonly sourceHeight: number;+ readonly effectsOutputSize: EffectsOutputSize | undefined;+}): EffectsOutputSize => {+ if (effectsOutputSize === undefined) {+ return {width: sourceWidth, height: sourceHeight};+ }++ const width = Math.round(effectsOutputSize.width);+ const height = Math.round(effectsOutputSize.height);++ if (+ !Number.isFinite(width) ||+ width <= 0 ||+ !Number.isFinite(height) ||+ height <= 0+ ) {+ throw new TypeError(+ '`effectsOutputSize` width and height must be positive finite numbers.',+ );+ }++ return {width, height};+};diff --git a/packages/media/src/video/video-for-preview.tsx b/packages/media/src/video/video-for-preview.tsxindex 44fe8a7bdf..a5f75ad8b9 100644--- a/packages/media/src/video/video-for-preview.tsx+++ b/packages/media/src/video/video-for-preview.tsx@@ -28,6 +28,7 @@ import {ProResDecoderNotEnabledError} from '../prores-error';import type {MediaRequestInit} from '../request-init';import {useCommonEffects} from '../use-common-effects';import type {+ EffectsOutputSize,FallbackOffthreadVideoProps,NativeVideoProps,VideoObjectFit,@@ -78,6 +79,7 @@ type VideoForPreviewProps = NativeVideoProps & {readonly setMediaDurationInSeconds: (durationInSeconds: number) => void;readonly _experimentalInitiallyDrawCachedFrame: boolean;readonly effects: EffectDefinitionAndStack<unknown>[];+ readonly effectsOutputSize: EffectsOutputSize | undefined;readonly refForOutline: React.RefObject<HTMLElement | null>;};@@ -112,6 +114,7 @@ const VideoForPreviewAssertedShowing: React.FC<objectFit: objectFitProp,_experimentalInitiallyDrawCachedFrame,effects,+ effectsOutputSize,setMediaDurationInSeconds,refForOutline,...props@@ -171,6 +174,8 @@ const VideoForPreviewAssertedShowing: React.FC<const effectsRef = useRef(effects);effectsRef.current = effects;+ const effectsOutputSizeRef = useRef(effectsOutputSize);+ effectsOutputSizeRef.current = effectsOutputSize;const onErrorRef = useRef(onError);onErrorRef.current = onError;@@ -298,6 +303,7 @@ const VideoForPreviewAssertedShowing: React.FC<getEffects: () => effectsRef.current,getEffectChainState: (width, height) =>effectChainStateRef.current?.get(width, height)!,+ getEffectsOutputSize: () => effectsOutputSizeRef.current,});mediaPlayerRef.current = player;@@ -514,7 +520,7 @@ const VideoForPreviewAssertedShowing: React.FC<mediaPlayer.redrawVideoEffects().catch(() => {// Player may have been disposed between layout and the async redraw.});- }, [effects, mediaPlayerReady, mediaPlayerRef]);+ }, [effects, effectsOutputSize, mediaPlayerReady, mediaPlayerRef]);const actualStyle: React.CSSProperties = useMemo(() => {return {diff --git a/packages/media/src/video/video-for-rendering.tsx b/packages/media/src/video/video-for-rendering.tsxindex be25b10d83..55a480acc1 100644--- a/packages/media/src/video/video-for-rendering.tsx+++ b/packages/media/src/video/video-for-rendering.tsx@@ -29,10 +29,12 @@ import {ProResDecoderNotEnabledError} from '../prores-error';import type {MediaRequestInit} from '../request-init';import {extractFrameViaBroadcastChannel} from '../video-extraction/extract-frame-via-broadcast-channel';import type {+ EffectsOutputSize,FallbackOffthreadVideoProps,NativeVideoProps,VideoObjectFit,} from './props';+import {resolveEffectsOutputSize} from './resolve-effects-output-size';import {warnAboutObjectFitInStyleOrClassName} from './warn-object-fit-css';type InnerVideoProps = NativeVideoProps & {@@ -61,6 +63,7 @@ type InnerVideoProps = NativeVideoProps & {readonly requestInit: MediaRequestInit | undefined;readonly objectFit: VideoObjectFit;readonly effects: EffectDefinitionAndStack<unknown>[];+ readonly effectsOutputSize: EffectsOutputSize | undefined;};type FallbackToOffthreadVideo = {@@ -93,6 +96,7 @@ export const VideoForRendering: React.FC<InnerVideoProps> = ({requestInit,objectFit: objectFitProp,effects,+ effectsOutputSize,...props}) => {if (!src) {@@ -321,24 +325,39 @@ export const VideoForRendering: React.FC<InnerVideoProps> = ({});// Could be in headless modeif (context) {- context.canvas.width = imageBitmap.width;- context.canvas.height = imageBitmap.height;+ const outputSize =+ effects.length > 0+ ? resolveEffectsOutputSize({+ sourceWidth: imageBitmap.width,+ sourceHeight: imageBitmap.height,+ effectsOutputSize,+ })+ : {width: imageBitmap.width, height: imageBitmap.height};++ context.canvas.width = outputSize.width;+ context.canvas.height = outputSize.height;context.canvas.style.aspectRatio = `${context.canvas.width} / ${context.canvas.height}`;- context.drawImage(imageBitmap, 0, 0);+ context.drawImage(+ imageBitmap,+ 0,+ 0,+ outputSize.width,+ outputSize.height,+ );if (effects.length > 0) {const completed = await Internals.runEffectChain({state: effectChainState.get(- imageBitmap.width,- imageBitmap.height,+ outputSize.width,+ outputSize.height,)!,source: context.canvas,effects,output: context.canvas,- width: imageBitmap.width,- height: imageBitmap.height,+ width: outputSize.width,+ height: outputSize.height,});if (!completed || mediaCache.isDisposed()) {@@ -449,6 +468,7 @@ export const VideoForRendering: React.FC<InnerVideoProps> = ({credentials,effectChainState,effects,+ effectsOutputSize,initialRequestInit,mediaCache,]);diff --git a/packages/media/src/video/video.tsx b/packages/media/src/video/video.tsxindex 9046bf97ff..881cb5eaa4 100644--- a/packages/media/src/video/video.tsx+++ b/packages/media/src/video/video.tsx@@ -106,6 +106,7 @@ const InnerVideo: React.FC<objectFit,_experimentalInitiallyDrawCachedFrame,effects,+ effectsOutputSize,setMediaDurationInSeconds,refForOutline,...props@@ -170,6 +171,7 @@ const InnerVideo: React.FC<requestInit={requestInit}objectFit={objectFit}effects={effects}+ effectsOutputSize={effectsOutputSize}/>);}@@ -204,6 +206,7 @@ const InnerVideo: React.FC<controls={controls}objectFit={objectFit}effects={effects}+ effectsOutputSize={effectsOutputSize}_experimentalInitiallyDrawCachedFrame={_experimentalInitiallyDrawCachedFrame}@@ -246,6 +249,7 @@ const VideoInner: React.FC<objectFit,_experimentalInitiallyDrawCachedFrame,effects,+ effectsOutputSize,durationInFrames,from,freeze,@@ -430,6 +434,7 @@ const VideoInner: React.FC<_experimentalInitiallyDrawCachedFrame ?? false}effects={memoizedEffects}+ effectsOutputSize={effectsOutputSize}setMediaDurationInSeconds={setMediaDurationInSeconds}refForOutline={refForOutline}/>