Skip to content

🎥 Make videos programmatically with React

18.7 KB
Raw Blame
import React, {
	useContext,
	useLayoutEffect,
	useMemo,
	useRef,
	useState,
} from 'react';
import type {IsExact} from './audio/props.js';
import type {ImageFit} from './calculate-image-fit.js';
import {CanvasImage} from './canvas-image/index.js';
import type {
	CanvasImageCanvasProps,
	CanvasImageProps,
} from './canvas-image/props.js';
import type {SequenceControls} from './CompositionManager.js';
import type {EffectsProp, EffectsOutputSize} from './effects/effect-types.js';
import {addSequenceStackTraces} from './enable-sequence-stack-traces.js';
import {Freeze} from './freeze.js';
import {getCrossOriginValue} from './get-cross-origin-value.js';
import type {
	InteractiveBaseProps,
	InteractiveCropProps,
	InteractivePremountProps,
} from './Interactive.js';
import {
	backgroundSchema,
	baseSchemaWithoutPlaybackRate,
	borderRadiusSchema,
	borderSchema,
	cropSchema,
	premountSchema,
	transformSchema,
	type InteractivitySchema,
} from './interactivity-schema.js';
import {usePreload} from './prefetch.js';
import {resolveSequenceDuration} from './resolve-sequence-duration.js';
import {SequenceContent} from './sequence-activity-context.js';
import {Sequence} from './Sequence.js';
import {SequenceContext} from './SequenceContext.js';
import {truncateSrcForLabel} from './truncate-src-for-label.js';
import {useBufferState} from './use-buffer-state.js';
import {useCropStyle} from './use-crop-style.js';
import {useDelayRender} from './use-delay-render.js';
import {usePremounting} from './use-premounting.js';
import {useRemotionEnvironment} from './use-remotion-environment.js';
import {resolveV5Default} from './v5-flag.js';
import {withInteractivitySchema} from './with-interactivity-schema.js';

function exponentialBackoff(errorCount: number): number {
	return 1000 * 2 ** (errorCount - 1);
}

type NativeImgProps = Omit<
	React.DetailedHTMLProps<
		React.ImgHTMLAttributes<HTMLImageElement>,
		HTMLImageElement
	>,
	'src'
>;

export type ImgProps = NativeImgProps & {
	readonly maxRetries?: number;
	readonly pauseWhenLoading?: boolean;
	readonly delayRenderRetries?: number;
	readonly delayRenderTimeoutInMilliseconds?: number;
	readonly onImageFrame?: (imageElement: HTMLImageElement) => void;
	readonly onImageError?: (error: Error) => void;
	readonly src: string;
	readonly effects?: EffectsProp;
	readonly effectsOutputSize?: EffectsOutputSize;
	readonly showInTimeline?: boolean;
	readonly name?: string;
} & Omit<InteractiveBaseProps, 'playbackRate'> &
	InteractiveCropProps &
	InteractivePremountProps;

type Expected = Omit<
	NativeImgProps,
	'onError' | 'src' | 'crossOrigin' | 'ref' | 'hidden'
>;

type ImgContentProps = Omit<
	ImgProps,
	| 'hidden'
	| 'name'
	| 'showInTimeline'
	| 'from'
	| 'trimBefore'
	| 'loop'
	| 'durationInFrames'
	| 'freeze'
	| 'effects'
	| 'effectsOutputSize'
	| 'premountFor'
	| 'postmountFor'
	| 'styleWhilePremounted'
	| 'styleWhilePostmounted'
	| 'cropLeft'
	| 'cropRight'
	| 'cropTop'
	| 'cropBottom'
>;

// Bind only the inputs each persistent callback needs. A render-scope closure
// can otherwise keep replaced handlers alive, even when it never calls them.
const setImageRef = (
	imageRef: React.RefObject<HTMLImageElement | null>,
	ref: React.Ref<HTMLImageElement> | null,
	img: HTMLImageElement | null,
) => {
	imageRef.current = img;
	if (typeof ref === 'function') {
		ref(img);
	} else if (ref) {
		ref.current = img;
	}
};

const retryImage = (
	imageRef: React.RefObject<HTMLImageElement | null>,
	timeout: number,
) => {
	if (!imageRef.current) {
		return;
	}

	const currentSrc = imageRef.current.src;

	setTimeout(() => {
		if (!imageRef.current) {
			// Component has been unmounted, do not retry
			return;
		}

		const newSrc = imageRef.current?.src;
		if (newSrc !== currentSrc) {
			// src has changed, do not retry
			return;
		}

		imageRef.current.removeAttribute('src');
		imageRef.current.setAttribute('src', newSrc);
	}, timeout);
};

const handleImageError = (
	{
		errors,
		imageRef,
		onError,
		onImageError,
		maxRetries,
		cancelRender,
		retryIn,
	}: {
		errors: React.RefObject<Record<string, number>>;
		imageRef: React.RefObject<HTMLImageElement | null>;
		onError: NonNullable<ImgProps['onError']> | null;
		onImageError: NonNullable<ImgProps['onImageError']> | null;
		maxRetries: number;
		cancelRender: ReturnType<typeof useDelayRender>['cancelRender'];
		retryIn: (timeout: number) => void;
	},
	e: React.SyntheticEvent<HTMLImageElement, Event>,
) => {
	if (!errors.current) {
		return;
	}

	errors.current[imageRef.current?.src as string] =
		(errors.current[imageRef.current?.src as string] ?? 0) + 1;
	if (
		(onError || onImageError) &&
		(errors.current[imageRef.current?.src as string] ?? 0) > maxRetries
	) {
		onError?.(e);
		onImageError?.(
			new Error(
				'Error loading image with src: ' +
					truncateSrcForLabel(imageRef.current?.src as string),
			),
		);
		return;
	}

	if ((errors.current[imageRef.current?.src as string] ?? 0) <= maxRetries) {
		const backoff = exponentialBackoff(
			errors.current[imageRef.current?.src as string] ?? 0,
		);
		// eslint-disable-next-line no-console
		console.warn(
			`Could not load image with source ${truncateSrcForLabel(
				imageRef.current?.src as string,
			)}, retrying again in ${backoff}ms`,
		);

		retryIn(backoff);
		return;
	}

	try {
		cancelRender(
			'Error loading image with src: ' +
				truncateSrcForLabel(imageRef.current?.src as string),
		);
	} catch {
		// cancelRender() intentionally throws after storing the error in scope.
		// In async image callbacks, we rely on the stored error for renderer propagation.
	}
};

// Keep the loading cleanup outside the render scope too: it can outlive many
// handler replacements while the source and onImageFrame stay unchanged.
const loadImage = ({
	imageRef,
	errors,
	actualSrc,
	delayRenderRetries,
	delayRenderTimeoutInMilliseconds,
	onImageFrame,
	delayRender,
	continueRender,
	setIsLoading,
}: {
	imageRef: React.RefObject<HTMLImageElement | null>;
	errors: React.RefObject<Record<string, number>>;
	actualSrc: string;
	delayRenderRetries: number | null;
	delayRenderTimeoutInMilliseconds: number | null;
	onImageFrame: NonNullable<ImgProps['onImageFrame']> | null;
	delayRender: ReturnType<typeof useDelayRender>['delayRender'];
	continueRender: ReturnType<typeof useDelayRender>['continueRender'];
	setIsLoading: React.Dispatch<React.SetStateAction<boolean>>;
}) => {
	if (window.process?.env?.NODE_ENV === 'test') {
		if (imageRef.current) {
			imageRef.current.src = actualSrc;
		}

		return;
	}

	const {current} = imageRef;
	if (!current) {
		return;
	}

	setIsLoading(true);
	const newHandle = delayRender(
		'Loading <Img> with src=' + truncateSrcForLabel(actualSrc),
		{
			retries: delayRenderRetries ?? undefined,
			timeoutInMilliseconds: delayRenderTimeoutInMilliseconds ?? undefined,
		},
	);

	let unmounted = false;

	const onComplete = () => {
		// the decode() promise isn't cancelable -- it may still resolve after unmounting
		if (unmounted) {
			continueRender(newHandle);
			return;
		}

		if ((errors.current[imageRef.current?.src as string] ?? 0) > 0) {
			delete errors.current[imageRef.current?.src as string];
			// eslint-disable-next-line no-console
			console.info(
				`Retry successful - ${truncateSrcForLabel(
					imageRef.current?.src as string,
				)} is now loaded`,
			);
		}

		if (current) {
			onImageFrame?.(current);
		}

		setIsLoading(false);
		continueRender(newHandle);
	};

	if (!imageRef.current) {
		onComplete();
		return;
	}

	current.src = actualSrc;
	current
		.decode()
		.then(onComplete)
		.catch((err) => {
			// fall back to onload event if decode() fails
			// eslint-disable-next-line no-console
			console.warn(err);

			// HTMLImageElement.complete is also true for broken images (e.g. 404),
			// so only treat it as loaded if intrinsic dimensions are available.
			if (
				current.complete &&
				current.naturalWidth > 0 &&
				current.naturalHeight > 0
			) {
				onComplete();
			} else {
				current.addEventListener('load', onComplete);
			}
		});

	// If tag gets unmounted, clear pending handles because image is not going to load
	return () => {
		unmounted = true;
		current.removeEventListener('load', onComplete);
		continueRender(newHandle);
	};
};

const ImgContent: React.FC<ImgContentProps> = ({
	onError,
	onImageError,
	maxRetries = 2,
	src,
	pauseWhenLoading,
	delayRenderRetries,
	delayRenderTimeoutInMilliseconds,
	onImageFrame,
	crossOrigin,
	decoding,
	ref,
	...props
}) => {
	const imageRef = useRef<HTMLImageElement>(null);
	const errors = useRef<Record<string, number>>({});
	const {delayPlayback} = useBufferState();
	const sequenceContext = useContext(SequenceContext);
	const [isLoading, setIsLoading] = useState(false);

	const _propsValid: IsExact<typeof props, Omit<Expected, 'decoding'>> = true;

	if (!_propsValid) {
		throw new Error('typecheck error');
	}

	const imageCallbackRef = useMemo(
		() => setImageRef.bind(null, imageRef, ref ?? null),
		[ref],
	);

	const actualSrc = usePreload(src as string);

	const retryIn = useMemo(() => retryImage.bind(null, imageRef), []);

	const {delayRender, continueRender, cancelRender} = useDelayRender();
	const isPremounting = Boolean(sequenceContext?.premounting);
	const isPostmounting = Boolean(sequenceContext?.postmounting);

	const didGetError = useMemo(
		() =>
			handleImageError.bind(null, {
				errors,
				imageRef,
				onError: onError ?? null,
				onImageError: onImageError ?? null,
				maxRetries,
				cancelRender,
				retryIn,
			}),
		[cancelRender, maxRetries, onError, onImageError, retryIn],
	);

	if (typeof window !== 'undefined') {
		// eslint-disable-next-line react-hooks/rules-of-hooks
		useLayoutEffect(() => {
			if (!pauseWhenLoading || !isLoading || isPremounting || isPostmounting) {
				return;
			}

			return delayPlayback().unblock;
		}, [
			delayPlayback,
			isLoading,
			isPostmounting,
			isPremounting,
			pauseWhenLoading,
		]);

		// Binding prevents production minification from inlining this scope
		// back into the component and retaining unrelated handlers.
		// eslint-disable-next-line react-hooks/rules-of-hooks, react-hooks/exhaustive-deps
		useLayoutEffect(
			loadImage.bind(null, {
				imageRef,
				errors,
				actualSrc,
				delayRenderRetries: delayRenderRetries ?? null,
				delayRenderTimeoutInMilliseconds:
					delayRenderTimeoutInMilliseconds ?? null,
				onImageFrame: onImageFrame ?? null,
				delayRender,
				continueRender,
				setIsLoading,
			}),
			[
				actualSrc,
				delayRenderRetries,
				delayRenderTimeoutInMilliseconds,
				onImageFrame,
				continueRender,
				delayRender,
			],
		);
	}

	const {isClientSideRendering, isRendering} = useRemotionEnvironment();

	const crossOriginValue = getCrossOriginValue({
		crossOrigin,
		requestsVideoFrame: false,
		isClientSideRendering,
	});

	// `src` is assigned imperatively to this element in the layout effect above.
	// The element may paint while `decode()` is pending; `delayRender()` only
	// blocks frame rendering.
	return (
		<img
			{...props}
			ref={imageCallbackRef}
			crossOrigin={crossOriginValue}
			onError={didGetError}
			decoding={isRendering ? 'sync' : decoding}
		/>
	);
};

type NativeImgInnerProps = Omit<ImgProps, 'effects' | 'effectsOutputSize'> & {
	readonly controls: SequenceControls | undefined;
};

const NativeImgInner: React.FC<NativeImgInnerProps> = ({
	hidden,
	name,
	showInTimeline,
	src,
	from,
	trimBefore,
	loop,
	durationInFrames,
	freeze,
	premountFor,
	postmountFor,
	style,
	styleWhilePremounted,
	styleWhilePostmounted,
	cropLeft,
	cropRight,
	cropTop,
	cropBottom,
	controls,
	...props
}) => {
	if (!src) {
		throw new Error('No "src" prop was passed to <Img>.');
	}

	const isMedia = useMemo(
		() => ({
			type: 'image' as const,
			src,
		}),
		[src],
	);

	const {
		effectivePostmountFor,
		effectivePremountFor,
		freezeFrame,
		isPremountingOrPostmounting,
		postmountingActive,
		premountingActive,
		premountingStyle,
	} = usePremounting({
		from: from ?? 0,
		durationInFrames: resolveSequenceDuration({
			durationInFrames,
			playbackRate: undefined,
			loop,
		}),
		premountFor: premountFor ?? null,
		postmountFor: postmountFor ?? null,
		style: style ?? null,
		styleWhilePremounted: styleWhilePremounted ?? null,
		styleWhilePostmounted: styleWhilePostmounted ?? null,
		hideWhilePremounted: 'display-none',
	});
	const croppedStyle = useCropStyle({
		cropLeft,
		cropRight,
		cropTop,
		cropBottom,
		style: premountingStyle,
		componentName: '<Img />',
	});

	return (
		<Freeze
			frame={freezeFrame}
			active={isPremountingOrPostmounting}
			_remotionInternalIsPremounting={premountingActive}
		>
			<Sequence
				layout="none"
				from={from ?? 0}
				trimBefore={trimBefore}
				loop={loop}
				durationInFrames={durationInFrames}
				freeze={freeze}
				_remotionInternalDocumentationLink="https://www.remotion.dev/docs/img"
				_remotionInternalIsMedia={isMedia}
				_remotionInternalPremountDisplay={effectivePremountFor || null}
				_remotionInternalPostmountDisplay={effectivePostmountFor || null}
				_remotionInternalIsPremounting={premountingActive}
				_remotionInternalIsPostmounting={postmountingActive}
				name={name ?? '<Img>'}
				controls={controls}
				showInTimeline={showInTimeline ?? true}
				hidden={hidden}
			>
				<SequenceContent>
					<ImgContent src={src} style={croppedStyle ?? undefined} {...props} />
				</SequenceContent>
			</Sequence>
		</Freeze>
	);
};

const CanvasImageWithPrivateProps = CanvasImage as React.ComponentType<
	CanvasImageProps & {
		readonly controls?: SequenceControls | undefined;
	}
>;

export const imgSchema = {
	src: {
		type: 'asset',
		assetType: 'image',
		default: undefined,
		description: 'Source',
		keyframable: false,
	},
	...baseSchemaWithoutPlaybackRate,
	...cropSchema,
	...premountSchema,
	...transformSchema,
	...backgroundSchema,
	...borderSchema,
	...borderRadiusSchema,
} as const satisfies InteractivitySchema;

const imgCanvasFallbackIncompatibleProps = new Set([
	'alt',
	'decoding',
	'fetchPriority',
	'loading',
	'onError',
	'onImageFrame',
	'onLoad',
	'sizes',
	'srcSet',
	'useMap',
]);

const getIncompatiblePropNames = (props: Record<string, unknown>) =>
	Object.keys(props).filter(
		(key) =>
			props[key] !== undefined && imgCanvasFallbackIncompatibleProps.has(key),
	);

const formatPropList = (props: string[]) => {
	return props.map((prop) => `"${prop}"`).join(', ');
};

const validateCanvasImageFallbackProps = ({
	props,
	ref,
	width,
	height,
}: {
	readonly props: Record<string, unknown>;
	readonly ref: React.Ref<HTMLImageElement> | undefined;
	readonly width: ImgProps['width'];
	readonly height: ImgProps['height'];
}) => {
	if (typeof width === 'string' || typeof height === 'string') {
		throw new Error(
			'The "width" and "height" props must be numbers on <Img> when effects are passed, because <Img> renders a <CanvasImage>. Use numeric props or CSS dimensions in "style".',
		);
	}

	const conflictingProps = getIncompatiblePropNames(props);
	if (ref !== null && ref !== undefined) {
		conflictingProps.unshift('ref');
	}

	if (conflictingProps.length === 0) {
		return;
	}

	throw new Error(
		`The ${formatPropList(conflictingProps)} prop${
			conflictingProps.length === 1 ? '' : 's'
		} cannot be used on <Img> when effects are passed, because <Img> renders a <canvas> instead of a native <img>. Remove ${
			conflictingProps.length === 1 ? 'this prop' : 'these props'
		}.`,
	);
};

const getFitFromObjectFit = (
	style: React.CSSProperties | undefined,
): ImageFit | undefined => {
	const objectFit = style?.objectFit;

	if (
		objectFit === 'fill' ||
		objectFit === 'contain' ||
		objectFit === 'cover'
	) {
		return objectFit;
	}

	return undefined;
};

const ImgInner: React.FC<
	ImgProps & {
		readonly controls: SequenceControls | undefined;
	}
> = ({
	effects = [],
	effectsOutputSize,
	ref,
	hidden,
	name,
	showInTimeline,
	src,
	from,
	trimBefore,
	durationInFrames,
	freeze,
	premountFor,
	postmountFor,
	styleWhilePremounted,
	styleWhilePostmounted,
	controls,
	width,
	height,
	className,
	style,
	cropLeft,
	cropRight,
	cropTop,
	cropBottom,
	id,
	pauseWhenLoading,
	maxRetries,
	delayRenderRetries,
	delayRenderTimeoutInMilliseconds,
	onImageError,
	...props
}) => {
	const shouldPauseWhenLoading = resolveV5Default(pauseWhenLoading);

	if (effects.length === 0) {
		return (
			<NativeImgInner
				{...props}
				ref={ref}
				hidden={hidden}
				name={name}
				showInTimeline={showInTimeline}
				src={src}
				from={from}
				trimBefore={trimBefore}
				durationInFrames={durationInFrames}
				freeze={freeze}
				premountFor={premountFor}
				postmountFor={postmountFor}
				styleWhilePremounted={styleWhilePremounted}
				styleWhilePostmounted={styleWhilePostmounted}
				controls={controls}
				width={width}
				height={height}
				className={className}
				style={style}
				cropLeft={cropLeft}
				cropRight={cropRight}
				cropTop={cropTop}
				cropBottom={cropBottom}
				id={id}
				pauseWhenLoading={shouldPauseWhenLoading}
				maxRetries={maxRetries}
				delayRenderRetries={delayRenderRetries}
				delayRenderTimeoutInMilliseconds={delayRenderTimeoutInMilliseconds}
				onImageError={onImageError}
			/>
		);
	}

	if (!src) {
		throw new Error('No "src" prop was passed to <Img>.');
	}

	validateCanvasImageFallbackProps({
		props,
		ref,
		width,
		height,
	});

	const canvasWidth = typeof width === 'number' ? width : undefined;
	const canvasHeight = typeof height === 'number' ? height : undefined;
	const canvasProps = props as CanvasImageCanvasProps;
	const canvasFit = getFitFromObjectFit(style) ?? 'fill';

	return (
		<CanvasImageWithPrivateProps
			src={src}
			width={canvasWidth}
			height={canvasHeight}
			fit={canvasFit}
			effects={effects}
			effectsOutputSize={effectsOutputSize}
			className={className}
			style={style}
			cropLeft={cropLeft}
			cropRight={cropRight}
			cropTop={cropTop}
			cropBottom={cropBottom}
			id={id}
			onError={onImageError}
			pauseWhenLoading={shouldPauseWhenLoading}
			maxRetries={maxRetries}
			delayRenderRetries={delayRenderRetries}
			delayRenderTimeoutInMilliseconds={delayRenderTimeoutInMilliseconds}
			from={from}
			trimBefore={trimBefore}
			durationInFrames={durationInFrames}
			freeze={freeze}
			premountFor={premountFor}
			postmountFor={postmountFor}
			styleWhilePremounted={styleWhilePremounted}
			styleWhilePostmounted={styleWhilePostmounted}
			hidden={hidden}
			name={name ?? '<Img>'}
			showInTimeline={showInTimeline}
			_remotionInternalDocumentationLink="https://www.remotion.dev/docs/img"
			_remotionInternalCropComponentName="<Img />"
			controls={controls}
			{...canvasProps}
		/>
	);
};

/*
 * @description Works just like a regular HTML img tag. When you use the <Img> tag, Remotion will ensure that the image is loaded before rendering the frame.
 * @see [Documentation](https://remotion.dev/docs/img)
 */
export const Img = withInteractivitySchema({
	Component: ImgInner,
	componentName: '<Img>',
	componentIdentity: 'dev.remotion.remotion.Img',
	schema: imgSchema,
	supportsEffects: true,
});
addSequenceStackTraces(Img);