Skip to content

🎥 Make videos programmatically with React

19.1 KB
Raw Blame
import {
	addFrameToCache,
	aspectRatioCache,
	ensureSlots,
	extractFrames,
	fillFrameWhereItFits,
	fillWithCachedFrames,
	frameDatabase,
	getAspectRatioFromCache,
	getFrameDatabaseKeyPrefix,
	getLoopDisplaySegments,
	getSlotsToExtract,
	getTimestampFromFrameDatabaseKey,
	hydrateThumbnailsFromPersistentCache,
	makeFrameDatabaseKey,
	mediaDurationCache,
	resizeVideoFrame,
	snapCanvasPositionToDevicePixel,
	WEBCODECS_TIMESCALE,
	writeThumbnailToPersistentCache,
} from '@remotion/timeline-utils';
import React, {useLayoutEffect, useMemo, useRef, useState} from 'react';
import type {LoopDisplay} from 'remotion';
import {Internals, useVideoConfig} from 'remotion';
import {alignCanvasToDevicePixels} from '../../helpers/align-canvas-to-device-pixels';
import {BLACK_ALPHA_30} from '../../helpers/colors';
import {getStudioPixelRatio} from '../../helpers/studio-pixel-ratio';
import {
	TIMELINE_LAYER_FILMSTRIP_HEIGHT,
	TIMELINE_VIDEO_INFO_WAVEFORM_HEIGHT,
} from '../../helpers/timeline-layout';
import {AudioWaveform} from '../AudioWaveform';
import {getTimelineMediaStartFrame} from './get-timeline-media-start-frame';
import type {TimelineVolume} from './get-timeline-volume';
import {getTimelineVideoFilmstripTimes} from './timeline-video-filmstrip-times';

const outerStyle: React.CSSProperties = {
	width: '100%',
	height: '100%',
	display: 'flex',
	flexDirection: 'column',
};

const filmstripContainerStyle: React.CSSProperties = {
	height: TIMELINE_LAYER_FILMSTRIP_HEIGHT,
	backgroundColor: BLACK_ALPHA_30,
	display: 'flex',
	overflow: 'hidden',
	borderTopLeftRadius: 2,
	fontSize: 10,
	fontFamily: 'Arial, Helvetica',
};

const MAX_FROZEN_FRAME_CACHE_DEVIATION = WEBCODECS_TIMESCALE * 0.05;

const TimelineVideoInfoSegment: React.FC<{
	readonly src: string;
	readonly visualizationWidth: number;
	readonly startMediaFrom: number;
	readonly mediaFrameAtSequenceZero: number | null;
	readonly sequenceFrameOffset: number;
	readonly durationInFrames: number;
	readonly sourceOffsetInFrames: number;
	readonly tiledLoop: {
		readonly displayDurationInFrames: number;
		readonly displayOffsetInFrames: number;
		readonly loopDisplayOffsetInFrames: number;
		readonly loopDisplay: LoopDisplay;
		readonly loopWidth: number;
	} | null;
	readonly playbackRate: number;
	readonly volume: TimelineVolume;
	readonly muted: boolean;
	readonly frozenMediaFrame: number | null;
	readonly extendLastFrame: boolean;
}> = ({
	src,
	visualizationWidth,
	startMediaFrom,
	mediaFrameAtSequenceZero,
	sequenceFrameOffset,
	durationInFrames,
	sourceOffsetInFrames,
	tiledLoop,
	playbackRate,
	volume,
	muted,
	frozenMediaFrame,
	extendLastFrame,
}) => {
	const {fps} = useVideoConfig();
	const resolvedSrc = Internals.usePreload(src);
	const ref = useRef<HTMLCanvasElement>(null);
	const [error, setError] = useState<Error | null>(null);
	const aspectRatio = useRef<number | null>(getAspectRatioFromCache(src));
	const mediaStartFrame =
		getTimelineMediaStartFrame({
			startMediaFrom,
			mediaFrameAtSequenceZero,
			sequenceFrameOffset,
			playbackRate,
		}) +
		sourceOffsetInFrames * playbackRate;

	// for rendering frames
	useLayoutEffect(() => {
		const {current: canvas} = ref;
		if (!canvas) {
			return;
		}

		const ctx = canvas.getContext('2d');
		if (!ctx) {
			return;
		}

		if (error) {
			ctx.resetTransform();
			ctx.clearRect(0, 0, canvas.width, canvas.height);
			return;
		}

		const controller = new AbortController();
		const pixelRatio = getStudioPixelRatio();
		const {horizontalOffset} = alignCanvasToDevicePixels({
			canvas,
			cssHeight: TIMELINE_LAYER_FILMSTRIP_HEIGHT,
			cssWidth: visualizationWidth,
			pixelRatio,
		});
		ctx.setTransform(1, 0, 0, 1, horizontalOffset, 0);

		const clearCanvas = () => {
			ctx.save();
			ctx.resetTransform();
			ctx.clearRect(0, 0, canvas.width, canvas.height);
			ctx.restore();
		};

		const drawRepeatedFrame = (frame: VideoFrame) => {
			const thumbnailWidth = Math.max(1, frame.displayWidth);

			clearCanvas();
			for (let x = 0; x < canvas.width; x += thumbnailWidth) {
				const left = snapCanvasPositionToDevicePixel({
					horizontalOffset,
					position: x,
				});
				const right = snapCanvasPositionToDevicePixel({
					horizontalOffset,
					position: x + thumbnailWidth,
				});
				ctx.drawImage(frame, left, 0, right - left, canvas.height);
			}
		};

		const getCachedFrozenFrame = (timestamp: number) => {
			const prefix = getFrameDatabaseKeyPrefix(src);
			const keys = Array.from(frameDatabase.keys()).filter((k) =>
				k.startsWith(prefix),
			);
			let bestDistance = Infinity;
			let bestFrame: VideoFrame | null = null;

			for (const key of keys) {
				const frame = frameDatabase.get(key);
				if (!frame) {
					continue;
				}

				const distance = Math.abs(
					getTimestampFromFrameDatabaseKey(key) - timestamp,
				);
				if (distance < bestDistance) {
					bestDistance = distance;
					bestFrame = frame.frame;
				}
			}

			return bestDistance <= MAX_FROZEN_FRAME_CACHE_DEVIATION
				? bestFrame
				: null;
		};

		const times = getTimelineVideoFilmstripTimes({
			trimBefore: mediaStartFrame,
			durationInFrames,
			playbackRate,
			fps,
			loopDisplay: undefined,
			frozenMediaFrame,
		});

		if (times.type === 'frozen') {
			const timestamp = times.timestampInSeconds * WEBCODECS_TIMESCALE;
			const cachedFrame = getCachedFrozenFrame(timestamp);

			if (cachedFrame) {
				drawRepeatedFrame(cachedFrame);
				return;
			}

			hydrateThumbnailsFromPersistentCache({
				src,
				height: canvas.height,
				getTimestamps: () => [timestamp],
			})
				.then(() => {
					if (controller.signal.aborted) {
						return;
					}

					const persistedFrame = getCachedFrozenFrame(timestamp);
					if (persistedFrame) {
						drawRepeatedFrame(persistedFrame);
						return;
					}

					return extractFrames({
						repeatLastFrame: extendLastFrame,
						timestampsInSeconds: ({
							track,
							durationInSeconds,
						}: {
							track: {height: number; width: number};
							durationInSeconds: number | null;
						}) => {
							aspectRatio.current = track.width / track.height;
							aspectRatioCache.set(src, aspectRatio.current);
							if (durationInSeconds !== null) {
								mediaDurationCache.set(src, durationInSeconds);
							}

							return [times.timestampInSeconds];
						},
						src: resolvedSrc,
						onVideoSample: (sample) => {
							let frame: VideoFrame | undefined;
							try {
								frame = sample.toVideoFrame();
								const scale = canvas.height / frame.displayHeight;

								const transformed = resizeVideoFrame({
									frame,
									scale,
								});

								if (transformed !== frame) {
									frame.close();
								}

								frame = undefined;

								const databaseKey = makeFrameDatabaseKey(
									src,
									transformed.timestamp,
								);

								addFrameToCache(databaseKey, transformed);
								writeThumbnailToPersistentCache({src, frame: transformed});

								drawRepeatedFrame(transformed);
							} catch (e) {
								if (frame) {
									frame.close();
								}

								throw e;
							} finally {
								sample.close();
							}
						},
						signal: controller.signal,
					});
				})
				.catch((e: unknown) => {
					setError(e as Error);
				});

			return () => {
				controller.abort();
			};
		}

		const targetCanvas = tiledLoop ? document.createElement('canvas') : canvas;
		if (tiledLoop) {
			targetCanvas.width = Math.max(
				1,
				Math.ceil(tiledLoop.loopWidth * pixelRatio),
			);
			targetCanvas.height = canvas.height;
		}

		const targetCtx = tiledLoop ? targetCanvas.getContext('2d') : ctx;
		if (!targetCtx) {
			return;
		}

		// desired-timestamp -> filled-timestamp
		const filledSlots = new Map<number, number | undefined>();
		const frameDrawOffset = tiledLoop ? 0 : horizontalOffset;

		const {fromSeconds, toSeconds} = times;
		// Keep the time-to-pixel scale independent of the integer canvas backing size.
		const targetWidth = tiledLoop
			? tiledLoop.loopWidth * pixelRatio
			: visualizationWidth * pixelRatio;
		const repeatTarget = () => {
			if (!tiledLoop) {
				return;
			}

			const pattern = ctx.createPattern(targetCanvas, 'repeat-x');
			if (!pattern) {
				return;
			}

			const phase =
				((tiledLoop.displayOffsetInFrames +
					tiledLoop.loopDisplayOffsetInFrames) %
					tiledLoop.loopDisplay.durationInFrames) *
				((tiledLoop.loopWidth * pixelRatio) /
					tiledLoop.loopDisplay.durationInFrames);
			pattern.setTransform(
				new DOMMatrix([
					(tiledLoop.loopWidth * pixelRatio) / targetCanvas.width,
					0,
					0,
					1,
					-phase,
					0,
				]),
			);
			clearCanvas();
			ctx.fillStyle = pattern;
			ctx.fillRect(0, 0, canvas.width, canvas.height);
		};

		// Returns whether every slot could be filled with a frame from memory.
		const fillWithFramesInMemory = () => {
			if (aspectRatio.current === null) {
				return false;
			}

			ensureSlots({
				filledSlots,
				naturalWidth: targetWidth,
				fromSeconds,
				toSeconds,
				aspectRatio: aspectRatio.current,
				frameHeight: canvas.height,
			});

			fillWithCachedFrames({
				ctx: targetCtx,
				naturalWidth: targetWidth,
				filledSlots,
				src,
				segmentDuration: toSeconds - fromSeconds,
				fromSeconds,
				devicePixelRatio: 1,
				frameHeight: canvas.height,
				horizontalOffset: frameDrawOffset,
			});
			repeatTarget();

			return getSlotsToExtract({filledSlots, src}).length === 0;
		};

		// Don't extract frames if every slot has a fitting frame
		if (fillWithFramesInMemory()) {
			return;
		}

		hydrateThumbnailsFromPersistentCache({
			src,
			height: canvas.height,
			getTimestamps: () => {
				// A persisted source also restores its aspect ratio.
				if (aspectRatio.current === null) {
					aspectRatio.current = getAspectRatioFromCache(src);
				}

				if (aspectRatio.current === null) {
					return [];
				}

				ensureSlots({
					filledSlots,
					naturalWidth: targetWidth,
					fromSeconds,
					toSeconds,
					aspectRatio: aspectRatio.current,
					frameHeight: canvas.height,
				});
				return Array.from(filledSlots.keys());
			},
		})
			.then(() => {
				if (controller.signal.aborted) {
					return;
				}

				if (fillWithFramesInMemory()) {
					return;
				}

				return extractFrames({
					repeatLastFrame: extendLastFrame,
					timestampsInSeconds: ({
						track,
						durationInSeconds,
					}: {
						track: {height: number; width: number};
						durationInSeconds: number | null;
					}) => {
						aspectRatio.current = track.width / track.height;
						aspectRatioCache.set(src, aspectRatio.current);
						if (durationInSeconds !== null) {
							mediaDurationCache.set(src, durationInSeconds);
						}

						ensureSlots({
							filledSlots,
							fromSeconds,
							toSeconds,
							naturalWidth: targetWidth,
							aspectRatio: aspectRatio.current,
							frameHeight: canvas.height,
						});
						fillWithCachedFrames({
							ctx: targetCtx,
							naturalWidth: targetWidth,
							filledSlots,
							src,
							segmentDuration: toSeconds - fromSeconds,
							fromSeconds,
							devicePixelRatio: 1,
							frameHeight: canvas.height,
							horizontalOffset: frameDrawOffset,
						});
						repeatTarget();

						return getSlotsToExtract({filledSlots, src}).map(
							(timestamp) => timestamp / WEBCODECS_TIMESCALE,
						);
					},
					src: resolvedSrc,
					onVideoSample: (sample) => {
						let frame: VideoFrame | undefined;
						try {
							frame = sample.toVideoFrame();
							const scale = canvas.height / frame.displayHeight;

							const transformed = resizeVideoFrame({
								frame,
								scale,
							});

							if (transformed !== frame) {
								frame.close();
							}

							frame = undefined;

							const databaseKey = makeFrameDatabaseKey(
								src,
								transformed.timestamp,
							);

							addFrameToCache(databaseKey, transformed);
							if (aspectRatio.current === null) {
								throw new Error('Aspect ratio is not set');
							}

							writeThumbnailToPersistentCache({src, frame: transformed});

							ensureSlots({
								filledSlots,
								fromSeconds,
								toSeconds,
								naturalWidth: targetWidth,
								aspectRatio: aspectRatio.current,
								frameHeight: canvas.height,
							});
							fillFrameWhereItFits({
								ctx: targetCtx,
								src,
								filledSlots,
								visualizationWidth: targetWidth,
								frame: transformed,
								segmentDuration: toSeconds - fromSeconds,
								fromSeconds,
								devicePixelRatio: 1,
								frameHeight: canvas.height,
								horizontalOffset: frameDrawOffset,
							});
							repeatTarget();
						} catch (e) {
							if (frame) {
								frame.close();
							}

							throw e;
						} finally {
							sample.close();
						}
					},
					signal: controller.signal,
				}).then(() => {
					if (controller.signal.aborted) {
						return;
					}

					fillWithCachedFrames({
						ctx: targetCtx,
						naturalWidth: targetWidth,
						filledSlots,
						src,
						segmentDuration: toSeconds - fromSeconds,
						fromSeconds,
						devicePixelRatio: 1,
						frameHeight: canvas.height,
						horizontalOffset: frameDrawOffset,
					});
					repeatTarget();
				});
			})
			.catch((e: unknown) => {
				setError(e as Error);
			});

		return () => {
			controller.abort();
		};
	}, [
		durationInFrames,
		error,
		extendLastFrame,
		fps,
		frozenMediaFrame,
		mediaStartFrame,
		playbackRate,
		resolvedSrc,
		src,
		tiledLoop,
		visualizationWidth,
	]);

	const audioWidth = visualizationWidth;

	const filmstripStyle: React.CSSProperties = useMemo(() => {
		return {
			...filmstripContainerStyle,
			width: visualizationWidth,
		};
	}, [visualizationWidth]);

	const audioStyle: React.CSSProperties = useMemo(() => {
		return {
			width: audioWidth,
			position: 'relative',
		};
	}, [audioWidth]);
	const segmentStyle: React.CSSProperties = useMemo(() => {
		return {
			...outerStyle,
			width: visualizationWidth,
			flexShrink: 0,
		};
	}, [visualizationWidth]);

	return (
		<div style={segmentStyle}>
			<div style={filmstripStyle}>
				<canvas ref={ref} />
			</div>
			<div style={audioStyle}>
				<AudioWaveform
					src={src}
					height={TIMELINE_VIDEO_INFO_WAVEFORM_HEIGHT}
					visualizationWidth={audioWidth}
					startFrom={mediaStartFrame}
					durationInFrames={durationInFrames}
					displayOffsetInFrames={tiledLoop?.displayOffsetInFrames ?? 0}
					loopDisplayOffsetInFrames={tiledLoop?.loopDisplayOffsetInFrames ?? 0}
					displayDurationInFrames={
						tiledLoop?.displayDurationInFrames ?? durationInFrames
					}
					volume={volume}
					muted={muted}
					playbackRate={playbackRate}
					loopDisplay={tiledLoop?.loopDisplay}
				/>
			</div>
		</div>
	);
};

const getVisibleSegments = ({
	displayDurationInFrames,
	displayOffsetInFrames,
	loopDisplay,
}: {
	readonly displayDurationInFrames: number;
	readonly displayOffsetInFrames: number;
	readonly loopDisplay: LoopDisplay | undefined;
}) => {
	if (!loopDisplay || loopDisplay.durationInFrames <= 0) {
		return [
			{
				key: 'single',
				displayOffsetInFrames,
				durationInFrames: displayDurationInFrames,
				sourceOffsetInFrames: displayOffsetInFrames,
			},
		];
	}

	return getLoopDisplaySegments({
		displayDurationInFrames,
		displayOffsetInFrames,
		loopDurationInFrames: loopDisplay.durationInFrames,
	}).map((segment) => {
		return {
			key: `loop-${segment.loopIndex}`,
			displayOffsetInFrames: segment.absoluteOffsetInFrames,
			durationInFrames: segment.durationInFrames,
			sourceOffsetInFrames: segment.loopOffsetInFrames,
		};
	});
};

const TimelineVideoInfoInner: React.FC<{
	readonly src: string;
	readonly visualizationWidth: number;
	readonly displayOffsetInFrames: number;
	readonly displayDurationInFrames: number;
	readonly startMediaFrom: number;
	readonly mediaFrameAtSequenceZero: number | null;
	readonly sequenceFrameOffset: number;
	readonly playbackRate: number;
	readonly volume: TimelineVolume;
	readonly muted: boolean;
	readonly marginLeft: number;
	readonly loopDisplay: LoopDisplay | undefined;
	readonly loopDisplayOffsetInFrames: number;
	readonly frozenMediaFrame: number | null;
	readonly extendLastFrame: boolean;
}> = ({
	src,
	visualizationWidth,
	displayOffsetInFrames,
	displayDurationInFrames,
	startMediaFrom,
	mediaFrameAtSequenceZero,
	sequenceFrameOffset,
	playbackRate,
	volume,
	muted,
	marginLeft,
	loopDisplay,
	loopDisplayOffsetInFrames,
	frozenMediaFrame,
	extendLastFrame,
}) => {
	const pixelsPerFrame = visualizationWidth / displayDurationInFrames;
	const loopWidth = loopDisplay
		? loopDisplay.durationInFrames * pixelsPerFrame
		: null;
	const shouldTileLoop =
		loopDisplay !== undefined &&
		loopWidth !== null &&
		loopWidth <= visualizationWidth;
	const tiledLoop = useMemo(() => {
		if (!shouldTileLoop || !loopDisplay || loopWidth === null) {
			return null;
		}

		return {
			displayDurationInFrames,
			displayOffsetInFrames,
			loopDisplayOffsetInFrames,
			loopDisplay,
			loopWidth,
		};
	}, [
		displayDurationInFrames,
		displayOffsetInFrames,
		loopDisplayOffsetInFrames,
		loopDisplay,
		loopWidth,
		shouldTileLoop,
	]);
	const segments = useMemo(() => {
		if (shouldTileLoop) {
			return [];
		}

		return getVisibleSegments({
			displayDurationInFrames,
			displayOffsetInFrames: displayOffsetInFrames + loopDisplayOffsetInFrames,
			loopDisplay,
		});
	}, [
		displayDurationInFrames,
		displayOffsetInFrames,
		loopDisplayOffsetInFrames,
		loopDisplay,
		shouldTileLoop,
	]);
	const getSegmentVolume = (
		segmentDisplayOffsetInFrames: number,
		segmentDurationInFrames: number,
	): TimelineVolume => {
		if (volume.type === 'constant') {
			return volume;
		}

		if (volume.type === 'linear') {
			return {
				type: 'linear',
				points: volume.points.map((point) => ({
					frame: point.frame - segmentDisplayOffsetInFrames,
					value: point.value,
				})),
			};
		}

		return {
			type: 'sampled',
			samples: volume.samples.slice(
				Math.max(0, Math.floor(segmentDisplayOffsetInFrames)),
				Math.min(
					volume.samples.length,
					Math.ceil(segmentDisplayOffsetInFrames + segmentDurationInFrames),
				),
			),
		};
	};

	return (
		<div style={{display: 'flex', marginLeft, height: '100%'}}>
			{tiledLoop ? (
				<TimelineVideoInfoSegment
					src={src}
					visualizationWidth={visualizationWidth}
					startMediaFrom={startMediaFrom}
					mediaFrameAtSequenceZero={mediaFrameAtSequenceZero}
					sequenceFrameOffset={sequenceFrameOffset}
					durationInFrames={tiledLoop.loopDisplay.durationInFrames}
					sourceOffsetInFrames={0}
					tiledLoop={tiledLoop}
					playbackRate={playbackRate}
					volume={volume}
					muted={muted}
					frozenMediaFrame={frozenMediaFrame}
					extendLastFrame={extendLastFrame}
				/>
			) : (
				segments.map((segment) => (
					<TimelineVideoInfoSegment
						key={segment.key}
						src={src}
						visualizationWidth={segment.durationInFrames * pixelsPerFrame}
						startMediaFrom={startMediaFrom}
						mediaFrameAtSequenceZero={mediaFrameAtSequenceZero}
						sequenceFrameOffset={sequenceFrameOffset}
						durationInFrames={segment.durationInFrames}
						sourceOffsetInFrames={segment.sourceOffsetInFrames}
						tiledLoop={null}
						playbackRate={playbackRate}
						volume={getSegmentVolume(
							segment.displayOffsetInFrames - loopDisplayOffsetInFrames,
							segment.durationInFrames,
						)}
						muted={muted}
						frozenMediaFrame={frozenMediaFrame}
						extendLastFrame={extendLastFrame}
					/>
				))
			)}
		</div>
	);
};

export const TimelineVideoInfo = React.memo(TimelineVideoInfoInner);