Skip to content

🎥 Make videos programmatically with React

8.7 KB
Raw Blame
import {
	drawBars,
	subscribeToWaveformPeaks,
	type WaveformDrawRange,
} from '@remotion/timeline-utils';
import React, {useLayoutEffect, useMemo, useRef, useState} from 'react';
import type {LoopDisplay} from 'remotion';
import {Internals} from 'remotion';
import {alignCanvasToDevicePixels} from '../helpers/align-canvas-to-device-pixels';
import {BLACK_ALPHA_60, WHITE_ALPHA_60} from '../helpers/colors';
import {TIMELINE_FRAME_WIDTH_AT_MAX_ZOOM} from '../helpers/get-timeline-max-zoom';
import {resolveStudioColor} from '../helpers/resolve-studio-color';
import {getStudioPixelRatio} from '../helpers/studio-pixel-ratio';
import type {TimelineVolume} from './Timeline/get-timeline-volume';

const EMPTY_PEAKS = new Float32Array(0);

const getContainerStyle = (height: number): React.CSSProperties => {
	return {
		display: 'flex',
		flexDirection: 'row',
		alignItems: 'center',
		position: 'relative',
		width: '100%',
		height,
		overflow: 'hidden',
	};
};

const waveformCanvasStyle: React.CSSProperties = {
	pointerEvents: 'none',
	flexShrink: 0,
	position: 'relative',
};

const volumeCanvasStyle: React.CSSProperties = {
	pointerEvents: 'none',
	position: 'absolute',
};

const AudioWaveformInner: React.FC<{
	readonly src: string;
	readonly height: number;
	readonly visualizationWidth: number;
	readonly startFrom: number;
	readonly durationInFrames: number;
	readonly displayOffsetInFrames: number;
	readonly displayDurationInFrames: number;
	readonly volume: TimelineVolume;
	readonly muted: boolean;
	readonly playbackRate: number;
	readonly loopDisplay: LoopDisplay | undefined;
	readonly loopDisplayOffsetInFrames: number;
}> = ({
	src,
	height,
	startFrom,
	durationInFrames,
	displayOffsetInFrames,
	displayDurationInFrames,
	visualizationWidth,
	volume,
	muted,
	playbackRate,
	loopDisplay,
	loopDisplayOffsetInFrames,
}) => {
	const [peaks, setPeaks] = useState<Float32Array | null>(null);
	const [error, setError] = useState<Error | null>(null);
	const vidConf = Internals.useUnsafeVideoConfig();
	if (vidConf === null) {
		throw new Error('Expected video config');
	}

	const waveformSampleRate = Math.ceil(
		vidConf.fps * TIMELINE_FRAME_WIDTH_AT_MAX_ZOOM,
	);

	const waveformCanvas = useRef<HTMLCanvasElement>(null);
	const volumeCanvas = useRef<HTMLCanvasElement>(null);
	const shouldRenderVolumeOverlay = volume.type !== 'constant';
	const visualizationMaxVolume = useMemo(() => {
		if (volume.type === 'constant') {
			return Math.max(1, volume.value);
		}

		if (volume.type === 'linear') {
			return volume.points.reduce(
				(highest, point) => Math.max(highest, point.value),
				1,
			);
		}

		return volume.samples.reduce((highest, value) => {
			return Number.isFinite(value) ? Math.max(highest, value) : highest;
		}, 1);
	}, [volume]);
	const visibleVolume = useMemo((): TimelineVolume => {
		if (muted) {
			return {type: 'constant', value: 0};
		}

		if (volume.type !== 'sampled') {
			return volume;
		}

		if (
			!Number.isFinite(displayDurationInFrames) ||
			displayDurationInFrames <= 0
		) {
			return {type: 'sampled', samples: []};
		}

		// Samples already cover the composition timeline, including media loops.
		return {
			type: 'sampled',
			samples: volume.samples.slice(
				Math.max(0, Math.floor(displayOffsetInFrames)),
				Math.min(
					volume.samples.length,
					Math.ceil(displayOffsetInFrames + displayDurationInFrames),
				),
			),
		};
	}, [displayDurationInFrames, displayOffsetInFrames, muted, volume]);
	const waveformVolume =
		visibleVolume.type === 'constant' ? visibleVolume.value : 1;

	// Layout effect so that a cache hit sets the peaks synchronously and the
	// waveform is painted on the very first frame after mounting.
	useLayoutEffect(() => {
		setPeaks(null);
		setError(null);

		return subscribeToWaveformPeaks({
			src,
			waveformSampleRate,
			onPeaks: (p) => setPeaks(p),
			onError: (err) => setError(err),
		});
	}, [src, waveformSampleRate]);

	const peakRange = useMemo((): WaveformDrawRange => {
		const peaksPerFrame = (waveformSampleRate * playbackRate) / vidConf.fps;
		const loop = loopDisplay !== undefined && loopDisplay.durationInFrames > 0;
		return {
			sourceStart: (startFrom / vidConf.fps) * waveformSampleRate,
			sourceDuration:
				(loop ? loopDisplay.durationInFrames : durationInFrames) *
				peaksPerFrame,
			displayStart:
				(displayOffsetInFrames + loopDisplayOffsetInFrames) * peaksPerFrame,
			displayDuration: displayDurationInFrames * peaksPerFrame,
			loop,
		};
	}, [
		displayDurationInFrames,
		displayOffsetInFrames,
		durationInFrames,
		loopDisplay,
		loopDisplayOffsetInFrames,
		playbackRate,
		startFrom,
		vidConf.fps,
		waveformSampleRate,
	]);

	// Drawing must happen in a layout effect: when the timeline zooms, the
	// canvas CSS box resizes in the same commit, and painting the bitmap
	// before the browser paints avoids showing a stretched stale waveform.
	useLayoutEffect(() => {
		const {current: canvasElement} = waveformCanvas;
		if (!canvasElement) {
			return;
		}

		const pixelRatio = getStudioPixelRatio();
		const {horizontalOffset} = alignCanvasToDevicePixels({
			canvas: canvasElement,
			cssHeight: height,
			cssWidth: visualizationWidth,
			pixelRatio,
		});
		const drawingWidth = visualizationWidth * pixelRatio;

		drawBars({
			canvas: canvasElement,
			peaks: peaks ?? EMPTY_PEAKS,
			range: peakRange,
			color: resolveStudioColor(
				WHITE_ALPHA_60,
				getComputedStyle(canvasElement),
			),
			volume: waveformVolume,
			width: drawingWidth,
			horizontalOffset,
		});
	}, [height, peakRange, peaks, visualizationWidth, waveformVolume]);

	useLayoutEffect(() => {
		if (!shouldRenderVolumeOverlay || !peaks) {
			return;
		}

		const {current: volumeCanvasElement} = volumeCanvas;
		if (!volumeCanvasElement) {
			return;
		}

		const pixelRatio = getStudioPixelRatio();
		const {
			height: h,
			horizontalOffset,
			width: w,
		} = alignCanvasToDevicePixels({
			canvas: volumeCanvasElement,
			cssHeight: height,
			cssWidth: visualizationWidth,
			pixelRatio,
		});
		const drawingWidth = visualizationWidth * pixelRatio;
		const context = volumeCanvasElement.getContext('2d');
		if (!context) {
			return;
		}

		context.clearRect(0, 0, w, h);
		if (visibleVolume.type === 'constant' || drawingWidth <= 0 || h <= 0) {
			return;
		}

		context.setTransform(1, 0, 0, 1, horizontalOffset, 0);
		context.beginPath();
		context.moveTo(0, 0);
		if (visibleVolume.type === 'linear') {
			for (const point of visibleVolume.points) {
				context.lineTo(
					((point.frame - displayOffsetInFrames) / displayDurationInFrames) *
						drawingWidth,
					Math.max(
						0,
						Math.min(h, (1 - point.value / visualizationMaxVolume) * h),
					),
				);
			}
		} else {
			// The canvas only spans the virtualized range. Sampling by its physical
			// width keeps drawing work bounded to the visible part of the timeline.
			const numberOfPoints = Math.min(
				visibleVolume.samples.length,
				Math.max(1, Math.ceil(drawingWidth)),
			);
			for (let point = 0; point < numberOfPoints; point++) {
				const volumeIndex = Math.floor(
					(point / numberOfPoints) * visibleVolume.samples.length,
				);
				const nextVolumeIndex = Math.floor(
					((point + 1) / numberOfPoints) * visibleVolume.samples.length,
				);
				const firstFrame = Math.max(0, Math.floor(displayOffsetInFrames));
				const x = Math.max(
					0,
					((firstFrame + volumeIndex - displayOffsetInFrames) /
						displayDurationInFrames) *
						drawingWidth,
				);
				const nextX =
					point === numberOfPoints - 1
						? drawingWidth
						: Math.min(
								drawingWidth,
								((firstFrame + nextVolumeIndex - displayOffsetInFrames) /
									displayDurationInFrames) *
									drawingWidth,
							);
				const unclampedY =
					(1 -
						(visibleVolume.samples[volumeIndex] ?? 1) /
							visualizationMaxVolume) *
					h;
				const y = Math.max(0, Math.min(h, unclampedY));
				// Volume samples apply for one frame. Keep each value until the next
				// frame boundary instead of blending a hold into a diagonal ramp.
				context.lineTo(x, y);
				context.lineTo(nextX, y);
			}
		}

		context.lineTo(drawingWidth, 0);
		context.closePath();
		context.fillStyle = resolveStudioColor(
			BLACK_ALPHA_60,
			getComputedStyle(volumeCanvasElement),
		);
		context.fill();
	}, [
		displayDurationInFrames,
		displayOffsetInFrames,
		height,
		peaks,
		shouldRenderVolumeOverlay,
		visualizationMaxVolume,
		visibleVolume,
		visualizationWidth,
	]);

	if (error) {
		return null;
	}

	if (!peaks) {
		return null;
	}

	return (
		<div style={getContainerStyle(height)}>
			<canvas ref={waveformCanvas} style={waveformCanvasStyle} />
			{/* Paint the volume shade over the waveform as well as its background. */}
			{shouldRenderVolumeOverlay ? (
				<canvas ref={volumeCanvas} style={volumeCanvasStyle} />
			) : null}
		</div>
	);
};

export const AudioWaveform = React.memo(AudioWaveformInner);