Skip to content

🎥 Make videos programmatically with React

9.3 KB
Raw Blame
import type {Caption, TikTokPage, TikTokToken} from '@remotion/captions';
import {createTikTokStyleCaptions} from '@remotion/captions';
import {loadFont} from '@remotion/google-fonts/Montserrat';
import {fitText} from '@remotion/layout-utils';
import React, {
	useEffect,
	useLayoutEffect,
	useMemo,
	useRef,
	useState,
} from 'react';
import {
	cancelRender,
	Interactive,
	interpolate,
	spring,
	useCurrentFrame,
	useVideoConfig,
	type InteractiveTransformProps,
	type InteractivitySchema,
	type SequenceProps,
} from 'remotion';

type MovingPillCaptionsProps = InteractiveTransformProps &
	Pick<SequenceProps, 'width' | 'height'> & {
		readonly captions: Caption[];
		readonly combineTokensWithinMilliseconds?: number;
	};

const desiredFontSize = 80;
const maximumTextWidth = 800;
const fontWeight = '700';
const textColor = '#ffffff';
const backgroundColor = '#0b84f3';
const pillHorizontalPadding = 12;
const pillVerticalPadding = 12;
const pillBorderRadius = 10;
const pillMoveDurationInFrames = 5;
const defaultCombineTokensWithinMilliseconds = 800;
const defaultWidth = 682;
const defaultHeight = 252;

const movingPillCaptionsSchema = {
	...Interactive.captionsSchema,
	width: {
		type: 'number',
		min: 1,
		step: 1,
		default: undefined,
		description: 'Caption area width',
		hiddenFromList: false,
	},
	height: {
		type: 'number',
		min: 1,
		step: 1,
		default: undefined,
		description: 'Caption area height',
		hiddenFromList: false,
	},
	combineTokensWithinMilliseconds: {
		type: 'number',
		min: 0,
		step: 50,
		default: defaultCombineTokensWithinMilliseconds,
		description: 'Time between caption pages',
		hiddenFromList: false,
	},
} as const satisfies InteractivitySchema;

const {fontFamily, waitUntilDone} = loadFont('normal', {
	weights: [fontWeight],
	subsets: ['latin'],
});

const frameToMilliseconds = (frame: number, fps: number) =>
	(frame / fps) * 1000;

const isTimeWithinHalfOpenInterval = (
	timeMs: number,
	fromMs: number,
	toMs: number,
) => timeMs >= fromMs && timeMs < toMs;

const getActivePageIndex = (
	pages: readonly Pick<TikTokPage, 'startMs' | 'durationMs'>[],
	timeMs: number,
) =>
	pages.findIndex((page) =>
		isTimeWithinHalfOpenInterval(
			timeMs,
			page.startMs,
			page.startMs + page.durationMs,
		),
	);

const getLatestStartedTokenIndex = (
	tokens: readonly Pick<TikTokToken, 'fromMs'>[],
	timeMs: number,
) =>
	tokens.reduce(
		(latestIndex, token, tokenIndex) =>
			token.fromMs <= timeMs ? tokenIndex : latestIndex,
		-1,
	);

type TokenLayout = {
	readonly height: number;
	readonly left: number;
	readonly top: number;
	readonly width: number;
};

const CaptionPage: React.FC<{
	readonly captionAreaWidth: number | null;
	readonly currentTimeMs: number;
	readonly fps: number;
	readonly page: TikTokPage;
	readonly pageIndex: number;
}> = ({captionAreaWidth, currentTimeMs, fps, page, pageIndex}) => {
	const fontSize = useMemo(() => {
		const availableWidth = Math.min(
			maximumTextWidth,
			captionAreaWidth ?? maximumTextWidth,
		);
		const maximumTokenWidth = Math.max(
			1,
			availableWidth - pillHorizontalPadding * 2,
		);
		const tokenFontSizes = page.tokens
			.map((token) => token.text.trim())
			.filter(Boolean)
			.map(
				(text) =>
					fitText({
						fontFamily,
						fontWeight,
						text,
						validateFontIsLoaded: true,
						withinWidth: maximumTokenWidth,
					}).fontSize,
			);

		return Math.min(
			desiredFontSize,
			fitText({
				fontFamily,
				fontWeight,
				text: page.text,
				validateFontIsLoaded: true,
				withinWidth: maximumTextWidth,
			}).fontSize,
			...tokenFontSizes,
		);
	}, [captionAreaWidth, page.text, page.tokens]);
	const textContainerRef = useRef<HTMLDivElement>(null);
	const tokenRefs = useRef<Array<HTMLSpanElement | null>>([]);
	const [tokenLayouts, setTokenLayouts] = useState<TokenLayout[]>([]);

	useLayoutEffect(() => {
		const container = textContainerRef.current;

		if (!container) {
			return;
		}

		const updateTokenLayouts = () => {
			setTokenLayouts(
				page.tokens.map((_, tokenIndex) => {
					const token = tokenRefs.current[tokenIndex];

					if (!token) {
						return {height: 0, left: 0, top: 0, width: 0};
					}

					return {
						height: token.offsetHeight,
						left: token.offsetLeft,
						top: token.offsetTop,
						width: token.offsetWidth,
					};
				}),
			);
		};

		updateTokenLayouts();
		const resizeObserver = new ResizeObserver(updateTokenLayouts);
		resizeObserver.observe(container);

		return () => resizeObserver.disconnect();
	}, [fontSize, page.tokens]);
	const latestStartedTokenIndex = getLatestStartedTokenIndex(
		page.tokens,
		currentTimeMs,
	);
	const pageLocalFrame = ((currentTimeMs - page.startMs) / 1000) * fps;
	const backgroundWordIndex = page.tokens.reduce(
		(wordIndex, token, tokenIndex) => {
			if (tokenIndex === 0) {
				return wordIndex;
			}

			const tokenStartFrame = ((token.fromMs - page.startMs) / 1000) * fps;
			return (
				wordIndex +
				spring({
					config: {damping: 100},
					delay: Math.max(0, tokenStartFrame - pillMoveDurationInFrames / 2),
					durationInFrames: pillMoveDurationInFrames,
					fps,
					frame: pageLocalFrame,
				})
			);
		},
		0,
	);
	const hasTokenLayouts = tokenLayouts.length === page.tokens.length;
	const clampedBackgroundWordIndex = Math.min(
		Math.max(backgroundWordIndex, 0),
		Math.max(0, tokenLayouts.length - 1),
	);
	const interpolationInput = tokenLayouts.map((_, tokenIndex) =>
		Number(tokenIndex),
	);
	const interpolateTokenLayout = (values: number[]) => {
		if (values.length === 0) {
			return 0;
		}

		if (values.length === 1) {
			return values[0];
		}

		return interpolate(clampedBackgroundWordIndex, interpolationInput, values);
	};
	const pillHeight = fontSize + pillVerticalPadding * 2;
	const pillLeft = interpolateTokenLayout(
		tokenLayouts.map((measurement) => measurement.left),
	);
	const pillTop = interpolateTokenLayout(
		tokenLayouts.map(
			(measurement) => measurement.top + (measurement.height - pillHeight) / 2,
		),
	);
	const pillWidth = interpolateTokenLayout(
		tokenLayouts.map((measurement) => measurement.width),
	);
	const textStrokeWidth = fontSize / 7;

	return (
		<div
			aria-label={page.text}
			aria-live="off"
			role="group"
			style={{
				alignItems: 'center',
				display: 'flex',
				height: '100%',
				justifyContent: 'center',
				width: '100%',
			}}
		>
			<div
				ref={textContainerRef}
				aria-hidden="true"
				style={{
					color: textColor,
					fontFamily,
					fontSize,
					fontWeight,
					lineHeight: 1.5,
					maxWidth: maximumTextWidth,
					paintOrder: 'stroke fill',
					position: 'relative',
					textAlign: 'center',
					WebkitTextStroke: `${textStrokeWidth}px #000000`,
					whiteSpace: 'normal',
					width: '100%',
				}}
			>
				{hasTokenLayouts && latestStartedTokenIndex >= 0 ? (
					<div
						aria-hidden="true"
						style={{
							backgroundColor,
							borderRadius: pillBorderRadius,
							height: pillHeight,
							left: pillLeft - pillHorizontalPadding,
							position: 'absolute',
							top: pillTop,
							width: pillWidth + pillHorizontalPadding * 2,
						}}
					/>
				) : null}
				{page.tokens.map((token, tokenIndex) => {
					const visibleText = token.text.trim();
					const visibleTextIndex = token.text.indexOf(visibleText);
					const leadingWhitespace = visibleText
						? token.text.slice(0, visibleTextIndex)
						: token.text;
					const trailingWhitespace = visibleText
						? token.text.slice(visibleTextIndex + visibleText.length)
						: '';

					return (
						<React.Fragment
							key={`${pageIndex}-${page.startMs}-${token.fromMs}-${tokenIndex}`}
						>
							{leadingWhitespace}
							<span
								ref={(element) => {
									tokenRefs.current[tokenIndex] = element;
								}}
								style={{
									color: textColor,
									display: 'inline-block',
									position: 'relative',
									whiteSpace: 'pre',
									zIndex: 1,
								}}
							>
								{visibleText}
							</span>
							{trailingWhitespace}
						</React.Fragment>
					);
				})}
			</div>
		</div>
	);
};

const MovingPillCaptionsContent: React.FC<MovingPillCaptionsProps> = ({
	captions,
	combineTokensWithinMilliseconds = defaultCombineTokensWithinMilliseconds,
	height = defaultHeight,
	style,
	width = defaultWidth,
}) => {
	const [fontLoaded, setFontLoaded] = useState(false);

	useEffect(() => {
		waitUntilDone()
			.then(() => {
				setFontLoaded(true);
			})
			.catch((error) => {
				cancelRender(error instanceof Error ? error : new Error(String(error)));
			});
	}, []);

	const frame = useCurrentFrame();
	const {fps} = useVideoConfig();
	const pages = useMemo(
		() =>
			createTikTokStyleCaptions({
				captions,
				combineTokensWithinMilliseconds,
			}).pages,
		[captions, combineTokensWithinMilliseconds],
	);
	const currentTimeMs = frameToMilliseconds(frame, fps);
	const activePageIndex = getActivePageIndex(pages, currentTimeMs);
	const page = pages[activePageIndex];

	return (
		<div
			style={{
				height,
				marginInline: 'auto',
				width,
				...style,
			}}
		>
			{fontLoaded && page ? (
				<CaptionPage
					key={`${activePageIndex}-${page.startMs}`}
					captionAreaWidth={width ?? null}
					currentTimeMs={currentTimeMs}
					fps={fps}
					page={page}
					pageIndex={activePageIndex}
				/>
			) : null}
		</div>
	);
};

export const MovingPillCaptions = Interactive.create({
	Component: MovingPillCaptionsContent,
	componentName: '<MovingPillCaptions>',
	schema: movingPillCaptionsSchema,
});