Skip to content

🎥 Make videos programmatically with React

18.0 KB
Raw Blame
import type {NewCompositionAsset} from '@remotion/studio-shared';
import {formatBytes} from '@remotion/studio-shared';
import React, {useCallback, useContext, useMemo, useState} from 'react';
import {Internals, staticFile} from 'remotion';
import {getBrowserStudioOperations} from '../helpers/browser-studio-operations';
import {StudioServerConnectionCtx} from '../helpers/client-id';
import {CURRENT_COLOR, LIGHT_TEXT} from '../helpers/colors';
import {formatMediaDuration} from '../helpers/format-media-duration';
import {getFileManagerName} from '../helpers/get-file-manager-name';
import {getPreviewFileType} from '../helpers/get-preview-file-type';
import {TIMELINE_FRAME_WIDTH_AT_MAX_ZOOM} from '../helpers/get-timeline-max-zoom';
import {installRequiredPackages} from '../helpers/install-required-package';
import {openInRemotionConvert} from '../helpers/open-in-remotion-convert';
import {
	renderHumanReadableAudioCodec,
	renderHumanReadableVideoCodec,
} from '../helpers/render-codec-label';
import {
	getUniqueCompositionName,
	useCreateComposition,
} from '../helpers/use-create-composition';
import {useImageMetadata} from '../helpers/use-image-metadata';
import type {MediaMetadata} from '../helpers/use-media-metadata';
import {useMediaMetadata} from '../helpers/use-media-metadata';
import {BackgroundRemovalIcon} from '../icons/background-removal';
import {ExpandedFolderIcon} from '../icons/folder';
import {RemotionConvertIcon} from '../icons/remotion-convert';
import {TranscriptionIcon} from '../icons/transcription';
import {TrashIcon} from '../icons/trash';
import {FilmIcon} from '../icons/video';
import {SetSelectedModalContext} from '../state/modals';
import {AssetAudioVolume} from './AssetAudioVolume';
import {InlineEditableTitle} from './InlineEditableTitle';
import {InspectorInfoHeader} from './InspectorInfoHeader';
import {CollapsibleInspectorSection} from './InspectorPanel/CollapsibleInspectorSection';
import {
	InspectorDetailRow,
	InspectorQuickAction,
	InspectorQuickActionsSection,
} from './InspectorPanel/common';
import {INSPECTOR_PANEL_HORIZONTAL_PADDING} from './InspectorPanelLayout';
import {COMPACT_CONTROL_ROW_HEIGHT} from './layout';
import {getNewCompositionDefaults} from './NewComposition/get-new-composition-defaults';
import {
	getStaticFileRenameSelection,
	useRenameStaticFile,
} from './NewComposition/use-rename-static-file';
import {showNotification} from './Notifications/NotificationCenter';
import {openInFileExplorer} from './RenderQueue/actions';
import {useDeleteAsset} from './use-delete-asset';
import {useStaticFiles} from './use-static-files';

export const CURRENT_ASSET_HEIGHT = COMPACT_CONTROL_ROW_HEIGHT;

const quickActionIconStyle: React.CSSProperties = {
	display: 'block',
	height: 18,
	width: 18,
};

const convertArrowStyle: React.CSSProperties = {
	display: 'inline-block',
	height: 12,
	marginLeft: 4,
	verticalAlign: -2,
	width: 12,
};

const assetMetadataStyle: React.CSSProperties = {
	padding: `0 ${INSPECTOR_PANEL_HORIZONTAL_PADDING}px`,
};

const assetMetadataValueStyle: React.CSSProperties = {
	color: LIGHT_TEXT,
	fontFamily: 'sans-serif',
	fontSize: 13,
	lineHeight: '20px',
};

const assetEmptyStateStyle: React.CSSProperties = {
	color: LIGHT_TEXT,
	fontFamily: 'sans-serif',
	fontSize: 12,
	lineHeight: 1.4,
	padding: `0 ${INSPECTOR_PANEL_HORIZONTAL_PADDING}px`,
};

export const getCurrentAssetMetadataSource = (assetName: string | null) => {
	if (!assetName) {
		return null;
	}

	const fileType = getPreviewFileType(assetName);
	return fileType === 'audio' || fileType === 'video'
		? staticFile(assetName)
		: null;
};

export const getCurrentAssetImageMetadataSource = (
	assetName: string | null,
) => {
	if (!assetName) {
		return null;
	}

	return getPreviewFileType(assetName) === 'image'
		? staticFile(assetName)
		: null;
};

const formatFps = (fps: number) => `${fps.toFixed(2)} FPS`;

type CurrentAssetDetail = {
	readonly label: string;
	readonly value: string;
};

export const getCurrentAssetMediaSections = (mediaMetadata: MediaMetadata) => {
	const hasVideo =
		mediaMetadata.hasVideoTrack === true ||
		mediaMetadata.width !== null ||
		mediaMetadata.height !== null ||
		mediaMetadata.videoCodec !== null ||
		mediaMetadata.fps !== null ||
		mediaMetadata.isHdr !== null;
	const hasAudio =
		mediaMetadata.hasAudioTrack === true ||
		mediaMetadata.audioCodec !== null ||
		mediaMetadata.sampleRate !== null;
	const video: CurrentAssetDetail[] = [];
	const audio: CurrentAssetDetail[] = [];

	if (hasVideo) {
		if (mediaMetadata.width !== null && mediaMetadata.height !== null) {
			video.push({
				label: 'Dimensions',
				value: `${mediaMetadata.width} × ${mediaMetadata.height}`,
			});
		}

		if (mediaMetadata.fps !== null) {
			video.push({label: 'Frame rate', value: formatFps(mediaMetadata.fps)});
		}

		video.push({
			label: 'Duration',
			value: formatMediaDuration(mediaMetadata.duration),
		});
		video.push({
			label: 'Codec',
			value: renderHumanReadableVideoCodec(mediaMetadata.videoCodec),
		});

		if (mediaMetadata.isHdr !== null) {
			video.push({
				label: 'HDR',
				value: mediaMetadata.isHdr ? 'Yes' : 'No',
			});
		}
	}

	if (hasAudio) {
		audio.push({
			label: 'Duration',
			value: formatMediaDuration(mediaMetadata.duration),
		});
		audio.push({
			label: 'Codec',
			value: renderHumanReadableAudioCodec(mediaMetadata.audioCodec),
		});

		if (mediaMetadata.sampleRate !== null) {
			audio.push({
				label: 'Sample rate',
				value: `${mediaMetadata.sampleRate} Hz`,
			});
		}
	}

	return {
		audio:
			hasAudio || (hasVideo && mediaMetadata.hasAudioTrack === false)
				? audio
				: null,
		video: hasVideo ? video : null,
	};
};

export const AssetInfo: React.FC<{
	readonly assetName: string | null;
	readonly contentSized?: boolean;
	readonly onAssetClick?: () => void;
	readonly readOnlyStudio: boolean;
}> = ({assetName, contentSized = false, onAssetClick, readOnlyStudio}) => {
	const {compositions, currentAssetMetadata} = useContext(
		Internals.CompositionManager,
	);
	const [isMakingComposition, setIsMakingComposition] = useState(false);
	const volumeMetadata =
		currentAssetMetadata?.asset === assetName ? currentAssetMetadata : null;
	const connectionStatus = useContext(StudioServerConnectionCtx)
		.previewServerState.type;
	const browserStudioOperations = getBrowserStudioOperations();
	const {setSelectedModal} = useContext(SetSelectedModalContext);

	const staticFiles = useStaticFiles();
	const renameFile = useRenameStaticFile({
		relativePath: assetName ?? '',
		staticFiles,
	});

	const sizeInBytes = useMemo(() => {
		if (!assetName) {
			return null;
		}

		const file = staticFiles.find((f) => f.name === assetName);
		return file?.sizeInBytes ?? null;
	}, [assetName, staticFiles]);

	const src = getCurrentAssetMetadataSource(assetName);
	const mediaMetadata = useMediaMetadata(src);
	const imageSrc = getCurrentAssetImageMetadataSource(assetName);
	const imageMetadata = useImageMetadata(imageSrc);
	const mutationsDisabled =
		browserStudioOperations === null &&
		(readOnlyStudio || connectionStatus !== 'connected');
	const fileName = assetName?.split('/').pop() ?? '';
	const fileType = assetName ? getPreviewFileType(assetName) : null;
	const defaultComposition = getNewCompositionDefaults(null, null);
	const compositionFps =
		fileType === 'video' &&
		mediaMetadata !== null &&
		mediaMetadata.fps !== null &&
		Number.isFinite(mediaMetadata.fps) &&
		mediaMetadata.fps > 0
			? mediaMetadata.fps
			: defaultComposition.fps;
	const assetForComposition = useMemo((): NewCompositionAsset | null => {
		if (assetName === null || fileType === null) {
			return null;
		}

		if (fileType === 'image') {
			if (imageMetadata === null) {
				return null;
			}

			return {
				type: 'image',
				src: assetName,
				durationInFrames: Math.round(compositionFps * 3),
			};
		}

		if (
			(fileType !== 'audio' && fileType !== 'video') ||
			mediaMetadata === null ||
			!Number.isFinite(mediaMetadata.duration) ||
			mediaMetadata.duration <= 0
		) {
			return null;
		}

		return {
			type: fileType,
			src: assetName,
			durationInFrames: Math.max(
				1,
				Math.ceil(mediaMetadata.duration * compositionFps),
			),
		};
	}, [assetName, compositionFps, fileType, imageMetadata, mediaMetadata]);
	const newCompositionId = useMemo(() => {
		const nameWithoutExtension = fileName.replace(/\.[^/.]+$/, '');
		return getUniqueCompositionName(compositions, nameWithoutExtension || null);
	}, [compositions, fileName]);
	const compositionSize = useMemo(() => {
		if (fileType === 'image' && imageMetadata !== null) {
			return {width: imageMetadata.width, height: imageMetadata.height};
		}

		if (
			fileType === 'video' &&
			mediaMetadata !== null &&
			mediaMetadata.width !== null &&
			mediaMetadata.height !== null
		) {
			return {
				width: mediaMetadata.width,
				height: mediaMetadata.height,
			};
		}

		return {
			width: defaultComposition.width,
			height: defaultComposition.height,
		};
	}, [
		defaultComposition.height,
		defaultComposition.width,
		fileType,
		imageMetadata,
		mediaMetadata,
	]);
	const {createComposition: makeComposition} = useCreateComposition({
		asset: assetForComposition,
		canvasCapture: null,
		compositions,
		durationInFrames:
			assetForComposition?.durationInFrames ??
			defaultComposition.durationInFrames,
		folderName: null,
		newId: newCompositionId,
		parentName: null,
		selectedFrameRate: compositionFps,
		size: compositionSize,
	});
	const onMakeComposition = useCallback(async () => {
		if (
			assetForComposition === null ||
			mutationsDisabled ||
			isMakingComposition
		) {
			return;
		}

		setIsMakingComposition(true);
		try {
			if (assetForComposition.type !== 'image') {
				await installRequiredPackages([
					{name: '@remotion/media', version: null},
				]);
			}

			const result = await makeComposition({
				signal: new AbortController().signal,
				symbolicatedStack: null,
			});
			if (!result.success) {
				showNotification(
					`Could not create composition: ${result.reason}`,
					4000,
				);
			}
		} catch (error) {
			showNotification(
				`Could not create composition: ${(error as Error).message}`,
				4000,
			);
		} finally {
			setIsMakingComposition(false);
		}
	}, [
		assetForComposition,
		isMakingComposition,
		makeComposition,
		mutationsDisabled,
	]);
	const onTranscribe = useCallback(() => {
		if (src === null || mutationsDisabled) {
			return;
		}

		setSelectedModal({
			type: 'transcribe',
			captionStyle: null,
			src,
			displayName: fileName,
			audioStreamIndex: null,
			requestInit: null,
			target: null,
		});
	}, [fileName, mutationsDisabled, setSelectedModal, src]);
	const onTrackMatting = useCallback(() => {
		if (src === null || fileType !== 'video' || mutationsDisabled) {
			return;
		}

		setSelectedModal({
			type: 'video-matting',
			src,
			displayName: fileName,
			target: null,
		});
	}, [fileName, fileType, mutationsDisabled, setSelectedModal, src]);
	const canRename =
		onAssetClick === undefined &&
		(browserStudioOperations !== null ||
			(connectionStatus === 'connected' && !readOnlyStudio));
	const onRename = useCallback(
		(newName: string) => {
			renameFile(newName).catch(() => undefined);
		},
		[renameFile],
	);
	const onOpenConvert = useCallback(() => {
		if (assetName === null) {
			return;
		}

		openInRemotionConvert({relativePath: assetName});
	}, [assetName]);
	const onShowInFileManager = useCallback(() => {
		if (assetName === null || !window.remotion_publicFolderExists) {
			showNotification('Could not find the public folder', 2000);
			return;
		}

		openInFileExplorer({
			directory: window.remotion_publicFolderExists + '/' + assetName,
		}).catch((err) => {
			showNotification(`Could not open file: ${err.message}`, 2000);
		});
	}, [assetName]);
	const onDelete = useDeleteAsset(assetName);

	if (!assetName) {
		return (
			<InspectorInfoHeader
				contentSized={contentSized}
				minHeight={CURRENT_ASSET_HEIGHT}
			/>
		);
	}

	const fileDetails: CurrentAssetDetail[] = [];
	if (imageMetadata !== null) {
		fileDetails.push({
			label: 'Dimensions',
			value: `${imageMetadata.width} × ${imageMetadata.height}`,
		});
	}

	const container = mediaMetadata?.format ?? imageMetadata?.format ?? null;
	if (container !== null) {
		fileDetails.push({label: 'Container', value: container});
	}

	if (sizeInBytes !== null) {
		fileDetails.push({label: 'Size', value: formatBytes(sizeInBytes)});
	}

	const mediaSections = mediaMetadata
		? getCurrentAssetMediaSections(mediaMetadata)
		: null;
	const fileManagerAvailable = browserStudioOperations === null;
	const fileManagerDisabled =
		window.remotion_publicFolderExists === null ||
		readOnlyStudio ||
		connectionStatus !== 'connected';
	const fileManagerName = getFileManagerName(
		window.remotion_fileSystemPlatform,
	);
	return (
		<>
			<InspectorInfoHeader
				contentSized={contentSized}
				minHeight={CURRENT_ASSET_HEIGHT}
				padding={
					contentSized
						? `0 ${INSPECTOR_PANEL_HORIZONTAL_PADDING}px 6px`
						: '4px 0'
				}
			>
				<InlineEditableTitle
					value={fileName}
					canRename={canRename}
					getInitialSelection={getStaticFileRenameSelection}
					onClick={onAssetClick}
					onCommit={onRename}
					size={contentSized ? 'default' : 'inspector'}
					aria-label={assetName}
				/>
			</InspectorInfoHeader>
			{fileDetails.length > 0 ? (
				<CollapsibleInspectorSection
					bodyPaddingBottom={8}
					collapsible
					label="File"
					sectionId="asset-file"
				>
					<div style={assetMetadataStyle}>
						{fileDetails.map((detail) => (
							<InspectorDetailRow key={detail.label} label={detail.label}>
								<span style={assetMetadataValueStyle}>{detail.value}</span>
							</InspectorDetailRow>
						))}
					</div>
				</CollapsibleInspectorSection>
			) : null}
			{mediaSections && mediaSections.video ? (
				<CollapsibleInspectorSection
					bodyPaddingBottom={8}
					collapsible
					label="Video"
					sectionId="asset-video"
				>
					<div style={assetMetadataStyle}>
						{mediaSections.video.map((detail) => (
							<InspectorDetailRow key={detail.label} label={detail.label}>
								<span style={assetMetadataValueStyle}>{detail.value}</span>
							</InspectorDetailRow>
						))}
					</div>
				</CollapsibleInspectorSection>
			) : null}
			{mediaSections && mediaSections.audio !== null ? (
				<CollapsibleInspectorSection
					bodyPaddingBottom={8}
					collapsible={mediaSections.audio.length > 0}
					label="Audio"
					sectionId="asset-audio"
				>
					{mediaSections.audio.length === 0 ? (
						<div style={assetEmptyStateStyle}>None</div>
					) : (
						<div style={assetMetadataStyle}>
							{mediaSections.audio.map((detail) => (
								<InspectorDetailRow key={detail.label} label={detail.label}>
									<span style={assetMetadataValueStyle}>{detail.value}</span>
								</InspectorDetailRow>
							))}
							{src ? (
								<InspectorDetailRow label="Average volume">
									<span style={assetMetadataValueStyle}>
										<AssetAudioVolume
											key={volumeMetadata?.src ?? src}
											src={volumeMetadata?.src ?? null}
											waveformSampleRate={
												volumeMetadata
													? Math.ceil(
															volumeMetadata.fps *
																TIMELINE_FRAME_WIDTH_AT_MAX_ZOOM,
														)
													: null
											}
										/>
									</span>
								</InspectorDetailRow>
							) : null}
						</div>
					)}
				</CollapsibleInspectorSection>
			) : null}
			<CollapsibleInspectorSection
				bodyPaddingBottom={8}
				collapsible
				label="Actions"
				sectionId="asset-actions"
			>
				<InspectorQuickActionsSection>
					{fileType === 'audio' ||
					fileType === 'image' ||
					fileType === 'video' ? (
						<InspectorQuickAction
							disabled={
								mutationsDisabled ||
								assetForComposition === null ||
								isMakingComposition
							}
							onClick={onMakeComposition}
							renderIcon={(color) => (
								<FilmIcon color={color} style={quickActionIconStyle} />
							)}
						>
							Make composition
						</InspectorQuickAction>
					) : null}
					{fileManagerAvailable ? (
						<InspectorQuickAction
							disabled={fileManagerDisabled}
							onClick={onShowInFileManager}
							renderIcon={(color) => (
								<ExpandedFolderIcon
									color={color}
									style={quickActionIconStyle}
								/>
							)}
						>
							Show in {fileManagerName}
						</InspectorQuickAction>
					) : null}
					{src && mediaMetadata?.hasAudioTrack !== false ? (
						<InspectorQuickAction
							disabled={mutationsDisabled}
							onClick={onTranscribe}
							renderIcon={(color) => (
								<TranscriptionIcon color={color} style={quickActionIconStyle} />
							)}
						>
							Transcribe
						</InspectorQuickAction>
					) : null}
					{fileType === 'video' ? (
						<InspectorQuickAction
							disabled={mutationsDisabled}
							onClick={onTrackMatting}
							renderIcon={(color) => (
								<BackgroundRemovalIcon
									color={color}
									style={quickActionIconStyle}
								/>
							)}
						>
							Remove background
						</InspectorQuickAction>
					) : null}
					{src ? (
						<InspectorQuickAction
							disabled={false}
							onClick={onOpenConvert}
							renderIcon={(color) => (
								<RemotionConvertIcon
									color={color}
									style={quickActionIconStyle}
								/>
							)}
						>
							Convert
							<svg
								aria-hidden="true"
								viewBox="0 0 16 16"
								style={convertArrowStyle}
							>
								<path
									d="M4 12 12 4M6 4h6v6"
									fill="none"
									stroke={CURRENT_COLOR}
									strokeLinecap="round"
									strokeLinejoin="round"
									strokeWidth="1.5"
								/>
							</svg>
						</InspectorQuickAction>
					) : null}
					<InspectorQuickAction
						disabled={mutationsDisabled}
						onClick={onDelete}
						renderIcon={(color) => (
							<TrashIcon color={color} style={quickActionIconStyle} />
						)}
					>
						Delete
					</InspectorQuickAction>
				</InspectorQuickActionsSection>
			</CollapsibleInspectorSection>
		</>
	);
};

export const CurrentAsset: React.FC<{
	readonly readOnlyStudio: boolean;
}> = ({readOnlyStudio}) => {
	const {canvasContent} = useContext(Internals.CompositionManager);
	const assetName =
		canvasContent?.type === 'asset' ? canvasContent.asset : null;

	return <AssetInfo assetName={assetName} readOnlyStudio={readOnlyStudio} />;
};