🎥 Make videos programmatically with React
Add video matting to Studio
JonnyBurgerb0d5529904f4
diff --git a/bun.lock b/bun.lockindex bd4cfcee73..f1fb5027be 100644--- a/bun.lock+++ b/bun.lock@@ -1829,6 +1829,7 @@"devDependencies": {"@happy-dom/global-registrator": "14.5.1","@remotion/eslint-config-internal": "workspace:*",+ "@remotion/video-matting": "workspace:*","@remotion/whisper-webgpu": "workspace:*","@testing-library/react": "16.1.0","@types/semver": "7.5.3",@@ -1838,11 +1839,13 @@"react-dom": "catalog:",},"peerDependencies": {+ "@remotion/video-matting": "workspace:*","@remotion/whisper-webgpu": "workspace:*","react": ">=16.8.0","react-dom": ">=16.8.0",},"optionalPeers": [+ "@remotion/video-matting","@remotion/whisper-webgpu",],},diff --git a/packages/example/e2e/transcription-modal.test.mts b/packages/example/e2e/transcription-modal.test.mtsindex 266039238b..c393b1dbad 100644--- a/packages/example/e2e/transcription-modal.test.mts+++ b/packages/example/e2e/transcription-modal.test.mts@@ -46,9 +46,35 @@ test.describe('transcription modal', () => {await expect(actions.getByRole('button')).toHaveText([/^Show in (Finder|File Explorer|File Manager)$/,'Transcribe',+ 'Track matting','Convert','Delete',]);+ await page.evaluate(() => {+ window.remotion_installedPackages = [+ ...(window.remotion_installedPackages ?? []),+ '@remotion/video-matting',+ ];+ });+ await actions.getByRole('button', {name: 'Track matting'}).click();+ const mattingDialog = page.getByRole('dialog', {+ name: 'Track matting vp8-vorbis.webm',+ });+ await expect(mattingDialog.getByTitle('Model')).toContainText('modnet');+ await expect(+ mattingDialog.getByRole('textbox', {name: 'Base video output file'}),+ ).toHaveValue(+ 'vp8-vorbis-base.webm',+ );+ await expect(+ mattingDialog.getByRole('textbox', {+ name: 'Foreground video output file',+ }),+ ).toHaveValue(+ 'vp8-vorbis-foreground.webm',+ );+ await page.keyboard.press('Escape');+ await expect(mattingDialog).toBeHidden();await transcribe.click();const dialog = page.getByRole('dialog');diff --git a/packages/studio/bundle.ts b/packages/studio/bundle.tsindex 9ae5a711c4..f31e221aa5 100644--- a/packages/studio/bundle.ts+++ b/packages/studio/bundle.ts@@ -17,6 +17,7 @@ const external = ['@remotion/renderer/pure','@remotion/web-renderer','@remotion/whisper-webgpu',+ '@remotion/video-matting','@remotion/renderer/error-handling','@jridgewell/trace-mapping','zod',diff --git a/packages/studio/package.json b/packages/studio/package.jsonindex ddc8f71ce5..86010c4e49 100644--- a/packages/studio/package.json+++ b/packages/studio/package.json@@ -20,11 +20,15 @@"url": "https://github.com/remotion-dev/remotion/issues"},"peerDependencies": {+ "@remotion/video-matting": "workspace:*","@remotion/whisper-webgpu": "workspace:*","react": ">=16.8.0","react-dom": ">=16.8.0"},"peerDependenciesMeta": {+ "@remotion/video-matting": {+ "optional": true+ },"@remotion/whisper-webgpu": {"optional": true}@@ -54,6 +58,7 @@},"devDependencies": {"@happy-dom/global-registrator": "14.5.1",+ "@remotion/video-matting": "workspace:*","@remotion/whisper-webgpu": "workspace:*","react": "catalog:","react-dom": "catalog:",diff --git a/packages/studio/src/components/CurrentAsset.tsx b/packages/studio/src/components/CurrentAsset.tsxindex 6bc2896982..37f6e574de 100644--- a/packages/studio/src/components/CurrentAsset.tsx+++ b/packages/studio/src/components/CurrentAsset.tsx@@ -15,6 +15,7 @@ import {import {useImageMetadata} from '../helpers/use-image-metadata';import type {MediaMetadata} from '../helpers/use-media-metadata';import {useMediaMetadata} from '../helpers/use-media-metadata';+import {EffectsIcon} from '../icons/effects';import {ExpandedFolderIcon} from '../icons/folder';import {RemotionConvertIcon} from '../icons/remotion-convert';import {TranscriptionIcon} from '../icons/transcription';@@ -210,6 +211,7 @@ export const AssetInfo: React.FC<{browserStudioOperations === null &&(readOnlyStudio || connectionStatus !== 'connected');const fileName = assetName?.split('/').pop() ?? '';+ const fileType = assetName ? getPreviewFileType(assetName) : null;const onTranscribe = useCallback(() => {if (src === null || mutationsDisabled) {return;@@ -223,6 +225,17 @@ export const AssetInfo: React.FC<{requestInit: null,});}, [fileName, mutationsDisabled, setSelectedModal, src]);+ const onTrackMatting = useCallback(() => {+ if (src === null || fileType !== 'video' || mutationsDisabled) {+ return;+ }++ setSelectedModal({+ type: 'video-matting',+ src,+ displayName: fileName,+ });+ }, [fileName, fileType, mutationsDisabled, setSelectedModal, src]);const canRename =onAssetClick === undefined &&(browserStudioOperations !== null ||@@ -373,6 +386,17 @@ export const AssetInfo: React.FC<{Transcribe</InspectorQuickAction>) : null}+ {fileType === 'video' ? (+ <InspectorQuickAction+ disabled={mutationsDisabled}+ onClick={onTrackMatting}+ renderIcon={(color) => (+ <EffectsIcon color={color} style={quickActionIconStyle} />+ )}+ >+ Track matting+ </InspectorQuickAction>+ ) : null}{src ? (<InspectorQuickActiondisabled={false}diff --git a/packages/studio/src/components/EditorContexts.tsx b/packages/studio/src/components/EditorContexts.tsxindex 0999b4a695..db16dc7d07 100644--- a/packages/studio/src/components/EditorContexts.tsx+++ b/packages/studio/src/components/EditorContexts.tsx@@ -15,6 +15,7 @@ import {ModalsProvider} from './ModalsProvider';import {CaptionQueueProcessorLoader} from './RenderQueue/CaptionQueueProcessorLoader';import {ClientRenderQueueProcessor} from './RenderQueue/ClientRenderQueueProcessor';import {RenderQueueContextProvider} from './RenderQueue/context';+import {VideoMattingQueueProcessorLoader} from './RenderQueue/VideoMattingQueueProcessorLoader';import {SetTimelineInOutProvider} from './SetTimelineInOutProvider';import {SettingsProvider} from './SettingsContext';import {ShowGuidesProvider} from './ShowGuidesProvider';@@ -37,6 +38,7 @@ export const EditorContexts: React.FC<{<RenderQueueContextProvider><ClientRenderQueueProcessor /><CaptionQueueProcessorLoader />+ <VideoMattingQueueProcessorLoader /><KeybindingContextProvider><CheckerboardProvider><ZoomGesturesProvider>diff --git a/packages/studio/src/components/Modals.tsx b/packages/studio/src/components/Modals.tsxindex 968d9f2c5c..bf19c856d3 100644--- a/packages/studio/src/components/Modals.tsx+++ b/packages/studio/src/components/Modals.tsx@@ -30,6 +30,7 @@ import {WebRenderModalWithLoader} from './RenderModal/WebRenderModal';import {SettingsModal} from './SettingsModal';import {SvgImportDialog} from './SvgImportDialog';import {TranscriptionModalWithOptionalWhisper} from './Transcription/TranscriptionModalWithOptionalWhisper';+import {VideoMattingModalWithOptionalPackage} from './VideoMatting/VideoMattingModalWithOptionalPackage';export const Modals: React.FC<{readonly readOnlyStudio: boolean;@@ -263,6 +264,9 @@ export const Modals: React.FC<{{modalContextType && modalContextType.type === 'transcribe' ? (<TranscriptionModalWithOptionalWhisper state={modalContextType} />) : null}+ {modalContextType && modalContextType.type === 'video-matting' ? (+ <VideoMattingModalWithOptionalPackage state={modalContextType} />+ ) : null}{modalContextType && modalContextType.type === 'fix-computed-value' && (<FixComputedValueModal state={modalContextType} />diff --git a/packages/studio/src/components/OptionalPackageModal.tsx b/packages/studio/src/components/OptionalPackageModal.tsxnew file mode 100644index 0000000000..43c06c0361--- /dev/null+++ b/packages/studio/src/components/OptionalPackageModal.tsx@@ -0,0 +1,157 @@+import React, {useCallback, useContext, useEffect, useState} from 'react';+import {installPackages} from '../api/install-package';+import {LIGHT_TEXT, WARNING_COLOR, WHITE} from '../helpers/colors';+import {SetSelectedModalContext} from '../state/modals';+import {Button} from './Button';+import {Flex, Row, Spacing} from './layout';+import {ModalButton} from './ModalButton';+import {ModalContainer} from './ModalContainer';+import {ModalFooterContainer} from './ModalFooter';+import {ModalHeader} from './ModalHeader';+import {Spinner} from './Spinner';++type InstallState =+ | {type: 'idle'}+ | {type: 'installing'}+ | {type: 'error'; error: Error};++const panelStyle: React.CSSProperties = {+ width: 'min(520px, calc(100vw - 40px))',+};++const contentStyle: React.CSSProperties = {+ color: WHITE,+ fontFamily: 'sans-serif',+ fontSize: 14,+ lineHeight: 1.5,+ padding: 16,+};++const codeStyle: React.CSSProperties = {+ color: 'inherit',+ fontFamily: 'monospace',+ fontSize: 'inherit',+};++const statusStyle: React.CSSProperties = {+ alignItems: 'center',+ color: LIGHT_TEXT,+ display: 'flex',+ fontFamily: 'sans-serif',+ fontSize: 14,+ gap: 10,+ lineHeight: 1.5,+};++const errorStyle: React.CSSProperties = {+ color: WARNING_COLOR,+ marginTop: 12,+ whiteSpace: 'pre-wrap',+};++const cancelStyle: React.CSSProperties = {minWidth: 90};++export const isOptionalPackageInstalled = (packageName: string) =>+ window.remotion_installedPackages?.includes(packageName) ?? false;++const OPTIONAL_PACKAGE_INSTALLED_EVENT = 'remotion-optional-package-installed';++export const useOptionalPackageInstalled = (packageName: string) => {+ const [installed, setInstalled] = useState(() =>+ isOptionalPackageInstalled(packageName),+ );++ useEffect(() => {+ const onInstalled = (event: Event) => {+ if ((event as CustomEvent<string>).detail === packageName) {+ setInstalled(true);+ }+ };++ window.addEventListener(OPTIONAL_PACKAGE_INSTALLED_EVENT, onInstalled);+ return () =>+ window.removeEventListener(OPTIONAL_PACKAGE_INSTALLED_EVENT, onInstalled);+ }, [packageName]);++ return installed;+};++export const OptionalPackageModal: React.FC<{+ readonly ariaLabel: string;+ readonly children: React.ReactNode;+ readonly packageName: string;+ readonly title: string;+}> = ({ariaLabel, children, packageName, title}) => {+ const {setSelectedModal} = useContext(SetSelectedModalContext);+ const installed = useOptionalPackageInstalled(packageName);+ const [installState, setInstallState] = useState<InstallState>({+ type: 'idle',+ });+ const busy = installState.type === 'installing';++ const dismiss = useCallback(() => {+ if (!busy) {+ setSelectedModal(null);+ }+ }, [busy, setSelectedModal]);++ const install = useCallback(async () => {+ setInstallState({type: 'installing'});+ try {+ await installPackages([{name: packageName, version: null}]);+ window.remotion_installedPackages = Array.from(+ new Set([...(window.remotion_installedPackages ?? []), packageName]),+ );+ window.dispatchEvent(+ new CustomEvent(OPTIONAL_PACKAGE_INSTALLED_EVENT, {+ detail: packageName,+ }),+ );+ } catch (error) {+ setInstallState({type: 'error', error: error as Error});+ }+ }, [packageName]);++ if (installed) {+ return children;+ }++ return (+ <ModalContainer+ ariaLabel={ariaLabel}+ onEscape={dismiss}+ onOutsideClick={dismiss}+ panelStyle={panelStyle}+ >+ <ModalHeader title={title} onClose={dismiss} />+ <div style={contentStyle}>+ {busy ? (+ <div style={statusStyle}>+ <Spinner duration={0.5} size={18} />+ {`Installing ${packageName}…`}+ </div>+ ) : (+ <>+ This requires installing{' '}+ <code style={codeStyle}>{packageName}</code>.{' Continue?'}+ {installState.type === 'error' ? (+ <div style={errorStyle}>{installState.error.message}</div>+ ) : null}+ </>+ )}+ </div>+ <ModalFooterContainer>+ <Row align="center">+ <Flex />+ <Button disabled={busy} onClick={dismiss} style={cancelStyle}>+ Cancel+ </Button>+ <Spacing x={1} />+ <ModalButton disabled={busy} onClick={install} autoFocus>+ {installState.type === 'error' ? 'Retry' : 'Continue'}+ </ModalButton>+ </Row>+ </ModalFooterContainer>+ </ModalContainer>+ );+};diff --git a/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx b/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsxindex 6e4e4f5062..5588dfeeaf 100644--- a/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx+++ b/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx@@ -12,6 +12,7 @@ import {writeStaticFile} from '../../api/write-static-file';import {resampleMediaTo16Khz} from '../Transcription/resample-media-to-16-khz';import type {CaptionJob} from './caption-job-types';import {RenderQueueContext} from './context';+import {loadModelForJob} from './load-model-for-job';export const CaptionQueueProcessor: React.FC = () => {const {@@ -40,19 +41,18 @@ export const CaptionQueueProcessor: React.FC = () => {value: 0.02,});await clearStaleModels();- const modelIsCached = await isWhisperModelCached({model: job.model});- await loadWhisperModel({+ await loadModelForJob({model: job.model,- onProgress: (progress) => {- const percentage =- progress.progress === null- ? ''- : ` ${Math.round(progress.progress * 100)}%`;- updateCaptionJobProgress(job.id, {- message: `${modelIsCached ? 'Loading' : 'Downloading'} ${job.model}${percentage}`,- value: 0.03 + (progress.progress ?? 0) * 0.27,- });- },+ progressStart: 0.03,+ progressSpan: 0.27,+ isModelCached: (model) => isWhisperModelCached({model}),+ loadModel: (model, onProgress) =>+ loadWhisperModel({+ model,+ onProgress: (progress) => onProgress(progress.progress),+ }),+ updateProgress: (progress) =>+ updateCaptionJobProgress(job.id, progress),});updateCaptionJobProgress(job.id, {diff --git a/packages/studio/src/components/RenderQueue/CaptionQueueProcessorLoader.tsx b/packages/studio/src/components/RenderQueue/CaptionQueueProcessorLoader.tsxindex 6f8ba1753e..403dfb5fe9 100644--- a/packages/studio/src/components/RenderQueue/CaptionQueueProcessorLoader.tsx+++ b/packages/studio/src/components/RenderQueue/CaptionQueueProcessorLoader.tsx@@ -1,5 +1,6 @@import React, {Suspense, useContext} from 'react';-import {isWhisperWebGpuInstalled} from '../Transcription/whisper-webgpu-capability';+import {useOptionalPackageInstalled} from '../OptionalPackageModal';+import {WHISPER_WEBGPU_PACKAGE} from '../Transcription/whisper-webgpu-capability';import {RenderQueueContext} from './context';const LazyCaptionQueueProcessor = React.lazy(async () => {@@ -9,7 +10,8 @@ const LazyCaptionQueueProcessor = React.lazy(async () => {export const CaptionQueueProcessorLoader: React.FC = () => {const {captionJobs} = useContext(RenderQueueContext);- if (!isWhisperWebGpuInstalled() || captionJobs.length === 0) {+ const installed = useOptionalPackageInstalled(WHISPER_WEBGPU_PACKAGE);+ if (!installed || captionJobs.length === 0) {return null;}diff --git a/packages/studio/src/components/RenderQueue/VideoMattingQueueItem.tsx b/packages/studio/src/components/RenderQueue/VideoMattingQueueItem.tsxnew file mode 100644index 0000000000..cee5bd4407--- /dev/null+++ b/packages/studio/src/components/RenderQueue/VideoMattingQueueItem.tsx@@ -0,0 +1,164 @@+import React, {useCallback, useContext, useMemo, useState} from 'react';+import {+ CURRENT_COLOR,+ FAIL_COLOR,+ getBackgroundFromHoverState,+ LIGHT_TEXT,+} from '../../helpers/colors';+import {pushUrl} from '../../helpers/url-state';+import type {RenderInlineAction} from '../InlineAction';+import {InlineAction} from '../InlineAction';+import {Row, Spacing} from '../layout';+import {useSelectAsset} from '../use-select-asset';+import {+ CircularProgress,+ RENDER_STATUS_INDICATOR_SIZE,+} from './CircularProgress';+import {RenderQueueContext} from './context';+import {renderQueueItemSubtitleStyle} from './item-style';+import {SuccessIcon} from './SuccessIcon';+import type {VideoMattingJob} from './video-matting-job-types';++const container: React.CSSProperties = {+ borderRadius: 4,+ display: 'flex',+ marginBottom: 1,+ marginLeft: 4,+ padding: 12,+ paddingBottom: 10,+ paddingRight: 4,+ width: 'calc(100% - 4px)',+};+const title: React.CSSProperties = {fontSize: 13, lineHeight: 1};+const right: React.CSSProperties = {+ display: 'flex',+ flex: 1,+ flexDirection: 'column',+ overflow: 'hidden',+};+const subtitle: React.CSSProperties = {+ ...renderQueueItemSubtitleStyle,+ cursor: 'default',+};+const statusIcon: React.CSSProperties = {+ height: RENDER_STATUS_INDICATOR_SIZE,+ width: RENDER_STATUS_INDICATOR_SIZE,+};+const removeIcon: React.CSSProperties = {color: CURRENT_COLOR, height: 16};++const Status: React.FC<{readonly job: VideoMattingJob}> = ({job}) => {+ if (job.status === 'running') {+ return (+ <div+ style={statusIcon}+ role="progressbar"+ aria-label="Video matting progress"+ aria-valuemin={0}+ aria-valuemax={100}+ aria-valuenow={Math.round(job.progress.value * 100)}+ aria-valuetext={job.progress.message}+ >+ <CircularProgress progress={Math.max(0.07, job.progress.value)} />+ </div>+ );+ }++ if (job.status === 'done') return <SuccessIcon />;+ if (job.status === 'failed')+ return (+ <svg style={statusIcon} viewBox="0 0 512 512">+ <path+ fill={FAIL_COLOR}+ d="M0 160V352L160 512H352L512 352V160L352 0H160L0 160zm353.9 32-64 64 64 64L320 353.9l-64-64-64 64L158.1 320l64-64-64-64L192 158.1l64 64 64-64z"+ />+ </svg>+ );+ return (+ <svg style={statusIcon} viewBox="0 0 512 512">+ <path+ fill={LIGHT_TEXT}+ d="M256 512C114.6 512 0 397.4 0 256S114.6 0 256 0s256 114.6 256 256-114.6 256-256 256zM232 120v136c0 8 4 15.5 10.7 20l96 64 26.6-39.9-85.3-56.9V120z"+ />+ </svg>+ );+};++export const VideoMattingQueueItem: React.FC<{+ readonly job: VideoMattingJob;+ readonly selected: boolean;+}> = ({job, selected}) => {+ const [hovered, setHovered] = useState(false);+ const {removeVideoMattingJob} = useContext(RenderQueueContext);+ const selectAsset = useSelectAsset();+ const done = job.status === 'done';+ const style = useMemo(+ () => ({+ ...container,+ backgroundColor: getBackgroundFromHoverState({+ hovered: done && hovered,+ selected,+ }),+ userSelect: 'none' as const,+ }),+ [done, hovered, selected],+ );+ const message =+ job.status === 'idle'+ ? 'Queued for video matting'+ : job.status === 'running'+ ? job.progress.message+ : job.status === 'failed'+ ? job.error.message+ : `${job.baseOutName} · ${job.foregroundOutName}`;+ const onClick = useCallback(() => {+ if (!done) return;+ selectAsset(job.foregroundOutName);+ pushUrl(`/assets/${job.foregroundOutName}`);+ }, [done, job.foregroundOutName, selectAsset]);+ const onRemove: React.MouseEventHandler = useCallback(+ (event) => {+ event.stopPropagation();+ removeVideoMattingJob(job.id);+ },+ [job.id, removeVideoMattingJob],+ );+ const renderRemove: RenderInlineAction = useCallback(+ (color) => (+ <svg style={removeIcon} viewBox="0 0 320 512">+ <path+ fill={color}+ d="M310.6 150.6 205.3 256l105.3 105.4-45.3 45.3L160 301.3 54.6 406.6 9.4 361.4 114.7 256 9.4 150.6l45.2-45.2L160 210.7l105.3-105.3z"+ />+ </svg>+ ),+ [],+ );+ return (+ <Row+ data-render-queue-item={job.id}+ style={style}+ align="center"+ onClick={onClick}+ onPointerEnter={() => setHovered(true)}+ onPointerLeave={() => setHovered(false)}+ >+ <Status job={job} />+ <Spacing x={1} />+ <div style={right}>+ <div style={title}>{job.displayName}</div>+ <span style={subtitle} title={message}>+ {message}+ </span>+ </div>+ <Spacing x={1} />+ {job.status === 'running' ? null : (+ <InlineAction+ renderAction={renderRemove}+ onClick={onRemove}+ title="Remove"+ variant={null}+ />+ )}+ </Row>+ );+};diff --git a/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsxnew file mode 100644index 0000000000..57e6b7d921--- /dev/null+++ b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx@@ -0,0 +1,121 @@+import {+ canUseVideoMatting,+ disposeVideoMattingModel,+ isVideoMattingModelCached,+ loadVideoMattingModel,+ separateVideoLayers,+} from '@remotion/video-matting';+import {useCallback, useContext, useEffect} from 'react';+import {writeStaticFile} from '../../api/write-static-file';+import {RenderQueueContext} from './context';+import {loadModelForJob} from './load-model-for-job';+import type {VideoMattingJob} from './video-matting-job-types';++export const VideoMattingQueueProcessor: React.FC = () => {+ const {+ markVideoMattingJobDone,+ markVideoMattingJobFailed,+ setProcessVideoMattingJobCallback,+ updateVideoMattingJobProgress,+ } = useContext(RenderQueueContext);++ const processJob = useCallback(+ async (job: VideoMattingJob) => {+ let outputs: Awaited<ReturnType<typeof separateVideoLayers>> | null =+ null;+ let processingError: Error | null = null;+ try {+ updateVideoMattingJobProgress(job.id, {+ message: 'Checking WebGPU support...',+ value: 0,+ });+ const support = await canUseVideoMatting({model: job.model});+ if (!support.supported) {+ throw new Error(support.detailedReason);+ }++ await loadModelForJob({+ model: job.model,+ progressStart: 0,+ progressSpan: 0.2,+ isModelCached: (model) => isVideoMattingModelCached({model}),+ loadModel: (model, onProgress) =>+ loadVideoMattingModel({+ model,+ onProgress: (progress) => onProgress(progress.progress),+ }),+ updateProgress: (progress) =>+ updateVideoMattingJobProgress(job.id, progress),+ });++ outputs = await separateVideoLayers({+ src: job.src,+ model: job.model,+ audio: job.audio,+ videoBitrate: job.videoBitrate,+ onProgress: (progress) => {+ updateVideoMattingJobProgress(job.id, {+ message:+ progress.stage === 'finalizing'+ ? 'Finalizing video layers...'+ : `Separating ${job.displayName}...`,+ value: 0.2 + (progress.progress ?? 1) * 0.65,+ });+ },+ });++ updateVideoMattingJobProgress(job.id, {+ message: 'Saving video layers...',+ value: 0.88,+ });+ const [base, foreground] = await Promise.all([+ outputs.base.getBlob(),+ outputs.foreground.getBlob(),+ ]);+ await Promise.all([+ base+ .arrayBuffer()+ .then((contents) =>+ writeStaticFile({contents, filePath: job.baseOutName}),+ ),+ foreground+ .arrayBuffer()+ .then((contents) =>+ writeStaticFile({contents, filePath: job.foregroundOutName}),+ ),+ ]);+ } catch (error) {+ processingError =+ error instanceof Error ? error : new Error(String(error));+ }++ try {+ await Promise.all([+ outputs?.base.dispose(),+ outputs?.foreground.dispose(),+ ]);+ await disposeVideoMattingModel({model: job.model});+ } catch {+ // Cleanup errors must not hide successfully written outputs.+ }++ if (processingError) {+ markVideoMattingJobFailed(job.id, processingError);+ } else {+ markVideoMattingJobDone(job.id);+ }+ },+ [+ markVideoMattingJobDone,+ markVideoMattingJobFailed,+ updateVideoMattingJobProgress,+ ],+ );++ useEffect(() => {+ setProcessVideoMattingJobCallback(processJob);+ return () => setProcessVideoMattingJobCallback(null);+ }, [processJob, setProcessVideoMattingJobCallback]);++ return null;+};diff --git a/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessorLoader.tsx b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessorLoader.tsxnew file mode 100644index 0000000000..c150af5a87--- /dev/null+++ b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessorLoader.tsx@@ -0,0 +1,24 @@+import React, {Suspense, useContext} from 'react';+import {useOptionalPackageInstalled} from '../OptionalPackageModal';+import {VIDEO_MATTING_PACKAGE} from '../VideoMatting/video-matting-capability';+import {RenderQueueContext} from './context';++const LazyProcessor = React.lazy(async () => {+ const {VideoMattingQueueProcessor} =+ await import('./VideoMattingQueueProcessor');+ return {default: VideoMattingQueueProcessor};+});++export const VideoMattingQueueProcessorLoader: React.FC = () => {+ const installed = useOptionalPackageInstalled(VIDEO_MATTING_PACKAGE);+ const {videoMattingJobs} = useContext(RenderQueueContext);+ if (!installed || videoMattingJobs.length === 0) {+ return null;+ }++ return (+ <Suspense fallback={null}>+ <LazyProcessor />+ </Suspense>+ );+};diff --git a/packages/studio/src/components/RenderQueue/context.tsx b/packages/studio/src/components/RenderQueue/context.tsxindex 86b7d3d503..83b3adade3 100644--- a/packages/studio/src/components/RenderQueue/context.tsx+++ b/packages/studio/src/components/RenderQueue/context.tsx@@ -33,6 +33,11 @@ import type {GetBlobCallback,RestoredClientRenderJob,} from './client-side-render-types';+import type {+ AddVideoMattingJobParams,+ VideoMattingJob,+ VideoMattingJobProgress,+} from './video-matting-job-types';declare global {interface Window {@@ -61,6 +66,7 @@ type RenderQueueContextType = {serverJobs: RenderJob[];clientJobs: ClientRenderJob[];captionJobs: CaptionJob[];+ videoMattingJobs: VideoMattingJob[];addClientStillJob: (params: AddClientStillJobParams,compositionRef: CompositionRef,@@ -97,6 +103,17 @@ type RenderQueueContextType = {setProcessCaptionJobCallback: (callback: ((job: CaptionJob) => Promise<void>) | null,) => void;+ addVideoMattingJob: (params: AddVideoMattingJobParams) => string;+ updateVideoMattingJobProgress: (+ jobId: string,+ progress: VideoMattingJobProgress,+ ) => void;+ markVideoMattingJobDone: (jobId: string) => void;+ markVideoMattingJobFailed: (jobId: string, error: Error) => void;+ removeVideoMattingJob: (jobId: string) => void;+ setProcessVideoMattingJobCallback: (+ callback: ((job: VideoMattingJob) => Promise<void>) | null,+ ) => void;getAbortController: (jobId: string) => AbortController;getCompositionForJob: (jobId: string) => CompositionRef | undefined;};@@ -109,6 +126,7 @@ export const RenderQueueContext = React.createContext<RenderQueueContextType>({serverJobs: [],clientJobs: [],captionJobs: [],+ videoMattingJobs: [],addClientStillJob: noopString,addClientVideoJob: noopString,updateClientJobProgress: noop,@@ -125,6 +143,12 @@ export const RenderQueueContext = React.createContext<RenderQueueContextType>({markCaptionJobFailed: noop,removeCaptionJob: noop,setProcessCaptionJobCallback: noop,+ addVideoMattingJob: noopString,+ updateVideoMattingJobProgress: noop,+ markVideoMattingJobDone: noop,+ markVideoMattingJobFailed: noop,+ removeVideoMattingJob: noop,+ setProcessVideoMattingJobCallback: noop,getAbortController: () => new AbortController(),getCompositionForJob: () => undefined,});@@ -144,6 +168,9 @@ export const RenderQueueContextProvider: React.FC<{restorePersistedClientRenders,);const [captionJobs, setCaptionJobs] = useState<CaptionJob[]>([]);+ const [videoMattingJobs, setVideoMattingJobs] = useState<VideoMattingJob[]>(+ [],+ );const [currentlyProcessing, setCurrentlyProcessing] = useState<string | null>(null,);@@ -156,6 +183,9 @@ export const RenderQueueContextProvider: React.FC<{const processCaptionJobCallbackRef = useRef<((job: CaptionJob) => Promise<void>) | null>(null);+ const processVideoMattingJobCallbackRef = useRef<+ ((job: VideoMattingJob) => Promise<void>) | null+ >(null);const [processorCallbacksVersion, setProcessorCallbacksVersion] = useState(0);const getLocalJobStartedAt = useCallback(() => {@@ -171,7 +201,7 @@ export const RenderQueueContextProvider: React.FC<{return;}- const nextJob = [...clientJobs, ...captionJobs]+ const nextJob = [...clientJobs, ...captionJobs, ...videoMattingJobs].filter((job) => job.status === 'idle').sort((a, b) => a.startedAt - b.startedAt)[0];if (!nextJob) {@@ -202,6 +232,27 @@ export const RenderQueueContextProvider: React.FC<{return;}+ if (nextJob.type === 'video-matting') {+ if (!processVideoMattingJobCallbackRef.current) {+ return;+ }++ setCurrentlyProcessing(nextJob.id);+ setVideoMattingJobs((prev) =>+ prev.map((job) =>+ job.id === nextJob.id+ ? {+ ...job,+ status: 'running',+ progress: {message: 'Starting video matting...', value: 0},+ }+ : job,+ ),+ );+ processVideoMattingJobCallbackRef.current(nextJob);+ return;+ }+if (!processJobCallbackRef.current) {return;}@@ -225,7 +276,13 @@ export const RenderQueueContextProvider: React.FC<{),);processJobCallbackRef.current(nextJob);- }, [captionJobs, clientJobs, currentlyProcessing, processorCallbacksVersion]);+ }, [+ captionJobs,+ clientJobs,+ currentlyProcessing,+ processorCallbacksVersion,+ videoMattingJobs,+ ]);const addClientStillJob = useCallback((@@ -457,6 +514,81 @@ export const RenderQueueContextProvider: React.FC<{[],);+ const addVideoMattingJob = useCallback(+ (params: AddVideoMattingJobParams): string => {+ const id = `video-matting-${Date.now()}-${Math.random()+ .toString(36)+ .substring(2, 11)}`;+ setVideoMattingJobs((prev) => [+ ...prev,+ {+ ...params,+ id,+ type: 'video-matting',+ startedAt: getLocalJobStartedAt(),+ status: 'idle',+ },+ ]);+ return id;+ },+ [getLocalJobStartedAt],+ );++ const updateVideoMattingJobProgress = useCallback(+ (jobId: string, progress: VideoMattingJobProgress): void => {+ setVideoMattingJobs((prev) =>+ prev.map((job) =>+ job.id === jobId ? {...job, status: 'running', progress} : job,+ ),+ );+ },+ [],+ );++ const markVideoMattingJobDone = useCallback((jobId: string): void => {+ setVideoMattingJobs((prev) =>+ prev.map((job) => (job.id === jobId ? {...job, status: 'done'} : job)),+ );+ setCurrentlyProcessing((current) => (current === jobId ? null : current));+ }, []);++ const markVideoMattingJobFailed = useCallback(+ (jobId: string, error: Error): void => {+ setVideoMattingJobs((prev) =>+ prev.map((job) =>+ job.id === jobId+ ? {+ ...job,+ status: 'failed',+ error: {message: error.message, stack: error.stack},+ }+ : job,+ ),+ );+ setCurrentlyProcessing((current) => (current === jobId ? null : current));+ },+ [],+ );++ const removeVideoMattingJob = useCallback((jobId: string): void => {+ setVideoMattingJobs((prev) => {+ const jobToRemove = prev.find((job) => job.id === jobId);+ if (jobToRemove?.status === 'running') {+ return prev;+ }++ return prev.filter((job) => job.id !== jobId);+ });+ }, []);++ const setProcessVideoMattingJobCallback = useCallback(+ (callback: ((job: VideoMattingJob) => Promise<void>) | null): void => {+ processVideoMattingJobCallbackRef.current = callback;+ setProcessorCallbacksVersion((version) => version + 1);+ },+ [],+ );+useImperativeHandle(renderJobsRef,() => ({@@ -498,6 +630,7 @@ export const RenderQueueContextProvider: React.FC<{serverJobs,clientJobs,captionJobs,+ videoMattingJobs,addClientStillJob,addClientVideoJob,updateClientJobProgress,@@ -514,6 +647,12 @@ export const RenderQueueContextProvider: React.FC<{markCaptionJobFailed,removeCaptionJob,setProcessCaptionJobCallback,+ addVideoMattingJob,+ updateVideoMattingJobProgress,+ markVideoMattingJobDone,+ markVideoMattingJobFailed,+ removeVideoMattingJob,+ setProcessVideoMattingJobCallback,getAbortController,getCompositionForJob,};@@ -521,6 +660,7 @@ export const RenderQueueContextProvider: React.FC<{serverJobs,clientJobs,captionJobs,+ videoMattingJobs,addClientStillJob,addClientVideoJob,updateClientJobProgress,@@ -537,6 +677,12 @@ export const RenderQueueContextProvider: React.FC<{markCaptionJobFailed,removeCaptionJob,setProcessCaptionJobCallback,+ addVideoMattingJob,+ updateVideoMattingJobProgress,+ markVideoMattingJobDone,+ markVideoMattingJobFailed,+ removeVideoMattingJob,+ setProcessVideoMattingJobCallback,]);return (diff --git a/packages/studio/src/components/RenderQueue/index.tsx b/packages/studio/src/components/RenderQueue/index.tsxindex 92bf4bb574..ae10ab843e 100644--- a/packages/studio/src/components/RenderQueue/index.tsx+++ b/packages/studio/src/components/RenderQueue/index.tsx@@ -7,6 +7,8 @@ import {isCaptionJob} from './caption-job-types';import {CaptionQueueItem} from './CaptionQueueItem';import {RenderQueueContext} from './context';import {RenderQueueItem} from './RenderQueueItem';+import {isVideoMattingJob} from './video-matting-job-types';+import {VideoMattingQueueItem} from './VideoMattingQueueItem';const errorExplanation: React.CSSProperties = {fontSize: 13,@@ -35,11 +37,14 @@ const renderQueue: React.CSSProperties = {};export const RenderQueue: React.FC = () => {- const {jobs, captionJobs} = useContext(RenderQueueContext);+ const {jobs, captionJobs, videoMattingJobs} = useContext(RenderQueueContext);const {canvasContent} = useContext(Internals.CompositionManager);const allJobs = useMemo(- () => [...jobs, ...captionJobs].sort((a, b) => a.startedAt - b.startedAt),- [jobs, captionJobs],+ () =>+ [...jobs, ...captionJobs, ...videoMattingJobs].sort(+ (a, b) => a.startedAt - b.startedAt,+ ),+ [jobs, captionJobs, videoMattingJobs],);const previousJobCount = React.useRef(allJobs.length);const jobCount = allJobs.length;@@ -70,7 +75,7 @@ export const RenderQueue: React.FC = () => {if (canvasContent.type === 'output') {for (let i = 0; i < allJobs.length; i++) {const job = allJobs[i];- if (isCaptionJob(job)) {+ if (isCaptionJob(job) || isVideoMattingJob(job)) {continue;}@@ -90,6 +95,15 @@ export const RenderQueue: React.FC = () => {) {return i;}++ if (+ isVideoMattingJob(job) &&+ job.status === 'done' &&+ (canvasContent.asset === job.baseOutName ||+ canvasContent.asset === job.foregroundOutName)+ ) {+ return i;+ }}}@@ -121,6 +135,12 @@ export const RenderQueue: React.FC = () => {job={job}selected={selectedJob === index}/>+ ) : isVideoMattingJob(job) ? (+ <VideoMattingQueueItem+ key={job.id}+ job={job}+ selected={selectedJob === index}+ />) : (<RenderQueueItemkey={job.id}diff --git a/packages/studio/src/components/RenderQueue/load-model-for-job.ts b/packages/studio/src/components/RenderQueue/load-model-for-job.tsnew file mode 100644index 0000000000..86acba1997--- /dev/null+++ b/packages/studio/src/components/RenderQueue/load-model-for-job.ts@@ -0,0 +1,28 @@+export const loadModelForJob = async <Model extends string>({+ isModelCached,+ loadModel,+ model,+ progressSpan,+ progressStart,+ updateProgress,+}: {+ isModelCached: (model: Model) => Promise<boolean>;+ loadModel: (+ model: Model,+ onProgress: (progress: number | null) => void,+ ) => Promise<unknown>;+ model: Model;+ progressSpan: number;+ progressStart: number;+ updateProgress: (progress: {message: string; value: number}) => void;+}) => {+ const cached = await isModelCached(model);+ await loadModel(model, (progress) => {+ const percentage =+ progress === null ? '' : ` ${Math.round(progress * 100)}%`;+ updateProgress({+ message: `${cached ? 'Loading' : 'Downloading'} ${model}${percentage}`,+ value: progressStart + (progress ?? 0) * progressSpan,+ });+ });+};diff --git a/packages/studio/src/components/RenderQueue/video-matting-job-types.ts b/packages/studio/src/components/RenderQueue/video-matting-job-types.tsnew file mode 100644index 0000000000..0c4fa18e53--- /dev/null+++ b/packages/studio/src/components/RenderQueue/video-matting-job-types.ts@@ -0,0 +1,36 @@+import type {+ VideoLayerAudio,+ VideoMattingBitrate,+ VideoMattingModel,+} from '@remotion/video-matting';++export type VideoMattingJobProgress = {+ message: string;+ value: number;+};++type VideoMattingJobStatus =+ | {status: 'idle'}+ | {status: 'running'; progress: VideoMattingJobProgress}+ | {status: 'done'}+ | {status: 'failed'; error: {message: string; stack: string | undefined}};++export type AddVideoMattingJobParams = {+ src: string;+ displayName: string;+ baseOutName: string;+ foregroundOutName: string;+ model: VideoMattingModel;+ audio: VideoLayerAudio;+ videoBitrate: VideoMattingBitrate;+};++export type VideoMattingJob = AddVideoMattingJobParams & {+ id: string;+ type: 'video-matting';+ startedAt: number;+} & VideoMattingJobStatus;++export const isVideoMattingJob = (job: {+ type: string;+}): job is VideoMattingJob => job.type === 'video-matting';diff --git a/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx b/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsxindex 38f4b492eb..b035bdcbb7 100644--- a/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx+++ b/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx@@ -1,145 +1,25 @@-import React, {Suspense, useCallback, useContext, useState} from 'react';-import {installPackages} from '../../api/install-package';-import {LIGHT_TEXT, WARNING_COLOR, WHITE} from '../../helpers/colors';+import React, {Suspense} from 'react';import type {TranscriptionModalState} from '../../state/modals';-import {SetSelectedModalContext} from '../../state/modals';-import {Button} from '../Button';-import {Flex, Row, Spacing} from '../layout';-import {ModalButton} from '../ModalButton';-import {ModalContainer} from '../ModalContainer';-import {ModalFooterContainer} from '../ModalFooter';-import {ModalHeader} from '../ModalHeader';-import {Spinner} from '../Spinner';-import {- isWhisperWebGpuInstalled,- WHISPER_WEBGPU_PACKAGE,-} from './whisper-webgpu-capability';+import {OptionalPackageModal} from '../OptionalPackageModal';+import {WHISPER_WEBGPU_PACKAGE} from './whisper-webgpu-capability';const LazyTranscriptionModal = React.lazy(async () => {const {TranscriptionModal} = await import('./TranscriptionModal');return {default: TranscriptionModal};});-type InstallState =- | {type: 'idle'}- | {type: 'installing'}- | {type: 'error'; error: Error};--const panelStyle: React.CSSProperties = {- width: 'min(520px, calc(100vw - 40px))',-};--const contentStyle: React.CSSProperties = {- color: WHITE,- fontFamily: 'sans-serif',- fontSize: 14,- lineHeight: 1.5,- padding: 16,-};--const codeStyle: React.CSSProperties = {- color: 'inherit',- fontFamily: 'monospace',- fontSize: 'inherit',-};--const statusStyle: React.CSSProperties = {- alignItems: 'center',- color: LIGHT_TEXT,- display: 'flex',- fontFamily: 'sans-serif',- fontSize: 14,- gap: 10,- lineHeight: 1.5,-};--const errorStyle: React.CSSProperties = {- color: WARNING_COLOR,- marginTop: 12,- whiteSpace: 'pre-wrap',-};--const cancelStyle: React.CSSProperties = {- minWidth: 90,-};-export const TranscriptionModalWithOptionalWhisper: React.FC<{readonly state: TranscriptionModalState;}> = ({state}) => {- const {setSelectedModal} = useContext(SetSelectedModalContext);- const [installed, setInstalled] = useState(isWhisperWebGpuInstalled);- const [installState, setInstallState] = useState<InstallState>({- type: 'idle',- });-- const busy = installState.type === 'installing';- const dismiss = useCallback(() => {- if (!busy) {- setSelectedModal(null);- }- }, [busy, setSelectedModal]);-- const install = useCallback(async () => {- setInstallState({type: 'installing'});- try {- await installPackages([{name: WHISPER_WEBGPU_PACKAGE, version: null}]);- window.remotion_installedPackages = Array.from(- new Set([- ...(window.remotion_installedPackages ?? []),- WHISPER_WEBGPU_PACKAGE,- ]),- );-- setInstalled(true);- } catch (error) {- setInstallState({type: 'error', error: error as Error});- }- }, []);-- if (installed) {- return (- <Suspense fallback={null}>- <LazyTranscriptionModal {...state} />- </Suspense>- );- }-return (- <ModalContainer+ <OptionalPackageModalariaLabel="Install transcription package"- onEscape={dismiss}- onOutsideClick={dismiss}- panelStyle={panelStyle}+ packageName={WHISPER_WEBGPU_PACKAGE}+ title="Transcribe">- <ModalHeader title="Transcribe" onClose={dismiss} />- <div style={contentStyle}>- {busy ? (- <div style={statusStyle}>- <Spinner duration={0.5} size={18} />- {`Installing ${WHISPER_WEBGPU_PACKAGE}…`}- </div>- ) : (- <>- This requires installing{' '}- <code style={codeStyle}>{WHISPER_WEBGPU_PACKAGE}</code>. Continue?- {installState.type === 'error' ? (- <div style={errorStyle}>{installState.error.message}</div>- ) : null}- </>- )}- </div>- <ModalFooterContainer>- <Row align="center">- <Flex />- <Button disabled={busy} onClick={dismiss} style={cancelStyle}>- Cancel- </Button>- <Spacing x={1} />- <ModalButton disabled={busy} onClick={install} autoFocus>- {installState.type === 'error' ? 'Retry' : 'Continue'}- </ModalButton>- </Row>- </ModalFooterContainer>- </ModalContainer>+ <Suspense fallback={null}>+ <LazyTranscriptionModal {...state} />+ </Suspense>+ </OptionalPackageModal>);};diff --git a/packages/studio/src/components/Transcription/caption-output-name.ts b/packages/studio/src/components/Transcription/caption-output-name.tsindex 458f2bb38a..06696e6b26 100644--- a/packages/studio/src/components/Transcription/caption-output-name.ts+++ b/packages/studio/src/components/Transcription/caption-output-name.ts@@ -1,86 +1,14 @@-const invalidCharacters = ['?', '*', '+', ':', '%'];+import {+ getDefaultOutputBaseName,+ validatePublicOutputName,+} from '../public-output-name';export const getDefaultCaptionOutputName = (src: string,displayName = 'captions',): string => {- let pathname = displayName;- if (!/^(data|blob):/i.test(src.trimStart())) {- pathname = src.split(/[?#]/)[0] ?? src;- try {- pathname = new URL(- src,- typeof window === 'undefined'- ? 'http://localhost'- : window.location.href,- ).pathname;- } catch {- // Fall back to the source as-is if it cannot be parsed as a URL.- }- }-- const lastPathSegment = pathname.split('/').filter(Boolean).at(-1);- let decoded = lastPathSegment ?? 'captions';- try {- decoded = decodeURIComponent(decoded);- } catch {- // Keep the encoded name if it contains an invalid escape sequence.- }-- const withoutExtension = decoded.replace(/\.[^/.]+$/, '');- const safeBaseName = withoutExtension- .replace(/[^a-zA-Z0-9-_ ]/g, '-')- .trim()- .replace(/\s+/g, '-')- .replace(/^-+|-+$/g, '');-- return `${safeBaseName || 'captions'}-captions.json`;+ return `${getDefaultOutputBaseName(src, displayName, 'captions')}-captions.json`;};-export const validateCaptionOutputName = (outName: string): string | null => {- if (outName.trim() === '') {- return 'Enter an output file';- }-- if (outName !== outName.trim()) {- return 'The output file must not start or end with whitespace';- }-- if (outName.startsWith('/') || /^[a-zA-Z]:/.test(outName)) {- return 'The output file must be relative to the public folder';- }-- if (outName.includes('\\')) {- return 'Use forward slashes in the output file';- }-- if (outName.includes('\0')) {- return 'The output file must not contain null characters';- }-- const segments = outName.split('/');- if (segments.some((segment) => segment === '')) {- return 'The output file must not contain empty path segments';- }-- if (segments.some((segment) => segment === '.')) {- return 'The output file must not contain "." path segments';- }-- if (segments.some((segment) => segment.startsWith('..'))) {- return 'The output file must stay inside the public folder';- }-- const invalidCharacter = invalidCharacters.find((character) =>- outName.includes(character),- );- if (invalidCharacter) {- return `The output file must not contain "${invalidCharacter}"`;- }-- if (!outName.toLowerCase().endsWith('.json')) {- return 'The output file must end in .json';- }-- return null;-};+export const validateCaptionOutputName = (outName: string): string | null =>+ validatePublicOutputName({extension: '.json', outName});diff --git a/packages/studio/src/components/Transcription/whisper-webgpu-capability.ts b/packages/studio/src/components/Transcription/whisper-webgpu-capability.tsindex 46a9d8ee6d..0e9564bf3d 100644--- a/packages/studio/src/components/Transcription/whisper-webgpu-capability.ts+++ b/packages/studio/src/components/Transcription/whisper-webgpu-capability.ts@@ -1,7 +1,6 @@+import {isOptionalPackageInstalled} from '../OptionalPackageModal';+export const WHISPER_WEBGPU_PACKAGE = '@remotion/whisper-webgpu';-export const isWhisperWebGpuInstalled = () => {- return (- window.remotion_installedPackages?.includes(WHISPER_WEBGPU_PACKAGE) ?? false- );-};+export const isWhisperWebGpuInstalled = () =>+ isOptionalPackageInstalled(WHISPER_WEBGPU_PACKAGE);diff --git a/packages/studio/src/components/VideoMatting/VideoMattingModal.tsx b/packages/studio/src/components/VideoMatting/VideoMattingModal.tsxnew file mode 100644index 0000000000..2b5dfb9e52--- /dev/null+++ b/packages/studio/src/components/VideoMatting/VideoMattingModal.tsx@@ -0,0 +1,380 @@+import {formatBytes} from '@remotion/studio-shared';+import {+ canUseVideoMatting,+ getAvailableModels,+ type VideoLayerAudio,+ type VideoMattingBitrate,+ type VideoMattingModel,+} from '@remotion/video-matting';+import React, {+ useCallback,+ useContext,+ useEffect,+ useMemo,+ useState,+} from 'react';+import {BLUE_DISABLED, WHITE} from '../../helpers/colors';+import {Checkmark} from '../../icons/Checkmark';+import type {VideoMattingModalState} from '../../state/modals';+import {SetSelectedModalContext} from '../../state/modals';+import {SidebarContext} from '../../state/sidebar';+import {Button} from '../Button';+import {VERTICAL_SCROLLBAR_CLASSNAME} from '../Menu/is-menu-item';+import {ModalHeader} from '../ModalHeader';+import type {ComboboxValue} from '../NewComposition/ComboBox';+import {Combobox} from '../NewComposition/ComboBox';+import {DismissableModal} from '../NewComposition/DismissableModal';+import {RemotionInput} from '../NewComposition/RemInput';+import {ValidationMessage} from '../NewComposition/ValidationMessage';+import {optionsSidebarTabs} from '../options-sidebar-tabs';+import {persistSelectedOptionsSidebarPanel} from '../OptionsPanel';+import {+ getDefaultOutputBaseName,+ validatePublicOutputName,+} from '../public-output-name';+import {input, label, optionRow, rightRow} from '../RenderModal/layout';+import {+ buttonStyle,+ container,+ flexer,+ optionsPanel,+ outerModalStyle,+} from '../RenderModal/render-modals';+import {RenderModalHr} from '../RenderModal/RenderModalHr';+import {RenderQueueContext} from '../RenderQueue/context';+import {useStaticFiles} from '../use-static-files';++const MODELS = getAvailableModels();+const controlStyle: React.CSSProperties = {width: 330, maxWidth: '100%'};+const panelStyle: React.CSSProperties = {+ ...optionsPanel,+ flexDirection: 'column',+ paddingTop: 16,+};+const modalStyle: React.CSSProperties = {+ ...outerModalStyle,+ height: 'auto',+ maxHeight: 'calc(100vh - 40px)',+ outline: 'none',+};+const outputRow: React.CSSProperties = {...optionRow, alignItems: 'flex-start'};+const fieldLabel: React.CSSProperties = {+ color: WHITE,+ fontFamily: 'sans-serif',+ fontSize: 13,+ lineHeight: '32px',+};+const validationStyle: React.CSSProperties = {padding: '0 16px 8px'};++type SupportState =+ | {type: 'checking'}+ | {type: 'supported'}+ | {type: 'unsupported'; message: string};++const makeOptions = <Value extends string>({+ items,+ selected,+ setSelected,+}: {+ items: readonly {id: Value; label: React.ReactNode}[];+ selected: Value;+ setSelected: (value: Value) => void;+}): ComboboxValue[] =>+ items.map(({id, label: optionLabel}) => ({+ type: 'item',+ id,+ value: id,+ label: optionLabel,+ leftItem: id === selected ? <Checkmark /> : null,+ keyHint: null,+ quickSwitcherLabel: null,+ subMenu: null,+ disabled: false,+ onClick: () => setSelected(id),+ }));++export const VideoMattingModal: React.FC<VideoMattingModalState> = ({+ displayName,+ src,+}) => {+ const baseName = useMemo(+ () => getDefaultOutputBaseName(src, displayName, 'video'),+ [displayName, src],+ );+ const [baseOutName, setBaseOutName] = useState(`${baseName}-base.webm`);+ const [foregroundOutName, setForegroundOutName] = useState(+ `${baseName}-foreground.webm`,+ );+ const [model, setModel] = useState<VideoMattingModel>('modnet');+ const [audio, setAudio] = useState<VideoLayerAudio>('base');+ const [videoBitrate, setVideoBitrate] =+ useState<VideoMattingBitrate>('very-high');+ const [support, setSupport] = useState<SupportState>({type: 'checking'});+ const staticFiles = useStaticFiles();+ const {addVideoMattingJob, videoMattingJobs} = useContext(RenderQueueContext);+ const {setSelectedModal} = useContext(SetSelectedModalContext);+ const {setSidebarCollapsedState} = useContext(SidebarContext);++ useEffect(() => {+ let cancelled = false;+ setSupport({type: 'checking'});+ canUseVideoMatting({model}).then((result) => {+ if (!cancelled) {+ setSupport(+ result.supported+ ? {type: 'supported'}+ : {type: 'unsupported', message: result.detailedReason},+ );+ }+ });+ return () => {+ cancelled = true;+ };+ }, [model]);++ const normalizedBase = baseOutName.normalize('NFC').toLowerCase();+ const normalizedForeground = foregroundOutName.normalize('NFC').toLowerCase();+ const duplicateOutput = normalizedBase === normalizedForeground;+ const queuedOutputs = new Set(+ videoMattingJobs+ .filter((job) => job.status === 'idle' || job.status === 'running')+ .flatMap((job) => [job.baseOutName, job.foregroundOutName])+ .map((name) => name.normalize('NFC').toLowerCase()),+ );+ const baseError =+ validatePublicOutputName({extension: '.webm', outName: baseOutName}) ??+ (duplicateOutput+ ? 'Base and foreground outputs must be different'+ : queuedOutputs.has(normalizedBase)+ ? 'Another video matting job is using this output file'+ : null);+ const foregroundError =+ validatePublicOutputName({+ extension: '.webm',+ outName: foregroundOutName,+ }) ??+ (duplicateOutput+ ? 'Base and foreground outputs must be different'+ : queuedOutputs.has(normalizedForeground)+ ? 'Another video matting job is using this output file'+ : null);+ const baseExists = staticFiles.some(+ (file) => file.name.normalize('NFC').toLowerCase() === normalizedBase,+ );+ const foregroundExists = staticFiles.some(+ (file) => file.name.normalize('NFC').toLowerCase() === normalizedForeground,+ );+ const canSubmit =+ support.type === 'supported' &&+ baseError === null &&+ foregroundError === null;++ const modelOptions = useMemo(+ () =>+ makeOptions({+ items: MODELS.map((item) => ({+ id: item.name,+ label: `${item.name} · ${formatBytes(item.webGpuDownloadSize)}`,+ })),+ selected: model,+ setSelected: setModel,+ }),+ [model],+ );+ const audioOptions = useMemo(+ () =>+ makeOptions({+ items: [+ {id: 'base' as const, label: 'Base layer'},+ {id: 'foreground' as const, label: 'Foreground layer'},+ {id: 'both' as const, label: 'Both layers'},+ {id: 'none' as const, label: 'No audio'},+ ],+ selected: audio,+ setSelected: setAudio,+ }),+ [audio],+ );+ const qualityOptions = useMemo(+ () =>+ makeOptions({+ items: ['very-low', 'low', 'medium', 'high', 'very-high'].map((id) => ({+ id: id as VideoMattingBitrate & string,+ label: id+ .split('-')+ .map((part) => part[0]!.toUpperCase() + part.slice(1))+ .join(' '),+ })),+ selected: videoBitrate as VideoMattingBitrate & string,+ setSelected: setVideoBitrate,+ }),+ [videoBitrate],+ );++ const submit = useCallback(() => {+ if (!canSubmit) return;+ addVideoMattingJob({+ src,+ displayName,+ baseOutName,+ foregroundOutName,+ model,+ audio,+ videoBitrate,+ });+ setSidebarCollapsedState({left: null, right: 'expanded'});+ persistSelectedOptionsSidebarPanel('renders');+ optionsSidebarTabs.current?.selectRendersPanel();+ setSelectedModal(null);+ }, [+ addVideoMattingJob,+ audio,+ baseOutName,+ canSubmit,+ displayName,+ foregroundOutName,+ model,+ setSelectedModal,+ setSidebarCollapsedState,+ src,+ videoBitrate,+ ]);++ return (+ <DismissableModal ariaLabel={`Track matting ${displayName}`}>+ <div style={modalStyle}>+ <ModalHeader title={`Track matting ${displayName}`} />+ <div style={container}>+ <div style={flexer} />+ <Button+ disabled={!canSubmit}+ onClick={submit}+ title={support.type === 'unsupported' ? support.message : undefined}+ style={{+ ...buttonStyle,+ backgroundColor: canSubmit+ ? buttonStyle.backgroundColor+ : BLUE_DISABLED,+ }}+ >+ Track matting+ </Button>+ </div>+ <div style={panelStyle} className={VERTICAL_SCROLLBAR_CLASSNAME}>+ <div style={outputRow}>+ <div style={fieldLabel}>Base output in public/</div>+ <div style={rightRow}>+ <RemotionInput+ aria-label="Base video output file"+ status={baseError ? 'error' : baseExists ? 'warning' : 'ok'}+ rightAlign+ value={baseOutName}+ onChange={(event) => setBaseOutName(event.target.value)}+ style={{...input, ...controlStyle}}+ />+ </div>+ </div>+ {baseError ? (+ <div style={validationStyle}>+ <ValidationMessage+ align="flex-end"+ message={baseError}+ type="error"+ />+ </div>+ ) : baseExists ? (+ <div style={validationStyle}>+ <ValidationMessage+ align="flex-end"+ message="Exists, will be overwritten"+ type="warning"+ />+ </div>+ ) : null}+ <div style={outputRow}>+ <div style={fieldLabel}>Foreground output in public/</div>+ <div style={rightRow}>+ <RemotionInput+ aria-label="Foreground video output file"+ status={+ foregroundError+ ? 'error'+ : foregroundExists+ ? 'warning'+ : 'ok'+ }+ rightAlign+ value={foregroundOutName}+ onChange={(event) => setForegroundOutName(event.target.value)}+ style={{...input, ...controlStyle}}+ />+ </div>+ </div>+ {foregroundError ? (+ <div style={validationStyle}>+ <ValidationMessage+ align="flex-end"+ message={foregroundError}+ type="error"+ />+ </div>+ ) : foregroundExists ? (+ <div style={validationStyle}>+ <ValidationMessage+ align="flex-end"+ message="Exists, will be overwritten"+ type="warning"+ />+ </div>+ ) : null}+ <RenderModalHr />+ <div style={optionRow}>+ <div style={label}>Model</div>+ <div style={rightRow}>+ <Combobox+ values={modelOptions}+ selectedId={model}+ title="Model"+ disabled={false}+ style={controlStyle}+ />+ </div>+ </div>+ {support.type === 'unsupported' ? (+ <div style={validationStyle}>+ <ValidationMessage+ align="flex-end"+ message={support.message}+ type="error"+ />+ </div>+ ) : null}+ <div style={optionRow}>+ <div style={label}>Audio</div>+ <div style={rightRow}>+ <Combobox+ values={audioOptions}+ selectedId={audio}+ title="Audio"+ disabled={false}+ style={controlStyle}+ />+ </div>+ </div>+ <div style={optionRow}>+ <div style={label}>Video quality</div>+ <div style={rightRow}>+ <Combobox+ values={qualityOptions}+ selectedId={String(videoBitrate)}+ title="Video quality"+ disabled={false}+ style={controlStyle}+ />+ </div>+ </div>+ </div>+ </div>+ </DismissableModal>+ );+};diff --git a/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx b/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsxnew file mode 100644index 0000000000..43bb871e7d--- /dev/null+++ b/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx@@ -0,0 +1,23 @@+import React, {Suspense} from 'react';+import type {VideoMattingModalState} from '../../state/modals';+import {OptionalPackageModal} from '../OptionalPackageModal';+import {VIDEO_MATTING_PACKAGE} from './video-matting-capability';++const LazyVideoMattingModal = React.lazy(async () => {+ const {VideoMattingModal} = await import('./VideoMattingModal');+ return {default: VideoMattingModal};+});++export const VideoMattingModalWithOptionalPackage: React.FC<{+ readonly state: VideoMattingModalState;+}> = ({state}) => (+ <OptionalPackageModal+ ariaLabel="Install video matting package"+ packageName={VIDEO_MATTING_PACKAGE}+ title="Track matting"+ >+ <Suspense fallback={null}>+ <LazyVideoMattingModal {...state} />+ </Suspense>+ </OptionalPackageModal>+);diff --git a/packages/studio/src/components/VideoMatting/video-matting-capability.ts b/packages/studio/src/components/VideoMatting/video-matting-capability.tsnew file mode 100644index 0000000000..66b376e54f--- /dev/null+++ b/packages/studio/src/components/VideoMatting/video-matting-capability.ts@@ -0,0 +1,6 @@+import {isOptionalPackageInstalled} from '../OptionalPackageModal';++export const VIDEO_MATTING_PACKAGE = '@remotion/video-matting';++export const isVideoMattingInstalled = () =>+ isOptionalPackageInstalled(VIDEO_MATTING_PACKAGE);diff --git a/packages/studio/src/components/public-output-name.ts b/packages/studio/src/components/public-output-name.tsnew file mode 100644index 0000000000..82f2e192f4--- /dev/null+++ b/packages/studio/src/components/public-output-name.ts@@ -0,0 +1,73 @@+const invalidCharacters = ['?', '*', '+', ':', '%'];++export const getDefaultOutputBaseName = (+ src: string,+ displayName: string,+ fallbackName: string,+): string => {+ let pathname = displayName;+ if (!/^(data|blob):/i.test(src.trimStart())) {+ pathname = src.split(/[?#]/)[0] ?? src;+ try {+ pathname = new URL(+ src,+ typeof window === 'undefined'+ ? 'http://localhost'+ : window.location.href,+ ).pathname;+ } catch {+ // Fall back to the source as-is if it cannot be parsed as a URL.+ }+ }++ const lastPathSegment = pathname.split('/').filter(Boolean).at(-1);+ let decoded = lastPathSegment ?? displayName;+ try {+ decoded = decodeURIComponent(decoded);+ } catch {+ // Keep the encoded name if it contains an invalid escape sequence.+ }++ return (+ decoded+ .replace(/\.[^/.]+$/, '')+ .replace(/[^a-zA-Z0-9-_ ]/g, '-')+ .trim()+ .replace(/\s+/g, '-')+ .replace(/^-+|-+$/g, '') || fallbackName+ );+};++export const validatePublicOutputName = ({+ extension,+ outName,+}: {+ extension: string;+ outName: string;+}): string | null => {+ if (outName.trim() === '') return 'Enter an output file';+ if (outName !== outName.trim())+ return 'The output file must not start or end with whitespace';+ if (outName.startsWith('/') || /^[a-zA-Z]:/.test(outName))+ return 'The output file must be relative to the public folder';+ if (outName.includes('\\')) return 'Use forward slashes in the output file';+ if (outName.includes('\0'))+ return 'The output file must not contain null characters';++ const segments = outName.split('/');+ if (segments.some((segment) => segment === ''))+ return 'The output file must not contain empty path segments';+ if (segments.some((segment) => segment === '.'))+ return 'The output file must not contain "." path segments';+ if (segments.some((segment) => segment.startsWith('..')))+ return 'The output file must stay inside the public folder';++ const invalidCharacter = invalidCharacters.find((character) =>+ outName.includes(character),+ );+ if (invalidCharacter)+ return `The output file must not contain "${invalidCharacter}"`;+ if (!outName.toLowerCase().endsWith(extension))+ return `The output file must end in ${extension}`;+ return null;+};diff --git a/packages/studio/src/state/modals.ts b/packages/studio/src/state/modals.tsindex 1e2f148c45..bfd2abad41 100644--- a/packages/studio/src/state/modals.ts+++ b/packages/studio/src/state/modals.ts@@ -148,6 +148,12 @@ export type TranscriptionModalState = {requestInit: Omit<RequestInit, 'signal'> | null;};+export type VideoMattingModalState = {+ type: 'video-matting';+ src: string;+ displayName: string;+};+export type CanvasCaptureImport = {readonly data: CanvasCaptureData;readonly durationInSeconds: number;@@ -264,6 +270,7 @@ export type ModalState =| ElementInstallModalState| AddEffectModalState| TranscriptionModalState+ | VideoMattingModalState| ConfirmationDialogState| SvgImportDialogState;diff --git a/packages/studio/src/test/video-matting-queue-processor.test.tsx b/packages/studio/src/test/video-matting-queue-processor.test.tsxnew file mode 100644index 0000000000..6b34484bf2--- /dev/null+++ b/packages/studio/src/test/video-matting-queue-processor.test.tsx@@ -0,0 +1,129 @@+import {afterEach, expect, mock, test} from 'bun:test';+import {act, cleanup, render, waitFor} from '@testing-library/react';+import type {ContextType} from 'react';+import {RenderQueueContext} from '../components/RenderQueue/context';+import type {VideoMattingJob} from '../components/RenderQueue/video-matting-job-types';+import {makeBrowserStudioOperations} from './make-browser-studio-operations';++const calls: string[] = [];+let processJob: ((job: VideoMattingJob) => Promise<void>) | null = null;++mock.module('@remotion/video-matting', () => ({+ canUseVideoMatting: () => Promise.resolve({supported: true}),+ isVideoMattingModelCached: () => Promise.resolve(false),+ loadVideoMattingModel: ({+ onProgress,+ }: {+ onProgress: (progress: {progress: number}) => void;+ }) => {+ calls.push('load-model');+ onProgress({progress: 1});+ return Promise.resolve({alreadyLoaded: false});+ },+ separateVideoLayers: ({+ onProgress,+ }: {+ onProgress: (progress: {stage: string; progress: number}) => void;+ }) => {+ calls.push('separate');+ onProgress({stage: 'processing', progress: 0.5});+ return Promise.resolve({+ base: {+ getBlob: () => Promise.resolve(new Blob(['base'])),+ dispose: () => Promise.resolve(),+ },+ foreground: {+ getBlob: () => Promise.resolve(new Blob(['foreground'])),+ dispose: () => Promise.resolve(),+ },+ });+ },+ disposeVideoMattingModel: () => {+ calls.push('dispose-model');+ return Promise.resolve();+ },+}));++afterEach(() => {+ cleanup();+ calls.length = 0;+ processJob = null;+});++test('loads the model, separates the layers and writes both outputs', async () => {+ const originalBrowserStudio = Object.getOwnPropertyDescriptor(+ window,+ 'remotion_browserStudio',+ );+ Object.defineProperty(window, 'remotion_browserStudio', {+ configurable: true,+ value: makeBrowserStudioOperations({+ writeStaticFile: ({filePath}) => {+ calls.push(`write:${filePath}`);+ return Promise.resolve();+ },+ }),+ });+ const progress: string[] = [];+ let done = false;+ let failed: Error | null = null;+ const value = {+ setProcessVideoMattingJobCallback: (callback: typeof processJob) => {+ processJob = callback;+ },+ updateVideoMattingJobProgress: (_id: string, update: {message: string}) => {+ progress.push(update.message);+ },+ markVideoMattingJobDone: () => {+ done = true;+ },+ markVideoMattingJobFailed: (_id: string, error: Error) => {+ failed = error;+ },+ } as unknown as ContextType<typeof RenderQueueContext>;++ const {VideoMattingQueueProcessor} =+ await import('../components/RenderQueue/VideoMattingQueueProcessor');+ render(+ <RenderQueueContext.Provider value={value}>+ <VideoMattingQueueProcessor />+ </RenderQueueContext.Provider>,+ );+ await waitFor(() => expect(processJob).not.toBeNull());++ await act(() =>+ processJob?.({+ id: 'matting-1',+ type: 'video-matting',+ status: 'idle',+ startedAt: 1,+ src: '/input.webm',+ displayName: 'input.webm',+ baseOutName: 'input-base.webm',+ foregroundOutName: 'input-foreground.webm',+ model: 'modnet',+ audio: 'base',+ videoBitrate: 'very-high',+ }),+ );++ expect(failed).toBeNull();+ expect(done).toBe(true);+ expect(calls).toEqual([+ 'load-model',+ 'separate',+ 'write:input-base.webm',+ 'write:input-foreground.webm',+ 'dispose-model',+ ]);+ expect(progress).toContain('Downloading modnet 100%');+ if (originalBrowserStudio) {+ Object.defineProperty(+ window,+ 'remotion_browserStudio',+ originalBrowserStudio,+ );+ } else {+ Reflect.deleteProperty(window, 'remotion_browserStudio');+ }+});diff --git a/packages/studio/src/test/whisper-webgpu-capability.test.tsx b/packages/studio/src/test/whisper-webgpu-capability.test.tsxindex 6f02eaffe9..78238fbcb0 100644--- a/packages/studio/src/test/whisper-webgpu-capability.test.tsx+++ b/packages/studio/src/test/whisper-webgpu-capability.test.tsx@@ -6,6 +6,8 @@ import {isWhisperWebGpuInstalled,WHISPER_WEBGPU_PACKAGE,} from '../components/Transcription/whisper-webgpu-capability';+import {VIDEO_MATTING_PACKAGE} from '../components/VideoMatting/video-matting-capability';+import {VideoMattingModalWithOptionalPackage} from '../components/VideoMatting/VideoMattingModalWithOptionalPackage';const originalInstalledPackages = window.remotion_installedPackages;@@ -43,6 +45,26 @@ test('asks before installing Whisper for transcription', () => {expect(getByRole('button', {name: 'Continue'})).toBeDefined();});+test('uses the same install confirmation for video matting', () => {+ window.remotion_installedPackages = [];+ const {container, getByRole} = render(+ <ModalsProvider>+ <VideoMattingModalWithOptionalPackage+ state={{+ type: 'video-matting',+ src: '/video.webm',+ displayName: 'video.webm',+ }}+ />+ </ModalsProvider>,+ );++ expect(container.textContent).toContain(+ `This requires installing ${VIDEO_MATTING_PACKAGE}. Continue?`,+ );+ expect(getByRole('button', {name: 'Continue'})).toBeDefined();+});+test('opens transcription after installing Whisper without restarting', async () => {const originalIsStudio = window.remotion_isStudio;const originalIsReadOnlyStudio = window.remotion_isReadOnlyStudio;diff --git a/packages/studio/tsconfig.json b/packages/studio/tsconfig.jsonindex 56fe86b3c2..49796a7c5a 100644--- a/packages/studio/tsconfig.json+++ b/packages/studio/tsconfig.json@@ -44,6 +44,9 @@},{"path": "../whisper-webgpu"+ },+ {+ "path": "../video-matting"}]}