Skip to content

🎥 Make videos programmatically with React

Add video matting to Studio

JonnyBurgerb0d5529904f4
diff --git a/bun.lock b/bun.lock
index bd4cfcee73..f1fb5027be 100644
--- a/bun.lock
+++ b/bun.lock
@@ -1829,6 +1829,7 @@
"devDependencies": {
"@happy-dom/global-registrator": "14.5.1",
"@remotion/eslint-config-internal": "workspace:*",
+ "@remotion/video-matting": "workspace:*",
"@remotion/whisper-webgpu": "workspace:*",
"@testing-library/react": "16.1.0",
"@types/semver": "7.5.3",
@@ -1838,11 +1839,13 @@
"react-dom": "catalog:",
},
"peerDependencies": {
+ "@remotion/video-matting": "workspace:*",
"@remotion/whisper-webgpu": "workspace:*",
"react": ">=16.8.0",
"react-dom": ">=16.8.0",
},
"optionalPeers": [
+ "@remotion/video-matting",
"@remotion/whisper-webgpu",
],
},
diff --git a/packages/example/e2e/transcription-modal.test.mts b/packages/example/e2e/transcription-modal.test.mts
index 266039238b..c393b1dbad 100644
--- a/packages/example/e2e/transcription-modal.test.mts
+++ b/packages/example/e2e/transcription-modal.test.mts
@@ -46,9 +46,35 @@ test.describe('transcription modal', () => {
await expect(actions.getByRole('button')).toHaveText([
/^Show in (Finder|File Explorer|File Manager)$/,
'Transcribe',
+ 'Track matting',
'Convert',
'Delete',
]);
+ await page.evaluate(() => {
+ window.remotion_installedPackages = [
+ ...(window.remotion_installedPackages ?? []),
+ '@remotion/video-matting',
+ ];
+ });
+ await actions.getByRole('button', {name: 'Track matting'}).click();
+ const mattingDialog = page.getByRole('dialog', {
+ name: 'Track matting vp8-vorbis.webm',
+ });
+ await expect(mattingDialog.getByTitle('Model')).toContainText('modnet');
+ await expect(
+ mattingDialog.getByRole('textbox', {name: 'Base video output file'}),
+ ).toHaveValue(
+ 'vp8-vorbis-base.webm',
+ );
+ await expect(
+ mattingDialog.getByRole('textbox', {
+ name: 'Foreground video output file',
+ }),
+ ).toHaveValue(
+ 'vp8-vorbis-foreground.webm',
+ );
+ await page.keyboard.press('Escape');
+ await expect(mattingDialog).toBeHidden();
await transcribe.click();
const dialog = page.getByRole('dialog');
diff --git a/packages/studio/bundle.ts b/packages/studio/bundle.ts
index 9ae5a711c4..f31e221aa5 100644
--- a/packages/studio/bundle.ts
+++ b/packages/studio/bundle.ts
@@ -17,6 +17,7 @@ const external = [
'@remotion/renderer/pure',
'@remotion/web-renderer',
'@remotion/whisper-webgpu',
+ '@remotion/video-matting',
'@remotion/renderer/error-handling',
'@jridgewell/trace-mapping',
'zod',
diff --git a/packages/studio/package.json b/packages/studio/package.json
index ddc8f71ce5..86010c4e49 100644
--- a/packages/studio/package.json
+++ b/packages/studio/package.json
@@ -20,11 +20,15 @@
"url": "https://github.com/remotion-dev/remotion/issues"
},
"peerDependencies": {
+ "@remotion/video-matting": "workspace:*",
"@remotion/whisper-webgpu": "workspace:*",
"react": ">=16.8.0",
"react-dom": ">=16.8.0"
},
"peerDependenciesMeta": {
+ "@remotion/video-matting": {
+ "optional": true
+ },
"@remotion/whisper-webgpu": {
"optional": true
}
@@ -54,6 +58,7 @@
},
"devDependencies": {
"@happy-dom/global-registrator": "14.5.1",
+ "@remotion/video-matting": "workspace:*",
"@remotion/whisper-webgpu": "workspace:*",
"react": "catalog:",
"react-dom": "catalog:",
diff --git a/packages/studio/src/components/CurrentAsset.tsx b/packages/studio/src/components/CurrentAsset.tsx
index 6bc2896982..37f6e574de 100644
--- a/packages/studio/src/components/CurrentAsset.tsx
+++ b/packages/studio/src/components/CurrentAsset.tsx
@@ -15,6 +15,7 @@ import {
import {useImageMetadata} from '../helpers/use-image-metadata';
import type {MediaMetadata} from '../helpers/use-media-metadata';
import {useMediaMetadata} from '../helpers/use-media-metadata';
+import {EffectsIcon} from '../icons/effects';
import {ExpandedFolderIcon} from '../icons/folder';
import {RemotionConvertIcon} from '../icons/remotion-convert';
import {TranscriptionIcon} from '../icons/transcription';
@@ -210,6 +211,7 @@ export const AssetInfo: React.FC<{
browserStudioOperations === null &&
(readOnlyStudio || connectionStatus !== 'connected');
const fileName = assetName?.split('/').pop() ?? '';
+ const fileType = assetName ? getPreviewFileType(assetName) : null;
const onTranscribe = useCallback(() => {
if (src === null || mutationsDisabled) {
return;
@@ -223,6 +225,17 @@ export const AssetInfo: React.FC<{
requestInit: null,
});
}, [fileName, mutationsDisabled, setSelectedModal, src]);
+ const onTrackMatting = useCallback(() => {
+ if (src === null || fileType !== 'video' || mutationsDisabled) {
+ return;
+ }
+
+ setSelectedModal({
+ type: 'video-matting',
+ src,
+ displayName: fileName,
+ });
+ }, [fileName, fileType, mutationsDisabled, setSelectedModal, src]);
const canRename =
onAssetClick === undefined &&
(browserStudioOperations !== null ||
@@ -373,6 +386,17 @@ export const AssetInfo: React.FC<{
Transcribe
</InspectorQuickAction>
) : null}
+ {fileType === 'video' ? (
+ <InspectorQuickAction
+ disabled={mutationsDisabled}
+ onClick={onTrackMatting}
+ renderIcon={(color) => (
+ <EffectsIcon color={color} style={quickActionIconStyle} />
+ )}
+ >
+ Track matting
+ </InspectorQuickAction>
+ ) : null}
{src ? (
<InspectorQuickAction
disabled={false}
diff --git a/packages/studio/src/components/EditorContexts.tsx b/packages/studio/src/components/EditorContexts.tsx
index 0999b4a695..db16dc7d07 100644
--- a/packages/studio/src/components/EditorContexts.tsx
+++ b/packages/studio/src/components/EditorContexts.tsx
@@ -15,6 +15,7 @@ import {ModalsProvider} from './ModalsProvider';
import {CaptionQueueProcessorLoader} from './RenderQueue/CaptionQueueProcessorLoader';
import {ClientRenderQueueProcessor} from './RenderQueue/ClientRenderQueueProcessor';
import {RenderQueueContextProvider} from './RenderQueue/context';
+import {VideoMattingQueueProcessorLoader} from './RenderQueue/VideoMattingQueueProcessorLoader';
import {SetTimelineInOutProvider} from './SetTimelineInOutProvider';
import {SettingsProvider} from './SettingsContext';
import {ShowGuidesProvider} from './ShowGuidesProvider';
@@ -37,6 +38,7 @@ export const EditorContexts: React.FC<{
<RenderQueueContextProvider>
<ClientRenderQueueProcessor />
<CaptionQueueProcessorLoader />
+ <VideoMattingQueueProcessorLoader />
<KeybindingContextProvider>
<CheckerboardProvider>
<ZoomGesturesProvider>
diff --git a/packages/studio/src/components/Modals.tsx b/packages/studio/src/components/Modals.tsx
index 968d9f2c5c..bf19c856d3 100644
--- a/packages/studio/src/components/Modals.tsx
+++ b/packages/studio/src/components/Modals.tsx
@@ -30,6 +30,7 @@ import {WebRenderModalWithLoader} from './RenderModal/WebRenderModal';
import {SettingsModal} from './SettingsModal';
import {SvgImportDialog} from './SvgImportDialog';
import {TranscriptionModalWithOptionalWhisper} from './Transcription/TranscriptionModalWithOptionalWhisper';
+import {VideoMattingModalWithOptionalPackage} from './VideoMatting/VideoMattingModalWithOptionalPackage';
export const Modals: React.FC<{
readonly readOnlyStudio: boolean;
@@ -263,6 +264,9 @@ export const Modals: React.FC<{
{modalContextType && modalContextType.type === 'transcribe' ? (
<TranscriptionModalWithOptionalWhisper state={modalContextType} />
) : null}
+ {modalContextType && modalContextType.type === 'video-matting' ? (
+ <VideoMattingModalWithOptionalPackage state={modalContextType} />
+ ) : null}
{modalContextType && modalContextType.type === 'fix-computed-value' && (
<FixComputedValueModal state={modalContextType} />
diff --git a/packages/studio/src/components/OptionalPackageModal.tsx b/packages/studio/src/components/OptionalPackageModal.tsx
new file mode 100644
index 0000000000..43c06c0361
--- /dev/null
+++ b/packages/studio/src/components/OptionalPackageModal.tsx
@@ -0,0 +1,157 @@
+import React, {useCallback, useContext, useEffect, useState} from 'react';
+import {installPackages} from '../api/install-package';
+import {LIGHT_TEXT, WARNING_COLOR, WHITE} from '../helpers/colors';
+import {SetSelectedModalContext} from '../state/modals';
+import {Button} from './Button';
+import {Flex, Row, Spacing} from './layout';
+import {ModalButton} from './ModalButton';
+import {ModalContainer} from './ModalContainer';
+import {ModalFooterContainer} from './ModalFooter';
+import {ModalHeader} from './ModalHeader';
+import {Spinner} from './Spinner';
+
+type InstallState =
+ | {type: 'idle'}
+ | {type: 'installing'}
+ | {type: 'error'; error: Error};
+
+const panelStyle: React.CSSProperties = {
+ width: 'min(520px, calc(100vw - 40px))',
+};
+
+const contentStyle: React.CSSProperties = {
+ color: WHITE,
+ fontFamily: 'sans-serif',
+ fontSize: 14,
+ lineHeight: 1.5,
+ padding: 16,
+};
+
+const codeStyle: React.CSSProperties = {
+ color: 'inherit',
+ fontFamily: 'monospace',
+ fontSize: 'inherit',
+};
+
+const statusStyle: React.CSSProperties = {
+ alignItems: 'center',
+ color: LIGHT_TEXT,
+ display: 'flex',
+ fontFamily: 'sans-serif',
+ fontSize: 14,
+ gap: 10,
+ lineHeight: 1.5,
+};
+
+const errorStyle: React.CSSProperties = {
+ color: WARNING_COLOR,
+ marginTop: 12,
+ whiteSpace: 'pre-wrap',
+};
+
+const cancelStyle: React.CSSProperties = {minWidth: 90};
+
+export const isOptionalPackageInstalled = (packageName: string) =>
+ window.remotion_installedPackages?.includes(packageName) ?? false;
+
+const OPTIONAL_PACKAGE_INSTALLED_EVENT = 'remotion-optional-package-installed';
+
+export const useOptionalPackageInstalled = (packageName: string) => {
+ const [installed, setInstalled] = useState(() =>
+ isOptionalPackageInstalled(packageName),
+ );
+
+ useEffect(() => {
+ const onInstalled = (event: Event) => {
+ if ((event as CustomEvent<string>).detail === packageName) {
+ setInstalled(true);
+ }
+ };
+
+ window.addEventListener(OPTIONAL_PACKAGE_INSTALLED_EVENT, onInstalled);
+ return () =>
+ window.removeEventListener(OPTIONAL_PACKAGE_INSTALLED_EVENT, onInstalled);
+ }, [packageName]);
+
+ return installed;
+};
+
+export const OptionalPackageModal: React.FC<{
+ readonly ariaLabel: string;
+ readonly children: React.ReactNode;
+ readonly packageName: string;
+ readonly title: string;
+}> = ({ariaLabel, children, packageName, title}) => {
+ const {setSelectedModal} = useContext(SetSelectedModalContext);
+ const installed = useOptionalPackageInstalled(packageName);
+ const [installState, setInstallState] = useState<InstallState>({
+ type: 'idle',
+ });
+ const busy = installState.type === 'installing';
+
+ const dismiss = useCallback(() => {
+ if (!busy) {
+ setSelectedModal(null);
+ }
+ }, [busy, setSelectedModal]);
+
+ const install = useCallback(async () => {
+ setInstallState({type: 'installing'});
+ try {
+ await installPackages([{name: packageName, version: null}]);
+ window.remotion_installedPackages = Array.from(
+ new Set([...(window.remotion_installedPackages ?? []), packageName]),
+ );
+ window.dispatchEvent(
+ new CustomEvent(OPTIONAL_PACKAGE_INSTALLED_EVENT, {
+ detail: packageName,
+ }),
+ );
+ } catch (error) {
+ setInstallState({type: 'error', error: error as Error});
+ }
+ }, [packageName]);
+
+ if (installed) {
+ return children;
+ }
+
+ return (
+ <ModalContainer
+ ariaLabel={ariaLabel}
+ onEscape={dismiss}
+ onOutsideClick={dismiss}
+ panelStyle={panelStyle}
+ >
+ <ModalHeader title={title} onClose={dismiss} />
+ <div style={contentStyle}>
+ {busy ? (
+ <div style={statusStyle}>
+ <Spinner duration={0.5} size={18} />
+ {`Installing ${packageName}…`}
+ </div>
+ ) : (
+ <>
+ This requires installing{' '}
+ <code style={codeStyle}>{packageName}</code>.{' Continue?'}
+ {installState.type === 'error' ? (
+ <div style={errorStyle}>{installState.error.message}</div>
+ ) : null}
+ </>
+ )}
+ </div>
+ <ModalFooterContainer>
+ <Row align="center">
+ <Flex />
+ <Button disabled={busy} onClick={dismiss} style={cancelStyle}>
+ Cancel
+ </Button>
+ <Spacing x={1} />
+ <ModalButton disabled={busy} onClick={install} autoFocus>
+ {installState.type === 'error' ? 'Retry' : 'Continue'}
+ </ModalButton>
+ </Row>
+ </ModalFooterContainer>
+ </ModalContainer>
+ );
+};
diff --git a/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx b/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx
index 6e4e4f5062..5588dfeeaf 100644
--- a/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx
+++ b/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx
@@ -12,6 +12,7 @@ import {writeStaticFile} from '../../api/write-static-file';
import {resampleMediaTo16Khz} from '../Transcription/resample-media-to-16-khz';
import type {CaptionJob} from './caption-job-types';
import {RenderQueueContext} from './context';
+import {loadModelForJob} from './load-model-for-job';
export const CaptionQueueProcessor: React.FC = () => {
const {
@@ -40,19 +41,18 @@ export const CaptionQueueProcessor: React.FC = () => {
value: 0.02,
});
await clearStaleModels();
- const modelIsCached = await isWhisperModelCached({model: job.model});
- await loadWhisperModel({
+ await loadModelForJob({
model: job.model,
- onProgress: (progress) => {
- const percentage =
- progress.progress === null
- ? ''
- : ` ${Math.round(progress.progress * 100)}%`;
- updateCaptionJobProgress(job.id, {
- message: `${modelIsCached ? 'Loading' : 'Downloading'} ${job.model}${percentage}`,
- value: 0.03 + (progress.progress ?? 0) * 0.27,
- });
- },
+ progressStart: 0.03,
+ progressSpan: 0.27,
+ isModelCached: (model) => isWhisperModelCached({model}),
+ loadModel: (model, onProgress) =>
+ loadWhisperModel({
+ model,
+ onProgress: (progress) => onProgress(progress.progress),
+ }),
+ updateProgress: (progress) =>
+ updateCaptionJobProgress(job.id, progress),
});
updateCaptionJobProgress(job.id, {
diff --git a/packages/studio/src/components/RenderQueue/CaptionQueueProcessorLoader.tsx b/packages/studio/src/components/RenderQueue/CaptionQueueProcessorLoader.tsx
index 6f8ba1753e..403dfb5fe9 100644
--- a/packages/studio/src/components/RenderQueue/CaptionQueueProcessorLoader.tsx
+++ b/packages/studio/src/components/RenderQueue/CaptionQueueProcessorLoader.tsx
@@ -1,5 +1,6 @@
import React, {Suspense, useContext} from 'react';
-import {isWhisperWebGpuInstalled} from '../Transcription/whisper-webgpu-capability';
+import {useOptionalPackageInstalled} from '../OptionalPackageModal';
+import {WHISPER_WEBGPU_PACKAGE} from '../Transcription/whisper-webgpu-capability';
import {RenderQueueContext} from './context';
const LazyCaptionQueueProcessor = React.lazy(async () => {
@@ -9,7 +10,8 @@ const LazyCaptionQueueProcessor = React.lazy(async () => {
export const CaptionQueueProcessorLoader: React.FC = () => {
const {captionJobs} = useContext(RenderQueueContext);
- if (!isWhisperWebGpuInstalled() || captionJobs.length === 0) {
+ const installed = useOptionalPackageInstalled(WHISPER_WEBGPU_PACKAGE);
+ if (!installed || captionJobs.length === 0) {
return null;
}
diff --git a/packages/studio/src/components/RenderQueue/VideoMattingQueueItem.tsx b/packages/studio/src/components/RenderQueue/VideoMattingQueueItem.tsx
new file mode 100644
index 0000000000..cee5bd4407
--- /dev/null
+++ b/packages/studio/src/components/RenderQueue/VideoMattingQueueItem.tsx
@@ -0,0 +1,164 @@
+import React, {useCallback, useContext, useMemo, useState} from 'react';
+import {
+ CURRENT_COLOR,
+ FAIL_COLOR,
+ getBackgroundFromHoverState,
+ LIGHT_TEXT,
+} from '../../helpers/colors';
+import {pushUrl} from '../../helpers/url-state';
+import type {RenderInlineAction} from '../InlineAction';
+import {InlineAction} from '../InlineAction';
+import {Row, Spacing} from '../layout';
+import {useSelectAsset} from '../use-select-asset';
+import {
+ CircularProgress,
+ RENDER_STATUS_INDICATOR_SIZE,
+} from './CircularProgress';
+import {RenderQueueContext} from './context';
+import {renderQueueItemSubtitleStyle} from './item-style';
+import {SuccessIcon} from './SuccessIcon';
+import type {VideoMattingJob} from './video-matting-job-types';
+
+const container: React.CSSProperties = {
+ borderRadius: 4,
+ display: 'flex',
+ marginBottom: 1,
+ marginLeft: 4,
+ padding: 12,
+ paddingBottom: 10,
+ paddingRight: 4,
+ width: 'calc(100% - 4px)',
+};
+const title: React.CSSProperties = {fontSize: 13, lineHeight: 1};
+const right: React.CSSProperties = {
+ display: 'flex',
+ flex: 1,
+ flexDirection: 'column',
+ overflow: 'hidden',
+};
+const subtitle: React.CSSProperties = {
+ ...renderQueueItemSubtitleStyle,
+ cursor: 'default',
+};
+const statusIcon: React.CSSProperties = {
+ height: RENDER_STATUS_INDICATOR_SIZE,
+ width: RENDER_STATUS_INDICATOR_SIZE,
+};
+const removeIcon: React.CSSProperties = {color: CURRENT_COLOR, height: 16};
+
+const Status: React.FC<{readonly job: VideoMattingJob}> = ({job}) => {
+ if (job.status === 'running') {
+ return (
+ <div
+ style={statusIcon}
+ role="progressbar"
+ aria-label="Video matting progress"
+ aria-valuemin={0}
+ aria-valuemax={100}
+ aria-valuenow={Math.round(job.progress.value * 100)}
+ aria-valuetext={job.progress.message}
+ >
+ <CircularProgress progress={Math.max(0.07, job.progress.value)} />
+ </div>
+ );
+ }
+
+ if (job.status === 'done') return <SuccessIcon />;
+ if (job.status === 'failed')
+ return (
+ <svg style={statusIcon} viewBox="0 0 512 512">
+ <path
+ fill={FAIL_COLOR}
+ d="M0 160V352L160 512H352L512 352V160L352 0H160L0 160zm353.9 32-64 64 64 64L320 353.9l-64-64-64 64L158.1 320l64-64-64-64L192 158.1l64 64 64-64z"
+ />
+ </svg>
+ );
+ return (
+ <svg style={statusIcon} viewBox="0 0 512 512">
+ <path
+ fill={LIGHT_TEXT}
+ d="M256 512C114.6 512 0 397.4 0 256S114.6 0 256 0s256 114.6 256 256-114.6 256-256 256zM232 120v136c0 8 4 15.5 10.7 20l96 64 26.6-39.9-85.3-56.9V120z"
+ />
+ </svg>
+ );
+};
+
+export const VideoMattingQueueItem: React.FC<{
+ readonly job: VideoMattingJob;
+ readonly selected: boolean;
+}> = ({job, selected}) => {
+ const [hovered, setHovered] = useState(false);
+ const {removeVideoMattingJob} = useContext(RenderQueueContext);
+ const selectAsset = useSelectAsset();
+ const done = job.status === 'done';
+ const style = useMemo(
+ () => ({
+ ...container,
+ backgroundColor: getBackgroundFromHoverState({
+ hovered: done && hovered,
+ selected,
+ }),
+ userSelect: 'none' as const,
+ }),
+ [done, hovered, selected],
+ );
+ const message =
+ job.status === 'idle'
+ ? 'Queued for video matting'
+ : job.status === 'running'
+ ? job.progress.message
+ : job.status === 'failed'
+ ? job.error.message
+ : `${job.baseOutName} · ${job.foregroundOutName}`;
+ const onClick = useCallback(() => {
+ if (!done) return;
+ selectAsset(job.foregroundOutName);
+ pushUrl(`/assets/${job.foregroundOutName}`);
+ }, [done, job.foregroundOutName, selectAsset]);
+ const onRemove: React.MouseEventHandler = useCallback(
+ (event) => {
+ event.stopPropagation();
+ removeVideoMattingJob(job.id);
+ },
+ [job.id, removeVideoMattingJob],
+ );
+ const renderRemove: RenderInlineAction = useCallback(
+ (color) => (
+ <svg style={removeIcon} viewBox="0 0 320 512">
+ <path
+ fill={color}
+ d="M310.6 150.6 205.3 256l105.3 105.4-45.3 45.3L160 301.3 54.6 406.6 9.4 361.4 114.7 256 9.4 150.6l45.2-45.2L160 210.7l105.3-105.3z"
+ />
+ </svg>
+ ),
+ [],
+ );
+ return (
+ <Row
+ data-render-queue-item={job.id}
+ style={style}
+ align="center"
+ onClick={onClick}
+ onPointerEnter={() => setHovered(true)}
+ onPointerLeave={() => setHovered(false)}
+ >
+ <Status job={job} />
+ <Spacing x={1} />
+ <div style={right}>
+ <div style={title}>{job.displayName}</div>
+ <span style={subtitle} title={message}>
+ {message}
+ </span>
+ </div>
+ <Spacing x={1} />
+ {job.status === 'running' ? null : (
+ <InlineAction
+ renderAction={renderRemove}
+ onClick={onRemove}
+ title="Remove"
+ variant={null}
+ />
+ )}
+ </Row>
+ );
+};
diff --git a/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx
new file mode 100644
index 0000000000..57e6b7d921
--- /dev/null
+++ b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx
@@ -0,0 +1,121 @@
+import {
+ canUseVideoMatting,
+ disposeVideoMattingModel,
+ isVideoMattingModelCached,
+ loadVideoMattingModel,
+ separateVideoLayers,
+} from '@remotion/video-matting';
+import {useCallback, useContext, useEffect} from 'react';
+import {writeStaticFile} from '../../api/write-static-file';
+import {RenderQueueContext} from './context';
+import {loadModelForJob} from './load-model-for-job';
+import type {VideoMattingJob} from './video-matting-job-types';
+
+export const VideoMattingQueueProcessor: React.FC = () => {
+ const {
+ markVideoMattingJobDone,
+ markVideoMattingJobFailed,
+ setProcessVideoMattingJobCallback,
+ updateVideoMattingJobProgress,
+ } = useContext(RenderQueueContext);
+
+ const processJob = useCallback(
+ async (job: VideoMattingJob) => {
+ let outputs: Awaited<ReturnType<typeof separateVideoLayers>> | null =
+ null;
+ let processingError: Error | null = null;
+ try {
+ updateVideoMattingJobProgress(job.id, {
+ message: 'Checking WebGPU support...',
+ value: 0,
+ });
+ const support = await canUseVideoMatting({model: job.model});
+ if (!support.supported) {
+ throw new Error(support.detailedReason);
+ }
+
+ await loadModelForJob({
+ model: job.model,
+ progressStart: 0,
+ progressSpan: 0.2,
+ isModelCached: (model) => isVideoMattingModelCached({model}),
+ loadModel: (model, onProgress) =>
+ loadVideoMattingModel({
+ model,
+ onProgress: (progress) => onProgress(progress.progress),
+ }),
+ updateProgress: (progress) =>
+ updateVideoMattingJobProgress(job.id, progress),
+ });
+
+ outputs = await separateVideoLayers({
+ src: job.src,
+ model: job.model,
+ audio: job.audio,
+ videoBitrate: job.videoBitrate,
+ onProgress: (progress) => {
+ updateVideoMattingJobProgress(job.id, {
+ message:
+ progress.stage === 'finalizing'
+ ? 'Finalizing video layers...'
+ : `Separating ${job.displayName}...`,
+ value: 0.2 + (progress.progress ?? 1) * 0.65,
+ });
+ },
+ });
+
+ updateVideoMattingJobProgress(job.id, {
+ message: 'Saving video layers...',
+ value: 0.88,
+ });
+ const [base, foreground] = await Promise.all([
+ outputs.base.getBlob(),
+ outputs.foreground.getBlob(),
+ ]);
+ await Promise.all([
+ base
+ .arrayBuffer()
+ .then((contents) =>
+ writeStaticFile({contents, filePath: job.baseOutName}),
+ ),
+ foreground
+ .arrayBuffer()
+ .then((contents) =>
+ writeStaticFile({contents, filePath: job.foregroundOutName}),
+ ),
+ ]);
+ } catch (error) {
+ processingError =
+ error instanceof Error ? error : new Error(String(error));
+ }
+
+ try {
+ await Promise.all([
+ outputs?.base.dispose(),
+ outputs?.foreground.dispose(),
+ ]);
+ await disposeVideoMattingModel({model: job.model});
+ } catch {
+ // Cleanup errors must not hide successfully written outputs.
+ }
+
+ if (processingError) {
+ markVideoMattingJobFailed(job.id, processingError);
+ } else {
+ markVideoMattingJobDone(job.id);
+ }
+ },
+ [
+ markVideoMattingJobDone,
+ markVideoMattingJobFailed,
+ updateVideoMattingJobProgress,
+ ],
+ );
+
+ useEffect(() => {
+ setProcessVideoMattingJobCallback(processJob);
+ return () => setProcessVideoMattingJobCallback(null);
+ }, [processJob, setProcessVideoMattingJobCallback]);
+
+ return null;
+};
diff --git a/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessorLoader.tsx b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessorLoader.tsx
new file mode 100644
index 0000000000..c150af5a87
--- /dev/null
+++ b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessorLoader.tsx
@@ -0,0 +1,24 @@
+import React, {Suspense, useContext} from 'react';
+import {useOptionalPackageInstalled} from '../OptionalPackageModal';
+import {VIDEO_MATTING_PACKAGE} from '../VideoMatting/video-matting-capability';
+import {RenderQueueContext} from './context';
+
+const LazyProcessor = React.lazy(async () => {
+ const {VideoMattingQueueProcessor} =
+ await import('./VideoMattingQueueProcessor');
+ return {default: VideoMattingQueueProcessor};
+});
+
+export const VideoMattingQueueProcessorLoader: React.FC = () => {
+ const installed = useOptionalPackageInstalled(VIDEO_MATTING_PACKAGE);
+ const {videoMattingJobs} = useContext(RenderQueueContext);
+ if (!installed || videoMattingJobs.length === 0) {
+ return null;
+ }
+
+ return (
+ <Suspense fallback={null}>
+ <LazyProcessor />
+ </Suspense>
+ );
+};
diff --git a/packages/studio/src/components/RenderQueue/context.tsx b/packages/studio/src/components/RenderQueue/context.tsx
index 86b7d3d503..83b3adade3 100644
--- a/packages/studio/src/components/RenderQueue/context.tsx
+++ b/packages/studio/src/components/RenderQueue/context.tsx
@@ -33,6 +33,11 @@ import type {
GetBlobCallback,
RestoredClientRenderJob,
} from './client-side-render-types';
+import type {
+ AddVideoMattingJobParams,
+ VideoMattingJob,
+ VideoMattingJobProgress,
+} from './video-matting-job-types';
declare global {
interface Window {
@@ -61,6 +66,7 @@ type RenderQueueContextType = {
serverJobs: RenderJob[];
clientJobs: ClientRenderJob[];
captionJobs: CaptionJob[];
+ videoMattingJobs: VideoMattingJob[];
addClientStillJob: (
params: AddClientStillJobParams,
compositionRef: CompositionRef,
@@ -97,6 +103,17 @@ type RenderQueueContextType = {
setProcessCaptionJobCallback: (
callback: ((job: CaptionJob) => Promise<void>) | null,
) => void;
+ addVideoMattingJob: (params: AddVideoMattingJobParams) => string;
+ updateVideoMattingJobProgress: (
+ jobId: string,
+ progress: VideoMattingJobProgress,
+ ) => void;
+ markVideoMattingJobDone: (jobId: string) => void;
+ markVideoMattingJobFailed: (jobId: string, error: Error) => void;
+ removeVideoMattingJob: (jobId: string) => void;
+ setProcessVideoMattingJobCallback: (
+ callback: ((job: VideoMattingJob) => Promise<void>) | null,
+ ) => void;
getAbortController: (jobId: string) => AbortController;
getCompositionForJob: (jobId: string) => CompositionRef | undefined;
};
@@ -109,6 +126,7 @@ export const RenderQueueContext = React.createContext<RenderQueueContextType>({
serverJobs: [],
clientJobs: [],
captionJobs: [],
+ videoMattingJobs: [],
addClientStillJob: noopString,
addClientVideoJob: noopString,
updateClientJobProgress: noop,
@@ -125,6 +143,12 @@ export const RenderQueueContext = React.createContext<RenderQueueContextType>({
markCaptionJobFailed: noop,
removeCaptionJob: noop,
setProcessCaptionJobCallback: noop,
+ addVideoMattingJob: noopString,
+ updateVideoMattingJobProgress: noop,
+ markVideoMattingJobDone: noop,
+ markVideoMattingJobFailed: noop,
+ removeVideoMattingJob: noop,
+ setProcessVideoMattingJobCallback: noop,
getAbortController: () => new AbortController(),
getCompositionForJob: () => undefined,
});
@@ -144,6 +168,9 @@ export const RenderQueueContextProvider: React.FC<{
restorePersistedClientRenders,
);
const [captionJobs, setCaptionJobs] = useState<CaptionJob[]>([]);
+ const [videoMattingJobs, setVideoMattingJobs] = useState<VideoMattingJob[]>(
+ [],
+ );
const [currentlyProcessing, setCurrentlyProcessing] = useState<string | null>(
null,
);
@@ -156,6 +183,9 @@ export const RenderQueueContextProvider: React.FC<{
const processCaptionJobCallbackRef = useRef<
((job: CaptionJob) => Promise<void>) | null
>(null);
+ const processVideoMattingJobCallbackRef = useRef<
+ ((job: VideoMattingJob) => Promise<void>) | null
+ >(null);
const [processorCallbacksVersion, setProcessorCallbacksVersion] = useState(0);
const getLocalJobStartedAt = useCallback(() => {
@@ -171,7 +201,7 @@ export const RenderQueueContextProvider: React.FC<{
return;
}
- const nextJob = [...clientJobs, ...captionJobs]
+ const nextJob = [...clientJobs, ...captionJobs, ...videoMattingJobs]
.filter((job) => job.status === 'idle')
.sort((a, b) => a.startedAt - b.startedAt)[0];
if (!nextJob) {
@@ -202,6 +232,27 @@ export const RenderQueueContextProvider: React.FC<{
return;
}
+ if (nextJob.type === 'video-matting') {
+ if (!processVideoMattingJobCallbackRef.current) {
+ return;
+ }
+
+ setCurrentlyProcessing(nextJob.id);
+ setVideoMattingJobs((prev) =>
+ prev.map((job) =>
+ job.id === nextJob.id
+ ? {
+ ...job,
+ status: 'running',
+ progress: {message: 'Starting video matting...', value: 0},
+ }
+ : job,
+ ),
+ );
+ processVideoMattingJobCallbackRef.current(nextJob);
+ return;
+ }
+
if (!processJobCallbackRef.current) {
return;
}
@@ -225,7 +276,13 @@ export const RenderQueueContextProvider: React.FC<{
),
);
processJobCallbackRef.current(nextJob);
- }, [captionJobs, clientJobs, currentlyProcessing, processorCallbacksVersion]);
+ }, [
+ captionJobs,
+ clientJobs,
+ currentlyProcessing,
+ processorCallbacksVersion,
+ videoMattingJobs,
+ ]);
const addClientStillJob = useCallback(
(
@@ -457,6 +514,81 @@ export const RenderQueueContextProvider: React.FC<{
[],
);
+ const addVideoMattingJob = useCallback(
+ (params: AddVideoMattingJobParams): string => {
+ const id = `video-matting-${Date.now()}-${Math.random()
+ .toString(36)
+ .substring(2, 11)}`;
+ setVideoMattingJobs((prev) => [
+ ...prev,
+ {
+ ...params,
+ id,
+ type: 'video-matting',
+ startedAt: getLocalJobStartedAt(),
+ status: 'idle',
+ },
+ ]);
+ return id;
+ },
+ [getLocalJobStartedAt],
+ );
+
+ const updateVideoMattingJobProgress = useCallback(
+ (jobId: string, progress: VideoMattingJobProgress): void => {
+ setVideoMattingJobs((prev) =>
+ prev.map((job) =>
+ job.id === jobId ? {...job, status: 'running', progress} : job,
+ ),
+ );
+ },
+ [],
+ );
+
+ const markVideoMattingJobDone = useCallback((jobId: string): void => {
+ setVideoMattingJobs((prev) =>
+ prev.map((job) => (job.id === jobId ? {...job, status: 'done'} : job)),
+ );
+ setCurrentlyProcessing((current) => (current === jobId ? null : current));
+ }, []);
+
+ const markVideoMattingJobFailed = useCallback(
+ (jobId: string, error: Error): void => {
+ setVideoMattingJobs((prev) =>
+ prev.map((job) =>
+ job.id === jobId
+ ? {
+ ...job,
+ status: 'failed',
+ error: {message: error.message, stack: error.stack},
+ }
+ : job,
+ ),
+ );
+ setCurrentlyProcessing((current) => (current === jobId ? null : current));
+ },
+ [],
+ );
+
+ const removeVideoMattingJob = useCallback((jobId: string): void => {
+ setVideoMattingJobs((prev) => {
+ const jobToRemove = prev.find((job) => job.id === jobId);
+ if (jobToRemove?.status === 'running') {
+ return prev;
+ }
+
+ return prev.filter((job) => job.id !== jobId);
+ });
+ }, []);
+
+ const setProcessVideoMattingJobCallback = useCallback(
+ (callback: ((job: VideoMattingJob) => Promise<void>) | null): void => {
+ processVideoMattingJobCallbackRef.current = callback;
+ setProcessorCallbacksVersion((version) => version + 1);
+ },
+ [],
+ );
+
useImperativeHandle(
renderJobsRef,
() => ({
@@ -498,6 +630,7 @@ export const RenderQueueContextProvider: React.FC<{
serverJobs,
clientJobs,
captionJobs,
+ videoMattingJobs,
addClientStillJob,
addClientVideoJob,
updateClientJobProgress,
@@ -514,6 +647,12 @@ export const RenderQueueContextProvider: React.FC<{
markCaptionJobFailed,
removeCaptionJob,
setProcessCaptionJobCallback,
+ addVideoMattingJob,
+ updateVideoMattingJobProgress,
+ markVideoMattingJobDone,
+ markVideoMattingJobFailed,
+ removeVideoMattingJob,
+ setProcessVideoMattingJobCallback,
getAbortController,
getCompositionForJob,
};
@@ -521,6 +660,7 @@ export const RenderQueueContextProvider: React.FC<{
serverJobs,
clientJobs,
captionJobs,
+ videoMattingJobs,
addClientStillJob,
addClientVideoJob,
updateClientJobProgress,
@@ -537,6 +677,12 @@ export const RenderQueueContextProvider: React.FC<{
markCaptionJobFailed,
removeCaptionJob,
setProcessCaptionJobCallback,
+ addVideoMattingJob,
+ updateVideoMattingJobProgress,
+ markVideoMattingJobDone,
+ markVideoMattingJobFailed,
+ removeVideoMattingJob,
+ setProcessVideoMattingJobCallback,
]);
return (
diff --git a/packages/studio/src/components/RenderQueue/index.tsx b/packages/studio/src/components/RenderQueue/index.tsx
index 92bf4bb574..ae10ab843e 100644
--- a/packages/studio/src/components/RenderQueue/index.tsx
+++ b/packages/studio/src/components/RenderQueue/index.tsx
@@ -7,6 +7,8 @@ import {isCaptionJob} from './caption-job-types';
import {CaptionQueueItem} from './CaptionQueueItem';
import {RenderQueueContext} from './context';
import {RenderQueueItem} from './RenderQueueItem';
+import {isVideoMattingJob} from './video-matting-job-types';
+import {VideoMattingQueueItem} from './VideoMattingQueueItem';
const errorExplanation: React.CSSProperties = {
fontSize: 13,
@@ -35,11 +37,14 @@ const renderQueue: React.CSSProperties = {
};
export const RenderQueue: React.FC = () => {
- const {jobs, captionJobs} = useContext(RenderQueueContext);
+ const {jobs, captionJobs, videoMattingJobs} = useContext(RenderQueueContext);
const {canvasContent} = useContext(Internals.CompositionManager);
const allJobs = useMemo(
- () => [...jobs, ...captionJobs].sort((a, b) => a.startedAt - b.startedAt),
- [jobs, captionJobs],
+ () =>
+ [...jobs, ...captionJobs, ...videoMattingJobs].sort(
+ (a, b) => a.startedAt - b.startedAt,
+ ),
+ [jobs, captionJobs, videoMattingJobs],
);
const previousJobCount = React.useRef(allJobs.length);
const jobCount = allJobs.length;
@@ -70,7 +75,7 @@ export const RenderQueue: React.FC = () => {
if (canvasContent.type === 'output') {
for (let i = 0; i < allJobs.length; i++) {
const job = allJobs[i];
- if (isCaptionJob(job)) {
+ if (isCaptionJob(job) || isVideoMattingJob(job)) {
continue;
}
@@ -90,6 +95,15 @@ export const RenderQueue: React.FC = () => {
) {
return i;
}
+
+ if (
+ isVideoMattingJob(job) &&
+ job.status === 'done' &&
+ (canvasContent.asset === job.baseOutName ||
+ canvasContent.asset === job.foregroundOutName)
+ ) {
+ return i;
+ }
}
}
@@ -121,6 +135,12 @@ export const RenderQueue: React.FC = () => {
job={job}
selected={selectedJob === index}
/>
+ ) : isVideoMattingJob(job) ? (
+ <VideoMattingQueueItem
+ key={job.id}
+ job={job}
+ selected={selectedJob === index}
+ />
) : (
<RenderQueueItem
key={job.id}
diff --git a/packages/studio/src/components/RenderQueue/load-model-for-job.ts b/packages/studio/src/components/RenderQueue/load-model-for-job.ts
new file mode 100644
index 0000000000..86acba1997
--- /dev/null
+++ b/packages/studio/src/components/RenderQueue/load-model-for-job.ts
@@ -0,0 +1,28 @@
+export const loadModelForJob = async <Model extends string>({
+ isModelCached,
+ loadModel,
+ model,
+ progressSpan,
+ progressStart,
+ updateProgress,
+}: {
+ isModelCached: (model: Model) => Promise<boolean>;
+ loadModel: (
+ model: Model,
+ onProgress: (progress: number | null) => void,
+ ) => Promise<unknown>;
+ model: Model;
+ progressSpan: number;
+ progressStart: number;
+ updateProgress: (progress: {message: string; value: number}) => void;
+}) => {
+ const cached = await isModelCached(model);
+ await loadModel(model, (progress) => {
+ const percentage =
+ progress === null ? '' : ` ${Math.round(progress * 100)}%`;
+ updateProgress({
+ message: `${cached ? 'Loading' : 'Downloading'} ${model}${percentage}`,
+ value: progressStart + (progress ?? 0) * progressSpan,
+ });
+ });
+};
diff --git a/packages/studio/src/components/RenderQueue/video-matting-job-types.ts b/packages/studio/src/components/RenderQueue/video-matting-job-types.ts
new file mode 100644
index 0000000000..0c4fa18e53
--- /dev/null
+++ b/packages/studio/src/components/RenderQueue/video-matting-job-types.ts
@@ -0,0 +1,36 @@
+import type {
+ VideoLayerAudio,
+ VideoMattingBitrate,
+ VideoMattingModel,
+} from '@remotion/video-matting';
+
+export type VideoMattingJobProgress = {
+ message: string;
+ value: number;
+};
+
+type VideoMattingJobStatus =
+ | {status: 'idle'}
+ | {status: 'running'; progress: VideoMattingJobProgress}
+ | {status: 'done'}
+ | {status: 'failed'; error: {message: string; stack: string | undefined}};
+
+export type AddVideoMattingJobParams = {
+ src: string;
+ displayName: string;
+ baseOutName: string;
+ foregroundOutName: string;
+ model: VideoMattingModel;
+ audio: VideoLayerAudio;
+ videoBitrate: VideoMattingBitrate;
+};
+
+export type VideoMattingJob = AddVideoMattingJobParams & {
+ id: string;
+ type: 'video-matting';
+ startedAt: number;
+} & VideoMattingJobStatus;
+
+export const isVideoMattingJob = (job: {
+ type: string;
+}): job is VideoMattingJob => job.type === 'video-matting';
diff --git a/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx b/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx
index 38f4b492eb..b035bdcbb7 100644
--- a/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx
+++ b/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx
@@ -1,145 +1,25 @@
-import React, {Suspense, useCallback, useContext, useState} from 'react';
-import {installPackages} from '../../api/install-package';
-import {LIGHT_TEXT, WARNING_COLOR, WHITE} from '../../helpers/colors';
+import React, {Suspense} from 'react';
import type {TranscriptionModalState} from '../../state/modals';
-import {SetSelectedModalContext} from '../../state/modals';
-import {Button} from '../Button';
-import {Flex, Row, Spacing} from '../layout';
-import {ModalButton} from '../ModalButton';
-import {ModalContainer} from '../ModalContainer';
-import {ModalFooterContainer} from '../ModalFooter';
-import {ModalHeader} from '../ModalHeader';
-import {Spinner} from '../Spinner';
-import {
- isWhisperWebGpuInstalled,
- WHISPER_WEBGPU_PACKAGE,
-} from './whisper-webgpu-capability';
+import {OptionalPackageModal} from '../OptionalPackageModal';
+import {WHISPER_WEBGPU_PACKAGE} from './whisper-webgpu-capability';
const LazyTranscriptionModal = React.lazy(async () => {
const {TranscriptionModal} = await import('./TranscriptionModal');
return {default: TranscriptionModal};
});
-type InstallState =
- | {type: 'idle'}
- | {type: 'installing'}
- | {type: 'error'; error: Error};
-
-const panelStyle: React.CSSProperties = {
- width: 'min(520px, calc(100vw - 40px))',
-};
-
-const contentStyle: React.CSSProperties = {
- color: WHITE,
- fontFamily: 'sans-serif',
- fontSize: 14,
- lineHeight: 1.5,
- padding: 16,
-};
-
-const codeStyle: React.CSSProperties = {
- color: 'inherit',
- fontFamily: 'monospace',
- fontSize: 'inherit',
-};
-
-const statusStyle: React.CSSProperties = {
- alignItems: 'center',
- color: LIGHT_TEXT,
- display: 'flex',
- fontFamily: 'sans-serif',
- fontSize: 14,
- gap: 10,
- lineHeight: 1.5,
-};
-
-const errorStyle: React.CSSProperties = {
- color: WARNING_COLOR,
- marginTop: 12,
- whiteSpace: 'pre-wrap',
-};
-
-const cancelStyle: React.CSSProperties = {
- minWidth: 90,
-};
-
export const TranscriptionModalWithOptionalWhisper: React.FC<{
readonly state: TranscriptionModalState;
}> = ({state}) => {
- const {setSelectedModal} = useContext(SetSelectedModalContext);
- const [installed, setInstalled] = useState(isWhisperWebGpuInstalled);
- const [installState, setInstallState] = useState<InstallState>({
- type: 'idle',
- });
-
- const busy = installState.type === 'installing';
- const dismiss = useCallback(() => {
- if (!busy) {
- setSelectedModal(null);
- }
- }, [busy, setSelectedModal]);
-
- const install = useCallback(async () => {
- setInstallState({type: 'installing'});
- try {
- await installPackages([{name: WHISPER_WEBGPU_PACKAGE, version: null}]);
- window.remotion_installedPackages = Array.from(
- new Set([
- ...(window.remotion_installedPackages ?? []),
- WHISPER_WEBGPU_PACKAGE,
- ]),
- );
-
- setInstalled(true);
- } catch (error) {
- setInstallState({type: 'error', error: error as Error});
- }
- }, []);
-
- if (installed) {
- return (
- <Suspense fallback={null}>
- <LazyTranscriptionModal {...state} />
- </Suspense>
- );
- }
-
return (
- <ModalContainer
+ <OptionalPackageModal
ariaLabel="Install transcription package"
- onEscape={dismiss}
- onOutsideClick={dismiss}
- panelStyle={panelStyle}
+ packageName={WHISPER_WEBGPU_PACKAGE}
+ title="Transcribe"
>
- <ModalHeader title="Transcribe" onClose={dismiss} />
- <div style={contentStyle}>
- {busy ? (
- <div style={statusStyle}>
- <Spinner duration={0.5} size={18} />
- {`Installing ${WHISPER_WEBGPU_PACKAGE}…`}
- </div>
- ) : (
- <>
- This requires installing{' '}
- <code style={codeStyle}>{WHISPER_WEBGPU_PACKAGE}</code>. Continue?
- {installState.type === 'error' ? (
- <div style={errorStyle}>{installState.error.message}</div>
- ) : null}
- </>
- )}
- </div>
- <ModalFooterContainer>
- <Row align="center">
- <Flex />
- <Button disabled={busy} onClick={dismiss} style={cancelStyle}>
- Cancel
- </Button>
- <Spacing x={1} />
- <ModalButton disabled={busy} onClick={install} autoFocus>
- {installState.type === 'error' ? 'Retry' : 'Continue'}
- </ModalButton>
- </Row>
- </ModalFooterContainer>
- </ModalContainer>
+ <Suspense fallback={null}>
+ <LazyTranscriptionModal {...state} />
+ </Suspense>
+ </OptionalPackageModal>
);
};
diff --git a/packages/studio/src/components/Transcription/caption-output-name.ts b/packages/studio/src/components/Transcription/caption-output-name.ts
index 458f2bb38a..06696e6b26 100644
--- a/packages/studio/src/components/Transcription/caption-output-name.ts
+++ b/packages/studio/src/components/Transcription/caption-output-name.ts
@@ -1,86 +1,14 @@
-const invalidCharacters = ['?', '*', '+', ':', '%'];
+import {
+ getDefaultOutputBaseName,
+ validatePublicOutputName,
+} from '../public-output-name';
export const getDefaultCaptionOutputName = (
src: string,
displayName = 'captions',
): string => {
- let pathname = displayName;
- if (!/^(data|blob):/i.test(src.trimStart())) {
- pathname = src.split(/[?#]/)[0] ?? src;
- try {
- pathname = new URL(
- src,
- typeof window === 'undefined'
- ? 'http://localhost'
- : window.location.href,
- ).pathname;
- } catch {
- // Fall back to the source as-is if it cannot be parsed as a URL.
- }
- }
-
- const lastPathSegment = pathname.split('/').filter(Boolean).at(-1);
- let decoded = lastPathSegment ?? 'captions';
- try {
- decoded = decodeURIComponent(decoded);
- } catch {
- // Keep the encoded name if it contains an invalid escape sequence.
- }
-
- const withoutExtension = decoded.replace(/\.[^/.]+$/, '');
- const safeBaseName = withoutExtension
- .replace(/[^a-zA-Z0-9-_ ]/g, '-')
- .trim()
- .replace(/\s+/g, '-')
- .replace(/^-+|-+$/g, '');
-
- return `${safeBaseName || 'captions'}-captions.json`;
+ return `${getDefaultOutputBaseName(src, displayName, 'captions')}-captions.json`;
};
-export const validateCaptionOutputName = (outName: string): string | null => {
- if (outName.trim() === '') {
- return 'Enter an output file';
- }
-
- if (outName !== outName.trim()) {
- return 'The output file must not start or end with whitespace';
- }
-
- if (outName.startsWith('/') || /^[a-zA-Z]:/.test(outName)) {
- return 'The output file must be relative to the public folder';
- }
-
- if (outName.includes('\\')) {
- return 'Use forward slashes in the output file';
- }
-
- if (outName.includes('\0')) {
- return 'The output file must not contain null characters';
- }
-
- const segments = outName.split('/');
- if (segments.some((segment) => segment === '')) {
- return 'The output file must not contain empty path segments';
- }
-
- if (segments.some((segment) => segment === '.')) {
- return 'The output file must not contain "." path segments';
- }
-
- if (segments.some((segment) => segment.startsWith('..'))) {
- return 'The output file must stay inside the public folder';
- }
-
- const invalidCharacter = invalidCharacters.find((character) =>
- outName.includes(character),
- );
- if (invalidCharacter) {
- return `The output file must not contain "${invalidCharacter}"`;
- }
-
- if (!outName.toLowerCase().endsWith('.json')) {
- return 'The output file must end in .json';
- }
-
- return null;
-};
+export const validateCaptionOutputName = (outName: string): string | null =>
+ validatePublicOutputName({extension: '.json', outName});
diff --git a/packages/studio/src/components/Transcription/whisper-webgpu-capability.ts b/packages/studio/src/components/Transcription/whisper-webgpu-capability.ts
index 46a9d8ee6d..0e9564bf3d 100644
--- a/packages/studio/src/components/Transcription/whisper-webgpu-capability.ts
+++ b/packages/studio/src/components/Transcription/whisper-webgpu-capability.ts
@@ -1,7 +1,6 @@
+import {isOptionalPackageInstalled} from '../OptionalPackageModal';
+
export const WHISPER_WEBGPU_PACKAGE = '@remotion/whisper-webgpu';
-export const isWhisperWebGpuInstalled = () => {
- return (
- window.remotion_installedPackages?.includes(WHISPER_WEBGPU_PACKAGE) ?? false
- );
-};
+export const isWhisperWebGpuInstalled = () =>
+ isOptionalPackageInstalled(WHISPER_WEBGPU_PACKAGE);
diff --git a/packages/studio/src/components/VideoMatting/VideoMattingModal.tsx b/packages/studio/src/components/VideoMatting/VideoMattingModal.tsx
new file mode 100644
index 0000000000..2b5dfb9e52
--- /dev/null
+++ b/packages/studio/src/components/VideoMatting/VideoMattingModal.tsx
@@ -0,0 +1,380 @@
+import {formatBytes} from '@remotion/studio-shared';
+import {
+ canUseVideoMatting,
+ getAvailableModels,
+ type VideoLayerAudio,
+ type VideoMattingBitrate,
+ type VideoMattingModel,
+} from '@remotion/video-matting';
+import React, {
+ useCallback,
+ useContext,
+ useEffect,
+ useMemo,
+ useState,
+} from 'react';
+import {BLUE_DISABLED, WHITE} from '../../helpers/colors';
+import {Checkmark} from '../../icons/Checkmark';
+import type {VideoMattingModalState} from '../../state/modals';
+import {SetSelectedModalContext} from '../../state/modals';
+import {SidebarContext} from '../../state/sidebar';
+import {Button} from '../Button';
+import {VERTICAL_SCROLLBAR_CLASSNAME} from '../Menu/is-menu-item';
+import {ModalHeader} from '../ModalHeader';
+import type {ComboboxValue} from '../NewComposition/ComboBox';
+import {Combobox} from '../NewComposition/ComboBox';
+import {DismissableModal} from '../NewComposition/DismissableModal';
+import {RemotionInput} from '../NewComposition/RemInput';
+import {ValidationMessage} from '../NewComposition/ValidationMessage';
+import {optionsSidebarTabs} from '../options-sidebar-tabs';
+import {persistSelectedOptionsSidebarPanel} from '../OptionsPanel';
+import {
+ getDefaultOutputBaseName,
+ validatePublicOutputName,
+} from '../public-output-name';
+import {input, label, optionRow, rightRow} from '../RenderModal/layout';
+import {
+ buttonStyle,
+ container,
+ flexer,
+ optionsPanel,
+ outerModalStyle,
+} from '../RenderModal/render-modals';
+import {RenderModalHr} from '../RenderModal/RenderModalHr';
+import {RenderQueueContext} from '../RenderQueue/context';
+import {useStaticFiles} from '../use-static-files';
+
+const MODELS = getAvailableModels();
+const controlStyle: React.CSSProperties = {width: 330, maxWidth: '100%'};
+const panelStyle: React.CSSProperties = {
+ ...optionsPanel,
+ flexDirection: 'column',
+ paddingTop: 16,
+};
+const modalStyle: React.CSSProperties = {
+ ...outerModalStyle,
+ height: 'auto',
+ maxHeight: 'calc(100vh - 40px)',
+ outline: 'none',
+};
+const outputRow: React.CSSProperties = {...optionRow, alignItems: 'flex-start'};
+const fieldLabel: React.CSSProperties = {
+ color: WHITE,
+ fontFamily: 'sans-serif',
+ fontSize: 13,
+ lineHeight: '32px',
+};
+const validationStyle: React.CSSProperties = {padding: '0 16px 8px'};
+
+type SupportState =
+ | {type: 'checking'}
+ | {type: 'supported'}
+ | {type: 'unsupported'; message: string};
+
+const makeOptions = <Value extends string>({
+ items,
+ selected,
+ setSelected,
+}: {
+ items: readonly {id: Value; label: React.ReactNode}[];
+ selected: Value;
+ setSelected: (value: Value) => void;
+}): ComboboxValue[] =>
+ items.map(({id, label: optionLabel}) => ({
+ type: 'item',
+ id,
+ value: id,
+ label: optionLabel,
+ leftItem: id === selected ? <Checkmark /> : null,
+ keyHint: null,
+ quickSwitcherLabel: null,
+ subMenu: null,
+ disabled: false,
+ onClick: () => setSelected(id),
+ }));
+
+export const VideoMattingModal: React.FC<VideoMattingModalState> = ({
+ displayName,
+ src,
+}) => {
+ const baseName = useMemo(
+ () => getDefaultOutputBaseName(src, displayName, 'video'),
+ [displayName, src],
+ );
+ const [baseOutName, setBaseOutName] = useState(`${baseName}-base.webm`);
+ const [foregroundOutName, setForegroundOutName] = useState(
+ `${baseName}-foreground.webm`,
+ );
+ const [model, setModel] = useState<VideoMattingModel>('modnet');
+ const [audio, setAudio] = useState<VideoLayerAudio>('base');
+ const [videoBitrate, setVideoBitrate] =
+ useState<VideoMattingBitrate>('very-high');
+ const [support, setSupport] = useState<SupportState>({type: 'checking'});
+ const staticFiles = useStaticFiles();
+ const {addVideoMattingJob, videoMattingJobs} = useContext(RenderQueueContext);
+ const {setSelectedModal} = useContext(SetSelectedModalContext);
+ const {setSidebarCollapsedState} = useContext(SidebarContext);
+
+ useEffect(() => {
+ let cancelled = false;
+ setSupport({type: 'checking'});
+ canUseVideoMatting({model}).then((result) => {
+ if (!cancelled) {
+ setSupport(
+ result.supported
+ ? {type: 'supported'}
+ : {type: 'unsupported', message: result.detailedReason},
+ );
+ }
+ });
+ return () => {
+ cancelled = true;
+ };
+ }, [model]);
+
+ const normalizedBase = baseOutName.normalize('NFC').toLowerCase();
+ const normalizedForeground = foregroundOutName.normalize('NFC').toLowerCase();
+ const duplicateOutput = normalizedBase === normalizedForeground;
+ const queuedOutputs = new Set(
+ videoMattingJobs
+ .filter((job) => job.status === 'idle' || job.status === 'running')
+ .flatMap((job) => [job.baseOutName, job.foregroundOutName])
+ .map((name) => name.normalize('NFC').toLowerCase()),
+ );
+ const baseError =
+ validatePublicOutputName({extension: '.webm', outName: baseOutName}) ??
+ (duplicateOutput
+ ? 'Base and foreground outputs must be different'
+ : queuedOutputs.has(normalizedBase)
+ ? 'Another video matting job is using this output file'
+ : null);
+ const foregroundError =
+ validatePublicOutputName({
+ extension: '.webm',
+ outName: foregroundOutName,
+ }) ??
+ (duplicateOutput
+ ? 'Base and foreground outputs must be different'
+ : queuedOutputs.has(normalizedForeground)
+ ? 'Another video matting job is using this output file'
+ : null);
+ const baseExists = staticFiles.some(
+ (file) => file.name.normalize('NFC').toLowerCase() === normalizedBase,
+ );
+ const foregroundExists = staticFiles.some(
+ (file) => file.name.normalize('NFC').toLowerCase() === normalizedForeground,
+ );
+ const canSubmit =
+ support.type === 'supported' &&
+ baseError === null &&
+ foregroundError === null;
+
+ const modelOptions = useMemo(
+ () =>
+ makeOptions({
+ items: MODELS.map((item) => ({
+ id: item.name,
+ label: `${item.name} · ${formatBytes(item.webGpuDownloadSize)}`,
+ })),
+ selected: model,
+ setSelected: setModel,
+ }),
+ [model],
+ );
+ const audioOptions = useMemo(
+ () =>
+ makeOptions({
+ items: [
+ {id: 'base' as const, label: 'Base layer'},
+ {id: 'foreground' as const, label: 'Foreground layer'},
+ {id: 'both' as const, label: 'Both layers'},
+ {id: 'none' as const, label: 'No audio'},
+ ],
+ selected: audio,
+ setSelected: setAudio,
+ }),
+ [audio],
+ );
+ const qualityOptions = useMemo(
+ () =>
+ makeOptions({
+ items: ['very-low', 'low', 'medium', 'high', 'very-high'].map((id) => ({
+ id: id as VideoMattingBitrate & string,
+ label: id
+ .split('-')
+ .map((part) => part[0]!.toUpperCase() + part.slice(1))
+ .join(' '),
+ })),
+ selected: videoBitrate as VideoMattingBitrate & string,
+ setSelected: setVideoBitrate,
+ }),
+ [videoBitrate],
+ );
+
+ const submit = useCallback(() => {
+ if (!canSubmit) return;
+ addVideoMattingJob({
+ src,
+ displayName,
+ baseOutName,
+ foregroundOutName,
+ model,
+ audio,
+ videoBitrate,
+ });
+ setSidebarCollapsedState({left: null, right: 'expanded'});
+ persistSelectedOptionsSidebarPanel('renders');
+ optionsSidebarTabs.current?.selectRendersPanel();
+ setSelectedModal(null);
+ }, [
+ addVideoMattingJob,
+ audio,
+ baseOutName,
+ canSubmit,
+ displayName,
+ foregroundOutName,
+ model,
+ setSelectedModal,
+ setSidebarCollapsedState,
+ src,
+ videoBitrate,
+ ]);
+
+ return (
+ <DismissableModal ariaLabel={`Track matting ${displayName}`}>
+ <div style={modalStyle}>
+ <ModalHeader title={`Track matting ${displayName}`} />
+ <div style={container}>
+ <div style={flexer} />
+ <Button
+ disabled={!canSubmit}
+ onClick={submit}
+ title={support.type === 'unsupported' ? support.message : undefined}
+ style={{
+ ...buttonStyle,
+ backgroundColor: canSubmit
+ ? buttonStyle.backgroundColor
+ : BLUE_DISABLED,
+ }}
+ >
+ Track matting
+ </Button>
+ </div>
+ <div style={panelStyle} className={VERTICAL_SCROLLBAR_CLASSNAME}>
+ <div style={outputRow}>
+ <div style={fieldLabel}>Base output in public/</div>
+ <div style={rightRow}>
+ <RemotionInput
+ aria-label="Base video output file"
+ status={baseError ? 'error' : baseExists ? 'warning' : 'ok'}
+ rightAlign
+ value={baseOutName}
+ onChange={(event) => setBaseOutName(event.target.value)}
+ style={{...input, ...controlStyle}}
+ />
+ </div>
+ </div>
+ {baseError ? (
+ <div style={validationStyle}>
+ <ValidationMessage
+ align="flex-end"
+ message={baseError}
+ type="error"
+ />
+ </div>
+ ) : baseExists ? (
+ <div style={validationStyle}>
+ <ValidationMessage
+ align="flex-end"
+ message="Exists, will be overwritten"
+ type="warning"
+ />
+ </div>
+ ) : null}
+ <div style={outputRow}>
+ <div style={fieldLabel}>Foreground output in public/</div>
+ <div style={rightRow}>
+ <RemotionInput
+ aria-label="Foreground video output file"
+ status={
+ foregroundError
+ ? 'error'
+ : foregroundExists
+ ? 'warning'
+ : 'ok'
+ }
+ rightAlign
+ value={foregroundOutName}
+ onChange={(event) => setForegroundOutName(event.target.value)}
+ style={{...input, ...controlStyle}}
+ />
+ </div>
+ </div>
+ {foregroundError ? (
+ <div style={validationStyle}>
+ <ValidationMessage
+ align="flex-end"
+ message={foregroundError}
+ type="error"
+ />
+ </div>
+ ) : foregroundExists ? (
+ <div style={validationStyle}>
+ <ValidationMessage
+ align="flex-end"
+ message="Exists, will be overwritten"
+ type="warning"
+ />
+ </div>
+ ) : null}
+ <RenderModalHr />
+ <div style={optionRow}>
+ <div style={label}>Model</div>
+ <div style={rightRow}>
+ <Combobox
+ values={modelOptions}
+ selectedId={model}
+ title="Model"
+ disabled={false}
+ style={controlStyle}
+ />
+ </div>
+ </div>
+ {support.type === 'unsupported' ? (
+ <div style={validationStyle}>
+ <ValidationMessage
+ align="flex-end"
+ message={support.message}
+ type="error"
+ />
+ </div>
+ ) : null}
+ <div style={optionRow}>
+ <div style={label}>Audio</div>
+ <div style={rightRow}>
+ <Combobox
+ values={audioOptions}
+ selectedId={audio}
+ title="Audio"
+ disabled={false}
+ style={controlStyle}
+ />
+ </div>
+ </div>
+ <div style={optionRow}>
+ <div style={label}>Video quality</div>
+ <div style={rightRow}>
+ <Combobox
+ values={qualityOptions}
+ selectedId={String(videoBitrate)}
+ title="Video quality"
+ disabled={false}
+ style={controlStyle}
+ />
+ </div>
+ </div>
+ </div>
+ </div>
+ </DismissableModal>
+ );
+};
diff --git a/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx b/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx
new file mode 100644
index 0000000000..43bb871e7d
--- /dev/null
+++ b/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx
@@ -0,0 +1,23 @@
+import React, {Suspense} from 'react';
+import type {VideoMattingModalState} from '../../state/modals';
+import {OptionalPackageModal} from '../OptionalPackageModal';
+import {VIDEO_MATTING_PACKAGE} from './video-matting-capability';
+
+const LazyVideoMattingModal = React.lazy(async () => {
+ const {VideoMattingModal} = await import('./VideoMattingModal');
+ return {default: VideoMattingModal};
+});
+
+export const VideoMattingModalWithOptionalPackage: React.FC<{
+ readonly state: VideoMattingModalState;
+}> = ({state}) => (
+ <OptionalPackageModal
+ ariaLabel="Install video matting package"
+ packageName={VIDEO_MATTING_PACKAGE}
+ title="Track matting"
+ >
+ <Suspense fallback={null}>
+ <LazyVideoMattingModal {...state} />
+ </Suspense>
+ </OptionalPackageModal>
+);
diff --git a/packages/studio/src/components/VideoMatting/video-matting-capability.ts b/packages/studio/src/components/VideoMatting/video-matting-capability.ts
new file mode 100644
index 0000000000..66b376e54f
--- /dev/null
+++ b/packages/studio/src/components/VideoMatting/video-matting-capability.ts
@@ -0,0 +1,6 @@
+import {isOptionalPackageInstalled} from '../OptionalPackageModal';
+
+export const VIDEO_MATTING_PACKAGE = '@remotion/video-matting';
+
+export const isVideoMattingInstalled = () =>
+ isOptionalPackageInstalled(VIDEO_MATTING_PACKAGE);
diff --git a/packages/studio/src/components/public-output-name.ts b/packages/studio/src/components/public-output-name.ts
new file mode 100644
index 0000000000..82f2e192f4
--- /dev/null
+++ b/packages/studio/src/components/public-output-name.ts
@@ -0,0 +1,73 @@
+const invalidCharacters = ['?', '*', '+', ':', '%'];
+
+export const getDefaultOutputBaseName = (
+ src: string,
+ displayName: string,
+ fallbackName: string,
+): string => {
+ let pathname = displayName;
+ if (!/^(data|blob):/i.test(src.trimStart())) {
+ pathname = src.split(/[?#]/)[0] ?? src;
+ try {
+ pathname = new URL(
+ src,
+ typeof window === 'undefined'
+ ? 'http://localhost'
+ : window.location.href,
+ ).pathname;
+ } catch {
+ // Fall back to the source as-is if it cannot be parsed as a URL.
+ }
+ }
+
+ const lastPathSegment = pathname.split('/').filter(Boolean).at(-1);
+ let decoded = lastPathSegment ?? displayName;
+ try {
+ decoded = decodeURIComponent(decoded);
+ } catch {
+ // Keep the encoded name if it contains an invalid escape sequence.
+ }
+
+ return (
+ decoded
+ .replace(/\.[^/.]+$/, '')
+ .replace(/[^a-zA-Z0-9-_ ]/g, '-')
+ .trim()
+ .replace(/\s+/g, '-')
+ .replace(/^-+|-+$/g, '') || fallbackName
+ );
+};
+
+export const validatePublicOutputName = ({
+ extension,
+ outName,
+}: {
+ extension: string;
+ outName: string;
+}): string | null => {
+ if (outName.trim() === '') return 'Enter an output file';
+ if (outName !== outName.trim())
+ return 'The output file must not start or end with whitespace';
+ if (outName.startsWith('/') || /^[a-zA-Z]:/.test(outName))
+ return 'The output file must be relative to the public folder';
+ if (outName.includes('\\')) return 'Use forward slashes in the output file';
+ if (outName.includes('\0'))
+ return 'The output file must not contain null characters';
+
+ const segments = outName.split('/');
+ if (segments.some((segment) => segment === ''))
+ return 'The output file must not contain empty path segments';
+ if (segments.some((segment) => segment === '.'))
+ return 'The output file must not contain "." path segments';
+ if (segments.some((segment) => segment.startsWith('..')))
+ return 'The output file must stay inside the public folder';
+
+ const invalidCharacter = invalidCharacters.find((character) =>
+ outName.includes(character),
+ );
+ if (invalidCharacter)
+ return `The output file must not contain "${invalidCharacter}"`;
+ if (!outName.toLowerCase().endsWith(extension))
+ return `The output file must end in ${extension}`;
+ return null;
+};
diff --git a/packages/studio/src/state/modals.ts b/packages/studio/src/state/modals.ts
index 1e2f148c45..bfd2abad41 100644
--- a/packages/studio/src/state/modals.ts
+++ b/packages/studio/src/state/modals.ts
@@ -148,6 +148,12 @@ export type TranscriptionModalState = {
requestInit: Omit<RequestInit, 'signal'> | null;
};
+export type VideoMattingModalState = {
+ type: 'video-matting';
+ src: string;
+ displayName: string;
+};
+
export type CanvasCaptureImport = {
readonly data: CanvasCaptureData;
readonly durationInSeconds: number;
@@ -264,6 +270,7 @@ export type ModalState =
| ElementInstallModalState
| AddEffectModalState
| TranscriptionModalState
+ | VideoMattingModalState
| ConfirmationDialogState
| SvgImportDialogState;
diff --git a/packages/studio/src/test/video-matting-queue-processor.test.tsx b/packages/studio/src/test/video-matting-queue-processor.test.tsx
new file mode 100644
index 0000000000..6b34484bf2
--- /dev/null
+++ b/packages/studio/src/test/video-matting-queue-processor.test.tsx
@@ -0,0 +1,129 @@
+import {afterEach, expect, mock, test} from 'bun:test';
+import {act, cleanup, render, waitFor} from '@testing-library/react';
+import type {ContextType} from 'react';
+import {RenderQueueContext} from '../components/RenderQueue/context';
+import type {VideoMattingJob} from '../components/RenderQueue/video-matting-job-types';
+import {makeBrowserStudioOperations} from './make-browser-studio-operations';
+
+const calls: string[] = [];
+let processJob: ((job: VideoMattingJob) => Promise<void>) | null = null;
+
+mock.module('@remotion/video-matting', () => ({
+ canUseVideoMatting: () => Promise.resolve({supported: true}),
+ isVideoMattingModelCached: () => Promise.resolve(false),
+ loadVideoMattingModel: ({
+ onProgress,
+ }: {
+ onProgress: (progress: {progress: number}) => void;
+ }) => {
+ calls.push('load-model');
+ onProgress({progress: 1});
+ return Promise.resolve({alreadyLoaded: false});
+ },
+ separateVideoLayers: ({
+ onProgress,
+ }: {
+ onProgress: (progress: {stage: string; progress: number}) => void;
+ }) => {
+ calls.push('separate');
+ onProgress({stage: 'processing', progress: 0.5});
+ return Promise.resolve({
+ base: {
+ getBlob: () => Promise.resolve(new Blob(['base'])),
+ dispose: () => Promise.resolve(),
+ },
+ foreground: {
+ getBlob: () => Promise.resolve(new Blob(['foreground'])),
+ dispose: () => Promise.resolve(),
+ },
+ });
+ },
+ disposeVideoMattingModel: () => {
+ calls.push('dispose-model');
+ return Promise.resolve();
+ },
+}));
+
+afterEach(() => {
+ cleanup();
+ calls.length = 0;
+ processJob = null;
+});
+
+test('loads the model, separates the layers and writes both outputs', async () => {
+ const originalBrowserStudio = Object.getOwnPropertyDescriptor(
+ window,
+ 'remotion_browserStudio',
+ );
+ Object.defineProperty(window, 'remotion_browserStudio', {
+ configurable: true,
+ value: makeBrowserStudioOperations({
+ writeStaticFile: ({filePath}) => {
+ calls.push(`write:${filePath}`);
+ return Promise.resolve();
+ },
+ }),
+ });
+ const progress: string[] = [];
+ let done = false;
+ let failed: Error | null = null;
+ const value = {
+ setProcessVideoMattingJobCallback: (callback: typeof processJob) => {
+ processJob = callback;
+ },
+ updateVideoMattingJobProgress: (_id: string, update: {message: string}) => {
+ progress.push(update.message);
+ },
+ markVideoMattingJobDone: () => {
+ done = true;
+ },
+ markVideoMattingJobFailed: (_id: string, error: Error) => {
+ failed = error;
+ },
+ } as unknown as ContextType<typeof RenderQueueContext>;
+
+ const {VideoMattingQueueProcessor} =
+ await import('../components/RenderQueue/VideoMattingQueueProcessor');
+ render(
+ <RenderQueueContext.Provider value={value}>
+ <VideoMattingQueueProcessor />
+ </RenderQueueContext.Provider>,
+ );
+ await waitFor(() => expect(processJob).not.toBeNull());
+
+ await act(() =>
+ processJob?.({
+ id: 'matting-1',
+ type: 'video-matting',
+ status: 'idle',
+ startedAt: 1,
+ src: '/input.webm',
+ displayName: 'input.webm',
+ baseOutName: 'input-base.webm',
+ foregroundOutName: 'input-foreground.webm',
+ model: 'modnet',
+ audio: 'base',
+ videoBitrate: 'very-high',
+ }),
+ );
+
+ expect(failed).toBeNull();
+ expect(done).toBe(true);
+ expect(calls).toEqual([
+ 'load-model',
+ 'separate',
+ 'write:input-base.webm',
+ 'write:input-foreground.webm',
+ 'dispose-model',
+ ]);
+ expect(progress).toContain('Downloading modnet 100%');
+ if (originalBrowserStudio) {
+ Object.defineProperty(
+ window,
+ 'remotion_browserStudio',
+ originalBrowserStudio,
+ );
+ } else {
+ Reflect.deleteProperty(window, 'remotion_browserStudio');
+ }
+});
diff --git a/packages/studio/src/test/whisper-webgpu-capability.test.tsx b/packages/studio/src/test/whisper-webgpu-capability.test.tsx
index 6f02eaffe9..78238fbcb0 100644
--- a/packages/studio/src/test/whisper-webgpu-capability.test.tsx
+++ b/packages/studio/src/test/whisper-webgpu-capability.test.tsx
@@ -6,6 +6,8 @@ import {
isWhisperWebGpuInstalled,
WHISPER_WEBGPU_PACKAGE,
} from '../components/Transcription/whisper-webgpu-capability';
+import {VIDEO_MATTING_PACKAGE} from '../components/VideoMatting/video-matting-capability';
+import {VideoMattingModalWithOptionalPackage} from '../components/VideoMatting/VideoMattingModalWithOptionalPackage';
const originalInstalledPackages = window.remotion_installedPackages;
@@ -43,6 +45,26 @@ test('asks before installing Whisper for transcription', () => {
expect(getByRole('button', {name: 'Continue'})).toBeDefined();
});
+test('uses the same install confirmation for video matting', () => {
+ window.remotion_installedPackages = [];
+ const {container, getByRole} = render(
+ <ModalsProvider>
+ <VideoMattingModalWithOptionalPackage
+ state={{
+ type: 'video-matting',
+ src: '/video.webm',
+ displayName: 'video.webm',
+ }}
+ />
+ </ModalsProvider>,
+ );
+
+ expect(container.textContent).toContain(
+ `This requires installing ${VIDEO_MATTING_PACKAGE}. Continue?`,
+ );
+ expect(getByRole('button', {name: 'Continue'})).toBeDefined();
+});
+
test('opens transcription after installing Whisper without restarting', async () => {
const originalIsStudio = window.remotion_isStudio;
const originalIsReadOnlyStudio = window.remotion_isReadOnlyStudio;
diff --git a/packages/studio/tsconfig.json b/packages/studio/tsconfig.json
index 56fe86b3c2..49796a7c5a 100644
--- a/packages/studio/tsconfig.json
+++ b/packages/studio/tsconfig.json
@@ -44,6 +44,9 @@
},
{
"path": "../whisper-webgpu"
+ },
+ {
+ "path": "../video-matting"
}
]
}