🎥 Make videos programmatically with React
Studio: Replace video source when removing background
Jonny Burgerd7aaba23a3c6
diff --git a/packages/browser-studio/src/browser-studio-operations.ts b/packages/browser-studio/src/browser-studio-operations.tsindex e075470586..2dad608138 100644--- a/packages/browser-studio/src/browser-studio-operations.ts+++ b/packages/browser-studio/src/browser-studio-operations.ts@@ -81,6 +81,7 @@ const {insertBasicCaptions: insertBasicCaptionsCodemod,insertJsxElementIntoProjectWithNodePathRemappings,insertVideoLayers: insertVideoLayersCodemod,+ replaceVideoSource: replaceVideoSourceCodemod,JsxElementIdentityMismatchError,JsxElementNotFoundAtLocationError,pasteEffects: pasteEffectsCodemod,@@ -2044,6 +2045,48 @@ export const createBrowserStudioOperations = ({}};+ const replaceVideoSource: BrowserStudioOperations['replaceVideoSource'] = ({+ fileName,+ nodePath,+ src,+ }) => {+ try {+ const project = getProject();+ const absolutePath = findProjectFile({filePath: fileName, project});+ const result = replaceVideoSourceCodemod({+ input: project.files[absolutePath],+ nodePath,+ src,+ });+ const nodePathMutation = controller.applyMutation({+ undoRedoNavigation: null,+ timelineSelection: null,+ fileName: absolutePath,+ mutate: () => ({+ ...project,+ files: {...project.files, [absolutePath]: result.output},+ }),+ nodePathMutationFiles: [+ {+ absolutePath,+ remappings: result.nodePathRemappings,+ },+ ],+ });+ if (nodePathMutation === null) {+ throw new Error('Could not replace video source');+ }++ return Promise.resolve({success: true as const, nodePathMutation});+ } catch (error) {+ return Promise.resolve({+ success: false as const,+ reason: error instanceof Error ? error.message : String(error),+ stack: error instanceof Error && error.stack ? error.stack : '',+ });+ }+ };+const insertJsxElement: BrowserStudioOperations['insertJsxElement'] = async (request,) => {@@ -2500,6 +2543,7 @@ export const createBrowserStudioOperations = ({splitVideoFromAudio,insertBasicCaptions,insertVideoLayers,+ replaceVideoSource,subscribeToDefaultProps: ({clientId, compositionId}) => {const clients =defaultPropsSubscriptions.get(compositionId) ?? new Set<string>();diff --git a/packages/codemods/src/index.ts b/packages/codemods/src/index.tsindex b59a60b3b2..9e3df70272 100644--- a/packages/codemods/src/index.ts+++ b/packages/codemods/src/index.ts@@ -21,6 +21,7 @@ import {getFolderFile,getRootFileForProject,} from './internals';+import {replaceVideoSource} from './replace-video-source';import {computeSequencePropsSubscriptionFromContent} from './sequence-props';import {JsxElementIdentityMismatchError} from './sequence-props/jsx-component-identity';import {JsxElementNotFoundAtLocationError} from './sequence-props/jsx-element-not-found-at-location-error';@@ -176,6 +177,7 @@ export const CodemodsInternals = {insertJsxElementIntoComposition,insertJsxElementIntoProjectWithNodePathRemappings,insertVideoLayers,+ replaceVideoSource,pasteEffects,resolveCompositionComponent,resolveCompositionComponentWithFile,diff --git a/packages/codemods/src/replace-video-source.ts b/packages/codemods/src/replace-video-source.tsnew file mode 100644index 0000000000..38d178daf1--- /dev/null+++ b/packages/codemods/src/replace-video-source.ts@@ -0,0 +1,121 @@+import type {JSXAttribute, JSXElement} from '@babel/types';+import type {SequenceNodePathRemapping} from '@remotion/studio-shared';+import * as recast from 'recast';+import type {SequenceNodePath} from 'remotion';+import {findJsxElementPathForDeletion} from './delete-jsx-nodes-internal';+import {+ captureJsxNodePaths,+ getNodePathRemappings,+} from './get-node-path-remappings';+import {recastLocToOffset} from './recast-loc-to-offset';+import {ensureNamedImport, getImportedName} from './sequence-props/imports';+import {parseAst} from './sequence-props/parse-ast';+import {+ applySourceEdits,+ captureImportSnapshots,+ getInsertImportSourceEdits,+} from './source-edits';+import {getPreferredQuote} from './source-style';++export const replaceVideoSource = ({+ input,+ nodePath,+ src,+ prettierConfigOverride = null,+}: {+ input: string;+ nodePath: SequenceNodePath;+ src: string;+ prettierConfigOverride?: Record<string, unknown> | null;+}): {+ output: string;+ logLine: number;+ nodePathRemappings: SequenceNodePathRemapping[];+} => {+ const ast = parseAst(input);+ const captured = captureJsxNodePaths(ast);+ const importSnapshots = captureImportSnapshots(ast);+ const videoPath = findJsxElementPathForDeletion(ast, nodePath);+ if (!videoPath) {+ throw new Error('Could not find the selected video element');+ }++ const video = videoPath.node as JSXElement;+ if (video.openingElement.name.type !== 'JSXIdentifier') {+ throw new Error('The selected video tag cannot be updated');+ }++ const tagName = video.openingElement.name.name;+ const importedVideo = ast.program.body.some(+ (statement) =>+ statement.type === 'ImportDeclaration' &&+ statement.importKind !== 'type' &&+ (statement.source.value === 'remotion' ||+ statement.source.value === '@remotion/media') &&+ statement.specifiers.some(+ (specifier) =>+ specifier.type === 'ImportSpecifier' &&+ specifier.local.name === tagName &&+ [+ 'Video',+ 'Html5Video',+ 'OffthreadVideo',+ 'experimental_Video',+ ].includes(getImportedName(specifier)),+ ),+ );+ if (!importedVideo) {+ throw new Error('The selected element is not a Remotion Video');+ }++ const srcAttribute = video.openingElement.attributes.find(+ (attribute): attribute is JSXAttribute =>+ attribute.type === 'JSXAttribute' &&+ attribute.name.type === 'JSXIdentifier' &&+ attribute.name.name === 'src',+ );+ if (!srcAttribute?.value?.loc) {+ throw new Error(`<${tagName}> has no src attribute`);+ }++ let staticFileLocalName = 'staticFile';+ let suffix = 1;+ while (new RegExp(`\\b${staticFileLocalName}\\b`).test(input)) {+ staticFileLocalName = `remotionStaticFile${suffix++}`;+ }++ staticFileLocalName = ensureNamedImport({+ ast,+ importedName: 'staticFile',+ localName: staticFileLocalName,+ sourcePath: 'remotion',+ });+ const stringLiteral = recast.prettyPrint(+ recast.types.builders.stringLiteral(src),+ {+ quote: getPreferredQuote(input, prettierConfigOverride),+ },+ ).code;+ const output = applySourceEdits({+ input,+ edits: [+ ...getInsertImportSourceEdits({+ ast,+ input,+ prettierConfigOverride,+ snapshots: importSnapshots,+ }),+ {+ start: recastLocToOffset(input, srcAttribute.value.loc.start),+ end: recastLocToOffset(input, srcAttribute.value.loc.end),+ replacement: `{${staticFileLocalName}(${stringLiteral})}`,+ },+ ],+ });+ const {nodePathRemappings} = getNodePathRemappings({ast, captured, output});+ return {+ output,+ logLine: video.openingElement.loc?.start.line ?? 1,+ nodePathRemappings,+ };+};diff --git a/packages/docs/docs/video-matting/index.mdx b/packages/docs/docs/video-matting/index.mdxindex 9b03d48a94..99ebf59540 100644--- a/packages/docs/docs/video-matting/index.mdx+++ b/packages/docs/docs/video-matting/index.mdx@@ -1,12 +1,12 @@---image: /generated/articles-docs-video-matting-index.pngtitle: '@remotion/video-matting'-crumb: 'Separate a video into layers'+crumb: 'Remove a video background'---# @remotion/video-matting<AvailableFrom v="4.0.523" />-Separate a video into background and foreground layers, locally in the browser or [Node.js](/docs/video-matting/node) using WebGPU.+Remove a video's background or separate it into background and foreground layers, locally in the browser or [Node.js](/docs/video-matting/node) using WebGPU.import {TableOfContents} from './table-of-contents';@@ -16,11 +16,11 @@ import {TableOfContents} from './table-of-contents';## Example-```ts twoslash title="separate-video.ts"+```ts twoslash title="remove-video-background.ts"import {downloadVideoMattingModel,loadVideoMattingModel,- separateVideoLayers,+ removeVideoBackground,} from '@remotion/video-matting';export const matteVideo = async (file: File) => {@@ -34,21 +34,18 @@ export const matteVideo = async (file: File) => {});console.log('Model already in memory:', modelHandle.alreadyLoaded);- await using layers = await separateVideoLayers({+ await using result = await removeVideoBackground({src: file,model,- onProgress: ({progress}) => console.log('Separating layers:', progress),+ onProgress: ({progress}) => console.log('Removing background:', progress),});- const [baseBlob, foregroundBlob] = await Promise.all([- layers.base.getBlob(),- layers.foreground.getBlob(),- ]);-- return {baseBlob, foregroundBlob};+ return result.video.getBlob();};```+To preserve the original video alongside the transparent result, duplicate its layer in Studio before removing the background.+## Model hostingWe noticed that hosting the models on R2 leads to faster downloading than through Hugging Face directly.@@ -65,7 +62,7 @@ A GPU is needed for this.Use [`canUseVideoMatting()`](/docs/video-matting/can-use-video-matting) to check whether your system can use video matting.The browser must also be able to decode the input video and encode VP9 with alpha through WebCodecs.-[`separateVideoLayers()`](/docs/video-matting/separate-video-layers) checks these media capabilities for the selected input.+[`removeVideoBackground()`](/docs/video-matting/remove-video-background) and [`separateVideoLayers()`](/docs/video-matting/separate-video-layers) check these media capabilities for the selected input.## Licensediff --git a/packages/docs/docs/video-matting/remove-video-background.mdx b/packages/docs/docs/video-matting/remove-video-background.mdxnew file mode 100644index 0000000000..324b851182--- /dev/null+++ b/packages/docs/docs/video-matting/remove-video-background.mdx@@ -0,0 +1,82 @@+---+image: /generated/articles-docs-video-matting-remove-video-background.png+title: removeVideoBackground()+crumb: '@remotion/video-matting'+---++# removeVideoBackground()<AvailableFrom v="4.0.528" />++Removes a video's background and returns one transparent WebM. The original audio is kept by default.++```ts twoslash title="remove-background.ts"+import {downloadVideoMattingModel, removeVideoBackground} from '@remotion/video-matting';++await downloadVideoMattingModel({model: 'modnet'});+const file = new File([], 'input.mp4');++await using result = await removeVideoBackground({src: file, model: 'modnet'});+const transparentVideo = await result.video.getBlob();+```++## Options++### src++A `string`, `URL`, or `Blob` containing the input video. A `File` may be passed because it extends `Blob`. In [Node.js](/docs/video-matting/node), strings can also be local file paths.++### model?++The model to use. Default: `modnet`. See [`getAvailableModels()`](/docs/video-matting/get-available-models).++### audio?++`keep` retains the input video's primary audio track in the output. `none` produces a silent video. Default: `keep`.++### output?++Configures the output destination using `VideoLayerOutputOptions`. Default: `web-fs` when available, otherwise `arraybuffer`.++#### output.outputTarget?++Set to `arraybuffer` to keep the encoded video in memory, or `web-fs` to store it in the browser's origin-private file system. Default: automatic selection.++#### output.outputWritable?++A `WritableStream<StreamTargetChunk>` receiving the encoded video. It cannot be combined with `outputTarget`. When set, `result.video.getBlob()` rejects because the bytes were sent to the stream.++### videoBitrate?++The VP9 video quality. Default: `very-high`. Accepts `very-low`, `low`, `medium`, `high`, `very-high`, or a positive integer bitrate in bits per second.++### audioBitrate?++The Opus audio quality when transcoding. Default: `medium`.++### keyframeIntervalInSeconds?++The maximum time between video keyframes. Default: `1`.++### signal?++An `AbortSignal` to cancel processing.++### onModelLoadProgress?++Called while initializing a downloaded model.++### onProgress?++Called while processing and finalizing the output. The callback receives the same progress shape as [`separateVideoLayers()`](/docs/video-matting/separate-video-layers).++## Return value++The result includes `video`, a `VideoLayerOutput` with `getBlob()` and `dispose()` methods, plus `model`, `width`, `height`, `durationInSeconds`, and `processedFrames`. Dispose the result after use. `await using` does this automatically.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs bun="" serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />++## See also++- [`separateVideoLayers()`](/docs/video-matting/separate-video-layers)+- [`canUseVideoMatting()`](/docs/video-matting/can-use-video-matting)diff --git a/packages/docs/docs/video-matting/table-of-contents.tsx b/packages/docs/docs/video-matting/table-of-contents.tsxindex 0e4a81d1ea..648a74530a 100644--- a/packages/docs/docs/video-matting/table-of-contents.tsx+++ b/packages/docs/docs/video-matting/table-of-contents.tsx@@ -33,6 +33,10 @@ export const TableOfContents: React.FC = () => {<strong>separateVideoLayers()</strong><div>Create base and foreground WebM layers</div></TOCItem>+ <TOCItem link="/docs/video-matting/remove-video-background">+ <strong>removeVideoBackground()</strong>+ <div>Create a transparent WebM</div>+ </TOCItem><TOCItem link="/docs/video-matting/dispose-video-matting-model"><strong>disposeVideoMattingModel()</strong><div>Release model memory</div>diff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.tsindex 96073a98ce..44a74e0f15 100644--- a/packages/docs/sidebars.ts+++ b/packages/docs/sidebars.ts@@ -971,6 +971,7 @@ const sidebars: SidebarsConfig = {'video-matting/download-video-matting-model','video-matting/load-video-matting-model','video-matting/remove-video-matting-model',+ 'video-matting/remove-video-background','video-matting/separate-video-layers','video-matting/dispose-video-matting-model',],diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.tsindex a8a940d1c0..736a09ef2c 100644--- a/packages/docs/src/data/articles.ts+++ b/packages/docs/src/data/articles.ts@@ -8994,6 +8994,15 @@ export const articles = [noAi: false,slug: 'video-matting/node',},+ {+ id: 'video-matting/remove-video-background',+ title: 'removeVideoBackground()',+ relativePath: 'docs/video-matting/remove-video-background.mdx',+ compId: 'articles-docs-video-matting-remove-video-background',+ crumb: '@remotion/video-matting',+ noAi: false,+ slug: 'video-matting/remove-video-background',+ },{id: 'video-matting/remove-video-matting-model',title: 'removeVideoMattingModel()',diff --git a/packages/docs/static/generated/articles-docs-video-matting-remove-video-background.png b/packages/docs/static/generated/articles-docs-video-matting-remove-video-background.pngnew file mode 100644index 0000000000..990237ecb8Binary files /dev/null and b/packages/docs/static/generated/articles-docs-video-matting-remove-video-background.png differdiff --git a/packages/studio-server/src/preview-server/api-routes.ts b/packages/studio-server/src/preview-server/api-routes.tsindex 2e544e1161..061ecca2d3 100644--- a/packages/studio-server/src/preview-server/api-routes.ts+++ b/packages/studio-server/src/preview-server/api-routes.ts@@ -51,6 +51,7 @@ import {handleRemoveRender} from './routes/remove-render';import {renameStaticFileHandler} from './routes/rename-static-file';import {reorderEffectHandler} from './routes/reorder-effect';import {reorderSequenceHandler} from './routes/reorder-sequence';+import {replaceVideoSourceHandler} from './routes/replace-video-source';import {handleRestartStudio} from './routes/restart-studio';import {saveEffectPropsHandler} from './routes/save-effect-props';import {saveMultipleEffectPropsHandler} from './routes/save-multiple-effect-props';@@ -132,6 +133,7 @@ export const allApiRoutes: {'/api/split-video-from-audio': splitVideoFromAudioHandler,'/api/insert-basic-captions': insertBasicCaptionsHandler,'/api/insert-video-layers': insertVideoLayersHandler,+ '/api/replace-video-source': replaceVideoSourceHandler,'/api/update-available': handleUpdate,'/api/release-notes': getReleaseNotesHandler,'/api/remotion-skills-info': remotionSkillsInfoHandler,diff --git a/packages/studio-server/src/preview-server/routes/replace-video-source.ts b/packages/studio-server/src/preview-server/routes/replace-video-source.tsnew file mode 100644index 0000000000..9d7db7f4a3--- /dev/null+++ b/packages/studio-server/src/preview-server/routes/replace-video-source.ts@@ -0,0 +1,87 @@+import {readFileSync} from 'node:fs';+import {CodemodsInternals} from '@remotion/codemods';+import {RenderInternals} from '@remotion/renderer';+import type {+ ReplaceVideoSourceRequest,+ ReplaceVideoSourceResponse,+} from '@remotion/studio-shared';+import {writeFileAndNotifyFileWatchers} from '../../file-watcher';+import {resolveFileInsideProject} from '../../helpers/resolve-file-inside-project';+import type {ApiHandler} from '../api-types';+import {formatLogFileLocation} from '../format-log-file-location';+import {broadcastSequenceNodePathMutation} from '../sequence-node-path-mutation';+import {+ printUndoHint,+ pushToUndoStack,+ suppressUndoStackInvalidation,+} from '../undo-stack';+import {+ getCodemodTimingPrefix,+ withSourceFileWriteQueue,+} from './source-file-write-queue';++export const replaceVideoSourceHandler: ApiHandler<+ ReplaceVideoSourceRequest,+ ReplaceVideoSourceResponse+> = ({input: {fileName, nodePath, src}, remotionRoot, logLevel}) =>+ withSourceFileWriteQueue<ReplaceVideoSourceResponse>(() => {+ try {+ const {absolutePath} = resolveFileInsideProject({+ remotionRoot,+ fileName,+ action: 'modify',+ });+ const fileContents = readFileSync(absolutePath, 'utf-8');+ const {output, logLine, nodePathRemappings} =+ CodemodsInternals.replaceVideoSource({+ input: fileContents,+ nodePath,+ src,+ });+ const nodePathMutation = broadcastSequenceNodePathMutation(+ [{absolutePath, remappings: nodePathRemappings}],+ null,+ );++ pushToUndoStack({+ filePath: absolutePath,+ oldContents: fileContents,+ newContents: null,+ logLevel,+ remotionRoot,+ logLine,+ description: {+ undoMessage: '↩️ Removed video background',+ redoMessage: '↪️ Removed video background',+ },+ entryType: 'replace-video-source',+ suppressHmrOnFileRestore: false,+ nodePathRemappings,+ });+ suppressUndoStackInvalidation(absolutePath);+ writeFileAndNotifyFileWatchers({+ file: absolutePath,+ content: output,+ originatorClientId: undefined,+ metadata: {skipSequencePropsUpdate: true},+ });++ const locationLabel = formatLogFileLocation({+ remotionRoot,+ absolutePath,+ line: logLine,+ });+ RenderInternals.Log.info(+ {indent: false, logLevel},+ `${getCodemodTimingPrefix(logLevel)}${RenderInternals.chalk.blueBright(locationLabel)} Replaced video source`,+ );+ printUndoHint(logLevel);+ return Promise.resolve({success: true as const, nodePathMutation});+ } catch (error) {+ return Promise.resolve({+ success: false,+ reason: (error as Error).message,+ stack: (error as Error).stack ?? '',+ } as const);+ }+ });diff --git a/packages/studio-server/src/preview-server/undo-stack.ts b/packages/studio-server/src/preview-server/undo-stack.tsindex 915003e550..783bb83c77 100644--- a/packages/studio-server/src/preview-server/undo-stack.ts+++ b/packages/studio-server/src/preview-server/undo-stack.ts@@ -44,6 +44,7 @@ type UndoEntryType =| 'split-video-from-audio'| 'insert-basic-captions'| 'insert-video-layers'+ | 'replace-video-source'| 'insert-jsx-element'| 'delete-composition'| 'rename-composition'diff --git a/packages/studio-shared/src/api-requests.ts b/packages/studio-shared/src/api-requests.tsindex c946b851cc..f01708ce84 100644--- a/packages/studio-shared/src/api-requests.ts+++ b/packages/studio-shared/src/api-requests.ts@@ -951,6 +951,14 @@ export type InsertVideoLayersResponse =| {success: true; nodePathMutation: SequenceNodePathMutation}| {success: false; reason: string; stack: string};+export type ReplaceVideoSourceRequest = {+ fileName: string;+ nodePath: SequenceNodePath;+ src: string;+};++export type ReplaceVideoSourceResponse = InsertVideoLayersResponse;+export type InsertableCompositionElement =| {type: 'solid';@@ -1453,6 +1461,10 @@ export type ApiRoutes = {InsertVideoLayersRequest,InsertVideoLayersResponse>;+ '/api/replace-video-source': ReqAndRes<+ ReplaceVideoSourceRequest,+ ReplaceVideoSourceResponse+ >;'/api/insert-jsx-element': ReqAndRes<InsertJsxElementRequest,InsertJsxElementResponsediff --git a/packages/studio-shared/src/browser-studio-operations.ts b/packages/studio-shared/src/browser-studio-operations.tsindex 9ee6a767ef..573de4ec7c 100644--- a/packages/studio-shared/src/browser-studio-operations.ts+++ b/packages/studio-shared/src/browser-studio-operations.ts@@ -64,6 +64,8 @@ import type {InsertBasicCaptionsResponse,InsertVideoLayersRequest,InsertVideoLayersResponse,+ ReplaceVideoSourceRequest,+ ReplaceVideoSourceResponse,SubscribeToDefaultPropsRequest,SubscribeToDefaultPropsResponse,SubscribeToSequencePropsRequest,@@ -232,6 +234,9 @@ export type BrowserStudioOperations = {insertVideoLayers: (request: InsertVideoLayersRequest,) => Promise<InsertVideoLayersResponse>;+ replaceVideoSource?: (+ request: ReplaceVideoSourceRequest,+ ) => Promise<ReplaceVideoSourceResponse>;subscribeToDefaultProps: (request: SubscribeToDefaultPropsRequest,) => Promise<SubscribeToDefaultPropsResponse>;diff --git a/packages/studio-shared/src/index.ts b/packages/studio-shared/src/index.tsindex d55adce154..ec12529ec9 100644--- a/packages/studio-shared/src/index.ts+++ b/packages/studio-shared/src/index.ts@@ -77,6 +77,8 @@ export {InsertBasicCaptionsResponse,InsertVideoLayersRequest,InsertVideoLayersResponse,+ ReplaceVideoSourceRequest,+ ReplaceVideoSourceResponse,InsertElementRequest,InsertElementResponse,InsertJsxElementRequest,diff --git a/packages/studio/src/components/CurrentAsset.tsx b/packages/studio/src/components/CurrentAsset.tsxindex 9a1758fe6f..2868ebfe69 100644--- a/packages/studio/src/components/CurrentAsset.tsx+++ b/packages/studio/src/components/CurrentAsset.tsx@@ -437,7 +437,7 @@ export const AssetInfo: React.FC<{<SeparationIcon color={color} style={quickActionIconStyle} />)}>- Separate foreground+ Remove background</InspectorQuickAction>) : null}{src ? (diff --git a/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsx b/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsxindex 0ba62155fe..1f2a9f67db 100644--- a/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsx+++ b/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsx@@ -173,7 +173,7 @@ const SequenceSourceQuickActions: React.FC<{const videoMattingDisabledReason = sourceActionsDisabled? 'Studio is read-only': selection.nodePathInfo.numberOfSequencesWithThisNodePath > 1- ? 'Programmatically duplicated videos cannot be separated from source'+ ? 'Programmatically duplicated videos cannot have their background removed from source': undefined;const onGenerateCaptions = useCallback(() => {if (transcriptionDisabledReason !== undefined || mediaSequence === null) {@@ -204,7 +204,7 @@ const SequenceSourceQuickActions: React.FC<{mediaSequence,transcriptionDisabledReason,]);- const onSeparateForeground = useCallback(() => {+ const onRemoveBackground = useCallback(() => {if (videoMattingDisabledReason !== undefined ||track.sequence.type !== 'video'@@ -290,13 +290,13 @@ const SequenceSourceQuickActions: React.FC<{{track.sequence.type === 'video' ? (<InspectorQuickActiondisabled={videoMattingDisabledReason !== undefined}- onClick={onSeparateForeground}+ onClick={onRemoveBackground}title={videoMattingDisabledReason}renderIcon={(color) => (<SeparationIcon style={actionIconStyle} color={color} />)}>- Separate foreground+ Remove background</InspectorQuickAction>) : null}{track.sequence.type === 'video' ? (diff --git a/packages/studio/src/components/RenderQueue/RenderQueueItemCancelButton.tsx b/packages/studio/src/components/RenderQueue/RenderQueueItemCancelButton.tsxindex 3edb3e17d7..bfb5699801 100644--- a/packages/studio/src/components/RenderQueue/RenderQueueItemCancelButton.tsx+++ b/packages/studio/src/components/RenderQueue/RenderQueueItemCancelButton.tsx@@ -67,7 +67,9 @@ export const RenderQueueCancelButton: React.FC<{job.type === 'caption'? 'Cancel transcription': job.type === 'video-matting'- ? 'Cancel video matting'+ ? 'outName' in job+ ? 'Cancel background removal'+ : 'Cancel video matting': 'Cancel render'}renderAction={renderAction}diff --git a/packages/studio/src/components/RenderQueue/VideoMattingQueueItem.tsx b/packages/studio/src/components/RenderQueue/VideoMattingQueueItem.tsxindex fee70ac02f..2d4365e090 100644--- a/packages/studio/src/components/RenderQueue/VideoMattingQueueItem.tsx+++ b/packages/studio/src/components/RenderQueue/VideoMattingQueueItem.tsx@@ -68,7 +68,11 @@ const Status: React.FC<{readonly job: VideoMattingJob}> = ({job}) => {<divstyle={statusIcon}role="progressbar"- aria-label="Video matting progress"+ aria-label={+ 'outName' in job+ ? 'Background removal progress'+ : 'Video matting progress'+ }aria-valuemin={0}aria-valuemax={100}aria-valuenow={Math.round(job.progress.value * 100)}@@ -121,7 +125,11 @@ export const VideoMattingQueueItem: React.FC<{);const messages =job.status === 'idle'- ? ['Queued for video matting']+ ? [+ 'outName' in job+ ? 'Queued for background removal'+ : 'Queued for video matting',+ ]: job.status === 'running' || job.status === 'saving'? [job.progress.message, job.progress.detail].filter((message): message is string => message !== null,@@ -130,7 +138,9 @@ export const VideoMattingQueueItem: React.FC<{? [job.error.message]: job.status === 'cancelled'? ['Cancelled']- : [job.baseOutName, job.foregroundOutName];+ : 'outName' in job+ ? [job.outName]+ : [job.baseOutName, job.foregroundOutName];const tooltip = messages.join('\n');const revealAsset = useCallback((assetName: string) => {@@ -139,38 +149,27 @@ export const VideoMattingQueueItem: React.FC<{},[selectAsset],);+ const outputName = 'outName' in job ? job.outName : job.foregroundOutName;const onClick = useCallback(() => {if (!done) return;- revealAsset(job.foregroundOutName);- }, [done, job.foregroundOutName, revealAsset]);+ revealAsset(outputName);+ }, [done, outputName, revealAsset]);const revealItems = useMemo((): ComboboxValue[] => {return [{disabled: false,- id: 'reveal-foreground',+ id: 'reveal-output',keyHint: null,- label: 'Reveal foreground',+ label: 'Reveal output',leftItem: null,- onClick: () => revealAsset(job.foregroundOutName),+ onClick: () => revealAsset(outputName),quickSwitcherLabel: null,subMenu: null,type: 'item',- value: 'reveal-foreground',- },- {- disabled: false,- id: 'reveal-background',- keyHint: null,- label: 'Reveal background',- leftItem: null,- onClick: () => revealAsset(job.baseOutName),- quickSwitcherLabel: null,- subMenu: null,- type: 'item',- value: 'reveal-background',+ value: 'reveal-output',},];- }, [job.baseOutName, job.foregroundOutName, revealAsset]);+ }, [outputName, revealAsset]);const onRemove: React.MouseEventHandler = useCallback((event) => {event.stopPropagation();@@ -208,7 +207,11 @@ export const VideoMattingQueueItem: React.FC<{) : job.status === 'failed' ? (<QueueJobErrorerror={job.error}- modalTitle="Video matting failed"+ modalTitle={+ 'outName' in job+ ? 'Background removal failed'+ : 'Video matting failed'+ }/>) : (messages.map((message) => (@@ -222,7 +225,7 @@ export const VideoMattingQueueItem: React.FC<{<Spacing x={1} />{done ? (<ActionTooltip- label="Reveal video layer"+ label="Reveal output"shortcut={null}delay={800}dismissOnClick@@ -231,7 +234,7 @@ export const VideoMattingQueueItem: React.FC<{renderAction={(color) => (<EllipsisIcon fill={color} svgProps={ellipsisIconStyle} />)}- aria-label="Reveal video layer"+ aria-label="Reveal output"values={revealItems}variant={null}/>diff --git a/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsxindex 1e178439f5..b789c76d7c 100644--- a/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx+++ b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx@@ -4,7 +4,10 @@ import {downloadVideoMattingModel,isVideoMattingModelCached,loadVideoMattingModel,+ removeVideoBackground,separateVideoLayers,+ type RemoveVideoBackgroundResult,+ type SeparateVideoLayersProgress,} from '@remotion/video-matting';import {useCallback, useContext, useEffect} from 'react';import {writeStaticFile} from '../../api/write-static-file';@@ -28,8 +31,10 @@ export const VideoMattingQueueProcessor: React.FC = () => {const processJob = useCallback(async (job: VideoMattingJob) => {const {signal} = getAbortController(job.id);- let outputs: Awaited<ReturnType<typeof separateVideoLayers>> | null =- null;+ let outputs:+ | Awaited<ReturnType<typeof separateVideoLayers>>+ | RemoveVideoBackgroundResult+ | null = null;let processingError: Error | null = null;try {signal.throwIfAborted();@@ -65,13 +70,12 @@ export const VideoMattingQueueProcessor: React.FC = () => {}),});- outputs = await separateVideoLayers({+ const mattingOptions = {signal,src: job.src,model: job.model,- audio: job.audio,videoBitrate: job.videoBitrate,- onProgress: (progress) => {+ onProgress: (progress: SeparateVideoLayersProgress) => {updateVideoMattingJobProgress(job.id, {detail:progress.stage === 'finalizing'@@ -79,56 +83,94 @@ export const VideoMattingQueueProcessor: React.FC = () => {: `Processed ${progress.processedFrames} ${progress.processedFrames === 1 ? 'frame' : 'frames'} · ${Math.round(progress.progress * 100)}%`,message:progress.stage === 'finalizing'- ? 'Finalizing video layers...'- : 'Separating foreground...',+ ? 'Finalizing video...'+ : 'outName' in job+ ? 'Removing background...'+ : 'Separating foreground...',value: 0.2 + (progress.progress ?? 1) * 0.65,});},- });+ };+ outputs =+ 'outName' in job+ ? await removeVideoBackground({...mattingOptions, audio: job.audio})+ : await separateVideoLayers({...mattingOptions, audio: job.audio});+ if (outputs === null) {+ throw new Error('Video matting produced no output.');+ }signal.throwIfAborted();updateVideoMattingJobProgress(job.id, {detail: null,- message: 'Saving video layers...',+ message: 'Saving video...',value: 0.88,});- const [base, foreground] = await Promise.all([- outputs.base.getBlob(),- outputs.foreground.getBlob(),- ]);+ const foreground = await (+ 'video' in outputs ? outputs.video : outputs.foreground+ ).getBlob();+ const base = 'base' in outputs ? await outputs.base.getBlob() : null;signal.throwIfAborted();markVideoMattingJobSaving(job.id);- await Promise.all([- base+ if ('outName' in job) {+ await foreground.arrayBuffer().then((contents) =>- writeStaticFile({contents, filePath: job.baseOutName}),- ),- foreground- .arrayBuffer()- .then((contents) =>- writeStaticFile({contents, filePath: job.foregroundOutName}),- ),- ]);+ writeStaticFile({contents, filePath: job.outName}),+ );+ } else {+ await Promise.all([+ base!+ .arrayBuffer()+ .then((contents) =>+ writeStaticFile({contents, filePath: job.baseOutName}),+ ),+ foreground+ .arrayBuffer()+ .then((contents) =>+ writeStaticFile({contents, filePath: job.foregroundOutName}),+ ),+ ]);+ }if (job.target !== null) {updateVideoMattingJobProgress(job.id, {detail: null,- message: 'Adding separated video layers...',+ message: 'Replacing video source...',value: 0.97,});- const request = {- fileName: job.target.fileName,- nodePath: job.target.nodePath.nodePath,- baseSrc: job.baseOutName,- foregroundSrc: job.foregroundOutName,- };const browserStudioOperations = getBrowserStudioOperations();- const response = browserStudioOperations- ? await browserStudioOperations.insertVideoLayers(request)- : await callApi('/api/insert-video-layers', request);- if (!response.success) {- throw new Error(response.reason);+ if ('outName' in job) {+ const request = {+ fileName: job.target.fileName,+ nodePath: job.target.nodePath.nodePath,+ src: job.outName,+ };+ const replaceSource = browserStudioOperations?.replaceVideoSource;+ if (browserStudioOperations && !replaceSource) {+ throw new Error(+ 'Browser Studio cannot replace the video source.',+ );+ }++ const response = replaceSource+ ? await replaceSource(request)+ : await callApi('/api/replace-video-source', request);+ if (!response.success) {+ throw new Error(response.reason);+ }+ } else {+ const request = {+ fileName: job.target.fileName,+ nodePath: job.target.nodePath.nodePath,+ baseSrc: job.baseOutName,+ foregroundSrc: job.foregroundOutName,+ };+ const response = browserStudioOperations+ ? await browserStudioOperations.insertVideoLayers(request)+ : await callApi('/api/insert-video-layers', request);+ if (!response.success) {+ throw new Error(response.reason);+ }}}} catch (error) {@@ -136,10 +178,15 @@ export const VideoMattingQueueProcessor: React.FC = () => {error instanceof Error ? error : new Error(String(error));}- await Promise.allSettled([- outputs?.base.dispose(),- outputs?.foreground.dispose(),- ]);+ if (outputs) {+ await Promise.allSettled([+ 'base' in outputs ? outputs.base.dispose() : null,+ 'video' in outputs+ ? outputs.video.dispose()+ : outputs.foreground.dispose(),+ ]);+ }+try {await disposeVideoMattingModel({model: job.model});} catch {diff --git a/packages/studio/src/components/RenderQueue/context.tsx b/packages/studio/src/components/RenderQueue/context.tsxindex 9834b461f5..bc33b5574b 100644--- a/packages/studio/src/components/RenderQueue/context.tsx+++ b/packages/studio/src/components/RenderQueue/context.tsx@@ -287,7 +287,10 @@ export const RenderQueueContextProvider: React.FC<{status: 'running',progress: {detail: null,- message: 'Starting video matting...',+ message:+ 'outName' in job+ ? 'Starting background removal...'+ : 'Starting video matting...',value: 0,},}diff --git a/packages/studio/src/components/RenderQueue/index.tsx b/packages/studio/src/components/RenderQueue/index.tsxindex 05caeae03a..a9b5c68319 100644--- a/packages/studio/src/components/RenderQueue/index.tsx+++ b/packages/studio/src/components/RenderQueue/index.tsx@@ -100,8 +100,10 @@ export const RenderQueue: React.FC = () => {if (isVideoMattingJob(job) &&job.status === 'done' &&- (canvasContent.asset === job.baseOutName ||- canvasContent.asset === job.foregroundOutName)+ ('outName' in job+ ? canvasContent.asset === job.outName+ : canvasContent.asset === job.baseOutName ||+ canvasContent.asset === job.foregroundOutName)) {return i;}@@ -116,7 +118,7 @@ export const RenderQueue: React.FC = () => {<div style={explainer}><Spacing y={5} /><div style={errorExplanation}>- Renders, transcriptions and video matting jobs will show up here.+ Renders, transcriptions and background removal jobs will show up here.</div><Spacing y={2} block /></div>diff --git a/packages/studio/src/components/RenderQueue/video-matting-job-types.ts b/packages/studio/src/components/RenderQueue/video-matting-job-types.tsindex 29925f16bb..d9e691894f 100644--- a/packages/studio/src/components/RenderQueue/video-matting-job-types.ts+++ b/packages/studio/src/components/RenderQueue/video-matting-job-types.ts@@ -18,13 +18,10 @@ type VideoMattingJobStatus =| {status: 'done'}| {status: 'failed'; error: {message: string; stack: string | undefined}};-export type AddVideoMattingJobParams = {+type VideoMattingJobCommon = {src: string;displayName: string;- baseOutName: string;- foregroundOutName: string;model: VideoMattingModel;- audio: VideoLayerAudio;videoBitrate: VideoMattingBitrate;target: {fileName: string;@@ -32,6 +29,19 @@ export type AddVideoMattingJobParams = {} | null;};+export type AddVideoMattingJobParams = VideoMattingJobCommon &+ (+ | {+ outName: string;+ audio: 'keep' | 'none';+ }+ | {+ baseOutName: string;+ foregroundOutName: string;+ audio: VideoLayerAudio;+ }+ );+export type VideoMattingJob = AddVideoMattingJobParams & {id: string;type: 'video-matting';diff --git a/packages/studio/src/components/VideoMatting/VideoMattingModal.tsx b/packages/studio/src/components/VideoMatting/VideoMattingModal.tsxindex a2627891d2..ad0be4499a 100644--- a/packages/studio/src/components/VideoMatting/VideoMattingModal.tsx+++ b/packages/studio/src/components/VideoMatting/VideoMattingModal.tsx@@ -3,7 +3,6 @@ import {canUseVideoMatting,getAvailableModels,isVideoMattingModelCached,- type VideoLayerAudio,type VideoMattingBitrate,type VideoMattingModel,} from '@remotion/video-matting';@@ -14,7 +13,7 @@ import React, {useMemo,useState,} from 'react';-import {BLUE_DISABLED} from '../../helpers/colors';+import {BLUE_DISABLED, LIGHT_TEXT} from '../../helpers/colors';import {Checkmark} from '../../icons/Checkmark';import {ModelsIcon} from '../../icons/models';import {SeparationIcon} from '../../icons/separation';@@ -75,13 +74,18 @@ const modalLayout: React.CSSProperties = {};const hiddenPanel: React.CSSProperties = {display: 'none'};const validationStyle: React.CSSProperties = {padding: '0 16px 8px'};+const replacementNoticeStyle: React.CSSProperties = {+ color: LIGHT_TEXT,+ fontSize: 12,+ padding: '0 16px 12px',+};type SupportState =| {type: 'checking'}| {type: 'supported'}| {type: 'unsupported'; message: string};-type Tab = 'separate' | 'models';+type Tab = 'remove' | 'models';const makeOptions = <Value extends string>({items,@@ -110,7 +114,7 @@ export const VideoMattingModal: React.FC<VideoMattingModalState> = ({src,target,}) => {- const [tab, setTab] = useState<Tab>('separate');+ const [tab, setTab] = useState<Tab>('remove');const isModelCached = useCallback((selectedModel: VideoMattingModel) =>isVideoMattingModelCached({model: selectedModel}),@@ -125,12 +129,9 @@ export const VideoMattingModal: React.FC<VideoMattingModalState> = ({() => getDefaultOutputBaseName(src, displayName, 'video'),[displayName, src],);- const [baseOutName, setBaseOutName] = useState(`${baseName}-base.webm`);- const [foregroundOutName, setForegroundOutName] = useState(- `${baseName}-foreground.webm`,- );+ const [outName, setOutName] = useState(`${baseName}-no-background.webm`);const [model, setModel] = useState<VideoMattingModel>('modnet');- const [audio, setAudio] = useState<VideoLayerAudio>('base');+ const [audio, setAudio] = useState<'keep' | 'none'>('keep');const [videoBitrate, setVideoBitrate] =useState<VideoMattingBitrate>('very-high');const [support, setSupport] = useState<SupportState>({type: 'checking'});@@ -156,9 +157,7 @@ export const VideoMattingModal: React.FC<VideoMattingModalState> = ({};}, [model]);- const normalizedBase = baseOutName.normalize('NFC').toLowerCase();- const normalizedForeground = foregroundOutName.normalize('NFC').toLowerCase();- const duplicateOutput = normalizedBase === normalizedForeground;+ const normalizedOutput = outName.normalize('NFC').toLowerCase();const queuedOutputs = new Set(videoMattingJobs.filter(@@ -167,36 +166,22 @@ export const VideoMattingModal: React.FC<VideoMattingModalState> = ({job.status === 'running' ||job.status === 'saving',)- .flatMap((job) => [job.baseOutName, job.foregroundOutName])+ .flatMap((job) =>+ 'outName' in job+ ? [job.outName]+ : [job.baseOutName, job.foregroundOutName],+ ).map((name) => name.normalize('NFC').toLowerCase()),);- const baseError =- validatePublicOutputName({extension: '.webm', outName: baseOutName}) ??- (duplicateOutput- ? 'Base and foreground outputs must be different'- : queuedOutputs.has(normalizedBase)- ? 'Another video matting job is using this output file'- : null);- const foregroundError =- validatePublicOutputName({- extension: '.webm',- outName: foregroundOutName,- }) ??- (duplicateOutput- ? 'Base and foreground outputs must be different'- : queuedOutputs.has(normalizedForeground)- ? 'Another video matting job is using this output file'- : null);- const baseExists = staticFiles.some(- (file) => file.name.normalize('NFC').toLowerCase() === normalizedBase,- );- const foregroundExists = staticFiles.some(- (file) => file.name.normalize('NFC').toLowerCase() === normalizedForeground,+ const outputError =+ validatePublicOutputName({extension: '.webm', outName}) ??+ (queuedOutputs.has(normalizedOutput)+ ? 'Another background removal job is using this output file'+ : null);+ const outputExists = staticFiles.some(+ (file) => file.name.normalize('NFC').toLowerCase() === normalizedOutput,);- const canSubmit =- support.type === 'supported' &&- baseError === null &&- foregroundError === null;+ const canSubmit = support.type === 'supported' && outputError === null;const modelOptions = useMemo(() =>@@ -214,9 +199,7 @@ export const VideoMattingModal: React.FC<VideoMattingModalState> = ({() =>makeOptions({items: [- {id: 'base' as const, label: 'Background layer'},- {id: 'foreground' as const, label: 'Foreground layer'},- {id: 'both' as const, label: 'Both layers'},+ {id: 'keep' as const, label: 'Keep original audio'},{id: 'none' as const, label: 'No audio'},],selected: audio,@@ -245,8 +228,7 @@ export const VideoMattingModal: React.FC<VideoMattingModalState> = ({addVideoMattingJob({src,displayName,- baseOutName,- foregroundOutName,+ outName,model,audio,videoBitrate,@@ -259,10 +241,9 @@ export const VideoMattingModal: React.FC<VideoMattingModalState> = ({}, [addVideoMattingJob,audio,- baseOutName,canSubmit,displayName,- foregroundOutName,+ outName,model,setSelectedModal,setSidebarCollapsedState,@@ -270,8 +251,7 @@ export const VideoMattingModal: React.FC<VideoMattingModalState> = ({target,videoBitrate,]);- const title =- target === null ? `Track matting ${displayName}` : 'Separate foreground';+ const title = `Remove background from ${displayName}`;return (<DismissableModal ariaLabel={title}>@@ -290,23 +270,23 @@ export const VideoMattingModal: React.FC<VideoMattingModalState> = ({: BLUE_DISABLED,}}>- Separate+ Remove background</Button></div><div style={modalLayout}><div style={leftSidebar}><VerticalTabautoFocus- onClick={() => setTab('separate')}+ onClick={() => setTab('remove')}renderIcon={(color) => (<div style={iconContainer}><SeparationIcon color={color} style={icon} /></div>)}- selected={tab === 'separate'}+ selected={tab === 'remove'}style={horizontalTab}>- Separate+ Remove background</VerticalTab><VerticalTabonClick={() => setTab('models')}@@ -322,38 +302,28 @@ export const VideoMattingModal: React.FC<VideoMattingModalState> = ({</VerticalTab></div><div- style={tab === 'separate' ? panelStyle : hiddenPanel}+ style={tab === 'remove' ? panelStyle : hiddenPanel}className={VERTICAL_SCROLLBAR_CLASSNAME}>+ {target !== null ? (+ <div style={replacementNoticeStyle}>+ This replaces the selected video's source. Duplicate the+ layer first to keep the original alongside it.+ </div>+ ) : null}<RenderModalOutputName- ariaLabel="Base video output file"- existingOutputPath={- window.remotion_publicFolderExists- ? `${window.remotion_publicFolderExists}/${baseOutName}`- : null- }- existence={baseExists}- inputStyle={input}- label="Base output in public/"- onValueChange={(event) => setBaseOutName(event.target.value)}- outName={baseOutName}- validationMessage={baseError}- />- <RenderModalOutputName- ariaLabel="Foreground video output file"+ ariaLabel="Video output file"existingOutputPath={window.remotion_publicFolderExists- ? `${window.remotion_publicFolderExists}/${foregroundOutName}`+ ? `${window.remotion_publicFolderExists}/${outName}`: null}- existence={foregroundExists}+ existence={outputExists}inputStyle={input}- label="Foreground output in public/"- onValueChange={(event) =>- setForegroundOutName(event.target.value)- }- outName={foregroundOutName}- validationMessage={foregroundError}+ label="Output in public/"+ onValueChange={(event) => setOutName(event.target.value)}+ outName={outName}+ validationMessage={outputError}/><RenderModalHr /><div style={optionRow}>diff --git a/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx b/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsxindex 32303e5b22..c86e0a36e3 100644--- a/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx+++ b/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx@@ -12,13 +12,9 @@ export const VideoMattingModalWithOptionalPackage: React.FC<{readonly state: VideoMattingModalState;}> = ({state}) => (<OptionalPackageModal- ariaLabel={- state.target === null- ? 'Install video matting package'- : 'Install video matting package to separate foreground'- }+ ariaLabel="Install video matting package to remove background"packageName={VIDEO_MATTING_PACKAGE}- title={state.target === null ? 'Track matting' : 'Separate foreground'}+ title="Remove background"><Suspense fallback={null}><LazyVideoMattingModal {...state} />diff --git a/packages/studio/src/components/WebMcp.tsx b/packages/studio/src/components/WebMcp.tsxindex 01351f7b4a..5e5a7fa46c 100644--- a/packages/studio/src/components/WebMcp.tsx+++ b/packages/studio/src/components/WebMcp.tsx@@ -634,10 +634,10 @@ export const WebMcp: FC = () => {),modelContext.registerTool({- name: 'separate_video_layers',- title: 'Separate Studio video layers',+ name: 'remove_video_background',+ title: 'Remove video background',description:- 'Separate a video asset from the public folder into background and foreground WebM files and add the work to the Jobs queue. If assetPath is omitted, the asset currently open in Studio is used.',+ 'Remove the background from a video asset in the public folder and add the work to the Jobs queue. If assetPath is omitted, the asset currently open in Studio is used.',inputSchema: {type: 'object',properties: {@@ -646,15 +646,10 @@ export const WebMcp: FC = () => {description:'Optional path relative to public/. Defaults to the asset currently open in Studio.',},- baseOutputPath: {- type: 'string',- description:- 'Optional background output path relative to public/.',- },- foregroundOutputPath: {+ outputPath: {type: 'string',description:- 'Optional foreground output path relative to public/.',+ 'Optional transparent video output path relative to public/.',},model: {type: 'string',@@ -662,8 +657,8 @@ export const WebMcp: FC = () => {},audio: {type: 'string',- enum: ['base', 'foreground', 'both', 'none'],- default: 'base',+ enum: ['keep', 'none'],+ default: 'keep',},videoBitrate: {oneOf: [@@ -690,7 +685,7 @@ export const WebMcp: FC = () => {staticFiles: staticFilesRef.current,});if (getPreviewFileType(assetPath) !== 'video') {- throw new Error('The separation asset must be a video.');+ throw new Error('The input asset must be a video.');}const videoMatting = await import('@remotion/video-matting');@@ -706,14 +701,9 @@ export const WebMcp: FC = () => {throw new Error(`Unknown video matting model: ${modelName}.`);}- const audio = input.audio ?? 'base';- if (- audio !== 'base' &&- audio !== 'foreground' &&- audio !== 'both' &&- audio !== 'none'- ) {- throw new Error('audio must be base, foreground, both, or none.');+ const audio = input.audio ?? 'keep';+ if (audio !== 'keep' && audio !== 'none') {+ throw new Error('audio must be keep or none.');}const videoBitrate = input.videoBitrate ?? 'very-high';@@ -737,53 +727,31 @@ export const WebMcp: FC = () => {displayName,'video',);- const baseOutputPath =- input.baseOutputPath ?? `${baseName}-base.webm`;- const foregroundOutputPath =- input.foregroundOutputPath ?? `${baseName}-foreground.webm`;- if (- typeof baseOutputPath !== 'string' ||- typeof foregroundOutputPath !== 'string'- ) {- throw new Error('Output paths must be strings.');+ const outputPath =+ input.outputPath ?? `${baseName}-no-background.webm`;+ if (typeof outputPath !== 'string') {+ throw new Error('Output path must be a string.');}- const baseError = validatePublicOutputName({- extension: '.webm',- outName: baseOutputPath,- });- const foregroundError = validatePublicOutputName({+ const outputError = validatePublicOutputName({extension: '.webm',- outName: foregroundOutputPath,+ outName: outputPath,});- if (baseError !== null || foregroundError !== null) {- throw new Error(- baseError ?? foregroundError ?? 'Invalid output path.',- );- }-- if (- baseOutputPath.normalize('NFC').toLowerCase() ===- foregroundOutputPath.normalize('NFC').toLowerCase()- ) {- throw new Error(- 'Background and foreground outputs must be different.',- );+ if (outputError !== null) {+ throw new Error(outputError);}const jobId = addVideoMattingJob({audio,- baseOutName: baseOutputPath,displayName,- foregroundOutName: foregroundOutputPath,+ outName: outputPath,model,src,target: null,videoBitrate,});return {- baseOutputPath,- foregroundOutputPath,+ outputPath,jobId,success: true,};diff --git a/packages/video-matting/src/create-video-layer-output.ts b/packages/video-matting/src/create-video-layer-output.tsindex 9168e8de8a..12f1dd4004 100644--- a/packages/video-matting/src/create-video-layer-output.ts+++ b/packages/video-matting/src/create-video-layer-output.ts@@ -147,7 +147,7 @@ export const createVideoLayerOutput = async <F extends OutputFormat>({const output = new Output({format, target});output.setMetadataTags({- comment: `Separated with @remotion/video-matting ${version}`,+ comment: `Processed with @remotion/video-matting ${version}`,});let finalizationPromise: Promise<VideoLayerOutput> | null = null;let cancellationPromise: Promise<void> | null = null;diff --git a/packages/video-matting/src/create-video-matting-frames.ts b/packages/video-matting/src/create-video-matting-frames.tsindex 5847fcf409..38a349e6da 100644--- a/packages/video-matting/src/create-video-matting-frames.ts+++ b/packages/video-matting/src/create-video-matting-frames.ts@@ -25,11 +25,12 @@ export type VideoMattingFramesOptions = {keyframeIntervalInSeconds: number;videoStartTimestamp: number;videoEndTimestamp: number;+ includeBase: boolean;};export type VideoMattingFrames = {frames: AsyncGenerator<VideoMattingFrame, void, unknown>;- baseSource: VideoSource;+ baseSource: VideoSource | null;foregroundSource: VideoSource;addFrame: (options: {frame: VideoMattingFrame;@@ -60,6 +61,7 @@ export const createVideoMattingFrames = async (keyframeIntervalInSeconds,videoStartTimestamp,videoEndTimestamp,+ includeBase,} = options;const sink = new CanvasSink(videoTrack, {alpha: true,@@ -68,16 +70,22 @@ export const createVideoMattingFrames = async (fit: 'fill',poolSize: 1,});- const baseCanvas = createVideoMattingCanvas({width, height});+ const baseCanvas = includeBase+ ? createVideoMattingCanvas({width, height})+ : null;const foregroundCanvas = createVideoMattingCanvas({width, height});- const baseContext = getVideoMattingCanvasContext(baseCanvas);+ const baseContext = baseCanvas+ ? getVideoMattingCanvasContext(baseCanvas)+ : null;const foregroundContext = getVideoMattingCanvasContext(foregroundCanvas);- const baseSource = new CanvasSource(baseCanvas, {- codec: 'vp9',- quality: videoQuality,- keyFrameInterval: keyframeIntervalInSeconds,- alpha: 'discard',- });+ const baseSource = baseCanvas+ ? new CanvasSource(baseCanvas, {+ codec: 'vp9',+ quality: videoQuality,+ keyFrameInterval: keyframeIntervalInSeconds,+ alpha: 'discard',+ })+ : null;const foregroundSource = new CanvasSource(foregroundCanvas, {codec: 'vp9',quality: videoQuality,@@ -105,12 +113,15 @@ export const createVideoMattingFrames = async (throw new Error('Expected a canvas frame.');}- drawOpaqueBaseFrame({- context: baseContext,- source: frame.image,- width,- height,- });+ if (baseContext) {+ drawOpaqueBaseFrame({+ context: baseContext,+ source: frame.image,+ width,+ height,+ });+ }+drawForegroundFrame({context: foregroundContext,result: foreground,@@ -119,7 +130,7 @@ export const createVideoMattingFrames = async (targetHeight: height,});await Promise.all([- baseSource.add(timestamp, duration),+ baseSource?.add(timestamp, duration),foregroundSource.add(timestamp, duration),]);},diff --git a/packages/video-matting/src/index.ts b/packages/video-matting/src/index.tsindex 57acd6497f..0e7e779616 100644--- a/packages/video-matting/src/index.ts+++ b/packages/video-matting/src/index.ts@@ -34,8 +34,13 @@ export type {VideoLayerOutputOptions,VideoLayerOutputTarget,} from './output-target';-export {separateVideoLayers} from './separate-video-layers';+export {+ removeVideoBackground,+ separateVideoLayers,+} from './separate-video-layers';export type {+ RemoveVideoBackgroundOptions,+ RemoveVideoBackgroundResult,SeparateVideoLayersOptions,SeparateVideoLayersProgress,SeparateVideoLayersResult,diff --git a/packages/video-matting/src/node-video-matting-frames.ts b/packages/video-matting/src/node-video-matting-frames.tsindex c638b28e2c..054d15c929 100644--- a/packages/video-matting/src/node-video-matting-frames.ts+++ b/packages/video-matting/src/node-video-matting-frames.ts@@ -12,14 +12,17 @@ export const createNodeVideoMattingFrames = ({keyframeIntervalInSeconds,videoStartTimestamp,videoEndTimestamp,+ includeBase,}: VideoMattingFramesOptions): VideoMattingFrames => {const sink = new VideoSampleSink(videoTrack);- const baseSource = new VideoSampleSource({- codec: 'vp9',- quality: videoQuality,- keyFrameInterval: keyframeIntervalInSeconds,- alpha: 'discard',- });+ const baseSource = includeBase+ ? new VideoSampleSource({+ codec: 'vp9',+ quality: videoQuality,+ keyFrameInterval: keyframeIntervalInSeconds,+ alpha: 'discard',+ })+ : null;const foregroundSource = new VideoSampleSource({codec: 'vp9',quality: videoQuality,@@ -64,7 +67,7 @@ export const createNodeVideoMattingFrames = ({throw new Error('Expected an RGBA frame.');}- const baseData = frame.image.data.slice();+ const baseData = includeBase ? frame.image.data.slice() : null;let foregroundData: Uint8ClampedArray<ArrayBuffer>;if (foreground.width === width && foreground.height === height) {foregroundData = foreground.data.slice();@@ -93,23 +96,28 @@ export const createNodeVideoMattingFrames = ({}}- for (let i = 0; i < baseData.length; i += 4) {- const alpha = baseData[i + 3]! / 255;+ for (let i = 0; i < foregroundData.length; i += 4) {+ const alpha = frame.image.data[i + 3]! / 255;// Match the browser's opaque black base and destination-in foreground.- baseData[i] = Math.round(baseData[i]! * alpha);- baseData[i + 1] = Math.round(baseData[i + 1]! * alpha);- baseData[i + 2] = Math.round(baseData[i + 2]! * alpha);- baseData[i + 3] = 255;+ if (baseData) {+ baseData[i] = Math.round(baseData[i]! * alpha);+ baseData[i + 1] = Math.round(baseData[i + 1]! * alpha);+ baseData[i + 2] = Math.round(baseData[i + 2]! * alpha);+ baseData[i + 3] = 255;+ }+foregroundData[i + 3] = Math.round(foregroundData[i + 3]! * alpha);}- const base = new VideoSample(baseData, {- format: 'RGBA',- codedWidth: width,- codedHeight: height,- timestamp,- duration,- });+ const base = baseData+ ? new VideoSample(baseData, {+ format: 'RGBA',+ codedWidth: width,+ codedHeight: height,+ timestamp,+ duration,+ })+ : null;const foregroundFrame = new VideoSample(foregroundData, {format: 'RGBA',codedWidth: width,@@ -119,7 +127,7 @@ export const createNodeVideoMattingFrames = ({});try {const results = await Promise.allSettled([- baseSource.add(base),+ baseSource?.add(base!),foregroundSource.add(foregroundFrame),]);for (const result of results) {@@ -128,7 +136,7 @@ export const createNodeVideoMattingFrames = ({}}} finally {- base.close();+ base?.close();foregroundFrame.close();}},diff --git a/packages/video-matting/src/prepare-audio.ts b/packages/video-matting/src/prepare-audio.tsindex bf0b108d06..a0ca61e02a 100644--- a/packages/video-matting/src/prepare-audio.ts+++ b/packages/video-matting/src/prepare-audio.ts@@ -48,7 +48,7 @@ export const prepareAudio = async <forceTranscode,}: {input: Input;- baseOutput: Output<WebMOutputFormat, BaseTarget>;+ baseOutput: Output<WebMOutputFormat, BaseTarget> | null;foregroundOutput: Output<WebMOutputFormat, ForegroundTarget>;destination: VideoMattingAudioDestination;videoStartTimestamp: number;@@ -109,11 +109,18 @@ export const prepareAudio = async <};}- const outputs =+ if (+ (destination === 'base' || destination === 'both') &&+ baseOutput === null+ ) {+ throw new Error('A base output is required for this audio destination.');+ }++ const outputs: Output<WebMOutputFormat, Target>[] =destination === 'both'- ? [baseOutput, foregroundOutput]+ ? [baseOutput!, foregroundOutput]: destination === 'base'- ? [baseOutput]+ ? [baseOutput!]: [foregroundOutput];const sourceCodec = await audioTrack.getCodec();const canCopyPackets =diff --git a/packages/video-matting/src/separate-video-layers.ts b/packages/video-matting/src/separate-video-layers.tsindex 822266e409..13ac142151 100644--- a/packages/video-matting/src/separate-video-layers.ts+++ b/packages/video-matting/src/separate-video-layers.ts@@ -76,6 +76,23 @@ export type SeparateVideoLayersResult = AsyncDisposable & {processedFrames: number;};+export type RemoveVideoBackgroundOptions = Omit<+ SeparateVideoLayersOptions,+ 'audio' | 'outputs'+> & {+ audio?: 'keep' | 'none';+ output?: VideoLayerOutputOptions;+};++export type RemoveVideoBackgroundResult = AsyncDisposable & {+ video: VideoLayerOutput;+ model: VideoMattingModel;+ width: number;+ height: number;+ durationInSeconds: number;+ processedFrames: number;+};+const AUDIO_DESTINATIONS: VideoLayerAudio[] = ['base','foreground',@@ -88,7 +105,7 @@ const createAbortError = (signal: AbortSignal): unknown => {return signal.reason;}- const error = new Error('Video layer separation was aborted.');+ const error = new Error('Video matting was aborted.');error.name = 'AbortError';return error;};@@ -151,9 +168,12 @@ const validateLayerOutputOptions = ({}};-const validateOptions = (options: SeparateVideoLayersOptions) => {+const validateOptions = (+ options: SeparateVideoLayersOptions,+ operationName: 'separateVideoLayers' | 'removeVideoBackground',+) => {if (!options || typeof options !== 'object') {- throw new TypeError('separateVideoLayers() expects an options object.');+ throw new TypeError(`${operationName}() expects an options object.`);}const isBlob = typeof Blob !== 'undefined' && options.src instanceof Blob;@@ -269,9 +289,11 @@ const makeInput = async (src: string | URL | Blob): Promise<Input> => {const probeVideoInput = async ({input,videoQuality,+ includeBase,}: {input: Input;videoQuality: Quality;+ includeBase: boolean;}): Promise<{videoTrack: InputVideoTrack; width: number; height: number}> => {if (!(await input.canRead())) {throw new Error('The input is not a supported media file.');@@ -300,12 +322,14 @@ const probeVideoInput = async ({}const [canEncodeBase, canEncodeForeground] = await Promise.all([- canEncodeVideo('vp9', {- width,- height,- quality: videoQuality,- alpha: 'discard',- }),+ includeBase+ ? canEncodeVideo('vp9', {+ width,+ height,+ quality: videoQuality,+ alpha: 'discard',+ })+ : Promise.resolve(true),canEncodeVideo('vp9', {width,height,@@ -315,21 +339,25 @@ const probeVideoInput = async ({]);if (!canEncodeBase || !canEncodeForeground) {throw new Error(- 'This environment cannot encode the VP9 video streams required for video layer separation.',+ 'This environment cannot encode the VP9 video streams required for video matting.',);}return {videoTrack, width, height};};-export const separateVideoLayers = async (+const runVideoMatting = async (options: SeparateVideoLayersOptions,-): Promise<SeparateVideoLayersResult> => {- validateOptions(options);+ includeBase: boolean,+): Promise<SeparateVideoLayersResult | RemoveVideoBackgroundResult> => {+ validateOptions(+ options,+ includeBase ? 'separateVideoLayers' : 'removeVideoBackground',+ );throwIfAborted(options.signal);const model = options.model ?? 'modnet';- const audio = options.audio ?? 'base';+ const audio = options.audio ?? (includeBase ? 'base' : 'foreground');const videoQuality = resolveVideoMattingQuality(options.videoBitrate ?? 'very-high',);@@ -346,6 +374,7 @@ export const separateVideoLayers = async (const {videoTrack, width, height} = await probeVideoInput({input,videoQuality,+ includeBase,});const [inputFirstVideoTimestamp, videoEndTimestamp] = await Promise.all([videoTrack.getFirstTimestamp(),@@ -405,6 +434,7 @@ export const separateVideoLayers = async (keyframeIntervalInSeconds,videoStartTimestamp,videoEndTimestamp,+ includeBase,});const iterator = videoFrames.frames;let nextFrame = await iterator.next();@@ -416,10 +446,13 @@ export const separateVideoLayers = async (throwIfAborted(options.signal);- baseOutput = await createVideoLayerOutput({- format: new WebMOutputFormat(),- options: options.outputs?.base,- });+ if (includeBase) {+ baseOutput = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: options.outputs?.base,+ });+ }+throwIfAborted(options.signal);foregroundOutput = await createVideoLayerOutput({format: new WebMOutputFormat(),@@ -427,12 +460,15 @@ export const separateVideoLayers = async (});throwIfAborted(options.signal);- baseOutput.output.addVideoTrack(videoFrames.baseSource);+ if (baseOutput && videoFrames.baseSource) {+ baseOutput.output.addVideoTrack(videoFrames.baseSource);+ }+foregroundOutput.output.addVideoTrack(videoFrames.foregroundSource);audioWriter = await prepareAudio({input,- baseOutput: baseOutput.output,+ baseOutput: baseOutput?.output ?? null,foregroundOutput: foregroundOutput.output,destination: audio,videoStartTimestamp,@@ -443,7 +479,7 @@ export const separateVideoLayers = async (throwIfAborted(options.signal);await Promise.all([- baseOutput.output.start(),+ baseOutput?.output.start(),foregroundOutput.output.start(),]);throwIfAborted(options.signal);@@ -514,28 +550,27 @@ export const separateVideoLayers = async (durationInSeconds,});await audioWriter.finishAudio();- videoFrames.baseSource.close();+ videoFrames.baseSource?.close();videoFrames.foregroundSource.close();const [base, foreground] = await Promise.all([- baseOutput.finalize(),+ baseOutput?.finalize() ?? Promise.resolve(null),foregroundOutput.finalize(),]);throwIfAborted(options.signal);completed = true;const separated = {- base,- foreground,+ ...(base ? {base, foreground} : {video: foreground}),model,width,height,durationInSeconds,processedFrames,- } as SeparateVideoLayersResult;+ } as SeparateVideoLayersResult | RemoveVideoBackgroundResult;Object.defineProperty(separated, Symbol.asyncDispose, {enumerable: false,value: async () => {- await Promise.all([base.dispose(), foreground.dispose()]);+ await Promise.all([base?.dispose(), foreground.dispose()]);},});return separated;@@ -573,3 +608,20 @@ export const separateVideoLayers = async (input.dispose();}};++export const separateVideoLayers = async (+ options: SeparateVideoLayersOptions,+): Promise<SeparateVideoLayersResult> =>+ (await runVideoMatting(options, true)) as SeparateVideoLayersResult;++export const removeVideoBackground = async (+ options: RemoveVideoBackgroundOptions,+): Promise<RemoveVideoBackgroundResult> =>+ (await runVideoMatting(+ {+ ...options,+ audio: options.audio === 'none' ? 'none' : 'foreground',+ outputs: {foreground: options.output},+ },+ false,+ )) as RemoveVideoBackgroundResult;