Skip to content

🎥 Make videos programmatically with React

@remotion/studio: Add Separate foreground action for videos

JonnyBurger20a8ca92495c
diff --git a/packages/browser-studio/src/browser-studio-operations.ts b/packages/browser-studio/src/browser-studio-operations.ts
index cccb54d14f..f18cdda6fd 100644
--- a/packages/browser-studio/src/browser-studio-operations.ts
+++ b/packages/browser-studio/src/browser-studio-operations.ts
@@ -17,6 +17,7 @@ import {
getRootFileForProject,
insertJsxElementIntoProjectWithNodePathRemappings,
insertBasicCaptions as insertBasicCaptionsCodemod,
+ insertVideoLayers as insertVideoLayersCodemod,
JsxElementIdentityMismatchError,
JsxElementNotFoundAtLocationError,
makeInMemoryInsertJsxElementCodemodEnvironment,
@@ -1986,6 +1987,50 @@ export const createBrowserStudioOperations = ({
}
};
+ const insertVideoLayers: BrowserStudioOperations['insertVideoLayers'] = ({
+ fileName,
+ nodePath,
+ baseSrc,
+ foregroundSrc,
+ }) => {
+ try {
+ const project = getProject();
+ const absolutePath = findProjectFile({filePath: fileName, project});
+ const result = insertVideoLayersCodemod({
+ input: project.files[absolutePath],
+ nodePath,
+ baseSrc,
+ foregroundSrc,
+ });
+ const nodePathMutation = controller.applyMutation({
+ undoRedoNavigation: null,
+ timelineSelection: null,
+ fileName: absolutePath,
+ mutate: () => ({
+ ...project,
+ files: {...project.files, [absolutePath]: result.output},
+ }),
+ nodePathMutationFiles: [
+ {
+ absolutePath,
+ remappings: result.nodePathRemappings,
+ },
+ ],
+ });
+ if (nodePathMutation === null) {
+ throw new Error('Could not insert separated video layers');
+ }
+
+ return Promise.resolve({success: true as const, nodePathMutation});
+ } catch (error) {
+ return Promise.resolve({
+ success: false as const,
+ reason: error instanceof Error ? error.message : String(error),
+ stack: error instanceof Error && error.stack ? error.stack : '',
+ });
+ }
+ };
+
const insertJsxElement: BrowserStudioOperations['insertJsxElement'] = async (
request,
) => {
@@ -2437,6 +2482,7 @@ export const createBrowserStudioOperations = ({
},
splitVideoFromAudio,
insertBasicCaptions,
+ insertVideoLayers,
subscribeToDefaultProps: ({clientId, compositionId}) => {
const clients =
defaultPropsSubscriptions.get(compositionId) ?? new Set<string>();
diff --git a/packages/browser-studio/src/test/browser-studio-operations.test.ts b/packages/browser-studio/src/test/browser-studio-operations.test.ts
index 20dda6bf17..682facbd93 100644
--- a/packages/browser-studio/src/test/browser-studio-operations.test.ts
+++ b/packages/browser-studio/src/test/browser-studio-operations.test.ts
@@ -404,6 +404,34 @@ registerRoot(Root);`,
expect(undoResult.success).toBe(true);
expect(currentProject.files[fileName]).toBe(initialSource);
+ events.length = 0;
+ const videoLayersResult = await operations.insertVideoLayers({
+ fileName: 'src/Composition.tsx',
+ nodePath: subscription.nodePath.nodePath,
+ baseSrc: 'video-base.webm',
+ foregroundSrc: 'video-foreground.webm',
+ });
+ if (!videoLayersResult.success) {
+ throw new Error(videoLayersResult.reason);
+ }
+
+ const videoLayersOutput = currentProject.files[fileName].replace(/\s+/g, ' ');
+ expect(currentProject.files[fileName]).toContain(
+ `import {staticFile} from 'remotion';`,
+ );
+ expect(videoLayersOutput).toContain(
+ "<Video src={staticFile('video-base.webm')} from={10} durationInFrames={20} volume={0.5} style={{opacity: 0.5}} />",
+ );
+ expect(videoLayersOutput).toContain(
+ "<Video src={staticFile('video-foreground.webm')} from={10} durationInFrames={20} volume={0.5} style={{opacity: 0.5}} />",
+ );
+ expect(events).toContainEqual({
+ type: 'sequence-node-paths-remapped',
+ mutation: videoLayersResult.nodePathMutation,
+ });
+ expect((await operations.undo()).success).toBe(true);
+ expect(currentProject.files[fileName]).toBe(initialSource);
+
events.length = 0;
const captionsResult = await operations.insertBasicCaptions({
fileName: 'src/Composition.tsx',
diff --git a/packages/studio-codemods/src/index.ts b/packages/studio-codemods/src/index.ts
index 7b6602d3f1..a26dec8e72 100644
--- a/packages/studio-codemods/src/index.ts
+++ b/packages/studio-codemods/src/index.ts
@@ -86,6 +86,7 @@ export {simpleDiff} from './simple-diff';
export {splitJsxSequence, splitJsxSequences} from './split-jsx-sequence';
export {splitVideoFromAudio} from './split-video-from-audio';
export {insertBasicCaptions} from './insert-basic-captions';
+export {insertVideoLayers} from './insert-video-layers';
export {
basicCaptionsElementSource,
getBasicCaptionsElementFile,
diff --git a/packages/studio-codemods/src/insert-video-layers.ts b/packages/studio-codemods/src/insert-video-layers.ts
new file mode 100644
index 0000000000..7729b5c883
--- /dev/null
+++ b/packages/studio-codemods/src/insert-video-layers.ts
@@ -0,0 +1,303 @@
+import type {
+ JSXAttribute,
+ JSXElement,
+ JSXExpressionContainer,
+ JSXFragment,
+} from '@babel/types';
+import type {SequenceNodePathRemapping} from '@remotion/studio-shared';
+import * as recast from 'recast';
+import type {SequenceNodePath} from 'remotion';
+import {findJsxElementPathForDeletion} from './delete-jsx-node';
+import {
+ captureJsxNodePaths,
+ getNodePathRemappings,
+} from './get-node-path-remappings';
+import {indentInsertedJsx} from './print-jsx';
+import {recastLocToOffset} from './recast-loc-to-offset';
+import {ensureNamedImport, getImportedName} from './sequence-props/imports';
+import {parseAst} from './sequence-props/parse-ast';
+import {
+ applySourceEdits,
+ captureImportSnapshots,
+ getAdjacentJsxInsertionSourceEdit,
+ getInsertImportSourceEdits,
+} from './source-edits';
+import {
+ getEndOfLine,
+ getIndentationUnit,
+ getLineIndent,
+ getPreferredQuote,
+ indentContinuationLines,
+} from './source-style';
+
+const b = recast.types.builders;
+
+const makeStaticFileSource = ({
+ input,
+ localName,
+ prettierConfigOverride,
+ src,
+}: {
+ input: string;
+ localName: string;
+ prettierConfigOverride: Record<string, unknown> | null;
+ src: string;
+}) => {
+ const stringLiteral = recast.prettyPrint(b.stringLiteral(src), {
+ quote: getPreferredQuote(input, prettierConfigOverride),
+ }).code;
+ return `{${localName}(${stringLiteral})}`;
+};
+
+const replaceSrcInElementSource = ({
+ element,
+ input,
+ srcAttribute,
+ srcSource,
+}: {
+ element: JSXElement;
+ input: string;
+ srcAttribute: JSXAttribute;
+ srcSource: string;
+}) => {
+ if (!element.loc || !srcAttribute.value?.loc) {
+ throw new Error('Could not locate the selected video source');
+ }
+
+ const elementStart = recastLocToOffset(input, element.loc.start);
+ const elementEnd = recastLocToOffset(input, element.loc.end);
+ const valueStart = recastLocToOffset(input, srcAttribute.value.loc.start);
+ const valueEnd = recastLocToOffset(input, srcAttribute.value.loc.end);
+ const originalIndent = getLineIndent({input, offset: elementStart});
+ const elementSource =
+ input.slice(elementStart, valueStart) +
+ srcSource +
+ input.slice(valueEnd, elementEnd);
+
+ return elementSource
+ .split(/\r?\n/)
+ .map((line, index) => {
+ if (index === 0) {
+ return line;
+ }
+
+ return line.startsWith(originalIndent)
+ ? line.slice(originalIndent.length)
+ : line.trimStart();
+ })
+ .join(getEndOfLine(input));
+};
+
+const parseVideoElement = (source: string): JSXElement => {
+ const statement = parseAst(`const element = (${source});`).program.body[0];
+ if (
+ statement?.type !== 'VariableDeclaration' ||
+ statement.declarations[0]?.init?.type !== 'JSXElement'
+ ) {
+ throw new Error('Could not generate the separated video layer');
+ }
+
+ return statement.declarations[0].init;
+};
+
+export const insertVideoLayers = ({
+ input,
+ nodePath,
+ baseSrc,
+ foregroundSrc,
+ prettierConfigOverride = null,
+}: {
+ input: string;
+ nodePath: SequenceNodePath;
+ baseSrc: string;
+ foregroundSrc: string;
+ prettierConfigOverride?: Record<string, unknown> | null;
+}): {
+ output: string;
+ logLine: number;
+ nodePathRemappings: SequenceNodePathRemapping[];
+} => {
+ const ast = parseAst(input);
+ const captured = captureJsxNodePaths(ast);
+ const importSnapshots = captureImportSnapshots(ast);
+ const videoPath = findJsxElementPathForDeletion(ast, nodePath);
+ if (!videoPath) {
+ throw new Error('Could not find the selected video element');
+ }
+
+ const video = videoPath.node as JSXElement;
+ if (video.openingElement.name.type !== 'JSXIdentifier') {
+ throw new Error('The selected video tag cannot be separated');
+ }
+
+ const tagName = video.openingElement.name.name;
+ const importedVideo = ast.program.body.some(
+ (statement) =>
+ statement.type === 'ImportDeclaration' &&
+ statement.importKind !== 'type' &&
+ (statement.source.value === 'remotion' ||
+ statement.source.value === '@remotion/media') &&
+ statement.specifiers.some(
+ (specifier) =>
+ specifier.type === 'ImportSpecifier' &&
+ specifier.local.name === tagName &&
+ [
+ 'Video',
+ 'Html5Video',
+ 'OffthreadVideo',
+ 'experimental_Video',
+ ].includes(getImportedName(specifier)),
+ ),
+ );
+ if (!importedVideo) {
+ throw new Error('The selected element is not a Remotion Video');
+ }
+
+ const srcAttribute = video.openingElement.attributes.find(
+ (attribute): attribute is JSXAttribute =>
+ attribute.type === 'JSXAttribute' &&
+ attribute.name.type === 'JSXIdentifier' &&
+ attribute.name.name === 'src',
+ );
+ if (!srcAttribute?.value) {
+ throw new Error(`<${tagName}> has no src attribute`);
+ }
+
+ if (!srcAttribute.value.loc) {
+ throw new Error('Could not locate the selected video source');
+ }
+
+ const srcValueStart = recastLocToOffset(input, srcAttribute.value.loc.start);
+ const srcValueEnd = recastLocToOffset(input, srcAttribute.value.loc.end);
+
+ let staticFileLocalName = 'staticFile';
+ let suffix = 1;
+ while (new RegExp(`\\b${staticFileLocalName}\\b`).test(input)) {
+ staticFileLocalName = `remotionStaticFile${suffix++}`;
+ }
+
+ staticFileLocalName = ensureNamedImport({
+ ast,
+ importedName: 'staticFile',
+ localName: staticFileLocalName,
+ sourcePath: 'remotion',
+ });
+ const baseSrcSource = makeStaticFileSource({
+ input,
+ localName: staticFileLocalName,
+ prettierConfigOverride,
+ src: baseSrc,
+ });
+ const foregroundSrcSource = makeStaticFileSource({
+ input,
+ localName: staticFileLocalName,
+ prettierConfigOverride,
+ src: foregroundSrc,
+ });
+ const baseVideoSource = replaceSrcInElementSource({
+ element: video,
+ input,
+ srcAttribute,
+ srcSource: baseSrcSource,
+ });
+ const foregroundVideoSource = replaceSrcInElementSource({
+ element: video,
+ input,
+ srcAttribute,
+ srcSource: foregroundSrcSource,
+ });
+ const foregroundVideo = parseVideoElement(foregroundVideoSource);
+
+ srcAttribute.value = b.jsxExpressionContainer(
+ b.callExpression(b.identifier(staticFileLocalName), [
+ b.stringLiteral(baseSrc),
+ ]),
+ ) as unknown as JSXExpressionContainer;
+
+ const parent = videoPath.parentPath?.node;
+ if (!parent) {
+ throw new Error('Could not locate the video element parent');
+ }
+
+ let sourceEdit;
+ if (
+ (parent.type === 'JSXElement' || parent.type === 'JSXFragment') &&
+ parent.children.includes(video)
+ ) {
+ const index = parent.children.indexOf(video);
+ parent.children.splice(index + 1, 0, foregroundVideo);
+ sourceEdit = [
+ {
+ start: srcValueStart,
+ end: srcValueEnd,
+ replacement: baseSrcSource,
+ },
+ getAdjacentJsxInsertionSourceEdit({
+ input,
+ insertion: foregroundVideoSource,
+ position: 'after',
+ target: video,
+ }),
+ ];
+ } else {
+ const fragment: JSXFragment = {
+ type: 'JSXFragment',
+ openingFragment: {type: 'JSXOpeningFragment'},
+ closingFragment: {type: 'JSXClosingFragment'},
+ children: [video, foregroundVideo],
+ };
+ videoPath.replace(fragment);
+ if (!video.loc) {
+ throw new Error('Could not locate the video element in source');
+ }
+
+ const start = recastLocToOffset(input, video.loc.start);
+ const end = recastLocToOffset(input, video.loc.end);
+ const indent = getLineIndent({input, offset: start});
+ const unit = getIndentationUnit(input, prettierConfigOverride);
+ const eol = getEndOfLine(input);
+ sourceEdit = [
+ {
+ start,
+ end,
+ replacement: indentContinuationLines({
+ indent,
+ input,
+ printed: [
+ '<>',
+ indentInsertedJsx({indent: unit, insertion: baseVideoSource}),
+ indentInsertedJsx({
+ indent: unit,
+ insertion: foregroundVideoSource,
+ }),
+ '</>',
+ ].join(eol),
+ }),
+ },
+ ];
+ }
+
+ const output = applySourceEdits({
+ input,
+ edits: [
+ ...getInsertImportSourceEdits({
+ ast,
+ input,
+ prettierConfigOverride,
+ snapshots: importSnapshots,
+ }),
+ ...sourceEdit,
+ ],
+ });
+ const {nodePathRemappings} = getNodePathRemappings({
+ ast,
+ captured,
+ output,
+ });
+
+ return {
+ output,
+ logLine: video.openingElement.loc?.start.line ?? 1,
+ nodePathRemappings,
+ };
+};
diff --git a/packages/studio-server/src/codemods/insert-video-layers.ts b/packages/studio-server/src/codemods/insert-video-layers.ts
new file mode 100644
index 0000000000..2e51d6c795
--- /dev/null
+++ b/packages/studio-server/src/codemods/insert-video-layers.ts
@@ -0,0 +1 @@
+export {insertVideoLayers} from '@remotion/studio-codemods';
diff --git a/packages/studio-server/src/preview-server/api-routes.ts b/packages/studio-server/src/preview-server/api-routes.ts
index 5ecb29f436..6e7cdd35de 100644
--- a/packages/studio-server/src/preview-server/api-routes.ts
+++ b/packages/studio-server/src/preview-server/api-routes.ts
@@ -28,6 +28,7 @@ import {findInFileHandler} from './routes/find-in-file';
import {insertBasicCaptionsHandler} from './routes/insert-basic-captions';
import {insertElementHandler} from './routes/insert-element';
import {insertJsxElementHandler} from './routes/insert-jsx-element';
+import {insertVideoLayersHandler} from './routes/insert-video-layers';
import {
installRemotionSkillHandler,
removeRemotionSkillHandler,
@@ -130,6 +131,7 @@ export const allApiRoutes: {
'/api/split-jsx-sequence': splitJsxSequenceHandler,
'/api/split-video-from-audio': splitVideoFromAudioHandler,
'/api/insert-basic-captions': insertBasicCaptionsHandler,
+ '/api/insert-video-layers': insertVideoLayersHandler,
'/api/update-available': handleUpdate,
'/api/release-notes': getReleaseNotesHandler,
'/api/remotion-skills-info': remotionSkillsInfoHandler,
diff --git a/packages/studio-server/src/preview-server/routes/insert-video-layers.ts b/packages/studio-server/src/preview-server/routes/insert-video-layers.ts
new file mode 100644
index 0000000000..6078375661
--- /dev/null
+++ b/packages/studio-server/src/preview-server/routes/insert-video-layers.ts
@@ -0,0 +1,91 @@
+import {readFileSync} from 'node:fs';
+import {RenderInternals} from '@remotion/renderer';
+import type {
+ InsertVideoLayersRequest,
+ InsertVideoLayersResponse,
+} from '@remotion/studio-shared';
+import {insertVideoLayers} from '../../codemods/insert-video-layers';
+import {writeFileAndNotifyFileWatchers} from '../../file-watcher';
+import {resolveFileInsideProject} from '../../helpers/resolve-file-inside-project';
+import type {ApiHandler} from '../api-types';
+import {formatLogFileLocation} from '../format-log-file-location';
+import {broadcastSequenceNodePathMutation} from '../sequence-node-path-mutation';
+import {
+ printUndoHint,
+ pushToUndoStack,
+ suppressUndoStackInvalidation,
+} from '../undo-stack';
+import {
+ getCodemodTimingPrefix,
+ withSourceFileWriteQueue,
+} from './source-file-write-queue';
+
+export const insertVideoLayersHandler: ApiHandler<
+ InsertVideoLayersRequest,
+ InsertVideoLayersResponse
+> = ({
+ input: {fileName, nodePath, baseSrc, foregroundSrc},
+ remotionRoot,
+ logLevel,
+}) =>
+ withSourceFileWriteQueue<InsertVideoLayersResponse>(() => {
+ try {
+ const {absolutePath} = resolveFileInsideProject({
+ remotionRoot,
+ fileName,
+ action: 'modify',
+ });
+ const fileContents = readFileSync(absolutePath, 'utf-8');
+ const {output, logLine, nodePathRemappings} = insertVideoLayers({
+ input: fileContents,
+ nodePath,
+ baseSrc,
+ foregroundSrc,
+ });
+ const nodePathMutation = broadcastSequenceNodePathMutation(
+ [{absolutePath, remappings: nodePathRemappings}],
+ null,
+ );
+
+ pushToUndoStack({
+ filePath: absolutePath,
+ oldContents: fileContents,
+ newContents: null,
+ logLevel,
+ remotionRoot,
+ logLine,
+ description: {
+ undoMessage: '↩️ Insertion of separated video layers',
+ redoMessage: '↪️ Insertion of separated video layers',
+ },
+ entryType: 'insert-video-layers',
+ suppressHmrOnFileRestore: false,
+ nodePathRemappings,
+ });
+ suppressUndoStackInvalidation(absolutePath);
+ writeFileAndNotifyFileWatchers({
+ file: absolutePath,
+ content: output,
+ originatorClientId: undefined,
+ metadata: {skipSequencePropsUpdate: true},
+ });
+
+ const locationLabel = formatLogFileLocation({
+ remotionRoot,
+ absolutePath,
+ line: logLine,
+ });
+ RenderInternals.Log.info(
+ {indent: false, logLevel},
+ `${getCodemodTimingPrefix(logLevel)}${RenderInternals.chalk.blueBright(locationLabel)} Inserted separated video layers`,
+ );
+ printUndoHint(logLevel);
+ return Promise.resolve({success: true as const, nodePathMutation});
+ } catch (error) {
+ return Promise.resolve({
+ success: false,
+ reason: (error as Error).message,
+ stack: (error as Error).stack ?? '',
+ } as const);
+ }
+ });
diff --git a/packages/studio-server/src/preview-server/undo-stack.ts b/packages/studio-server/src/preview-server/undo-stack.ts
index dc73309521..b400df8d11 100644
--- a/packages/studio-server/src/preview-server/undo-stack.ts
+++ b/packages/studio-server/src/preview-server/undo-stack.ts
@@ -43,6 +43,7 @@ type UndoEntryType =
| 'split-jsx-sequence'
| 'split-video-from-audio'
| 'insert-basic-captions'
+ | 'insert-video-layers'
| 'insert-jsx-element'
| 'delete-composition'
| 'rename-composition'
diff --git a/packages/studio-server/src/test/insert-video-layers.test.ts b/packages/studio-server/src/test/insert-video-layers.test.ts
new file mode 100644
index 0000000000..60756cd9ed
--- /dev/null
+++ b/packages/studio-server/src/test/insert-video-layers.test.ts
@@ -0,0 +1,85 @@
+import {expect, test} from 'bun:test';
+import {mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs';
+import {tmpdir} from 'node:os';
+import path from 'node:path';
+import {
+ createFileWatcherRegistry,
+ setFileWatcherRegistry,
+} from '../file-watcher';
+import {setLiveEventsListener} from '../preview-server/live-events';
+import {insertVideoLayersHandler} from '../preview-server/routes/insert-video-layers';
+import {getUndoStack} from '../preview-server/undo-stack';
+import {lineContainingToNodePath} from './test-utils';
+
+test('video matting inserts base and foreground videos through the Studio route', async () => {
+ const remotionRoot = mkdtempSync(path.join(tmpdir(), 'remotion-matting-'));
+ const entryPoint = path.join(remotionRoot, 'Root.tsx');
+ const cleanupFileWatcher = setFileWatcherRegistry(
+ createFileWatcherRegistry(),
+ );
+ const cleanupLiveEvents = setLiveEventsListener({
+ sendEventToClient: () => undefined,
+ sendEventToClientId: () => true,
+ router: () => Promise.resolve(),
+ closeConnections: () => Promise.resolve(),
+ addNewClientListener: () => () => undefined,
+ });
+ const input = `import {Video as Clip} from '@remotion/media';
+
+const source = 'input.mp4';
+const untouched = {keep : 'spacing'};
+
+export const Comp = () => (
+ <Clip
+ src={source}
+ from={10}
+ style={{opacity: 0.5}}
+ />
+);
+`;
+
+ try {
+ (getUndoStack() as unknown as unknown[]).length = 0;
+ writeFileSync(entryPoint, input);
+ const result = await insertVideoLayersHandler({
+ input: {
+ fileName: entryPoint,
+ nodePath: lineContainingToNodePath(input, '<Clip'),
+ baseSrc: 'input-base.webm',
+ foregroundSrc: 'input-foreground.webm',
+ },
+ entryPoint,
+ remotionRoot,
+ request: {} as never,
+ response: {} as never,
+ logLevel: 'error',
+ methods: {
+ removeJob: () => undefined,
+ cancelJob: () => undefined,
+ addJob: () => undefined,
+ },
+ publicDir: remotionRoot,
+ binariesDirectory: null,
+ configFile: null,
+ getDefaultCodingAgent: () => null,
+ getDefaultEditor: () => null,
+ });
+
+ expect(result.success).toBe(true);
+ const written = readFileSync(entryPoint, 'utf-8');
+ const singleLine = written.replace(/\s+/g, ' ');
+ expect(written).toContain(`import {staticFile} from 'remotion';`);
+ expect(singleLine).toContain(
+ "<Clip src={staticFile('input-base.webm')} from={10} style={{opacity: 0.5}} />",
+ );
+ expect(singleLine).toContain(
+ "<Clip src={staticFile('input-foreground.webm')} from={10} style={{opacity: 0.5}} />",
+ );
+ expect(written).toContain("const untouched = {keep : 'spacing'};");
+ expect(getUndoStack()).toHaveLength(1);
+ } finally {
+ cleanupFileWatcher();
+ cleanupLiveEvents();
+ rmSync(remotionRoot, {recursive: true, force: true});
+ }
+});
diff --git a/packages/studio-shared/src/api-requests.ts b/packages/studio-shared/src/api-requests.ts
index 356362ec26..030b17447c 100644
--- a/packages/studio-shared/src/api-requests.ts
+++ b/packages/studio-shared/src/api-requests.ts
@@ -940,6 +940,17 @@ export type InsertBasicCaptionsResponse =
| {success: true; nodePathMutation: SequenceNodePathMutation}
| {success: false; reason: string; stack: string};
+export type InsertVideoLayersRequest = {
+ fileName: string;
+ nodePath: SequenceNodePath;
+ baseSrc: string;
+ foregroundSrc: string;
+};
+
+export type InsertVideoLayersResponse =
+ | {success: true; nodePathMutation: SequenceNodePathMutation}
+ | {success: false; reason: string; stack: string};
+
export type InsertableCompositionElement =
| {
type: 'solid';
@@ -1438,6 +1449,10 @@ export type ApiRoutes = {
InsertBasicCaptionsRequest,
InsertBasicCaptionsResponse
>;
+ '/api/insert-video-layers': ReqAndRes<
+ InsertVideoLayersRequest,
+ InsertVideoLayersResponse
+ >;
'/api/insert-jsx-element': ReqAndRes<
InsertJsxElementRequest,
InsertJsxElementResponse
diff --git a/packages/studio-shared/src/browser-studio-operations.ts b/packages/studio-shared/src/browser-studio-operations.ts
index 6ef1759824..6edf02f27d 100644
--- a/packages/studio-shared/src/browser-studio-operations.ts
+++ b/packages/studio-shared/src/browser-studio-operations.ts
@@ -62,6 +62,8 @@ import type {
SplitVideoFromAudioResponse,
InsertBasicCaptionsRequest,
InsertBasicCaptionsResponse,
+ InsertVideoLayersRequest,
+ InsertVideoLayersResponse,
SubscribeToDefaultPropsRequest,
SubscribeToDefaultPropsResponse,
SubscribeToSequencePropsRequest,
@@ -227,6 +229,9 @@ export type BrowserStudioOperations = {
insertBasicCaptions: (
request: InsertBasicCaptionsRequest,
) => Promise<InsertBasicCaptionsResponse>;
+ insertVideoLayers: (
+ request: InsertVideoLayersRequest,
+ ) => Promise<InsertVideoLayersResponse>;
subscribeToDefaultProps: (
request: SubscribeToDefaultPropsRequest,
) => Promise<SubscribeToDefaultPropsResponse>;
diff --git a/packages/studio-shared/src/index.ts b/packages/studio-shared/src/index.ts
index 2fbdbd0659..247644e736 100644
--- a/packages/studio-shared/src/index.ts
+++ b/packages/studio-shared/src/index.ts
@@ -75,6 +75,8 @@ export {
InsertElementFileConflict,
InsertBasicCaptionsRequest,
InsertBasicCaptionsResponse,
+ InsertVideoLayersRequest,
+ InsertVideoLayersResponse,
InsertElementRequest,
InsertElementResponse,
InsertJsxElementRequest,
diff --git a/packages/studio/src/components/CurrentAsset.tsx b/packages/studio/src/components/CurrentAsset.tsx
index 091d2c9fa5..a34c721c73 100644
--- a/packages/studio/src/components/CurrentAsset.tsx
+++ b/packages/studio/src/components/CurrentAsset.tsx
@@ -239,6 +239,7 @@ export const AssetInfo: React.FC<{
type: 'video-matting',
src,
displayName: fileName,
+ target: null,
});
}, [fileName, fileType, mutationsDisabled, setSelectedModal, src]);
const canRename =
diff --git a/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsx b/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsx
index 1bd2892414..689dde809b 100644
--- a/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsx
+++ b/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsx
@@ -6,6 +6,7 @@ import {isStudioInteractivityEnabled} from '../../helpers/interactivity-enabled'
import {AudioIcon} from '../../icons/audio';
import {DuplicateIcon} from '../../icons/duplicate';
import {ScissorsIcon} from '../../icons/scissors';
+import {SeparationIcon} from '../../icons/separation';
import {SnowflakeIcon} from '../../icons/snowflake';
import {TranscriptionIcon} from '../../icons/transcription';
import {TrashIcon} from '../../icons/trash';
@@ -167,6 +168,11 @@ const SequenceSourceQuickActions: React.FC<{
: selection.nodePathInfo.numberOfSequencesWithThisNodePath > 1
? 'Programmatically duplicated media cannot be transcribed from source'
: undefined;
+ const videoMattingDisabledReason = sourceActionsDisabled
+ ? 'Studio is read-only'
+ : selection.nodePathInfo.numberOfSequencesWithThisNodePath > 1
+ ? 'Programmatically duplicated videos cannot be separated from source'
+ : undefined;
const onGenerateCaptions = useCallback(() => {
if (transcriptionDisabledReason !== undefined || mediaSequence === null) {
return;
@@ -196,6 +202,33 @@ const SequenceSourceQuickActions: React.FC<{
mediaSequence,
transcriptionDisabledReason,
]);
+ const onSeparateForeground = useCallback(() => {
+ if (
+ videoMattingDisabledReason !== undefined ||
+ track.sequence.type !== 'video'
+ ) {
+ return;
+ }
+
+ const nodePath = selection.nodePathInfo.sequenceSubscriptionKey;
+ setSelectedModal({
+ type: 'video-matting',
+ src: track.sequence.src,
+ displayName: getMediaFileName(
+ track.sequence.src,
+ track.sequence.displayName,
+ ),
+ target: {
+ fileName: nodePath.absolutePath,
+ nodePath,
+ },
+ });
+ }, [
+ selection.nodePathInfo,
+ setSelectedModal,
+ track.sequence,
+ videoMattingDisabledReason,
+ ]);
const onDuplicate = useCallback(() => {
if (sourceActionsDisabled) {
return;
@@ -252,6 +285,18 @@ const SequenceSourceQuickActions: React.FC<{
{freezeFrameMenuItem.label}
</InspectorQuickAction>
) : null}
+ {track.sequence.type === 'video' ? (
+ <InspectorQuickAction
+ disabled={videoMattingDisabledReason !== undefined}
+ onClick={onSeparateForeground}
+ title={videoMattingDisabledReason}
+ renderIcon={(color) => (
+ <SeparationIcon style={actionIconStyle} color={color} />
+ )}
+ >
+ Separate foreground
+ </InspectorQuickAction>
+ ) : null}
{track.sequence.type === 'video' ? (
<InspectorQuickAction
disabled={splitVideoFromAudioDisabledReason !== undefined}
diff --git a/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx
index 8063683a57..220f52b0df 100644
--- a/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx
+++ b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx
@@ -7,6 +7,8 @@ import {
} from '@remotion/video-matting';
import {useCallback, useContext, useEffect} from 'react';
import {writeStaticFile} from '../../api/write-static-file';
+import {getBrowserStudioOperations} from '../../helpers/browser-studio-operations';
+import {callApi} from '../call-api';
import {RenderQueueContext} from './context';
import {loadModelForJob} from './load-model-for-job';
import type {VideoMattingJob} from './video-matting-job-types';
@@ -93,6 +95,27 @@ export const VideoMattingQueueProcessor: React.FC = () => {
writeStaticFile({contents, filePath: job.foregroundOutName}),
),
]);
+
+ if (job.target !== null) {
+ updateVideoMattingJobProgress(job.id, {
+ detail: null,
+ message: 'Adding separated video layers...',
+ value: 0.97,
+ });
+ const request = {
+ fileName: job.target.fileName,
+ nodePath: job.target.nodePath.nodePath,
+ baseSrc: job.baseOutName,
+ foregroundSrc: job.foregroundOutName,
+ };
+ const browserStudioOperations = getBrowserStudioOperations();
+ const response = browserStudioOperations
+ ? await browserStudioOperations.insertVideoLayers(request)
+ : await callApi('/api/insert-video-layers', request);
+ if (!response.success) {
+ throw new Error(response.reason);
+ }
+ }
} catch (error) {
processingError =
error instanceof Error ? error : new Error(String(error));
diff --git a/packages/studio/src/components/RenderQueue/video-matting-job-types.ts b/packages/studio/src/components/RenderQueue/video-matting-job-types.ts
index 7ecd97d7c4..864adf14f6 100644
--- a/packages/studio/src/components/RenderQueue/video-matting-job-types.ts
+++ b/packages/studio/src/components/RenderQueue/video-matting-job-types.ts
@@ -3,6 +3,7 @@ import type {
VideoMattingBitrate,
VideoMattingModel,
} from '@remotion/video-matting';
+import type {SequencePropsSubscriptionKey} from 'remotion';
export type VideoMattingJobProgress = {
detail: string | null;
@@ -24,6 +25,10 @@ export type AddVideoMattingJobParams = {
model: VideoMattingModel;
audio: VideoLayerAudio;
videoBitrate: VideoMattingBitrate;
+ target: {
+ fileName: string;
+ nodePath: SequencePropsSubscriptionKey;
+ } | null;
};
export type VideoMattingJob = AddVideoMattingJobParams & {
diff --git a/packages/studio/src/components/VideoMatting/VideoMattingModal.tsx b/packages/studio/src/components/VideoMatting/VideoMattingModal.tsx
index 9006e0ccf2..a0e6dd67dc 100644
--- a/packages/studio/src/components/VideoMatting/VideoMattingModal.tsx
+++ b/packages/studio/src/components/VideoMatting/VideoMattingModal.tsx
@@ -108,6 +108,7 @@ const makeOptions = <Value extends string>({
export const VideoMattingModal: React.FC<VideoMattingModalState> = ({
displayName,
src,
+ target,
}) => {
const [tab, setTab] = useState<Tab>('separate');
const isModelCached = useCallback(
@@ -244,6 +245,7 @@ export const VideoMattingModal: React.FC<VideoMattingModalState> = ({
model,
audio,
videoBitrate,
+ target,
});
setSidebarCollapsedState({left: null, right: 'expanded'});
persistSelectedOptionsSidebarPanel('renders');
@@ -260,13 +262,16 @@ export const VideoMattingModal: React.FC<VideoMattingModalState> = ({
setSelectedModal,
setSidebarCollapsedState,
src,
+ target,
videoBitrate,
]);
+ const title =
+ target === null ? `Track matting ${displayName}` : 'Separate foreground';
return (
- <DismissableModal ariaLabel={`Track matting ${displayName}`}>
+ <DismissableModal ariaLabel={title}>
<div style={modalStyle}>
- <ModalHeader title={`Track matting ${displayName}`} />
+ <ModalHeader title={title} />
<div style={container}>
<div style={flexer} />
<Button
diff --git a/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx b/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx
index 43bb871e7d..32303e5b22 100644
--- a/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx
+++ b/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx
@@ -12,9 +12,13 @@ export const VideoMattingModalWithOptionalPackage: React.FC<{
readonly state: VideoMattingModalState;
}> = ({state}) => (
<OptionalPackageModal
- ariaLabel="Install video matting package"
+ ariaLabel={
+ state.target === null
+ ? 'Install video matting package'
+ : 'Install video matting package to separate foreground'
+ }
packageName={VIDEO_MATTING_PACKAGE}
- title="Track matting"
+ title={state.target === null ? 'Track matting' : 'Separate foreground'}
>
<Suspense fallback={null}>
<LazyVideoMattingModal {...state} />
diff --git a/packages/studio/src/components/WebMcp.tsx b/packages/studio/src/components/WebMcp.tsx
index 7b1da56d9e..01351f7b4a 100644
--- a/packages/studio/src/components/WebMcp.tsx
+++ b/packages/studio/src/components/WebMcp.tsx
@@ -778,6 +778,7 @@ export const WebMcp: FC = () => {
foregroundOutName: foregroundOutputPath,
model,
src,
+ target: null,
videoBitrate,
});
return {
diff --git a/packages/studio/src/state/modals.ts b/packages/studio/src/state/modals.ts
index 842b9915c6..a644e34f69 100644
--- a/packages/studio/src/state/modals.ts
+++ b/packages/studio/src/state/modals.ts
@@ -170,6 +170,10 @@ export type VideoMattingModalState = {
type: 'video-matting';
src: string;
displayName: string;
+ target: {
+ fileName: string;
+ nodePath: SequencePropsSubscriptionKey;
+ } | null;
};
export type CanvasCaptureImport = {
diff --git a/packages/studio/src/test/make-browser-studio-operations.ts b/packages/studio/src/test/make-browser-studio-operations.ts
index 16af2c2bd1..ff9b1a6992 100644
--- a/packages/studio/src/test/make-browser-studio-operations.ts
+++ b/packages/studio/src/test/make-browser-studio-operations.ts
@@ -54,6 +54,7 @@ export const makeBrowserStudioOperations = (
splitJsxSequence: () => unusedOperation('splitJsxSequence'),
splitVideoFromAudio: () => unusedOperation('splitVideoFromAudio'),
insertBasicCaptions: () => unusedOperation('insertBasicCaptions'),
+ insertVideoLayers: () => unusedOperation('insertVideoLayers'),
subscribeToDefaultProps: () => unusedOperation('subscribeToDefaultProps'),
subscribeToEvent: () => unusedOperation('subscribeToEvent'),
subscribeToSequenceProps: () => unusedOperation('subscribeToSequenceProps'),
diff --git a/packages/studio/src/test/render-queue-scheduler.test.tsx b/packages/studio/src/test/render-queue-scheduler.test.tsx
index 9fe2c713f5..c8a575d643 100644
--- a/packages/studio/src/test/render-queue-scheduler.test.tsx
+++ b/packages/studio/src/test/render-queue-scheduler.test.tsx
@@ -66,6 +66,7 @@ const videoMattingParams: AddVideoMattingJobParams = {
foregroundOutName: 'video-foreground.webm',
model: 'ben2-base',
src: '/video.mp4',
+ target: null,
videoBitrate: 'very-high',
};
diff --git a/packages/studio/src/test/video-matting-queue-processor.test.tsx b/packages/studio/src/test/video-matting-queue-processor.test.tsx
index aa1cf3c2d2..8f35effcf2 100644
--- a/packages/studio/src/test/video-matting-queue-processor.test.tsx
+++ b/packages/studio/src/test/video-matting-queue-processor.test.tsx
@@ -57,7 +57,7 @@ afterEach(() => {
processJob = null;
});
-test('loads the model, separates the layers and writes both outputs', async () => {
+test('writes both layers and inserts them into the selected video source', async () => {
const originalBrowserStudio = Object.getOwnPropertyDescriptor(
window,
'remotion_browserStudio',
@@ -65,6 +65,15 @@ test('loads the model, separates the layers and writes both outputs', async () =
Object.defineProperty(window, 'remotion_browserStudio', {
configurable: true,
value: makeBrowserStudioOperations({
+ insertVideoLayers: ({baseSrc, foregroundSrc, fileName, nodePath}) => {
+ calls.push(
+ `insert:${fileName}:${nodePath.join('.')}:${baseSrc}:${foregroundSrc}`,
+ );
+ return Promise.resolve({
+ success: true,
+ nodePathMutation: {} as never,
+ });
+ },
writeStaticFile: ({filePath}) => {
calls.push(`write:${filePath}`);
return Promise.resolve();
@@ -113,6 +122,16 @@ test('loads the model, separates the layers and writes both outputs', async () =
foregroundOutName: 'input-foreground.webm',
model: 'modnet',
audio: 'base',
+ target: {
+ fileName: '/project/Composition.tsx',
+ nodePath: {
+ absolutePath: '/project/Composition.tsx',
+ effectKeys: [],
+ nodePath: ['Comp', 0],
+ sequenceKeys: [],
+ videoConfigValues: null,
+ },
+ },
videoBitrate: 'very-high',
}),
);
@@ -124,6 +143,7 @@ test('loads the model, separates the layers and writes both outputs', async () =
'separate',
'write:input-base.webm',
'write:input-foreground.webm',
+ 'insert:/project/Composition.tsx:Comp.0:input-base.webm:input-foreground.webm',
'dispose-model',
]);
expect(progress).toContainEqual({
diff --git a/packages/studio/src/test/whisper-webgpu-capability.test.tsx b/packages/studio/src/test/whisper-webgpu-capability.test.tsx
index a5fd478a6b..2422967d67 100644
--- a/packages/studio/src/test/whisper-webgpu-capability.test.tsx
+++ b/packages/studio/src/test/whisper-webgpu-capability.test.tsx
@@ -70,6 +70,7 @@ test('uses the same install confirmation for video matting', () => {
type: 'video-matting',
src: '/video.webm',
displayName: 'video.webm',
+ target: null,
}}
/>
</ModalsProvider>,