Skip to content

🎥 Make videos programmatically with React

Replace WebMCP currentComposition with currentContent

JonnyBurger1eb6f8e27578
diff --git a/packages/docs/docs/ai/webmcp.mdx b/packages/docs/docs/ai/webmcp.mdx
index 7696c65b06..948fc1ba44 100644
--- a/packages/docs/docs/ai/webmcp.mdx
+++ b/packages/docs/docs/ai/webmcp.mdx
@@ -43,6 +43,24 @@ The shape of inputs and outputs is not currently stable.
Tools that modify the timeline, playback or guides require a composition to be open.
+### `currentContent`<AvailableFrom v="4.0.523" />
+
+Tool responses identify the content open in the canvas using `currentContent`, replacing `currentComposition`. It is `null` when no content is open. Browser-rendered outputs include metadata without the blob itself.
+
+```ts title="CurrentContent"
+type CurrentContent =
+ | {type: 'composition'; compositionId: string}
+ | {type: 'asset'; asset: string}
+ | {type: 'output'; path: string}
+ | {
+ type: 'output-blob';
+ displayName: string;
+ width: number;
+ height: number;
+ sizeInBytes: number;
+ };
+```
+
### `restart_studio`<AvailableFrom v="4.0.521" />
Restarts the Studio server using [`restartStudio()`](/docs/studio/restart-studio). Requires a writable Studio with a running server. No composition needs to be open. The browser temporarily disconnects and reconnects when Studio is ready.
@@ -139,7 +157,7 @@ Opens a registered composition by name.
```ts title="Output"
{
- currentComposition: string;
+ currentContent: {type: 'composition'; compositionId: string};
}
```
@@ -153,7 +171,7 @@ Returns the timeline sequences mounted for the current composition, including th
```ts title="Output"
{
- currentComposition: string | null;
+ currentContent: CurrentContent | null;
sequences: Array<{
sequenceId: string;
name: string | null;
@@ -181,7 +199,7 @@ Selects and reveals a sequence using a `sequenceId` returned by `get_sequences`.
```ts title="Output"
{
- currentComposition: string;
+ currentContent: {type: 'composition'; compositionId: string};
selectedSequence: {
sequenceId: string;
name: string | null;
@@ -227,7 +245,7 @@ Returns the rendered composition HTML at the current frame without the surroundi
```ts title="Output"
{
- currentComposition: string | null;
+ currentContent: CurrentContent | null;
currentFrame: number | null;
html: string | null;
htmlLength: number | null;
@@ -245,7 +263,7 @@ Returns the measurable selectable component outlines active at the current frame
```ts title="Output"
{
- currentComposition: string | null;
+ currentContent: CurrentContent | null;
currentFrame: number | null;
outlines: Array<{
sequenceId: string;
@@ -283,7 +301,7 @@ Returns the current playhead, playback, audio, looping and timeline zoom state.
```ts title="Output"
{
- currentComposition: string | null;
+ currentContent: CurrentContent | null;
currentFrame: number | null;
playing: boolean | null;
muted: boolean | null;
@@ -305,7 +323,7 @@ Returns source-code context when exactly one supported timeline item is selected
```ts title="Output"
{
currentFrame: number;
- currentComposition: string | null;
+ currentContent: CurrentContent | null;
currentSelection: string | null;
selectionType:
| 'guide'
@@ -342,7 +360,7 @@ Returns guides for the current composition. Vertical positions are x-coordinates
```ts title="Output"
{
- currentComposition: string | null;
+ currentContent: CurrentContent | null;
guidesVisible: boolean;
guides: Array<{
id: string;
@@ -365,7 +383,7 @@ Shows or hides every guide in the current composition.
```ts title="Output"
{
- currentComposition: string;
+ currentContent: {type: 'composition'; compositionId: string};
guidesVisible: boolean;
}
```
@@ -383,7 +401,7 @@ Adds and shows a guide in the current composition. Positions use composition pix
```ts title="Output"
{
- currentComposition: string;
+ currentContent: {type: 'composition'; compositionId: string};
guide: {
id: string;
orientation: 'horizontal' | 'vertical';
@@ -405,7 +423,7 @@ Removes a guide using an ID returned by `get_guides` or `add_guide`.
```ts title="Output"
{
- currentComposition: string;
+ currentContent: {type: 'composition'; compositionId: string};
guideId: string;
removed: true;
}
@@ -421,7 +439,7 @@ Starts playback from the current frame. Still compositions cannot be played.
```ts title="Output"
{
- currentComposition: string;
+ currentContent: {type: 'composition'; compositionId: string};
playing: true;
}
```
@@ -436,7 +454,7 @@ Pauses playback at the current frame.
```ts title="Output"
{
- currentComposition: string;
+ currentContent: {type: 'composition'; compositionId: string};
playing: false;
}
```
@@ -451,7 +469,7 @@ Mutes audio playback.
```ts title="Output"
{
- currentComposition: string;
+ currentContent: {type: 'composition'; compositionId: string};
muted: true;
}
```
@@ -466,7 +484,7 @@ Unmutes audio playback.
```ts title="Output"
{
- currentComposition: string;
+ currentContent: {type: 'composition'; compositionId: string};
muted: false;
}
```
@@ -485,7 +503,7 @@ Sets the timeline zoom using a normalized value between `0` (fully zoomed out) a
```ts title="Output"
{
- currentComposition: string;
+ currentContent: {type: 'composition'; compositionId: string};
timelineZoom: number;
}
```
@@ -513,7 +531,7 @@ Sets the playback multiplier. Negative values play backwards.
```ts title="Output"
{
- currentComposition: string;
+ currentContent: {type: 'composition'; compositionId: string};
playbackRate: number;
}
```
@@ -532,7 +550,7 @@ Seeks to a zero-based frame. Values beyond the duration are clamped to the final
```ts title="Output"
{
- currentComposition: string;
+ currentContent: {type: 'composition'; compositionId: string};
currentFrame: number;
}
```
diff --git a/packages/example/e2e/studio.test.mts b/packages/example/e2e/studio.test.mts
index 1719cd7cbb..7f309f8494 100644
--- a/packages/example/e2e/studio.test.mts
+++ b/packages/example/e2e/studio.test.mts
@@ -3035,7 +3035,10 @@ export const SequenceShiftRepro = () => {
expect(webMcpSelection).toEqual({
currentFrame: 3,
currentSelection: contextForAgents,
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
selectionType: 'sequence',
selectedSequence: expect.objectContaining({
sequenceId: expect.any(String),
@@ -3078,7 +3081,10 @@ export const SequenceShiftRepro = () => {
}
).sequences;
expect(webMcpSequences).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
sequences: expect.arrayContaining([
expect.objectContaining({
sequenceId: selectedSequence.sequenceId,
@@ -3117,7 +3123,10 @@ export const SequenceShiftRepro = () => {
{sequenceId: sequenceToSelect?.sequenceId ?? ''},
);
expect(webMcpSelectSequenceResult).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
selectedSequence: expect.objectContaining({
sequenceId: sequenceToSelect?.sequenceId,
selectable: true,
@@ -3199,7 +3208,10 @@ export const SequenceShiftRepro = () => {
}
).html;
expect(webMcpCanvasHtml).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
currentFrame: 3,
html: expect.any(String),
htmlLength: canvasHtml.length,
@@ -3250,7 +3262,10 @@ export const SequenceShiftRepro = () => {
}
).outlines.find((outline) => outline.name === '0% gridline');
expect(webMcpOutlines).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
currentFrame: 3,
outlines: expect.any(Array),
});
@@ -3343,7 +3358,10 @@ export const SequenceShiftRepro = () => {
return tool.execute({compositionName: 'package-absolute-fill'});
});
expect(webMcpSelectCompositionResult).toEqual({
- currentComposition: 'package-absolute-fill',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'package-absolute-fill',
+ },
});
await expect
.poll(() => new URL(page.url()).pathname)
@@ -3411,7 +3429,10 @@ export const SequenceShiftRepro = () => {
return tool.execute({});
});
expect(webMcpGuides).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
guidesVisible: true,
guides: [
{
@@ -3450,7 +3471,10 @@ export const SequenceShiftRepro = () => {
return tool.execute({visible: false});
});
expect(webMcpHideGuidesResult).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
guidesVisible: false,
});
await expect(page.locator('.__remotion_editor_guide')).toHaveCount(0);
@@ -3482,7 +3506,10 @@ export const SequenceShiftRepro = () => {
return tool.execute({visible: true});
});
expect(webMcpShowGuidesResult).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
guidesVisible: true,
});
await expect(page.locator('.__remotion_editor_guide')).toHaveCount(2);
@@ -3514,7 +3541,10 @@ export const SequenceShiftRepro = () => {
return tool.execute({orientation: 'vertical', position: 640});
});
expect(webMcpAddGuideResult).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
guide: {
id: expect.any(String),
orientation: 'vertical',
@@ -3563,7 +3593,10 @@ export const SequenceShiftRepro = () => {
{guideId: addedGuideId},
);
expect(webMcpRemoveGuideResult).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
guideId: addedGuideId,
removed: true,
});
@@ -3602,7 +3635,10 @@ export const SequenceShiftRepro = () => {
return tool.execute({playbackRate: 1.5});
});
expect(webMcpPlaybackRateResult).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
playbackRate: 1.5,
});
await expect(
@@ -3632,7 +3668,10 @@ export const SequenceShiftRepro = () => {
return tool.execute({zoom: 0.5});
});
expect(webMcpTimelineZoomResult).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
timelineZoom: expect.any(Number),
});
const normalizedTimelineZoom = (
@@ -3663,7 +3702,10 @@ export const SequenceShiftRepro = () => {
return tool.execute({zoom: 1});
});
expect(webMcpMaximumTimelineZoomResult).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
timelineZoom: 1,
});
await expect(
@@ -3694,7 +3736,10 @@ export const SequenceShiftRepro = () => {
return tool.execute({});
});
expect(webMcpMuteResult).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
muted: true,
});
await expect(
@@ -3721,7 +3766,10 @@ export const SequenceShiftRepro = () => {
return tool.execute({});
});
expect(webMcpUnmuteResult).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
muted: false,
});
await expect(
@@ -3748,7 +3796,10 @@ export const SequenceShiftRepro = () => {
return tool.execute({});
});
expect(webMcpPlayResult).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
playing: true,
});
await expect(
@@ -3775,7 +3826,10 @@ export const SequenceShiftRepro = () => {
return tool.execute({});
});
expect(webMcpPauseResult).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
playing: false,
});
await expect(
@@ -3803,7 +3857,10 @@ export const SequenceShiftRepro = () => {
});
expect(webMcpSeekResult).toEqual({
currentFrame: 179,
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
});
await expect(
page.getByRole('button', {name: '179', exact: true}),
@@ -3825,7 +3882,10 @@ export const SequenceShiftRepro = () => {
return tool.execute();
});
expect(webMcpPlaybackState).toEqual({
- currentComposition: 'AnimatedBarChart',
+ currentContent: {
+ type: 'composition',
+ compositionId: 'AnimatedBarChart',
+ },
currentFrame: 179,
playing: false,
muted: false,
@@ -3964,7 +4024,7 @@ export const SequenceShiftRepro = () => {
}),
)
.toEqual({
- currentComposition: null,
+ currentContent: {type: 'asset', asset: 'test.gif'},
currentFrame: null,
playing: null,
muted: null,
@@ -3989,7 +4049,7 @@ export const SequenceShiftRepro = () => {
}),
)
.toEqual({
- currentComposition: null,
+ currentContent: {type: 'asset', asset: 'test.gif'},
currentFrame: null,
html: null,
htmlLength: null,
@@ -4011,10 +4071,37 @@ export const SequenceShiftRepro = () => {
}),
)
.toEqual({
- currentComposition: null,
+ currentContent: {type: 'asset', asset: 'test.gif'},
currentFrame: null,
outlines: [],
});
+ for (const preview of [
+ {
+ route: '/assets/test.gif',
+ content: {type: 'asset', asset: 'test.gif'},
+ },
+ {
+ route: '/outputs/public%2Ftest.gif',
+ content: {type: 'output', path: 'public/test.gif'},
+ },
+ ]) {
+ await page.goto(`${STUDIO_URL}${preview.route}`);
+ await expect
+ .poll(() =>
+ page.evaluate(async () => {
+ const tools = (
+ window as typeof window & {
+ readonly __remotion_webmcp_tools: Map<
+ string,
+ {readonly execute: () => Promise<unknown>}
+ >;
+ }
+ ).__remotion_webmcp_tools;
+ return tools.get('get_selection')?.execute() ?? null;
+ }),
+ )
+ .toMatchObject({currentContent: preview.content});
+ }
} finally {
fs.writeFileSync(configFile, configBeforeTest);
}
diff --git a/packages/studio/src/components/WebMcp.tsx b/packages/studio/src/components/WebMcp.tsx
index 3b9198989d..7c065b6c20 100644
--- a/packages/studio/src/components/WebMcp.tsx
+++ b/packages/studio/src/components/WebMcp.tsx
@@ -242,6 +242,21 @@ export const WebMcp: FC = () => {
) ?? null
);
}, [canvasContent, compositions]);
+ const currentContent = useMemo(() => {
+ if (canvasContent?.type === 'output-blob') {
+ return {
+ type: canvasContent.type,
+ displayName: canvasContent.displayName,
+ width: canvasContent.width,
+ height: canvasContent.height,
+ sizeInBytes: canvasContent.sizeInBytes,
+ };
+ }
+
+ return canvasContent;
+ }, [canvasContent]);
+ const currentContentRef = useRef(currentContent);
+ currentContentRef.current = currentContent;
const currentComposition = currentCompositionDefinition?.id ?? null;
const currentCompositionRef = useRef(currentComposition);
currentCompositionRef.current = currentComposition;
@@ -424,7 +439,12 @@ export const WebMcp: FC = () => {
}
selectComposition(composition, true);
- return Promise.resolve({currentComposition: composition.id});
+ return Promise.resolve({
+ currentContent: {
+ type: 'composition',
+ compositionId: composition.id,
+ },
+ });
},
},
{signal: controller.signal},
@@ -444,7 +464,7 @@ export const WebMcp: FC = () => {
execute: () => {
const compositionId = currentCompositionRef.current;
return Promise.resolve({
- currentComposition: compositionId,
+ currentContent: currentContentRef.current,
sequences:
compositionId === null
? []
@@ -505,7 +525,7 @@ export const WebMcp: FC = () => {
selectItems([selection], {reveal: true});
return Promise.resolve({
- currentComposition: compositionId,
+ currentContent: currentContentRef.current,
selectedSequence: serializeSequence(timelineTrack),
});
},
@@ -571,7 +591,7 @@ export const WebMcp: FC = () => {
const compositionId = currentCompositionRef.current;
if (compositionId === null) {
return Promise.resolve({
- currentComposition: null,
+ currentContent: currentContentRef.current,
currentFrame: null,
html: null,
htmlLength: null,
@@ -581,7 +601,7 @@ export const WebMcp: FC = () => {
const outerHtml = Internals.portalNode().outerHTML;
return Promise.resolve({
- currentComposition: compositionId,
+ currentContent: currentContentRef.current,
currentFrame: getCurrentFrame(),
html: outerHtml.slice(0, MAX_CANVAS_HTML_LENGTH),
htmlLength: outerHtml.length,
@@ -607,7 +627,7 @@ export const WebMcp: FC = () => {
const composition = currentCompositionDefinitionRef.current;
if (composition === null) {
return {
- currentComposition: null,
+ currentContent: currentContentRef.current,
currentFrame: null,
outlines: [],
};
@@ -719,7 +739,7 @@ export const WebMcp: FC = () => {
);
return {
- currentComposition: composition.id,
+ currentContent: currentContentRef.current,
currentFrame,
outlines: outlines.filter(
(outline): outline is NonNullable<typeof outline> =>
@@ -746,7 +766,7 @@ export const WebMcp: FC = () => {
const compositionId = currentCompositionRef.current;
if (compositionId === null) {
return Promise.resolve({
- currentComposition: null,
+ currentContent: currentContentRef.current,
currentFrame: null,
playing: null,
muted: null,
@@ -771,7 +791,7 @@ export const WebMcp: FC = () => {
});
return Promise.resolve({
- currentComposition: compositionId,
+ currentContent: currentContentRef.current,
currentFrame: getCurrentFrame(),
playing: isPlaying(),
muted: playerMutedRef.current,
@@ -792,7 +812,7 @@ export const WebMcp: FC = () => {
name: 'get_selection',
title: 'Get Studio selection',
description:
- 'Read the current frame, composition, and source-code context for the item currently selected in Remotion Studio. The selection matches "Copy context for agents".',
+ 'Read the current frame, canvas content (composition, asset, or output), and source-code context for the item currently selected in Remotion Studio. The selection matches "Copy context for agents".',
inputSchema: {
type: 'object',
properties: {},
@@ -803,7 +823,7 @@ export const WebMcp: FC = () => {
Promise.resolve({
currentFrame: getCurrentFrame(),
currentSelection: currentSelectionRef.current,
- currentComposition: currentCompositionRef.current,
+ currentContent: currentContentRef.current,
selectionType:
selectedItemsRef.current.length === 1
? selectedItemsRef.current[0].type
@@ -830,7 +850,7 @@ export const WebMcp: FC = () => {
const guidesAreVisible = editorShowGuidesRef.current;
return Promise.resolve({
- currentComposition: compositionId,
+ currentContent: currentContentRef.current,
guidesVisible: guidesAreVisible,
guides:
compositionId === null
@@ -879,7 +899,7 @@ export const WebMcp: FC = () => {
editorShowGuidesRef.current = visible;
setEditorShowGuides(() => visible);
return Promise.resolve({
- currentComposition: compositionId,
+ currentContent: currentContentRef.current,
guidesVisible: visible,
});
},
@@ -941,7 +961,7 @@ export const WebMcp: FC = () => {
setEditorShowGuides(() => true);
return Promise.resolve({
- currentComposition: compositionId,
+ currentContent: currentContentRef.current,
guide: {
id: guide.id,
orientation: guide.orientation,
@@ -1009,7 +1029,7 @@ export const WebMcp: FC = () => {
}
return Promise.resolve({
- currentComposition: compositionId,
+ currentContent: currentContentRef.current,
guideId,
removed: true,
});
@@ -1047,7 +1067,7 @@ export const WebMcp: FC = () => {
play();
return Promise.resolve({
- currentComposition: compositionId,
+ currentContent: currentContentRef.current,
playing: true,
});
},
@@ -1078,7 +1098,7 @@ export const WebMcp: FC = () => {
pause();
return Promise.resolve({
- currentComposition: compositionId,
+ currentContent: currentContentRef.current,
playing: false,
});
},
@@ -1106,7 +1126,7 @@ export const WebMcp: FC = () => {
setPlayerMuted(true);
persistMuteOption(true);
return Promise.resolve({
- currentComposition: compositionId,
+ currentContent: currentContentRef.current,
muted: true,
});
},
@@ -1134,7 +1154,7 @@ export const WebMcp: FC = () => {
setPlayerMuted(false);
persistMuteOption(false);
return Promise.resolve({
- currentComposition: compositionId,
+ currentContent: currentContentRef.current,
muted: false,
});
},
@@ -1204,7 +1224,7 @@ export const WebMcp: FC = () => {
});
return Promise.resolve({
- currentComposition: compositionId,
+ currentContent: currentContentRef.current,
timelineZoom: timelineZoomToNormalized({
zoom: timelineZoom,
minZoom,
@@ -1252,7 +1272,7 @@ export const WebMcp: FC = () => {
setPlaybackRate(() => playbackRate);
persistPlaybackRate(playbackRate);
return Promise.resolve({
- currentComposition: compositionId,
+ currentContent: currentContentRef.current,
playbackRate,
});
},
@@ -1295,7 +1315,7 @@ export const WebMcp: FC = () => {
seek(currentFrame);
return Promise.resolve({
currentFrame,
- currentComposition: compositionId,
+ currentContent: currentContentRef.current,
});
},
},