Skip to content

🎥 Make videos programmatically with React

Add more Studio WebMCP tools

JonnyBurger346e2e15a8dd
diff --git a/packages/canvas/src/outlines.ts b/packages/canvas/src/outlines.ts
index e6ee8f7898..915d375b44 100644
--- a/packages/canvas/src/outlines.ts
+++ b/packages/canvas/src/outlines.ts
@@ -398,7 +398,7 @@ export const cropCanvasOutlinePoints = (
};
export const measureCanvasOutlines = <Target extends CanvasOutlineTarget>(
- container: SVGSVGElement,
+ container: Element,
targets: readonly Target[],
): CanvasOutline[] => {
resetBoxQuadsCache();
diff --git a/packages/docs/docs/ai/webmcp.mdx b/packages/docs/docs/ai/webmcp.mdx
index 99e06708c4..4160eeca76 100644
--- a/packages/docs/docs/ai/webmcp.mdx
+++ b/packages/docs/docs/ai/webmcp.mdx
@@ -27,9 +27,15 @@ No Remotion configuration is required. Studio registers its tools when the brows
| -------------------- | ---------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- |
| `get_compositions` | None | Returns mounted compositions as a nested tree of folders and compositions. |
| `select_composition` | `compositionName`: string | Opens a registered composition in Studio. |
+| `get_sequences` | None | Returns mounted timeline sequences with IDs, hierarchy, timing, type and source context. |
+| `select_sequence` | `sequenceId`: string | Selects and reveals a sequence returned by `get_sequences`. |
| `get_composition` | None | Returns composition metadata and the current frame, or `null` fields for other canvas content. |
-| `get_selection` | None | Returns the current frame, composition and source-code context for the selected item. |
+| `get_canvas_html` | None | Returns the rendered composition HTML at the current frame, capped at 100,000 characters. |
+| `get_outlines` | None | Returns active component outlines with source context and composition-space geometry. |
+| `get_playback_state` | None | Returns the current playback, audio, looping and timeline zoom state. |
+| `get_selection` | None | Returns the selection type, owning sequence metadata and source-code context. |
| `get_guides` | None | Returns the horizontal and vertical guides drawn over the canvas. |
+| `set_guides_visible` | `visible`: boolean | Shows or hides all guides in the current composition. |
| `add_guide` | `orientation`: `horizontal` or `vertical`; `position`: number | Adds and shows a guide in the current composition. |
| `remove_guide` | `guideId`: string | Removes a guide from the current composition. |
| `seek_to_frame` | `frame`: a non-negative integer | Seeks the timeline. Values beyond the duration are clamped to the final frame. |
@@ -42,7 +48,11 @@ No Remotion configuration is required. Studio registers its tools when the brows
Tools that change playback require a composition to be open. `play` is unavailable for still compositions.
-`get_selection` returns source-code context when exactly one sequence or property is selected. It returns `null` when there is no selection, multiple items are selected or a guide is selected.
+`get_selection` returns source-code context when exactly one sequence or property is selected. Its `selectedSequence` uses the same descriptor and `sequenceId` as `get_sequences`, including when a property, effect or keyframe inside that sequence is selected. Source context is `null` when there is no selection, multiple items are selected or a guide is selected.
+
+`get_canvas_html` returns only the composition canvas markup, not the surrounding Studio interface. It reports the full `htmlLength` and whether the returned HTML was `truncated`. Canvas and WebGL pixels are not represented in HTML.
+
+`get_outlines` returns the selectable component outlines active at the current frame. Each outline includes the `sequenceId` accepted by `select_sequence`, a node-path-based `outlineId`, source-code context, and a four-point polygon with its bounding box in composition pixels. Components without measurable DOM geometry are omitted.
## Example prompts
@@ -51,9 +61,14 @@ You can ask a browser agent:
- “What is currently selected in the Studio?”
- “List all mounted compositions and their folders.”
- “Open the `Shapes` composition.”
+- “List the sequences and select the `Title` sequence.”
+- “Inspect the HTML rendered on the canvas.”
+- “List the visible components and their canvas geometry.”
+- “What is the current playback state?”
- “Set the timeline zoom to 2.5x.”
- “Seek to frame 90, set the playback rate to 0.5x and play.”
- “List the guides in the current composition.”
+- “Hide the guides.”
- “Add a vertical guide at 640 pixels.”
- “Mute the preview.”
diff --git a/packages/example/e2e/studio.test.mts b/packages/example/e2e/studio.test.mts
index a3d9f7db19..84fac30431 100644
--- a/packages/example/e2e/studio.test.mts
+++ b/packages/example/e2e/studio.test.mts
@@ -1834,6 +1834,7 @@ test.describe('visual mode', () => {
});
await page.addInitScript(() => {
window.localStorage.setItem('remotion.mute', 'false');
+ window.localStorage.setItem('remotion.loop', 'true');
window.localStorage.setItem('remotion.editorShowGuides', 'true');
window.localStorage.setItem(
'remotion.guidesList',
@@ -2085,9 +2086,15 @@ test.describe('visual mode', () => {
return (
tools.has('get_compositions') &&
tools.has('select_composition') &&
+ tools.has('get_sequences') &&
+ tools.has('select_sequence') &&
tools.has('get_composition') &&
+ tools.has('get_canvas_html') &&
+ tools.has('get_outlines') &&
+ tools.has('get_playback_state') &&
tools.has('get_selection') &&
tools.has('get_guides') &&
+ tools.has('set_guides_visible') &&
tools.has('add_guide') &&
tools.has('remove_guide') &&
tools.has('play') &&
@@ -2121,7 +2128,120 @@ test.describe('visual mode', () => {
currentFrame: 3,
currentSelection: contextForAgents,
currentComposition: 'AnimatedBarChart',
+ selectionType: 'sequence',
+ selectedSequence: expect.objectContaining({
+ sequenceId: expect.any(String),
+ name: '0% gridline',
+ type: 'sequence',
+ stack: expect.any(String),
+ selectable: true,
+ }),
+ });
+ const selectedSequence = (
+ webMcpSelection as {
+ readonly selectedSequence: {
+ readonly sequenceId: string;
+ };
+ }
+ ).selectedSequence;
+ const webMcpSequences = await page.evaluate(async () => {
+ const tools = (
+ window as typeof window & {
+ readonly __remotion_webmcp_tools: Map<
+ string,
+ {readonly execute: () => Promise<unknown>}
+ >;
+ }
+ ).__remotion_webmcp_tools;
+ const tool = tools.get('get_sequences');
+ if (!tool) {
+ throw new Error('get_sequences was not registered');
+ }
+
+ return tool.execute();
+ });
+ const sequenceList = (
+ webMcpSequences as {
+ readonly sequences: readonly {
+ readonly sequenceId: string;
+ readonly selectable: boolean;
+ }[];
+ }
+ ).sequences;
+ expect(webMcpSequences).toEqual({
+ currentComposition: 'AnimatedBarChart',
+ sequences: expect.arrayContaining([
+ expect.objectContaining({
+ sequenceId: selectedSequence.sequenceId,
+ name: '0% gridline',
+ selectable: true,
+ }),
+ ]),
+ });
+ const sequenceToSelect = sequenceList.find(
+ (sequence) =>
+ sequence.selectable &&
+ sequence.sequenceId !== selectedSequence.sequenceId,
+ );
+ expect(sequenceToSelect).toBeDefined();
+ const webMcpSelectSequenceResult = await page.evaluate(
+ async ({sequenceId}) => {
+ const tools = (
+ window as typeof window & {
+ readonly __remotion_webmcp_tools: Map<
+ string,
+ {
+ readonly execute: (
+ input: Record<string, unknown>,
+ ) => Promise<unknown>;
+ }
+ >;
+ }
+ ).__remotion_webmcp_tools;
+ const tool = tools.get('select_sequence');
+ if (!tool) {
+ throw new Error('select_sequence was not registered');
+ }
+
+ return tool.execute({sequenceId});
+ },
+ {sequenceId: sequenceToSelect?.sequenceId ?? ''},
+ );
+ expect(webMcpSelectSequenceResult).toEqual({
+ currentComposition: 'AnimatedBarChart',
+ selectedSequence: expect.objectContaining({
+ sequenceId: sequenceToSelect?.sequenceId,
+ selectable: true,
+ }),
});
+ await expect
+ .poll(() =>
+ page.evaluate(async () => {
+ const tools = (
+ window as typeof window & {
+ readonly __remotion_webmcp_tools: Map<
+ string,
+ {readonly execute: () => Promise<unknown>}
+ >;
+ }
+ ).__remotion_webmcp_tools;
+ const tool = tools.get('get_selection');
+ const selection = (await tool?.execute()) as {
+ readonly selectionType?: string;
+ readonly selectedSequence?: {
+ readonly sequenceId?: string;
+ };
+ };
+ return {
+ selectionType: selection.selectionType,
+ sequenceId: selection.selectedSequence?.sequenceId,
+ };
+ }),
+ )
+ .toEqual({
+ selectionType: 'sequence',
+ sequenceId: sequenceToSelect?.sequenceId,
+ });
const webMcpComposition = await page.evaluate(async () => {
const tools = (
window as typeof window & {
@@ -2147,6 +2267,109 @@ test.describe('visual mode', () => {
fps: 30,
currentFrame: 3,
});
+ const webMcpCanvasHtml = await page.evaluate(async () => {
+ const tools = (
+ window as typeof window & {
+ readonly __remotion_webmcp_tools: Map<
+ string,
+ {readonly execute: () => Promise<unknown>}
+ >;
+ }
+ ).__remotion_webmcp_tools;
+ const tool = tools.get('get_canvas_html');
+ if (!tool) {
+ throw new Error('get_canvas_html was not registered');
+ }
+
+ return tool.execute();
+ });
+ const canvasHtml = (
+ webMcpCanvasHtml as {
+ readonly html: string;
+ readonly htmlLength: number;
+ }
+ ).html;
+ expect(webMcpCanvasHtml).toEqual({
+ currentComposition: 'AnimatedBarChart',
+ currentFrame: 3,
+ html: expect.any(String),
+ htmlLength: canvasHtml.length,
+ truncated: false,
+ });
+ expect(canvasHtml).toContain('Performance overview');
+ expect(canvasHtml).toContain('Regional growth');
+ expect(canvasHtml).not.toContain('Change the playback rate');
+ const webMcpOutlines = await page.evaluate(async () => {
+ const tools = (
+ window as typeof window & {
+ readonly __remotion_webmcp_tools: Map<
+ string,
+ {readonly execute: () => Promise<unknown>}
+ >;
+ }
+ ).__remotion_webmcp_tools;
+ const tool = tools.get('get_outlines');
+ if (!tool) {
+ throw new Error('get_outlines was not registered');
+ }
+
+ return tool.execute();
+ });
+ const gridlineOutline = (
+ webMcpOutlines as {
+ readonly outlines: readonly {
+ readonly name: string | null;
+ readonly sequenceId: string;
+ readonly outlineId: string;
+ readonly context: string | null;
+ readonly geometry: {
+ readonly points: readonly {
+ readonly x: number;
+ readonly y: number;
+ }[];
+ readonly boundingBox: {
+ readonly x: number;
+ readonly y: number;
+ readonly width: number;
+ readonly height: number;
+ };
+ };
+ }[];
+ }
+ ).outlines.find((outline) => outline.name === '0% gridline');
+ expect(webMcpOutlines).toEqual({
+ currentComposition: 'AnimatedBarChart',
+ currentFrame: 3,
+ outlines: expect.any(Array),
+ });
+ expect(gridlineOutline).toEqual({
+ outlineId: expect.any(String),
+ sequenceId: selectedSequence.sequenceId,
+ name: '0% gridline',
+ depth: expect.any(Number),
+ context: contextForAgents,
+ geometry: {
+ points: expect.arrayContaining([
+ expect.objectContaining({
+ x: expect.any(Number),
+ y: expect.any(Number),
+ }),
+ ]),
+ boundingBox: {
+ x: expect.any(Number),
+ y: expect.any(Number),
+ width: expect.any(Number),
+ height: expect.any(Number),
+ },
+ },
+ });
+ expect(gridlineOutline?.geometry.points).toHaveLength(4);
+ expect(gridlineOutline?.geometry.boundingBox.x).toBeGreaterThan(150);
+ expect(gridlineOutline?.geometry.boundingBox.x).toBeLessThan(180);
+ expect(gridlineOutline?.geometry.boundingBox.y).toBeGreaterThan(550);
+ expect(gridlineOutline?.geometry.boundingBox.y).toBeLessThan(610);
+ expect(gridlineOutline?.geometry.boundingBox.width).toBeGreaterThan(950);
+ expect(gridlineOutline?.geometry.boundingBox.width).toBeLessThan(1030);
const webMcpCompositions = await page.evaluate(async () => {
const tools = (
window as typeof window & {
@@ -2292,6 +2515,70 @@ test.describe('visual mode', () => {
],
});
await expect(page.locator('.__remotion_editor_guide')).toHaveCount(2);
+ const webMcpHideGuidesResult = await page.evaluate(async () => {
+ const tools = (
+ window as typeof window & {
+ readonly __remotion_webmcp_tools: Map<
+ string,
+ {
+ readonly execute: (
+ input: Record<string, unknown>,
+ ) => Promise<unknown>;
+ }
+ >;
+ }
+ ).__remotion_webmcp_tools;
+ const tool = tools.get('set_guides_visible');
+ if (!tool) {
+ throw new Error('set_guides_visible was not registered');
+ }
+
+ return tool.execute({visible: false});
+ });
+ expect(webMcpHideGuidesResult).toEqual({
+ currentComposition: 'AnimatedBarChart',
+ guidesVisible: false,
+ });
+ await expect(page.locator('.__remotion_editor_guide')).toHaveCount(0);
+ await expect
+ .poll(() =>
+ page.evaluate(() =>
+ localStorage.getItem('remotion.editorShowGuides'),
+ ),
+ )
+ .toBe('false');
+ const webMcpShowGuidesResult = await page.evaluate(async () => {
+ const tools = (
+ window as typeof window & {
+ readonly __remotion_webmcp_tools: Map<
+ string,
+ {
+ readonly execute: (
+ input: Record<string, unknown>,
+ ) => Promise<unknown>;
+ }
+ >;
+ }
+ ).__remotion_webmcp_tools;
+ const tool = tools.get('set_guides_visible');
+ if (!tool) {
+ throw new Error('set_guides_visible was not registered');
+ }
+
+ return tool.execute({visible: true});
+ });
+ expect(webMcpShowGuidesResult).toEqual({
+ currentComposition: 'AnimatedBarChart',
+ guidesVisible: true,
+ });
+ await expect(page.locator('.__remotion_editor_guide')).toHaveCount(2);
+ await expect
+ .poll(() =>
+ page.evaluate(() =>
+ localStorage.getItem('remotion.editorShowGuides'),
+ ),
+ )
+ .toBe('true');
const webMcpAddGuideResult = await page.evaluate(async () => {
const tools = (
window as typeof window & {
@@ -2572,6 +2859,32 @@ test.describe('visual mode', () => {
await expect(
page.getByRole('button', {name: '179', exact: true}),
).toBeVisible();
+ const webMcpPlaybackState = await page.evaluate(async () => {
+ const tools = (
+ window as typeof window & {
+ readonly __remotion_webmcp_tools: Map<
+ string,
+ {readonly execute: () => Promise<unknown>}
+ >;
+ }
+ ).__remotion_webmcp_tools;
+ const tool = tools.get('get_playback_state');
+ if (!tool) {
+ throw new Error('get_playback_state was not registered');
+ }
+
+ return tool.execute();
+ });
+ expect(webMcpPlaybackState).toEqual({
+ currentComposition: 'AnimatedBarChart',
+ currentFrame: 179,
+ playing: false,
+ muted: false,
+ volume: 1,
+ playbackRate: 1.5,
+ looping: true,
+ timelineZoom: 2.5,
+ });
fs.writeFileSync(
configFile,
@@ -2686,6 +2999,73 @@ test.describe('visual mode', () => {
fps: null,
currentFrame: null,
});
+ await expect
+ .poll(() =>
+ page.evaluate(async () => {
+ const tools = (
+ window as typeof window & {
+ readonly __remotion_webmcp_tools: Map<
+ string,
+ {readonly execute: () => Promise<unknown>}
+ >;
+ }
+ ).__remotion_webmcp_tools;
+ const tool = tools.get('get_playback_state');
+ return tool?.execute() ?? null;
+ }),
+ )
+ .toEqual({
+ currentComposition: null,
+ currentFrame: null,
+ playing: null,
+ muted: null,
+ volume: null,
+ playbackRate: null,
+ looping: null,
+ timelineZoom: null,
+ });
+ await expect
+ .poll(() =>
+ page.evaluate(async () => {
+ const tools = (
+ window as typeof window & {
+ readonly __remotion_webmcp_tools: Map<
+ string,
+ {readonly execute: () => Promise<unknown>}
+ >;
+ }
+ ).__remotion_webmcp_tools;
+ const tool = tools.get('get_canvas_html');
+ return tool?.execute() ?? null;
+ }),
+ )
+ .toEqual({
+ currentComposition: null,
+ currentFrame: null,
+ html: null,
+ htmlLength: null,
+ truncated: false,
+ });
+ await expect
+ .poll(() =>
+ page.evaluate(async () => {
+ const tools = (
+ window as typeof window & {
+ readonly __remotion_webmcp_tools: Map<
+ string,
+ {readonly execute: () => Promise<unknown>}
+ >;
+ }
+ ).__remotion_webmcp_tools;
+ const tool = tools.get('get_outlines');
+ return tool?.execute() ?? null;
+ }),
+ )
+ .toEqual({
+ currentComposition: null,
+ currentFrame: null,
+ outlines: [],
+ });
} finally {
fs.writeFileSync(configFile, configBeforeTest);
}
diff --git a/packages/studio/src/components/WebMcp.tsx b/packages/studio/src/components/WebMcp.tsx
index 81a23a77aa..466a463ae3 100644
--- a/packages/studio/src/components/WebMcp.tsx
+++ b/packages/studio/src/components/WebMcp.tsx
@@ -1,16 +1,26 @@
+import {
+ getCanvasSelectableOutlines,
+ measureCanvasOutlines,
+} from '@remotion/canvas';
import {useContext, useEffect, useMemo, useRef, type FC} from 'react';
import {Internals} from 'remotion';
import {pause} from '../api/pause';
import {play} from '../api/play';
import {seek} from '../api/seek';
+import {calculateTimeline} from '../helpers/calculate-timeline';
import {createFolderTree} from '../helpers/create-folder-tree';
import {formatContextForAgents} from '../helpers/format-file-location';
-import {clampTimelineZoom} from '../helpers/get-timeline-max-zoom';
+import {
+ clampTimelineZoom,
+ TIMELINE_MIN_ZOOM,
+} from '../helpers/get-timeline-max-zoom';
+import type {TimelineTrackData} from '../helpers/get-timeline-sequence-sort-key';
import {
EditorShowGuidesContext,
persistGuidesList,
type Guide,
} from '../state/editor-guides';
+import {loadLoopOption} from '../state/loop';
import {persistMuteOption} from '../state/mute';
import {commonPlaybackRates, persistPlaybackRate} from '../state/playbackrate';
import {TimelineZoomCtx} from '../state/timeline-zoom';
@@ -18,7 +28,12 @@ import {useSelectComposition} from './InitialCompositionLoader';
import {findTrackForNodePathInfo} from './Timeline/find-track-for-node-path-info';
import {getCurrentDuration, getCurrentFrame} from './Timeline/imperative-state';
import {parseKeyframeFieldFromNodePath} from './Timeline/parse-keyframe-field-from-node-path';
-import {useTimelineSelection} from './Timeline/TimelineSelection';
+import {shouldShowTrackInTimeline} from './Timeline/should-show-track-in-timeline';
+import {
+ getTimelineSelectionFromNodePathInfo,
+ useTimelineSelection,
+} from './Timeline/TimelineSelection';
+import {getOriginalLocationFromStack} from './Timeline/TimelineStack/get-stack';
import {useResolveStackAndReactToChange} from './Timeline/use-resolved-stack-react-to-change';
type WebMcpTool = {
@@ -48,6 +63,37 @@ type WebMcpCompositionTreeItem =
readonly children: WebMcpCompositionTreeItem[];
};
+type WebMcpSequence = {
+ readonly sequenceId: string;
+ readonly name: string | null;
+ readonly type: TimelineTrackData['sequence']['type'];
+ readonly parentSequenceId: string | null;
+ readonly depth: number;
+ readonly startFrame: number;
+ readonly endFrame: number;
+ readonly durationInFrames: number;
+ readonly stack: string | null;
+ readonly selectable: boolean;
+};
+
+const serializeSequence = (track: TimelineTrackData): WebMcpSequence => {
+ return {
+ sequenceId: track.sequence.id,
+ name:
+ track.sequence.displayName ||
+ track.sequence.controls?.componentName ||
+ null,
+ type: track.sequence.type,
+ parentSequenceId: track.sequence.parent,
+ depth: track.depth,
+ startFrame: track.sequence.from,
+ endFrame: track.sequence.from + track.sequence.duration - 1,
+ durationInFrames: track.sequence.duration,
+ stack: track.sequence.getStack(),
+ selectable: track.nodePathInfo !== null,
+ };
+};
+
const serializeCompositionTree = (
items: ReturnType<typeof createFolderTree>,
): WebMcpCompositionTreeItem[] => {
@@ -68,14 +114,20 @@ const serializeCompositionTree = (
};
const getNoStack = () => null;
+const MAX_CANVAS_HTML_LENGTH = 100_000;
export const WebMcp: FC = () => {
- const {clearSelection, selectedItems} = useTimelineSelection();
+ const {canSelect, clearSelection, selectedItems, selectItems} =
+ useTimelineSelection();
const {canvasContent, compositions, currentCompositionMetadata, folders} =
useContext(Internals.CompositionManager);
- const {setPlaybackRate} = Internals.usePlaybackRate();
+ const {playbackRate: currentPlaybackRate, setPlaybackRate} =
+ Internals.usePlaybackRate();
+ const [, , imperativePlaying] = Internals.Timeline.usePlayingState();
+ const {mediaVolume, playerMuted} = useContext(Internals.MediaVolumeContext);
const {setPlayerMuted} = useContext(Internals.SetMediaVolumeContext);
- const {setZoom: setTimelineZoom} = useContext(TimelineZoomCtx);
+ const {setZoom: setTimelineZoom, zoom: timelineZoomMap} =
+ useContext(TimelineZoomCtx);
const selectComposition = useSelectComposition();
const {editorShowGuides, guidesList, setEditorShowGuides, setGuidesList} =
useContext(EditorShowGuidesContext);
@@ -159,8 +211,16 @@ export const WebMcp: FC = () => {
}, [currentResolvedLocation, selectedItem, selectedItems.length, track]);
const currentSelectionRef = useRef(currentSelection);
currentSelectionRef.current = currentSelection;
+ const selectedSequence = useMemo(
+ () => (track === null ? null : serializeSequence(track)),
+ [track],
+ );
+ const selectedSequenceRef = useRef(selectedSequence);
+ selectedSequenceRef.current = selectedSequence;
const selectedItemsRef = useRef(selectedItems);
selectedItemsRef.current = selectedItems;
+ const canSelectRef = useRef(canSelect);
+ canSelectRef.current = canSelect;
const currentCompositionDefinition = useMemo(() => {
if (canvasContent?.type !== 'composition') {
return null;
@@ -179,6 +239,10 @@ export const WebMcp: FC = () => {
currentCompositionDefinitionRef.current = currentCompositionDefinition;
const compositionsRef = useRef(compositions);
compositionsRef.current = compositions;
+ const sequencesRef = useRef(sequences);
+ sequencesRef.current = sequences;
+ const overrideIdToNodePathMappingsRef = useRef(overrideIdToNodePathMappings);
+ overrideIdToNodePathMappingsRef.current = overrideIdToNodePathMappings;
const foldersRef = useRef(folders);
foldersRef.current = folders;
const currentCompositionMetadataRef = useRef(currentCompositionMetadata);
@@ -187,6 +251,14 @@ export const WebMcp: FC = () => {
guidesListRef.current = guidesList;
const editorShowGuidesRef = useRef(editorShowGuides);
editorShowGuidesRef.current = editorShowGuides;
+ const mediaVolumeRef = useRef(mediaVolume);
+ mediaVolumeRef.current = mediaVolume;
+ const playbackRateRef = useRef(currentPlaybackRate);
+ playbackRateRef.current = currentPlaybackRate;
+ const playerMutedRef = useRef(playerMuted);
+ playerMutedRef.current = playerMuted;
+ const timelineZoomRef = useRef(timelineZoomMap);
+ timelineZoomRef.current = timelineZoomMap;
useEffect(() => {
const {modelContext} = document as Document & {
@@ -197,6 +269,26 @@ export const WebMcp: FC = () => {
}
const controller = new AbortController();
+ const getCurrentTimeline = () => {
+ const composition = currentCompositionDefinitionRef.current;
+ if (composition === null) {
+ return [];
+ }
+
+ const durationInFrames =
+ currentCompositionMetadataRef.current?.durationInFrames ??
+ composition.durationInFrames ??
+ getCurrentDuration();
+
+ return calculateTimeline({
+ sequences: sequencesRef.current,
+ overrideIdsToNodePaths: overrideIdToNodePathMappingsRef.current,
+ compositions: compositionsRef.current,
+ }).filter((timelineTrack) =>
+ shouldShowTrackInTimeline(timelineTrack, durationInFrames),
+ );
+ };
+
Promise.all([
modelContext.registerTool(
{
@@ -259,6 +351,89 @@ export const WebMcp: FC = () => {
},
{signal: controller.signal},
),
+ modelContext.registerTool(
+ {
+ name: 'get_sequences',
+ title: 'Get Studio sequences',
+ description:
+ 'Read the mounted sequences in the current Remotion Studio timeline, including their IDs, hierarchy, timing, type, source stack, and whether they can be selected.',
+ inputSchema: {
+ type: 'object',
+ properties: {},
+ additionalProperties: false,
+ },
+ annotations: {readOnlyHint: true},
+ execute: () => {
+ const compositionId = currentCompositionRef.current;
+ return Promise.resolve({
+ currentComposition: compositionId,
+ sequences:
+ compositionId === null
+ ? []
+ : getCurrentTimeline().map(serializeSequence),
+ });
+ },
+ },
+ {signal: controller.signal},
+ ),
+ modelContext.registerTool(
+ {
+ name: 'select_sequence',
+ title: 'Select Studio sequence',
+ description:
+ 'Select and reveal a sequence in the current Remotion Studio timeline by the sequence ID returned by get_sequences.',
+ inputSchema: {
+ type: 'object',
+ properties: {
+ sequenceId: {
+ type: 'string',
+ minLength: 1,
+ description: 'The sequence ID returned by get_sequences.',
+ },
+ },
+ required: ['sequenceId'],
+ additionalProperties: false,
+ },
+ annotations: {readOnlyHint: false},
+ execute: ({sequenceId}) => {
+ if (typeof sequenceId !== 'string' || sequenceId.length === 0) {
+ throw new Error('sequenceId must be a non-empty string.');
+ }
+
+ const compositionId = currentCompositionRef.current;
+ if (compositionId === null) {
+ throw new Error('No composition is currently selected.');
+ }
+
+ if (!canSelectRef.current) {
+ throw new Error('Studio sequence selection is unavailable.');
+ }
+
+ const timelineTrack = getCurrentTimeline().find(
+ (candidate) => candidate.sequence.id === sequenceId,
+ );
+ if (!timelineTrack) {
+ throw new Error(
+ `Sequence ${sequenceId} not found in the current composition.`,
+ );
+ }
+
+ const selection = getTimelineSelectionFromNodePathInfo(
+ timelineTrack.nodePathInfo,
+ );
+ if (selection === null) {
+ throw new Error(`Sequence ${sequenceId} cannot be selected.`);
+ }
+
+ selectItems([selection], {reveal: true});
+ return Promise.resolve({
+ currentComposition: compositionId,
+ selectedSequence: serializeSequence(timelineTrack),
+ });
+ },
+ },
+ {signal: controller.signal},
+ ),
modelContext.registerTool(
{
name: 'get_composition',
@@ -302,6 +477,225 @@ export const WebMcp: FC = () => {
},
{signal: controller.signal},
),
+ modelContext.registerTool(
+ {
+ name: 'get_canvas_html',
+ title: 'Get Studio canvas HTML',
+ description:
+ 'Read the HTML of the rendered composition at the current frame. The result is limited to the composition canvas and does not include the Studio interface. Canvas and WebGL pixels are not included.',
+ inputSchema: {
+ type: 'object',
+ properties: {},
+ additionalProperties: false,
+ },
+ annotations: {readOnlyHint: true},
+ execute: () => {
+ const compositionId = currentCompositionRef.current;
+ if (compositionId === null) {
+ return Promise.resolve({
+ currentComposition: null,
+ currentFrame: null,
+ html: null,
+ htmlLength: null,
+ truncated: false,
+ });
+ }
+
+ const outerHtml = Internals.portalNode().outerHTML;
+ return Promise.resolve({
+ currentComposition: compositionId,
+ currentFrame: getCurrentFrame(),
+ html: outerHtml.slice(0, MAX_CANVAS_HTML_LENGTH),
+ htmlLength: outerHtml.length,
+ truncated: outerHtml.length > MAX_CANVAS_HTML_LENGTH,
+ });
+ },
+ },
+ {signal: controller.signal},
+ ),
+ modelContext.registerTool(
+ {
+ name: 'get_outlines',
+ title: 'Get Studio canvas outlines',
+ description:
+ 'Read the active selectable component outlines in the current Remotion Studio canvas, including sequence identity, source-code context, and geometry in composition pixels.',
+ inputSchema: {
+ type: 'object',
+ properties: {},
+ additionalProperties: false,
+ },
+ annotations: {readOnlyHint: true},
+ execute: async () => {
+ const composition = currentCompositionDefinitionRef.current;
+ if (composition === null) {
+ return {
+ currentComposition: null,
+ currentFrame: null,
+ outlines: [],
+ };
+ }
+
+ const currentFrame = getCurrentFrame();
+ const selectableOutlines = getCanvasSelectableOutlines({
+ sequences: sequencesRef.current,
+ overrideIdsToNodePaths: overrideIdToNodePathMappingsRef.current,
+ compositions: compositionsRef.current,
+ timelinePosition: currentFrame,
+ });
+ const portalNode = Internals.portalNode();
+ const portalRect = portalNode.getBoundingClientRect();
+ const metadata = currentCompositionMetadataRef.current;
+ const compositionWidth =
+ metadata?.width ?? composition.width ?? portalNode.offsetWidth;
+ const compositionHeight =
+ metadata?.height ?? composition.height ?? portalNode.offsetHeight;
+ const scaleX = portalRect.width / compositionWidth;
+ const scaleY = portalRect.height / compositionHeight;
+ if (
+ !Number.isFinite(scaleX) ||
+ scaleX === 0 ||
+ !Number.isFinite(scaleY) ||
+ scaleY === 0
+ ) {
+ throw new Error('The Studio canvas is not ready to be measured.');
+ }
+
+ const measuredOutlines = measureCanvasOutlines(
+ portalNode,
+ selectableOutlines.map((outline) => {
+ if (outline.sequence.refForOutline === null) {
+ throw new Error('Expected an outline ref.');
+ }
+
+ return {
+ key: outline.key,
+ ref: outline.sequence.refForOutline,
+ crop: {left: 0, right: 0, top: 0, bottom: 0},
+ includeOutsideContainer: true,
+ };
+ }),
+ );
+ const measurementsByKey = new Map(
+ measuredOutlines.map((outline) => [outline.key, outline]),
+ );
+ const contextByStack = new Map<
+ string,
+ Promise<Awaited<ReturnType<typeof getOriginalLocationFromStack>>>
+ >();
+ const outlines = await Promise.all(
+ selectableOutlines.map(async (outline) => {
+ const measurement = measurementsByKey.get(outline.key);
+ if (!measurement) {
+ return null;
+ }
+
+ const outlineStack = outline.sequence.getStack();
+ let location = null;
+ if (outlineStack !== null) {
+ let promise = contextByStack.get(outlineStack);
+ if (!promise) {
+ promise = getOriginalLocationFromStack(
+ outlineStack,
+ 'sequence',
+ ).catch(() => null);
+ contextByStack.set(outlineStack, promise);
+ }
+
+ location = await promise;
+ }
+
+ const name =
+ outline.sequence.displayName ||
+ outline.sequence.controls?.componentName ||
+ null;
+ const points = measurement.points.map((point) => ({
+ x: point.x / scaleX,
+ y: point.y / scaleY,
+ }));
+ const xValues = points.map((point) => point.x);
+ const yValues = points.map((point) => point.y);
+ const left = Math.min(...xValues);
+ const top = Math.min(...yValues);
+ const right = Math.max(...xValues);
+ const bottom = Math.max(...yValues);
+
+ return {
+ outlineId: outline.key,
+ sequenceId: outline.sequence.id,
+ name,
+ depth: outline.depth,
+ context: formatContextForAgents({
+ location,
+ name,
+ root: window.remotion_cwd,
+ }),
+ geometry: {
+ points,
+ boundingBox: {
+ x: left,
+ y: top,
+ width: right - left,
+ height: bottom - top,
+ },
+ },
+ };
+ }),
+ );
+
+ return {
+ currentComposition: composition.id,
+ currentFrame,
+ outlines: outlines.filter(
+ (outline): outline is NonNullable<typeof outline> =>
+ outline !== null,
+ ),
+ };
+ },
+ },
+ {signal: controller.signal},
+ ),
+ modelContext.registerTool(
+ {
+ name: 'get_playback_state',
+ title: 'Get Studio playback state',
+ description:
+ 'Read the current frame, playing state, audio state, playback rate, looping state, and timeline zoom for the composition open in Remotion Studio. All playback fields are null when the canvas is not showing a composition.',
+ inputSchema: {
+ type: 'object',
+ properties: {},
+ additionalProperties: false,
+ },
+ annotations: {readOnlyHint: true},
+ execute: () => {
+ const compositionId = currentCompositionRef.current;
+ if (compositionId === null) {
+ return Promise.resolve({
+ currentComposition: null,
+ currentFrame: null,
+ playing: null,
+ muted: null,
+ volume: null,
+ playbackRate: null,
+ looping: null,
+ timelineZoom: null,
+ });
+ }
+
+ return Promise.resolve({
+ currentComposition: compositionId,
+ currentFrame: getCurrentFrame(),
+ playing: imperativePlaying.current,
+ muted: playerMutedRef.current,
+ volume: mediaVolumeRef.current,
+ playbackRate: playbackRateRef.current,
+ looping: loadLoopOption(),
+ timelineZoom:
+ timelineZoomRef.current[compositionId] ?? TIMELINE_MIN_ZOOM,
+ });
+ },
+ },
+ {signal: controller.signal},
+ ),
modelContext.registerTool(
{
name: 'get_selection',
@@ -319,6 +713,11 @@ export const WebMcp: FC = () => {
currentFrame: getCurrentFrame(),
currentSelection: currentSelectionRef.current,
currentComposition: currentCompositionRef.current,
+ selectionType:
+ selectedItemsRef.current.length === 1
+ ? selectedItemsRef.current[0].type
+ : null,
+ selectedSequence: selectedSequenceRef.current,
}),
},
{signal: controller.signal},
@@ -358,6 +757,44 @@ export const WebMcp: FC = () => {
},
{signal: controller.signal},
),
+ modelContext.registerTool(
+ {
+ name: 'set_guides_visible',
+ title: 'Set Studio guides visibility',
+ description:
+ 'Show or hide all guides for the current Remotion Studio composition.',
+ inputSchema: {
+ type: 'object',
+ properties: {
+ visible: {
+ type: 'boolean',
+ description: 'Whether guides should be visible.',
+ },
+ },
+ required: ['visible'],
+ additionalProperties: false,
+ },
+ annotations: {readOnlyHint: false},
+ execute: ({visible}) => {
+ if (typeof visible !== 'boolean') {
+ throw new Error('visible must be a boolean.');
+ }
+
+ const compositionId = currentCompositionRef.current;
+ if (compositionId === null) {
+ throw new Error('No composition is currently selected.');
+ }
+
+ editorShowGuidesRef.current = visible;
+ setEditorShowGuides(() => visible);
+ return Promise.resolve({
+ currentComposition: compositionId,
+ guidesVisible: visible,
+ });
+ },
+ },
+ {signal: controller.signal},
+ ),
modelContext.registerTool(
{
name: 'add_guide',
@@ -764,7 +1201,9 @@ export const WebMcp: FC = () => {
};
}, [
clearSelection,
+ imperativePlaying,
selectComposition,
+ selectItems,
setEditorShowGuides,
setGuidesList,
setPlaybackRate,