🎥 Make videos programmatically with React
Replace WebMCP currentComposition with currentContent
JonnyBurger1eb6f8e27578
diff --git a/packages/docs/docs/ai/webmcp.mdx b/packages/docs/docs/ai/webmcp.mdxindex 7696c65b06..948fc1ba44 100644--- a/packages/docs/docs/ai/webmcp.mdx+++ b/packages/docs/docs/ai/webmcp.mdx@@ -43,6 +43,24 @@ The shape of inputs and outputs is not currently stable.Tools that modify the timeline, playback or guides require a composition to be open.+### `currentContent`<AvailableFrom v="4.0.523" />++Tool responses identify the content open in the canvas using `currentContent`, replacing `currentComposition`. It is `null` when no content is open. Browser-rendered outputs include metadata without the blob itself.++```ts title="CurrentContent"+type CurrentContent =+ | {type: 'composition'; compositionId: string}+ | {type: 'asset'; asset: string}+ | {type: 'output'; path: string}+ | {+ type: 'output-blob';+ displayName: string;+ width: number;+ height: number;+ sizeInBytes: number;+ };+```+### `restart_studio`<AvailableFrom v="4.0.521" />Restarts the Studio server using [`restartStudio()`](/docs/studio/restart-studio). Requires a writable Studio with a running server. No composition needs to be open. The browser temporarily disconnects and reconnects when Studio is ready.@@ -139,7 +157,7 @@ Opens a registered composition by name.```ts title="Output"{- currentComposition: string;+ currentContent: {type: 'composition'; compositionId: string};}```@@ -153,7 +171,7 @@ Returns the timeline sequences mounted for the current composition, including th```ts title="Output"{- currentComposition: string | null;+ currentContent: CurrentContent | null;sequences: Array<{sequenceId: string;name: string | null;@@ -181,7 +199,7 @@ Selects and reveals a sequence using a `sequenceId` returned by `get_sequences`.```ts title="Output"{- currentComposition: string;+ currentContent: {type: 'composition'; compositionId: string};selectedSequence: {sequenceId: string;name: string | null;@@ -227,7 +245,7 @@ Returns the rendered composition HTML at the current frame without the surroundi```ts title="Output"{- currentComposition: string | null;+ currentContent: CurrentContent | null;currentFrame: number | null;html: string | null;htmlLength: number | null;@@ -245,7 +263,7 @@ Returns the measurable selectable component outlines active at the current frame```ts title="Output"{- currentComposition: string | null;+ currentContent: CurrentContent | null;currentFrame: number | null;outlines: Array<{sequenceId: string;@@ -283,7 +301,7 @@ Returns the current playhead, playback, audio, looping and timeline zoom state.```ts title="Output"{- currentComposition: string | null;+ currentContent: CurrentContent | null;currentFrame: number | null;playing: boolean | null;muted: boolean | null;@@ -305,7 +323,7 @@ Returns source-code context when exactly one supported timeline item is selected```ts title="Output"{currentFrame: number;- currentComposition: string | null;+ currentContent: CurrentContent | null;currentSelection: string | null;selectionType:| 'guide'@@ -342,7 +360,7 @@ Returns guides for the current composition. Vertical positions are x-coordinates```ts title="Output"{- currentComposition: string | null;+ currentContent: CurrentContent | null;guidesVisible: boolean;guides: Array<{id: string;@@ -365,7 +383,7 @@ Shows or hides every guide in the current composition.```ts title="Output"{- currentComposition: string;+ currentContent: {type: 'composition'; compositionId: string};guidesVisible: boolean;}```@@ -383,7 +401,7 @@ Adds and shows a guide in the current composition. Positions use composition pix```ts title="Output"{- currentComposition: string;+ currentContent: {type: 'composition'; compositionId: string};guide: {id: string;orientation: 'horizontal' | 'vertical';@@ -405,7 +423,7 @@ Removes a guide using an ID returned by `get_guides` or `add_guide`.```ts title="Output"{- currentComposition: string;+ currentContent: {type: 'composition'; compositionId: string};guideId: string;removed: true;}@@ -421,7 +439,7 @@ Starts playback from the current frame. Still compositions cannot be played.```ts title="Output"{- currentComposition: string;+ currentContent: {type: 'composition'; compositionId: string};playing: true;}```@@ -436,7 +454,7 @@ Pauses playback at the current frame.```ts title="Output"{- currentComposition: string;+ currentContent: {type: 'composition'; compositionId: string};playing: false;}```@@ -451,7 +469,7 @@ Mutes audio playback.```ts title="Output"{- currentComposition: string;+ currentContent: {type: 'composition'; compositionId: string};muted: true;}```@@ -466,7 +484,7 @@ Unmutes audio playback.```ts title="Output"{- currentComposition: string;+ currentContent: {type: 'composition'; compositionId: string};muted: false;}```@@ -485,7 +503,7 @@ Sets the timeline zoom using a normalized value between `0` (fully zoomed out) a```ts title="Output"{- currentComposition: string;+ currentContent: {type: 'composition'; compositionId: string};timelineZoom: number;}```@@ -513,7 +531,7 @@ Sets the playback multiplier. Negative values play backwards.```ts title="Output"{- currentComposition: string;+ currentContent: {type: 'composition'; compositionId: string};playbackRate: number;}```@@ -532,7 +550,7 @@ Seeks to a zero-based frame. Values beyond the duration are clamped to the final```ts title="Output"{- currentComposition: string;+ currentContent: {type: 'composition'; compositionId: string};currentFrame: number;}```diff --git a/packages/example/e2e/studio.test.mts b/packages/example/e2e/studio.test.mtsindex 1719cd7cbb..7f309f8494 100644--- a/packages/example/e2e/studio.test.mts+++ b/packages/example/e2e/studio.test.mts@@ -3035,7 +3035,10 @@ export const SequenceShiftRepro = () => {expect(webMcpSelection).toEqual({currentFrame: 3,currentSelection: contextForAgents,- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },selectionType: 'sequence',selectedSequence: expect.objectContaining({sequenceId: expect.any(String),@@ -3078,7 +3081,10 @@ export const SequenceShiftRepro = () => {}).sequences;expect(webMcpSequences).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },sequences: expect.arrayContaining([expect.objectContaining({sequenceId: selectedSequence.sequenceId,@@ -3117,7 +3123,10 @@ export const SequenceShiftRepro = () => {{sequenceId: sequenceToSelect?.sequenceId ?? ''},);expect(webMcpSelectSequenceResult).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },selectedSequence: expect.objectContaining({sequenceId: sequenceToSelect?.sequenceId,selectable: true,@@ -3199,7 +3208,10 @@ export const SequenceShiftRepro = () => {}).html;expect(webMcpCanvasHtml).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },currentFrame: 3,html: expect.any(String),htmlLength: canvasHtml.length,@@ -3250,7 +3262,10 @@ export const SequenceShiftRepro = () => {}).outlines.find((outline) => outline.name === '0% gridline');expect(webMcpOutlines).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },currentFrame: 3,outlines: expect.any(Array),});@@ -3343,7 +3358,10 @@ export const SequenceShiftRepro = () => {return tool.execute({compositionName: 'package-absolute-fill'});});expect(webMcpSelectCompositionResult).toEqual({- currentComposition: 'package-absolute-fill',+ currentContent: {+ type: 'composition',+ compositionId: 'package-absolute-fill',+ },});await expect.poll(() => new URL(page.url()).pathname)@@ -3411,7 +3429,10 @@ export const SequenceShiftRepro = () => {return tool.execute({});});expect(webMcpGuides).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },guidesVisible: true,guides: [{@@ -3450,7 +3471,10 @@ export const SequenceShiftRepro = () => {return tool.execute({visible: false});});expect(webMcpHideGuidesResult).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },guidesVisible: false,});await expect(page.locator('.__remotion_editor_guide')).toHaveCount(0);@@ -3482,7 +3506,10 @@ export const SequenceShiftRepro = () => {return tool.execute({visible: true});});expect(webMcpShowGuidesResult).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },guidesVisible: true,});await expect(page.locator('.__remotion_editor_guide')).toHaveCount(2);@@ -3514,7 +3541,10 @@ export const SequenceShiftRepro = () => {return tool.execute({orientation: 'vertical', position: 640});});expect(webMcpAddGuideResult).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },guide: {id: expect.any(String),orientation: 'vertical',@@ -3563,7 +3593,10 @@ export const SequenceShiftRepro = () => {{guideId: addedGuideId},);expect(webMcpRemoveGuideResult).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },guideId: addedGuideId,removed: true,});@@ -3602,7 +3635,10 @@ export const SequenceShiftRepro = () => {return tool.execute({playbackRate: 1.5});});expect(webMcpPlaybackRateResult).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },playbackRate: 1.5,});await expect(@@ -3632,7 +3668,10 @@ export const SequenceShiftRepro = () => {return tool.execute({zoom: 0.5});});expect(webMcpTimelineZoomResult).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },timelineZoom: expect.any(Number),});const normalizedTimelineZoom = (@@ -3663,7 +3702,10 @@ export const SequenceShiftRepro = () => {return tool.execute({zoom: 1});});expect(webMcpMaximumTimelineZoomResult).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },timelineZoom: 1,});await expect(@@ -3694,7 +3736,10 @@ export const SequenceShiftRepro = () => {return tool.execute({});});expect(webMcpMuteResult).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },muted: true,});await expect(@@ -3721,7 +3766,10 @@ export const SequenceShiftRepro = () => {return tool.execute({});});expect(webMcpUnmuteResult).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },muted: false,});await expect(@@ -3748,7 +3796,10 @@ export const SequenceShiftRepro = () => {return tool.execute({});});expect(webMcpPlayResult).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },playing: true,});await expect(@@ -3775,7 +3826,10 @@ export const SequenceShiftRepro = () => {return tool.execute({});});expect(webMcpPauseResult).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },playing: false,});await expect(@@ -3803,7 +3857,10 @@ export const SequenceShiftRepro = () => {});expect(webMcpSeekResult).toEqual({currentFrame: 179,- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },});await expect(page.getByRole('button', {name: '179', exact: true}),@@ -3825,7 +3882,10 @@ export const SequenceShiftRepro = () => {return tool.execute();});expect(webMcpPlaybackState).toEqual({- currentComposition: 'AnimatedBarChart',+ currentContent: {+ type: 'composition',+ compositionId: 'AnimatedBarChart',+ },currentFrame: 179,playing: false,muted: false,@@ -3964,7 +4024,7 @@ export const SequenceShiftRepro = () => {}),).toEqual({- currentComposition: null,+ currentContent: {type: 'asset', asset: 'test.gif'},currentFrame: null,playing: null,muted: null,@@ -3989,7 +4049,7 @@ export const SequenceShiftRepro = () => {}),).toEqual({- currentComposition: null,+ currentContent: {type: 'asset', asset: 'test.gif'},currentFrame: null,html: null,htmlLength: null,@@ -4011,10 +4071,37 @@ export const SequenceShiftRepro = () => {}),).toEqual({- currentComposition: null,+ currentContent: {type: 'asset', asset: 'test.gif'},currentFrame: null,outlines: [],});+ for (const preview of [+ {+ route: '/assets/test.gif',+ content: {type: 'asset', asset: 'test.gif'},+ },+ {+ route: '/outputs/public%2Ftest.gif',+ content: {type: 'output', path: 'public/test.gif'},+ },+ ]) {+ await page.goto(`${STUDIO_URL}${preview.route}`);+ await expect+ .poll(() =>+ page.evaluate(async () => {+ const tools = (+ window as typeof window & {+ readonly __remotion_webmcp_tools: Map<+ string,+ {readonly execute: () => Promise<unknown>}+ >;+ }+ ).__remotion_webmcp_tools;+ return tools.get('get_selection')?.execute() ?? null;+ }),+ )+ .toMatchObject({currentContent: preview.content});+ }} finally {fs.writeFileSync(configFile, configBeforeTest);}diff --git a/packages/studio/src/components/WebMcp.tsx b/packages/studio/src/components/WebMcp.tsxindex 3b9198989d..7c065b6c20 100644--- a/packages/studio/src/components/WebMcp.tsx+++ b/packages/studio/src/components/WebMcp.tsx@@ -242,6 +242,21 @@ export const WebMcp: FC = () => {) ?? null);}, [canvasContent, compositions]);+ const currentContent = useMemo(() => {+ if (canvasContent?.type === 'output-blob') {+ return {+ type: canvasContent.type,+ displayName: canvasContent.displayName,+ width: canvasContent.width,+ height: canvasContent.height,+ sizeInBytes: canvasContent.sizeInBytes,+ };+ }++ return canvasContent;+ }, [canvasContent]);+ const currentContentRef = useRef(currentContent);+ currentContentRef.current = currentContent;const currentComposition = currentCompositionDefinition?.id ?? null;const currentCompositionRef = useRef(currentComposition);currentCompositionRef.current = currentComposition;@@ -424,7 +439,12 @@ export const WebMcp: FC = () => {}selectComposition(composition, true);- return Promise.resolve({currentComposition: composition.id});+ return Promise.resolve({+ currentContent: {+ type: 'composition',+ compositionId: composition.id,+ },+ });},},{signal: controller.signal},@@ -444,7 +464,7 @@ export const WebMcp: FC = () => {execute: () => {const compositionId = currentCompositionRef.current;return Promise.resolve({- currentComposition: compositionId,+ currentContent: currentContentRef.current,sequences:compositionId === null? []@@ -505,7 +525,7 @@ export const WebMcp: FC = () => {selectItems([selection], {reveal: true});return Promise.resolve({- currentComposition: compositionId,+ currentContent: currentContentRef.current,selectedSequence: serializeSequence(timelineTrack),});},@@ -571,7 +591,7 @@ export const WebMcp: FC = () => {const compositionId = currentCompositionRef.current;if (compositionId === null) {return Promise.resolve({- currentComposition: null,+ currentContent: currentContentRef.current,currentFrame: null,html: null,htmlLength: null,@@ -581,7 +601,7 @@ export const WebMcp: FC = () => {const outerHtml = Internals.portalNode().outerHTML;return Promise.resolve({- currentComposition: compositionId,+ currentContent: currentContentRef.current,currentFrame: getCurrentFrame(),html: outerHtml.slice(0, MAX_CANVAS_HTML_LENGTH),htmlLength: outerHtml.length,@@ -607,7 +627,7 @@ export const WebMcp: FC = () => {const composition = currentCompositionDefinitionRef.current;if (composition === null) {return {- currentComposition: null,+ currentContent: currentContentRef.current,currentFrame: null,outlines: [],};@@ -719,7 +739,7 @@ export const WebMcp: FC = () => {);return {- currentComposition: composition.id,+ currentContent: currentContentRef.current,currentFrame,outlines: outlines.filter((outline): outline is NonNullable<typeof outline> =>@@ -746,7 +766,7 @@ export const WebMcp: FC = () => {const compositionId = currentCompositionRef.current;if (compositionId === null) {return Promise.resolve({- currentComposition: null,+ currentContent: currentContentRef.current,currentFrame: null,playing: null,muted: null,@@ -771,7 +791,7 @@ export const WebMcp: FC = () => {});return Promise.resolve({- currentComposition: compositionId,+ currentContent: currentContentRef.current,currentFrame: getCurrentFrame(),playing: isPlaying(),muted: playerMutedRef.current,@@ -792,7 +812,7 @@ export const WebMcp: FC = () => {name: 'get_selection',title: 'Get Studio selection',description:- 'Read the current frame, composition, and source-code context for the item currently selected in Remotion Studio. The selection matches "Copy context for agents".',+ 'Read the current frame, canvas content (composition, asset, or output), and source-code context for the item currently selected in Remotion Studio. The selection matches "Copy context for agents".',inputSchema: {type: 'object',properties: {},@@ -803,7 +823,7 @@ export const WebMcp: FC = () => {Promise.resolve({currentFrame: getCurrentFrame(),currentSelection: currentSelectionRef.current,- currentComposition: currentCompositionRef.current,+ currentContent: currentContentRef.current,selectionType:selectedItemsRef.current.length === 1? selectedItemsRef.current[0].type@@ -830,7 +850,7 @@ export const WebMcp: FC = () => {const guidesAreVisible = editorShowGuidesRef.current;return Promise.resolve({- currentComposition: compositionId,+ currentContent: currentContentRef.current,guidesVisible: guidesAreVisible,guides:compositionId === null@@ -879,7 +899,7 @@ export const WebMcp: FC = () => {editorShowGuidesRef.current = visible;setEditorShowGuides(() => visible);return Promise.resolve({- currentComposition: compositionId,+ currentContent: currentContentRef.current,guidesVisible: visible,});},@@ -941,7 +961,7 @@ export const WebMcp: FC = () => {setEditorShowGuides(() => true);return Promise.resolve({- currentComposition: compositionId,+ currentContent: currentContentRef.current,guide: {id: guide.id,orientation: guide.orientation,@@ -1009,7 +1029,7 @@ export const WebMcp: FC = () => {}return Promise.resolve({- currentComposition: compositionId,+ currentContent: currentContentRef.current,guideId,removed: true,});@@ -1047,7 +1067,7 @@ export const WebMcp: FC = () => {play();return Promise.resolve({- currentComposition: compositionId,+ currentContent: currentContentRef.current,playing: true,});},@@ -1078,7 +1098,7 @@ export const WebMcp: FC = () => {pause();return Promise.resolve({- currentComposition: compositionId,+ currentContent: currentContentRef.current,playing: false,});},@@ -1106,7 +1126,7 @@ export const WebMcp: FC = () => {setPlayerMuted(true);persistMuteOption(true);return Promise.resolve({- currentComposition: compositionId,+ currentContent: currentContentRef.current,muted: true,});},@@ -1134,7 +1154,7 @@ export const WebMcp: FC = () => {setPlayerMuted(false);persistMuteOption(false);return Promise.resolve({- currentComposition: compositionId,+ currentContent: currentContentRef.current,muted: false,});},@@ -1204,7 +1224,7 @@ export const WebMcp: FC = () => {});return Promise.resolve({- currentComposition: compositionId,+ currentContent: currentContentRef.current,timelineZoom: timelineZoomToNormalized({zoom: timelineZoom,minZoom,@@ -1252,7 +1272,7 @@ export const WebMcp: FC = () => {setPlaybackRate(() => playbackRate);persistPlaybackRate(playbackRate);return Promise.resolve({- currentComposition: compositionId,+ currentContent: currentContentRef.current,playbackRate,});},@@ -1295,7 +1315,7 @@ export const WebMcp: FC = () => {seek(currentFrame);return Promise.resolve({currentFrame,- currentComposition: compositionId,+ currentContent: currentContentRef.current,});},},