🎥 Make videos programmatically with React
@remotion/studio: Add generate with agent action
Jonny Burger94c69b99bca6
diff --git a/packages/studio/src/components/AgentPrompt.tsx b/packages/studio/src/components/AgentPrompt.tsxnew file mode 100644index 0000000000..c05d2040f8--- /dev/null+++ b/packages/studio/src/components/AgentPrompt.tsx@@ -0,0 +1,195 @@+import React, {useCallback, useContext} from 'react';+import {StudioServerConnectionCtx} from '../helpers/client-id';+import {+ BLACK_ALPHA_30,+ BLUE,+ BORDER_WHITE_ALPHA_12,+ LIGHT_TEXT,+ WHITE,+} from '../helpers/colors';+import {copyText} from '../helpers/copy-text';+import {useCopyFeedback} from '../helpers/use-copy-feedback';+import {CopyIcon} from '../icons/copy';+import {SkillsIcon} from '../icons/skills';+import {CodingAgentButton} from './CodingAgentButton';+import type {RenderInlineAction} from './InlineAction';+import {InlineAction} from './InlineAction';+import {ValidationMessage} from './NewComposition/ValidationMessage';+import {showNotification} from './Notifications/NotificationCenter';+import {useSettings} from './SettingsContext';+import {SkillSettingsRow} from './SkillsSettings';++const text: React.CSSProperties = {+ color: LIGHT_TEXT,+ fontFamily: 'sans-serif',+ fontSize: 14,+ lineHeight: 1.5,+};++const commandField: React.CSSProperties = {+ alignItems: 'flex-start',+ background: BLACK_ALPHA_30,+ borderRadius: 6,+ boxSizing: 'border-box',+ display: 'flex',+ gap: 8,+ marginTop: 10,+ padding: '8px 8px 8px 10px',+ width: '100%',+};++const code: React.CSSProperties = {+ color: WHITE,+ flex: 1,+ fontFamily: 'monospace',+ fontSize: 13,+ fontWeight: 'normal',+ lineHeight: 1.5,+ margin: 0,+ minWidth: 0,+ overflowWrap: 'anywhere',+ whiteSpace: 'pre-wrap',+};++const copyAction: React.CSSProperties = {flexShrink: 0};+const copyIcon: React.CSSProperties = {flexShrink: 0, height: 12, width: 12};+const skillsIcon: React.CSSProperties = {+ display: 'inline-block',+ height: 18,+ marginRight: 6,+ verticalAlign: 'middle',+ width: 18,+};+const promptHeader: React.CSSProperties = {+ alignItems: 'center',+ display: 'flex',+ gap: 12,+ justifyContent: 'space-between',+};+const skillRowContainer: React.CSSProperties = {+ borderTop: BORDER_WHITE_ALPHA_12,+ marginTop: 10,+};+const skillError: React.CSSProperties = {marginTop: 10};++export const AgentPrompt: React.FC<{+ readonly availableText: string;+ readonly promptDetails: string;+ readonly skillId: string;+}> = ({availableText, promptDetails, skillId}) => {+ const {codingAgentInfo, remotionSkillsInfo, skillActionError} = useSettings();+ const {previewServerState} = useContext(StudioServerConnectionCtx);+ const skill = remotionSkillsInfo?.skills.find(({name}) => name === skillId);+ const skillAvailable = Boolean(+ skill?.installedInProject || skill?.installedGlobally,+ );+ const canInstallSkill =+ !window.remotion_isReadOnlyStudio &&+ previewServerState.type === 'connected' &&+ remotionSkillsInfo !== null;+ const skillName = `/${skillId}`;+ const prompt = `${skillName}${promptDetails}`;+ const installCommand = 'npx remotion skills add';+ const hasCodingAgent =+ (codingAgentInfo?.installedCodingAgents.length ?? 0) > 0;+ const {copied: promptCopied, markCopied: markPromptCopied} =+ useCopyFeedback();+ const {copied: installCopied, markCopied: markInstallCopied} =+ useCopyFeedback();++ const copy = useCallback((value: string, markCopied: () => void) => {+ copyText(value)+ .then(markCopied)+ .catch((err) => showNotification(`Could not copy: ${err.message}`, 2000));+ }, []);+ const renderPromptCopy: RenderInlineAction = useCallback(+ (color) => (+ <CopyIcon copied={promptCopied} color={color} style={copyIcon} />+ ),+ [promptCopied],+ );+ const renderInstallCopy: RenderInlineAction = useCallback(+ (color) => (+ <CopyIcon copied={installCopied} color={color} style={copyIcon} />+ ),+ [installCopied],+ );++ return (+ <>+ {skillAvailable ? null : (+ <>+ <div style={text}>Install the Agent Skill first:</div>+ {canInstallSkill && skill ? (+ <div+ aria-label="Remotion Agent Skills"+ role="list"+ style={skillRowContainer}+ >+ <SkillSettingsRow isLast={false} skill={skill} />+ </div>+ ) : (+ <div style={commandField}>+ <pre style={code}>{installCommand}</pre>+ <InlineAction+ onClick={() => copy(installCommand, markInstallCopied)}+ renderAction={renderInstallCopy}+ style={copyAction}+ title="Copy command"+ variant={null}+ />+ </div>+ )}+ {skillActionError ? (+ <div style={skillError}>+ <ValidationMessage+ align="flex-start"+ message={skillActionError}+ type="error"+ />+ </div>+ ) : null}+ </>+ )}+ <div style={{...promptHeader, marginTop: skillAvailable ? 0 : 16}}>+ <div style={text}>+ {skillAvailable+ ? availableText+ : `Then ${availableText.toLowerCase()}`}+ </div>+ {hasCodingAgent ? (+ <CodingAgentButton+ label="Open in"+ prompt={prompt}+ size="compact"+ style={null}+ />+ ) : null}+ </div>+ <div style={commandField}>+ <pre style={code}>+ <SkillsIcon color={BLUE} style={skillsIcon} aria-hidden />+ <span+ style={{+ color: BLUE,+ fontFamily: 'inherit',+ fontSize: 'inherit',+ fontWeight: 'inherit',+ lineHeight: 'inherit',+ }}+ >+ {skillName}+ </span>+ {promptDetails}+ </pre>+ <InlineAction+ onClick={() => copy(prompt, markPromptCopied)}+ renderAction={renderPromptCopy}+ style={copyAction}+ title="Copy prompt"+ variant={null}+ />+ </div>+ </>+ );+};diff --git a/packages/studio/src/components/FixComputedValueModal.tsx b/packages/studio/src/components/FixComputedValueModal.tsxindex d16b104b97..ac3ab92968 100644--- a/packages/studio/src/components/FixComputedValueModal.tsx+++ b/packages/studio/src/components/FixComputedValueModal.tsx@@ -1,32 +1,16 @@import type {SymbolicatedStackFrame} from '@remotion/studio-shared';-import React, {useCallback, useContext, useEffect, useState} from 'react';+import React, {useContext, useEffect, useState} from 'react';import {resolveFileSource} from '../error-overlay/react-overlay/effects/resolve-file-source';import {OpenInEditor} from '../error-overlay/remotion-overlay/OpenInEditor';import {StackElement} from '../error-overlay/remotion-overlay/StackFrame';import {StudioServerConnectionCtx} from '../helpers/client-id';-import {- BLACK_ALPHA_30,- BLUE,- BORDER_WHITE_ALPHA_12,- LIGHT_TEXT,- WHITE,-} from '../helpers/colors';-import {copyText} from '../helpers/copy-text';+import {LIGHT_TEXT} from '../helpers/colors';import {findOriginalPositionInFileAtProperty} from '../helpers/open-in-editor';-import {useCopyFeedback} from '../helpers/use-copy-feedback';-import {CopyIcon} from '../icons/copy';-import {SkillsIcon} from '../icons/skills';import type {ModalState} from '../state/modals';-import {CodingAgentButton} from './CodingAgentButton';-import type {RenderInlineAction} from './InlineAction';-import {InlineAction} from './InlineAction';+import {AgentPrompt} from './AgentPrompt';import {getMaxModalHeight, getMaxModalWidth} from './ModalContainer';import {ModalHeader} from './ModalHeader';import {DismissableModal} from './NewComposition/DismissableModal';-import {ValidationMessage} from './NewComposition/ValidationMessage';-import {showNotification} from './Notifications/NotificationCenter';-import {useSettings} from './SettingsContext';-import {SkillSettingsRow} from './SkillsSettings';import {useEditorOpening} from './use-default-editor-info';const panelStyle: React.CSSProperties = {@@ -52,55 +36,6 @@ const text: React.CSSProperties = {lineHeight: 1.5,};-const commandField: React.CSSProperties = {- alignItems: 'flex-start',- background: BLACK_ALPHA_30,- borderRadius: 6,- boxSizing: 'border-box',- display: 'flex',- gap: 8,- marginTop: 10,- padding: '8px 8px 8px 10px',- width: '100%',-};--const code: React.CSSProperties = {- color: WHITE,- flex: 1,- fontFamily: 'monospace',- fontSize: 13,- lineHeight: 1.5,- margin: 0,- minWidth: 0,- overflowWrap: 'anywhere',- whiteSpace: 'pre-wrap',-};--const copyAction: React.CSSProperties = {- flexShrink: 0,-};--const skillsIcon: React.CSSProperties = {- display: 'inline-block',- height: 18,- width: 18,- marginRight: 6,- verticalAlign: 'middle',-};--const copyIcon: React.CSSProperties = {- flexShrink: 0,- height: 12,- width: 12,-};--const agentPromptHeader: React.CSSProperties = {- alignItems: 'center',- display: 'flex',- gap: 12,- justifyContent: 'space-between',-};-const sourcePreviewContainer: React.CSSProperties = {marginBottom: 16,marginTop: 4,@@ -112,15 +47,6 @@ const sourcePreviewStatus: React.CSSProperties = {marginTop: 10,};-const skillRowContainer: React.CSSProperties = {- borderTop: BORDER_WHITE_ALPHA_12,- marginTop: 10,-};--const skillError: React.CSSProperties = {- marginTop: 10,-};-type FixComputedValueModalState = Extract<ModalState,{type: 'fix-computed-value'}@@ -142,41 +68,22 @@ type SourcePreviewState =export const FixComputedValueModal: React.FC<{readonly state: FixComputedValueModalState;}> = ({state}) => {- const {codingAgentInfo, remotionSkillsInfo, skillActionError} = useSettings();const {previewServerState} = useContext(StudioServerConnectionCtx);const {canOpenInEditor, defaultEditorId, defaultEditorName} =useEditorOpening(previewServerState.type === 'connected');const [sourcePreview, setSourcePreview] = useState<SourcePreviewState>({type: 'loading',});- const skillName = '/remotion-interactivity';- const skillId = 'remotion-interactivity';- const remotionInteractivitySkillAvailable =- remotionSkillsInfo?.remotionInteractivitySkillAvailable ?? false;- const interactivitySkill =- remotionSkillsInfo?.skills.find((skill) => skill.name === skillId) ?? null;- const canInstallSkill =- !window.remotion_isReadOnlyStudio &&- previewServerState.type === 'connected' &&- remotionSkillsInfo !== null;const promptLocation =sourcePreview.type === 'loaded'? `${sourcePreview.stack.originalFileName}:${sourcePreview.stack.originalLineNumber}:${sourcePreview.stack.originalColumnNumber}`: `${state.location.source}:${state.location.line}:${state.location.column}`;const promptDetails = ` ${promptLocation} make "${state.prop}" interactive`;- const prompt = `${skillName}${promptDetails}`;- const installCommand = 'npx remotion skills add';- const hasCodingAgent =- (codingAgentInfo?.installedCodingAgents.length ?? 0) > 0;const canOpenSourceInEditor =sourcePreview.type === 'loaded' &&canOpenInEditor &&defaultEditorId !== null &&defaultEditorName !== null;- const {copied: promptCopied, markCopied: markPromptCopied} =- useCopyFeedback();- const {copied: installCommandCopied, markCopied: markInstallCommandCopied} =- useCopyFeedback();useEffect(() => {let cancelled = false;@@ -216,41 +123,6 @@ export const FixComputedValueModal: React.FC<{};}, [state.location, state.prop]);- const onCopyPrompt = useCallback(() => {- copyText(prompt)- .then(markPromptCopied)- .catch((err) => {- showNotification(`Could not copy: ${err.message}`, 2000);- });- }, [markPromptCopied, prompt]);-- const onCopyInstallCommand = useCallback(() => {- copyText(installCommand)- .then(markInstallCommandCopied)- .catch((err) => {- showNotification(`Could not copy: ${err.message}`, 2000);- });- }, [installCommand, markInstallCommandCopied]);-- const renderPromptCopyAction: RenderInlineAction = useCallback(- (color) => {- return <CopyIcon copied={promptCopied} color={color} style={copyIcon} />;- },- [promptCopied],- );- const renderInstallCommandCopyAction: RenderInlineAction = useCallback(- (color) => {- return (- <CopyIcon- copied={installCommandCopied}- color={color}- style={copyIcon}- />- );- },- [installCommandCopied],- );-return (<DismissableModal panelStyle={panelStyle}><ModalHeader title="Fix computed value" />@@ -298,97 +170,11 @@ export const FixComputedValueModal: React.FC<{: `Could not load code preview: ${sourcePreview.message}`}</div>)}- {remotionInteractivitySkillAvailable ? null : (- <div style={text}>You can ask an agent to restructure the code.</div>- )}- {remotionInteractivitySkillAvailable ? null : (- <>- <div style={text}>Install the Agent Skill first:</div>- {canInstallSkill && interactivitySkill ? (- <div- aria-label="Remotion Agent Skills"- role="list"- style={skillRowContainer}- >- <SkillSettingsRow isLast={false} skill={interactivitySkill} />- </div>- ) : (- <div style={commandField}>- <pre style={code}>{installCommand}</pre>- <InlineAction- style={copyAction}- variant={null}- onClick={onCopyInstallCommand}- renderAction={renderInstallCommandCopyAction}- title="Copy command"- />- </div>- )}- {skillActionError ? (- <div style={skillError}>- <ValidationMessage- align="flex-start"- message={skillActionError}- type="error"- />- </div>- ) : null}- </>- )}- <div- style={{- ...agentPromptHeader,- marginTop: remotionInteractivitySkillAvailable ? 0 : 16,- }}- >- <div>- {remotionInteractivitySkillAvailable ? (- <>- <div style={text}>- You can ask an agent to restructure the code.- </div>- <div style={text}>- Use this prompt to make this value editable:- </div>- </>- ) : (- <div style={text}>- Then use this prompt to make the value editable:- </div>- )}- </div>- {hasCodingAgent ? (- <CodingAgentButton- label="Open in"- prompt={prompt}- size="compact"- style={null}- />- ) : null}- </div>- <div style={commandField}>- <pre style={code}>- <SkillsIcon color={BLUE} style={skillsIcon} aria-hidden />- <span- style={{- color: BLUE,- fontFamily: 'inherit',- fontSize: 'inherit',- lineHeight: 'inherit',- }}- >- {skillName}- </span>- {promptDetails}- </pre>- <InlineAction- style={copyAction}- variant={null}- onClick={onCopyPrompt}- renderAction={renderPromptCopyAction}- title="Copy prompt"- />- </div>+ <AgentPrompt+ availableText="Use this prompt to make this value editable:"+ promptDetails={promptDetails}+ skillId="remotion-interactivity"+ /></div></DismissableModal>);diff --git a/packages/studio/src/components/GenerateWithAgentModal.tsx b/packages/studio/src/components/GenerateWithAgentModal.tsxnew file mode 100644index 0000000000..4b3aa5d69a--- /dev/null+++ b/packages/studio/src/components/GenerateWithAgentModal.tsx@@ -0,0 +1,53 @@+import React from 'react';+import {LIGHT_TEXT} from '../helpers/colors';+import {formatFileLocation} from '../helpers/format-file-location';+import type {ModalState} from '../state/modals';+import {AgentPrompt} from './AgentPrompt';+import {getMaxModalHeight, getMaxModalWidth} from './ModalContainer';+import {ModalHeader} from './ModalHeader';+import {DismissableModal} from './NewComposition/DismissableModal';++const panelStyle: React.CSSProperties = {+ borderRadius: 6,+ display: 'flex',+ flexDirection: 'column',+ maxHeight: getMaxModalHeight(800),+ minWidth: 0,+ overflow: 'hidden',+ width: getMaxModalWidth(560),+};+const container: React.CSSProperties = {padding: 16};+const text: React.CSSProperties = {+ color: LIGHT_TEXT,+ fontFamily: 'sans-serif',+ fontSize: 14,+ lineHeight: 1.5,+ marginBottom: 16,+};++type State = Extract<ModalState, {type: 'generate-with-agent'}>;++export const GenerateWithAgentModal: React.FC<{readonly state: State}> = ({+ state,+}) => {+ const location = formatFileLocation({+ location: state.location,+ root: window.remotion_cwd,+ });++ return (+ <DismissableModal panelStyle={panelStyle}>+ <ModalHeader title="Generate with agent" />+ <div style={container}>+ <div style={text}>+ Ask a coding agent to generate content in this composition.+ </div>+ <AgentPrompt+ availableText="Start your prompt with:"+ promptDetails={location ? ` ${location}` : ''}+ skillId="remotion-markup"+ />+ </div>+ </DismissableModal>+ );+};diff --git a/packages/studio/src/components/InspectorPanel/CompositionInspector.tsx b/packages/studio/src/components/InspectorPanel/CompositionInspector.tsxindex 95189a7a22..1bfb1320de 100644--- a/packages/studio/src/components/InspectorPanel/CompositionInspector.tsx+++ b/packages/studio/src/components/InspectorPanel/CompositionInspector.tsx@@ -17,6 +17,7 @@ import {isStudioInteractivityEnabled} from '../../helpers/interactivity-enabled'import {CloudDownloadIcon} from '../../icons/cloud-download';import {PicIcon} from '../../icons/frame';import {SolidIcon} from '../../icons/solid';+import {SparklesIcon} from '../../icons/sparkles';import {FilmIcon} from '../../icons/video';import {VisualControlsContext} from '../../visual-controls/VisualControls';import {useConfirmationDialog} from '../ConfirmationDialog';@@ -94,6 +95,8 @@ const CompositionActions: React.FC = () => {insertAsset,insertComposition,insertSolid,+ generateWithAgent,+ canShowGenerateWithAgent,} = useCompositionActions();const downloadProject = getBrowserStudioOperations()?.downloadProject ?? null;const confirm = useConfirmationDialog();@@ -153,6 +156,7 @@ const CompositionActions: React.FC = () => {!canShowInsertAsset &&!canShowInsertComposition &&!canShowInsertSolid &&+ !canShowGenerateWithAgent &&downloadProject === null) {return null;@@ -198,6 +202,17 @@ const CompositionActions: React.FC = () => {Add composition...</InspectorQuickAction>) : null}+ {canShowGenerateWithAgent ? (+ <InspectorQuickAction+ disabled={false}+ onClick={generateWithAgent}+ renderIcon={(color) => (+ <SparklesIcon color={color} style={actionIconStyle} />+ )}+ >+ Generate with agent...+ </InspectorQuickAction>+ ) : null}{canShowInsertAsset ? <ElementLibraryButton /> : null}{downloadProject ? (<InspectorQuickActiondiff --git a/packages/studio/src/components/InspectorPanel/use-composition-actions.ts b/packages/studio/src/components/InspectorPanel/use-composition-actions.tsindex 8c3c249452..5255113d43 100644--- a/packages/studio/src/components/InspectorPanel/use-composition-actions.ts+++ b/packages/studio/src/components/InspectorPanel/use-composition-actions.ts@@ -77,6 +77,10 @@ export const useCompositionActions = () => {const canInsertAsset = canShowInsertAsset && !isAddingAsset;const canShowInsertComposition = canShowInsertAsset;const canInsertComposition = canShowInsertComposition && !isAddingComposition;+ const canShowGenerateWithAgent =+ previewInteractive &&+ !window.remotion_isReadOnlyStudio &&+ currentCompositionId !== null;const insertSolid = useCallback(async () => {if (@@ -283,6 +287,17 @@ export const useCompositionActions = () => {setSelectedModal,]);+ const generateWithAgent = useCallback(() => {+ if (currentCompositionId === null) {+ return;+ }++ setSelectedModal({+ type: 'generate-with-agent',+ location: compositionComponentInfo?.location ?? null,+ });+ }, [compositionComponentInfo, currentCompositionId, setSelectedModal]);+return {canInsertAsset,canInsertComposition,@@ -290,8 +305,10 @@ export const useCompositionActions = () => {canShowInsertAsset,canShowInsertComposition,canShowInsertSolid,+ canShowGenerateWithAgent,insertAsset,insertComposition,insertSolid,+ generateWithAgent,};};diff --git a/packages/studio/src/components/Modals.tsx b/packages/studio/src/components/Modals.tsxindex cd49948c95..b9918826df 100644--- a/packages/studio/src/components/Modals.tsx+++ b/packages/studio/src/components/Modals.tsx@@ -14,6 +14,7 @@ import {} from './ElementInstallConfirmation';import {ElementLibraryModal} from './ElementLibraryModal';import {FixComputedValueModal} from './FixComputedValueModal';+import {GenerateWithAgentModal} from './GenerateWithAgentModal';import {DeleteComposition} from './NewComposition/DeleteComposition';import {DeleteFolder} from './NewComposition/DeleteFolder';import {DuplicateComposition} from './NewComposition/DuplicateComposition';@@ -268,6 +269,9 @@ export const Modals: React.FC<{{modalContextType && modalContextType.type === 'fix-computed-value' && (<FixComputedValueModal state={modalContextType} />)}+ {modalContextType && modalContextType.type === 'generate-with-agent' ? (+ <GenerateWithAgentModal state={modalContextType} />+ ) : null}{modalContextType && modalContextType.type === 'quick-switcher' && (<QuickSwitcherreadOnlyStudio={readOnlyStudio}diff --git a/packages/studio/src/icons/sparkles.tsx b/packages/studio/src/icons/sparkles.tsxnew file mode 100644index 0000000000..9c41499eab--- /dev/null+++ b/packages/studio/src/icons/sparkles.tsx@@ -0,0 +1,9 @@+import type {SVGProps} from 'react';++export const SparklesIcon: React.FC<+ SVGProps<SVGSVGElement> & {readonly color: string}+> = ({color, ...props}) => (+ <svg {...props} fill={color} viewBox="0 0 640 640">+ <path d="M390.6 268.1 529.2 320 390.6 371.9C381.9 375.1 375.1 382 371.8 390.7L320 529.2 268.1 390.6C264.9 381.9 258 375.1 249.3 371.8L110.8 320 249.4 268.1C258.1 264.9 264.9 258 268.2 249.3L320 110.8 371.9 249.4C375.1 258.1 382 264.9 390.7 268.2ZM401.8 238.1 342.5 79.6C339 70.2 330 64 320 64S301 70.2 297.5 79.6L238.2 238.2 79.6 297.5C70.2 301 64 310 64 320S70.2 339 79.6 342.5L238.2 401.9 297.6 560.5C301 569.8 310 576 320 576S339 569.8 342.5 560.4L401.9 401.8 560.5 342.4C569.8 339 576 330 576 320S569.8 301 560.4 297.5L401.8 238.2Z" />+ </svg>+);diff --git a/packages/studio/src/state/modals.ts b/packages/studio/src/state/modals.tsindex 653c190590..b33f0566ef 100644--- a/packages/studio/src/state/modals.ts+++ b/packages/studio/src/state/modals.ts@@ -266,6 +266,13 @@ export type ModalState =column: number;};}+ | {+ type: 'generate-with-agent';+ location: {+ source: string;+ line: number;+ } | null;+ }| {type: 'quick-switcher';mode: QuickSwitcherMode;