🎥 Make videos programmatically with React
`@remotion/studio`: Add ChatGPT actions for pre-compose and generation
Jonny Burger40b22580135a
diff --git a/packages/studio/src/components/AgentPrompt.tsx b/packages/studio/src/components/AgentPrompt.tsxindex f47d797eb1..d8455f4839 100644--- a/packages/studio/src/components/AgentPrompt.tsx+++ b/packages/studio/src/components/AgentPrompt.tsx@@ -66,10 +66,11 @@ const skillRowContainer: React.CSSProperties = {const skillError: React.CSSProperties = {marginTop: 10};export const AgentPrompt: React.FC<{+ readonly action: React.ReactNode;readonly availableText: string;readonly promptDetails: string;readonly skillId: string;-}> = ({availableText, promptDetails, skillId}) => {+}> = ({action, availableText, promptDetails, skillId}) => {const {codingAgentInfo, remotionSkillsInfo, skillActionError} = useSettings();const {previewServerState} = useContext(StudioServerConnectionCtx);const skill = remotionSkillsInfo?.skills.find(({name}) => name === skillId);@@ -150,14 +151,15 @@ export const AgentPrompt: React.FC<{? availableText: `Then ${availableText.toLowerCase()}`}</div>- {hasCodingAgent ? (- <CodingAgentButton- label="Open in"- prompt={prompt}- size="compact"- style={null}- />- ) : null}+ {action ??+ (hasCodingAgent ? (+ <CodingAgentButton+ label="Open in"+ prompt={prompt}+ size="compact"+ style={null}+ />+ ) : null)}</div><div style={commandField}><pre style={code}>diff --git a/packages/studio/src/components/FixComputedValueModal.tsx b/packages/studio/src/components/FixComputedValueModal.tsxindex ac3ab92968..f3e49e4820 100644--- a/packages/studio/src/components/FixComputedValueModal.tsx+++ b/packages/studio/src/components/FixComputedValueModal.tsx@@ -171,6 +171,7 @@ export const FixComputedValueModal: React.FC<{</div>)}<AgentPrompt+ action={null}availableText="Use this prompt to make this value editable:"promptDetails={promptDetails}skillId="remotion-interactivity"diff --git a/packages/studio/src/components/GenerateWithAgentModal.tsx b/packages/studio/src/components/GenerateWithAgentModal.tsxindex 4b3aa5d69a..a0a3cfef7c 100644--- a/packages/studio/src/components/GenerateWithAgentModal.tsx+++ b/packages/studio/src/components/GenerateWithAgentModal.tsx@@ -43,6 +43,7 @@ export const GenerateWithAgentModal: React.FC<{readonly state: State}> = ({Ask a coding agent to generate content in this composition.</div><AgentPrompt+ action={null}availableText="Start your prompt with:"promptDetails={location ? ` ${location}` : ''}skillId="remotion-markup"diff --git a/packages/studio/src/components/InspectorPanel/use-composition-actions.ts b/packages/studio/src/components/InspectorPanel/use-composition-actions.tsindex 8dd8d28eca..372c3db4c3 100644--- a/packages/studio/src/components/InspectorPanel/use-composition-actions.ts+++ b/packages/studio/src/components/InspectorPanel/use-composition-actions.ts@@ -1,10 +1,19 @@import type {InsertCompositionElementRequest} from '@remotion/studio-shared';-import {useCallback, useContext, useMemo, useState} from 'react';+import {+ useCallback,+ useContext,+ useMemo,+ useState,+ type MouseEvent,+} from 'react';import {Internals, type _InternalTypes} from 'remotion';import {getBrowserStudioOperations} from '../../helpers/browser-studio-operations';import {StudioServerConnectionCtx} from '../../helpers/client-id';+import {getRelativeFileLocation} from '../../helpers/format-file-location';+import {getCodexAnnotation} from '../../helpers/get-codex-annotation';import {isStudioInteractivityEnabled} from '../../helpers/interactivity-enabled';import {useCachedCompositionComponentInfo} from '../../helpers/open-in-editor';+import {requestCodexAnnotation} from '../../helpers/request-codex-annotation';import {SetSelectedModalContext} from '../../state/modals';import {callApi} from '../call-api';import type {CompositionDragData} from '../composition-drag-data';@@ -15,6 +24,7 @@ import {pickFilesToImport,} from '../import-assets';import {showNotification} from '../Notifications/NotificationCenter';+import {useSettings} from '../SettingsContext';import {getOriginalLocationFromStack} from '../Timeline/TimelineStack/get-stack';import {useResolvedStack} from '../Timeline/use-resolved-stack';@@ -28,6 +38,13 @@ export const useCompositionActions = () => {const [isAddingComposition, setIsAddingComposition] = useState(false);const {setSelectedModal} = useContext(SetSelectedModalContext);const {previewServerState} = useContext(StudioServerConnectionCtx);+ const {remotionSkillsInfo} = useSettings();+ const markupSkill = remotionSkillsInfo?.skills.find(+ ({name}) => name === 'remotion-markup',+ );+ const markupSkillAvailable = Boolean(+ markupSkill?.installedInProject || markupSkill?.installedGlobally,+ );const previewConnected = previewServerState.type === 'connected';const previewInteractive = previewConnected && isStudioInteractivityEnabled();const browserStudioOperations = getBrowserStudioOperations();@@ -78,9 +95,9 @@ export const useCompositionActions = () => {const canShowInsertComposition = canShowInsertAsset && videoConfig !== null;const canInsertComposition = canShowInsertComposition && !isAddingComposition;const canShowGenerateWithAgent =- previewInteractive &&!window.remotion_isReadOnlyStudio &&- browserStudioOperations === null &&+ ((previewInteractive && browserStudioOperations === null) ||+ getCodexAnnotation() !== null) &¤tCompositionId !== null;const insertSolid = useCallback(async () => {@@ -293,16 +310,60 @@ export const useCompositionActions = () => {setSelectedModal,]);- const generateWithAgent = useCallback(() => {- if (currentCompositionId === null) {- return;- }+ const generateWithAgent = useCallback(+ (event: MouseEvent<HTMLButtonElement>) => {+ if (!canShowGenerateWithAgent || currentCompositionId === null) {+ return;+ }- setSelectedModal({- type: 'generate-with-agent',- location: compositionComponentInfo?.location ?? null,- });- }, [compositionComponentInfo, currentCompositionId, setSelectedModal]);+ const location = compositionComponentInfo?.location ?? null;+ if (markupSkillAvailable && getCodexAnnotation() !== null) {+ const sourceLocation = getRelativeFileLocation({+ location,+ root: window.remotion_cwd,+ });+ const metadata: Record<string, string | number> = {};+ if (currentCompositionId.length <= 256) {+ metadata.composition = currentCompositionId;+ }++ if (sourceLocation && sourceLocation.filename.length <= 256) {+ const withSource = {+ ...metadata,+ source: sourceLocation.filename,+ line: sourceLocation.line,+ };+ if (+ new TextEncoder().encode(JSON.stringify(withSource)).length <= 2048+ ) {+ metadata.source = sourceLocation.filename;+ metadata.line = sourceLocation.line;+ }+ }++ const accepted = requestCodexAnnotation({+ target: event.currentTarget,+ initialComment: '$remotion-markup',+ metadata,+ });+ if (accepted) {+ return;+ }+ }++ setSelectedModal({+ type: 'generate-with-agent',+ location,+ });+ },+ [+ canShowGenerateWithAgent,+ compositionComponentInfo,+ currentCompositionId,+ markupSkillAvailable,+ setSelectedModal,+ ],+ );return {canInsertAsset,diff --git a/packages/studio/src/components/PrecomposeRefactorModal.tsx b/packages/studio/src/components/PrecomposeRefactorModal.tsxindex db685b114e..a75b044e33 100644--- a/packages/studio/src/components/PrecomposeRefactorModal.tsx+++ b/packages/studio/src/components/PrecomposeRefactorModal.tsx@@ -1,10 +1,13 @@-import React, {useContext} from 'react';+import React, {useCallback, useContext, useRef} from 'react';import {getBrowserStudioOperations} from '../helpers/browser-studio-operations';import {StudioServerConnectionCtx} from '../helpers/client-id';import {LIGHT_TEXT} from '../helpers/colors';import {formatFileLocation} from '../helpers/format-file-location';+import {getCodexAnnotation} from '../helpers/get-codex-annotation';+import {requestCodexAnnotation} from '../helpers/request-codex-annotation';import type {ModalState} from '../state/modals';import {AgentPrompt} from './AgentPrompt';+import {ModalButton} from './ModalButton';import {getMaxModalHeight, getMaxModalWidth} from './ModalContainer';import {ModalHeader} from './ModalHeader';import {DismissableModal} from './NewComposition/DismissableModal';@@ -41,6 +44,17 @@ export const PrecomposeRefactorModal: React.FC<{readonly state: State}> = ({}) => {const {previewServerState} = useContext(StudioServerConnectionCtx);const {error, remotionSkillsInfo} = useSettings();+ const annotationTarget = useRef<HTMLDivElement>(null);+ const markupSkill = remotionSkillsInfo?.skills.find(+ ({name}) => name === 'remotion-markup',+ );+ const markupSkillAvailable = Boolean(+ markupSkill?.installedInProject || markupSkill?.installedGlobally,+ );+ const canAnnotate =+ markupSkillAvailable &&+ !window.remotion_isReadOnlyStudio &&+ getCodexAnnotation() !== null;const isBrowserStudio = getBrowserStudioOperations() !== null;const canSuggestAgent =!isBrowserStudio &&@@ -63,17 +77,36 @@ export const PrecomposeRefactorModal: React.FC<{readonly state: State}> = ({targets.length === 1? ` Pre-compose ${targets[0]}`: ` Pre-compose these sequences: ${targets.join('; ')}`;+ const selection = targets.join('; ');+ const onSendToChatGPT = useCallback(() => {+ requestCodexAnnotation({+ target: annotationTarget.current,+ initialComment: '$remotion-markup',+ metadata: {+ action: 'Pre-compose',+ sequenceCount: targets.length,+ ...(selection.length <= 256 ? {selection} : {}),+ },+ });+ }, [selection, targets.length]);return (<DismissableModal panelStyle={panelStyle}><ModalHeader title="Pre-compose" />- <div style={container}>+ <div ref={annotationTarget} style={container}><div style={text}>Move these items into their own composition and re-import them intothis composition.</div>- {canSuggestAgent ? (+ {canSuggestAgent || canAnnotate ? (<AgentPrompt+ action={+ canAnnotate ? (+ <ModalButton size="compact" onClick={onSendToChatGPT}>+ Send to ChatGPT+ </ModalButton>+ ) : null+ }availableText="You can pre-compose using an agent:"promptDetails={promptDetails}skillId="remotion-markup"diff --git a/packages/studio/src/components/WrapRefactorModal.tsx b/packages/studio/src/components/WrapRefactorModal.tsxindex 10801c057e..794544e353 100644--- a/packages/studio/src/components/WrapRefactorModal.tsx+++ b/packages/studio/src/components/WrapRefactorModal.tsx@@ -70,6 +70,7 @@ export const WrapRefactorModal: React.FC<{readonly state: State}> = ({</div>{canSuggestAgent ? (<AgentPrompt+ action={null}availableText="You can wrap it using an agent:"promptDetails={` Wrap ${target} in <${state.wrapper}> from '${wrapperImportPath}'`}skillId="remotion-markup"diff --git a/packages/studio/src/helpers/get-codex-annotation.ts b/packages/studio/src/helpers/get-codex-annotation.tsindex 649c4bfd2b..13b12b20bc 100644--- a/packages/studio/src/helpers/get-codex-annotation.ts+++ b/packages/studio/src/helpers/get-codex-annotation.ts@@ -1,9 +1,9 @@type CodexAnnotation = {readonly request: (target: Element,- options: {- metadata: Record<string, string | number>;- },+ options:+ | {metadata: Record<string, string | number>}+ | {initialComment: string},) => {accepted: boolean};};diff --git a/packages/studio/src/helpers/request-codex-annotation.ts b/packages/studio/src/helpers/request-codex-annotation.tsnew file mode 100644index 0000000000..49b509d41e--- /dev/null+++ b/packages/studio/src/helpers/request-codex-annotation.ts@@ -0,0 +1,37 @@+import {showNotification} from '../components/Notifications/NotificationCenter';+import {getCodexAnnotation} from './get-codex-annotation';++export const requestCodexAnnotation = ({+ target,+ initialComment,+ metadata,+}: {+ readonly target: Element | null;+ readonly initialComment: string;+ readonly metadata: Record<string, string | number> | null;+}): boolean => {+ const annotation = getCodexAnnotation();+ if (annotation === null || target === null) {+ showNotification('The annotation editor is unavailable', 3000);+ return false;+ }++ try {+ const result = annotation.request(target, {+ initialComment,+ ...(metadata === null ? {} : {metadata}),+ });+ if (result?.accepted) {+ return true;+ }++ showNotification('Could not open the annotation editor', 3000);+ } catch (err) {+ showNotification(+ `Could not open the annotation editor: ${(err as Error).message}`,+ 3000,+ );+ }++ return false;+};