Skip to content

🎥 Make videos programmatically with React

`@remotion/studio`: Add ChatGPT actions for pre-compose and generation

Jonny Burger40b22580135a
diff --git a/packages/studio/src/components/AgentPrompt.tsx b/packages/studio/src/components/AgentPrompt.tsx
index f47d797eb1..d8455f4839 100644
--- a/packages/studio/src/components/AgentPrompt.tsx
+++ b/packages/studio/src/components/AgentPrompt.tsx
@@ -66,10 +66,11 @@ const skillRowContainer: React.CSSProperties = {
const skillError: React.CSSProperties = {marginTop: 10};
export const AgentPrompt: React.FC<{
+ readonly action: React.ReactNode;
readonly availableText: string;
readonly promptDetails: string;
readonly skillId: string;
-}> = ({availableText, promptDetails, skillId}) => {
+}> = ({action, availableText, promptDetails, skillId}) => {
const {codingAgentInfo, remotionSkillsInfo, skillActionError} = useSettings();
const {previewServerState} = useContext(StudioServerConnectionCtx);
const skill = remotionSkillsInfo?.skills.find(({name}) => name === skillId);
@@ -150,14 +151,15 @@ export const AgentPrompt: React.FC<{
? availableText
: `Then ${availableText.toLowerCase()}`}
</div>
- {hasCodingAgent ? (
- <CodingAgentButton
- label="Open in"
- prompt={prompt}
- size="compact"
- style={null}
- />
- ) : null}
+ {action ??
+ (hasCodingAgent ? (
+ <CodingAgentButton
+ label="Open in"
+ prompt={prompt}
+ size="compact"
+ style={null}
+ />
+ ) : null)}
</div>
<div style={commandField}>
<pre style={code}>
diff --git a/packages/studio/src/components/FixComputedValueModal.tsx b/packages/studio/src/components/FixComputedValueModal.tsx
index ac3ab92968..f3e49e4820 100644
--- a/packages/studio/src/components/FixComputedValueModal.tsx
+++ b/packages/studio/src/components/FixComputedValueModal.tsx
@@ -171,6 +171,7 @@ export const FixComputedValueModal: React.FC<{
</div>
)}
<AgentPrompt
+ action={null}
availableText="Use this prompt to make this value editable:"
promptDetails={promptDetails}
skillId="remotion-interactivity"
diff --git a/packages/studio/src/components/GenerateWithAgentModal.tsx b/packages/studio/src/components/GenerateWithAgentModal.tsx
index 4b3aa5d69a..a0a3cfef7c 100644
--- a/packages/studio/src/components/GenerateWithAgentModal.tsx
+++ b/packages/studio/src/components/GenerateWithAgentModal.tsx
@@ -43,6 +43,7 @@ export const GenerateWithAgentModal: React.FC<{readonly state: State}> = ({
Ask a coding agent to generate content in this composition.
</div>
<AgentPrompt
+ action={null}
availableText="Start your prompt with:"
promptDetails={location ? ` ${location}` : ''}
skillId="remotion-markup"
diff --git a/packages/studio/src/components/InspectorPanel/use-composition-actions.ts b/packages/studio/src/components/InspectorPanel/use-composition-actions.ts
index 8dd8d28eca..372c3db4c3 100644
--- a/packages/studio/src/components/InspectorPanel/use-composition-actions.ts
+++ b/packages/studio/src/components/InspectorPanel/use-composition-actions.ts
@@ -1,10 +1,19 @@
import type {InsertCompositionElementRequest} from '@remotion/studio-shared';
-import {useCallback, useContext, useMemo, useState} from 'react';
+import {
+ useCallback,
+ useContext,
+ useMemo,
+ useState,
+ type MouseEvent,
+} from 'react';
import {Internals, type _InternalTypes} from 'remotion';
import {getBrowserStudioOperations} from '../../helpers/browser-studio-operations';
import {StudioServerConnectionCtx} from '../../helpers/client-id';
+import {getRelativeFileLocation} from '../../helpers/format-file-location';
+import {getCodexAnnotation} from '../../helpers/get-codex-annotation';
import {isStudioInteractivityEnabled} from '../../helpers/interactivity-enabled';
import {useCachedCompositionComponentInfo} from '../../helpers/open-in-editor';
+import {requestCodexAnnotation} from '../../helpers/request-codex-annotation';
import {SetSelectedModalContext} from '../../state/modals';
import {callApi} from '../call-api';
import type {CompositionDragData} from '../composition-drag-data';
@@ -15,6 +24,7 @@ import {
pickFilesToImport,
} from '../import-assets';
import {showNotification} from '../Notifications/NotificationCenter';
+import {useSettings} from '../SettingsContext';
import {getOriginalLocationFromStack} from '../Timeline/TimelineStack/get-stack';
import {useResolvedStack} from '../Timeline/use-resolved-stack';
@@ -28,6 +38,13 @@ export const useCompositionActions = () => {
const [isAddingComposition, setIsAddingComposition] = useState(false);
const {setSelectedModal} = useContext(SetSelectedModalContext);
const {previewServerState} = useContext(StudioServerConnectionCtx);
+ const {remotionSkillsInfo} = useSettings();
+ const markupSkill = remotionSkillsInfo?.skills.find(
+ ({name}) => name === 'remotion-markup',
+ );
+ const markupSkillAvailable = Boolean(
+ markupSkill?.installedInProject || markupSkill?.installedGlobally,
+ );
const previewConnected = previewServerState.type === 'connected';
const previewInteractive = previewConnected && isStudioInteractivityEnabled();
const browserStudioOperations = getBrowserStudioOperations();
@@ -78,9 +95,9 @@ export const useCompositionActions = () => {
const canShowInsertComposition = canShowInsertAsset && videoConfig !== null;
const canInsertComposition = canShowInsertComposition && !isAddingComposition;
const canShowGenerateWithAgent =
- previewInteractive &&
!window.remotion_isReadOnlyStudio &&
- browserStudioOperations === null &&
+ ((previewInteractive && browserStudioOperations === null) ||
+ getCodexAnnotation() !== null) &&
currentCompositionId !== null;
const insertSolid = useCallback(async () => {
@@ -293,16 +310,60 @@ export const useCompositionActions = () => {
setSelectedModal,
]);
- const generateWithAgent = useCallback(() => {
- if (currentCompositionId === null) {
- return;
- }
+ const generateWithAgent = useCallback(
+ (event: MouseEvent<HTMLButtonElement>) => {
+ if (!canShowGenerateWithAgent || currentCompositionId === null) {
+ return;
+ }
- setSelectedModal({
- type: 'generate-with-agent',
- location: compositionComponentInfo?.location ?? null,
- });
- }, [compositionComponentInfo, currentCompositionId, setSelectedModal]);
+ const location = compositionComponentInfo?.location ?? null;
+ if (markupSkillAvailable && getCodexAnnotation() !== null) {
+ const sourceLocation = getRelativeFileLocation({
+ location,
+ root: window.remotion_cwd,
+ });
+ const metadata: Record<string, string | number> = {};
+ if (currentCompositionId.length <= 256) {
+ metadata.composition = currentCompositionId;
+ }
+
+ if (sourceLocation && sourceLocation.filename.length <= 256) {
+ const withSource = {
+ ...metadata,
+ source: sourceLocation.filename,
+ line: sourceLocation.line,
+ };
+ if (
+ new TextEncoder().encode(JSON.stringify(withSource)).length <= 2048
+ ) {
+ metadata.source = sourceLocation.filename;
+ metadata.line = sourceLocation.line;
+ }
+ }
+
+ const accepted = requestCodexAnnotation({
+ target: event.currentTarget,
+ initialComment: '$remotion-markup',
+ metadata,
+ });
+ if (accepted) {
+ return;
+ }
+ }
+
+ setSelectedModal({
+ type: 'generate-with-agent',
+ location,
+ });
+ },
+ [
+ canShowGenerateWithAgent,
+ compositionComponentInfo,
+ currentCompositionId,
+ markupSkillAvailable,
+ setSelectedModal,
+ ],
+ );
return {
canInsertAsset,
diff --git a/packages/studio/src/components/PrecomposeRefactorModal.tsx b/packages/studio/src/components/PrecomposeRefactorModal.tsx
index db685b114e..a75b044e33 100644
--- a/packages/studio/src/components/PrecomposeRefactorModal.tsx
+++ b/packages/studio/src/components/PrecomposeRefactorModal.tsx
@@ -1,10 +1,13 @@
-import React, {useContext} from 'react';
+import React, {useCallback, useContext, useRef} from 'react';
import {getBrowserStudioOperations} from '../helpers/browser-studio-operations';
import {StudioServerConnectionCtx} from '../helpers/client-id';
import {LIGHT_TEXT} from '../helpers/colors';
import {formatFileLocation} from '../helpers/format-file-location';
+import {getCodexAnnotation} from '../helpers/get-codex-annotation';
+import {requestCodexAnnotation} from '../helpers/request-codex-annotation';
import type {ModalState} from '../state/modals';
import {AgentPrompt} from './AgentPrompt';
+import {ModalButton} from './ModalButton';
import {getMaxModalHeight, getMaxModalWidth} from './ModalContainer';
import {ModalHeader} from './ModalHeader';
import {DismissableModal} from './NewComposition/DismissableModal';
@@ -41,6 +44,17 @@ export const PrecomposeRefactorModal: React.FC<{readonly state: State}> = ({
}) => {
const {previewServerState} = useContext(StudioServerConnectionCtx);
const {error, remotionSkillsInfo} = useSettings();
+ const annotationTarget = useRef<HTMLDivElement>(null);
+ const markupSkill = remotionSkillsInfo?.skills.find(
+ ({name}) => name === 'remotion-markup',
+ );
+ const markupSkillAvailable = Boolean(
+ markupSkill?.installedInProject || markupSkill?.installedGlobally,
+ );
+ const canAnnotate =
+ markupSkillAvailable &&
+ !window.remotion_isReadOnlyStudio &&
+ getCodexAnnotation() !== null;
const isBrowserStudio = getBrowserStudioOperations() !== null;
const canSuggestAgent =
!isBrowserStudio &&
@@ -63,17 +77,36 @@ export const PrecomposeRefactorModal: React.FC<{readonly state: State}> = ({
targets.length === 1
? ` Pre-compose ${targets[0]}`
: ` Pre-compose these sequences: ${targets.join('; ')}`;
+ const selection = targets.join('; ');
+ const onSendToChatGPT = useCallback(() => {
+ requestCodexAnnotation({
+ target: annotationTarget.current,
+ initialComment: '$remotion-markup',
+ metadata: {
+ action: 'Pre-compose',
+ sequenceCount: targets.length,
+ ...(selection.length <= 256 ? {selection} : {}),
+ },
+ });
+ }, [selection, targets.length]);
return (
<DismissableModal panelStyle={panelStyle}>
<ModalHeader title="Pre-compose" />
- <div style={container}>
+ <div ref={annotationTarget} style={container}>
<div style={text}>
Move these items into their own composition and re-import them into
this composition.
</div>
- {canSuggestAgent ? (
+ {canSuggestAgent || canAnnotate ? (
<AgentPrompt
+ action={
+ canAnnotate ? (
+ <ModalButton size="compact" onClick={onSendToChatGPT}>
+ Send to ChatGPT
+ </ModalButton>
+ ) : null
+ }
availableText="You can pre-compose using an agent:"
promptDetails={promptDetails}
skillId="remotion-markup"
diff --git a/packages/studio/src/components/WrapRefactorModal.tsx b/packages/studio/src/components/WrapRefactorModal.tsx
index 10801c057e..794544e353 100644
--- a/packages/studio/src/components/WrapRefactorModal.tsx
+++ b/packages/studio/src/components/WrapRefactorModal.tsx
@@ -70,6 +70,7 @@ export const WrapRefactorModal: React.FC<{readonly state: State}> = ({
</div>
{canSuggestAgent ? (
<AgentPrompt
+ action={null}
availableText="You can wrap it using an agent:"
promptDetails={` Wrap ${target} in <${state.wrapper}> from '${wrapperImportPath}'`}
skillId="remotion-markup"
diff --git a/packages/studio/src/helpers/get-codex-annotation.ts b/packages/studio/src/helpers/get-codex-annotation.ts
index 649c4bfd2b..13b12b20bc 100644
--- a/packages/studio/src/helpers/get-codex-annotation.ts
+++ b/packages/studio/src/helpers/get-codex-annotation.ts
@@ -1,9 +1,9 @@
type CodexAnnotation = {
readonly request: (
target: Element,
- options: {
- metadata: Record<string, string | number>;
- },
+ options:
+ | {metadata: Record<string, string | number>}
+ | {initialComment: string},
) => {accepted: boolean};
};
diff --git a/packages/studio/src/helpers/request-codex-annotation.ts b/packages/studio/src/helpers/request-codex-annotation.ts
new file mode 100644
index 0000000000..49b509d41e
--- /dev/null
+++ b/packages/studio/src/helpers/request-codex-annotation.ts
@@ -0,0 +1,37 @@
+import {showNotification} from '../components/Notifications/NotificationCenter';
+import {getCodexAnnotation} from './get-codex-annotation';
+
+export const requestCodexAnnotation = ({
+ target,
+ initialComment,
+ metadata,
+}: {
+ readonly target: Element | null;
+ readonly initialComment: string;
+ readonly metadata: Record<string, string | number> | null;
+}): boolean => {
+ const annotation = getCodexAnnotation();
+ if (annotation === null || target === null) {
+ showNotification('The annotation editor is unavailable', 3000);
+ return false;
+ }
+
+ try {
+ const result = annotation.request(target, {
+ initialComment,
+ ...(metadata === null ? {} : {metadata}),
+ });
+ if (result?.accepted) {
+ return true;
+ }
+
+ showNotification('Could not open the annotation editor', 3000);
+ } catch (err) {
+ showNotification(
+ `Could not open the annotation editor: ${(err as Error).message}`,
+ 3000,
+ );
+ }
+
+ return false;
+};