Skip to content

🎥 Make videos programmatically with React

`@remotion/studio`: Add ChatGPT annotations to source and repair actions

Jonny Burgera4c470df028a
diff --git a/packages/studio/src/components/CanvasTabs.tsx b/packages/studio/src/components/CanvasTabs.tsx
index 75089c01b7..ee35ea91b3 100644
--- a/packages/studio/src/components/CanvasTabs.tsx
+++ b/packages/studio/src/components/CanvasTabs.tsx
@@ -428,6 +428,9 @@ const CanvasTab: React.FC<{
triggerStyle={tooltipTriggerStyle}
>
<span
+ data-compname={
+ tab.type === 'composition' ? tab.compositionId : undefined
+ }
style={{...labelStyle, color: selected ? WHITE : LIGHT_TEXT}}
onPointerEnter={() => {
setTooltipHovered(true);
diff --git a/packages/studio/src/components/FixComputedValueModal.tsx b/packages/studio/src/components/FixComputedValueModal.tsx
index f3e49e4820..6910bbbee7 100644
--- a/packages/studio/src/components/FixComputedValueModal.tsx
+++ b/packages/studio/src/components/FixComputedValueModal.tsx
@@ -1,16 +1,26 @@
import type {SymbolicatedStackFrame} from '@remotion/studio-shared';
-import React, {useContext, useEffect, useState} from 'react';
+import React, {
+ useCallback,
+ useContext,
+ useEffect,
+ useRef,
+ useState,
+} from 'react';
import {resolveFileSource} from '../error-overlay/react-overlay/effects/resolve-file-source';
import {OpenInEditor} from '../error-overlay/remotion-overlay/OpenInEditor';
import {StackElement} from '../error-overlay/remotion-overlay/StackFrame';
import {StudioServerConnectionCtx} from '../helpers/client-id';
import {LIGHT_TEXT} from '../helpers/colors';
+import {getCodexAnnotation} from '../helpers/get-codex-annotation';
import {findOriginalPositionInFileAtProperty} from '../helpers/open-in-editor';
+import {requestCodexAnnotation} from '../helpers/request-codex-annotation';
import type {ModalState} from '../state/modals';
import {AgentPrompt} from './AgentPrompt';
+import {ModalButton} from './ModalButton';
import {getMaxModalHeight, getMaxModalWidth} from './ModalContainer';
import {ModalHeader} from './ModalHeader';
import {DismissableModal} from './NewComposition/DismissableModal';
+import {useSettings} from './SettingsContext';
import {useEditorOpening} from './use-default-editor-info';
const panelStyle: React.CSSProperties = {
@@ -69,6 +79,15 @@ export const FixComputedValueModal: React.FC<{
readonly state: FixComputedValueModalState;
}> = ({state}) => {
const {previewServerState} = useContext(StudioServerConnectionCtx);
+ const {remotionSkillsInfo} = useSettings();
+ const annotationTarget = useRef<HTMLDivElement>(null);
+ const interactivitySkill = remotionSkillsInfo?.skills.find(
+ ({name}) => name === 'remotion-interactivity',
+ );
+ const interactivitySkillAvailable = Boolean(
+ interactivitySkill?.installedInProject ||
+ interactivitySkill?.installedGlobally,
+ );
const {canOpenInEditor, defaultEditorId, defaultEditorName} =
useEditorOpening(previewServerState.type === 'connected');
const [sourcePreview, setSourcePreview] = useState<SourcePreviewState>({
@@ -79,6 +98,19 @@ export const FixComputedValueModal: React.FC<{
? `${sourcePreview.stack.originalFileName}:${sourcePreview.stack.originalLineNumber}:${sourcePreview.stack.originalColumnNumber}`
: `${state.location.source}:${state.location.line}:${state.location.column}`;
const promptDetails = ` ${promptLocation} make "${state.prop}" interactive`;
+ const canAnnotate =
+ sourcePreview.type === 'loaded' &&
+ !window.remotion_isReadOnlyStudio &&
+ getCodexAnnotation() !== null;
+ const onSendToChatGPT = useCallback(() => {
+ requestCodexAnnotation({
+ target: annotationTarget.current,
+ initialComment: interactivitySkillAvailable
+ ? '$remotion-interactivity Make this computed value interactive.'
+ : 'Make this computed value interactive.',
+ metadata: {property: state.prop, source: promptLocation},
+ });
+ }, [interactivitySkillAvailable, promptLocation, state.prop]);
const canOpenSourceInEditor =
sourcePreview.type === 'loaded' &&
canOpenInEditor &&
@@ -131,7 +163,7 @@ export const FixComputedValueModal: React.FC<{
This value cannot be interactively edited because it is computed:
</div>
{sourcePreview.type === 'loaded' ? (
- <div style={sourcePreviewContainer}>
+ <div ref={annotationTarget} style={sourcePreviewContainer}>
<StackElement
collapsible={false}
defaultFunctionName={null}
@@ -171,7 +203,13 @@ export const FixComputedValueModal: React.FC<{
</div>
)}
<AgentPrompt
- action={null}
+ action={
+ canAnnotate ? (
+ <ModalButton size="compact" onClick={onSendToChatGPT}>
+ Send to ChatGPT
+ </ModalButton>
+ ) : null
+ }
availableText="Use this prompt to make this value editable:"
promptDetails={promptDetails}
skillId="remotion-interactivity"
diff --git a/packages/studio/src/components/InspectorLocationCopy.tsx b/packages/studio/src/components/InspectorLocationCopy.tsx
index 8e9d478e0a..eab5dcfdc1 100644
--- a/packages/studio/src/components/InspectorLocationCopy.tsx
+++ b/packages/studio/src/components/InspectorLocationCopy.tsx
@@ -106,6 +106,7 @@ export const InspectorLocationCopy: React.FC<{
}}
>
<InspectorOpenInEditor
+ annotationName={name}
locationType={null}
contextForAgents={contextForAgents}
location={openInEditorLocation}
diff --git a/packages/studio/src/components/InspectorOpenInEditor.tsx b/packages/studio/src/components/InspectorOpenInEditor.tsx
index e6c07b1981..1a3b9f69e1 100644
--- a/packages/studio/src/components/InspectorOpenInEditor.tsx
+++ b/packages/studio/src/components/InspectorOpenInEditor.tsx
@@ -1,11 +1,12 @@
import type {DefaultCodingAgent} from '@remotion/renderer';
import type {EditorPickerId} from '@remotion/studio-shared';
-import React, {useCallback, useContext, useMemo} from 'react';
+import React, {useCallback, useContext, useMemo, useRef} from 'react';
import type {OriginalPosition} from '../error-overlay/react-overlay/utils/get-source-map';
import {getBrowserStudioOperations} from '../helpers/browser-studio-operations';
import {StudioServerConnectionCtx} from '../helpers/client-id';
import {LIGHT_TEXT} from '../helpers/colors';
import {copyText} from '../helpers/copy-text';
+import {formatFileLocation} from '../helpers/format-file-location';
import {
getDefaultOpenInTarget,
openGitSource,
@@ -19,6 +20,7 @@ import {
import {CaretDown} from '../icons/caret';
import {EditorIcon} from '../icons/editor';
import {GitHubIcon} from '../icons/github';
+import {getAnnotateWithChatGPTMenuItems} from './get-annotate-with-chatgpt-menu-item';
import {getOpenInMenuItems} from './get-open-in-menu-items';
import type {ComboboxValue} from './NewComposition/ComboBox';
import {showNotification} from './Notifications/NotificationCenter';
@@ -46,12 +48,14 @@ const editorButtonIconSize = 18;
const githubButtonIconSize = 16;
export const InspectorOpenInEditor: React.FC<{
+ readonly annotationName: string | null;
readonly contextForAgents?: string | null;
readonly location: OriginalPosition | null;
readonly label?: React.ReactNode;
readonly locationType: 'file' | 'folder' | null;
readonly showTooltips: boolean;
}> = ({
+ annotationName,
contextForAgents = null,
label,
location,
@@ -59,6 +63,7 @@ export const InspectorOpenInEditor: React.FC<{
showTooltips,
}) => {
const {previewServerState} = useContext(StudioServerConnectionCtx);
+ const annotationTarget = useRef<HTMLDivElement>(null);
const configureDefaultApps = useConfigureDefaultApps();
const {
canConfigureApps,
@@ -210,8 +215,33 @@ export const InspectorOpenInEditor: React.FC<{
},
});
- return items;
+ return [
+ ...items,
+ ...(annotationName
+ ? getAnnotateWithChatGPTMenuItems({
+ id: 'annotate-inspector-source',
+ getTarget: () =>
+ annotationTarget.current?.closest(
+ '[aria-label="Inspector source location"]',
+ ) ?? null,
+ initialComment: null,
+ metadata: {
+ selection: annotationName,
+ ...(location?.source
+ ? {
+ source:
+ formatFileLocation({
+ location,
+ root: window.remotion_cwd,
+ }) ?? location.source,
+ }
+ : {}),
+ },
+ })
+ : []),
+ ];
}, [
+ annotationName,
codingAgentInfo,
canConfigureApps,
canOpenInEditor,
@@ -295,5 +325,9 @@ export const InspectorOpenInEditor: React.FC<{
return null;
}
- return <SegmentedButton segments={segments} style={null} />;
+ return (
+ <div ref={annotationTarget}>
+ <SegmentedButton segments={segments} style={null} />
+ </div>
+ );
};
diff --git a/packages/studio/src/components/InspectorPanel/ConnectedCompositionsSection.tsx b/packages/studio/src/components/InspectorPanel/ConnectedCompositionsSection.tsx
index c3036a99f4..41ea214084 100644
--- a/packages/studio/src/components/InspectorPanel/ConnectedCompositionsSection.tsx
+++ b/packages/studio/src/components/InspectorPanel/ConnectedCompositionsSection.tsx
@@ -110,7 +110,7 @@ const ConnectedCompositionRow: React.FC<{
/>
)}
>
- {composition.id}
+ <span data-compname={composition.id}>{composition.id}</span>
</InspectorQuickAction>
</ContextMenu>
);
diff --git a/packages/studio/src/components/MenuBuildIndicator.tsx b/packages/studio/src/components/MenuBuildIndicator.tsx
index 9676ea840f..0de93e16b6 100644
--- a/packages/studio/src/components/MenuBuildIndicator.tsx
+++ b/packages/studio/src/components/MenuBuildIndicator.tsx
@@ -67,6 +67,7 @@ export const MenuBuildIndicator: React.FC<{
<Spacing x={0.5} />
<div style={openInEditor}>
<InspectorOpenInEditor
+ annotationName={null}
location={folderLocation}
locationType="folder"
showTooltips
diff --git a/packages/studio/src/components/QuickSwitcher/QuickSwitcherResult.tsx b/packages/studio/src/components/QuickSwitcher/QuickSwitcherResult.tsx
index 3757f407da..d389a3ddba 100644
--- a/packages/studio/src/components/QuickSwitcher/QuickSwitcherResult.tsx
+++ b/packages/studio/src/components/QuickSwitcher/QuickSwitcherResult.tsx
@@ -227,8 +227,9 @@ export const QuickSwitcherResult: React.FC<{
const row = (
<div
- ref={ref}
key={result.id}
+ ref={ref}
+ data-compname={composition?.id}
style={style}
onClick={onSelected ?? undefined}
>
diff --git a/packages/studio/src/components/SettingsModalFooter.tsx b/packages/studio/src/components/SettingsModalFooter.tsx
index 4b3aef1f96..fdfc2adc41 100644
--- a/packages/studio/src/components/SettingsModalFooter.tsx
+++ b/packages/studio/src/components/SettingsModalFooter.tsx
@@ -92,6 +92,7 @@ export const SettingsModalFooter: React.FC<{
Changes save to
<Spacing x={0.5} />
<InspectorOpenInEditor
+ annotationName={null}
locationType={null}
location={configFileLocation}
label={
diff --git a/packages/studio/src/components/Timeline/TimelineEffectItem.tsx b/packages/studio/src/components/Timeline/TimelineEffectItem.tsx
index 5cc18eb9f2..223997c1a4 100644
--- a/packages/studio/src/components/Timeline/TimelineEffectItem.tsx
+++ b/packages/studio/src/components/Timeline/TimelineEffectItem.tsx
@@ -1,11 +1,14 @@
-import React, {useCallback, useContext, useMemo, useState} from 'react';
+import React, {useCallback, useContext, useMemo, useRef, useState} from 'react';
import type {SequencePropsSubscriptionKey, InteractivitySchema} from 'remotion';
import {Internals} from 'remotion';
import type {CodePosition} from '../../error-overlay/react-overlay/utils/get-source-map';
import {canUseEffectOperations} from '../../helpers/browser-studio-operations';
import {StudioServerConnectionCtx} from '../../helpers/client-id';
import {TIMELINE_BLUE, WHITE_ALPHA_80} from '../../helpers/colors';
-import {formatContextForAgents} from '../../helpers/format-file-location';
+import {
+ formatContextForAgents,
+ formatFileLocation,
+} from '../../helpers/format-file-location';
import type {SequenceNodePathInfo} from '../../helpers/get-timeline-sequence-sort-key';
import {
EXPANDED_SECTION_PADDING_RIGHT,
@@ -14,8 +17,10 @@ import {
import {ContextMenu} from '../ContextMenu';
import {deleteEffects, reorderEffect} from '../effect-operations-api';
import type {GetIsExpanded} from '../ExpandedTracksProvider';
+import {getAnnotateWithChatGPTMenuItems} from '../get-annotate-with-chatgpt-menu-item';
import type {ComboboxValue} from '../NewComposition/ComboBox';
import {showNotification} from '../Notifications/NotificationCenter';
+import {useSettings} from '../SettingsContext';
import {getCopyContextForAgentsMenuItem} from './get-copy-context-for-agents-menu-item';
import {saveEffectProp} from './save-effect-prop';
import {
@@ -145,6 +150,14 @@ export const TimelineEffectItem: React.FC<{
toggleTrack,
}) => {
const {previewServerState} = useContext(StudioServerConnectionCtx);
+ const annotationTarget = useRef<HTMLDivElement>(null);
+ const {remotionSkillsInfo} = useSettings();
+ const markupSkill = remotionSkillsInfo?.skills.find(
+ ({name}) => name === 'remotion-markup',
+ );
+ const markupSkillAvailable = Boolean(
+ markupSkill?.installedInProject || markupSkill?.installedGlobally,
+ );
const previewConnected = previewServerState.type === 'connected';
const canMutateEffects = canUseEffectOperations();
const {propStatuses} = useContext(Internals.VisualModePropStatusesContext);
@@ -233,6 +246,22 @@ export const TimelineEffectItem: React.FC<{
root: window.remotion_cwd,
}),
}),
+ ...getAnnotateWithChatGPTMenuItems({
+ id: 'annotate-effect',
+ getTarget: () => annotationTarget.current,
+ initialComment: markupSkillAvailable ? '$remotion-markup' : null,
+ metadata: {
+ layer: nodePath.nodePath.join('.'),
+ layerKeys: JSON.stringify(nodePath.sequenceKeys),
+ effect: label,
+ effectIndex,
+ source:
+ formatFileLocation({
+ location: validatedLocation,
+ root: window.remotion_cwd,
+ }) ?? validatedLocation.source,
+ },
+ }),
];
if (!previewConnected) {
@@ -289,6 +318,9 @@ export const TimelineEffectItem: React.FC<{
deleteDisabled,
documentationLink,
label,
+ effectIndex,
+ markupSkillAvailable,
+ nodePath,
onDeleteEffectFromSource,
previewConnected,
selection,
@@ -545,9 +577,9 @@ export const TimelineEffectItem: React.FC<{
row
);
- return previewConnected ? (
- <ContextMenu getItems={getContextMenuItems}>{draggableRow}</ContextMenu>
- ) : (
- draggableRow
+ return (
+ <ContextMenu ref={annotationTarget} getItems={getContextMenuItems}>
+ {draggableRow}
+ </ContextMenu>
);
};
diff --git a/packages/studio/src/components/Timeline/TimelineEffectPropItem.tsx b/packages/studio/src/components/Timeline/TimelineEffectPropItem.tsx
index 5b7d8ee83e..b65706f763 100644
--- a/packages/studio/src/components/Timeline/TimelineEffectPropItem.tsx
+++ b/packages/studio/src/components/Timeline/TimelineEffectPropItem.tsx
@@ -3,7 +3,7 @@ import {
isSchemaFieldKeyframable,
optimisticUpdateForEffectPropStatuses,
} from '@remotion/studio-shared';
-import React, {useCallback, useContext, useMemo} from 'react';
+import React, {useCallback, useContext, useMemo, useRef} from 'react';
import type {
CanUpdateSequencePropStatus,
CanUpdateSequencePropStatusFalse,
@@ -15,14 +15,19 @@ import {Internals} from 'remotion';
import type {CodePosition} from '../../error-overlay/react-overlay/utils/get-source-map';
import {canUseEffectOperations} from '../../helpers/browser-studio-operations';
import {StudioServerConnectionCtx} from '../../helpers/client-id';
-import {formatContextForAgents} from '../../helpers/format-file-location';
+import {
+ formatContextForAgents,
+ formatFileLocation,
+} from '../../helpers/format-file-location';
import type {SequenceNodePathInfo} from '../../helpers/get-timeline-sequence-sort-key';
import {openOriginalPositionInEditorAtProperty} from '../../helpers/open-in-editor';
import type {EffectSchemaFieldInfo} from '../../helpers/timeline-layout';
import {useRuntimeStoreValue} from '../../helpers/use-runtime-values';
import {ContextMenu} from '../ContextMenu';
import {saveEffectProps} from '../effect-operations-api';
+import {getAnnotateWithChatGPTMenuItems} from '../get-annotate-with-chatgpt-menu-item';
import type {ComboboxValue} from '../NewComposition/ComboBox';
+import {useSettings} from '../SettingsContext';
import {useEditorOpening} from '../use-default-editor-info';
import {callAddEffectKeyframe} from './call-add-keyframe';
import {getCopyContextForAgentsMenuItem} from './get-copy-context-for-agents-menu-item';
@@ -450,6 +455,14 @@ export const TimelineEffectPropItem: React.FC<{
runtimeValueStore,
}) => {
const {previewServerState} = useContext(StudioServerConnectionCtx);
+ const annotationTarget = useRef<HTMLDivElement>(null);
+ const {remotionSkillsInfo} = useSettings();
+ const markupSkill = remotionSkillsInfo?.skills.find(
+ ({name}) => name === 'remotion-markup',
+ );
+ const markupSkillAvailable = Boolean(
+ markupSkill?.installedInProject || markupSkill?.installedGlobally,
+ );
const {canOpenInEditor, defaultEditorId} = useEditorOpening(
previewServerState.type === 'connected',
);
@@ -592,6 +605,25 @@ export const TimelineEffectPropItem: React.FC<{
root: window.remotion_cwd,
}),
}),
+ ...getAnnotateWithChatGPTMenuItems({
+ id: 'annotate-effect-property',
+ getTarget: () => annotationTarget.current,
+ initialComment: markupSkillAvailable ? '$remotion-markup' : null,
+ metadata: {
+ layer: nodePath.nodePath.join('.'),
+ layerKeys: JSON.stringify(nodePath.sequenceKeys),
+ effectIndex: field.effectIndex,
+ effectKeys: JSON.stringify(
+ nodePath.effectKeys[field.effectIndex] ?? [],
+ ),
+ property: field.key,
+ source:
+ formatFileLocation({
+ location: validatedLocation,
+ root: window.remotion_cwd,
+ }) ?? validatedLocation.source,
+ },
+ }),
{
type: 'divider',
id: 'copy-context-for-agents-divider',
@@ -609,7 +641,16 @@ export const TimelineEffectPropItem: React.FC<{
value: 'reset-effect-field',
},
];
- }, [canShowReset, field.key, onReset, selection, validatedLocation]);
+ }, [
+ canShowReset,
+ field.effectIndex,
+ field.key,
+ markupSkillAvailable,
+ nodePath,
+ onReset,
+ selection,
+ validatedLocation,
+ ]);
const onPropertyDoubleClick = useCallback<
React.MouseEventHandler<HTMLDivElement>
@@ -662,5 +703,9 @@ export const TimelineEffectPropItem: React.FC<{
</TimelineRowChrome>
);
- return <ContextMenu getItems={getContextMenuItems}>{row}</ContextMenu>;
+ return (
+ <ContextMenu ref={annotationTarget} getItems={getContextMenuItems}>
+ {row}
+ </ContextMenu>
+ );
};
diff --git a/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts b/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts
index 575984df30..7fe1d72b2d 100644
--- a/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts
+++ b/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts
@@ -8,13 +8,14 @@ import type {ResolvedStackLocation, TSequence} from 'remotion';
import {NoReactInternals} from 'remotion/no-react';
import {copyText} from '../../helpers/copy-text';
import {formatContextForAgents} from '../../helpers/format-file-location';
-import {getCodexAnnotation} from '../../helpers/get-codex-annotation';
import {
getDefaultOpenInTarget,
getGitSourceName,
openGitSource,
} from '../../helpers/get-git-menu-item';
+import {getVisibleAnnotationTarget} from '../../helpers/request-codex-annotation';
import {getSequenceAnnotationMetadata} from '../../helpers/sequence-annotation';
+import {getAnnotateWithChatGPTMenuItems} from '../get-annotate-with-chatgpt-menu-item';
import {getOpenInMenuItems} from '../get-open-in-menu-items';
import type {ComboboxValue} from '../NewComposition/ComboBox';
import {showNotification} from '../Notifications/NotificationCenter';
@@ -347,69 +348,19 @@ export const getSequenceContextMenuItems = (
}
: null,
getCopyContextForAgentsMenuItem({contextForAgents}),
- getCodexAnnotation() !== null
- ? {
- type: 'item' as const,
- id: 'annotate-layer',
- keyHint: null,
- label: 'Annotate with ChatGPT',
- leftItem: null,
- disabled: false,
- onClick: () => {
- const target = [
- sequence.refForOutline?.current,
- contextMenuTarget,
- ].find((element) => {
- if (!element?.isConnected) {
- return false;
- }
-
- const rect = element.getBoundingClientRect();
- return (
- rect.width > 0 &&
- rect.height > 0 &&
- rect.bottom > 0 &&
- rect.right > 0 &&
- rect.top < window.innerHeight &&
- rect.left < window.innerWidth
- );
- });
- if (!target) {
- showNotification(
- 'The annotation target is no longer visible',
- 3000,
- );
- return;
- }
-
- const annotation = getCodexAnnotation();
- if (annotation === null) {
- showNotification('The annotation editor is unavailable', 3000);
- return;
- }
-
- try {
- const result = annotation.request(target, {
- metadata: getSequenceAnnotationMetadata({
- sequence,
- location: originalLocation,
- }),
- });
- if (!result?.accepted) {
- showNotification('Could not open the annotation editor', 3000);
- }
- } catch (err) {
- showNotification(
- `Could not open the annotation editor: ${(err as Error).message}`,
- 3000,
- );
- }
- },
- quickSwitcherLabel: null,
- subMenu: null,
- value: 'annotate-layer',
- }
- : null,
+ ...getAnnotateWithChatGPTMenuItems({
+ id: 'annotate-layer',
+ getTarget: () =>
+ getVisibleAnnotationTarget([
+ sequence.refForOutline?.current,
+ contextMenuTarget,
+ ]),
+ initialComment: null,
+ metadata: getSequenceAnnotationMetadata({
+ sequence,
+ location: originalLocation,
+ }),
+ }),
assetLinkInfo
? {
type: 'item' as const,
diff --git a/packages/studio/src/components/WrapRefactorModal.tsx b/packages/studio/src/components/WrapRefactorModal.tsx
index 794544e353..14b0307d6b 100644
--- a/packages/studio/src/components/WrapRefactorModal.tsx
+++ b/packages/studio/src/components/WrapRefactorModal.tsx
@@ -1,10 +1,13 @@
-import React, {useContext} from 'react';
+import React, {useCallback, useContext, useRef} from 'react';
import {getBrowserStudioOperations} from '../helpers/browser-studio-operations';
import {StudioServerConnectionCtx} from '../helpers/client-id';
import {LIGHT_TEXT} from '../helpers/colors';
import {formatFileLocation} from '../helpers/format-file-location';
+import {getCodexAnnotation} from '../helpers/get-codex-annotation';
+import {requestCodexAnnotation} from '../helpers/request-codex-annotation';
import type {ModalState} from '../state/modals';
import {AgentPrompt} from './AgentPrompt';
+import {ModalButton} from './ModalButton';
import {getMaxModalHeight, getMaxModalWidth} from './ModalContainer';
import {ModalHeader} from './ModalHeader';
import {DismissableModal} from './NewComposition/DismissableModal';
@@ -41,6 +44,15 @@ export const WrapRefactorModal: React.FC<{readonly state: State}> = ({
}) => {
const {previewServerState} = useContext(StudioServerConnectionCtx);
const {error, remotionSkillsInfo} = useSettings();
+ const annotationTarget = useRef<HTMLDivElement>(null);
+ const markupSkill = remotionSkillsInfo?.skills.find(
+ ({name}) => name === 'remotion-markup',
+ );
+ const markupSkillAvailable = Boolean(
+ markupSkill?.installedInProject || markupSkill?.installedGlobally,
+ );
+ const canAnnotate =
+ !window.remotion_isReadOnlyStudio && getCodexAnnotation() !== null;
const isBrowserStudio = getBrowserStudioOperations() !== null;
const canSuggestAgent =
!isBrowserStudio &&
@@ -59,18 +71,45 @@ export const WrapRefactorModal: React.FC<{readonly state: State}> = ({
state.wrapper === 'HtmlInCanvasMotionBlur'
? '@remotion/motion-blur'
: 'remotion';
+ const onSendToChatGPT = useCallback(() => {
+ requestCodexAnnotation({
+ target: annotationTarget.current,
+ initialComment: markupSkillAvailable
+ ? '$remotion-markup Wrap this item.'
+ : 'Wrap this item.',
+ metadata: {
+ action: 'Wrap item',
+ wrapper: state.wrapper,
+ import: wrapperImportPath,
+ ...(state.displayName ? {selection: state.displayName} : {}),
+ source: location,
+ },
+ });
+ }, [
+ location,
+ markupSkillAvailable,
+ state.displayName,
+ state.wrapper,
+ wrapperImportPath,
+ ]);
return (
<DismissableModal panelStyle={panelStyle}>
<ModalHeader title="Wrap item" />
- <div style={container}>
+ <div ref={annotationTarget} style={container}>
<div style={text}>
Studio could not safely wrap this item in
{` <${state.wrapper}>`} with an automated code change.
</div>
- {canSuggestAgent ? (
+ {canSuggestAgent || canAnnotate ? (
<AgentPrompt
- action={null}
+ action={
+ canAnnotate ? (
+ <ModalButton size="compact" onClick={onSendToChatGPT}>
+ Send to ChatGPT
+ </ModalButton>
+ ) : null
+ }
availableText="You can wrap it using an agent:"
promptDetails={` Wrap ${target} in <${state.wrapper}> from '${wrapperImportPath}'`}
skillId="remotion-markup"
diff --git a/packages/studio/src/components/composition-menu-items.ts b/packages/studio/src/components/composition-menu-items.ts
index 86d3e8d5ca..18951bd458 100644
--- a/packages/studio/src/components/composition-menu-items.ts
+++ b/packages/studio/src/components/composition-menu-items.ts
@@ -20,7 +20,9 @@ import {
openOriginalPositionInEditor,
} from '../helpers/open-in-editor';
import type {PreviewServerConnectionState} from '../helpers/preview-server-events';
+import {getVisibleAnnotationTarget} from '../helpers/request-codex-annotation';
import type {ModalState} from '../state/modals';
+import {getAnnotateWithChatGPTMenuItems} from './get-annotate-with-chatgpt-menu-item';
import {getOpenInMenuItems} from './get-open-in-menu-items';
import type {ComboboxValue} from './NewComposition/ComboBox';
import {showNotification} from './Notifications/NotificationCenter';
@@ -432,6 +434,25 @@ export const getCompositionMenuItems = ({
value: 'copy-context-for-agents',
disabled: !contextForAgents,
},
+ ...(composition
+ ? getAnnotateWithChatGPTMenuItems({
+ id: 'annotate-composition',
+ getTarget: () =>
+ getVisibleAnnotationTarget(
+ Array.from(document.querySelectorAll('[data-compname]'))
+ .reverse()
+ .filter(
+ (element) =>
+ element.getAttribute('data-compname') === composition.id,
+ ),
+ ),
+ initialComment: null,
+ metadata: {
+ composition: composition.id,
+ ...(fileLocation ? {source: fileLocation} : {}),
+ },
+ })
+ : []),
{
id: 'copy-file-location',
keyHint: null,
diff --git a/packages/studio/src/components/get-annotate-with-chatgpt-menu-item.ts b/packages/studio/src/components/get-annotate-with-chatgpt-menu-item.ts
new file mode 100644
index 0000000000..45095d52c8
--- /dev/null
+++ b/packages/studio/src/components/get-annotate-with-chatgpt-menu-item.ts
@@ -0,0 +1,36 @@
+import {getCodexAnnotation} from '../helpers/get-codex-annotation';
+import {requestCodexAnnotation} from '../helpers/request-codex-annotation';
+import type {ComboboxValue} from './NewComposition/ComboBox';
+
+export const getAnnotateWithChatGPTMenuItems = ({
+ id,
+ getTarget,
+ initialComment,
+ metadata,
+}: {
+ readonly id: string;
+ readonly getTarget: () => Element | null;
+ readonly initialComment: string | null;
+ readonly metadata: Record<string, string | number>;
+}): ComboboxValue[] => {
+ if (getCodexAnnotation() === null) {
+ return [];
+ }
+
+ return [
+ {
+ type: 'item',
+ id,
+ keyHint: null,
+ label: 'Annotate with ChatGPT',
+ leftItem: null,
+ disabled: false,
+ onClick: () => {
+ requestCodexAnnotation({target: getTarget(), initialComment, metadata});
+ },
+ quickSwitcherLabel: null,
+ subMenu: null,
+ value: id,
+ },
+ ];
+};
diff --git a/packages/studio/src/error-overlay/remotion-overlay/ErrorDisplay.tsx b/packages/studio/src/error-overlay/remotion-overlay/ErrorDisplay.tsx
index 494d4b5c19..64b2f68045 100644
--- a/packages/studio/src/error-overlay/remotion-overlay/ErrorDisplay.tsx
+++ b/packages/studio/src/error-overlay/remotion-overlay/ErrorDisplay.tsx
@@ -1,9 +1,11 @@
import {getLocationFromBuildError} from '@remotion/studio-shared';
-import React, {useContext, useMemo} from 'react';
+import React, {useCallback, useContext, useMemo, useRef} from 'react';
import {MediaPlaybackError} from 'remotion';
import {CodingAgentButton} from '../../components/CodingAgentButton';
import {Spacing} from '../../components/layout';
import {HORIZONTAL_SCROLLBAR_CLASSNAME} from '../../components/Menu/is-menu-item';
+import {ModalButton} from '../../components/ModalButton';
+import {useSettings} from '../../components/SettingsContext';
import {useEditorOpening} from '../../components/use-default-editor-info';
import {StudioServerConnectionCtx} from '../../helpers/client-id';
import {
@@ -11,6 +13,9 @@ import {
WHITE,
WHITE_ALPHA_60,
} from '../../helpers/colors';
+import {formatFileLocation} from '../../helpers/format-file-location';
+import {getCodexAnnotation} from '../../helpers/get-codex-annotation';
+import {requestCodexAnnotation} from '../../helpers/request-codex-annotation';
import type {ErrorRecord} from '../react-overlay/listen-to-runtime-errors';
import {AskOnDiscord} from './AskOnDiscord';
import {CalculateMetadataErrorExplainer} from './CalculateMetadataErrorExplainer';
@@ -92,6 +97,28 @@ export const ErrorDisplay: React.FC<{
symbolicationFailure,
}) => {
const {previewServerState} = useContext(StudioServerConnectionCtx);
+ const annotationTarget = useRef<HTMLDivElement>(null);
+ const {remotionSkillsInfo} = useSettings();
+ const canAnnotate =
+ !window.remotion_isReadOnlyStudio && getCodexAnnotation() !== null;
+ const buildErrorLocation = getLocationFromBuildError(display.error);
+ const errorKind = calculateMetadata
+ ? 'calculateMetadata'
+ : display.error instanceof MediaPlaybackError
+ ? 'media playback'
+ : buildErrorLocation
+ ? 'compilation'
+ : 'runtime';
+ const skillId =
+ errorKind === 'media playback'
+ ? 'remotion-multimedia'
+ : errorKind === 'runtime'
+ ? 'remotion-markup'
+ : null;
+ const skill = remotionSkillsInfo?.skills.find(({name}) => name === skillId);
+ const skillAvailable = Boolean(
+ skill?.installedInProject || skill?.installedGlobally,
+ );
const {canOpenInEditor, defaultEditorId, defaultEditorName} =
useEditorOpening(previewServerState.type === 'connected');
const stackFrames = useMemo(() => {
@@ -149,9 +176,53 @@ export const ErrorDisplay: React.FC<{
return display.error.stack || header;
}, [display.error, message, stackFrames]);
const fixWithAgentPrompt = `Fix this error in my Remotion project:\n\n${errorTextForCopy}`;
+ const onSendToChatGPT = useCallback(() => {
+ const frame = stackFrames[0];
+ const location = frame?.originalFileName
+ ? {
+ source: frame.originalFileName,
+ line: frame.originalLineNumber,
+ column: frame.originalColumnNumber,
+ }
+ : buildErrorLocation
+ ? {
+ source: buildErrorLocation.fileName,
+ line: buildErrorLocation.lineNumber,
+ column: buildErrorLocation.columnNumber,
+ }
+ : null;
+ const source = formatFileLocation({
+ location,
+ root: window.remotion_cwd,
+ });
+ requestCodexAnnotation({
+ target: annotationTarget.current,
+ initialComment: `${skillAvailable ? `$${skillId} ` : ''}Fix this ${errorKind} error.`,
+ metadata: {
+ kind: errorKind,
+ error: display.error.name,
+ message: message.slice(0, 256),
+ ...(source ? {source} : {}),
+ ...(location?.column !== null && location?.column !== undefined
+ ? {column: location.column}
+ : {}),
+ ...(display.error instanceof MediaPlaybackError
+ ? {media: display.error.src}
+ : {}),
+ },
+ });
+ }, [
+ buildErrorLocation,
+ display.error,
+ errorKind,
+ message,
+ skillAvailable,
+ skillId,
+ stackFrames,
+ ]);
return (
- <div>
+ <div ref={annotationTarget}>
<ErrorTitle
symbolicating={false}
name={display.error.name}
@@ -190,6 +261,12 @@ export const ErrorDisplay: React.FC<{
size="default"
style={codingAgentButton}
/>
+ {canAnnotate ? (
+ <>
+ <ModalButton onClick={onSendToChatGPT}>Send to ChatGPT</ModalButton>
+ <div style={spacer} />
+ </>
+ ) : null}
<CopyStackTrace errorText={errorTextForCopy} />
<div style={spacer} />
<SearchGithubIssues
@@ -257,6 +334,15 @@ export const ErrorDisplay: React.FC<{
) : null}
</>
)}
+ {canAnnotate && stackFrames.length > 0 && display.error.stack ? (
+ <pre
+ aria-label="Complete stack trace"
+ className={HORIZONTAL_SCROLLBAR_CLASSNAME}
+ style={rawStack}
+ >
+ {display.error.stack}
+ </pre>
+ ) : null}
</div>
);
};
diff --git a/packages/studio/src/helpers/get-codex-annotation.ts b/packages/studio/src/helpers/get-codex-annotation.ts
index 13b12b20bc..767fe6364f 100644
--- a/packages/studio/src/helpers/get-codex-annotation.ts
+++ b/packages/studio/src/helpers/get-codex-annotation.ts
@@ -1,9 +1,10 @@
type CodexAnnotation = {
readonly request: (
target: Element,
- options:
- | {metadata: Record<string, string | number>}
- | {initialComment: string},
+ options: {
+ metadata?: Record<string, string | number>;
+ initialComment?: string;
+ },
) => {accepted: boolean};
};
diff --git a/packages/studio/src/helpers/request-codex-annotation.ts b/packages/studio/src/helpers/request-codex-annotation.ts
index 49b509d41e..0e74e72406 100644
--- a/packages/studio/src/helpers/request-codex-annotation.ts
+++ b/packages/studio/src/helpers/request-codex-annotation.ts
@@ -1,25 +1,76 @@
import {showNotification} from '../components/Notifications/NotificationCenter';
import {getCodexAnnotation} from './get-codex-annotation';
+export const getVisibleAnnotationTarget = (
+ targets: readonly (Element | null | undefined)[],
+): Element | null => {
+ return (
+ targets.find((element) => {
+ if (!element?.isConnected) {
+ return false;
+ }
+
+ const rect = element.getBoundingClientRect();
+ const style = window.getComputedStyle(element);
+ return (
+ style.visibility !== 'hidden' &&
+ style.display !== 'none' &&
+ rect.width > 0 &&
+ rect.height > 0 &&
+ rect.bottom > 0 &&
+ rect.right > 0 &&
+ rect.top < window.innerHeight &&
+ rect.left < window.innerWidth
+ );
+ }) ?? null
+ );
+};
+
export const requestCodexAnnotation = ({
target,
initialComment,
metadata,
}: {
readonly target: Element | null;
- readonly initialComment: string;
+ readonly initialComment: string | null;
readonly metadata: Record<string, string | number> | null;
}): boolean => {
const annotation = getCodexAnnotation();
- if (annotation === null || target === null) {
+ if (annotation === null) {
showNotification('The annotation editor is unavailable', 3000);
return false;
}
+ if (target === null || getVisibleAnnotationTarget([target]) === null) {
+ showNotification('The annotation target is no longer visible', 3000);
+ return false;
+ }
+
+ // Omit oversized facts instead of truncating source paths or identifiers.
+ const boundedMetadata: Record<string, string | number> = {};
+ for (const [key, value] of Object.entries(metadata ?? {})) {
+ if (
+ Object.keys(boundedMetadata).length >= 6 ||
+ key.length > 64 ||
+ !/^[A-Za-z][A-Za-z0-9 _-]*$/.test(key) ||
+ key.includes(' ') ||
+ (typeof value === 'string' ? value.length > 256 : !Number.isFinite(value))
+ ) {
+ continue;
+ }
+
+ const next = {...boundedMetadata, [key]: value};
+ if (new TextEncoder().encode(JSON.stringify(next)).length <= 2048) {
+ boundedMetadata[key] = value;
+ }
+ }
+
try {
const result = annotation.request(target, {
- initialComment,
- ...(metadata === null ? {} : {metadata}),
+ ...(initialComment === null ? {} : {initialComment}),
+ ...(Object.keys(boundedMetadata).length === 0
+ ? {}
+ : {metadata: boundedMetadata}),
});
if (result?.accepted) {
return true;