🎥 Make videos programmatically with React
`@remotion/studio`: Add ChatGPT annotations to source and repair actions
Jonny Burgera4c470df028a
diff --git a/packages/studio/src/components/CanvasTabs.tsx b/packages/studio/src/components/CanvasTabs.tsxindex 75089c01b7..ee35ea91b3 100644--- a/packages/studio/src/components/CanvasTabs.tsx+++ b/packages/studio/src/components/CanvasTabs.tsx@@ -428,6 +428,9 @@ const CanvasTab: React.FC<{triggerStyle={tooltipTriggerStyle}><span+ data-compname={+ tab.type === 'composition' ? tab.compositionId : undefined+ }style={{...labelStyle, color: selected ? WHITE : LIGHT_TEXT}}onPointerEnter={() => {setTooltipHovered(true);diff --git a/packages/studio/src/components/FixComputedValueModal.tsx b/packages/studio/src/components/FixComputedValueModal.tsxindex f3e49e4820..6910bbbee7 100644--- a/packages/studio/src/components/FixComputedValueModal.tsx+++ b/packages/studio/src/components/FixComputedValueModal.tsx@@ -1,16 +1,26 @@import type {SymbolicatedStackFrame} from '@remotion/studio-shared';-import React, {useContext, useEffect, useState} from 'react';+import React, {+ useCallback,+ useContext,+ useEffect,+ useRef,+ useState,+} from 'react';import {resolveFileSource} from '../error-overlay/react-overlay/effects/resolve-file-source';import {OpenInEditor} from '../error-overlay/remotion-overlay/OpenInEditor';import {StackElement} from '../error-overlay/remotion-overlay/StackFrame';import {StudioServerConnectionCtx} from '../helpers/client-id';import {LIGHT_TEXT} from '../helpers/colors';+import {getCodexAnnotation} from '../helpers/get-codex-annotation';import {findOriginalPositionInFileAtProperty} from '../helpers/open-in-editor';+import {requestCodexAnnotation} from '../helpers/request-codex-annotation';import type {ModalState} from '../state/modals';import {AgentPrompt} from './AgentPrompt';+import {ModalButton} from './ModalButton';import {getMaxModalHeight, getMaxModalWidth} from './ModalContainer';import {ModalHeader} from './ModalHeader';import {DismissableModal} from './NewComposition/DismissableModal';+import {useSettings} from './SettingsContext';import {useEditorOpening} from './use-default-editor-info';const panelStyle: React.CSSProperties = {@@ -69,6 +79,15 @@ export const FixComputedValueModal: React.FC<{readonly state: FixComputedValueModalState;}> = ({state}) => {const {previewServerState} = useContext(StudioServerConnectionCtx);+ const {remotionSkillsInfo} = useSettings();+ const annotationTarget = useRef<HTMLDivElement>(null);+ const interactivitySkill = remotionSkillsInfo?.skills.find(+ ({name}) => name === 'remotion-interactivity',+ );+ const interactivitySkillAvailable = Boolean(+ interactivitySkill?.installedInProject ||+ interactivitySkill?.installedGlobally,+ );const {canOpenInEditor, defaultEditorId, defaultEditorName} =useEditorOpening(previewServerState.type === 'connected');const [sourcePreview, setSourcePreview] = useState<SourcePreviewState>({@@ -79,6 +98,19 @@ export const FixComputedValueModal: React.FC<{? `${sourcePreview.stack.originalFileName}:${sourcePreview.stack.originalLineNumber}:${sourcePreview.stack.originalColumnNumber}`: `${state.location.source}:${state.location.line}:${state.location.column}`;const promptDetails = ` ${promptLocation} make "${state.prop}" interactive`;+ const canAnnotate =+ sourcePreview.type === 'loaded' &&+ !window.remotion_isReadOnlyStudio &&+ getCodexAnnotation() !== null;+ const onSendToChatGPT = useCallback(() => {+ requestCodexAnnotation({+ target: annotationTarget.current,+ initialComment: interactivitySkillAvailable+ ? '$remotion-interactivity Make this computed value interactive.'+ : 'Make this computed value interactive.',+ metadata: {property: state.prop, source: promptLocation},+ });+ }, [interactivitySkillAvailable, promptLocation, state.prop]);const canOpenSourceInEditor =sourcePreview.type === 'loaded' &&canOpenInEditor &&@@ -131,7 +163,7 @@ export const FixComputedValueModal: React.FC<{This value cannot be interactively edited because it is computed:</div>{sourcePreview.type === 'loaded' ? (- <div style={sourcePreviewContainer}>+ <div ref={annotationTarget} style={sourcePreviewContainer}><StackElementcollapsible={false}defaultFunctionName={null}@@ -171,7 +203,13 @@ export const FixComputedValueModal: React.FC<{</div>)}<AgentPrompt- action={null}+ action={+ canAnnotate ? (+ <ModalButton size="compact" onClick={onSendToChatGPT}>+ Send to ChatGPT+ </ModalButton>+ ) : null+ }availableText="Use this prompt to make this value editable:"promptDetails={promptDetails}skillId="remotion-interactivity"diff --git a/packages/studio/src/components/InspectorLocationCopy.tsx b/packages/studio/src/components/InspectorLocationCopy.tsxindex 8e9d478e0a..eab5dcfdc1 100644--- a/packages/studio/src/components/InspectorLocationCopy.tsx+++ b/packages/studio/src/components/InspectorLocationCopy.tsx@@ -106,6 +106,7 @@ export const InspectorLocationCopy: React.FC<{}}><InspectorOpenInEditor+ annotationName={name}locationType={null}contextForAgents={contextForAgents}location={openInEditorLocation}diff --git a/packages/studio/src/components/InspectorOpenInEditor.tsx b/packages/studio/src/components/InspectorOpenInEditor.tsxindex e6c07b1981..1a3b9f69e1 100644--- a/packages/studio/src/components/InspectorOpenInEditor.tsx+++ b/packages/studio/src/components/InspectorOpenInEditor.tsx@@ -1,11 +1,12 @@import type {DefaultCodingAgent} from '@remotion/renderer';import type {EditorPickerId} from '@remotion/studio-shared';-import React, {useCallback, useContext, useMemo} from 'react';+import React, {useCallback, useContext, useMemo, useRef} from 'react';import type {OriginalPosition} from '../error-overlay/react-overlay/utils/get-source-map';import {getBrowserStudioOperations} from '../helpers/browser-studio-operations';import {StudioServerConnectionCtx} from '../helpers/client-id';import {LIGHT_TEXT} from '../helpers/colors';import {copyText} from '../helpers/copy-text';+import {formatFileLocation} from '../helpers/format-file-location';import {getDefaultOpenInTarget,openGitSource,@@ -19,6 +20,7 @@ import {import {CaretDown} from '../icons/caret';import {EditorIcon} from '../icons/editor';import {GitHubIcon} from '../icons/github';+import {getAnnotateWithChatGPTMenuItems} from './get-annotate-with-chatgpt-menu-item';import {getOpenInMenuItems} from './get-open-in-menu-items';import type {ComboboxValue} from './NewComposition/ComboBox';import {showNotification} from './Notifications/NotificationCenter';@@ -46,12 +48,14 @@ const editorButtonIconSize = 18;const githubButtonIconSize = 16;export const InspectorOpenInEditor: React.FC<{+ readonly annotationName: string | null;readonly contextForAgents?: string | null;readonly location: OriginalPosition | null;readonly label?: React.ReactNode;readonly locationType: 'file' | 'folder' | null;readonly showTooltips: boolean;}> = ({+ annotationName,contextForAgents = null,label,location,@@ -59,6 +63,7 @@ export const InspectorOpenInEditor: React.FC<{showTooltips,}) => {const {previewServerState} = useContext(StudioServerConnectionCtx);+ const annotationTarget = useRef<HTMLDivElement>(null);const configureDefaultApps = useConfigureDefaultApps();const {canConfigureApps,@@ -210,8 +215,33 @@ export const InspectorOpenInEditor: React.FC<{},});- return items;+ return [+ ...items,+ ...(annotationName+ ? getAnnotateWithChatGPTMenuItems({+ id: 'annotate-inspector-source',+ getTarget: () =>+ annotationTarget.current?.closest(+ '[aria-label="Inspector source location"]',+ ) ?? null,+ initialComment: null,+ metadata: {+ selection: annotationName,+ ...(location?.source+ ? {+ source:+ formatFileLocation({+ location,+ root: window.remotion_cwd,+ }) ?? location.source,+ }+ : {}),+ },+ })+ : []),+ ];}, [+ annotationName,codingAgentInfo,canConfigureApps,canOpenInEditor,@@ -295,5 +325,9 @@ export const InspectorOpenInEditor: React.FC<{return null;}- return <SegmentedButton segments={segments} style={null} />;+ return (+ <div ref={annotationTarget}>+ <SegmentedButton segments={segments} style={null} />+ </div>+ );};diff --git a/packages/studio/src/components/InspectorPanel/ConnectedCompositionsSection.tsx b/packages/studio/src/components/InspectorPanel/ConnectedCompositionsSection.tsxindex c3036a99f4..41ea214084 100644--- a/packages/studio/src/components/InspectorPanel/ConnectedCompositionsSection.tsx+++ b/packages/studio/src/components/InspectorPanel/ConnectedCompositionsSection.tsx@@ -110,7 +110,7 @@ const ConnectedCompositionRow: React.FC<{/>)}>- {composition.id}+ <span data-compname={composition.id}>{composition.id}</span></InspectorQuickAction></ContextMenu>);diff --git a/packages/studio/src/components/MenuBuildIndicator.tsx b/packages/studio/src/components/MenuBuildIndicator.tsxindex 9676ea840f..0de93e16b6 100644--- a/packages/studio/src/components/MenuBuildIndicator.tsx+++ b/packages/studio/src/components/MenuBuildIndicator.tsx@@ -67,6 +67,7 @@ export const MenuBuildIndicator: React.FC<{<Spacing x={0.5} /><div style={openInEditor}><InspectorOpenInEditor+ annotationName={null}location={folderLocation}locationType="folder"showTooltipsdiff --git a/packages/studio/src/components/QuickSwitcher/QuickSwitcherResult.tsx b/packages/studio/src/components/QuickSwitcher/QuickSwitcherResult.tsxindex 3757f407da..d389a3ddba 100644--- a/packages/studio/src/components/QuickSwitcher/QuickSwitcherResult.tsx+++ b/packages/studio/src/components/QuickSwitcher/QuickSwitcherResult.tsx@@ -227,8 +227,9 @@ export const QuickSwitcherResult: React.FC<{const row = (<div- ref={ref}key={result.id}+ ref={ref}+ data-compname={composition?.id}style={style}onClick={onSelected ?? undefined}>diff --git a/packages/studio/src/components/SettingsModalFooter.tsx b/packages/studio/src/components/SettingsModalFooter.tsxindex 4b3aef1f96..fdfc2adc41 100644--- a/packages/studio/src/components/SettingsModalFooter.tsx+++ b/packages/studio/src/components/SettingsModalFooter.tsx@@ -92,6 +92,7 @@ export const SettingsModalFooter: React.FC<{Changes save to<Spacing x={0.5} /><InspectorOpenInEditor+ annotationName={null}locationType={null}location={configFileLocation}label={diff --git a/packages/studio/src/components/Timeline/TimelineEffectItem.tsx b/packages/studio/src/components/Timeline/TimelineEffectItem.tsxindex 5cc18eb9f2..223997c1a4 100644--- a/packages/studio/src/components/Timeline/TimelineEffectItem.tsx+++ b/packages/studio/src/components/Timeline/TimelineEffectItem.tsx@@ -1,11 +1,14 @@-import React, {useCallback, useContext, useMemo, useState} from 'react';+import React, {useCallback, useContext, useMemo, useRef, useState} from 'react';import type {SequencePropsSubscriptionKey, InteractivitySchema} from 'remotion';import {Internals} from 'remotion';import type {CodePosition} from '../../error-overlay/react-overlay/utils/get-source-map';import {canUseEffectOperations} from '../../helpers/browser-studio-operations';import {StudioServerConnectionCtx} from '../../helpers/client-id';import {TIMELINE_BLUE, WHITE_ALPHA_80} from '../../helpers/colors';-import {formatContextForAgents} from '../../helpers/format-file-location';+import {+ formatContextForAgents,+ formatFileLocation,+} from '../../helpers/format-file-location';import type {SequenceNodePathInfo} from '../../helpers/get-timeline-sequence-sort-key';import {EXPANDED_SECTION_PADDING_RIGHT,@@ -14,8 +17,10 @@ import {import {ContextMenu} from '../ContextMenu';import {deleteEffects, reorderEffect} from '../effect-operations-api';import type {GetIsExpanded} from '../ExpandedTracksProvider';+import {getAnnotateWithChatGPTMenuItems} from '../get-annotate-with-chatgpt-menu-item';import type {ComboboxValue} from '../NewComposition/ComboBox';import {showNotification} from '../Notifications/NotificationCenter';+import {useSettings} from '../SettingsContext';import {getCopyContextForAgentsMenuItem} from './get-copy-context-for-agents-menu-item';import {saveEffectProp} from './save-effect-prop';import {@@ -145,6 +150,14 @@ export const TimelineEffectItem: React.FC<{toggleTrack,}) => {const {previewServerState} = useContext(StudioServerConnectionCtx);+ const annotationTarget = useRef<HTMLDivElement>(null);+ const {remotionSkillsInfo} = useSettings();+ const markupSkill = remotionSkillsInfo?.skills.find(+ ({name}) => name === 'remotion-markup',+ );+ const markupSkillAvailable = Boolean(+ markupSkill?.installedInProject || markupSkill?.installedGlobally,+ );const previewConnected = previewServerState.type === 'connected';const canMutateEffects = canUseEffectOperations();const {propStatuses} = useContext(Internals.VisualModePropStatusesContext);@@ -233,6 +246,22 @@ export const TimelineEffectItem: React.FC<{root: window.remotion_cwd,}),}),+ ...getAnnotateWithChatGPTMenuItems({+ id: 'annotate-effect',+ getTarget: () => annotationTarget.current,+ initialComment: markupSkillAvailable ? '$remotion-markup' : null,+ metadata: {+ layer: nodePath.nodePath.join('.'),+ layerKeys: JSON.stringify(nodePath.sequenceKeys),+ effect: label,+ effectIndex,+ source:+ formatFileLocation({+ location: validatedLocation,+ root: window.remotion_cwd,+ }) ?? validatedLocation.source,+ },+ }),];if (!previewConnected) {@@ -289,6 +318,9 @@ export const TimelineEffectItem: React.FC<{deleteDisabled,documentationLink,label,+ effectIndex,+ markupSkillAvailable,+ nodePath,onDeleteEffectFromSource,previewConnected,selection,@@ -545,9 +577,9 @@ export const TimelineEffectItem: React.FC<{row);- return previewConnected ? (- <ContextMenu getItems={getContextMenuItems}>{draggableRow}</ContextMenu>- ) : (- draggableRow+ return (+ <ContextMenu ref={annotationTarget} getItems={getContextMenuItems}>+ {draggableRow}+ </ContextMenu>);};diff --git a/packages/studio/src/components/Timeline/TimelineEffectPropItem.tsx b/packages/studio/src/components/Timeline/TimelineEffectPropItem.tsxindex 5b7d8ee83e..b65706f763 100644--- a/packages/studio/src/components/Timeline/TimelineEffectPropItem.tsx+++ b/packages/studio/src/components/Timeline/TimelineEffectPropItem.tsx@@ -3,7 +3,7 @@ import {isSchemaFieldKeyframable,optimisticUpdateForEffectPropStatuses,} from '@remotion/studio-shared';-import React, {useCallback, useContext, useMemo} from 'react';+import React, {useCallback, useContext, useMemo, useRef} from 'react';import type {CanUpdateSequencePropStatus,CanUpdateSequencePropStatusFalse,@@ -15,14 +15,19 @@ import {Internals} from 'remotion';import type {CodePosition} from '../../error-overlay/react-overlay/utils/get-source-map';import {canUseEffectOperations} from '../../helpers/browser-studio-operations';import {StudioServerConnectionCtx} from '../../helpers/client-id';-import {formatContextForAgents} from '../../helpers/format-file-location';+import {+ formatContextForAgents,+ formatFileLocation,+} from '../../helpers/format-file-location';import type {SequenceNodePathInfo} from '../../helpers/get-timeline-sequence-sort-key';import {openOriginalPositionInEditorAtProperty} from '../../helpers/open-in-editor';import type {EffectSchemaFieldInfo} from '../../helpers/timeline-layout';import {useRuntimeStoreValue} from '../../helpers/use-runtime-values';import {ContextMenu} from '../ContextMenu';import {saveEffectProps} from '../effect-operations-api';+import {getAnnotateWithChatGPTMenuItems} from '../get-annotate-with-chatgpt-menu-item';import type {ComboboxValue} from '../NewComposition/ComboBox';+import {useSettings} from '../SettingsContext';import {useEditorOpening} from '../use-default-editor-info';import {callAddEffectKeyframe} from './call-add-keyframe';import {getCopyContextForAgentsMenuItem} from './get-copy-context-for-agents-menu-item';@@ -450,6 +455,14 @@ export const TimelineEffectPropItem: React.FC<{runtimeValueStore,}) => {const {previewServerState} = useContext(StudioServerConnectionCtx);+ const annotationTarget = useRef<HTMLDivElement>(null);+ const {remotionSkillsInfo} = useSettings();+ const markupSkill = remotionSkillsInfo?.skills.find(+ ({name}) => name === 'remotion-markup',+ );+ const markupSkillAvailable = Boolean(+ markupSkill?.installedInProject || markupSkill?.installedGlobally,+ );const {canOpenInEditor, defaultEditorId} = useEditorOpening(previewServerState.type === 'connected',);@@ -592,6 +605,25 @@ export const TimelineEffectPropItem: React.FC<{root: window.remotion_cwd,}),}),+ ...getAnnotateWithChatGPTMenuItems({+ id: 'annotate-effect-property',+ getTarget: () => annotationTarget.current,+ initialComment: markupSkillAvailable ? '$remotion-markup' : null,+ metadata: {+ layer: nodePath.nodePath.join('.'),+ layerKeys: JSON.stringify(nodePath.sequenceKeys),+ effectIndex: field.effectIndex,+ effectKeys: JSON.stringify(+ nodePath.effectKeys[field.effectIndex] ?? [],+ ),+ property: field.key,+ source:+ formatFileLocation({+ location: validatedLocation,+ root: window.remotion_cwd,+ }) ?? validatedLocation.source,+ },+ }),{type: 'divider',id: 'copy-context-for-agents-divider',@@ -609,7 +641,16 @@ export const TimelineEffectPropItem: React.FC<{value: 'reset-effect-field',},];- }, [canShowReset, field.key, onReset, selection, validatedLocation]);+ }, [+ canShowReset,+ field.effectIndex,+ field.key,+ markupSkillAvailable,+ nodePath,+ onReset,+ selection,+ validatedLocation,+ ]);const onPropertyDoubleClick = useCallback<React.MouseEventHandler<HTMLDivElement>@@ -662,5 +703,9 @@ export const TimelineEffectPropItem: React.FC<{</TimelineRowChrome>);- return <ContextMenu getItems={getContextMenuItems}>{row}</ContextMenu>;+ return (+ <ContextMenu ref={annotationTarget} getItems={getContextMenuItems}>+ {row}+ </ContextMenu>+ );};diff --git a/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts b/packages/studio/src/components/Timeline/get-sequence-context-menu-items.tsindex 575984df30..7fe1d72b2d 100644--- a/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts+++ b/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts@@ -8,13 +8,14 @@ import type {ResolvedStackLocation, TSequence} from 'remotion';import {NoReactInternals} from 'remotion/no-react';import {copyText} from '../../helpers/copy-text';import {formatContextForAgents} from '../../helpers/format-file-location';-import {getCodexAnnotation} from '../../helpers/get-codex-annotation';import {getDefaultOpenInTarget,getGitSourceName,openGitSource,} from '../../helpers/get-git-menu-item';+import {getVisibleAnnotationTarget} from '../../helpers/request-codex-annotation';import {getSequenceAnnotationMetadata} from '../../helpers/sequence-annotation';+import {getAnnotateWithChatGPTMenuItems} from '../get-annotate-with-chatgpt-menu-item';import {getOpenInMenuItems} from '../get-open-in-menu-items';import type {ComboboxValue} from '../NewComposition/ComboBox';import {showNotification} from '../Notifications/NotificationCenter';@@ -347,69 +348,19 @@ export const getSequenceContextMenuItems = (}: null,getCopyContextForAgentsMenuItem({contextForAgents}),- getCodexAnnotation() !== null- ? {- type: 'item' as const,- id: 'annotate-layer',- keyHint: null,- label: 'Annotate with ChatGPT',- leftItem: null,- disabled: false,- onClick: () => {- const target = [- sequence.refForOutline?.current,- contextMenuTarget,- ].find((element) => {- if (!element?.isConnected) {- return false;- }-- const rect = element.getBoundingClientRect();- return (- rect.width > 0 &&- rect.height > 0 &&- rect.bottom > 0 &&- rect.right > 0 &&- rect.top < window.innerHeight &&- rect.left < window.innerWidth- );- });- if (!target) {- showNotification(- 'The annotation target is no longer visible',- 3000,- );- return;- }-- const annotation = getCodexAnnotation();- if (annotation === null) {- showNotification('The annotation editor is unavailable', 3000);- return;- }-- try {- const result = annotation.request(target, {- metadata: getSequenceAnnotationMetadata({- sequence,- location: originalLocation,- }),- });- if (!result?.accepted) {- showNotification('Could not open the annotation editor', 3000);- }- } catch (err) {- showNotification(- `Could not open the annotation editor: ${(err as Error).message}`,- 3000,- );- }- },- quickSwitcherLabel: null,- subMenu: null,- value: 'annotate-layer',- }- : null,+ ...getAnnotateWithChatGPTMenuItems({+ id: 'annotate-layer',+ getTarget: () =>+ getVisibleAnnotationTarget([+ sequence.refForOutline?.current,+ contextMenuTarget,+ ]),+ initialComment: null,+ metadata: getSequenceAnnotationMetadata({+ sequence,+ location: originalLocation,+ }),+ }),assetLinkInfo? {type: 'item' as const,diff --git a/packages/studio/src/components/WrapRefactorModal.tsx b/packages/studio/src/components/WrapRefactorModal.tsxindex 794544e353..14b0307d6b 100644--- a/packages/studio/src/components/WrapRefactorModal.tsx+++ b/packages/studio/src/components/WrapRefactorModal.tsx@@ -1,10 +1,13 @@-import React, {useContext} from 'react';+import React, {useCallback, useContext, useRef} from 'react';import {getBrowserStudioOperations} from '../helpers/browser-studio-operations';import {StudioServerConnectionCtx} from '../helpers/client-id';import {LIGHT_TEXT} from '../helpers/colors';import {formatFileLocation} from '../helpers/format-file-location';+import {getCodexAnnotation} from '../helpers/get-codex-annotation';+import {requestCodexAnnotation} from '../helpers/request-codex-annotation';import type {ModalState} from '../state/modals';import {AgentPrompt} from './AgentPrompt';+import {ModalButton} from './ModalButton';import {getMaxModalHeight, getMaxModalWidth} from './ModalContainer';import {ModalHeader} from './ModalHeader';import {DismissableModal} from './NewComposition/DismissableModal';@@ -41,6 +44,15 @@ export const WrapRefactorModal: React.FC<{readonly state: State}> = ({}) => {const {previewServerState} = useContext(StudioServerConnectionCtx);const {error, remotionSkillsInfo} = useSettings();+ const annotationTarget = useRef<HTMLDivElement>(null);+ const markupSkill = remotionSkillsInfo?.skills.find(+ ({name}) => name === 'remotion-markup',+ );+ const markupSkillAvailable = Boolean(+ markupSkill?.installedInProject || markupSkill?.installedGlobally,+ );+ const canAnnotate =+ !window.remotion_isReadOnlyStudio && getCodexAnnotation() !== null;const isBrowserStudio = getBrowserStudioOperations() !== null;const canSuggestAgent =!isBrowserStudio &&@@ -59,18 +71,45 @@ export const WrapRefactorModal: React.FC<{readonly state: State}> = ({state.wrapper === 'HtmlInCanvasMotionBlur'? '@remotion/motion-blur': 'remotion';+ const onSendToChatGPT = useCallback(() => {+ requestCodexAnnotation({+ target: annotationTarget.current,+ initialComment: markupSkillAvailable+ ? '$remotion-markup Wrap this item.'+ : 'Wrap this item.',+ metadata: {+ action: 'Wrap item',+ wrapper: state.wrapper,+ import: wrapperImportPath,+ ...(state.displayName ? {selection: state.displayName} : {}),+ source: location,+ },+ });+ }, [+ location,+ markupSkillAvailable,+ state.displayName,+ state.wrapper,+ wrapperImportPath,+ ]);return (<DismissableModal panelStyle={panelStyle}><ModalHeader title="Wrap item" />- <div style={container}>+ <div ref={annotationTarget} style={container}><div style={text}>Studio could not safely wrap this item in{` <${state.wrapper}>`} with an automated code change.</div>- {canSuggestAgent ? (+ {canSuggestAgent || canAnnotate ? (<AgentPrompt- action={null}+ action={+ canAnnotate ? (+ <ModalButton size="compact" onClick={onSendToChatGPT}>+ Send to ChatGPT+ </ModalButton>+ ) : null+ }availableText="You can wrap it using an agent:"promptDetails={` Wrap ${target} in <${state.wrapper}> from '${wrapperImportPath}'`}skillId="remotion-markup"diff --git a/packages/studio/src/components/composition-menu-items.ts b/packages/studio/src/components/composition-menu-items.tsindex 86d3e8d5ca..18951bd458 100644--- a/packages/studio/src/components/composition-menu-items.ts+++ b/packages/studio/src/components/composition-menu-items.ts@@ -20,7 +20,9 @@ import {openOriginalPositionInEditor,} from '../helpers/open-in-editor';import type {PreviewServerConnectionState} from '../helpers/preview-server-events';+import {getVisibleAnnotationTarget} from '../helpers/request-codex-annotation';import type {ModalState} from '../state/modals';+import {getAnnotateWithChatGPTMenuItems} from './get-annotate-with-chatgpt-menu-item';import {getOpenInMenuItems} from './get-open-in-menu-items';import type {ComboboxValue} from './NewComposition/ComboBox';import {showNotification} from './Notifications/NotificationCenter';@@ -432,6 +434,25 @@ export const getCompositionMenuItems = ({value: 'copy-context-for-agents',disabled: !contextForAgents,},+ ...(composition+ ? getAnnotateWithChatGPTMenuItems({+ id: 'annotate-composition',+ getTarget: () =>+ getVisibleAnnotationTarget(+ Array.from(document.querySelectorAll('[data-compname]'))+ .reverse()+ .filter(+ (element) =>+ element.getAttribute('data-compname') === composition.id,+ ),+ ),+ initialComment: null,+ metadata: {+ composition: composition.id,+ ...(fileLocation ? {source: fileLocation} : {}),+ },+ })+ : []),{id: 'copy-file-location',keyHint: null,diff --git a/packages/studio/src/components/get-annotate-with-chatgpt-menu-item.ts b/packages/studio/src/components/get-annotate-with-chatgpt-menu-item.tsnew file mode 100644index 0000000000..45095d52c8--- /dev/null+++ b/packages/studio/src/components/get-annotate-with-chatgpt-menu-item.ts@@ -0,0 +1,36 @@+import {getCodexAnnotation} from '../helpers/get-codex-annotation';+import {requestCodexAnnotation} from '../helpers/request-codex-annotation';+import type {ComboboxValue} from './NewComposition/ComboBox';++export const getAnnotateWithChatGPTMenuItems = ({+ id,+ getTarget,+ initialComment,+ metadata,+}: {+ readonly id: string;+ readonly getTarget: () => Element | null;+ readonly initialComment: string | null;+ readonly metadata: Record<string, string | number>;+}): ComboboxValue[] => {+ if (getCodexAnnotation() === null) {+ return [];+ }++ return [+ {+ type: 'item',+ id,+ keyHint: null,+ label: 'Annotate with ChatGPT',+ leftItem: null,+ disabled: false,+ onClick: () => {+ requestCodexAnnotation({target: getTarget(), initialComment, metadata});+ },+ quickSwitcherLabel: null,+ subMenu: null,+ value: id,+ },+ ];+};diff --git a/packages/studio/src/error-overlay/remotion-overlay/ErrorDisplay.tsx b/packages/studio/src/error-overlay/remotion-overlay/ErrorDisplay.tsxindex 494d4b5c19..64b2f68045 100644--- a/packages/studio/src/error-overlay/remotion-overlay/ErrorDisplay.tsx+++ b/packages/studio/src/error-overlay/remotion-overlay/ErrorDisplay.tsx@@ -1,9 +1,11 @@import {getLocationFromBuildError} from '@remotion/studio-shared';-import React, {useContext, useMemo} from 'react';+import React, {useCallback, useContext, useMemo, useRef} from 'react';import {MediaPlaybackError} from 'remotion';import {CodingAgentButton} from '../../components/CodingAgentButton';import {Spacing} from '../../components/layout';import {HORIZONTAL_SCROLLBAR_CLASSNAME} from '../../components/Menu/is-menu-item';+import {ModalButton} from '../../components/ModalButton';+import {useSettings} from '../../components/SettingsContext';import {useEditorOpening} from '../../components/use-default-editor-info';import {StudioServerConnectionCtx} from '../../helpers/client-id';import {@@ -11,6 +13,9 @@ import {WHITE,WHITE_ALPHA_60,} from '../../helpers/colors';+import {formatFileLocation} from '../../helpers/format-file-location';+import {getCodexAnnotation} from '../../helpers/get-codex-annotation';+import {requestCodexAnnotation} from '../../helpers/request-codex-annotation';import type {ErrorRecord} from '../react-overlay/listen-to-runtime-errors';import {AskOnDiscord} from './AskOnDiscord';import {CalculateMetadataErrorExplainer} from './CalculateMetadataErrorExplainer';@@ -92,6 +97,28 @@ export const ErrorDisplay: React.FC<{symbolicationFailure,}) => {const {previewServerState} = useContext(StudioServerConnectionCtx);+ const annotationTarget = useRef<HTMLDivElement>(null);+ const {remotionSkillsInfo} = useSettings();+ const canAnnotate =+ !window.remotion_isReadOnlyStudio && getCodexAnnotation() !== null;+ const buildErrorLocation = getLocationFromBuildError(display.error);+ const errorKind = calculateMetadata+ ? 'calculateMetadata'+ : display.error instanceof MediaPlaybackError+ ? 'media playback'+ : buildErrorLocation+ ? 'compilation'+ : 'runtime';+ const skillId =+ errorKind === 'media playback'+ ? 'remotion-multimedia'+ : errorKind === 'runtime'+ ? 'remotion-markup'+ : null;+ const skill = remotionSkillsInfo?.skills.find(({name}) => name === skillId);+ const skillAvailable = Boolean(+ skill?.installedInProject || skill?.installedGlobally,+ );const {canOpenInEditor, defaultEditorId, defaultEditorName} =useEditorOpening(previewServerState.type === 'connected');const stackFrames = useMemo(() => {@@ -149,9 +176,53 @@ export const ErrorDisplay: React.FC<{return display.error.stack || header;}, [display.error, message, stackFrames]);const fixWithAgentPrompt = `Fix this error in my Remotion project:\n\n${errorTextForCopy}`;+ const onSendToChatGPT = useCallback(() => {+ const frame = stackFrames[0];+ const location = frame?.originalFileName+ ? {+ source: frame.originalFileName,+ line: frame.originalLineNumber,+ column: frame.originalColumnNumber,+ }+ : buildErrorLocation+ ? {+ source: buildErrorLocation.fileName,+ line: buildErrorLocation.lineNumber,+ column: buildErrorLocation.columnNumber,+ }+ : null;+ const source = formatFileLocation({+ location,+ root: window.remotion_cwd,+ });+ requestCodexAnnotation({+ target: annotationTarget.current,+ initialComment: `${skillAvailable ? `$${skillId} ` : ''}Fix this ${errorKind} error.`,+ metadata: {+ kind: errorKind,+ error: display.error.name,+ message: message.slice(0, 256),+ ...(source ? {source} : {}),+ ...(location?.column !== null && location?.column !== undefined+ ? {column: location.column}+ : {}),+ ...(display.error instanceof MediaPlaybackError+ ? {media: display.error.src}+ : {}),+ },+ });+ }, [+ buildErrorLocation,+ display.error,+ errorKind,+ message,+ skillAvailable,+ skillId,+ stackFrames,+ ]);return (- <div>+ <div ref={annotationTarget}><ErrorTitlesymbolicating={false}name={display.error.name}@@ -190,6 +261,12 @@ export const ErrorDisplay: React.FC<{size="default"style={codingAgentButton}/>+ {canAnnotate ? (+ <>+ <ModalButton onClick={onSendToChatGPT}>Send to ChatGPT</ModalButton>+ <div style={spacer} />+ </>+ ) : null}<CopyStackTrace errorText={errorTextForCopy} /><div style={spacer} /><SearchGithubIssues@@ -257,6 +334,15 @@ export const ErrorDisplay: React.FC<{) : null}</>)}+ {canAnnotate && stackFrames.length > 0 && display.error.stack ? (+ <pre+ aria-label="Complete stack trace"+ className={HORIZONTAL_SCROLLBAR_CLASSNAME}+ style={rawStack}+ >+ {display.error.stack}+ </pre>+ ) : null}</div>);};diff --git a/packages/studio/src/helpers/get-codex-annotation.ts b/packages/studio/src/helpers/get-codex-annotation.tsindex 13b12b20bc..767fe6364f 100644--- a/packages/studio/src/helpers/get-codex-annotation.ts+++ b/packages/studio/src/helpers/get-codex-annotation.ts@@ -1,9 +1,10 @@type CodexAnnotation = {readonly request: (target: Element,- options:- | {metadata: Record<string, string | number>}- | {initialComment: string},+ options: {+ metadata?: Record<string, string | number>;+ initialComment?: string;+ },) => {accepted: boolean};};diff --git a/packages/studio/src/helpers/request-codex-annotation.ts b/packages/studio/src/helpers/request-codex-annotation.tsindex 49b509d41e..0e74e72406 100644--- a/packages/studio/src/helpers/request-codex-annotation.ts+++ b/packages/studio/src/helpers/request-codex-annotation.ts@@ -1,25 +1,76 @@import {showNotification} from '../components/Notifications/NotificationCenter';import {getCodexAnnotation} from './get-codex-annotation';+export const getVisibleAnnotationTarget = (+ targets: readonly (Element | null | undefined)[],+): Element | null => {+ return (+ targets.find((element) => {+ if (!element?.isConnected) {+ return false;+ }++ const rect = element.getBoundingClientRect();+ const style = window.getComputedStyle(element);+ return (+ style.visibility !== 'hidden' &&+ style.display !== 'none' &&+ rect.width > 0 &&+ rect.height > 0 &&+ rect.bottom > 0 &&+ rect.right > 0 &&+ rect.top < window.innerHeight &&+ rect.left < window.innerWidth+ );+ }) ?? null+ );+};+export const requestCodexAnnotation = ({target,initialComment,metadata,}: {readonly target: Element | null;- readonly initialComment: string;+ readonly initialComment: string | null;readonly metadata: Record<string, string | number> | null;}): boolean => {const annotation = getCodexAnnotation();- if (annotation === null || target === null) {+ if (annotation === null) {showNotification('The annotation editor is unavailable', 3000);return false;}+ if (target === null || getVisibleAnnotationTarget([target]) === null) {+ showNotification('The annotation target is no longer visible', 3000);+ return false;+ }++ // Omit oversized facts instead of truncating source paths or identifiers.+ const boundedMetadata: Record<string, string | number> = {};+ for (const [key, value] of Object.entries(metadata ?? {})) {+ if (+ Object.keys(boundedMetadata).length >= 6 ||+ key.length > 64 ||+ !/^[A-Za-z][A-Za-z0-9 _-]*$/.test(key) ||+ key.includes(' ') ||+ (typeof value === 'string' ? value.length > 256 : !Number.isFinite(value))+ ) {+ continue;+ }++ const next = {...boundedMetadata, [key]: value};+ if (new TextEncoder().encode(JSON.stringify(next)).length <= 2048) {+ boundedMetadata[key] = value;+ }+ }+try {const result = annotation.request(target, {- initialComment,- ...(metadata === null ? {} : {metadata}),+ ...(initialComment === null ? {} : {initialComment}),+ ...(Object.keys(boundedMetadata).length === 0+ ? {}+ : {metadata: boundedMetadata}),});if (result?.accepted) {return true;