Skip to content

🎥 Make videos programmatically with React

`@remotion/studio`: Add layer annotations with ChatGPT

Jonny Burgerf27a9687b61e
diff --git a/packages/studio/src/components/SelectedOutlineElement.tsx b/packages/studio/src/components/SelectedOutlineElement.tsx
index 0e2c02fcbf..96b2c84906 100644
--- a/packages/studio/src/components/SelectedOutlineElement.tsx
+++ b/packages/studio/src/components/SelectedOutlineElement.tsx
@@ -13,6 +13,7 @@ import {
openInCodingAgent as launchCodingAgent,
openOriginalPositionInEditor,
} from '../helpers/open-in-editor';
+import {getSequenceAnnotationAttributes} from '../helpers/sequence-annotation';
import {SetSelectedModalContext} from '../state/modals';
import {Transform3DModeStateContext} from '../state/transform-3d-mode';
import {useConfirmationDialog} from './ConfirmationDialog';
@@ -45,6 +46,7 @@ import {
} from './Timeline/TimelineSelection';
import {getOriginalLocationFromStack} from './Timeline/TimelineStack/get-stack';
import {useDeleteTimelineItems} from './Timeline/use-delete-timeline-items';
+import {useResolvedStack} from './Timeline/use-resolved-stack';
import {
useDefaultCodingAgentInfo,
useEditorOpening,
@@ -122,6 +124,9 @@ const SelectedOutlineElementUnmemoized: React.FC<
getLatestTargetByKey,
layoutTarget,
});
+ const annotationLocation = useResolvedStack(
+ layoutTarget?.sequence.getStack() ?? null,
+ );
const resolveOriginalLocation = React.useCallback(
async (resolveTarget: SelectedOutlineTarget) => {
@@ -464,6 +469,15 @@ const SelectedOutlineElementUnmemoized: React.FC<
return (
<>
<SelectedOutlinePolygon
+ annotationAttributes={
+ layoutTarget
+ ? getSequenceAnnotationAttributes({
+ sequence: layoutTarget.sequence,
+ location: annotationLocation,
+ surface: 'outline',
+ })
+ : null
+ }
compositionHeight={compositionHeight}
compositionWidth={compositionWidth}
containsSelection={layoutTarget?.containsSelection === true}
diff --git a/packages/studio/src/components/SelectedOutlinePolygon.tsx b/packages/studio/src/components/SelectedOutlinePolygon.tsx
index e98b9e6ff0..19096758a4 100644
--- a/packages/studio/src/components/SelectedOutlinePolygon.tsx
+++ b/packages/studio/src/components/SelectedOutlinePolygon.tsx
@@ -10,6 +10,7 @@ import {
import {createDragAwareDoubleClickTracker} from '../helpers/drag-aware-double-click';
import {isStudioInteractivityEnabled} from '../helpers/interactivity-enabled';
import {isMac} from '../helpers/is-mac';
+import type {getSequenceAnnotationAttributes} from '../helpers/sequence-annotation';
import {EditorShowGuidesContext} from '../state/editor-guides';
import {EditorSnappingContext} from '../state/editor-snapping';
import {
@@ -60,6 +61,9 @@ export const SELECTED_OUTLINE_KEY_ATTR =
'data-remotion-studio-selected-outline-key';
const SelectedOutlinePolygonUnmemoized: React.FC<{
+ readonly annotationAttributes: ReturnType<
+ typeof getSequenceAnnotationAttributes
+ > | null;
readonly compositionHeight: number;
readonly compositionWidth: number;
readonly containsSelection: boolean;
@@ -90,6 +94,7 @@ const SelectedOutlinePolygonUnmemoized: React.FC<{
readonly showSelectedOutline: boolean;
readonly translateWithCommandKey: boolean;
}> = ({
+ annotationAttributes,
compositionHeight,
compositionWidth,
containsSelection,
@@ -440,6 +445,7 @@ const SelectedOutlinePolygonUnmemoized: React.FC<{
return (
<CanvasOutlinePolygon
ref={polygonRef}
+ {...annotationAttributes}
{...{
[PREVENT_CLEAR_SELECTION_ON_POINTER_DOWN_ATTR]: 'true',
[SELECTED_OUTLINE_KEY_ATTR]: outline.key,
diff --git a/packages/studio/src/components/SelectedOutlineRenderer.tsx b/packages/studio/src/components/SelectedOutlineRenderer.tsx
index 9896d309ab..0048b59a6a 100644
--- a/packages/studio/src/components/SelectedOutlineRenderer.tsx
+++ b/packages/studio/src/components/SelectedOutlineRenderer.tsx
@@ -230,6 +230,7 @@ const SelectedOutlineRendererUnmemoized: React.FC<{
return (
<svg
ref={overlayRef}
+ {...{'oai-annotation-container': ''}}
style={outlineContainer}
width="100%"
height="100%"
diff --git a/packages/studio/src/components/Timeline/TimelineList.tsx b/packages/studio/src/components/Timeline/TimelineList.tsx
index 4d85afaa7f..e71ea00479 100644
--- a/packages/studio/src/components/Timeline/TimelineList.tsx
+++ b/packages/studio/src/components/Timeline/TimelineList.tsx
@@ -47,7 +47,10 @@ export const TimelineList: React.FC = () => {
const {rows, tracksEnd, virtualItems} = useTimelineVirtualization();
return (
- <div style={{...container, height: tracksEnd}}>
+ <div
+ style={{...container, height: tracksEnd}}
+ {...{'oai-annotation-container': ''}}
+ >
<style>{`.remotion-timeline-sequence-name-measure::before {
content: attr(data-name);
display: block;
diff --git a/packages/studio/src/components/Timeline/TimelineSequence.tsx b/packages/studio/src/components/Timeline/TimelineSequence.tsx
index 21c04b8645..5c4c6af0c5 100644
--- a/packages/studio/src/components/Timeline/TimelineSequence.tsx
+++ b/packages/studio/src/components/Timeline/TimelineSequence.tsx
@@ -1,5 +1,5 @@
-import {CanvasInternals} from '@remotion/sdk';
import type {TimelineTrackData} from '@remotion/sdk';
+import {CanvasInternals} from '@remotion/sdk';
import {stringifySequenceSubscriptionKey} from '@remotion/studio-shared';
import type {WaveformVolume} from '@remotion/timeline-utils';
import React, {
@@ -12,6 +12,7 @@ import React, {
} from 'react';
import type {
_InternalTypes,
+ ResolvedStackLocation,
SequencePropsSubscriptionKey,
TSequence,
} from 'remotion';
@@ -38,6 +39,7 @@ import {getTimelineSequenceLayout} from '../../helpers/get-timeline-sequence-lay
import type {SequenceNodePathInfo} from '../../helpers/get-timeline-sequence-sort-key';
import {isStudioInteractivityEnabled} from '../../helpers/interactivity-enabled';
import {isVideoWithLastFrameHold} from '../../helpers/is-video-with-last-frame-hold';
+import {getSequenceAnnotationAttributes} from '../../helpers/sequence-annotation';
import {
getTimelineLayerHeight,
TIMELINE_LAYER_HEIGHT_AUDIO,
@@ -295,6 +297,7 @@ const TimelineSequenceNegativeStart = React.memo(
const TimelineSequenceCurrentFrame: React.FC<{
readonly s: TSequence;
+ readonly annotationLocation: ResolvedStackLocation | null;
readonly activeTrimEdge: 'left' | 'right' | null;
readonly displayDurationInFrames: number;
readonly premount: {readonly left: number; readonly width: number} | null;
@@ -324,6 +327,7 @@ const TimelineSequenceCurrentFrame: React.FC<{
}> = ({
s,
activeTrimEdge,
+ annotationLocation,
displayDurationInFrames,
premount,
postmount,
@@ -506,6 +510,11 @@ const TimelineSequenceCurrentFrame: React.FC<{
<div
ref={ref}
role="group"
+ {...getSequenceAnnotationAttributes({
+ sequence: s,
+ location: annotationLocation,
+ surface: 'layer',
+ })}
{...{[TIMELINE_MARQUEE_ITEM_ATTR]: true}}
style={actualStyle}
aria-label={s.displayName}
@@ -1036,135 +1045,145 @@ const TimelineSequenceInner: React.FC<{
setPropStatuses,
validatedLocation?.source,
]);
- const getContextMenuItems = useCallback(() => {
- if (assetContextMenu !== null) {
- return assetContextMenu;
- }
+ const getContextMenuItems = useCallback(
+ (event: MouseEvent) => {
+ const contextMenuTarget =
+ event.target instanceof Element
+ ? event.target.closest<HTMLElement>(`[${TIMELINE_MARQUEE_ITEM_ATTR}]`)
+ : null;
+ if (assetContextMenu !== null) {
+ return assetContextMenu;
+ }
- if (selectable && !selected) {
- onSelect({shiftKey: false, toggleKey: false});
- }
+ if (selectable && !selected) {
+ onSelect({shiftKey: false, toggleKey: false});
+ }
- if (selectedSequenceNodePathInfos !== null) {
- return getMultiSequenceContextMenuItems({
- getContextForAgents: () =>
- getSequencesContextForAgents({
- nodePathInfos: selectedSequenceNodePathInfos,
- overrideIdsToNodePaths: overrideIdToNodePathMappingsRef.current,
- sequences,
- }),
- deleteDisabled: !previewInteractive,
- duplicateDisabled: !previewInteractive,
- splitDisabled: !previewInteractive,
- onDeleteSelectedSequences,
- onDuplicateSelectedSequences,
- onSplitSelectedSequences,
- });
- }
+ if (selectedSequenceNodePathInfos !== null) {
+ return getMultiSequenceContextMenuItems({
+ getContextForAgents: () =>
+ getSequencesContextForAgents({
+ nodePathInfos: selectedSequenceNodePathInfos,
+ overrideIdsToNodePaths: overrideIdToNodePathMappingsRef.current,
+ sequences,
+ }),
+ deleteDisabled: !previewInteractive,
+ duplicateDisabled: !previewInteractive,
+ splitDisabled: !previewInteractive,
+ onDeleteSelectedSequences,
+ onDuplicateSelectedSequences,
+ onSplitSelectedSequences,
+ });
+ }
- const splitMenuItem = getSequenceSplitMenuItem({
- nodePathInfo,
- sequence: s,
- propStatuses: propStatusesForOverride,
- splitFrame: getCurrentFrame(),
- keyframeDisplayOffset,
- keyframePlaybackRate,
- canEditSource: previewInteractive && Boolean(validatedLocation?.source),
- hasMultipleSelection: selected && selectedItems.length > 1,
- });
+ const splitMenuItem = getSequenceSplitMenuItem({
+ nodePathInfo,
+ sequence: s,
+ propStatuses: propStatusesForOverride,
+ splitFrame: getCurrentFrame(),
+ keyframeDisplayOffset,
+ keyframePlaybackRate,
+ canEditSource: previewInteractive && Boolean(validatedLocation?.source),
+ hasMultipleSelection: selected && selectedItems.length > 1,
+ });
- const freezeFrameMenuItem = getSequenceFreezeFrameMenuItem({
- clientId:
- previewInteractive && previewServerState.type === 'connected'
- ? previewServerState.clientId
- : null,
- nodePath,
- propStatusesForOverride,
- sequence: s,
- sequenceFrameOffset,
- setPropStatuses,
- timelinePosition: getCurrentFrame(),
- validatedSource: validatedLocation?.source ?? null,
- });
+ const freezeFrameMenuItem = getSequenceFreezeFrameMenuItem({
+ clientId:
+ previewInteractive && previewServerState.type === 'connected'
+ ? previewServerState.clientId
+ : null,
+ nodePath,
+ propStatusesForOverride,
+ sequence: s,
+ sequenceFrameOffset,
+ setPropStatuses,
+ timelinePosition: getCurrentFrame(),
+ validatedSource: validatedLocation?.source ?? null,
+ });
- return getSequenceContextMenuItems({
- assetLinkInfo: mediaSrc ? getTimelineAssetLinkInfo(mediaSrc) : null,
+ return getSequenceContextMenuItems(
+ {
+ assetLinkInfo: mediaSrc ? getTimelineAssetLinkInfo(mediaSrc) : null,
+ canOpenInEditor,
+ codingAgentInfo,
+ copyImageElement: null,
+ deleteDisabled,
+ disableInteractivityDisabled,
+ duplicateDisabled,
+ editorInfo,
+ includeSourceEditItems: isStudioInteractivityEnabled(),
+ isProgrammaticallyDuplicated,
+ onConfigureApps: canConfigureApps
+ ? () => {
+ setSelectedModal({
+ type: 'settings',
+ initialStudioPane: null,
+ initialTab: 'apps',
+ initialPublicLicenseKey:
+ window.remotion_renderDefaults?.publicLicenseKey ?? null,
+ });
+ }
+ : null,
+ onDeleteSequenceFromSource,
+ onDisableSequenceInteractivity,
+ onDuplicateSequenceFromSource,
+ openInCodingAgent,
+ openInEditor,
+ originalLocation,
+ selectAsset,
+ sequence: s,
+ sourceActions: isStudioInteractivityEnabled()
+ ? [
+ ...(splitMenuItem ? [splitMenuItem] : []),
+ ...(freezeFrameMenuItem ? [freezeFrameMenuItem] : []),
+ ]
+ : [],
+ },
+ contextMenuTarget,
+ );
+ },
+ [
+ assetContextMenu,
canOpenInEditor,
+ canConfigureApps,
codingAgentInfo,
- copyImageElement: null,
deleteDisabled,
disableInteractivityDisabled,
duplicateDisabled,
editorInfo,
- includeSourceEditItems: isStudioInteractivityEnabled(),
isProgrammaticallyDuplicated,
- onConfigureApps: canConfigureApps
- ? () => {
- setSelectedModal({
- type: 'settings',
- initialStudioPane: null,
- initialTab: 'apps',
- initialPublicLicenseKey:
- window.remotion_renderDefaults?.publicLicenseKey ?? null,
- });
- }
- : null,
+ keyframeDisplayOffset,
+ keyframePlaybackRate,
+ mediaSrc,
+ nodePath,
+ nodePathInfo,
+ onSelect,
onDeleteSequenceFromSource,
+ onDeleteSelectedSequences,
onDisableSequenceInteractivity,
onDuplicateSequenceFromSource,
+ onDuplicateSelectedSequences,
+ onSplitSelectedSequences,
openInCodingAgent,
openInEditor,
originalLocation,
+ overrideIdToNodePathMappingsRef,
+ previewInteractive,
+ previewServerState,
+ propStatusesForOverride,
+ s,
selectAsset,
- sequence: s,
- sourceActions: isStudioInteractivityEnabled()
- ? [
- ...(splitMenuItem ? [splitMenuItem] : []),
- ...(freezeFrameMenuItem ? [freezeFrameMenuItem] : []),
- ]
- : [],
- });
- }, [
- assetContextMenu,
- canOpenInEditor,
- canConfigureApps,
- codingAgentInfo,
- deleteDisabled,
- disableInteractivityDisabled,
- duplicateDisabled,
- editorInfo,
- isProgrammaticallyDuplicated,
- keyframeDisplayOffset,
- keyframePlaybackRate,
- mediaSrc,
- nodePath,
- nodePathInfo,
- onSelect,
- onDeleteSequenceFromSource,
- onDeleteSelectedSequences,
- onDisableSequenceInteractivity,
- onDuplicateSequenceFromSource,
- onDuplicateSelectedSequences,
- onSplitSelectedSequences,
- openInCodingAgent,
- openInEditor,
- originalLocation,
- overrideIdToNodePathMappingsRef,
- previewInteractive,
- previewServerState,
- propStatusesForOverride,
- s,
- selectAsset,
- selectable,
- selected,
- selectedItems.length,
- selectedSequenceNodePathInfos,
- sequenceFrameOffset,
- sequences,
- setPropStatuses,
- setSelectedModal,
- validatedLocation?.source,
- ]);
+ selectable,
+ selected,
+ selectedItems.length,
+ selectedSequenceNodePathInfos,
+ sequenceFrameOffset,
+ sequences,
+ setPropStatuses,
+ setSelectedModal,
+ validatedLocation?.source,
+ ],
+ );
const {frozenFrame} = s;
const {onPointerDown: onMoveDragPointerDown} = useTimelineSequenceFromDrag({
@@ -1462,6 +1481,7 @@ const TimelineSequenceInner: React.FC<{
const sequence = (
<TimelineSequenceCurrentFrame
s={s}
+ annotationLocation={originalLocation}
activeTrimEdge={activeTrimEdge}
displayDurationInFrames={displayDurationInFrames}
premount={visibleLayout.premount}
diff --git a/packages/studio/src/components/Timeline/TimelineSequenceItem.tsx b/packages/studio/src/components/Timeline/TimelineSequenceItem.tsx
index 3bff2ac619..2d5f0fe62c 100644
--- a/packages/studio/src/components/Timeline/TimelineSequenceItem.tsx
+++ b/packages/studio/src/components/Timeline/TimelineSequenceItem.tsx
@@ -36,6 +36,7 @@ import {
isPointerSessionRelease,
startDeferredCapturedPointerSession,
} from '../../helpers/pointer-session';
+import {getSequenceAnnotationAttributes} from '../../helpers/sequence-annotation';
import {getStudioKeyboardShortcutsEnabled} from '../../helpers/studio-runtime-config';
import {
getTimelineLayerHeight,
@@ -1236,215 +1237,225 @@ const TimelineSequenceItemInner: React.FC<{
);
}, [canRotate, nodePathInfo, selectItem, setManuallyEnabled]);
- const getContextMenuItems = useCallback(() => {
- if (assetContextMenu !== null) {
- return assetContextMenu;
- }
+ const getContextMenuItems = useCallback(
+ (event: MouseEvent) => {
+ const contextMenuTarget =
+ event.currentTarget instanceof HTMLElement ? event.currentTarget : null;
+ if (assetContextMenu !== null) {
+ return assetContextMenu;
+ }
- if (selectable && !selected) {
- onSelect({shiftKey: false, toggleKey: false});
- }
+ if (selectable && !selected) {
+ onSelect({shiftKey: false, toggleKey: false});
+ }
- if (selectedSequenceNodePathInfos !== null) {
- return getMultiSequenceContextMenuItems({
- getContextForAgents: () =>
- getSequencesContextForAgents({
- nodePathInfos: selectedSequenceNodePathInfos,
- overrideIdsToNodePaths: overrideIdToNodePathMappingsRef.current,
- sequences: sequencesRef.current,
- }),
- deleteDisabled: !previewInteractive,
- duplicateDisabled: !previewInteractive,
- splitDisabled: !previewInteractive,
- onDeleteSelectedSequences,
- onDuplicateSelectedSequences,
- onSplitSelectedSequences,
- });
- }
+ if (selectedSequenceNodePathInfos !== null) {
+ return getMultiSequenceContextMenuItems({
+ getContextForAgents: () =>
+ getSequencesContextForAgents({
+ nodePathInfos: selectedSequenceNodePathInfos,
+ overrideIdsToNodePaths: overrideIdToNodePathMappingsRef.current,
+ sequences: sequencesRef.current,
+ }),
+ deleteDisabled: !previewInteractive,
+ duplicateDisabled: !previewInteractive,
+ splitDisabled: !previewInteractive,
+ onDeleteSelectedSequences,
+ onDuplicateSelectedSequences,
+ onSplitSelectedSequences,
+ });
+ }
- const splitMenuItem = getSequenceSplitMenuItem({
- nodePathInfo,
- sequence,
- propStatuses: propStatusesForOverride,
- splitFrame: getCurrentFrame(),
- keyframeDisplayOffset,
- keyframePlaybackRate,
- canEditSource: previewInteractive && Boolean(validatedLocation?.source),
- hasMultipleSelection: selected && selectedItems.length > 1,
- });
+ const splitMenuItem = getSequenceSplitMenuItem({
+ nodePathInfo,
+ sequence,
+ propStatuses: propStatusesForOverride,
+ splitFrame: getCurrentFrame(),
+ keyframeDisplayOffset,
+ keyframePlaybackRate,
+ canEditSource: previewInteractive && Boolean(validatedLocation?.source),
+ hasMultipleSelection: selected && selectedItems.length > 1,
+ });
- const freezeFrameMenuItem = getSequenceFreezeFrameMenuItem({
- clientId:
- previewInteractive && previewServerState.type === 'connected'
- ? previewServerState.clientId
- : null,
- nodePath,
- propStatusesForOverride,
- sequence,
- sequenceFrameOffset,
- setPropStatuses,
- timelinePosition: getCurrentFrame(),
- validatedSource: validatedLocation?.source ?? null,
- });
+ const freezeFrameMenuItem = getSequenceFreezeFrameMenuItem({
+ clientId:
+ previewInteractive && previewServerState.type === 'connected'
+ ? previewServerState.clientId
+ : null,
+ nodePath,
+ propStatusesForOverride,
+ sequence,
+ sequenceFrameOffset,
+ setPropStatuses,
+ timelinePosition: getCurrentFrame(),
+ validatedSource: validatedLocation?.source ?? null,
+ });
- return getSequenceContextMenuItems({
- assetLinkInfo: mediaSrc ? getTimelineAssetLinkInfo(mediaSrc) : null,
+ return getSequenceContextMenuItems(
+ {
+ assetLinkInfo: mediaSrc ? getTimelineAssetLinkInfo(mediaSrc) : null,
+ canOpenInEditor,
+ codingAgentInfo,
+ copyImageElement: null,
+ deleteDisabled,
+ disableInteractivityDisabled,
+ duplicateDisabled,
+ editorInfo,
+ includeSourceEditItems: isStudioInteractivityEnabled(),
+ isProgrammaticallyDuplicated,
+ onConfigureApps: canConfigureApps
+ ? () => {
+ setSelectedModal({
+ type: 'settings',
+ initialStudioPane: null,
+ initialTab: 'apps',
+ initialPublicLicenseKey:
+ window.remotion_renderDefaults?.publicLicenseKey ?? null,
+ });
+ }
+ : null,
+ onDeleteSequenceFromSource,
+ onDisableSequenceInteractivity,
+ onDuplicateSequenceFromSource,
+ openInCodingAgent,
+ openInEditor,
+ originalLocation,
+ selectAsset,
+ sequence,
+ sourceActions: isStudioInteractivityEnabled()
+ ? [
+ ...(nodePathInfo?.supportsEffects
+ ? [
+ {
+ type: 'item' as const,
+ id: 'add-effect',
+ keyHint: null,
+ label: 'Add effect...',
+ leftItem: null,
+ disabled: !canAddEffect,
+ onClick: onAddEffect,
+ quickSwitcherLabel: null,
+ subMenu: null,
+ value: 'add-effect',
+ },
+ ]
+ : []),
+ ...(canCrop
+ ? [
+ {
+ type: 'item' as const,
+ id: 'crop',
+ keyHint: null,
+ label: isProgrammaticallyDuplicated
+ ? 'Crop all'
+ : 'Crop',
+ leftItem: null,
+ disabled: false,
+ onClick: onCrop,
+ quickSwitcherLabel: null,
+ subMenu: null,
+ value: 'crop',
+ },
+ ]
+ : []),
+ ...(canRotate
+ ? [
+ {
+ type: 'item' as const,
+ id: 'rotate',
+ keyHint: null,
+ label: isProgrammaticallyDuplicated
+ ? 'Rotate all'
+ : 'Rotate',
+ leftItem: null,
+ disabled: false,
+ onClick: onRotate,
+ quickSwitcherLabel: null,
+ subMenu: null,
+ value: 'rotate',
+ },
+ ]
+ : []),
+ ...(canCrop || canRotate
+ ? [
+ {
+ type: 'divider' as const,
+ id: 'transform-controls-divider',
+ },
+ ]
+ : []),
+ {
+ type: 'item' as const,
+ id: 'rename-sequence',
+ keyHint: null,
+ label: 'Rename...',
+ leftItem: null,
+ disabled: !canRenameThisSequence,
+ onClick: () => {
+ onRenameSequence();
+ },
+ quickSwitcherLabel: null,
+ subMenu: null,
+ value: 'rename-sequence',
+ },
+ ...(splitMenuItem ? [splitMenuItem] : []),
+ ...(freezeFrameMenuItem ? [freezeFrameMenuItem] : []),
+ ]
+ : [],
+ },
+ contextMenuTarget,
+ );
+ },
+ [
+ assetContextMenu,
+ canAddEffect,
+ canCrop,
+ canRotate,
+ canConfigureApps,
canOpenInEditor,
+ canRenameThisSequence,
codingAgentInfo,
- copyImageElement: null,
deleteDisabled,
disableInteractivityDisabled,
duplicateDisabled,
editorInfo,
- includeSourceEditItems: isStudioInteractivityEnabled(),
isProgrammaticallyDuplicated,
- onConfigureApps: canConfigureApps
- ? () => {
- setSelectedModal({
- type: 'settings',
- initialStudioPane: null,
- initialTab: 'apps',
- initialPublicLicenseKey:
- window.remotion_renderDefaults?.publicLicenseKey ?? null,
- });
- }
- : null,
+ keyframeDisplayOffset,
+ keyframePlaybackRate,
+ mediaSrc,
+ nodePath,
+ nodePathInfo,
+ onAddEffect,
+ onCrop,
+ onRotate,
onDeleteSequenceFromSource,
+ onDeleteSelectedSequences,
onDisableSequenceInteractivity,
onDuplicateSequenceFromSource,
+ onDuplicateSelectedSequences,
+ onSplitSelectedSequences,
+ onRenameSequence,
+ onSelect,
openInCodingAgent,
openInEditor,
originalLocation,
+ overrideIdToNodePathMappingsRef,
+ previewInteractive,
+ previewServerState,
+ propStatusesForOverride,
selectAsset,
+ selectable,
+ selected,
+ selectedItems.length,
+ selectedSequenceNodePathInfos,
sequence,
- sourceActions: isStudioInteractivityEnabled()
- ? [
- ...(nodePathInfo?.supportsEffects
- ? [
- {
- type: 'item' as const,
- id: 'add-effect',
- keyHint: null,
- label: 'Add effect...',
- leftItem: null,
- disabled: !canAddEffect,
- onClick: onAddEffect,
- quickSwitcherLabel: null,
- subMenu: null,
- value: 'add-effect',
- },
- ]
- : []),
- ...(canCrop
- ? [
- {
- type: 'item' as const,
- id: 'crop',
- keyHint: null,
- label: isProgrammaticallyDuplicated ? 'Crop all' : 'Crop',
- leftItem: null,
- disabled: false,
- onClick: onCrop,
- quickSwitcherLabel: null,
- subMenu: null,
- value: 'crop',
- },
- ]
- : []),
- ...(canRotate
- ? [
- {
- type: 'item' as const,
- id: 'rotate',
- keyHint: null,
- label: isProgrammaticallyDuplicated
- ? 'Rotate all'
- : 'Rotate',
- leftItem: null,
- disabled: false,
- onClick: onRotate,
- quickSwitcherLabel: null,
- subMenu: null,
- value: 'rotate',
- },
- ]
- : []),
- ...(canCrop || canRotate
- ? [
- {
- type: 'divider' as const,
- id: 'transform-controls-divider',
- },
- ]
- : []),
- {
- type: 'item' as const,
- id: 'rename-sequence',
- keyHint: null,
- label: 'Rename...',
- leftItem: null,
- disabled: !canRenameThisSequence,
- onClick: () => {
- onRenameSequence();
- },
- quickSwitcherLabel: null,
- subMenu: null,
- value: 'rename-sequence',
- },
- ...(splitMenuItem ? [splitMenuItem] : []),
- ...(freezeFrameMenuItem ? [freezeFrameMenuItem] : []),
- ]
- : [],
- });
- }, [
- assetContextMenu,
- canAddEffect,
- canCrop,
- canRotate,
- canConfigureApps,
- canOpenInEditor,
- canRenameThisSequence,
- codingAgentInfo,
- deleteDisabled,
- disableInteractivityDisabled,
- duplicateDisabled,
- editorInfo,
- isProgrammaticallyDuplicated,
- keyframeDisplayOffset,
- keyframePlaybackRate,
- mediaSrc,
- nodePath,
- nodePathInfo,
- onAddEffect,
- onCrop,
- onRotate,
- onDeleteSequenceFromSource,
- onDeleteSelectedSequences,
- onDisableSequenceInteractivity,
- onDuplicateSequenceFromSource,
- onDuplicateSelectedSequences,
- onSplitSelectedSequences,
- onRenameSequence,
- onSelect,
- openInCodingAgent,
- openInEditor,
- originalLocation,
- overrideIdToNodePathMappingsRef,
- previewInteractive,
- previewServerState,
- propStatusesForOverride,
- selectAsset,
- selectable,
- selected,
- selectedItems.length,
- selectedSequenceNodePathInfos,
- sequence,
- sequenceFrameOffset,
- sequencesRef,
- setSelectedModal,
- setPropStatuses,
- validatedLocation?.source,
- ]);
+ sequenceFrameOffset,
+ sequencesRef,
+ setSelectedModal,
+ setPropStatuses,
+ validatedLocation?.source,
+ ],
+ );
const canDropEffect =
canMutateEffects &&
nodePath !== null &&
@@ -1626,14 +1637,26 @@ const TimelineSequenceItemInner: React.FC<{
trackRow
);
+ const annotatedTrackRow = (
+ <div
+ {...getSequenceAnnotationAttributes({
+ sequence,
+ location: originalLocation,
+ surface: 'track',
+ })}
+ >
+ {reorderableTrackRow}
+ </div>
+ );
+
return (
<>
{previewConnected || window.remotion_isReadOnlyStudio ? (
<ContextMenu getItems={getContextMenuItems}>
- {reorderableTrackRow}
+ {annotatedTrackRow}
</ContextMenu>
) : (
- reorderableTrackRow
+ annotatedTrackRow
)}
{previewConnected &&
isStudioInteractivityEnabled() &&
diff --git a/packages/studio/src/components/Timeline/TimelineTrack.tsx b/packages/studio/src/components/Timeline/TimelineTrack.tsx
index 997a725fd3..38f799d9b8 100644
--- a/packages/studio/src/components/Timeline/TimelineTrack.tsx
+++ b/packages/studio/src/components/Timeline/TimelineTrack.tsx
@@ -1,6 +1,7 @@
import React, {useContext, useMemo} from 'react';
import {StudioServerConnectionCtx} from '../../helpers/client-id';
import type {TimelineTrackData} from '../../helpers/get-timeline-sequence-sort-key';
+import {getSequenceAnnotationAttributes} from '../../helpers/sequence-annotation';
import {
getTimelineLayerHeight,
TIMELINE_ITEM_BORDER_BOTTOM,
@@ -15,12 +16,14 @@ import {
} from './TimelineSelection';
import {TimelineSequence} from './TimelineSequence';
import {TimelineWidthContext} from './TimelineWidthProvider';
+import {useResolvedStack} from './use-resolved-stack';
const emptyConnectedCompositions = [] as const;
const TimelineTrackUnmemoized: React.FC<{
readonly track: TimelineTrackData;
}> = ({track}) => {
+ const annotationLocation = useResolvedStack(track.sequence.getStack());
const {getIsExpanded} = useContext(ExpandedTracksGetterContext);
const {previewServerState} = useContext(StudioServerConnectionCtx);
const previewServerConnected = previewServerState.type === 'connected';
@@ -52,7 +55,14 @@ const TimelineTrackUnmemoized: React.FC<{
return (
<div onPointerEnter={onPointerEnter} onPointerLeave={onPointerLeave}>
- <div style={layerStyle}>
+ <div
+ style={layerStyle}
+ {...getSequenceAnnotationAttributes({
+ sequence: track.sequence,
+ location: annotationLocation,
+ surface: 'track',
+ })}
+ >
{rowHighlightBackground && timelineWidth !== null ? (
<div
style={getTimelineSelectedTrackHighlightStyle(
diff --git a/packages/studio/src/components/Timeline/TimelineTracks.tsx b/packages/studio/src/components/Timeline/TimelineTracks.tsx
index 4b94bae99f..d1346d43c1 100644
--- a/packages/studio/src/components/Timeline/TimelineTracks.tsx
+++ b/packages/studio/src/components/Timeline/TimelineTracks.tsx
@@ -26,7 +26,7 @@ const TimelineTracksInner: React.FC<{
}, []);
return (
- <div style={timelineStyle}>
+ <div style={timelineStyle} {...{'oai-annotation-container': ''}}>
<div style={{...content, height: tracksEnd}}>
{virtualItems.map((virtualItem) => (
<div
diff --git a/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts b/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts
index a258d2f149..575984df30 100644
--- a/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts
+++ b/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts
@@ -8,11 +8,13 @@ import type {ResolvedStackLocation, TSequence} from 'remotion';
import {NoReactInternals} from 'remotion/no-react';
import {copyText} from '../../helpers/copy-text';
import {formatContextForAgents} from '../../helpers/format-file-location';
+import {getCodexAnnotation} from '../../helpers/get-codex-annotation';
import {
getDefaultOpenInTarget,
getGitSourceName,
openGitSource,
} from '../../helpers/get-git-menu-item';
+import {getSequenceAnnotationMetadata} from '../../helpers/sequence-annotation';
import {getOpenInMenuItems} from '../get-open-in-menu-items';
import type {ComboboxValue} from '../NewComposition/ComboBox';
import {showNotification} from '../Notifications/NotificationCenter';
@@ -186,53 +188,56 @@ export const getMultiSequenceContextMenuItems = ({
];
};
-export const getSequenceContextMenuItems = ({
- assetLinkInfo,
- canOpenInEditor,
- copyImageElement,
- deleteDisabled,
- disableInteractivityDisabled,
- duplicateDisabled,
- isProgrammaticallyDuplicated,
- includeSourceEditItems,
- codingAgentInfo,
- editorInfo,
- onConfigureApps,
- onDeleteSequenceFromSource,
- onDisableSequenceInteractivity,
- onDuplicateSequenceFromSource,
- openInCodingAgent,
- openInEditor,
- originalLocation,
- selectAsset,
- sequence,
- sourceActions = [],
-}: {
- readonly assetLinkInfo: TimelineAssetLinkInfo | null;
- readonly canOpenInEditor: boolean;
- readonly copyImageElement: Element | null;
- readonly deleteDisabled: boolean;
- readonly disableInteractivityDisabled: boolean;
- readonly duplicateDisabled: boolean;
- readonly isProgrammaticallyDuplicated: boolean;
- readonly includeSourceEditItems: boolean;
- readonly codingAgentInfo: GetDefaultCodingAgentInfoResponse | null;
- readonly editorInfo: GetDefaultEditorInfoResponse | null;
- readonly onConfigureApps: (() => void) | null;
- readonly onDeleteSequenceFromSource: () => void;
- readonly onDisableSequenceInteractivity: () => void;
- readonly onDuplicateSequenceFromSource: () => void;
- readonly openInCodingAgent: (
- codingAgentId: DefaultCodingAgent,
- codingAgentName: string,
- contextForAgents: string | null,
- ) => void;
- readonly openInEditor: (editorId: EditorPickerId | null) => void;
- readonly originalLocation: ResolvedStackLocation | null;
- readonly selectAsset: (src: string) => void;
- readonly sequence: TSequence;
- readonly sourceActions?: readonly ComboboxValue[];
-}): ComboboxValue[] => {
+export const getSequenceContextMenuItems = (
+ {
+ assetLinkInfo,
+ canOpenInEditor,
+ copyImageElement,
+ deleteDisabled,
+ disableInteractivityDisabled,
+ duplicateDisabled,
+ isProgrammaticallyDuplicated,
+ includeSourceEditItems,
+ codingAgentInfo,
+ editorInfo,
+ onConfigureApps,
+ onDeleteSequenceFromSource,
+ onDisableSequenceInteractivity,
+ onDuplicateSequenceFromSource,
+ openInCodingAgent,
+ openInEditor,
+ originalLocation,
+ selectAsset,
+ sequence,
+ sourceActions = [],
+ }: {
+ readonly assetLinkInfo: TimelineAssetLinkInfo | null;
+ readonly canOpenInEditor: boolean;
+ readonly copyImageElement: Element | null;
+ readonly deleteDisabled: boolean;
+ readonly disableInteractivityDisabled: boolean;
+ readonly duplicateDisabled: boolean;
+ readonly isProgrammaticallyDuplicated: boolean;
+ readonly includeSourceEditItems: boolean;
+ readonly codingAgentInfo: GetDefaultCodingAgentInfoResponse | null;
+ readonly editorInfo: GetDefaultEditorInfoResponse | null;
+ readonly onConfigureApps: (() => void) | null;
+ readonly onDeleteSequenceFromSource: () => void;
+ readonly onDisableSequenceInteractivity: () => void;
+ readonly onDuplicateSequenceFromSource: () => void;
+ readonly openInCodingAgent: (
+ codingAgentId: DefaultCodingAgent,
+ codingAgentName: string,
+ contextForAgents: string | null,
+ ) => void;
+ readonly openInEditor: (editorId: EditorPickerId | null) => void;
+ readonly originalLocation: ResolvedStackLocation | null;
+ readonly selectAsset: (src: string) => void;
+ readonly sequence: TSequence;
+ readonly sourceActions?: readonly ComboboxValue[];
+ },
+ contextMenuTarget: HTMLElement | null = null,
+): ComboboxValue[] => {
const isInteractiveSvg =
sequence.controls?.componentIdentity === interactiveSvgComponentIdentity;
const installedEditors = editorInfo?.installedEditors ?? [];
@@ -342,6 +347,69 @@ export const getSequenceContextMenuItems = ({
}
: null,
getCopyContextForAgentsMenuItem({contextForAgents}),
+ getCodexAnnotation() !== null
+ ? {
+ type: 'item' as const,
+ id: 'annotate-layer',
+ keyHint: null,
+ label: 'Annotate with ChatGPT',
+ leftItem: null,
+ disabled: false,
+ onClick: () => {
+ const target = [
+ sequence.refForOutline?.current,
+ contextMenuTarget,
+ ].find((element) => {
+ if (!element?.isConnected) {
+ return false;
+ }
+
+ const rect = element.getBoundingClientRect();
+ return (
+ rect.width > 0 &&
+ rect.height > 0 &&
+ rect.bottom > 0 &&
+ rect.right > 0 &&
+ rect.top < window.innerHeight &&
+ rect.left < window.innerWidth
+ );
+ });
+ if (!target) {
+ showNotification(
+ 'The annotation target is no longer visible',
+ 3000,
+ );
+ return;
+ }
+
+ const annotation = getCodexAnnotation();
+ if (annotation === null) {
+ showNotification('The annotation editor is unavailable', 3000);
+ return;
+ }
+
+ try {
+ const result = annotation.request(target, {
+ metadata: getSequenceAnnotationMetadata({
+ sequence,
+ location: originalLocation,
+ }),
+ });
+ if (!result?.accepted) {
+ showNotification('Could not open the annotation editor', 3000);
+ }
+ } catch (err) {
+ showNotification(
+ `Could not open the annotation editor: ${(err as Error).message}`,
+ 3000,
+ );
+ }
+ },
+ quickSwitcherLabel: null,
+ subMenu: null,
+ value: 'annotate-layer',
+ }
+ : null,
assetLinkInfo
? {
type: 'item' as const,
diff --git a/packages/studio/src/helpers/get-codex-annotation.ts b/packages/studio/src/helpers/get-codex-annotation.ts
new file mode 100644
index 0000000000..649c4bfd2b
--- /dev/null
+++ b/packages/studio/src/helpers/get-codex-annotation.ts
@@ -0,0 +1,24 @@
+type CodexAnnotation = {
+ readonly request: (
+ target: Element,
+ options: {
+ metadata: Record<string, string | number>;
+ },
+ ) => {accepted: boolean};
+};
+
+export const getCodexAnnotation = (): CodexAnnotation | null => {
+ if (typeof document === 'undefined') {
+ return null;
+ }
+
+ // The Codex browser injects this bridge. Check it when used, since the
+ // browser can install or remove it after the page has loaded.
+ const annotation = (
+ document as Document & {
+ oai: {annotation: CodexAnnotation | null} | null;
+ }
+ ).oai?.annotation;
+
+ return typeof annotation?.request === 'function' ? annotation : null;
+};
diff --git a/packages/studio/src/helpers/sequence-annotation.ts b/packages/studio/src/helpers/sequence-annotation.ts
new file mode 100644
index 0000000000..20181df795
--- /dev/null
+++ b/packages/studio/src/helpers/sequence-annotation.ts
@@ -0,0 +1,65 @@
+import type {ResolvedStackLocation, TSequence} from 'remotion';
+import {getRelativeFileLocation} from './format-file-location';
+
+export const getSequenceAnnotationMetadata = ({
+ sequence,
+ location,
+}: {
+ readonly sequence: TSequence;
+ readonly location: ResolvedStackLocation | null;
+}): Record<string, string | number> => {
+ const metadata: Record<string, string | number> = {
+ layer: (
+ sequence.displayName ||
+ sequence.controls?.componentName ||
+ sequence.id
+ ).slice(0, 128),
+ sequenceId: sequence.id.slice(0, 128),
+ ...(Number.isFinite(sequence.from) ? {from: sequence.from} : {}),
+ ...(Number.isFinite(sequence.duration)
+ ? {duration: sequence.duration}
+ : {}),
+ };
+ const relativeLocation = getRelativeFileLocation({
+ location,
+ root: window.remotion_cwd,
+ });
+ if (relativeLocation && relativeLocation.filename.length <= 256) {
+ const withSource = {
+ ...metadata,
+ source: relativeLocation.filename,
+ line: relativeLocation.line,
+ };
+ // The browser accepts at most six keys and 2,048 UTF-8 bytes. Omit
+ // oversized source context instead of truncating the filename.
+ if (new TextEncoder().encode(JSON.stringify(withSource)).length <= 2048) {
+ return withSource;
+ }
+ }
+
+ return metadata;
+};
+
+export const getSequenceAnnotationAttributes = ({
+ sequence,
+ location,
+ surface,
+}: {
+ readonly sequence: TSequence;
+ readonly location: ResolvedStackLocation | null;
+ readonly surface: 'outline' | 'layer' | 'track';
+}) => {
+ const metadata = getSequenceAnnotationMetadata({sequence, location});
+ return {
+ 'data-remotion-annotation-surface': surface,
+ 'data-remotion-layer': metadata.layer,
+ 'data-remotion-sequence-id': metadata.sequenceId,
+ 'data-remotion-sequence-type': sequence.type,
+ 'data-remotion-from': metadata.from,
+ 'data-remotion-duration': metadata.duration,
+ 'data-remotion-source': metadata.source,
+ 'data-remotion-line': metadata.line,
+ 'oai-annotatable': metadata.layer,
+ 'oai-annotation-metadata': JSON.stringify(metadata),
+ };
+};