π₯ Make videos programmatically with React
feat(studio): highlight and follow active captions
Jonny Burger1c2a7ebf58c4
diff --git a/packages/studio/src/components/CaptionPlayheadHighlight.tsx b/packages/studio/src/components/CaptionPlayheadHighlight.tsxnew file mode 100644index 0000000000..f0ffed17c0--- /dev/null+++ b/packages/studio/src/components/CaptionPlayheadHighlight.tsx@@ -0,0 +1,166 @@+import type {Caption} from '@remotion/captions';+import type React from 'react';+import {useContext, useEffect, useRef} from 'react';+import {Internals} from 'remotion';+import {WHITE_ALPHA_06} from '../helpers/colors';+import {getCurrentFps} from './Timeline/imperative-state';+import {+ useCurrentTimelineSelectionStateAsRef,+ type TimelineSelection,+} from './Timeline/TimelineSelection';++// Subscribe beside the editor so frame changes do not render the caption rows.+export const CaptionPlayheadHighlight: React.FC<{+ readonly captions: Caption[];+ readonly getSequenceFrame: () => number;+ readonly containerRef: React.RefObject<HTMLDivElement | null>;+ readonly onInitialCaptionScrollRef: React.RefObject<(() => void) | null>;+}> = ({+ captions,+ getSequenceFrame,+ containerRef,+ onInitialCaptionScrollRef,+}) => {+ const frame = Internals.Timeline.useTimelinePosition();+ const {seek} = useContext(Internals.SetTimelineContext);+ const selection = useCurrentTimelineSelectionStateAsRef();+ const seekRevision = seek?.revision.current ?? 0;+ const {selectedItems} = selection.current;+ const suspended = useRef<{+ seekRevision: number;+ selectedItems: readonly TimelineSelection[];+ } | null>(null);+ const scrollContainer = useRef<{+ element: HTMLElement;+ scrollTop: number;+ } | null>(null);+ const highlighted = useRef<{+ captions: Caption[];+ index: number;+ row: HTMLDivElement | null;+ } | null>(null);++ useEffect(() => {+ let element = containerRef.current?.parentElement ?? null;+ while (+ element &&+ !['auto', 'scroll'].includes(getComputedStyle(element).overflowY)+ ) {+ element = element.parentElement;+ }++ if (!element) {+ return;+ }++ const scroller = element;+ scrollContainer.current = {+ element: scroller,+ scrollTop: scroller.scrollTop,+ };+ onInitialCaptionScrollRef.current = () => {+ // The selection reveal is programmatic too: keep following after it+ // centers the caption instead of treating it as manual scrolling.+ suspended.current = null;+ if (scrollContainer.current) {+ scrollContainer.current.scrollTop = scroller.scrollTop;+ }+ };++ const onScroll = () => {+ // Ignore our own scrollIntoView. Any other scroll pauses following until+ // the user seeks or selects a sequence again, without React state.+ if (scroller.scrollTop === scrollContainer.current?.scrollTop) {+ return;+ }++ suspended.current = {+ seekRevision: seek?.revision.current ?? 0,+ selectedItems: selection.current.selectedItems,+ };+ };++ scroller.addEventListener('scroll', onScroll);++ return () => {+ scroller.removeEventListener('scroll', onScroll);+ scrollContainer.current = null;+ onInitialCaptionScrollRef.current = null;+ };+ }, [containerRef, onInitialCaptionScrollRef, seek, selection]);++ useEffect(() => {+ // Read after the Sequence has committed its local clock, including freezes+ // and loops. Only touch the DOM when the active caption or rows change,+ // or when a seek resumes following after editing or manual scrolling.+ const timeMs = (getSequenceFrame() / getCurrentFps()) * 1000;+ const index = captions.findIndex(+ (caption) => caption.startMs <= timeMs && caption.endMs > timeMs,+ );+ const isSuspended =+ suspended.current?.seekRevision === seekRevision &&+ suspended.current.selectedItems === selectedItems;+ const shouldResume = suspended.current !== null && !isSuspended;+ if (+ highlighted.current?.index === index &&+ highlighted.current.captions === captions &&+ !shouldResume+ ) {+ return;+ }++ const row =+ index === -1+ ? null+ : (containerRef.current?.querySelector<HTMLDivElement>(+ `[data-caption-row-index="${index}"]`,+ ) ?? null);+ if (highlighted.current?.row !== row) {+ highlighted.current?.row?.style.removeProperty(+ '--remotion-active-caption-background',+ );+ row?.style.setProperty(+ '--remotion-active-caption-background',+ WHITE_ALPHA_06,+ );+ }++ highlighted.current = {captions, index, row};+ if (containerRef.current?.querySelector('[data-caption-index]:focus')) {+ suspended.current = {+ seekRevision,+ selectedItems,+ };+ return;+ }++ if (isSuspended) {+ return;+ }++ suspended.current = null;+ row?.scrollIntoView({block: 'nearest'});+ if (scrollContainer.current) {+ scrollContainer.current.scrollTop =+ scrollContainer.current.element.scrollTop;+ }+ }, [+ captions,+ containerRef,+ frame,+ getSequenceFrame,+ seekRevision,+ selectedItems,+ ]);++ useEffect(() => {+ return () => {+ highlighted.current?.row?.style.removeProperty(+ '--remotion-active-caption-background',+ );+ highlighted.current = null;+ };+ }, []);++ return null;+};diff --git a/packages/studio/src/components/CaptionTextEditor.tsx b/packages/studio/src/components/CaptionTextEditor.tsxindex c2207e33e2..75db0043c2 100644--- a/packages/studio/src/components/CaptionTextEditor.tsx+++ b/packages/studio/src/components/CaptionTextEditor.tsx@@ -29,6 +29,7 @@ const list: React.CSSProperties = {const row: React.CSSProperties = {alignItems: 'center',+ backgroundColor: 'var(--remotion-active-caption-background, transparent)',borderBottom: `1px solid ${LINE_COLOR}`,display: 'grid',gap: 8,@@ -65,7 +66,7 @@ export const CaptionTextEditor: React.FC<{readonly onSave: ((captions: Caption[]) => void) | null;readonly onCancel: (() => void) | null;readonly readOnly: boolean;-}> = ({captions, onChange, onSave, onCancel, readOnly}) => {+}> = React.memo(({captions, onChange, onSave, onCancel, readOnly}) => {const listRef = useRef<HTMLDivElement>(null);const cancelledBlurIndexes = useRef(new Set<number>());const dirtyRef = useRef(false);@@ -234,7 +235,7 @@ export const CaptionTextEditor: React.FC<{: `Add page break after caption ${index + 1}`;return (- <div key={key} style={row}>+ <div key={key} data-caption-row-index={index} style={row}><div style={timing}>{formatMilliseconds(caption.startMs)} β{' '}{formatMilliseconds(caption.endMs)} ms@@ -330,4 +331,6 @@ export const CaptionTextEditor: React.FC<{</div></div>);-};+});++CaptionTextEditor.displayName = 'CaptionTextEditor';diff --git a/packages/studio/src/components/InlineCaptionInspector.tsx b/packages/studio/src/components/InlineCaptionInspector.tsxindex 5aee156b06..11ac230b96 100644--- a/packages/studio/src/components/InlineCaptionInspector.tsx+++ b/packages/studio/src/components/InlineCaptionInspector.tsx@@ -15,6 +15,7 @@ import {Internals} from 'remotion';import type {CodePosition} from '../error-overlay/react-overlay/utils/get-source-map';import {StudioServerConnectionCtx} from '../helpers/client-id';import {CaptionInspector} from './CaptionInspector';+import {CaptionPlayheadHighlight} from './CaptionPlayheadHighlight';import {saveInlineCaptionPatches,saveSequenceProps,@@ -132,8 +133,10 @@ export const InlineCaptionInspector: React.FC<{readonly captions: Caption[];readonly controls: SequenceRegistrationControls;readonly expanded: boolean;+ readonly getSequenceFrame: () => number;readonly nodePath: SequencePropsSubscriptionKey;readonly onCaptionsRendered: () => void;+ readonly onInitialCaptionScrollRef: React.RefObject<(() => void) | null>;readonly onToggle: () => void;readonly readOnlyStudio: boolean;readonly validatedLocation: CodePosition;@@ -141,8 +144,10 @@ export const InlineCaptionInspector: React.FC<{captions,controls,expanded,+ getSequenceFrame,nodePath,onCaptionsRendered,+ onInitialCaptionScrollRef,onToggle,readOnlyStudio,validatedLocation,@@ -163,6 +168,7 @@ export const InlineCaptionInspector: React.FC<{const canSave =!readOnlyStudio && clientId !== null && captionStatus?.status === 'static';const [draftCaptions, setDraftCaptions] = useState(captions);+ const containerRef = useRef<HTMLDivElement>(null);const savedCaptions = useRef(captions);const runtimeSignature = serializeCaptions(captions);const lastRuntimeSignature = useRef(runtimeSignature);@@ -298,16 +304,26 @@ export const InlineCaptionInspector: React.FC<{: 'Captions are not ready for editing';return (- <CaptionInspector- captions={draftCaptions}- expanded={expanded}- onTextChange={updateCaptions}- onTextSave={saveCaptions}- onTextCancel={cancelCaptions}- onToggle={onToggle}- readOnly={!canSave}- readOnlyTitle={canSave ? null : readOnlyTitle}- onReplaceCaptions={canSave ? replaceCaptions : null}- />+ <div ref={containerRef}>+ <CaptionInspector+ captions={draftCaptions}+ expanded={expanded}+ onTextChange={updateCaptions}+ onTextSave={saveCaptions}+ onTextCancel={cancelCaptions}+ onToggle={onToggle}+ readOnly={!canSave}+ readOnlyTitle={canSave ? null : readOnlyTitle}+ onReplaceCaptions={canSave ? replaceCaptions : null}+ />+ {expanded ? (+ <CaptionPlayheadHighlight+ captions={draftCaptions}+ containerRef={containerRef}+ getSequenceFrame={getSequenceFrame}+ onInitialCaptionScrollRef={onInitialCaptionScrollRef}+ />+ ) : null}+ </div>);};diff --git a/packages/studio/src/components/InspectorSequenceSection.tsx b/packages/studio/src/components/InspectorSequenceSection.tsxindex 8351c5613b..899cce294b 100644--- a/packages/studio/src/components/InspectorSequenceSection.tsx+++ b/packages/studio/src/components/InspectorSequenceSection.tsx@@ -352,6 +352,7 @@ export const InspectorSequenceSection: React.FC<{? (inspectorRevealRequest?.token ?? null): null;const captionsInspectorRef = useRef<HTMLDivElement>(null);+ const onInitialCaptionScrollRef = useRef<(() => void) | null>(null);const handledCaptionSelection = useRef<readonly TimelineSelection[] | null>(null,);@@ -756,6 +757,26 @@ export const InspectorSequenceSection: React.FC<{: null,[hasCaptionsSchema, runtimeValues.captions],);+ const {+ getCurrentFrame: getCommittedSequenceFrame,+ frozenFrame: sequenceFrozenFrame,+ from: sequenceFrom,+ sequencePlaybackRate,+ } = sequence;+ const getSequenceFrame = useCallback(+ () =>+ getCommittedSequenceFrame?.() ??+ sequenceFrozenFrame ??+ (getCurrentFrame() - sequenceFrom) * sequencePlaybackRate ++ sequenceFrameOffset,+ [+ getCommittedSequenceFrame,+ sequenceFrozenFrame,+ sequenceFrom,+ sequencePlaybackRate,+ sequenceFrameOffset,+ ],+ );useEffect(() => {if (handledCaptionSelection.current === selectedItems) {return;@@ -778,12 +799,7 @@ export const InspectorSequenceSection: React.FC<{}handledCaptionSelection.current = selectedItems;- const sequenceFrame =- sequence.getCurrentFrame?.() ??- sequence.frozenFrame ??- (getCurrentFrame() - sequence.from) * sequence.sequencePlaybackRate +- sequenceFrameOffset;- const timeMs = (sequenceFrame / getCurrentFps()) * 1000;+ const timeMs = (getSequenceFrame() / getCurrentFps()) * 1000;const index = inlineCaptions.findIndex((caption) => caption.startMs <= timeMs && caption.endMs > timeMs,);@@ -794,10 +810,9 @@ export const InspectorSequenceSection: React.FC<{pendingCaptionScroll.current = {selection: selectedItems, index};setAdditionalSectionExpanded('captions', true);}, [+ getSequenceFrame,inlineCaptions,selectedItems,- sequence,- sequenceFrameOffset,sequenceKey,setAdditionalSectionExpanded,]);@@ -820,6 +835,7 @@ export const InspectorSequenceSection: React.FC<{}input.scrollIntoView({block: 'center'});+ onInitialCaptionScrollRef.current?.();pendingCaptionScroll.current = null;}, [captionsExpanded, selectedItems]);@@ -1028,8 +1044,10 @@ export const InspectorSequenceSection: React.FC<{captions={inlineCaptions}controls={sequence.controls}expanded={captionsExpanded}+ getSequenceFrame={getSequenceFrame}nodePath={nodePathInfo.sequenceSubscriptionKey}onCaptionsRendered={revealPendingCaption}+ onInitialCaptionScrollRef={onInitialCaptionScrollRef}onToggle={() => toggleAdditionalSection('captions')}readOnlyStudio={readOnlyStudio}validatedLocation={validatedLocation}