Skip to content

πŸŽ₯ Make videos programmatically with React

feat(studio): highlight and follow active captions

Jonny Burger1c2a7ebf58c4
diff --git a/packages/studio/src/components/CaptionPlayheadHighlight.tsx b/packages/studio/src/components/CaptionPlayheadHighlight.tsx
new file mode 100644
index 0000000000..f0ffed17c0
--- /dev/null
+++ b/packages/studio/src/components/CaptionPlayheadHighlight.tsx
@@ -0,0 +1,166 @@
+import type {Caption} from '@remotion/captions';
+import type React from 'react';
+import {useContext, useEffect, useRef} from 'react';
+import {Internals} from 'remotion';
+import {WHITE_ALPHA_06} from '../helpers/colors';
+import {getCurrentFps} from './Timeline/imperative-state';
+import {
+ useCurrentTimelineSelectionStateAsRef,
+ type TimelineSelection,
+} from './Timeline/TimelineSelection';
+
+// Subscribe beside the editor so frame changes do not render the caption rows.
+export const CaptionPlayheadHighlight: React.FC<{
+ readonly captions: Caption[];
+ readonly getSequenceFrame: () => number;
+ readonly containerRef: React.RefObject<HTMLDivElement | null>;
+ readonly onInitialCaptionScrollRef: React.RefObject<(() => void) | null>;
+}> = ({
+ captions,
+ getSequenceFrame,
+ containerRef,
+ onInitialCaptionScrollRef,
+}) => {
+ const frame = Internals.Timeline.useTimelinePosition();
+ const {seek} = useContext(Internals.SetTimelineContext);
+ const selection = useCurrentTimelineSelectionStateAsRef();
+ const seekRevision = seek?.revision.current ?? 0;
+ const {selectedItems} = selection.current;
+ const suspended = useRef<{
+ seekRevision: number;
+ selectedItems: readonly TimelineSelection[];
+ } | null>(null);
+ const scrollContainer = useRef<{
+ element: HTMLElement;
+ scrollTop: number;
+ } | null>(null);
+ const highlighted = useRef<{
+ captions: Caption[];
+ index: number;
+ row: HTMLDivElement | null;
+ } | null>(null);
+
+ useEffect(() => {
+ let element = containerRef.current?.parentElement ?? null;
+ while (
+ element &&
+ !['auto', 'scroll'].includes(getComputedStyle(element).overflowY)
+ ) {
+ element = element.parentElement;
+ }
+
+ if (!element) {
+ return;
+ }
+
+ const scroller = element;
+ scrollContainer.current = {
+ element: scroller,
+ scrollTop: scroller.scrollTop,
+ };
+ onInitialCaptionScrollRef.current = () => {
+ // The selection reveal is programmatic too: keep following after it
+ // centers the caption instead of treating it as manual scrolling.
+ suspended.current = null;
+ if (scrollContainer.current) {
+ scrollContainer.current.scrollTop = scroller.scrollTop;
+ }
+ };
+
+ const onScroll = () => {
+ // Ignore our own scrollIntoView. Any other scroll pauses following until
+ // the user seeks or selects a sequence again, without React state.
+ if (scroller.scrollTop === scrollContainer.current?.scrollTop) {
+ return;
+ }
+
+ suspended.current = {
+ seekRevision: seek?.revision.current ?? 0,
+ selectedItems: selection.current.selectedItems,
+ };
+ };
+
+ scroller.addEventListener('scroll', onScroll);
+
+ return () => {
+ scroller.removeEventListener('scroll', onScroll);
+ scrollContainer.current = null;
+ onInitialCaptionScrollRef.current = null;
+ };
+ }, [containerRef, onInitialCaptionScrollRef, seek, selection]);
+
+ useEffect(() => {
+ // Read after the Sequence has committed its local clock, including freezes
+ // and loops. Only touch the DOM when the active caption or rows change,
+ // or when a seek resumes following after editing or manual scrolling.
+ const timeMs = (getSequenceFrame() / getCurrentFps()) * 1000;
+ const index = captions.findIndex(
+ (caption) => caption.startMs <= timeMs && caption.endMs > timeMs,
+ );
+ const isSuspended =
+ suspended.current?.seekRevision === seekRevision &&
+ suspended.current.selectedItems === selectedItems;
+ const shouldResume = suspended.current !== null && !isSuspended;
+ if (
+ highlighted.current?.index === index &&
+ highlighted.current.captions === captions &&
+ !shouldResume
+ ) {
+ return;
+ }
+
+ const row =
+ index === -1
+ ? null
+ : (containerRef.current?.querySelector<HTMLDivElement>(
+ `[data-caption-row-index="${index}"]`,
+ ) ?? null);
+ if (highlighted.current?.row !== row) {
+ highlighted.current?.row?.style.removeProperty(
+ '--remotion-active-caption-background',
+ );
+ row?.style.setProperty(
+ '--remotion-active-caption-background',
+ WHITE_ALPHA_06,
+ );
+ }
+
+ highlighted.current = {captions, index, row};
+ if (containerRef.current?.querySelector('[data-caption-index]:focus')) {
+ suspended.current = {
+ seekRevision,
+ selectedItems,
+ };
+ return;
+ }
+
+ if (isSuspended) {
+ return;
+ }
+
+ suspended.current = null;
+ row?.scrollIntoView({block: 'nearest'});
+ if (scrollContainer.current) {
+ scrollContainer.current.scrollTop =
+ scrollContainer.current.element.scrollTop;
+ }
+ }, [
+ captions,
+ containerRef,
+ frame,
+ getSequenceFrame,
+ seekRevision,
+ selectedItems,
+ ]);
+
+ useEffect(() => {
+ return () => {
+ highlighted.current?.row?.style.removeProperty(
+ '--remotion-active-caption-background',
+ );
+ highlighted.current = null;
+ };
+ }, []);
+
+ return null;
+};
diff --git a/packages/studio/src/components/CaptionTextEditor.tsx b/packages/studio/src/components/CaptionTextEditor.tsx
index c2207e33e2..75db0043c2 100644
--- a/packages/studio/src/components/CaptionTextEditor.tsx
+++ b/packages/studio/src/components/CaptionTextEditor.tsx
@@ -29,6 +29,7 @@ const list: React.CSSProperties = {
const row: React.CSSProperties = {
alignItems: 'center',
+ backgroundColor: 'var(--remotion-active-caption-background, transparent)',
borderBottom: `1px solid ${LINE_COLOR}`,
display: 'grid',
gap: 8,
@@ -65,7 +66,7 @@ export const CaptionTextEditor: React.FC<{
readonly onSave: ((captions: Caption[]) => void) | null;
readonly onCancel: (() => void) | null;
readonly readOnly: boolean;
-}> = ({captions, onChange, onSave, onCancel, readOnly}) => {
+}> = React.memo(({captions, onChange, onSave, onCancel, readOnly}) => {
const listRef = useRef<HTMLDivElement>(null);
const cancelledBlurIndexes = useRef(new Set<number>());
const dirtyRef = useRef(false);
@@ -234,7 +235,7 @@ export const CaptionTextEditor: React.FC<{
: `Add page break after caption ${index + 1}`;
return (
- <div key={key} style={row}>
+ <div key={key} data-caption-row-index={index} style={row}>
<div style={timing}>
{formatMilliseconds(caption.startMs)} β†’{' '}
{formatMilliseconds(caption.endMs)} ms
@@ -330,4 +331,6 @@ export const CaptionTextEditor: React.FC<{
</div>
</div>
);
-};
+});
+
+CaptionTextEditor.displayName = 'CaptionTextEditor';
diff --git a/packages/studio/src/components/InlineCaptionInspector.tsx b/packages/studio/src/components/InlineCaptionInspector.tsx
index 5aee156b06..11ac230b96 100644
--- a/packages/studio/src/components/InlineCaptionInspector.tsx
+++ b/packages/studio/src/components/InlineCaptionInspector.tsx
@@ -15,6 +15,7 @@ import {Internals} from 'remotion';
import type {CodePosition} from '../error-overlay/react-overlay/utils/get-source-map';
import {StudioServerConnectionCtx} from '../helpers/client-id';
import {CaptionInspector} from './CaptionInspector';
+import {CaptionPlayheadHighlight} from './CaptionPlayheadHighlight';
import {
saveInlineCaptionPatches,
saveSequenceProps,
@@ -132,8 +133,10 @@ export const InlineCaptionInspector: React.FC<{
readonly captions: Caption[];
readonly controls: SequenceRegistrationControls;
readonly expanded: boolean;
+ readonly getSequenceFrame: () => number;
readonly nodePath: SequencePropsSubscriptionKey;
readonly onCaptionsRendered: () => void;
+ readonly onInitialCaptionScrollRef: React.RefObject<(() => void) | null>;
readonly onToggle: () => void;
readonly readOnlyStudio: boolean;
readonly validatedLocation: CodePosition;
@@ -141,8 +144,10 @@ export const InlineCaptionInspector: React.FC<{
captions,
controls,
expanded,
+ getSequenceFrame,
nodePath,
onCaptionsRendered,
+ onInitialCaptionScrollRef,
onToggle,
readOnlyStudio,
validatedLocation,
@@ -163,6 +168,7 @@ export const InlineCaptionInspector: React.FC<{
const canSave =
!readOnlyStudio && clientId !== null && captionStatus?.status === 'static';
const [draftCaptions, setDraftCaptions] = useState(captions);
+ const containerRef = useRef<HTMLDivElement>(null);
const savedCaptions = useRef(captions);
const runtimeSignature = serializeCaptions(captions);
const lastRuntimeSignature = useRef(runtimeSignature);
@@ -298,16 +304,26 @@ export const InlineCaptionInspector: React.FC<{
: 'Captions are not ready for editing';
return (
- <CaptionInspector
- captions={draftCaptions}
- expanded={expanded}
- onTextChange={updateCaptions}
- onTextSave={saveCaptions}
- onTextCancel={cancelCaptions}
- onToggle={onToggle}
- readOnly={!canSave}
- readOnlyTitle={canSave ? null : readOnlyTitle}
- onReplaceCaptions={canSave ? replaceCaptions : null}
- />
+ <div ref={containerRef}>
+ <CaptionInspector
+ captions={draftCaptions}
+ expanded={expanded}
+ onTextChange={updateCaptions}
+ onTextSave={saveCaptions}
+ onTextCancel={cancelCaptions}
+ onToggle={onToggle}
+ readOnly={!canSave}
+ readOnlyTitle={canSave ? null : readOnlyTitle}
+ onReplaceCaptions={canSave ? replaceCaptions : null}
+ />
+ {expanded ? (
+ <CaptionPlayheadHighlight
+ captions={draftCaptions}
+ containerRef={containerRef}
+ getSequenceFrame={getSequenceFrame}
+ onInitialCaptionScrollRef={onInitialCaptionScrollRef}
+ />
+ ) : null}
+ </div>
);
};
diff --git a/packages/studio/src/components/InspectorSequenceSection.tsx b/packages/studio/src/components/InspectorSequenceSection.tsx
index 8351c5613b..899cce294b 100644
--- a/packages/studio/src/components/InspectorSequenceSection.tsx
+++ b/packages/studio/src/components/InspectorSequenceSection.tsx
@@ -352,6 +352,7 @@ export const InspectorSequenceSection: React.FC<{
? (inspectorRevealRequest?.token ?? null)
: null;
const captionsInspectorRef = useRef<HTMLDivElement>(null);
+ const onInitialCaptionScrollRef = useRef<(() => void) | null>(null);
const handledCaptionSelection = useRef<readonly TimelineSelection[] | null>(
null,
);
@@ -756,6 +757,26 @@ export const InspectorSequenceSection: React.FC<{
: null,
[hasCaptionsSchema, runtimeValues.captions],
);
+ const {
+ getCurrentFrame: getCommittedSequenceFrame,
+ frozenFrame: sequenceFrozenFrame,
+ from: sequenceFrom,
+ sequencePlaybackRate,
+ } = sequence;
+ const getSequenceFrame = useCallback(
+ () =>
+ getCommittedSequenceFrame?.() ??
+ sequenceFrozenFrame ??
+ (getCurrentFrame() - sequenceFrom) * sequencePlaybackRate +
+ sequenceFrameOffset,
+ [
+ getCommittedSequenceFrame,
+ sequenceFrozenFrame,
+ sequenceFrom,
+ sequencePlaybackRate,
+ sequenceFrameOffset,
+ ],
+ );
useEffect(() => {
if (handledCaptionSelection.current === selectedItems) {
return;
@@ -778,12 +799,7 @@ export const InspectorSequenceSection: React.FC<{
}
handledCaptionSelection.current = selectedItems;
- const sequenceFrame =
- sequence.getCurrentFrame?.() ??
- sequence.frozenFrame ??
- (getCurrentFrame() - sequence.from) * sequence.sequencePlaybackRate +
- sequenceFrameOffset;
- const timeMs = (sequenceFrame / getCurrentFps()) * 1000;
+ const timeMs = (getSequenceFrame() / getCurrentFps()) * 1000;
const index = inlineCaptions.findIndex(
(caption) => caption.startMs <= timeMs && caption.endMs > timeMs,
);
@@ -794,10 +810,9 @@ export const InspectorSequenceSection: React.FC<{
pendingCaptionScroll.current = {selection: selectedItems, index};
setAdditionalSectionExpanded('captions', true);
}, [
+ getSequenceFrame,
inlineCaptions,
selectedItems,
- sequence,
- sequenceFrameOffset,
sequenceKey,
setAdditionalSectionExpanded,
]);
@@ -820,6 +835,7 @@ export const InspectorSequenceSection: React.FC<{
}
input.scrollIntoView({block: 'center'});
+ onInitialCaptionScrollRef.current?.();
pendingCaptionScroll.current = null;
}, [captionsExpanded, selectedItems]);
@@ -1028,8 +1044,10 @@ export const InspectorSequenceSection: React.FC<{
captions={inlineCaptions}
controls={sequence.controls}
expanded={captionsExpanded}
+ getSequenceFrame={getSequenceFrame}
nodePath={nodePathInfo.sequenceSubscriptionKey}
onCaptionsRendered={revealPendingCaption}
+ onInitialCaptionScrollRef={onInitialCaptionScrollRef}
onToggle={() => toggleAdditionalSection('captions')}
readOnlyStudio={readOnlyStudio}
validatedLocation={validatedLocation}