Skip to content

🎥 Make videos programmatically with React

Show orange caption layers with transcript labels

Jonny Burgerbe37b0a145f5
diff --git a/packages/studio/src/components/Timeline/TimelineSequence.tsx b/packages/studio/src/components/Timeline/TimelineSequence.tsx
index e62b6fa74d..79fd410776 100644
--- a/packages/studio/src/components/Timeline/TimelineSequence.tsx
+++ b/packages/studio/src/components/Timeline/TimelineSequence.tsx
@@ -23,6 +23,7 @@ import {
BLUE,
TIMELINE_AUDIO_GRADIENT,
TIMELINE_BACKGROUND_COLOR,
+ TIMELINE_CAPTIONS_GRADIENT,
TIMELINE_NEGATIVE_START_BACKGROUND_COLOR,
TIMELINE_NEGATIVE_START_BORDER_COLOR,
TIMELINE_VIDEO_GRADIENT,
@@ -52,7 +53,6 @@ import {useMediaMetadata} from '../../helpers/use-media-metadata';
import {useRuntimeValueSelector} from '../../helpers/use-runtime-values';
import {SnowflakeIcon} from '../../icons/snowflake';
import {SetSelectedModalContext} from '../../state/modals';
-import {ActionTooltip} from '../ActionTooltip';
import {AudioWaveform} from '../AudioWaveform';
import {CompositionOrStillIcon} from '../CompositionOrStillIcon';
import {useConfirmationDialog} from '../ConfirmationDialog';
@@ -80,6 +80,10 @@ import {splitSelectedTimelineItems} from './split-selected-timeline-item';
import {getTimelineAssetLinkInfo} from './timeline-asset-link';
import {timelineLeftEdgeCursor} from './timeline-left-edge-cursor';
import {timelineLayerLayoutsRef} from './timeline-refs';
+import {
+ PACKED_LABEL_BOTTOM,
+ TimelineSequenceLabel,
+} from './timeline-sequence-label';
import {
TIMELINE_PACKED_AUXILIARY_ROW_HEIGHT,
TIMELINE_PACKED_TRACK_HEIGHT,
@@ -97,6 +101,7 @@ import {
useTimelineRowContainsSelection,
useTimelineRowSelection,
} from './TimelineSelection';
+import {TimelineSequenceCaptions} from './TimelineSequenceCaptions';
import {TimelineSequenceFrame} from './TimelineSequenceFrame';
import {TimelineSequenceMountIndicator} from './TimelineSequenceMountIndicator';
import {
@@ -126,8 +131,6 @@ import {
const NEGATIVE_START_BORDER_WIDTH = 1;
const EDGE_DRAG_HIGHLIGHT_WIDTH = 12;
const MIN_SECONDARY_LEFT_EDGE_ACTION_WIDTH = 32;
-const PACKED_LABEL_HEIGHT = 15;
-const PACKED_LABEL_BOTTOM = 1;
type TimelineEdgeHighlightEdge = 'left' | 'right' | 'source-only';
@@ -325,6 +328,7 @@ const TimelineSequenceNegativeStart = React.memo(
const TimelineSequenceBar: React.FC<{
readonly s: TSequence;
readonly labelOffset: number;
+ readonly hasCaptionsSchema: boolean;
readonly connectedComposition: _InternalTypes['AnyComposition'] | null;
readonly annotationLocation: ResolvedStackLocation | null;
readonly paintEndFrame: number | null;
@@ -367,6 +371,7 @@ const TimelineSequenceBar: React.FC<{
}> = ({
s,
labelOffset,
+ hasCaptionsSchema,
connectedComposition,
activeTrimEdge,
leftTrimHighlight,
@@ -544,7 +549,7 @@ const TimelineSequenceBar: React.FC<{
</svg>
)}
- {s.timelineTrack ? (
+ {s.timelineTrack || hasCaptionsSchema ? (
<div
style={{
position: 'absolute',
@@ -574,40 +579,17 @@ const TimelineSequenceBar: React.FC<{
}}
/>
) : null}
- <ActionTooltip
- label={
- s.timelineTrack.role === 'track' && frozenFrame !== null
- ? `${s.displayName} (frozen at frame ${frozenFrame})`
- : s.displayName
- }
- shortcut={null}
- delay={250}
- dismissOnClick
- triggerStyle={{
- alignSelf: 'flex-start',
- maxWidth: '100%',
- minWidth: 0,
- pointerEvents: 'auto',
- }}
- >
- <span
- style={{
- flex: 1,
- fontSize: 11,
- lineHeight: `${PACKED_LABEL_HEIGHT}px`,
- color: getTimelineColor(false, false),
- maskImage:
- 'linear-gradient(to right, black calc(100% - 5px), transparent)',
- minWidth: 0,
- // Keep the fade in empty space unless the label is clipped.
- paddingRight: 5,
- whiteSpace: 'nowrap',
- overflow: 'hidden',
- WebkitMaskImage:
- 'linear-gradient(to right, black calc(100% - 5px), transparent)',
- }}
+ {hasCaptionsSchema ? (
+ <TimelineSequenceCaptions s={s} />
+ ) : (
+ <TimelineSequenceLabel
+ label={
+ s.timelineTrack?.role === 'track' && frozenFrame !== null
+ ? `${s.displayName} (frozen at frame ${frozenFrame})`
+ : s.displayName
+ }
>
- {s.timelineTrack.role === 'track' && frozenFrame !== null ? (
+ {s.timelineTrack?.role === 'track' && frozenFrame !== null ? (
<SnowflakeIcon
aria-label={`Frozen at frame ${frozenFrame}`}
color={getTimelineColor(false, false)}
@@ -619,13 +601,14 @@ const TimelineSequenceBar: React.FC<{
}}
/>
) : null}
- {s.timelineTrack.role === 'overlay' ? 'Overlay' : s.displayName}
- </span>
- </ActionTooltip>
+ {s.timelineTrack?.role === 'overlay' ? 'Overlay' : s.displayName}
+ </TimelineSequenceLabel>
+ )}
</div>
) : null}
- {!s.timelineTrack &&
+ {!hasCaptionsSchema &&
+ !s.timelineTrack &&
s.type !== 'audio' &&
s.type !== 'video' &&
s.type !== 'image' &&
@@ -1702,6 +1685,9 @@ const TimelineSequenceInner: React.FC<{
visibleLayout?.rightEdgeVisible === true &&
(s.autoDuration || endsAtContainerBoundary || endsAtNaturalMediaDuration);
+ const hasCaptionsSchema = Object.values(s.controls?.schema ?? {}).some(
+ (field) => field.type === 'remotion-captions',
+ );
const style: React.CSSProperties = useMemo(() => {
const role = s.timelineTrack?.role;
return {
@@ -1712,7 +1698,9 @@ const TimelineSequenceInner: React.FC<{
? TIMELINE_AUDIO_GRADIENT
: s.type === 'video'
? TIMELINE_VIDEO_GRADIENT
- : BLUE,
+ : hasCaptionsSchema
+ ? TIMELINE_CAPTIONS_GRADIENT
+ : BLUE,
borderTopLeftRadius: showLeftBorderRadius ? 2 : 0,
borderBottomLeftRadius: showLeftBorderRadius ? 2 : 0,
borderTopRightRadius: showRightBorderRadius ? 2 : 0,
@@ -1728,6 +1716,7 @@ const TimelineSequenceInner: React.FC<{
overflow: 'visible',
};
}, [
+ hasCaptionsSchema,
s.type,
s.timelineTrack,
showLeftBorderRadius,
@@ -1781,6 +1770,7 @@ const TimelineSequenceInner: React.FC<{
const sequence = (
<TimelineSequenceBar
s={s}
+ hasCaptionsSchema={hasCaptionsSchema}
labelOffset={
labelStartFrame === null
? 0
diff --git a/packages/studio/src/components/Timeline/TimelineSequenceCaptions.tsx b/packages/studio/src/components/Timeline/TimelineSequenceCaptions.tsx
new file mode 100644
index 0000000000..7e8b3d7201
--- /dev/null
+++ b/packages/studio/src/components/Timeline/TimelineSequenceCaptions.tsx
@@ -0,0 +1,25 @@
+import type {Caption} from '@remotion/captions';
+import React from 'react';
+import type {TSequence} from 'remotion';
+import {useRuntimeValue} from '../../helpers/use-runtime-values';
+import {TimelineSequenceLabel} from './timeline-sequence-label';
+
+export const TimelineSequenceCaptions: React.FC<{
+ readonly s: TSequence;
+}> = ({s}) => {
+ const captionKey = Object.entries(s.controls?.schema ?? {}).find(
+ ([, field]) => field.type === 'remotion-captions',
+ )?.[0];
+ const value = useRuntimeValue(s.controls ?? null, captionKey ?? 'captions');
+ const captions = Array.isArray(value) ? (value as Caption[]) : [];
+ const transcript = captions
+ .map((caption) => caption.text)
+ .join('')
+ .trim();
+
+ return (
+ <TimelineSequenceLabel label={transcript}>
+ {transcript}
+ </TimelineSequenceLabel>
+ );
+};
diff --git a/packages/studio/src/components/Timeline/timeline-sequence-label.tsx b/packages/studio/src/components/Timeline/timeline-sequence-label.tsx
new file mode 100644
index 0000000000..193e714bf2
--- /dev/null
+++ b/packages/studio/src/components/Timeline/timeline-sequence-label.tsx
@@ -0,0 +1,41 @@
+import React from 'react';
+import {ActionTooltip} from '../ActionTooltip';
+import {getTimelineColor} from './TimelineSelection';
+
+export const PACKED_LABEL_BOTTOM = 1;
+export const timelineSequenceLabelStyle: React.CSSProperties = {
+ flex: 1,
+ fontSize: 11,
+ lineHeight: '15px',
+ color: getTimelineColor(false, false),
+ maskImage: 'linear-gradient(to right, black calc(100% - 5px), transparent)',
+ minWidth: 0,
+ // Keep the fade in empty space unless the label is clipped.
+ paddingRight: 5,
+ whiteSpace: 'nowrap',
+ overflow: 'hidden',
+ WebkitMaskImage:
+ 'linear-gradient(to right, black calc(100% - 5px), transparent)',
+};
+
+export const TimelineSequenceLabel: React.FC<{
+ readonly label: string;
+ readonly children: React.ReactNode;
+}> = ({label, children}) => {
+ return (
+ <ActionTooltip
+ label={label}
+ shortcut={null}
+ delay={250}
+ dismissOnClick
+ triggerStyle={{
+ alignSelf: 'flex-start',
+ maxWidth: '100%',
+ minWidth: 0,
+ pointerEvents: 'auto',
+ }}
+ >
+ <span style={timelineSequenceLabelStyle}>{children}</span>
+ </ActionTooltip>
+ );
+};
diff --git a/packages/studio/src/helpers/colors.ts b/packages/studio/src/helpers/colors.ts
index a61490b445..355daddf67 100644
--- a/packages/studio/src/helpers/colors.ts
+++ b/packages/studio/src/helpers/colors.ts
@@ -202,6 +202,8 @@ export const TIMELINE_AUDIO_GRADIENT =
'var(--remotion-studio-timeline-audio-gradient)';
export const TIMELINE_VIDEO_GRADIENT =
'var(--remotion-studio-timeline-video-gradient)';
+export const TIMELINE_CAPTIONS_GRADIENT =
+ 'var(--remotion-studio-timeline-captions-gradient)';
export const getBackgroundFromHoverState = ({
selected,
diff --git a/packages/studio/src/helpers/studio-css-variables.ts b/packages/studio/src/helpers/studio-css-variables.ts
index 3f35ae79d7..27107012d5 100644
--- a/packages/studio/src/helpers/studio-css-variables.ts
+++ b/packages/studio/src/helpers/studio-css-variables.ts
@@ -76,6 +76,7 @@ export const studioCssVariables = `
--remotion-studio-color-picker-hue-gradient: linear-gradient(to right, #f00 0%, #ff0 17%, #0f0 33%, #0ff 50%, #00f 67%, #f0f 83%, #f00 100%);
--remotion-studio-timeline-audio-gradient: linear-gradient(rgb(16 171 58), rgb(43 165 63) 60%);
--remotion-studio-timeline-video-gradient: linear-gradient(to top, #8e44ad, #9b59b6);
+ --remotion-studio-timeline-captions-gradient: linear-gradient(to top, #b96516, #cc7828);
--remotion-studio-focus-color: #555;
--remotion-studio-color-picker-checker-color: var(--remotion-studio-white-alpha-15);
--remotion-studio-info-blue-border-color: rgba(59, 130, 246, 0.4);