Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

feat: add caption layout and styling engine

webadderalled8739f0404d
diff --git a/src/components/video-editor/captionLayout.ts b/src/components/video-editor/captionLayout.ts
new file mode 100644
index 00000000..ca23ac1c
--- /dev/null
+++ b/src/components/video-editor/captionLayout.ts
@@ -0,0 +1,463 @@
+import type {
+ AutoCaptionAnimation,
+ AutoCaptionSettings,
+ CaptionCue,
+ CaptionCueWord,
+} from "./types";
+
+export type CaptionWordState = "spoken" | "active" | "upcoming";
+
+export interface CaptionWordLayout {
+ text: string;
+ index: number;
+ forcedBreakBefore: boolean;
+ leadingSpace: boolean;
+ startMs: number;
+ endMs: number;
+ state: CaptionWordState;
+}
+
+export interface CaptionLineLayout {
+ words: CaptionWordLayout[];
+ width: number;
+ startWordIndex: number;
+ endWordIndex: number;
+}
+
+interface CaptionPageLayout {
+ lines: CaptionLineLayout[];
+ startMs: number;
+ endMs: number;
+}
+
+export interface ActiveCaptionLayout {
+ cue: CaptionCue;
+ blockKey: string;
+ visibleLines: CaptionLineLayout[];
+ hasWordTimings: boolean;
+ activeWordIndex: number;
+ activeWordProgress: number;
+ visiblePageIndex: number;
+ opacity: number;
+ translateY: number;
+ scale: number;
+}
+
+type CaptionSourceWord = {
+ cueId: string;
+ text: string;
+ forcedBreakBefore: boolean;
+ leadingSpace?: boolean;
+ startMs?: number;
+ endMs?: number;
+};
+
+const CAPTION_ENTER_MS = 180;
+const CAPTION_EXIT_MS = 140;
+const CAPTION_BLOCK_GAP_BREAK_MS = 500;
+
+function clamp(value: number, min: number, max: number) {
+ return Math.min(max, Math.max(min, value));
+}
+
+function clamp01(value: number) {
+ return clamp(value, 0, 1);
+}
+
+function splitCaptionWordsFromText(text: string) {
+ const sourceLines = text
+ .split(/\r?\n/)
+ .map((line) => line.trim())
+ .filter((line) => line.length > 0);
+ const words: CaptionSourceWord[] = [];
+
+ sourceLines.forEach((line, lineIndex) => {
+ line.split(/\s+/).filter(Boolean).forEach((word, wordIndex) => {
+ words.push({
+ cueId: "",
+ text: word,
+ forcedBreakBefore: lineIndex > 0 && wordIndex === 0,
+ });
+ });
+ });
+
+ return words;
+}
+
+function splitCaptionWords(cue: CaptionCue) {
+ if (Array.isArray(cue.words) && cue.words.length > 0) {
+ return cue.words
+ .filter((word): word is CaptionCueWord => Boolean(word && typeof word.text === "string"))
+ .map((word) => ({
+ cueId: cue.id,
+ text: word.text.trim(),
+ forcedBreakBefore: false,
+ leadingSpace: Boolean(word.leadingSpace),
+ startMs: word.startMs,
+ endMs: word.endMs,
+ }))
+ .filter((word) => word.text.length > 0);
+ }
+
+ return splitCaptionWordsFromText(cue.text);
+}
+
+function getActiveCaptionCue(cues: CaptionCue[], timeMs: number) {
+ for (const cue of cues) {
+ if (timeMs >= cue.startMs && timeMs <= cue.endMs) {
+ return cue;
+ }
+ }
+
+ return null;
+}
+
+function flattenCaptionWords(cues: CaptionCue[]) {
+ const flattened: Array<{
+ cueId: string;
+ text: string;
+ forcedBreakBefore: boolean;
+ leadingSpace: boolean;
+ startMs: number;
+ endMs: number;
+ hasRealTiming: boolean;
+ }> = [];
+
+ cues.forEach((cue, cueIndex) => {
+ const sourceWords = splitCaptionWords(cue);
+ if (sourceWords.length === 0) {
+ return;
+ }
+
+ const cueHasRealWordTimings = sourceWords.every(
+ (word) => isFiniteNumber(word.startMs) && isFiniteNumber(word.endMs) && word.endMs > word.startMs,
+ );
+ const cueDuration = Math.max(1, cue.endMs - cue.startMs);
+ const fallbackWordDuration = cueDuration / sourceWords.length;
+ const previousCue = cueIndex > 0 ? cues[cueIndex - 1] : null;
+ const shouldForceCueBreak =
+ previousCue !== null && cue.startMs - previousCue.endMs >= CAPTION_BLOCK_GAP_BREAK_MS;
+
+ sourceWords.forEach((word, wordIndex) => {
+ const fallbackStartMs = cue.startMs + fallbackWordDuration * wordIndex;
+ const fallbackEndMs =
+ wordIndex === sourceWords.length - 1
+ ? cue.endMs
+ : cue.startMs + fallbackWordDuration * (wordIndex + 1);
+ const startsMergedCue = wordIndex === 0 && !word.forcedBreakBefore && !shouldForceCueBreak;
+ const leadingSpace = wordIndex === 0
+ ? flattened.length > 0 && startsMergedCue
+ : (word.leadingSpace ?? true);
+
+ flattened.push({
+ cueId: cue.id,
+ text: word.text,
+ forcedBreakBefore:
+ word.forcedBreakBefore || (wordIndex === 0 && shouldForceCueBreak),
+ leadingSpace,
+ startMs: cueHasRealWordTimings ? word.startMs! : fallbackStartMs,
+ endMs: cueHasRealWordTimings ? word.endMs! : Math.max(fallbackStartMs + 1, fallbackEndMs),
+ hasRealTiming: cueHasRealWordTimings,
+ });
+ });
+ });
+
+ return flattened;
+}
+
+function getCaptionAnimationState(
+ animationStyle: AutoCaptionAnimation,
+ enterProgress: number,
+ exitProgress: number,
+) {
+ const visibility = Math.min(enterProgress, exitProgress);
+
+ switch (animationStyle) {
+ case "none":
+ return {
+ opacity: 1,
+ translateY: 0,
+ scale: 1,
+ };
+ case "fade":
+ return {
+ opacity: 0.3 + visibility * 0.7,
+ translateY: 0,
+ scale: 1,
+ };
+ case "rise":
+ return {
+ opacity: 0.25 + visibility * 0.75,
+ translateY: (1 - visibility) * 18,
+ scale: 0.985 + visibility * 0.015,
+ };
+ case "pop":
+ default:
+ return {
+ opacity: 0.35 + visibility * 0.65,
+ translateY: (1 - visibility) * 8,
+ scale: 0.94 + visibility * 0.06,
+ };
+ }
+}
+
+function buildCaptionPages(options: {
+ lines: CaptionLineLayout[];
+ words: CaptionWordLayout[];
+ maxRows: number;
+ hasWordTimings: boolean;
+ cue: CaptionCue;
+}) {
+ const pages: CaptionPageLayout[] = [];
+ const totalPages = Math.max(1, Math.ceil(options.lines.length / options.maxRows));
+ let lineIndex = 0;
+ let remainingLines = options.lines.length;
+
+ for (let pageIndex = 0; pageIndex < totalPages; pageIndex += 1) {
+ const remainingPages = totalPages - pageIndex;
+ const pageSize = Math.min(
+ options.maxRows,
+ Math.max(1, Math.ceil(remainingLines / remainingPages)),
+ );
+ const pageLines = options.lines.slice(lineIndex, lineIndex + pageSize);
+ if (pageLines.length === 0) {
+ continue;
+ }
+
+ lineIndex += pageLines.length;
+ remainingLines -= pageLines.length;
+
+ if (options.hasWordTimings) {
+ const firstWord = options.words[pageLines[0].startWordIndex];
+ const lastWord = options.words[pageLines[pageLines.length - 1].endWordIndex];
+ pages.push({
+ lines: pageLines,
+ startMs: firstWord?.startMs ?? options.cue.startMs,
+ endMs: lastWord?.endMs ?? options.cue.endMs,
+ });
+ continue;
+ }
+
+ pages.push({
+ lines: pageLines,
+ startMs: 0,
+ endMs: 0,
+ });
+ }
+
+ if (!options.hasWordTimings) {
+ const totalWords = Math.max(1, options.words.length);
+ const cueDuration = Math.max(1, options.cue.endMs - options.cue.startMs);
+ let elapsedMs = options.cue.startMs;
+
+ pages.forEach((page, pageIndex) => {
+ const pageWordCount = page.lines.reduce((count, line) => count + line.words.length, 0);
+ const proportionalDuration = Math.round((cueDuration * pageWordCount) / totalWords);
+ const nextElapsed = pageIndex === pages.length - 1
+ ? options.cue.endMs
+ : Math.min(options.cue.endMs, elapsedMs + Math.max(1, proportionalDuration));
+
+ page.startMs = elapsedMs;
+ page.endMs = Math.max(elapsedMs + 1, nextElapsed);
+ elapsedMs = page.endMs;
+ });
+ }
+
+ for (let index = 0; index < pages.length - 1; index += 1) {
+ pages[index].endMs = Math.max(pages[index].startMs + 1, pages[index + 1].startMs);
+ }
+
+ if (pages.length > 0) {
+ pages[pages.length - 1].endMs = Math.max(pages[pages.length - 1].startMs + 1, options.cue.endMs);
+ }
+
+ return pages;
+}
+
+function buildCaptionLines(options: {
+ words: CaptionWordLayout[];
+ maxWidthPx: number;
+ measureText: (text: string) => number;
+}) {
+ const wordCount = options.words.length;
+ const segmentWidths = options.words.map((word, index) => {
+ if (index === 0) {
+ return options.measureText(word.text);
+ }
+
+ return options.measureText(`${word.leadingSpace ? " " : ""}${word.text}`);
+ });
+ const plainWordWidths = options.words.map((word) => options.measureText(word.text));
+ const bestCost = new Array<number>(wordCount + 1).fill(Number.POSITIVE_INFINITY);
+ const nextBreak = new Array<number>(wordCount).fill(wordCount);
+
+ bestCost[wordCount] = 0;
+
+ for (let startIndex = wordCount - 1; startIndex >= 0; startIndex -= 1) {
+ let lineWidth = plainWordWidths[startIndex];
+
+ for (let endIndex = startIndex; endIndex < wordCount; endIndex += 1) {
+ if (endIndex > startIndex) {
+ if (options.words[endIndex].forcedBreakBefore) {
+ break;
+ }
+
+ lineWidth += segmentWidths[endIndex];
+ }
+
+ if (lineWidth > options.maxWidthPx && endIndex > startIndex) {
+ break;
+ }
+
+ const slack = Math.max(0, options.maxWidthPx - lineWidth);
+ const fullness = options.maxWidthPx <= 0 ? 1 : lineWidth / options.maxWidthPx;
+ const isLastLine = endIndex === wordCount - 1;
+ const linePenalty = isLastLine
+ ? slack * slack * 0.18
+ : slack * slack + (fullness < 0.72 ? (0.72 - fullness) * 26000 : 0);
+ const candidateCost = linePenalty + bestCost[endIndex + 1];
+
+ if (candidateCost < bestCost[startIndex]) {
+ bestCost[startIndex] = candidateCost;
+ nextBreak[startIndex] = endIndex + 1;
+ }
+ }
+ }
+
+ const lines: CaptionLineLayout[] = [];
+ let startIndex = 0;
+
+ while (startIndex < wordCount) {
+ const endIndexExclusive = Math.max(startIndex + 1, nextBreak[startIndex]);
+ const lineWords = options.words.slice(startIndex, endIndexExclusive).map((word, index) => ({
+ ...word,
+ leadingSpace: index === 0 ? false : word.leadingSpace,
+ }));
+ const width = lineWords.reduce(
+ (total, word, index) => total + options.measureText(`${index === 0 ? "" : word.leadingSpace ? " " : ""}${word.text}`),
+ 0,
+ );
+
+ lines.push({
+ words: lineWords,
+ width,
+ startWordIndex: lineWords[0].index,
+ endWordIndex: lineWords[lineWords.length - 1].index,
+ });
+
+ startIndex = endIndexExclusive;
+ }
+
+ return lines;
+}
+
+function getVisibleCaptionPageIndex(pages: CaptionPageLayout[], timeMs: number) {
+ for (let index = 0; index < pages.length; index += 1) {
+ if (timeMs >= pages[index].startMs && timeMs <= pages[index].endMs) {
+ return index;
+ }
+ }
+
+ return -1;
+}
+
+export function buildActiveCaptionLayout(options: {
+ cues: CaptionCue[];
+ timeMs: number;
+ settings: AutoCaptionSettings;
+ maxWidthPx: number;
+ measureText: (text: string) => number;
+}) {
+ const sourceWords = flattenCaptionWords(options.cues);
+ if (sourceWords.length === 0) {
+ return null;
+ }
+
+ const hasWordTimings = sourceWords.every((word) => word.hasRealTiming);
+
+ let activeWordIndex = -1;
+ if (hasWordTimings) {
+ activeWordIndex = sourceWords.findIndex((word) => options.timeMs >= word.startMs && options.timeMs < word.endMs);
+ if (activeWordIndex < 0) {
+ activeWordIndex = sourceWords.findIndex((word) => options.timeMs < word.startMs);
+ activeWordIndex = activeWordIndex < 0 ? sourceWords.length - 1 : clamp(activeWordIndex - 1, 0, sourceWords.length - 1);
+ }
+ }
+ const maxRows = clamp(Math.round(options.settings.maxRows || 1), 1, 4);
+
+ const words: CaptionWordLayout[] = sourceWords.map((word, index) => {
+ return {
+ text: word.text,
+ index,
+ forcedBreakBefore: word.forcedBreakBefore,
+ leadingSpace: word.leadingSpace,
+ startMs: word.startMs,
+ endMs: word.endMs,
+ state: !hasWordTimings
+ ? "spoken"
+ : index < activeWordIndex
+ ? "spoken"
+ : index === activeWordIndex
+ ? "active"
+ : "upcoming",
+ };
+ });
+
+ const lines = buildCaptionLines({
+ words,
+ maxWidthPx: options.maxWidthPx,
+ measureText: options.measureText,
+ });
+
+ const pages = buildCaptionPages({
+ lines,
+ words,
+ maxRows,
+ hasWordTimings: true,
+ cue: {
+ id: sourceWords[0].cueId,
+ startMs: sourceWords[0].startMs,
+ endMs: sourceWords[sourceWords.length - 1].endMs,
+ text: "",
+ },
+ });
+ const visiblePageIndex = getVisibleCaptionPageIndex(pages, options.timeMs);
+ if (visiblePageIndex < 0) {
+ return null;
+ }
+ const visiblePage = pages[visiblePageIndex] ?? null;
+ const visibleLines = visiblePage?.lines ?? lines.slice(0, maxRows);
+ const activeWord = activeWordIndex >= 0 ? words[activeWordIndex] : null;
+ const activeWordProgress = activeWord
+ ? clamp01((options.timeMs - activeWord.startMs) / Math.max(1, activeWord.endMs - activeWord.startMs))
+ : 0;
+ const animationStartMs = visiblePage?.startMs ?? sourceWords[0].startMs;
+ const animationEndMs = visiblePage?.endMs ?? sourceWords[sourceWords.length - 1].endMs;
+ const pageStartWordIndex = visibleLines[0]?.startWordIndex ?? 0;
+ const pageCueId = sourceWords[pageStartWordIndex]?.cueId ?? sourceWords[0].cueId;
+ const activeCue = getActiveCaptionCue(options.cues, options.timeMs) ?? options.cues.find((candidate) => candidate.id === pageCueId) ?? options.cues[0];
+ const enterProgress = clamp01((options.timeMs - animationStartMs) / CAPTION_ENTER_MS);
+ const exitProgress = clamp01((animationEndMs - options.timeMs) / CAPTION_EXIT_MS);
+ const animation = getCaptionAnimationState(
+ options.settings.animationStyle,
+ enterProgress,
+ exitProgress,
+ );
+
+ return {
+ cue: activeCue,
+ blockKey: `${Math.round(animationStartMs)}-${Math.round(animationEndMs)}`,
+ visibleLines,
+ hasWordTimings,
+ activeWordIndex,
+ activeWordProgress,
+ visiblePageIndex,
+ opacity: animation.opacity,
+ translateY: animation.translateY,
+ scale: animation.scale,
+ } satisfies ActiveCaptionLayout;
+}
+
+function isFiniteNumber(value: unknown): value is number {
+ return typeof value === "number" && Number.isFinite(value);
+}
\ No newline at end of file
diff --git a/src/components/video-editor/captionStyle.ts b/src/components/video-editor/captionStyle.ts
new file mode 100644
index 00000000..47953092
--- /dev/null
+++ b/src/components/video-editor/captionStyle.ts
@@ -0,0 +1,68 @@
+import type { CaptionWordState } from "./captionLayout";
+import { DEFAULT_AUTO_CAPTION_SETTINGS } from "./types";
+
+export const CAPTION_FONT_WEIGHT = 400;
+export const CAPTION_LINE_HEIGHT = 1.32;
+
+const DEFAULT_CAPTION_REFERENCE_WIDTH =
+ 1920 * (DEFAULT_AUTO_CAPTION_SETTINGS.maxWidth / 100);
+
+export function getCaptionTargetWidth(containerWidth: number, maxWidthPercent: number) {
+ return Math.max(1, containerWidth * (maxWidthPercent / 100));
+}
+
+export function getCaptionScaledFontSize(
+ fontSize: number,
+ containerWidth: number,
+ maxWidthPercent: number,
+) {
+ return Math.max(
+ 14,
+ fontSize * (getCaptionTargetWidth(containerWidth, maxWidthPercent) / DEFAULT_CAPTION_REFERENCE_WIDTH),
+ );
+}
+
+export function getCaptionPadding(fontSize: number) {
+ return {
+ x: fontSize * 1.1,
+ y: fontSize * 0.78,
+ };
+}
+
+export function getCaptionScaledRadius(radius: number, fontSize: number) {
+ const baseline = Math.max(1, DEFAULT_AUTO_CAPTION_SETTINGS.fontSize);
+ return Math.max(0, radius * (fontSize / baseline));
+}
+
+export function getCaptionTextMaxWidth(containerWidth: number, maxWidthPercent: number, fontSize: number) {
+ const padding = getCaptionPadding(fontSize);
+ return Math.max(fontSize * 4, getCaptionTargetWidth(containerWidth, maxWidthPercent) - padding.x * 2);
+}
+
+export function getCaptionWordVisualState(hasWordTimings: boolean, state: CaptionWordState) {
+ if (!hasWordTimings) {
+ return {
+ isInactive: false,
+ opacity: 1,
+ };
+ }
+
+ switch (state) {
+ case "upcoming":
+ return {
+ isInactive: true,
+ opacity: 0.82,
+ };
+ case "spoken":
+ return {
+ isInactive: false,
+ opacity: 0.72,
+ };
+ case "active":
+ default:
+ return {
+ isInactive: false,
+ opacity: 1,
+ };
+ }
+}
\ No newline at end of file