Skip to content

Open source CapCut alternative.

TypeScript

feat(site): captions emphasize the spoken word in one click

David Led22213c3d33e
diff --git a/site/src/cut/components/SubtitlesPanel.tools.ts b/site/src/cut/components/SubtitlesPanel.tools.ts
index 455ac07e..2145e44e 100644
--- a/site/src/cut/components/SubtitlesPanel.tools.ts
+++ b/site/src/cut/components/SubtitlesPanel.tools.ts
@@ -7,6 +7,7 @@
*/
import { bool, num, obj, str, type AiToolDef } from "@/cut/lib/aiToolDef";
+import { CAPTION_STYLES, WORD_ACCENT_MODES } from "@/cut/lib/subtitles";
export const SUBTITLES_TOOLS = [
{
@@ -78,15 +79,19 @@ export const SUBTITLES_TOOLS = [
{
name: "set_caption_look",
description:
- "Style the caption track: a text-videos look id in `look` sets everything at once (preset, size, font, karaoke highlight, accent color, frame background), or set the pieces yourself. Restyling captions is a change the user asks for — leave the track as it is when they only asked for the words. Karaoke — word_highlight — lights each word as it is spoken or sung, in the preview and the export burn-in; it needs word timings, which transcription and sync_lyrics leave on the cues.",
+ "Style the caption track: a text-videos look id in `look` sets everything at once (preset, size, font, karaoke highlight, accent color, frame background), or set the pieces yourself. Restyling captions is a change the user asks for — leave the track as it is when they only asked for the words. Karaoke — word_highlight — keeps the whole line up and emphasizes each word as it is spoken or sung, in the preview and the export burn-in; it needs word timings, which transcription and sync_lyrics leave on the cues. Naming an accent_mode turns it on, so \"make the words pop as I say them\" is one call with accent_mode: pop.",
inputSchema: obj({
look: str("Text-videos look id — sets the whole caption look"),
- style: str("Caption preset: clean, hook, punchy, minimal, editorial, typewriter, block, highlight, bubble, neon"),
+ style: str(`Caption preset: ${Object.keys(CAPTION_STYLES).join(", ")}`),
size: num("Caption size in px at a 1080-wide frame"),
font: str("Font id"),
- word_highlight: bool("Light each word as it lands (karaoke)"),
+ word_highlight: bool("Emphasize each word as it lands (karaoke)"),
accent_color: str("Highlight color"),
- accent_mode: { type: "string", enum: ["color", "underline", "box"], description: "How the word lights up" },
+ accent_mode: {
+ type: "string",
+ enum: WORD_ACCENT_MODES.map((m) => m.id),
+ description: `How the spoken word is emphasized — ${WORD_ACCENT_MODES.map((m) => `${m.id}: ${m.hint}`).join("; ")}`,
+ },
x: num("Caption center x 0..1"),
y: num("Caption center y 0..1"),
background: bool("With `look`, also set the project background (default: whatever the look does — the designed looks paint it, plain and over-footage leave it alone)"),
diff --git a/site/src/cut/components/SubtitlesPanel.tsx b/site/src/cut/components/SubtitlesPanel.tsx
index 88949622..c6dc0d7b 100644
--- a/site/src/cut/components/SubtitlesPanel.tsx
+++ b/site/src/cut/components/SubtitlesPanel.tsx
@@ -17,6 +17,7 @@ import {
laneCues,
subtitleLaneCount,
trackLocale,
+ WORD_ACCENT_MODES,
} from "@/cut/lib/subtitles";
import { useElapsed } from "@/cut/hooks/useElapsed";
import { useCutCaps } from "@/cut/lib/backend/hooks";
@@ -28,7 +29,6 @@ import {
type FontId,
type SubtitleCue,
type SubtitlesBlock,
- type WordAccentMode,
} from "@/cut/lib/types";
import { cn } from "@/lib/utils";
import { ScrollArea } from "@/components/ui/scroll-area";
@@ -183,12 +183,6 @@ export function SubtitlesPanel() {
);
}
-const ACCENT_MODES: { id: WordAccentMode; label: string }[] = [
- { id: "color", label: "Color" },
- { id: "underline", label: "Underline" },
- { id: "box", label: "Highlight" },
-];
-
/** The Options tab: caption visibility, the karaoke word highlight with its
* treatment and color, position reset for a dragged caption, and the
* subtitle-voiceover generator. */
@@ -201,6 +195,7 @@ function OptionsTab() {
// Effective word treatment: the caption style's defaults with the user's
// overrides on top, so the controls always show what's on the video.
const look = karaokeLook(captionStyle(subtitles.style), subtitles);
+ const on = !!subtitles.wordHighlight;
return (
<ScrollArea
@@ -249,51 +244,51 @@ function OptionsTab() {
/>
</div>
</div>
- <label className="flex min-h-8 items-center justify-between text-xs font-medium">
- Highlight spoken word
- <Switch
- className="sub-word-highlight"
- checked={!!subtitles.wordHighlight}
- onCheckedChange={(v) =>
- useEditor.getState().setSubtitlesView({ wordHighlight: v || undefined })
- }
- />
- </label>
- {subtitles.wordHighlight && (
- <>
- <div className="flex min-h-8 items-center justify-between text-xs font-medium">
- Word style
- <div className="sub-accent-mode flex rounded-lg border border-input p-0.5">
- {ACCENT_MODES.map((m) => (
- <button
- key={m.id}
- className={cn(
- "rounded-md px-2 py-1 text-[11.5px] font-medium transition-colors",
- look.mode === m.id
- ? "bg-neutral-900 text-white"
- : "text-muted-foreground hover:text-foreground"
- )}
- aria-pressed={look.mode === m.id}
- onClick={() => useEditor.getState().setSubtitlesView({ accentMode: m.id })}
- >
- {m.label}
- </button>
- ))}
- </div>
- </div>
- <div className="flex min-h-8 items-center justify-between text-xs font-medium">
- Word color
- <div className="sub-accent-color flex items-center">
- <ColorField
- value={look.color}
- label="Word color"
- onBegin={() => useEditor.getState().pushHistory()}
- onLive={(c) => useEditor.getState().setSubtitlesView({ accentColor: c })}
- onCommit={(c) => useEditor.getState().setSubtitlesView({ accentColor: c })}
- />
- </div>
+ <div className="flex flex-col gap-1.5 text-xs font-medium">
+ Emphasize the spoken word
+ <div className="sub-word-highlight flex rounded-lg border border-input p-0.5">
+ <button
+ className={cn(
+ "flex-1 rounded-md px-1.5 py-1 text-[11.5px] font-medium transition-colors",
+ on ? "text-muted-foreground hover:text-foreground" : "bg-neutral-900 text-white"
+ )}
+ aria-pressed={!on}
+ onClick={() => useEditor.getState().setSubtitlesView({ wordHighlight: undefined })}
+ >
+ Off
+ </button>
+ {WORD_ACCENT_MODES.map((m) => (
+ <button
+ key={m.id}
+ className={cn(
+ "sub-accent-mode flex-1 rounded-md px-1.5 py-1 text-[11.5px] font-medium transition-colors",
+ on && look.mode === m.id
+ ? "bg-neutral-900 text-white"
+ : "text-muted-foreground hover:text-foreground"
+ )}
+ aria-pressed={on && look.mode === m.id}
+ onClick={() =>
+ useEditor.getState().setSubtitlesView({ wordHighlight: true, accentMode: m.id })
+ }
+ >
+ {m.label}
+ </button>
+ ))}
+ </div>
+ </div>
+ {on && (
+ <div className="flex min-h-8 items-center justify-between text-xs font-medium">
+ Word color
+ <div className="sub-accent-color flex items-center">
+ <ColorField
+ value={look.color}
+ label="Word color"
+ onBegin={() => useEditor.getState().pushHistory()}
+ onLive={(c) => useEditor.getState().setSubtitlesView({ accentColor: c })}
+ onCommit={(c) => useEditor.getState().setSubtitlesView({ accentColor: c })}
+ />
</div>
- </>
+ </div>
)}
<div className="flex min-h-8 items-center justify-between text-xs font-medium">
Position
diff --git a/site/src/cut/lib/subtitles.ts b/site/src/cut/lib/subtitles.ts
index 1f5610ce..1db18275 100644
--- a/site/src/cut/lib/subtitles.ts
+++ b/site/src/cut/lib/subtitles.ts
@@ -1,3 +1,11 @@
+import {
+ contrastText as kitContrastText,
+ WORD_ACCENT_DEFAULT,
+ WORD_ACCENT_LABELS,
+ WORD_ACCENT_MODE_IDS,
+ WORD_ACCENT_NOTES,
+ wordWindows,
+} from "@donkeycut/effects-kit";
import { formatTime } from "./time";
import type {
CaptionStyleId,
@@ -99,27 +107,36 @@ export interface CaptionStyle {
}
/** Spoken-word accent for styles that don't pick their own. */
-export const DEFAULT_ACCENT = "#FFE94A";
+export const DEFAULT_ACCENT = WORD_ACCENT_DEFAULT;
+
+/** The word emphasis treatments, in the order every picker offers them —
+ * the caption panel, the element animation grid, and the assistant's tool
+ * enums all read this one list, so a new treatment reaches all of them. */
+export const WORD_ACCENT_MODES: { id: WordAccentMode; label: string; hint: string }[] =
+ WORD_ACCENT_MODE_IDS.map((id) => ({
+ id,
+ label: WORD_ACCENT_LABELS[id],
+ hint: WORD_ACCENT_NOTES[id],
+ }));
/** Black or white, whichever reads on the given hex fill. */
-export function contrastText(hex: string): string {
- const m = /^#?([0-9a-fA-F]{6})$/.exec(hex.trim());
- if (!m) return "#111114";
- const n = parseInt(m[1], 16);
- const lum = 0.299 * ((n >> 16) & 255) + 0.587 * ((n >> 8) & 255) + 0.114 * (n & 255);
- return lum > 150 ? "#111114" : "#FFFFFF";
-}
+export const contrastText = kitContrastText;
/** The effective karaoke treatment for the spoken word: user overrides on the
- * subtitles block win over the caption style's defaults, and a custom color
- * picks its own contrast text for the box treatment. */
+ * subtitles block win over the caption style's defaults, a swell with no
+ * accent named keeps the caption's own color, and a custom color picks its
+ * own contrast text for the box treatment. */
export function karaokeLook(
style: CaptionStyle,
subs?: { accentMode?: WordAccentMode; accentColor?: string }
): { mode: WordAccentMode; color: string; text: string } {
- const color = subs?.accentColor ?? style.accent ?? DEFAULT_ACCENT;
+ const mode = subs?.accentMode ?? style.accentMode ?? "underline";
+ // A swell says it with size, so with nobody naming an accent it keeps the
+ // color the rest of the caption is written in.
+ const color =
+ subs?.accentColor ?? style.accent ?? (mode === "pop" ? style.color : DEFAULT_ACCENT);
return {
- mode: subs?.accentMode ?? style.accentMode ?? "underline",
+ mode,
color,
text: subs?.accentColor ? contrastText(color) : style.accentText ?? contrastText(color),
};
@@ -320,25 +337,12 @@ export function wrapCaptionForSize(text: string, size: number, frameW: number):
* otherwise (hand edits, social rewrites) splits proportionally by word length.
*/
export function cueWordWindows(cue: SubtitleCue): { start: number; end: number }[] {
- const words = cue.text.trim().split(/\s+/).filter(Boolean);
- const n = words.length;
- if (n === 0) return [];
- const starts: number[] = [];
- if (cue.words && cue.words.length === n) {
- for (let i = 0; i < n; i++) {
- const t = i === 0 ? cue.start : Math.min(Math.max(cue.words[i].t0, cue.start), cue.end);
- starts.push(i > 0 && t < starts[i - 1] ? starts[i - 1] : t);
- }
- } else {
- const weights = words.map((w) => w.length + 1);
- const total = weights.reduce((a, b) => a + b, 0);
- let acc = 0;
- for (const w of weights) {
- starts.push(cue.start + ((cue.end - cue.start) * acc) / total);
- acc += w;
- }
- }
- return starts.map((start, i) => ({ start, end: i === n - 1 ? cue.end : starts[i + 1] }));
+ const dur = Math.max(0, cue.end - cue.start);
+ return wordWindows(
+ cue.text,
+ dur,
+ cue.words?.map((w) => w.t0 - cue.start)
+ ).map((w) => ({ start: cue.start + w.start, end: cue.start + w.end }));
}
/** A cue as a synthetic overlay, so captions ride the title pipeline. The style