Open source CapCut alternative.
TypeScript
feat(site): videos made of words [rebuild]
David Le273c152101bb
diff --git a/site/src/cut/components/SubtitlesPanel.tools.ts b/site/src/cut/components/SubtitlesPanel.tools.tsindex b428e8f8..42f482aa 100644--- a/site/src/cut/components/SubtitlesPanel.tools.ts+++ b/site/src/cut/components/SubtitlesPanel.tools.ts@@ -36,6 +36,39 @@ export const SUBTITLES_TOOLS = [track: num("Subtitle track to write, 0-based (default: the active track)"),}),},+ {+ name: "sync_lyrics",+ description:+ "Time a text the user already has — song lyrics, a script, a poem — to the audio in the cut, and put it on a caption track. Transcribes the cut first when the track is empty, then aligns the user's words to the recognized ones and keeps the user's text exactly as written: the recognizer supplies the clock, never the wording, which is what makes it work on singing it half-mishears. Words it missed take interpolated times from the words around them. Pass `look` to dress the track in a text-videos look at the same time (karaoke word highlight included). For lyrics as full-frame cards rather than captions, follow this with add_text_sequence from_captions.",+ inputSchema: obj(+ {+ text: str("The lyrics or script, one line per screen (newline separated)"),+ track: num("Subtitle track to write, 0-based (default: the active track)"),+ look: str("Text-videos look id to dress the captions in"),+ from: num("Ignore audio before this timeline second"),+ to: num("Ignore audio after this timeline second"),+ background: bool("With `look`, also set the project background to its frame color (default true)"),+ },+ ["text"]+ ),+ },+ {+ name: "set_caption_look",+ description:+ "Style the caption track: a text-videos look id in `look` sets everything at once (preset, size, font, karaoke highlight, accent color, frame background), or set the pieces yourself. Karaoke — word_highlight — lights each word as it is spoken or sung, in the preview and the export burn-in; it needs word timings, which transcription and sync_lyrics leave on the cues.",+ inputSchema: obj({+ look: str("Text-videos look id — sets the whole caption look"),+ style: str("Caption preset: clean, hook, punchy, minimal, editorial, typewriter, block, highlight, bubble, neon"),+ size: num("Caption size in px at a 1080-wide frame"),+ font: str("Font id"),+ word_highlight: bool("Light each word as it lands (karaoke)"),+ accent_color: str("Highlight color"),+ accent_mode: { type: "string", enum: ["color", "underline", "box"], description: "How the word lights up" },+ x: num("Caption center x 0..1"),+ y: num("Caption center y 0..1"),+ background: bool("With `look`, also set the project background (default true)"),+ }),+ },{name: "subtitles_add_track",description:diff --git a/site/src/cut/components/Timeline.tools.ts b/site/src/cut/components/Timeline.tools.tsindex f2171f39..729fae06 100644--- a/site/src/cut/components/Timeline.tools.ts+++ b/site/src/cut/components/Timeline.tools.ts@@ -181,6 +181,32 @@ export const TIMELINE_TOOLS = [lane: num("Element row (0 = the bottom row). Elements on one row never overlap — a title over a shape needs a higher row than the shape."),}, ["text"]),},+ {+ name: "add_text_sequence",+ description:+ "Place a run of text on the timeline in one call — a lyric video, a kinetic-typography passage, a quote sequence — styled by a named look. `look` carries the whole design: the frame color, the card colors it cycles line to line, the type, and how each line arrives (ids and what each is for are in the text-videos skill). Lines come from `lines` ({text, start, end} in timeline seconds, plus an optional color/size override per line), or from the caption track with from_captions when a sync already timed them. Cards land on `lane` and the words one row above, so they stack instead of sliding apart. It sets the project background to the look's frame color unless background is false — words over the user's own footage want a look with no cards, and the footage keeps playing underneath.",+ inputSchema: obj({+ look: str("Look id from the text-videos skill (default lyric-card)"),+ lines: {+ type: "array",+ description: "The run, in order. Omit times on a line and it follows the one before it.",+ items: obj(+ {+ text: str("The line (\\n for a break inside it)"),+ start: num("Timeline start s"),+ end: num("Timeline end s"),+ color: str("Override the look's text color for this line — for an accent word or a section change"),+ size: num("Override the look's size for this line"),+ },+ ["text"]+ ),+ },+ from_captions: num("Build the run from this caption track's cues (0-based) instead of `lines`"),+ lane: num("Row for the color cards; the words go one row above (default 0)"),+ cards: bool("Paint the look's color cards behind each line (default: whatever the look does)"),+ background: bool("Set the project background to the look's frame color (default true)"),+ }),+ },{name: "freeze_frame",description:diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex c324a831..943f20a5 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -78,9 +78,11 @@ import { renderProjectFrame } from "./exportRender";import { rasterCanvasToDataUrl } from "./raster";import { buildAiContext } from "./aiContext";import { sampleClipFrameData } from "./previewCanvas";-import { laneCues, subtitleLaneCount } from "./subtitles";+import { CAPTION_STYLES, laneCues, subtitleLaneCount } from "./subtitles";import { fuseTimeline, renderFusedTimeline } from "./watch/fuse";import { mergeWatch, mergeWatchNotes, uncoveredSeconds, unnotedSpans } from "./watch/merge";+import { syncLines, type TimedWord } from "./lyricSync";+import { requireTextLook } from "./textLooks";import { queueWatchSweep, withSweepPaused } from "./watch/sweep";import { synthesizeMusic } from "./audioGen";import { composeMusicPrompt } from "./composeGen";@@ -1296,6 +1298,179 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {return { id: o.id, ...(isTextOverlay(o) ? { text: o.text } : {}), start: o.start, end: o.end };},+ add_text_sequence: (s, input) => {+ const look = requireTextLook(input.look);+ const lane = isNum(input.lane) ? Math.max(0, Math.round(input.lane)) : 0;+ const wantCards =+ typeof input.cards === "boolean" ? input.cards : look.frame.cards.length > 0;+ const cards = wantCards && look.frame.cards.length > 0 ? look.frame.cards : [];+ const lines = textSequenceLines(s, input);+ if (lines.length === 0)+ throw new ToolError(+ "Pass lines: [{text, start, end}], or from_captions: true to build the sequence from the caption track."+ );+ if (input.background !== false) s.setBackground(look.frame.background);+ const textLane = lane + (cards.length > 0 ? 1 : 0);+ const made: { id: string; start: number; end: number; text: string }[] = [];+ lines.forEach((line, i) => {+ const card = cards.length > 0 ? cards[i % cards.length] : null;+ if (card) {+ useEditor.getState().seek(line.start);+ useEditor.getState().addShape("rect", { lane });+ const sel = useEditor.getState().selection;+ if (sel?.kind === "overlay")+ applyOverlayPatchSettled(sel.id, {+ start: line.start,+ end: line.end,+ x: 0.5,+ y: 0.5,+ w: 1,+ h: 1,+ fill: card,+ });+ }+ useEditor.getState().seek(line.start);+ useEditor.getState().addOverlay({ lane: textLane });+ const sel = useEditor.getState().selection;+ if (sel?.kind !== "overlay") return;+ const onCard =+ card && look.text.onCards ? look.text.onCards[i % look.text.onCards.length] : undefined;+ const fit = fitTextToFrame(line.text, look, frameOf(s.aspect), line.size ?? look.text.size);+ applyOverlayPatchSettled(sel.id, {+ start: line.start,+ end: line.end,+ text: fit.text,+ x: look.text.x,+ y: look.text.y,+ size: fit.size,+ font: look.text.font,+ weight: look.text.weight,+ color: line.color ?? onCard ?? look.text.color,+ italic: look.text.italic,+ letterSpacing: look.text.letterSpacing,+ lineHeight: look.text.lineHeight,+ stroke: look.text.stroke,+ plate: look.text.plate ?? false,+ plateColor: look.text.plateColor,+ plateOpacity: look.text.plateOpacity,+ shadow: look.text.shadow ?? false,+ align: "center",+ });+ if (look.motion.in || look.motion.out || look.motion.loop)+ useEditor.getState().updateOverlay(sel.id, {+ anim: {+ ...(look.motion.in ? { in: look.motion.in } : {}),+ ...(look.motion.out ? { out: look.motion.out } : {}),+ ...(look.motion.loop+ ? { loop: { style: look.motion.loop.style, speed: look.motion.loop.speed ?? 1 } }+ : {}),+ },+ });+ const o = useEditor.getState().overlays.find((x) => x.id === sel.id);+ if (o) made.push({ id: o.id, start: round2(o.start), end: round2(o.end), text: line.text });+ });+ return {+ look: look.id,+ count: made.length,+ lanes: cards.length > 0 ? { cards: lane, text: textLane } : { text: textLane },+ background: useEditor.getState().background,+ items: made.slice(0, 40),+ ...(made.length > 40 ? { itemsTruncated: true } : {}),+ note:+ cards.length > 0+ ? `Cards on row ${lane}, words on row ${textLane}, colors cycling ${look.frame.cards.join(" → ")}.`+ : `Words on row ${textLane} over the ${look.frame.background} frame.`,+ };+ },++ set_caption_look: (s, input) => {+ const patch: Parameters<typeof s.setSubtitlesView>[0] = {};+ if (typeof input.look === "string") {+ const look = requireTextLook(input.look);+ patch.style = look.captions.style;+ patch.wordHighlight = look.captions.wordHighlight;+ if (look.captions.size !== undefined) patch.size = look.captions.size;+ if (look.captions.font) patch.font = look.captions.font;+ if (look.captions.accentColor) patch.accentColor = look.captions.accentColor;+ if (look.captions.accentMode) patch.accentMode = look.captions.accentMode;+ if (input.background !== false) s.setBackground(look.frame.background);+ }+ if (typeof input.style === "string") {+ if (!(input.style in CAPTION_STYLES))+ throw new ToolError(`Unknown caption style. Use one of: ${Object.keys(CAPTION_STYLES).join(", ")}.`);+ patch.style = input.style as CaptionStyleId;+ }+ if (isNum(input.size)) patch.size = clamp(Math.round(input.size), 16, 200);+ if (typeof input.font === "string" && allFonts().some((f) => f.id === input.font))+ patch.font = input.font;+ if (typeof input.word_highlight === "boolean") patch.wordHighlight = input.word_highlight;+ if (typeof input.accent_color === "string") patch.accentColor = input.accent_color;+ if (input.accent_mode === "color" || input.accent_mode === "underline" || input.accent_mode === "box")+ patch.accentMode = input.accent_mode;+ if (isNum(input.y)) patch.y = clamp(input.y, 0.02, 0.98);+ if (isNum(input.x)) patch.x = clamp(input.x, 0.02, 0.98);+ if (Object.keys(patch).length === 0) throw new ToolError("Nothing to change.");+ s.setSubtitlesView(patch);+ const cur = useEditor.getState().subtitles;+ return {+ style: cur.style ?? "clean",+ size: cur.size,+ font: cur.font,+ wordHighlight: cur.wordHighlight === true,+ accentColor: cur.accentColor,+ accentMode: cur.accentMode,+ };+ },++ sync_lyrics: async (s, input) => {+ const text = typeof input.text === "string" ? input.text : "";+ const lines = text+ .split(/\r?\n/)+ .map((l) => l.trim())+ .filter((l) => l.length > 0);+ if (lines.length === 0) throw new ToolError("Pass text — the lyrics or script, one line per screen.");+ const lane = targetSubtitleTrack(input);+ // The recording's own clock. Existing captions serve; without them the+ // cut is transcribed first, which is what puts times on the words.+ let cues = laneCues(useEditor.getState().subtitles, lane);+ if (cues.length === 0) {+ await useEditor.getState().generateSubtitles();+ const cur = useEditor.getState();+ if (cur.subtitleStatus === "error")+ throw new ToolError(cur.subtitleError ?? "Could not transcribe the audio.");+ cues = laneCues(cur.subtitles, lane);+ }+ const st = useEditor.getState();+ const span = {+ from: isNum(input.from) ? Math.max(0, input.from) : 0,+ to: isNum(input.to) ? input.to : Math.max(projectDuration(st), cues[cues.length - 1]?.end ?? 0),+ };+ const timed = syncLines(lines, cueWords(cues), span);+ st.setLaneCues(lane, timed);+ if (typeof input.look === "string") {+ const look = requireTextLook(input.look);+ st.setSubtitlesView({+ style: look.captions.style,+ wordHighlight: look.captions.wordHighlight,+ ...(look.captions.size !== undefined ? { size: look.captions.size } : {}),+ ...(look.captions.font ? { font: look.captions.font } : {}),+ ...(look.captions.accentColor ? { accentColor: look.captions.accentColor } : {}),+ ...(look.captions.accentMode ? { accentMode: look.captions.accentMode } : {}),+ });+ if (input.background !== false) st.setBackground(look.frame.background);+ }+ const heardWords = cueWords(cues).length;+ return {+ track: lane,+ count: timed.length,+ lines: timed.slice(0, 40).map((l) => ({ start: l.start, end: l.end, text: l.text })),+ ...(timed.length > 40 ? { linesTruncated: true } : {}),+ note:+ `Your text is on the track verbatim, timed against ${heardWords} recognized words. ` ++ "Words the recognizer missed took interpolated times, so spot-check a line or two with seek + capture_frame and retime with update_cue.",+ };+ },+update_overlay: (s, input) => {const o = requireItem(s.overlays, input.id, "overlay element");s.updateOverlay(o.id, overlayPatch(input, o.kind ?? "text"));@@ -2629,6 +2804,107 @@ async function synthesizeVoiceover(/** Resolve a watch/listen target: a timeline clip (its source plus trim and* placement) or a bare project asset. */+/** The words a caption track carries, on the timeline clock: the+ * transcriber's own word times where it left them, and an even split of each+ * cue's span where it did not. */+function cueWords(cues: { start: number; end: number; text: string; words?: TimedWord[] }[]): TimedWord[] {+ const out: TimedWord[] = [];+ for (const c of cues) {+ if (c.words && c.words.length > 0) {+ out.push(...c.words);+ continue;+ }+ const parts = c.text.split(/\s+/).filter((w) => w.length > 0);+ const step = parts.length > 0 ? (c.end - c.start) / parts.length : 0;+ parts.forEach((w, i) => out.push({ w, t0: c.start + step * i, t1: c.start + step * (i + 1) }));+ }+ return out;+}++/** A line broken and sized to the frame it is going into. The look's size is+ * a ceiling written for a wide frame; a vertical cut, or a long line, gets+ * breaks at word boundaries first and a smaller size only when breaking is+ * not enough. Type the user hand-sized (a per-line `size`) is left alone+ * apart from the breaks. */+function fitTextToFrame(+ text: string,+ look: { text: { size: number; widthRatio?: number; lineHeight?: number } },+ frame: { w: number; h: number },+ size: number+): { text: string; size: number } {+ if (text.includes("\n")) return { text, size }; // the caller broke it themselves+ const ratio = look.text.widthRatio ?? 0.55;+ const usable = frame.w * 0.86;+ const words = text.split(/\s+/).filter((w) => w.length > 0);+ if (words.length === 0) return { text, size };+ const maxLines = 3;+ // The widest a line may run, in characters, at this size.+ const budget = Math.max(6, Math.floor(usable / (size * ratio)));+ const lines: string[] = [];+ let cur = "";+ for (const w of words) {+ const next = cur ? `${cur} ${w}` : w;+ if (next.length <= budget || cur === "") cur = next;+ else {+ lines.push(cur);+ cur = w;+ }+ }+ if (cur) lines.push(cur);+ const longest = lines.reduce((n, l) => Math.max(n, l.length), 0);+ // Still over the frame (one long word, or more rows than the look wants):+ // shrink until it sits inside.+ let fitted = size;+ if (longest > budget) fitted = Math.min(fitted, Math.floor(usable / (longest * ratio)));+ if (lines.length > maxLines) fitted = Math.min(fitted, Math.floor(size * (maxLines / lines.length)));+ const rows = lines.length;+ const lineH = look.text.lineHeight ?? 1.25;+ const tall = rows * fitted * lineH;+ if (tall > frame.h * 0.8) fitted = Math.floor((frame.h * 0.8) / (rows * lineH));+ return { text: lines.join("\n"), size: Math.max(16, Math.min(size, fitted)) };+}++/** The lines a text sequence should place: the ones passed in, or the caption+ * track read as a running order. */+function textSequenceLines(+ s: ReturnType<typeof useEditor.getState>,+ input: Record<string, unknown>+): { text: string; start: number; end: number; color?: string; size?: number }[] {+ if (input.from_captions === true || isNum(input.from_captions)) {+ const lane = isNum(input.from_captions) ? Math.max(0, Math.round(input.from_captions)) : 0;+ return laneCues(s.subtitles, lane).map((c) => ({+ text: c.text,+ start: round2(c.start),+ end: round2(c.end),+ }));+ }+ const raw = Array.isArray(input.lines) ? input.lines : [];+ const lines = raw+ .filter((l): l is Record<string, unknown> => typeof l === "object" && l !== null)+ .map((l) => ({+ text: typeof l.text === "string" ? l.text.trim() : "",+ start: isNum(l.start) ? Math.max(0, l.start) : NaN,+ end: isNum(l.end) ? l.end : NaN,+ ...(typeof l.color === "string" ? { color: l.color } : {}),+ ...(isNum(l.size) ? { size: clamp(Math.round(l.size), 16, 320) } : {}),+ }))+ .filter((l) => l.text.length > 0);+ // A line with no times follows the one before it; the first starts at the+ // playhead. Every line gets a real span, so nothing lands on top of+ // anything.+ let cursor = playheadAt();+ return lines.map((l, i) => {+ const start = Number.isNaN(l.start) ? cursor : l.start;+ const nextStart = lines[i + 1]?.start ?? NaN;+ const fallbackEnd = Number.isNaN(nextStart) ? start + 2 : nextStart;+ // Every line lands at least 0.2s long, whichever way its end was+ // decided — a following line that starts earlier cannot invert it.+ const end = Math.max(Number.isNaN(l.end) ? fallbackEnd : l.end, start + 0.2);+ cursor = end;+ return { ...l, start: round2(start), end: round2(end) };+ });+}+/** Spans watched this session that nobody has written down yet, by asset id.* Contact sheets fall out of the conversation as it grows, so the record has* to be written while the frames are still on screen; a second look at thediff --git a/site/src/cut/lib/lyricSync.test.ts b/site/src/cut/lib/lyricSync.test.tsnew file mode 100644index 00000000..29d67f7e--- /dev/null+++ b/site/src/cut/lib/lyricSync.test.ts@@ -0,0 +1,100 @@+import { describe, expect, test } from "bun:test";+import { alignWords, syncLines, type TimedWord } from "./lyricSync";++const heard = (pairs: [string, number, number][]): TimedWord[] =>+ pairs.map(([w, t0, t1]) => ({ w, t0, t1 }));++describe("alignWords", () => {+ test("matches through insertions and mishearings", () => {+ const a = ["i", "got", "half", "of", "my", "clients"];+ const b = ["uh", "i", "got", "hoff", "of", "my", "clients"];+ expect(alignWords(a, b)).toEqual([1, 2, -1, 4, 5, 6]);+ });++ test("nothing in common matches nothing", () => {+ expect(alignWords(["red", "blue"], ["cat", "dog"])).toEqual([-1, -1]);+ });+});++describe("syncLines", () => {+ test("lines take the times of the words the recognizer heard", () => {+ const lines = ["Hold on tight", "We're going home"];+ const words = heard([+ ["hold", 1, 1.4],+ ["on", 1.4, 1.8],+ ["tight", 1.8, 2.4],+ ["were", 3, 3.4],+ ["going", 3.4, 3.9],+ ["home", 3.9, 4.6],+ ]);+ const out = syncLines(lines, words, { from: 0, to: 6 });+ expect(out).toHaveLength(2);+ expect(out[0].start).toBeCloseTo(1, 2);+ expect(out[0].end).toBeCloseTo(2.4, 2);+ expect(out[1].start).toBeCloseTo(3, 2);+ expect(out[1].end).toBeCloseTo(4.6, 2);+ });++ test("the user's words are kept verbatim, whatever was heard", () => {+ const out = syncLines(["Don't stop believin'"], heard([+ ["dont", 2, 2.5],+ ["stop", 2.5, 3],+ ["believing", 3, 4],+ ]), { from: 0, to: 5 });+ expect(out[0].text).toBe("Don't stop believin'");+ expect(out[0].words?.map((w) => w.w)).toEqual(["Don't", "stop", "believin'"]);+ });++ test("words nobody heard are interpolated between the ones that were", () => {+ const out = syncLines(["one two three four"], heard([+ ["one", 0, 0.5],+ ["four", 3.5, 4],+ ]), { from: 0, to: 5 });+ const times = out[0].words!.map((w) => w.t0);+ expect(times[0]).toBeCloseTo(0, 2);+ expect(times[1]).toBeGreaterThan(0.5);+ expect(times[2]).toBeGreaterThan(times[1]);+ expect(times[3]).toBeCloseTo(3.5, 2);+ });++ test("lines stay in order, never overlap, and stay inside the span", () => {+ const out = syncLines(["a b", "c d", "e f"], heard([+ ["a", 9, 9.2],+ ["b", 1, 1.2],+ ["e", 4, 4.5],+ ]), { from: 0, to: 6 });+ for (let i = 0; i + 1 < out.length; i++) expect(out[i].end).toBeLessThanOrEqual(out[i + 1].start);+ expect(out[0].start).toBeGreaterThanOrEqual(0);+ expect(out[out.length - 1].end).toBeLessThanOrEqual(6);+ });++ test("a line whose end was never heard holds only as long as it reads", () => {+ // A short script against a long recording: the tail must not stretch to+ // the end of the source.+ const out = syncLines(["the opening line", "a line nobody heard"], heard([+ ["the", 1, 1.3],+ ["opening", 1.3, 1.8],+ ["line", 1.8, 2.2],+ ]), { from: 0, to: 600 });+ expect(out[1].end - out[1].start).toBeLessThan(4);+ expect(out[1].end).toBeLessThan(10);+ });++ test("a line crowded against the end of the span stays inside it", () => {+ const out = syncLines(["early", "late one"], heard([+ ["early", 0.2, 0.6],+ ["late", 5.9, 5.95],+ ["one", 5.95, 6],+ ]), { from: 0, to: 6 });+ for (const l of out) {+ expect(l.end).toBeLessThanOrEqual(6);+ expect(l.start).toBeGreaterThanOrEqual(0);+ }+ });++ test("a transcript with nothing recognizable still spreads the lines", () => {+ const out = syncLines(["first", "second", "third"], [], { from: 0, to: 9 });+ expect(out.map((l) => l.start)).toEqual([0, 3, 6]);+ expect(out[2].end).toBe(9);+ });+});diff --git a/site/src/cut/lib/lyricSync.ts b/site/src/cut/lib/lyricSync.tsnew file mode 100644index 00000000..bdccdb56--- /dev/null+++ b/site/src/cut/lib/lyricSync.ts@@ -0,0 +1,202 @@+/**+ * Syncing known text to a recording: the user has the lyrics (or the script)+ * and the audio, and wants the words on screen at the moment they are sung.+ *+ * The transcript says WHEN with good timing and WHAT with poor accuracy —+ * singing, doubled vocals and music defeat a recognizer. The user's text says+ * WHAT exactly. So the two are aligned word by word, the user's text is kept+ * verbatim, and the transcript is used only for its clock. Words the+ * recognizer missed take interpolated times between the words around them,+ * which is why a line still lands even when half of it went unheard.+ */++export interface TimedWord {+ t0: number;+ t1: number;+ w: string;+}++export interface TimedLine {+ start: number;+ end: number;+ text: string;+ words?: TimedWord[];+}++/** Comparison form of a word: letters, digits and nothing else. Apostrophes+ * go too, so "don't" and "dont" are the same word. */+const norm = (w: string): string =>+ w+ .toLowerCase()+ .normalize("NFKD")+ .replace(/[^\p{L}\p{N}]/gu, "");++const MATCH = 2;+const MISMATCH = -1;+const GAP = -1;++/**+ * Word-for-word alignment of `a` onto `b` (Needleman–Wunsch), returning for+ * each index of `a` the index it matched in `b`, or -1. Only real matches+ * (equal normalized words) are reported, so a mismatched pair the aligner+ * carried through for the sake of the path never claims a time.+ */+export function alignWords(a: string[], b: string[]): number[] {+ const n = a.length;+ const m = b.length;+ const out = new Array<number>(n).fill(-1);+ if (n === 0 || m === 0) return out;+ // Score matrix in one flat array: (n+1) x (m+1).+ const w = m + 1;+ const score = new Float32Array((n + 1) * w);+ for (let i = 1; i <= n; i++) score[i * w] = i * GAP;+ for (let j = 1; j <= m; j++) score[j] = j * GAP;+ for (let i = 1; i <= n; i++) {+ for (let j = 1; j <= m; j++) {+ const diag = score[(i - 1) * w + (j - 1)] + (a[i - 1] === b[j - 1] ? MATCH : MISMATCH);+ const up = score[(i - 1) * w + j] + GAP;+ const left = score[i * w + (j - 1)] + GAP;+ score[i * w + j] = Math.max(diag, up, left);+ }+ }+ let i = n;+ let j = m;+ while (i > 0 && j > 0) {+ const here = score[i * w + j];+ const diag = score[(i - 1) * w + (j - 1)] + (a[i - 1] === b[j - 1] ? MATCH : MISMATCH);+ if (here === diag) {+ if (a[i - 1] === b[j - 1]) out[i - 1] = j - 1;+ i--;+ j--;+ } else if (here === score[(i - 1) * w + j] + GAP) {+ i--;+ } else {+ j--;+ }+ }+ return out;+}++const MIN_LINE_S = 0.4;++/**+ * The user's lines, timed against a transcript of the same audio. Lines come+ * back in order, non-overlapping, inside [from, to], each carrying per-word+ * times so a karaoke highlight has something to follow.+ */+export function syncLines(+ lines: string[],+ transcript: TimedWord[],+ span: { from: number; to: number }+): TimedLine[] {+ const kept = lines.map((l) => l.trim()).filter((l) => l.length > 0);+ if (kept.length === 0) return [];+ const from = Math.max(0, Math.min(span.from, span.to));+ const to = Math.max(span.from, span.to);++ // Every lyric word, flat, remembering the line it came from.+ const flat: { line: number; raw: string; key: string }[] = [];+ kept.forEach((line, li) => {+ for (const raw of line.split(/\s+/)) {+ const key = norm(raw);+ if (key.length > 0) flat.push({ line: li, raw, key });+ }+ });+ const heard = transcript+ .filter((x) => x.t1 > x.t0 - 1e-6 && norm(x.w).length > 0)+ .sort((x, y) => x.t0 - y.t0);+ const matched = alignWords(+ flat.map((f) => f.key),+ heard.map((h) => norm(h.w))+ );++ // Anchored times, then interpolation across the words nobody heard: the+ // gap between two anchors is split evenly, and the head and tail of the+ // song run off the span's own edges.+ const t0 = new Array<number>(flat.length).fill(NaN);+ const t1 = new Array<number>(flat.length).fill(NaN);+ for (let k = 0; k < flat.length; k++) {+ const j = matched[k];+ if (j >= 0) {+ t0[k] = heard[j].t0;+ t1[k] = Math.max(heard[j].t1, heard[j].t0 + 0.05);+ }+ }+ const anchors = t0.map((v, k) => (Number.isNaN(v) ? -1 : k)).filter((k) => k >= 0);+ if (anchors.length === 0) {+ // Nothing was recognized: spread the lines evenly over the span so the+ // user still gets a timed draft to nudge.+ const each = Math.max(MIN_LINE_S, (to - from) / kept.length);+ return kept.map((text, i) => ({+ start: round2(from + i * each),+ end: round2(Math.min(to, from + (i + 1) * each)),+ text,+ }));+ }+ const fill = (loK: number, hiK: number, loT: number, hiT: number) => {+ const steps = hiK - loK;+ if (steps <= 0) return;+ const step = (hiT - loT) / steps;+ for (let k = loK + 1; k < hiK; k++) {+ t0[k] = loT + step * (k - loK);+ t1[k] = t0[k] + step * 0.9;+ }+ };+ fill(-1, anchors[0], from, t0[anchors[0]]);+ for (let a = 0; a + 1 < anchors.length; a++)+ fill(anchors[a], anchors[a + 1], t1[anchors[a]], t0[anchors[a + 1]]);+ const last = anchors[anchors.length - 1];+ fill(last, flat.length, t1[last], to);+ if (Number.isNaN(t0[flat.length - 1])) {+ t0[flat.length - 1] = t1[last];+ t1[flat.length - 1] = to;+ }++ // Words back into their lines.+ const out: TimedLine[] = kept.map((text) => ({ start: NaN, end: NaN, text }));+ // Whether a line's last word was actually heard. A line that ends on+ // interpolated time has no real end — it would otherwise run to wherever+ // the interpolation reached, which on a source that outlasts the text is+ // the far end of the recording.+ const heardTail = kept.map(() => false);+ for (let k = 0; k < flat.length; k++) heardTail[flat[k].line] = matched[k] >= 0;+ for (let k = 0; k < flat.length; k++) {+ const line = out[flat[k].line];+ const wordT0 = clamp(t0[k], from, to);+ const wordT1 = clamp(Math.max(t1[k], wordT0 + 0.05), from, to);+ line.start = Number.isNaN(line.start) ? wordT0 : Math.min(line.start, wordT0);+ line.end = Number.isNaN(line.end) ? wordT1 : Math.max(line.end, wordT1);+ (line.words ??= []).push({ t0: round2(wordT0), t1: round2(wordT1), w: flat[k].raw });+ }++ // Order, no overlap, nothing shorter than a beat, nothing past the span.+ // The span itself is the outer bound: a line crowded against the end holds+ // for whatever room is left rather than running past `to`.+ let cursor = from;+ for (let i = 0; i < out.length; i++) {+ const line = out[i];+ if (Number.isNaN(line.start)) line.start = cursor;+ line.start = clamp(Math.max(line.start, cursor), from, Math.max(from, to - MIN_LINE_S));+ const nextStart = out[i + 1]?.start;+ const ceiling = Number.isNaN(nextStart ?? NaN) ? to : Math.min(to, nextStart ?? to);+ if (!(line.end > line.start)) line.end = line.start + MIN_LINE_S;+ // A line nobody was heard finishing holds for as long as it takes to+ // read, and no longer.+ if (!heardTail[i]) {+ const words = line.words?.length ?? line.text.split(/\s+/).length;+ line.end = Math.min(line.end, line.start + Math.max(1, 0.4 * words + 0.6));+ }+ line.end = clamp(+ Math.min(Math.max(line.end, line.start + MIN_LINE_S), Math.max(ceiling, line.start + MIN_LINE_S)),+ line.start,+ to+ );+ cursor = line.end;+ line.start = round2(line.start);+ line.end = round2(line.end);+ }+ return out;+}++const round2 = (n: number) => Math.round(n * 100) / 100;+const clamp = (n: number, lo: number, hi: number) => Math.min(hi, Math.max(lo, n));diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.tsindex d89db231..6f24d2f3 100644--- a/site/src/cut/lib/store.ts+++ b/site/src/cut/lib/store.ts@@ -536,6 +536,9 @@ export interface EditorState {* right past occupied stretches on the cue's own lane so cues never overlap,* and detach the word timings (they described the old window). One undo step. */setCueTiming: (id: string, start: number, end: number) => void;+ /** Replace one subtitle track's captions wholesale, in timeline seconds —+ * what a lyric sync writes once it knows when every line lands. */+ setLaneCues: (lane: number, cues: { start: number; end: number; text: string; words?: { t0: number; t1: number; w: string }[] }[]) => void;/** Re-time listed cues to a generated voiceover: set each cue's [start, end]* and spread its words across the new span (the AI voice paces differently* from the original recording, so the word highlighter would otherwise drift). */@@ -4040,6 +4043,29 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {},})),+ setLaneCues: (lane, cues) => {+ push();+ const made: SubtitleCue[] = cues+ .filter((c) => c.text.trim().length > 0 && c.end > c.start)+ .map((c) => ({+ id: uid(),+ start: Math.max(0, c.start),+ end: c.end,+ text: c.text.trim(),+ ...(c.words && c.words.length > 0 ? { words: c.words } : {}),+ ...(lane > 0 ? { lane } : {}),+ }));+ set((s) => ({+ subtitles: {+ ...s.subtitles,+ cues: [...s.subtitles.cues.filter((c) => (c.lane ?? 0) !== lane), ...made].sort(+ (a, b) => a.start - b.start+ ),+ },+ subtitleStatus: made.length > 0 ? "ready" : s.subtitleStatus,+ }));+ },+setCueTiming: (id, start, end) => {const cue = get().subtitles.cues.find((c) => c.id === id);if (!cue) return;diff --git a/site/src/cut/lib/textLooks.ts b/site/src/cut/lib/textLooks.tsnew file mode 100644index 00000000..00fb3c3c--- /dev/null+++ b/site/src/cut/lib/textLooks.ts@@ -0,0 +1,275 @@+import type { OverlayAnimStyle, OverlayLoopStyle, WordAccentMode } from "@donkeycut/effects-kit";+import type { CaptionStyleId, FontId } from "./types";++/**+ * Named looks for text-driven videos — lyric videos, kinetic typography,+ * quote cards, anything whose picture is words. One entry describes a whole+ * look: the frame behind the words, the type, how a line arrives and leaves,+ * and the caption settings that carry the same look when the words ride a+ * caption track instead of elements.+ *+ * This registry is the single source: add_text_sequence and set_caption_look+ * take these ids, and the text-videos skill is written from the entries, so a+ * new look becomes a thing the assistant knows the moment it lands here.+ */++export interface TextLook {+ id: string;+ label: string;+ /** What this look is for — the line the assistant picks from. */+ when: string;+ /** The frame behind the words. `background` is the project's frame color.+ * `cards` are full-frame color cards a sequence cycles through, one per+ * line; empty means the words sit straight on the frame or on footage. */+ frame: { background: string; cards: string[] };+ /** The type. `onCards` is the text color per card color, index-matched to+ * `frame.cards`; without cards, `color` carries every line. */+ text: {+ font: FontId;+ size: number;+ weight: 400 | 700;+ color: string;+ onCards?: string[];+ italic?: boolean;+ letterSpacing?: number;+ lineHeight?: number;+ stroke?: { color: string; width: number };+ plate?: boolean;+ plateColor?: string;+ plateOpacity?: number;+ shadow?: boolean;+ x: number;+ y: number;+ /** Average glyph width as a fraction of the type size, for this face.+ * A sequence uses it to break and size a line so it fits the frame it is+ * actually in — the same 132px that fills a 16:9 frame runs off a 9:16+ * one. Absent = 0.55, an ordinary sans. */+ widthRatio?: number;+ };+ /** The color that marks the word that matters: the karaoke highlight, an+ * accent line, a marker scribble. */+ accent: string;+ /** How each line arrives, leaves, and moves while it holds. */+ motion: {+ in?: { style: OverlayAnimStyle; seconds: number };+ out?: { style: OverlayAnimStyle; seconds: number };+ loop?: { style: OverlayLoopStyle; speed?: number };+ };+ /** The same look on a caption track — what set_caption_look writes when it+ * is handed this look id. */+ captions: {+ style: CaptionStyleId;+ wordHighlight: boolean;+ size?: number;+ font?: FontId;+ accentColor?: string;+ accentMode?: WordAccentMode;+ };+}++export const TEXT_LOOKS: Record<string, TextLook> = {+ "lyric-card": {+ id: "lyric-card",+ label: "Lyric cards",+ when: "One line at a time on a full-frame color card, the color changing line to line. The loud, readable default for lyrics and punchy talking-head text.",+ frame: { background: "#111114", cards: ["#111114", "#FFFFFF", "#FF5500"] },+ text: {+ font: "archivo-black",+ size: 104,+ weight: 700,+ color: "#FFFFFF",+ onCards: ["#FFFFFF", "#111114", "#FFFFFF"],+ lineHeight: 1.1,+ shadow: false,+ x: 0.5,+ y: 0.5,+ widthRatio: 0.62,+ },+ accent: "#FF5500",+ motion: { in: { style: "pop", seconds: 0.35 } },+ captions: { style: "punchy", wordHighlight: true, accentColor: "#FF5500" },+ },++ "karaoke-glow": {+ id: "karaoke-glow",+ label: "Karaoke glow",+ when: "The whole line stays up and each word lights as it is sung. The look for singing you can follow — needs word timings, so it rides a caption track.",+ frame: { background: "#0B0B12", cards: [] },+ text: {+ font: "montserrat",+ size: 84,+ weight: 700,+ color: "#FFFFFF",+ lineHeight: 1.2,+ shadow: true,+ x: 0.5,+ y: 0.5,+ widthRatio: 0.56,+ },+ accent: "#FFE94A",+ motion: { in: { style: "fade", seconds: 0.25 }, out: { style: "fade", seconds: 0.25 } },+ captions: { style: "highlight", wordHighlight: true, size: 84, accentColor: "#FFE94A" },+ },++ "neon-club": {+ id: "neon-club",+ label: "Neon club",+ when: "Near-black frame, tall condensed caps, an electric accent. Dance, hip-hop, anything with a night to it.",+ frame: { background: "#07070C", cards: [] },+ text: {+ font: "bebas",+ size: 132,+ weight: 400,+ color: "#F5F5FF",+ letterSpacing: 0.04,+ stroke: { color: "#22D3EE", width: 0.02 },+ lineHeight: 1.05,+ shadow: true,+ x: 0.5,+ y: 0.5,+ widthRatio: 0.42,+ },+ accent: "#F0ABFC",+ motion: { in: { style: "streak", seconds: 0.4 }, loop: { style: "flicker", speed: 0.6 } },+ captions: { style: "neon", wordHighlight: true, accentColor: "#F0ABFC" },+ },++ "marker-note": {+ id: "marker-note",+ label: "Marker note",+ when: "Handwriting on paper, ink arriving as it is written. Personal, confessional, singer-songwriter, or a note-to-camera.",+ frame: { background: "#F4F1E8", cards: ["#F4F1E8"] },+ text: {+ font: "caveat",+ size: 118,+ weight: 700,+ color: "#1B1B1B",+ onCards: ["#1B1B1B"],+ lineHeight: 1.15,+ shadow: false,+ x: 0.5,+ y: 0.48,+ widthRatio: 0.42,+ },+ accent: "#FF5500",+ motion: { in: { style: "typewriter", seconds: 0.5 } },+ captions: { style: "typewriter", wordHighlight: false, font: "caveat", accentColor: "#FF5500" },+ },++ "serif-mood": {+ id: "serif-mood",+ label: "Serif mood",+ when: "A wide-set serif holding still on black, fading between lines. Ballads, poetry, closing credits, anything slow.",+ frame: { background: "#000000", cards: [] },+ text: {+ font: "dm-serif",+ size: 76,+ weight: 400,+ color: "#F2EDE4",+ letterSpacing: 0.06,+ lineHeight: 1.4,+ shadow: false,+ x: 0.5,+ y: 0.5,+ widthRatio: 0.5,+ },+ accent: "#C9A227",+ motion: { in: { style: "fade", seconds: 0.8 }, out: { style: "fade", seconds: 0.8 } },+ captions: { style: "minimal", wordHighlight: false, font: "dm-serif", accentColor: "#C9A227" },+ },++ "type-out": {+ id: "type-out",+ label: "Type out",+ when: "Mono on a dark screen, each line typed in. Explainers, code, anything that should read like a terminal.",+ frame: { background: "#101317", cards: [] },+ text: {+ font: "mono",+ size: 62,+ weight: 400,+ color: "#9BE29B",+ letterSpacing: 0.02,+ lineHeight: 1.5,+ shadow: false,+ x: 0.5,+ y: 0.5,+ widthRatio: 0.6,+ },+ accent: "#F2F2F2",+ motion: { in: { style: "typewriter", seconds: 0.6 } },+ captions: { style: "typewriter", wordHighlight: false, font: "mono" },+ },++ "over-footage": {+ id: "over-footage",+ label: "Over footage",+ when: "No cards at all — heavy outlined white type over whatever is playing. The look for words on the user's own video.",+ frame: { background: "#000000", cards: [] },+ text: {+ font: "anton",+ size: 96,+ weight: 400,+ color: "#FFFFFF",+ stroke: { color: "#000000", width: 0.035 },+ lineHeight: 1.15,+ shadow: true,+ x: 0.5,+ y: 0.72,+ widthRatio: 0.46,+ },+ accent: "#FFE94A",+ motion: { in: { style: "rise", seconds: 0.35 }, out: { style: "fade", seconds: 0.2 } },+ captions: { style: "hook", wordHighlight: true, accentColor: "#FFE94A" },+ },++ "kinetic-scatter": {+ id: "kinetic-scatter",+ label: "Kinetic scatter",+ when: "Letters fly in and settle, the card color shifting under them. Busy, energetic, made to be watched twice.",+ frame: { background: "#1B1036", cards: ["#1B1036", "#2563EB", "#DB2777"] },+ text: {+ font: "poppins",+ size: 98,+ weight: 700,+ color: "#FFFFFF",+ onCards: ["#FFFFFF", "#FFFFFF", "#FFFFFF"],+ lineHeight: 1.1,+ shadow: false,+ x: 0.5,+ y: 0.5,+ widthRatio: 0.58,+ },+ accent: "#FDE047",+ motion: { in: { style: "scatter", seconds: 0.5 }, out: { style: "converge", seconds: 0.3 } },+ captions: { style: "bubble", wordHighlight: true, accentColor: "#FDE047" },+ },+};++export const TEXT_LOOK_IDS = Object.keys(TEXT_LOOKS);++export const DEFAULT_TEXT_LOOK = "lyric-card";++export function textLook(id: string | undefined): TextLook {+ return TEXT_LOOKS[id ?? ""] ?? TEXT_LOOKS[DEFAULT_TEXT_LOOK];+}++/** The look a caller named, or the default when they named none. An id the+ * registry does not hold throws: a look decides the frame color, the cards,+ * the type and the motion of the whole project, so quietly substituting a+ * different one would redress the video behind the caller's back. */+export function requireTextLook(id: unknown): TextLook {+ if (id === undefined || id === null || id === "") return TEXT_LOOKS[DEFAULT_TEXT_LOOK];+ const look = typeof id === "string" ? TEXT_LOOKS[id] : undefined;+ if (!look)+ throw new Error(`Unknown look "${String(id)}". Use one of: ${TEXT_LOOK_IDS.join(", ")}.`);+ return look;+}++/** The registry as prompt text: one line per look, so the skill teaches+ * whatever the registry currently holds. */+export const textLookCatalog = (): string =>+ TEXT_LOOK_IDS.map((id) => {+ const l = TEXT_LOOKS[id];+ const cards = l.frame.cards.length > 0 ? `cards ${l.frame.cards.join("/")}` : "no cards";+ return `- ${l.id} (${l.label}): ${l.when} Frame ${l.frame.background}, ${cards}; ${l.text.font} ${l.text.size}px; ${l.motion.in?.style ?? "no"} in; captions ${l.captions.style}${l.captions.wordHighlight ? " + karaoke" : ""}.`;+ }).join("\n");diff --git a/site/src/cut/lib/textVideo.test.ts b/site/src/cut/lib/textVideo.test.tsnew file mode 100644index 00000000..01b24c4a--- /dev/null+++ b/site/src/cut/lib/textVideo.test.ts@@ -0,0 +1,189 @@+import { beforeEach, describe, expect, test } from "bun:test";+import { runAiTool } from "./aiTools";+import { useEditor } from "./store";+import { TEXT_LOOKS, textLookCatalog } from "./textLooks";+import { emptySubtitles, frameOf, isTextOverlay, isShapeOverlay, type SubtitleCue } from "./types";++/**+ * Text-driven videos: a run of lines becomes a stack of color cards and words+ * carrying a named look, and the same look dresses a caption track.+ */++const reset = () =>+ useEditor.setState({+ projectId: "p1",+ aspect: "16:9",+ assets: [],+ clips: [],+ overlays: [],+ subtitles: emptySubtitles(),+ background: "#000000",+ });++const overlays = () => useEditor.getState().overlays;++describe("add_text_sequence", () => {+ beforeEach(reset);++ test("lays cards under words on their own rows, in the look's colors", async () => {+ const look = TEXT_LOOKS["lyric-card"];+ await runAiTool("add_text_sequence", {+ look: "lyric-card",+ lines: [+ { text: "first line", start: 0, end: 2 },+ { text: "second line", start: 2, end: 4 },+ { text: "third line", start: 4, end: 6 },+ ],+ });+ const cards = overlays().filter(isShapeOverlay);+ const words = overlays().filter(isTextOverlay);+ expect(cards).toHaveLength(3);+ expect(words).toHaveLength(3);+ expect(cards.map((c) => c.lane ?? 0)).toEqual([0, 0, 0]);+ expect(words.map((w) => w.lane ?? 0)).toEqual([1, 1, 1]);+ expect(cards.map((c) => c.fill)).toEqual(look.frame.cards);+ expect(words.map((w) => w.color)).toEqual(look.text.onCards!);+ expect(words.map((w) => w.text)).toEqual(["first line", "second line", "third line"]);+ expect(words[0].font).toBe(look.text.font);+ expect(words[0].anim?.in?.style).toBe(look.motion.in!.style);+ expect(useEditor.getState().background).toBe(look.frame.background);+ });++ test("a look with no cards puts the words straight on the frame", async () => {+ await runAiTool("add_text_sequence", {+ look: "over-footage",+ background: false,+ lines: [{ text: "over the shot", start: 1, end: 3 }],+ });+ expect(overlays().filter(isShapeOverlay)).toHaveLength(0);+ const word = overlays().filter(isTextOverlay)[0];+ expect(word.lane ?? 0).toBe(0);+ expect(word.y).toBeCloseTo(TEXT_LOOKS["over-footage"].text.y, 2);+ expect(useEditor.getState().background).toBe("#000000");+ });++ test("lines with no times follow one another", async () => {+ await runAiTool("add_text_sequence", {+ look: "serif-mood",+ lines: [{ text: "one" }, { text: "two" }, { text: "three" }],+ });+ const words = overlays().filter(isTextOverlay);+ for (let i = 0; i + 1 < words.length; i++)+ expect(words[i].end).toBeLessThanOrEqual(words[i + 1].start + 1e-6);+ });++ test("a long line breaks and shrinks to fit the frame it is in", async () => {+ useEditor.setState({ aspect: "9:16" });+ await runAiTool("add_text_sequence", {+ look: "neon-club",+ lines: [{ text: "Nobody knows my name tonight", start: 0, end: 2 }],+ });+ const word = overlays().filter(isTextOverlay)[0];+ expect(word.text).toContain("\n");+ const frame = frameOf("9:16");+ const ratio = TEXT_LOOKS["neon-club"].text.widthRatio!;+ for (const line of word.text.split("\n"))+ expect(line.length * word.size * ratio).toBeLessThanOrEqual(frame.w * 0.9);+ });++ test("a per-line color overrides the look", async () => {+ await runAiTool("add_text_sequence", {+ look: "lyric-card",+ lines: [{ text: "accent", start: 0, end: 1, color: "#00FF00" }],+ });+ expect(overlays().filter(isTextOverlay)[0].color).toBe("#00FF00");+ });++ test("the caption track can be the running order", async () => {+ const cues: SubtitleCue[] = [+ { id: "q1", start: 0.5, end: 1.5, text: "hold on" },+ { id: "q2", start: 1.5, end: 3, text: "we're going home" },+ ];+ useEditor.setState({ subtitles: { ...emptySubtitles(), cues } });+ await runAiTool("add_text_sequence", { look: "kinetic-scatter", from_captions: 0 });+ const words = overlays().filter(isTextOverlay);+ expect(words.map((w) => w.text)).toEqual(["hold on", "we're going home"]);+ expect(words[0].start).toBeCloseTo(0.5, 2);+ expect(words[1].end).toBeCloseTo(3, 2);+ });++ test("an unknown look is refused rather than quietly swapped", async () => {+ let message = "";+ try {+ await runAiTool("add_text_sequence", {+ look: "karaoke",+ lines: [{ text: "a line", start: 0, end: 1 }],+ });+ } catch (err) {+ message = err instanceof Error ? err.message : String(err);+ }+ expect(message).toContain("Unknown look");+ expect(overlays()).toHaveLength(0);+ expect(useEditor.getState().background).toBe("#000000");+ });++ test("a line with no end of its own never inverts", async () => {+ await runAiTool("add_text_sequence", {+ look: "serif-mood",+ lines: [{ text: "no times" }, { text: "starts earlier", start: 0, end: 1 }],+ });+ for (const w of overlays().filter(isTextOverlay)) expect(w.end).toBeGreaterThan(w.start);+ });++ test("a run with nothing to place is refused", async () => {+ let message = "";+ try {+ await runAiTool("add_text_sequence", { look: "lyric-card", lines: [] });+ } catch (err) {+ message = err instanceof Error ? err.message : String(err);+ }+ expect(message).toContain("lines");+ });+});++describe("set_caption_look", () => {+ beforeEach(reset);++ test("a look id sets the whole caption look", async () => {+ await runAiTool("set_caption_look", { look: "karaoke-glow" });+ const look = TEXT_LOOKS["karaoke-glow"];+ const subs = useEditor.getState().subtitles;+ expect(subs.style).toBe(look.captions.style);+ expect(subs.wordHighlight).toBe(true);+ expect(subs.accentColor).toBe(look.captions.accentColor);+ expect(useEditor.getState().background).toBe(look.frame.background);+ });++ test("fields can be set on their own", async () => {+ await runAiTool("set_caption_look", { style: "neon", word_highlight: false, size: 70 });+ const subs = useEditor.getState().subtitles;+ expect(subs.style).toBe("neon");+ expect(subs.wordHighlight).toBe(false);+ expect(subs.size).toBe(70);+ });++ test("an unknown preset is refused", async () => {+ let message = "";+ try {+ await runAiTool("set_caption_look", { style: "sparkle" });+ } catch (err) {+ message = err instanceof Error ? err.message : String(err);+ }+ expect(message).toContain("caption style");+ });+});++describe("the look registry", () => {+ test("every look reads as one prompt line", () => {+ const lines = textLookCatalog().split("\n");+ expect(lines).toHaveLength(Object.keys(TEXT_LOOKS).length);+ for (const l of lines) expect(l.startsWith("- ")).toBe(true);+ });++ test("card looks say what color the words go in on each card", () => {+ for (const look of Object.values(TEXT_LOOKS)) {+ if (look.frame.cards.length === 0) continue;+ expect(look.text.onCards).toHaveLength(look.frame.cards.length);+ }+ });+});diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex f107a25b..290d5f4b 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -10,6 +10,7 @@*/import { AI_PANEL_TOOLS } from "@/cut/components/AiPanel.tools";+import { textLookCatalog } from "@/cut/lib/textLooks";import { OVERLAY_ANIMATION_TOOLS } from "@/cut/components/AnimationTiles.tools";import { AUDIO_TOOLS } from "@/cut/components/AudioPanel.tools";import { EDITOR_TOOLS } from "@/cut/components/Editor.tools";@@ -156,7 +157,31 @@ Good TikTok titles: short punchy lines, high contrast (white/yellow + shadow orTasteful graphics: shapes read best as accents (a highlight box behind a stat, an arrow pointing at the subject, a color bar under a title) — semi-transparent fills (fill_opacity 0.2–0.5) sit better over footage than solid blocks. One or two stickers a scene; size 0.15–0.3 of the frame width.In the UI: the timeline toolbar adds Text; the Elements side-panel tab browses stickers and shapes and creates sticker images, and the Effects tab holds the effects — a click in any panel picks a tile, and dragging one onto the timeline is what places it; dragging in the preview places an element, its corner handle resizes, the top handle rotates; the Inspector edits every field.`,+ "text-videos": `# Text-driven videos: lyrics, kinetic type, quote cards+A video whose picture is words. Three pieces: the frame behind them (set_background, or the user's footage), the words themselves, and the clock that says when each line lands.++**Two ways to carry the words.** A caption track is one styled row of text that follows a voice — it holds word timings, so karaoke (each word lighting as it is sung) is only possible here, and one call restyles every line. Elements are separate titles and cards on the timeline — per-line color, per-glyph motion, a color card behind each line — which is what kinetic typography and lyric cards are made of. Captions for a song you want to sing along to; elements when each line is its own picture. Both burn into the export.++**The looks.** Every look sets the frame color, the card colors it cycles, the type, the motion, and the caption equivalent, so a whole video comes from one id:+${textLookCatalog()}++**A lyric video, from an audio track and the words.** The user has the song and the lyrics.+1. The song is in the cut (a soundtrack clip). \`get_state\` confirms it and gives the duration.+2. sync_lyrics with the lyrics text, one line per screen. It transcribes the cut, aligns the user's words to what it recognized, and writes the user's text — verbatim — onto a caption track with per-word times. Singing defeats recognizers; that is why the wording comes from the user and only the clock comes from the transcript.+3. Dress it: set_caption_look with a look id (karaoke-glow for words lighting up, neon-club, serif-mood…), or add_text_sequence from_captions to turn the same timed lines into full-frame cards.+4. Check it: seek + capture_frame at two or three lines, and listen_audio across one of them. A line that sits early or late retimes with update_cue — the recognizer's gaps are where drift comes from.+With no lyrics text, transcribe with subtitles_generate and work from the cues it wrote.++**Words over the user's footage.** Same flow, with the over-footage look (or any look with \`cards: false\`) and \`background: false\` — the clips keep playing and the words ride above them. Aim them off the caption band when captions are also on.++**Words on plain color.** A cut with nothing on track 0 is a complete video: set the background, place the run, export. Cards are how the color changes line to line — the sequence paints one per line on its own row and puts the words on the row above, so they stack instead of sliding apart.++**Timing.** One line per screen beats two. A line needs about 0.4s minimum to read; 1.5–2.5s is the normal beat for a spoken line, and a sung line takes whatever the vocal takes. Lines never overlap on a row — the sequence tool lays them end to end from the times you give it.++**Making it yours.** Every look is a starting point: after add_text_sequence, update_overlay retimes or recolors any line, set_overlay_animation swaps a line's motion, and set_overlay_keyframes moves it on a path. A per-line \`color\` on the sequence call marks the word that matters without touching the rest.`,+"audio-and-subtitles": `# Audio, voiceover & subtitles+Lyrics and scripts the user already has: sync_lyrics times their exact words against the audio (transcribing first when needed) and writes them to a caption track with per-word timings; set_caption_look then dresses the track — preset, size, font, karaoke word highlight, accent color — from a text-videos look id or field by field. The text-videos skill has the whole flow, including turning the same timed lines into full-frame cards.Soundtrack clips: volume 0..1.5, fadeIn/fadeOut seconds (max half the clip), start = timeline position, in/out = trim inside the source; clips can spread across several soundtrack lanes (the \`lane\` field), new sounds slide to free space in their lane. Fades render with ffmpeg afade on export.Ducking: a soundtrack clip's \`duck\` (0..1, via update_audio) lowers ALL other audio — video-clip sound and other music — to that gain while the clip plays; 1 clears it. Voiceovers set this so narration sits over quieter music. It applies in both the preview and the export.Voiceover (Donkey's hosted speech model — signed in, spends credits, like image/video generation):