Open source CapCut alternative.
TypeScript
feat(site): a caption holds as many words as you set [rebuild]
David Lec4da3e9a7d60
diff --git a/site/src/cut/components/SubtitlesPanel.tools.ts b/site/src/cut/components/SubtitlesPanel.tools.tsindex 48f9a836..da05316c 100644--- a/site/src/cut/components/SubtitlesPanel.tools.ts+++ b/site/src/cut/components/SubtitlesPanel.tools.ts@@ -7,6 +7,11 @@*/import { bool, num, obj, str, type AiToolDef } from "@/cut/lib/aiToolDef";+import {+ DEFAULT_WORDS_PER_CUE,+ MAX_WORDS_PER_CUE,+ MIN_WORDS_PER_CUE,+} from "@/cut/lib/cueChunk";import { CAPTION_STYLES } from "@/cut/lib/subtitles";import {WORD_EFFECT_MENU,@@ -85,11 +90,14 @@ export const SUBTITLES_TOOLS = [{name: "set_caption_look",description:- "Style the caption track: a text-videos look id in `look` sets everything at once (preset, size, font, word effect, accent color, frame background), or set the pieces yourself. Restyling captions is a change the user asks for — leave the track as it is when they only asked for the words. Word effects — word_highlight — play the caption word by word as it is spoken or sung, in the preview and the export burn-in; they need word timings, which transcription and sync_lyrics leave on the cues. An emphasis travels along a line that is fully up; a build assembles the line as it is spoken. Naming an accent_mode turns it on, so \"make the words pop as I say them\" is one call with accent_mode: pop, and \"have the caption appear word by word\" is accent_mode: build.",+ "Style the caption track: a text-videos look id in `look` sets everything at once (preset, size, font, word effect, accent color, frame background), or set the pieces yourself. `words_per_cue` is how the captions READ rather than how they look — it re-cuts the tracks and re-aligns them to the audio. Restyling captions is a change the user asks for — leave the track as it is when they only asked for the words. Word effects — word_highlight — play the caption word by word as it is spoken or sung, in the preview and the export burn-in; they need word timings, which transcription and sync_lyrics leave on the cues. An emphasis travels along a line that is fully up; a build assembles the line as it is spoken. Naming an accent_mode turns it on, so \"make the words pop as I say them\" is one call with accent_mode: pop, and \"have the caption appear word by word\" is accent_mode: build.",inputSchema: obj({look: str("Text-videos look id — sets the whole caption look"),style: str(`Caption preset: ${Object.keys(CAPTION_STYLES).join(", ")}`),size: num("Caption size in px at a 1080-wide frame"),+ words_per_cue: num(+ `How many words one caption holds at a time (${MIN_WORDS_PER_CUE}–${MAX_WORDS_PER_CUE}, default ${DEFAULT_WORDS_PER_CUE}) — the answer to "show fewer words at once", "one word at a time", "these captions are a wall of text". Every track is re-cut on its own measured words and then re-aligned to the speech in the mix, so the wording survives and the new lines land on the audio; a line somebody authored (a translation, lyrics, narration) is split when it runs long and never merged with the line beside it.`+ ),font: str("Font id"),word_highlight: bool("Play the caption word by word as it lands"),accent_color: str("Word effect accent color"),diff --git a/site/src/cut/components/SubtitlesPanel.tsx b/site/src/cut/components/SubtitlesPanel.tsxindex 031ad093..01491aa8 100644--- a/site/src/cut/components/SubtitlesPanel.tsx+++ b/site/src/cut/components/SubtitlesPanel.tsx@@ -18,6 +18,12 @@ import {subtitleLaneCount,trackLocale,} from "@/cut/lib/subtitles";+import {+ cueWordCount,+ MAX_WORDS_PER_CUE,+ MIN_WORDS_PER_CUE,+ DEFAULT_WORDS_PER_CUE,+} from "@/cut/lib/cueChunk";import { useElapsed } from "@/cut/hooks/useElapsed";import { useCutCaps } from "@/cut/lib/backend/hooks";import { useActiveWork, useGenNotify } from "@/cut/lib/genNotify";@@ -268,6 +274,7 @@ function OptionsTab() {/></div></div>+ <WordsPerCueRow /><div className="flex min-h-8 items-center justify-between text-xs font-medium">Word effect<PillSelect@@ -369,6 +376,49 @@ function OptionsTab() {);}+/** How many words a caption holds at a time. Committing re-cuts every track+ * on its own words and measures the new captions against the cut's mix, which+ * takes a moment on a long project — the slider shows the drag, the track is+ * re-cut once on release. */+function WordsPerCueRow() {+ const subtitles = useEditor((s) => s.subtitles);+ const [draft, setDraft] = useState<number | null>(null);+ const [working, setWorking] = useState(false);+ const value = draft ?? cueWordCount(subtitles);+ return (+ <div className="flex min-h-8 items-center justify-between text-xs font-medium">+ Words at a time+ <div className="sub-words-per-cue flex items-center gap-2">+ {working && <Loader2 className="size-3 animate-spin text-muted-foreground" />}+ <ValueSlider+ label="Words at a time"+ sliderClassName="data-horizontal:w-24"+ valueClassName="w-7 text-muted-foreground"+ value={value}+ min={MIN_WORDS_PER_CUE}+ max={MAX_WORDS_PER_CUE}+ step={1}+ snap={[DEFAULT_WORDS_PER_CUE]}+ format={(v) => String(Math.round(v))}+ parse={parseNumberInput}+ onDraft={(v) => setDraft(Math.round(v))}+ onCommit={(v) => {+ setDraft(Math.round(v));+ setWorking(true);+ void useEditor+ .getState()+ .setSubtitleWordsPerCue(v)+ .finally(() => {+ setWorking(false);+ setDraft(null);+ });+ }}+ />+ </div>+ </div>+ );+}+/** The Styles tab: the caption style list, each option rendered as a live* preview of that look. */function StylesTab() {diff --git a/site/src/cut/lib/aiContext.ts b/site/src/cut/lib/aiContext.tsindex fcaa4b4b..181bd6f4 100644--- a/site/src/cut/lib/aiContext.ts+++ b/site/src/cut/lib/aiContext.ts@@ -5,6 +5,7 @@ import { chatOwner } from "./chatAssets";import { useGenerate } from "./generate";import { getClipSpans, overlayLayers, resolveTransitions, totalDuration, useEditor } from "./store";import { playheadAt, skimAt } from "./playhead";+import { cueWordCount } from "./cueChunk";import { laneCues, subtitleLaneCount } from "./subtitles";import { watchSweepActive } from "./watch/sweep";import { libraryFontId, listLibraryFonts } from "./linkedLibrary";@@ -380,6 +381,9 @@ export function buildAiContext(opts?: { fullCues?: boolean; chatId?: string | nucues: laneCues(s.subtitles, i).length,...(s.subtitles.tracks?.[i]?.hidden ? { hidden: true } : {}),})),+ // How the captions READ: how many words one holds at a time. Changing it+ // re-cuts every track (set_caption_look words_per_cue).+ wordsPerCue: cueWordCount(s.subtitles),status: s.subtitleStatus,// A window of cues by default; when truncated the model calls get_state// for the whole transcript (e.g. "clean up all the captions").diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 07a9db16..38e95a12 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -109,6 +109,11 @@ import {type TextLayout,type TextVariation,} from "./textCompose";+import {+ cueWordCount,+ MAX_WORDS_PER_CUE,+ MIN_WORDS_PER_CUE,+} from "./cueChunk";import { partialWrites, syncToSpeech, type SyncReport } from "./cueSync";import { requireTextLook, type TextLook } from "./textLooks";import {@@ -1641,7 +1646,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {};},- set_caption_look: (s, input) => {+ set_caption_look: async (s, input) => {const patch: Parameters<typeof s.setSubtitlesView>[0] = {};if (typeof input.look === "string") {const look = requireTextLook(input.look);@@ -1675,13 +1680,23 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {}if (isNum(input.y)) patch.y = clamp(input.y, 0.02, 0.98);if (isNum(input.x)) patch.x = clamp(input.x, 0.02, 0.98);- if (Object.keys(patch).length === 0) throw new ToolError("Nothing to change.");- s.setSubtitlesView(patch);+ const per = isNum(input.words_per_cue)+ ? clamp(Math.round(input.words_per_cue), MIN_WORDS_PER_CUE, MAX_WORDS_PER_CUE)+ : null;+ if (Object.keys(patch).length === 0 && per === null)+ throw new ToolError("Nothing to change.");+ if (Object.keys(patch).length > 0) s.setSubtitlesView(patch);+ // Re-cutting the captions rewrites the track and measures it against the+ // mix, so the tool waits for the tracks it reports on.+ const before = per !== null ? useEditor.getState().subtitles.cues.length : 0;+ if (per !== null) await useEditor.getState().setSubtitleWordsPerCue(per);const cur = useEditor.getState().subtitles;return {style: cur.style ?? "clean",size: cur.size,font: cur.font,+ wordsPerCue: cueWordCount(cur),+ ...(per !== null ? { cues: cur.cues.length, cuesBefore: before } : {}),wordHighlight: cur.wordHighlight === true,accentColor: cur.accentColor,accentMode: cur.accentMode,diff --git a/site/src/cut/lib/cueChunk.test.ts b/site/src/cut/lib/cueChunk.test.tsnew file mode 100644index 00000000..76e56a1e--- /dev/null+++ b/site/src/cut/lib/cueChunk.test.ts@@ -0,0 +1,151 @@+import { describe, expect, test } from "bun:test";+import { chunkCues, cueWordCount, DEFAULT_WORDS_PER_CUE } from "./cueChunk";+import type { SubtitleCue } from "./types";++/**+ * Cutting a caption track to a word count.+ *+ * What these hold: the wording survives a re-cut word for word, the edges land+ * on the words the transcriber measured, captions never overlap, and lines+ * somebody wrote by hand are split but never glued together.+ */++/** A transcribed cue: every word measured, a word every 0.4s. */+function heard(id: string, from: number, text: string): SubtitleCue {+ const parts = text.split(" ");+ return {+ id,+ start: from,+ end: from + parts.length * 0.4,+ text,+ words: parts.map((w, i) => ({ w, t0: from + i * 0.4, t1: from + i * 0.4 + 0.35 })),+ };+}++/** An authored line: text and a span, no measured words. */+const written = (id: string, start: number, end: number, text: string): SubtitleCue => ({+ id,+ start,+ end,+ text,+});++const wordsOf = (cues: SubtitleCue[]) => cues.flatMap((c) => c.text.split(" "));++describe("cueWordCount", () => {+ test("defaults, and clamps what the project carries", () => {+ expect(cueWordCount({})).toBe(DEFAULT_WORDS_PER_CUE);+ expect(cueWordCount({ wordsPerCue: 3 })).toBe(3);+ expect(cueWordCount({ wordsPerCue: 0 })).toBe(1);+ expect(cueWordCount({ wordsPerCue: 99 })).toBe(12);+ });+});++describe("chunkCues", () => {+ test("cuts a transcribed line into groups of n, keeping every word", () => {+ const cues = [heard("a", 1, "Don't wait for someone to give you a chance")];+ const out = chunkCues(cues, 5);+ expect(out.map((c) => c.text)).toEqual(["Don't wait for someone to", "give you a chance"]);+ expect(wordsOf(out)).toEqual(wordsOf(cues));+ });++ test("edges are the words' own times", () => {+ const out = chunkCues([heard("a", 1, "one two three four five six")], 3);+ expect(out[0].start).toBeCloseTo(1, 3);+ expect(out[0].end).toBeCloseTo(2.15, 3);+ expect(out[1].start).toBeCloseTo(2.2, 3);+ expect(out[1].words?.map((w) => w.w)).toEqual(["four", "five", "six"]);+ });++ test("merges across measured lines when the words keep running", () => {+ const out = chunkCues([heard("a", 1, "one two"), heard("b", 1.8, "three four")], 4);+ expect(out).toHaveLength(1);+ expect(out[0].text).toBe("one two three four");+ expect(out[0].id).toBe("a");+ });++ test("a pause in the speech closes the caption early", () => {+ const out = chunkCues([heard("a", 1, "one two"), heard("b", 6, "three four")], 5);+ expect(out.map((c) => c.text)).toEqual(["one two", "three four"]);+ });++ test("a finished sentence closes the caption early", () => {+ const out = chunkCues([heard("a", 1, "so that's it. now watch this")], 6);+ expect(out.map((c) => c.text)).toEqual(["so that's it.", "now watch this"]);+ });++ test("authored lines split but never merge", () => {+ const cues = [+ written("a", 0, 2, "no esperes a que alguien te dé una oportunidad"),+ written("b", 2, 3, "hazlo tú"),+ ];+ const out = chunkCues(cues, 5);+ expect(out.map((c) => c.text)).toEqual([+ "no esperes a que alguien",+ "te dé una oportunidad",+ "hazlo tú",+ ]);+ // Times are a share of the line they came from, and nothing crosses into+ // the line beside it.+ expect(out[1].end).toBeLessThanOrEqual(2);+ expect(out[2].start).toBeGreaterThanOrEqual(2);+ // No measured words, so none are written — the word effect falls back to+ // its own share of the span.+ expect(out[0].words).toBeUndefined();+ });++ test("captions on a track never overlap, and stay in order", () => {+ const out = chunkCues(+ [heard("a", 1, "one two three four five six seven eight nine ten")],+ 2+ );+ for (let i = 0; i + 1 < out.length; i++) {+ expect(out[i].end).toBeLessThanOrEqual(out[i + 1].start);+ expect(out[i].end).toBeGreaterThan(out[i].start);+ }+ });++ test("each track is cut on its own words", () => {+ const out = chunkCues(+ [+ heard("a", 1, "one two three four"),+ { ...heard("b", 1, "uno dos tres cuatro"), lane: 1 },+ ],+ 2+ );+ expect(out.filter((c) => (c.lane ?? 0) === 0).map((c) => c.text)).toEqual([+ "one two",+ "three four",+ ]);+ expect(out.filter((c) => c.lane === 1).map((c) => c.text)).toEqual([+ "uno dos",+ "tres cuatro",+ ]);+ });++ test("re-cutting at the number a track already carries changes nothing", () => {+ const once = chunkCues([heard("a", 1, "one two three four five six seven")], 4);+ const twice = chunkCues(once, 4);+ expect(twice).toEqual(once);+ });++ test("a caption never holds the screen for more than a few seconds", () => {+ // A language written without spaces: the whole line is one "word".+ const cues = [+ written("a", 0, 4, "很长的一句话"),+ written("b", 4, 8, "另一句话"),+ written("c", 8, 12, "第三句话"),+ ];+ const out = chunkCues(cues, 5);+ for (const c of out) expect(c.end - c.start).toBeLessThanOrEqual(6.001);+ });++ test("a hand-edited line that lost its word count falls back to the spread", () => {+ const cue = { ...heard("a", 1, "one two three four"), text: "one two three" };+ const out = chunkCues([cue], 2);+ expect(out.map((c) => c.text)).toEqual(["one two", "three"]);+ expect(out[0].words).toBeUndefined();+ expect(out[0].start).toBeCloseTo(1, 3);+ expect(out[out.length - 1].end).toBeCloseTo(2.6, 3);+ });+});diff --git a/site/src/cut/lib/cueChunk.ts b/site/src/cut/lib/cueChunk.tsnew file mode 100644index 00000000..11a1aeaf--- /dev/null+++ b/site/src/cut/lib/cueChunk.ts@@ -0,0 +1,192 @@+/**+ * How many words a caption holds at once.+ *+ * A transcriber groups its words its own way — the on-device model breaks on+ * punctuation and a character budget — and that grouping is what a caption+ * reads like: three words at a time is a fast, punchy read, ten is a+ * paragraph parked over the picture. The project carries the number and this+ * is where a track is cut to it.+ *+ * Cues are cut from words, not from lines. A word carries the moment the+ * transcriber measured it at, so a group's edges are real onsets and the+ * caption changes when the speech does. A track with no measured words gets+ * times spread across each line by word length, which is arithmetic and not+ * evidence — the caller measures a re-cut track against the mix afterwards+ * (cueSync.ts), where the audio settles it.+ *+ * Lines merge only where both sides carry measured words. Authored lines — a+ * translation, a lyric sheet, narration written to picture — are somebody's+ * own breaks, so they are split when they run long and never glued together.+ */++import type { SubtitleCue } from "./types";++/** Words a caption holds when the project hasn't said otherwise. */+export const DEFAULT_WORDS_PER_CUE = 5;+export const MIN_WORDS_PER_CUE = 1;+export const MAX_WORDS_PER_CUE = 12;++/** A pause this long between two words breaks the caption, however few words+ * it holds: the speech stopped, so the line does too. */+const GAP = 0.6;+/** No caption holds the screen longer than this whatever the word count says.+ * One recognized "word" can be a whole line in a language written without+ * spaces, and five of those is a paragraph nobody can read. */+const MAX_SPAN = 6;+/** A caption never lands shorter than this. */+const MIN_SPAN = 0.2;+/** The end of a sentence ends the caption, closing quotes and brackets+ * included. */+const SENTENCE_END = /[.!?…。!?]["'”’)\]]?$/;++const round = (n: number) => Math.round(n * 1000) / 1000;+const uid = () => crypto.randomUUID().slice(0, 8);++/** The project's words-per-caption, clamped to what the UI offers. */+export function cueWordCount(subs: { wordsPerCue?: number }): number {+ const n = subs.wordsPerCue;+ if (typeof n !== "number" || !Number.isFinite(n)) return DEFAULT_WORDS_PER_CUE;+ return Math.min(MAX_WORDS_PER_CUE, Math.max(MIN_WORDS_PER_CUE, Math.round(n)));+}++/** Spread a line's words across [start, end], each word's slice proportional+ * to its length. What a caption with no measured timings falls back to, here+ * and when captions are re-timed onto a generated voiceover. */+export function spreadWordsEvenly(+ text: string,+ start: number,+ end: number+): { t0: number; t1: number; w: string }[] | undefined {+ const parts = text.split(/\s+/).filter(Boolean);+ if (parts.length === 0) return undefined;+ const lengths = parts.map((w) => Math.max(1, w.length));+ const total = lengths.reduce((a, b) => a + b, 0);+ const span = Math.max(0, end - start);+ let acc = 0;+ return parts.map((w, i) => {+ const t0 = start + (acc / total) * span;+ acc += lengths[i];+ const t1 = start + (acc / total) * span;+ return { t0, t1, w };+ });+}++interface StreamWord {+ w: string;+ t0: number;+ t1: number;+ /** The transcriber measured this word. False means its time is a share of+ * the line it sits in, which is what keeps authored lines from merging. */+ measured: boolean;+ /** The cue it came from: its index in the track and its id, which the+ * caption it lands in reuses when it opens on the same word. */+ from: number;+ id: string;+}++/** One track's words in order, each carrying where it came from. Times are+ * kept monotonic, so a group's edges can be read straight off its ends. */+function streamOf(cues: readonly SubtitleCue[]): StreamWord[] {+ const out: StreamWord[] = [];+ let floor = -Infinity;+ cues.forEach((cue, from) => {+ const tokens = cue.text.split(/\s+/).filter(Boolean);+ if (tokens.length === 0) return;+ // Word timings ride along only while they still describe the line: a+ // hand-edit that changed the word count left them behind, and the spread+ // is the honest reading then.+ const timed = cue.words && cue.words.length === tokens.length ? cue.words : null;+ const spread = timed ? null : spreadWordsEvenly(cue.text, cue.start, cue.end);+ tokens.forEach((w, i) => {+ const src = timed ? timed[i] : spread![i];+ const t0 = Math.max(src.t0, floor);+ const t1 = Math.max(src.t1, t0);+ floor = t1;+ out.push({ w, t0, t1, measured: !!timed, from, id: cue.id });+ });+ });+ return out;+}++/** The words cut into captions: `per` at most, and fewer wherever the speech,+ * the punctuation, or somebody's own line break says the caption ends. */+function groupsOf(stream: StreamWord[], per: number): StreamWord[][] {+ const out: StreamWord[][] = [];+ let cur: StreamWord[] = [];+ for (const word of stream) {+ const prev = cur[cur.length - 1];+ if (prev) {+ const authored = word.from !== prev.from && !(word.measured && prev.measured);+ if (+ cur.length >= per ||+ authored ||+ word.t0 - prev.t1 > GAP ||+ SENTENCE_END.test(prev.w) ||+ word.t1 - cur[0].t0 > MAX_SPAN+ ) {+ out.push(cur);+ cur = [];+ }+ }+ cur.push(word);+ }+ if (cur.length > 0) out.push(cur);+ return out;+}++/** One track's captions, re-cut to `per` words each. Cues come back in time+ * order, never overlapping, and a caption that opens on the word its old cue+ * opened on keeps that cue's id — so re-cutting a track at the number it+ * already carries leaves it exactly as it was. */+export function chunkLaneCues(cues: readonly SubtitleCue[], per: number): SubtitleCue[] {+ const n = cueWordCount({ wordsPerCue: per });+ const ordered = [...cues].sort((a, b) => a.start - b.start);+ const groups = groupsOf(streamOf(ordered), n);+ const lane = ordered[0]?.lane ?? 0;+ const used = new Set<string>();+ return groups.map((g, i) => {+ const first = g[0];+ const last = g[g.length - 1];+ const next = groups[i + 1]?.[0];+ const start = first.t0;+ // Captions on a track may never overlap, so a caption closes where the+ // next one opens whenever its own last word runs past it.+ const ceil = next ? Math.max(start, next.t0) : Infinity;+ const end = Math.min(Math.max(last.t1, start + MIN_SPAN), ceil);+ const id = used.has(first.id) ? uid() : first.id;+ used.add(id);+ const measured = g.every((w) => w.measured);+ return {+ id,+ start: round(start),+ end: round(end),+ text: g.map((w) => w.w).join(" "),+ ...(measured+ ? {+ words: g.map((w) => ({+ w: w.w,+ t0: round(Math.min(Math.max(w.t0, start), end)),+ t1: round(Math.min(Math.max(w.t1, start), end)),+ })),+ }+ : {}),+ ...(lane > 0 ? { lane } : {}),+ };+ });+}++/** Every track's captions re-cut to `per` words each, each track cut on its+ * own words. */+export function chunkCues(cues: readonly SubtitleCue[], per: number): SubtitleCue[] {+ if (cues.length === 0) return [];+ const lanes = new Map<number, SubtitleCue[]>();+ for (const cue of cues) {+ const lane = cue.lane ?? 0;+ const list = lanes.get(lane);+ if (list) list.push(cue);+ else lanes.set(lane, [cue]);+ }+ return [...lanes.values()]+ .flatMap((list) => chunkLaneCues(list, per))+ .sort((a, b) => a.start - b.start);+}diff --git a/site/src/cut/lib/cueSync.test.ts b/site/src/cut/lib/cueSync.test.tsindex 2fe73f70..fb6174e6 100644--- a/site/src/cut/lib/cueSync.test.ts+++ b/site/src/cut/lib/cueSync.test.ts@@ -102,6 +102,82 @@ describe("syncToSpeech", () => {});});+describe("words at a time", () => {+ /** Six measured words a quarter-second apart, filling one speech span. */+ const spoken = (id: string, from: number): SubtitleCue => {+ const words = Array.from({ length: 6 }, (_, i) => ({+ w: `w${id}${i}`,+ t0: from + i * 0.25,+ t1: from + i * 0.25 + 0.25,+ }));+ return {+ id,+ start: from,+ end: from + 1.5,+ text: words.map((w) => w.w).join(" "),+ words,+ };+ };++ const track = () => [spoken("a", 1), spoken("b", 4), spoken("c", 8)];++ const openProject = (cues: SubtitleCue[]) =>+ useEditor.setState({+ projectId: "p1",+ aspect: "9:16",+ assets: [],+ clips: [],+ audioClips: [],+ overlays: [],+ subtitles: { ...emptySubtitles(), cues },+ cutMixSamples: async () => ({ samples: audio(), sampleRate: RATE }),+ });++ test("re-cuts every track and keeps the wording word for word", async () => {+ openProject(track());+ const said = useEditor+ .getState()+ .subtitles.cues.flatMap((c) => c.text.split(" "));+ await useEditor.getState().setSubtitleWordsPerCue(3);+ const cues = useEditor.getState().subtitles.cues;+ expect(cues).toHaveLength(6);+ expect(cues.flatMap((c) => c.text.split(" "))).toEqual(said);+ expect(useEditor.getState().subtitles.wordsPerCue).toBe(3);+ });++ test("the new captions land on the speech, and never on each other", async () => {+ openProject(track());+ await useEditor.getState().setSubtitleWordsPerCue(2);+ const cues = useEditor.getState().subtitles.cues;+ expect(cues[0].start).toBeCloseTo(1, 1);+ expect(cues[cues.length - 1].end).toBeCloseTo(9.5, 1);+ for (let i = 0; i + 1 < cues.length; i++) {+ expect(cues[i].end).toBeLessThanOrEqual(cues[i + 1].start + 1e-6);+ // Each caption still sits inside a run of speech in the mix.+ expect(SPEECH.some(([f, t]) => cues[i].start >= f - 0.2 && cues[i].end <= t + 0.2)).toBe(true);+ }+ });++ test("the re-cut and its alignment undo together", async () => {+ openProject(track());+ await useEditor.getState().setSubtitleWordsPerCue(3);+ expect(useEditor.getState().subtitles.cues).toHaveLength(6);+ useEditor.getState().undo();+ const cues = useEditor.getState().subtitles.cues;+ expect(cues).toHaveLength(3);+ expect(cues.map((c) => c.id)).toEqual(["a", "b", "c"]);+ });++ test("a mix with nothing to read still re-cuts the captions", async () => {+ openProject(track());+ useEditor.setState({ cutMixSamples: async () => null });+ await useEditor.getState().setSubtitleWordsPerCue(3);+ // The cut is made on the words the transcriber measured; only the+ // measurement against the mix is skipped.+ expect(useEditor.getState().subtitles.cues).toHaveLength(6);+ });+});+describe("align_to_audio", () => {const reset = (cues: SubtitleCue[], overlays: unknown[] = []) =>useEditor.setState({diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.tsindex 2c330985..0dc3582d 100644--- a/site/src/cut/lib/store.ts+++ b/site/src/cut/lib/store.ts@@ -57,10 +57,18 @@ import {} from "./docCache";import { renderMix, transcribeSamples, type CloudTranscribeSpec } from "./cloudTranscribe";import { alignCues } from "./cueAlign";+import {+ chunkCues,+ cueWordCount,+ MAX_WORDS_PER_CUE,+ MIN_WORDS_PER_CUE,+ spreadWordsEvenly,+} from "./cueChunk";+import { syncToSpeech } from "./cueSync";import { useGenNotify } from "./genNotify";import { clampPlayhead, playheadAt, previewAt, setPlayhead, setSkim } from "./playhead";import { engineTranscribeSamples, withEngineStt } from "./localStt";-import { trackLocale } from "./subtitles";+import { laneCues, subtitleLaneCount, trackLocale } from "./subtitles";import { ANIM_STYLE_IDS, animStyleOfTransition, assetIsSilent, clipPoseAt, DEFAULT_BACKGROUND, emptySubtitles, frameOf, IMAGE_CLIP_SECONDS, isAudioTransition, isEffectOverlay, isStickerOverlay, MAX_SUBTITLE_LANES, mediaUrl, migrateBehindSubject, migrateLegacyTransitions, normalizeAspect, overlayAnimStyle, projectBackground, SPEED_FLOOR, SPEED_MIN, stampOverlayKinds, stripDefaultOverlayKinds, TRANSITION_MAX, TRANSITION_STYLE_IDS, transitionBarAt, transitionBarStart, transitionStyleOfAnim, type TransitionBoundaryKind } from "./types";import { liftMoveTracks } from "./textMotion";import { readTextStyle } from "./textStyle";@@ -584,6 +592,10 @@ export interface EditorState {* translation, so the new cues carry none. */translateSubtitleTrack: (fromLane: number) => Promise<void>;setSubtitlesView: (patch: Partial<Pick<SubtitlesBlock, "showOnVideo" | "showOnTimeline" | "locale" | "style" | "size" | "font" | "x" | "y" | "wordHighlight" | "accentMode" | "accentColor" | "accentScale" | "accentDim">>) => void;+ /** How many words a caption holds at a time. Every track is re-cut on its+ * own words and then measured against the cut's mix, so the new captions+ * land on the speech instead of on arithmetic. One undo step. */+ setSubtitleWordsPerCue: (n: number) => Promise<void>;/** The subtitle track (row) the panel edits and generation writes to. */subtitleLane: number;setSubtitleLane: (lane: number) => void;@@ -3694,9 +3706,12 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {const epoch = laneEpoch;set({ subtitleStatus: "running", subtitleError: null, subtitleStartedAt: Date.now() });try {- const cues = await runTranscription(projectId, spec);- if (cues === null) return; // switched projects mid-run+ const heard = await runTranscription(projectId, spec);+ if (heard === null) return; // switched projects mid-runif (laneEpoch !== epoch) return set(staleLaneError);+ // The transcriber groups its own words its own way; the project says+ // how many a caption holds, and the cut is made on the measured words.+ const cues = chunkCues(heard, cueWordCount(get().subtitles));// Only the active track's cues are replaced; other languages stay.const tagged = cues.map((c) => ({ ...c, ...(lane > 0 ? { lane } : {}) }));if (cues.length === 0) {@@ -3776,7 +3791,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {// word timings) onto the clip's timeline span. New cues replace any// on the active track that overlapped the clip; the rest of the// timeline — and every other track — keeps its cues.- const placed = cues.map((c) => ({+ const placed = chunkCues(cues, cueWordCount(get().subtitles)).map((c) => ({...c,start: c.start + sp.start,end: c.end + sp.start,@@ -3841,13 +3856,10 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {}if (get().projectId !== projectId) return; // switched projects mid-runif (laneEpoch !== epoch) return set(staleLaneError);- const cues: SubtitleCue[] = body.cues.map((c) => ({- id: uid(),- start: c.start,- end: c.end,- text: c.text,- ...(lane > 0 ? { lane } : {}),- }));+ const cues: SubtitleCue[] = chunkCues(+ body.cues.map((c) => ({ id: uid(), start: c.start, end: c.end, text: c.text })),+ cueWordCount(get().subtitles)+ ).map((c) => ({ ...c, ...(lane > 0 ? { lane } : {}) }));push();set((cur) => ({subtitles: {@@ -3910,19 +3922,31 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {// that reordered/split cues mid-request can't apply text by a stale// index onto the wrong cue.const byId = new Map(cues.map((c, i) => [c.id, body.texts![i]]));- set((cur) => ({- subtitles: {- ...cur.subtitles,- style,- // Rewriting the text drops per-word timings (they no longer match),- // but the cue's own start/end are untouched.- cues: cur.subtitles.cues.map((c) => {- const t = byId.get(c.id);- return t && t !== c.text ? { ...c, text: t, words: undefined } : c;- }),- },- subtitleStatus: "ready",- }));+ set((cur) => {+ // Rewriting the text drops per-word timings (they no longer match),+ // but the cue's own start/end are untouched.+ const written = cur.subtitles.cues.map((c) => {+ const t = byId.get(c.id);+ return t && t !== c.text ? { ...c, text: t, words: undefined } : c;+ });+ // A rewritten line still reads at the project's word count: one+ // that came back long is cut, one that came back short is left as+ // the model wrote it.+ const recut = chunkCues(+ written.filter((c) => laneOf(c) === lane),+ cueWordCount(cur.subtitles)+ );+ return {+ subtitles: {+ ...cur.subtitles,+ style,+ cues: [...written.filter((c) => laneOf(c) !== lane), ...recut].sort(+ (a, b) => a.start - b.start+ ),+ },+ subtitleStatus: "ready" as const,+ };+ });} else {// The style still applied; leave the text as-is.set({ subtitleStatus: "ready" });@@ -3973,13 +3997,15 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {...cur.subtitles,cues: [...cur.subtitles.cues.filter((c) => laneOf(c) !== lane),- ...source.map((c, i) => ({- id: uid(),- start: c.start,- end: c.end,- text: texts[i],- ...(lane > 0 ? { lane } : {}),- })),+ ...chunkCues(+ source.map((c, i) => ({+ id: uid(),+ start: c.start,+ end: c.end,+ text: texts[i],+ })),+ cueWordCount(cur.subtitles)+ ).map((c) => ({ ...c, ...(lane > 0 ? { lane } : {}) })),].sort((a, b) => a.start - b.start),generatedAt: Date.now(),},@@ -3997,6 +4023,58 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {setSubtitlesView: (patch) =>set((s) => ({ subtitles: { ...s.subtitles, ...patch } })),+ setSubtitleWordsPerCue: async (n) => {+ const s = get();+ const per = Math.min(MAX_WORDS_PER_CUE, Math.max(MIN_WORDS_PER_CUE, Math.round(n)));+ if (s.subtitles.wordsPerCue === per) return;+ const projectId = s.projectId;+ const epoch = laneEpoch;+ push();+ set((cur) => {+ const cues = chunkCues(cur.subtitles.cues, per);+ // Re-cutting mints a cue where a caption opens mid-line, so a+ // selection pointing at a cue that no longer exists is dropped.+ const alive = new Set(cues.map((c) => c.id));+ const kept = (sel: Selection) => !sel || sel.kind !== "cue" || alive.has(sel.id);+ const multiSelection = cur.multiSelection.filter(kept);+ return {+ subtitles: { ...cur.subtitles, wordsPerCue: per, cues },+ multiSelection,+ selection: kept(cur.selection)+ ? cur.selection+ : (multiSelection[multiSelection.length - 1] ?? null),+ };+ });+ // The new edges are word onsets where the transcriber measured them and+ // a share of a line where it didn't. The mix is what tells the two+ // apart, so the re-cut track is measured against it — the same pass+ // align_to_audio runs, folded into the change that needed it.+ const mix = await get().cutMixSamples().catch(() => null);+ if (!mix) return;+ if (get().projectId !== projectId || laneEpoch !== epoch) return;+ set((cur) => {+ if (cur.subtitles.wordsPerCue !== per) return {}; // changed again mid-render+ let cues = cur.subtitles.cues;+ for (let lane = 0; lane < subtitleLaneCount(cur.subtitles); lane++) {+ const list = laneCues({ ...cur.subtitles, cues }, lane);+ if (list.length === 0) continue;+ // A track cut from measured words is nearly right already; one whose+ // times are a share of a line (a translation, narration, a+ // hand-edit) earns the wider reach a guess deserves.+ const guessed = list.some((c) => !c.words || c.words.length === 0);+ const report = syncToSpeech(list, mix.samples, mix.sampleRate, {+ ...(guessed ? { reach: 0.6 } : {}),+ });+ if (!report.evidence) continue;+ const byId = new Map(report.items.map((c) => [c.id, c]));+ cues = cues.map((c) => byId.get(c.id) ?? c);+ }+ return {+ subtitles: { ...cur.subtitles, cues: [...cues].sort((a, b) => a.start - b.start) },+ };+ });+ },+setSubtitleLane: (lane) => {const count = Math.max(1,@@ -5522,26 +5600,5 @@ export function projectDuration(s: {return durValue;}-/** Spread a cue's words across [start, end], each word's slice proportional to- * its length. Used when re-timing captions to a generated voiceover, which- * carries no per-word timestamps of its own. */-function spreadWordsEvenly(- text: string,- start: number,- end: number,-): { t0: number; t1: number; w: string }[] | undefined {- const parts = text.split(/\s+/).filter(Boolean);- if (parts.length === 0) return undefined;- const lengths = parts.map((w) => Math.max(1, w.length));- const total = lengths.reduce((a, b) => a + b, 0);- const span = Math.max(0, end - start);- let acc = 0;- return parts.map((w, i) => {- const t0 = start + (acc / total) * span;- acc += lengths[i];- const t1 = start + (acc / total) * span;- return { t0, t1, w };- });-}export const useTotalDuration = () => useEditor((s) => totalDuration(s.clips));diff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.tsindex 677f16d4..5961473c 100644--- a/site/src/cut/lib/types.ts+++ b/site/src/cut/lib/types.ts@@ -1217,6 +1217,9 @@ export interface SubtitlesBlock {* per-track anchor in `tracks`. Read as the lane-0 fallback. */x?: number;y?: number;+ /** How many words one caption holds at a time; absent = the default read+ * (lib/cueChunk.ts). Changing it re-cuts every track on its own words. */+ wordsPerCue?: number;/** Word effects on: the caption plays word by word as it is spoken, in the* preview and the export burn-in. */wordHighlight?: boolean;diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex 5fc996f9..e2f3b7a5 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -12,6 +12,11 @@import { AI_PANEL_TOOLS } from "@/cut/components/AiPanel.tools";import { TEXT_LAYOUT_IDS, TEXT_LAYOUT_NOTES } from "@/cut/lib/textCompose";import { textLookCatalog } from "@/cut/lib/textLooks";+import {+ DEFAULT_WORDS_PER_CUE,+ MAX_WORDS_PER_CUE,+ MIN_WORDS_PER_CUE,+} from "@/cut/lib/cueChunk";import { CAPTION_STYLES } from "@/cut/lib/subtitles";import { wordEffectCatalog, WORD_EFFECT_IDS } from "@donkeycut/effects-kit";import { textMoveCatalog } from "@/cut/lib/textMotion";@@ -301,7 +306,7 @@ ${textMoveCatalog()}"audio-and-subtitles": `# Audio, voiceover & subtitlesLyrics and scripts the user already has: sync_lyrics times their exact words against the audio (transcribing first when needed) and writes them to a caption track with per-word timings; set_caption_look then dresses the track — preset, size, font, word effect, accent color — from a text-videos look id or field by field. The text-videos skill has the whole flow, including turning the same timed lines into full-frame cards.-Soundtrack clips: volume 0..1.5, fadeIn/fadeOut seconds (max half the clip), start = timeline position, in/out = trim inside the source; clips can spread across several soundtrack lanes (the \`lane\` field), new sounds slide to free space in their lane. Fades render with ffmpeg afade on export.+Soundtrack clips: volume 0..3 (1 = unchanged, above 1 boosts), fadeIn/fadeOut seconds (max half the clip), start = timeline position, in/out = trim inside the source; clips can spread across several soundtrack lanes (the \`lane\` field), new sounds slide to free space in their lane. Fades render with ffmpeg afade on export.Ducking: a soundtrack clip's \`duck\` (0..1, via update_audio) lowers ALL other audio — video-clip sound and other music — to that gain while the clip plays; 1 clears it. Voiceovers set this so narration sits over quieter music. It applies in both the preview and the export.Audio effects (add_effect): the treatments over the sound — ${AUDIO_EFFECT_IDS.join(", ")} — placed on the timeline as elements, each treating everything audible under its window at once. Reach for one when the ask is how a stretch SOUNDS ("make this line sound like a phone call", "echo on that word", "muffle the music behind the voiceover"); \`amount\` says how far the treatment goes, and the effect trims and moves like any element. Loudness, fades and ducking on one clip stay with the audio tools; a dissolve across a cut stays with the transition on that cut (the audiocross style).Voiceover (Donkey's hosted speech model — signed in, spends credits, like image/video generation):@@ -310,7 +315,8 @@ Voiceover (Donkey's hosted speech model — signed in, spends credits, like imagMusic (Donkey's hosted music model, Gemini/Lyria — signed in, spends credits):- generate_music(prompt, instrumental?, length?, add_to_timeline?, start?, volume?): renders a track from a mood/genre/tempo brief — a full song with sung vocals (instrumental:false; the model writes the lyrics, or pass your own [Verse]/[Chorus] lines) or a vocal-free bed (instrumental:true, the default). length "clip" (~30s, default) or "song" (~2min). It returns when the track lands and previews as a chat card; add_to_timeline (or a start) drops it on the soundtrack at a soft bed volume. For "background music for this video", keep it instrumental, read videoTrack/duration to fit the mood and length, and set any speech clips' \`duck\` so the bed sits under them. This is sung music — a spoken narration is voiceover_generate; the Audio tab's Voice/Music sub-tabs mirror the two.Subtitle tracks: a project carries up to 3, one language each (editor_state subtitles.tracks; each cue's \`track\`). Each track shows its own caption line, dragged to its own spot; all share one visual style. The panel and generation write to the ACTIVE track — the track tool param switches it. subtitles_add_track / subtitles_remove_track manage them; subtitles_translate_track("ko-KR") is the whole "add Korean subtitles" flow when captions exist (it adds/reuses the track and translates cue-for-cue).-Generating: subtitles_generate transcribes the cut's audio on-device (Apple speech, macOS 26) onto the target track; cues are caption-sized (≈38 chars). subtitles_from_visuals is the fallback for a cut with NO speech — it watches sampled frames (via the user's Claude login) and writes timed narration captions of what's on screen. So: speech present → subtitles_generate; silent/music-only footage the user wants captioned → subtitles_from_visuals. Never fabricate a spoken transcript.+How the captions read: one caption holds ${DEFAULT_WORDS_PER_CUE} words at a time by default, ${MIN_WORDS_PER_CUE}–${MAX_WORDS_PER_CUE} through set_caption_look words_per_cue (the current number is editor_state subtitles.wordsPerCue, and the Subtitles panel's Options tab offers the same slider). It answers "show fewer words at once", "one word at a time", "these captions are a wall of text". Every track is re-cut on its own measured words — the wording survives verbatim — and the re-cut track is then measured against the mix, so the new lines land on the speech. Lines somebody authored (a translation, lyrics, narration written to picture) are split when they run long and never glued to the line beside them, and a caption always breaks at a pause or the end of a sentence however few words it holds.+Generating: subtitles_generate transcribes the cut's audio on-device (Apple speech, macOS 26) onto the target track; cues are cut to the project's words-per-caption. subtitles_from_visuals is the fallback for a cut with NO speech — it watches sampled frames (via the user's Claude login) and writes timed narration captions of what's on screen. So: speech present → subtitles_generate; silent/music-only footage the user wants captioned → subtitles_from_visuals. Never fabricate a spoken transcript.captions_generate rewrites a track's cues into punchy social captions (emoji, curiosity-hook opener) in a style (clean/hook/punchy), keeping timings — use it when the ask is social/TikTok captions rather than a plain transcript.Editing: update_cue (text or retime), delete_cue, merge_cue (joins into the previous cue on the same track). In the panel, Return splits a caption at the cursor onto real word timings; hand-edited text drops its word timings.subtitles_set_view: showOnVideo (preview + export burn-in), showOnTimeline (amber cue rows).