Create polished demo videos without editing skills. Mac/Windows/Linux
feat(captions): segment captions by sentence using punctuation and pauses
Joe Hachemc31d78ad1275
diff --git a/electron/ipc/captions/generate.ts b/electron/ipc/captions/generate.tsindex 2cab7319..038d1f87 100644--- a/electron/ipc/captions/generate.ts+++ b/electron/ipc/captions/generate.ts@@ -9,9 +9,9 @@ import { getBundledWhisperExecutableCandidates } from "../paths/binaries";import { resolveRecordingSession } from "../project/session";import { normalizeVideoSourcePath } from "../utils";import { parseSrtCues, parseWhisperJsonCues, shouldRetryWhisperWithoutJson } from "./parser";+import { segmentCuesIntoPhrases } from "./segment";import {parseSilenceIntervals,- resegmentCuesBySilence,SILENCE_DETECT_MIN_S,SILENCE_NOISE_DB,type SilenceInterval,@@ -259,19 +259,28 @@ export async function generateAutoCaptionsFromVideo(options: {const timedCues = jsonEnabled? parseWhisperJsonCues(await fs.readFile(jsonPath, "utf-8")): [];+ if (jsonEnabled && timedCues.length === 0) {+ // JSON ran but yielded no word-timed cues (empty/malformed output). We fall back+ // to SRT, which has no word timings — captions are then split by sentence text and+ // silence rather than precise word timing. Surface it for diagnosis.+ console.warn(+ "[auto-captions] Whisper JSON produced no word-timed cues; falling back to SRT (no word timings).",+ );+ }const cues =timedCues.length > 0 ? timedCues : parseSrtCues(await fs.readFile(srtPath, "utf-8"));if (cues.length === 0) {throw new Error("Whisper completed, but no caption cues were produced.");}- // Whisper cues span silence and don't break on pauses. Re-segment them against- // ground-truth acoustic silence (ffmpeg `silencedetect`) so captions only cover- // real speech. Failure here must not block caption generation — fall back to raw.+ // Whisper cues run sentences together and don't break on pauses. Re-segment them+ // into one caption per sentence/phrase using Whisper's own word stream (punctuation+ // + pauses), backed by ground-truth acoustic silence (ffmpeg `silencedetect`).+ // Failure here must not block caption generation — fall back to raw.let cuesToReturn = cues;try {const silences = await detectSilenceIntervals({ ffmpegPath, wavPath });- const resegmented = resegmentCuesBySilence(cues, silences);+ const resegmented = segmentCuesIntoPhrases(cues, silences);if (resegmented.length > 0) {cuesToReturn = resegmented;}diff --git a/electron/ipc/captions/segment.test.ts b/electron/ipc/captions/segment.test.tsnew file mode 100644index 00000000..94344d59--- /dev/null+++ b/electron/ipc/captions/segment.test.ts@@ -0,0 +1,347 @@+import { describe, expect, it } from "vitest";+import type { CaptionCuePayload } from "../types";+import { endsSentence, segmentCuesIntoPhrases } from "./segment";++describe("endsSentence", () => {+ it("detects terminal punctuation, ignoring trailing closers", () => {+ expect(endsSentence("world.")).toBe(true);+ expect(endsSentence("you?")).toBe(true);+ expect(endsSentence("stop!")).toBe(true);+ expect(endsSentence("wait…")).toBe(true);+ expect(endsSentence('said."')).toBe(true);+ expect(endsSentence("done,")).toBe(false);+ expect(endsSentence("hello")).toBe(false);+ });++ it("does not treat common abbreviations or initialisms as a sentence end", () => {+ expect(endsSentence("Mr.")).toBe(false);+ expect(endsSentence("Dr.")).toBe(false);+ expect(endsSentence("etc.")).toBe(false);+ expect(endsSentence("e.g.")).toBe(false);+ expect(endsSentence("U.S.")).toBe(false);+ expect(endsSentence("J.")).toBe(false);+ // real sentence ends are unaffected+ expect(endsSentence("Smith.")).toBe(true);+ expect(endsSentence("5.")).toBe(true);+ // `?`/`!` after an abbreviation-like token still end the sentence+ expect(endsSentence("really?")).toBe(true);+ });+});++describe("segmentCuesIntoPhrases", () => {+ it("splits back-to-back sentences with no pause into separate captions", () => {+ const cues: CaptionCuePayload[] = [+ {+ id: "caption-1",+ startMs: 0,+ endMs: 1_800,+ text: "Hello world. How are you?",+ words: [+ { text: "Hello", startMs: 0, endMs: 400 },+ { text: "world.", startMs: 400, endMs: 800, leadingSpace: true },+ { text: "How", startMs: 800, endMs: 1_100, leadingSpace: true },+ { text: "are", startMs: 1_100, endMs: 1_400, leadingSpace: true },+ { text: "you?", startMs: 1_400, endMs: 1_800, leadingSpace: true },+ ],+ },+ ];+ const result = segmentCuesIntoPhrases(cues, []);+ expect(result).toHaveLength(2);+ expect(result[0].text).toBe("Hello world.");+ expect(result[1].text).toBe("How are you?");+ expect(result[0].endMs).toBeLessThanOrEqual(result[1].startMs);+ });++ it("does not leak the first word of the next sentence into the previous caption", () => {+ const cues: CaptionCuePayload[] = [+ {+ id: "caption-1",+ startMs: 0,+ endMs: 1_800,+ text: "Hello world. How are you?",+ words: [+ { text: "Hello", startMs: 0, endMs: 400 },+ { text: "world.", startMs: 400, endMs: 800, leadingSpace: true },+ { text: "How", startMs: 800, endMs: 1_100, leadingSpace: true },+ { text: "are", startMs: 1_100, endMs: 1_400, leadingSpace: true },+ { text: "you?", startMs: 1_400, endMs: 1_800, leadingSpace: true },+ ],+ },+ ];+ const result = segmentCuesIntoPhrases(cues, []);+ expect(result[1].words?.[0].text).toBe("How");+ expect(result[1].words?.[0].startMs).toBe(800);+ // the first word of a phrase has no leading space+ expect(result[1].words?.[0].leadingSpace).toBeUndefined();+ });++ it("merges rapid-fire short sentences into one caption (word-timed)", () => {+ const cues: CaptionCuePayload[] = [+ {+ id: "caption-1",+ startMs: 0,+ endMs: 800,+ text: "Okay. Great.",+ words: [+ { text: "Okay.", startMs: 0, endMs: 400 },+ { text: "Great.", startMs: 400, endMs: 800, leadingSpace: true },+ ],+ },+ ];+ const result = segmentCuesIntoPhrases(cues, []);+ expect(result).toHaveLength(1);+ expect(result[0].text).toBe("Okay. Great.");+ expect(result[0].words?.map((word) => word.text)).toEqual(["Okay.", "Great."]);+ });++ it("merges rapid-fire short sentences into one caption (word-less)", () => {+ const cues: CaptionCuePayload[] = [+ { id: "caption-1", startMs: 0, endMs: 400, text: "Okay." },+ { id: "caption-2", startMs: 400, endMs: 800, text: "Great." },+ ];+ const result = segmentCuesIntoPhrases(cues, []);+ expect(result).toHaveLength(1);+ expect(result[0].text).toBe("Okay. Great.");+ });++ it("does not merge a short sentence into a full-length one", () => {+ const cues: CaptionCuePayload[] = [+ {+ id: "caption-1",+ startMs: 0,+ endMs: 2_400,+ text: "Okay. This is a full length sentence that should stand alone.",+ words: [+ { text: "Okay.", startMs: 0, endMs: 400 },+ { text: "This", startMs: 400, endMs: 700, leadingSpace: true },+ { text: "is", startMs: 700, endMs: 900, leadingSpace: true },+ { text: "a", startMs: 900, endMs: 1_050, leadingSpace: true },+ { text: "full", startMs: 1_050, endMs: 1_350, leadingSpace: true },+ { text: "length", startMs: 1_350, endMs: 1_650, leadingSpace: true },+ { text: "sentence", startMs: 1_650, endMs: 1_950, leadingSpace: true },+ { text: "that", startMs: 1_950, endMs: 2_100, leadingSpace: true },+ { text: "should", startMs: 2_100, endMs: 2_250, leadingSpace: true },+ { text: "stand", startMs: 2_250, endMs: 2_350, leadingSpace: true },+ { text: "alone.", startMs: 2_350, endMs: 2_400, leadingSpace: true },+ ],+ },+ ];+ const result = segmentCuesIntoPhrases(cues, []);+ expect(result).toHaveLength(2);+ expect(result[0].text).toBe("Okay.");+ expect(result[1].text).toBe("This is a full length sentence that should stand alone.");+ });++ it("keeps a comma/clause inside one caption", () => {+ const cues: CaptionCuePayload[] = [+ {+ id: "caption-1",+ startMs: 0,+ endMs: 1_500,+ text: "When I'm done, I start.",+ words: [+ { text: "When", startMs: 0, endMs: 300 },+ { text: "I'm", startMs: 300, endMs: 600, leadingSpace: true },+ { text: "done,", startMs: 600, endMs: 900, leadingSpace: true },+ { text: "I", startMs: 900, endMs: 1_100, leadingSpace: true },+ { text: "start.", startMs: 1_100, endMs: 1_500, leadingSpace: true },+ ],+ },+ ];+ const result = segmentCuesIntoPhrases(cues, []);+ expect(result).toHaveLength(1);+ expect(result[0].text).toBe("When I'm done, I start.");+ });++ it("does not split on an abbreviation, only on the real sentence end", () => {+ const cues: CaptionCuePayload[] = [+ {+ id: "caption-1",+ startMs: 0,+ endMs: 1_800,+ text: "Hello Mr. Smith. How are you?",+ words: [+ { text: "Hello", startMs: 0, endMs: 300 },+ { text: "Mr.", startMs: 300, endMs: 600, leadingSpace: true },+ { text: "Smith.", startMs: 600, endMs: 900, leadingSpace: true },+ { text: "How", startMs: 900, endMs: 1_200, leadingSpace: true },+ { text: "are", startMs: 1_200, endMs: 1_500, leadingSpace: true },+ { text: "you?", startMs: 1_500, endMs: 1_800, leadingSpace: true },+ ],+ },+ ];+ const result = segmentCuesIntoPhrases(cues, []);+ expect(result).toHaveLength(2);+ expect(result[0].text).toBe("Hello Mr. Smith.");+ expect(result[1].text).toBe("How are you?");+ });++ it("splits on a real pause even without punctuation", () => {+ const cues: CaptionCuePayload[] = [+ {+ id: "caption-1",+ startMs: 0,+ endMs: 2_000,+ text: "one two three four",+ words: [+ { text: "one", startMs: 0, endMs: 300 },+ { text: "two", startMs: 300, endMs: 600, leadingSpace: true },+ // 800ms gap (>= 700ms pause) -> phrase boundary+ { text: "three", startMs: 1_400, endMs: 1_700, leadingSpace: true },+ { text: "four", startMs: 1_700, endMs: 2_000, leadingSpace: true },+ ],+ },+ ];+ const result = segmentCuesIntoPhrases(cues, []);+ expect(result).toHaveLength(2);+ expect(result[0].text).toBe("one two");+ expect(result[1].text).toBe("three four");+ });++ it("splits on a long acoustic silence in a word gap even when the word gap is short", () => {+ const cues: CaptionCuePayload[] = [+ {+ id: "caption-1",+ startMs: 0,+ endMs: 2_600,+ text: "alpha beta",+ words: [+ { text: "alpha", startMs: 0, endMs: 600 },+ { text: "beta", startMs: 2_200, endMs: 2_600, leadingSpace: true },+ ],+ },+ ];+ // pauseMs high so the word gap (1600ms) wouldn't split on its own; the 1500ms+ // silence sitting in the gap is what forces the break. beta starts after the+ // silence ends, so it is not dropped as a hallucination.+ const result = segmentCuesIntoPhrases(cues, [{ startMs: 600, endMs: 2_100 }], {+ pauseMs: 5_000,+ });+ expect(result).toHaveLength(2);+ expect(result[0].text).toBe("alpha");+ expect(result[1].text).toBe("beta");+ });++ it("does not split that same input when there is no silence and the gap is below the pause", () => {+ const cues: CaptionCuePayload[] = [+ {+ id: "caption-1",+ startMs: 0,+ endMs: 2_600,+ text: "alpha beta",+ words: [+ { text: "alpha", startMs: 0, endMs: 600 },+ { text: "beta", startMs: 2_200, endMs: 2_600, leadingSpace: true },+ ],+ },+ ];+ const result = segmentCuesIntoPhrases(cues, [], { pauseMs: 5_000 });+ expect(result).toHaveLength(1);+ expect(result[0].text).toBe("alpha beta");+ });++ it("splits a runaway phrase with no punctuation and no pause via the safety cap", () => {+ const words = Array.from({ length: 12 }, (_, index) => ({+ text: `w${index}`,+ startMs: index * 300,+ endMs: index * 300 + 300,+ ...(index > 0 ? { leadingSpace: true } : {}),+ }));+ const cues: CaptionCuePayload[] = [+ {+ id: "caption-1",+ startMs: 0,+ endMs: 3_600,+ text: words.map((w) => w.text).join(" "),+ words,+ },+ ];+ const result = segmentCuesIntoPhrases(cues, [], { maxPhraseMs: 2_000 });+ expect(result.length).toBeGreaterThan(1);+ expect(result.map((cue) => cue.text).join(" ")).toBe(words.map((w) => w.text).join(" "));+ });++ it("drops words that fall entirely inside a long silence (hallucination)", () => {+ const cues: CaptionCuePayload[] = [+ {+ id: "caption-1",+ startMs: 5_000,+ endMs: 6_000,+ text: "Thank you.",+ words: [+ { text: "Thank", startMs: 5_000, endMs: 5_500 },+ { text: "you.", startMs: 5_500, endMs: 6_000, leadingSpace: true },+ ],+ },+ ];+ const result = segmentCuesIntoPhrases(cues, [{ startMs: 4_000, endMs: 9_000 }]);+ expect(result).toHaveLength(0);+ });++ it("produces sorted, non-overlapping cues with sequential ids", () => {+ const cues: CaptionCuePayload[] = [+ {+ id: "caption-1",+ startMs: 0,+ endMs: 2_400,+ text: "First phrase here. Second phrase here. Third one.",+ words: [+ { text: "First", startMs: 0, endMs: 300 },+ { text: "phrase", startMs: 300, endMs: 600, leadingSpace: true },+ { text: "here.", startMs: 600, endMs: 900, leadingSpace: true },+ { text: "Second", startMs: 900, endMs: 1_200, leadingSpace: true },+ { text: "phrase", startMs: 1_200, endMs: 1_500, leadingSpace: true },+ { text: "here.", startMs: 1_500, endMs: 1_800, leadingSpace: true },+ { text: "Third", startMs: 1_800, endMs: 2_100, leadingSpace: true },+ { text: "one.", startMs: 2_100, endMs: 2_400, leadingSpace: true },+ ],+ },+ ];+ const result = segmentCuesIntoPhrases(cues, []);+ expect(result).toHaveLength(3);+ for (let index = 0; index < result.length - 1; index += 1) {+ expect(result[index].startMs).toBeLessThanOrEqual(result[index + 1].startMs);+ expect(result[index].endMs).toBeLessThanOrEqual(result[index + 1].startMs);+ }+ expect(result.map((cue) => cue.id)).toEqual(+ result.map((_, index) => `caption-${index + 1}`),+ );+ });++ it("keeps a single-sentence word-less cue intact (legacy silence trim preserved)", () => {+ const cues: CaptionCuePayload[] = [+ { id: "caption-1", startMs: 0, endMs: 13_920, text: "Hello, this is a test" },+ ];+ // One sentence -> unchanged: trim leading silence, keep text (legacy behavior).+ const result = segmentCuesIntoPhrases(cues, [{ startMs: 0, endMs: 3_000 }]);+ expect(result).toHaveLength(1);+ expect(result[0].startMs).toBe(3_000 - 80);+ expect(result[0].endMs).toBe(13_920 + 80);+ expect(result[0].text).toBe("Hello, this is a test");+ });++ it("splits a continuous word-less paragraph into one caption per sentence", () => {+ // The reported bug: no word timings (SRT path) + continuous speech collapsed into one+ // caption. It must now split on sentence punctuation even without word timings.+ const cues: CaptionCuePayload[] = [+ { id: "caption-1", startMs: 0, endMs: 3_000, text: "This is one." },+ { id: "caption-2", startMs: 3_000, endMs: 6_000, text: "This is two." },+ { id: "caption-3", startMs: 6_000, endMs: 9_000, text: "This is three." },+ ];+ const result = segmentCuesIntoPhrases(cues, []);+ expect(result).toHaveLength(3);+ expect(result.map((cue) => cue.text)).toEqual([+ "This is one.",+ "This is two.",+ "This is three.",+ ]);+ for (let index = 0; index < result.length - 1; index += 1) {+ expect(result[index].endMs).toBeLessThanOrEqual(result[index + 1].startMs);+ }+ expect(result.map((cue) => cue.id)).toEqual(["caption-1", "caption-2", "caption-3"]);+ });++ it("returns an empty array for empty input", () => {+ expect(segmentCuesIntoPhrases([], [])).toEqual([]);+ });+});diff --git a/electron/ipc/captions/segment.ts b/electron/ipc/captions/segment.tsnew file mode 100644index 00000000..912f089c--- /dev/null+++ b/electron/ipc/captions/segment.ts@@ -0,0 +1,451 @@+import type { CaptionCuePayload, CaptionWordPayload } from "../types";+import { buildCaptionTextFromWords } from "./parser";+import { padSpans, resegmentCuesBySilence, type SilenceInterval } from "./silence";++/**+ * Phrase-aware caption segmentation.+ *+ * Whisper breaks speech on its own internal boundaries — not on sentences — so a single+ * cue can run two phrases together, and re-segmenting purely on acoustic silence merges+ * back-to-back sentences and misassigns boundary words. We instead walk Whisper's own+ * word stream (which carries punctuation) and start a new caption at a real boundary:+ * - the end of a sentence (`.`, `?`, `!`, `…`), or+ * - a real pause — a large gap between two consecutive words, or a long ffmpeg+ * `silencedetect` interval sitting in that gap.+ *+ * Because every break happens *between two consecutive words*, a word can never leak into+ * the wrong caption (the failure mode of center-time region assignment). Commas/clauses+ * stay inside a caption and a whole sentence is allowed to be one caption; only a high+ * safety cap splits a runaway phrase with no punctuation and no pause.+ *+ * When the transcript has no word timings (SRT fallback) we first re-segment by acoustic+ * silence (`resegmentCuesBySilence`) and then split each cue on its sentence punctuation, so+ * a continuous paragraph still becomes one caption per sentence (timing is proportional).+ */++/** A gap (ms) between two consecutive words this long or longer starts a new phrase. */+const DEFAULT_PHRASE_PAUSE_MS = 700;+/** An ffmpeg `silencedetect` interval this long inside a word gap also starts a new phrase. */+const DEFAULT_SPLIT_SILENCE_MS = 1_500;+/** Padding (ms) kept around each phrase so captions don't feel clipped. */+const DEFAULT_EDGE_PAD_MS = 80;+/** Safety cap: a phrase with no sentence end and no pause is split once it gets this long. */+const DEFAULT_MAX_PHRASE_MS = 12_000;++/**+ * A caption shorter than this (ms) is "too quick" and may be merged with an adjacent short+ * caption so rapid-fire one-word sentences ("Okay." "Great.") don't each flash by alone.+ */+const DEFAULT_MIN_CAPTION_MS = 800;+/** Only merge short captions separated by at most this gap (ms) — never across a real pause. */+const DEFAULT_MERGE_GAP_MS = 400;+/** A merged short-caption run never grows past this duration (ms) or character count. */+const DEFAULT_MAX_MERGED_MS = 2_500;+const DEFAULT_MAX_MERGED_CHARS = 80;++export interface SegmentOptions {+ /** Word gap (ms) that splits one phrase into two. Lower = more, shorter captions. */+ pauseMs?: number;+ /** Minimum acoustic silence (ms) inside a word gap that also splits a phrase. */+ splitSilenceMs?: number;+ /** Padding (ms) kept around each phrase. */+ edgePadMs?: number;+ /** Safety cap (ms) that splits a punctuation-less, pause-less runaway phrase. */+ maxPhraseMs?: number;+ /** A caption shorter than this (ms) may be merged with an adjacent short caption. */+ minCaptionMs?: number;+ /** Only merge short captions separated by at most this gap (ms). */+ mergeGapMs?: number;+ /** A merged short-caption run never grows past this duration (ms). */+ maxMergedMs?: number;+ /** A merged short-caption run never grows past this character count. */+ maxMergedChars?: number;+}++interface CaptionPiece {+ startMs: number;+ endMs: number;+ text: string;+ words: CaptionWordPayload[];+}++const SENTENCE_END = /[.?!…。!?]$/;+/** Closing quotes/brackets that can trail terminal punctuation, e.g. `said."` */+const TRAILING_CLOSERS = /[)\]}"'”’»」』)】]}>]+$/u;++/**+ * Unambiguous English titles that take a trailing period mid-sentence. Kept deliberately+ * short: only words that are never themselves a sentence (so we don't suppress a real+ * break — e.g. "no" is excluded because "No." is a valid sentence). Dotted initialisms+ * like "e.g."/"U.S."/"a.m." are handled by the regex below, not this list. `?`/`!`/`…`+ * always end a sentence. For non-English audio this simply never matches.+ */+const ABBREVIATIONS = new Set(["mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "vs", "etc"]);++/** A trailing period belongs to an abbreviation/initialism rather than ending a sentence. */+function isAbbreviation(text: string): boolean {+ const trimmed = text.trim().replace(TRAILING_CLOSERS, "").trim();+ if (!trimmed.endsWith(".")) {+ return false; // only a plain period can be an abbreviation marker+ }+ const core = trimmed.slice(0, -1).toLowerCase();+ if (core.length === 0) {+ return false;+ }+ // Single-letter initial ("J.", "U.") or dotted initialism ("U.S.", "e.g.", "a.m.").+ if (/^[a-z]$/.test(core) || /^[a-z](\.[a-z])+$/.test(core)) {+ return true;+ }+ return ABBREVIATIONS.has(core);+}++/**+ * True when a word's text ends a sentence, ignoring trailing closing quotes/brackets and+ * common abbreviations (so "Mr. Smith" or "e.g." don't start a new caption).+ */+export function endsSentence(text: string): boolean {+ const trimmed = text.trim().replace(TRAILING_CLOSERS, "").trim();+ if (!SENTENCE_END.test(trimmed)) {+ return false;+ }+ return !isAbbreviation(text);+}++/** Every cue carries usable word timing, so we can segment on the word stream. */+function hasWordTimings(cues: CaptionCuePayload[]): boolean {+ return cues.length > 0 && cues.every((cue) => Array.isArray(cue.words) && cue.words.length > 0);+}++/** Flatten all cues' words into one time-ordered stream, spacing across cue joins. */+function flattenWords(cues: CaptionCuePayload[]): CaptionWordPayload[] {+ const stream: CaptionWordPayload[] = [];+ for (const cue of cues) {+ const words = (cue.words ?? []) as CaptionWordPayload[];+ words.forEach((word, index) => {+ // A new cue continues the speech, so its first word leads with a space.+ const leadingSpace =+ stream.length > 0 && (index === 0 ? true : word.leadingSpace !== false);+ stream.push({+ text: word.text,+ startMs: word.startMs,+ endMs: word.endMs,+ ...(leadingSpace ? { leadingSpace: true } : {}),+ });+ });+ }+ return stream.sort((left, right) => left.startMs - right.startMs || left.endMs - right.endMs);+}++/** Drop words that sit entirely inside a long detected silence (Whisper hallucinations). */+function dropHallucinations(+ words: CaptionWordPayload[],+ silences: SilenceInterval[],+ splitSilenceMs: number,+): CaptionWordPayload[] {+ const longSilences = silences.filter(+ (silence) => silence.endMs - silence.startMs >= splitSilenceMs,+ );+ if (longSilences.length === 0) {+ return words;+ }+ return words.filter(+ (word) =>+ !longSilences.some(+ (silence) => silence.startMs <= word.startMs && word.endMs <= silence.endMs,+ ),+ );+}++/** A long silence interval overlaps the gap between two consecutive words. */+function silenceInGap(+ gapStartMs: number,+ gapEndMs: number,+ silences: SilenceInterval[],+ splitSilenceMs: number,+): boolean {+ return silences.some(+ (silence) =>+ silence.endMs - silence.startMs >= splitSilenceMs &&+ silence.startMs < gapEndMs &&+ silence.endMs > gapStartMs,+ );+}++/** Reset the first word's leading space so a phrase reads as its own line. */+function normalizePhraseWords(words: CaptionWordPayload[]): CaptionWordPayload[] {+ return words.map((word, index) => {+ if (index === 0 && word.leadingSpace) {+ const { leadingSpace: _leadingSpace, ...rest } = word;+ return rest;+ }+ return word;+ });+}++/** Group a cue's words into runs that each end on a sentence boundary. */+function groupWordsBySentence(words: CaptionWordPayload[]): CaptionWordPayload[][] {+ const groups: CaptionWordPayload[][] = [];+ let current: CaptionWordPayload[] = [];+ words.forEach((word, index) => {+ current.push(word);+ if (endsSentence(word.text) && index < words.length - 1) {+ groups.push(current);+ current = [];+ }+ });+ if (current.length > 0) {+ groups.push(current);+ }+ return groups;+}++/**+ * Split a word-less cue's text into one cue per sentence, distributing the cue's time span+ * across sentences by character length. Used on the fallback (no word timing) path so a+ * continuous paragraph still becomes one caption per sentence.+ */+function splitTextBySentence(cue: CaptionCuePayload): CaptionCuePayload[] {+ const tokens = cue.text.trim().split(/\s+/).filter(Boolean);+ if (tokens.length <= 1) {+ return [cue];+ }++ const groups: string[][] = [];+ let current: string[] = [];+ tokens.forEach((token, index) => {+ current.push(token);+ if (endsSentence(token) && index < tokens.length - 1) {+ groups.push(current);+ current = [];+ }+ });+ if (current.length > 0) {+ groups.push(current);+ }+ if (groups.length <= 1) {+ return [cue];+ }++ const texts = groups.map((group) => group.join(" "));+ const totalChars = texts.reduce((sum, text) => sum + text.length, 0) || 1;+ const spanMs = Math.max(1, cue.endMs - cue.startMs);+ let cursorMs = cue.startMs;+ return texts.map((text, index) => {+ const startMs = cursorMs;+ const endMs =+ index === texts.length - 1+ ? cue.endMs+ : Math.min(+ cue.endMs - 1,+ Math.round(startMs + (spanMs * text.length) / totalChars),+ );+ cursorMs = Math.max(startMs + 1, endMs);+ return { id: cue.id, startMs, endMs: Math.max(startMs + 1, endMs), text };+ });+}++/** Split one re-segmented cue into one cue per sentence (by words if present, else text). */+function splitCueBySentences(cue: CaptionCuePayload): CaptionCuePayload[] {+ const words = Array.isArray(cue.words) ? (cue.words as CaptionWordPayload[]) : [];+ if (words.length === 0) {+ return splitTextBySentence(cue);+ }++ const groups = groupWordsBySentence(words);+ if (groups.length <= 1) {+ return [cue];+ }+ return groups.map((group) => {+ const phraseWords = normalizePhraseWords(group);+ return {+ id: cue.id,+ startMs: phraseWords[0].startMs,+ endMs: phraseWords[phraseWords.length - 1].endMs,+ text: buildCaptionTextFromWords(phraseWords),+ words: phraseWords,+ };+ });+}++/** Concatenate two adjacent cues into one, joining words (with a space) when both have them. */+function mergeTwoCues(left: CaptionCuePayload, right: CaptionCuePayload): CaptionCuePayload {+ const leftWords = Array.isArray(left.words) ? (left.words as CaptionWordPayload[]) : [];+ const rightWords = Array.isArray(right.words) ? (right.words as CaptionWordPayload[]) : [];+ if (leftWords.length > 0 && rightWords.length > 0) {+ // The right cue's first word started its own phrase (no leading space) — restore it.+ const joined = normalizePhraseWords([+ ...leftWords,+ ...rightWords.map((word, index) =>+ index === 0 ? { ...word, leadingSpace: true } : word,+ ),+ ]);+ return {+ id: left.id,+ startMs: left.startMs,+ endMs: right.endMs,+ text: buildCaptionTextFromWords(joined),+ words: joined,+ };+ }+ return {+ id: left.id,+ startMs: left.startMs,+ endMs: right.endMs,+ text: `${left.text} ${right.text}`.trim(),+ };+}++interface MergeOptions {+ minCaptionMs: number;+ mergeGapMs: number;+ maxMergedMs: number;+ maxMergedChars: number;+}++/**+ * Merge adjacent captions that are BOTH short and rapid-fire (tiny gap), so quick one-word+ * sentences like "Okay." "Great." read as one caption instead of flashing by individually.+ * Only merges when both sides are short, so a short caption never absorbs a full-length one,+ * and never across a real pause or past the size caps.+ */+function mergeShortAdjacentCaptions(+ cues: CaptionCuePayload[],+ options: MergeOptions,+): CaptionCuePayload[] {+ if (cues.length <= 1) {+ return cues;+ }++ const merged: CaptionCuePayload[] = [];+ let group = cues[0];+ for (let index = 1; index < cues.length; index += 1) {+ const next = cues[index];+ const groupDurationMs = group.endMs - group.startMs;+ const nextDurationMs = next.endMs - next.startMs;+ const gapMs = next.startMs - group.endMs;+ const combinedDurationMs = next.endMs - group.startMs;+ const combinedChars = group.text.length + next.text.length + 1;++ const canMerge =+ groupDurationMs < options.minCaptionMs &&+ nextDurationMs < options.minCaptionMs &&+ gapMs <= options.mergeGapMs &&+ combinedDurationMs <= options.maxMergedMs &&+ combinedChars <= options.maxMergedChars;++ if (canMerge) {+ group = mergeTwoCues(group, next);+ } else {+ merged.push(group);+ group = next;+ }+ }+ merged.push(group);+ return merged;+}++/** Assign sequential, stable ids to the final cue list. */+function renumberCues(cues: CaptionCuePayload[]): CaptionCuePayload[] {+ return cues.map((cue, index) => ({ ...cue, id: `caption-${index + 1}` }));+}++/**+ * Re-segment Whisper cues into one caption per sentence/phrase, then merge rapid-fire short+ * sentences back together. Returns sorted, non-overlapping cues with fresh ids. Falls back to+ * silence-only re-segmentation (plus sentence splitting) when the transcript has no word timings.+ */+export function segmentCuesIntoPhrases(+ cues: CaptionCuePayload[],+ silences: SilenceInterval[],+ options: SegmentOptions = {},+): CaptionCuePayload[] {+ const pauseMs = options.pauseMs ?? DEFAULT_PHRASE_PAUSE_MS;+ const splitSilenceMs = options.splitSilenceMs ?? DEFAULT_SPLIT_SILENCE_MS;+ const edgePadMs = options.edgePadMs ?? DEFAULT_EDGE_PAD_MS;+ const maxPhraseMs = options.maxPhraseMs ?? DEFAULT_MAX_PHRASE_MS;+ const mergeOptions: MergeOptions = {+ minCaptionMs: options.minCaptionMs ?? DEFAULT_MIN_CAPTION_MS,+ mergeGapMs: options.mergeGapMs ?? DEFAULT_MERGE_GAP_MS,+ maxMergedMs: options.maxMergedMs ?? DEFAULT_MAX_MERGED_MS,+ maxMergedChars: options.maxMergedChars ?? DEFAULT_MAX_MERGED_CHARS,+ };++ if (cues.length === 0) {+ return [];+ }++ // No word timings (SRT path): the silence-only segmenter trims/merges by acoustic+ // silence but can't see sentence boundaries, so a continuous paragraph would collapse+ // into one caption. Re-segment by silence first, then split each cue on its sentence+ // punctuation so we still get one caption per sentence.+ if (!hasWordTimings(cues)) {+ const base = resegmentCuesBySilence(cues, silences, { splitSilenceMs, edgePadMs });+ const sentences = base.flatMap(splitCueBySentences);+ return renumberCues(mergeShortAdjacentCaptions(sentences, mergeOptions));+ }++ const sortedCues = [...cues].sort(+ (left, right) => left.startMs - right.startMs || left.endMs - right.endMs,+ );+ const stream = dropHallucinations(flattenWords(sortedCues), silences, splitSilenceMs);+ if (stream.length === 0) {+ return [];+ }++ const phrases: CaptionWordPayload[][] = [];+ let current: CaptionWordPayload[] = [];++ for (let index = 0; index < stream.length; index += 1) {+ const word = stream[index];+ current.push(word);++ const next = stream[index + 1];+ if (!next) {+ break;+ }++ const gapMs = next.startMs - word.endMs;+ const phraseDurationMs = word.endMs - current[0].startMs;+ const shouldBreak =+ endsSentence(word.text) ||+ gapMs >= pauseMs ||+ silenceInGap(word.endMs, next.startMs, silences, splitSilenceMs) ||+ phraseDurationMs >= maxPhraseMs;++ if (shouldBreak) {+ phrases.push(current);+ current = [];+ }+ }+ if (current.length > 0) {+ phrases.push(current);+ }++ const pieces: CaptionPiece[] = phrases+ .map((words) => {+ const phraseWords = normalizePhraseWords(words);+ return {+ startMs: phraseWords[0].startMs,+ endMs: phraseWords[phraseWords.length - 1].endMs,+ text: buildCaptionTextFromWords(phraseWords),+ words: phraseWords,+ };+ })+ .filter((piece) => piece.text.trim().length > 0);++ if (pieces.length === 0) {+ return [];+ }++ pieces.sort((left, right) => left.startMs - right.startMs || left.endMs - right.endMs);+ padSpans(pieces, edgePadMs);++ const sentenceCues: CaptionCuePayload[] = pieces.map((piece) => ({+ id: "",+ startMs: piece.startMs,+ endMs: piece.endMs,+ text: piece.text,+ ...(piece.words.length > 0 ? { words: piece.words } : {}),+ }));+ return renumberCues(mergeShortAdjacentCaptions(sentenceCues, mergeOptions));+}diff --git a/electron/ipc/captions/silence.ts b/electron/ipc/captions/silence.tsindex 2ddb7faa..c8750ca1 100644--- a/electron/ipc/captions/silence.ts+++ b/electron/ipc/captions/silence.ts@@ -171,7 +171,7 @@ function splitTextProportionally(text: string, overlaps: Span[]): string[] {}/** Pad spans toward neighbors by at most half the silent gap, so cues never overlap. */-function padSpans(spans: Span[], edgePadMs: number): void {+export function padSpans(spans: Span[], edgePadMs: number): void {for (let index = 0; index < spans.length; index += 1) {const prevEndMs = index > 0 ? spans[index - 1].endMs : null;const nextStartMs = index < spans.length - 1 ? spans[index + 1].startMs : null;diff --git a/src/components/video-editor/captionLayout.test.ts b/src/components/video-editor/captionLayout.test.tsnew file mode 100644index 00000000..9cdc733b--- /dev/null+++ b/src/components/video-editor/captionLayout.test.ts@@ -0,0 +1,41 @@+import { describe, expect, it } from "vitest";+import { flattenCaptionWords } from "./captionLayout";+import type { CaptionCue } from "./types";++describe("flattenCaptionWords", () => {+ it("forces a break at every cue boundary so each phrase shows on its own", () => {+ const cues: CaptionCue[] = [+ {+ id: "a",+ startMs: 0,+ endMs: 1_000,+ text: "hello world",+ words: [+ { text: "hello", startMs: 0, endMs: 500 },+ { text: "world", startMs: 500, endMs: 1_000, leadingSpace: true },+ ],+ },+ {+ // back-to-back with cue "a" (no gap) — would previously be re-packed by width+ id: "b",+ startMs: 1_000,+ endMs: 2_000,+ text: "next one",+ words: [+ { text: "next", startMs: 1_000, endMs: 1_500 },+ { text: "one", startMs: 1_500, endMs: 2_000, leadingSpace: true },+ ],+ },+ ];++ const flattened = flattenCaptionWords(cues);+ const firstWordOfSecondCue = flattened.find(+ (word) => word.cueId === "b" && word.cueWordIndex === 0,+ );++ expect(firstWordOfSecondCue?.forcedBreakBefore).toBe(true);+ expect(firstWordOfSecondCue?.leadingSpace).toBe(false);+ // the very first word of the first cue never forces a break+ expect(flattened[0].forcedBreakBefore).toBe(false);+ });+});diff --git a/src/components/video-editor/captionLayout.ts b/src/components/video-editor/captionLayout.tsindex 8b2faad2..adacaae6 100644--- a/src/components/video-editor/captionLayout.ts+++ b/src/components/video-editor/captionLayout.ts@@ -179,8 +179,10 @@ export function flattenCaptionWords(cues: CaptionCue[]) {const cueDuration = Math.max(1, cue.endMs - cue.startMs);const fallbackWordDuration = cueDuration / sourceWords.length;const previousCue = cueIndex > 0 ? cues[cueIndex - 1] : null;- const shouldForceCueBreak =- previousCue !== null && cue.startMs - previousCue.endMs >= CAPTION_BLOCK_GAP_BREAK_MS;+ // Each cue is its own phrase, so always start a new line/page at a cue boundary.+ // Otherwise back-to-back phrases (a small gap) get re-packed together by width and+ // their boundary disappears on screen — we want one phrase shown at a time.+ const shouldForceCueBreak = previousCue !== null;sourceWords.forEach((word, wordIndex) => {const fallbackStartMs = cue.startMs + fallbackWordDuration * wordIndex;