Create polished demo videos without editing skills. Mac/Windows/Linux
fix(captions): trim silence and split phrases on real pauses
Joe Hachem01ed7ea6f392
diff --git a/electron/ipc/captions/generate.ts b/electron/ipc/captions/generate.tsindex 33c34584..2cab7319 100644--- a/electron/ipc/captions/generate.ts+++ b/electron/ipc/captions/generate.ts@@ -1,14 +1,21 @@+import { execFile, spawnSync } from "node:child_process";import { constants as fsConstants } from "node:fs";import fs from "node:fs/promises";import path from "node:path";-import { execFile, spawnSync } from "node:child_process";import { promisify } from "node:util";import { app } from "electron";import { getFfmpegBinaryPath } from "../ffmpeg/binary";import { getBundledWhisperExecutableCandidates } from "../paths/binaries";-import { parseWhisperJsonCues, parseSrtCues, shouldRetryWhisperWithoutJson } from "./parser";-import { normalizeVideoSourcePath } from "../utils";import { resolveRecordingSession } from "../project/session";+import { normalizeVideoSourcePath } from "../utils";+import { parseSrtCues, parseWhisperJsonCues, shouldRetryWhisperWithoutJson } from "./parser";+import {+ parseSilenceIntervals,+ resegmentCuesBySilence,+ SILENCE_DETECT_MIN_S,+ SILENCE_NOISE_DB,+ type SilenceInterval,+} from "./silence";const execFileAsync = promisify(execFile);@@ -155,6 +162,30 @@ export async function extractCaptionAudioSource(options: {);}+export async function detectSilenceIntervals(options: {+ ffmpegPath: string;+ wavPath: string;+}): Promise<SilenceInterval[]> {+ // ffmpeg writes silencedetect results to stderr; the null muxer just runs the filter.+ const { stderr } = await execFileAsync(+ options.ffmpegPath,+ [+ "-hide_banner",+ "-nostats",+ "-i",+ options.wavPath,+ "-af",+ `silencedetect=noise=${SILENCE_NOISE_DB}dB:d=${SILENCE_DETECT_MIN_S}`,+ "-f",+ "null",+ "-",+ ],+ { timeout: 5 * 60 * 1000, maxBuffer: 20 * 1024 * 1024 },+ );++ return parseSilenceIntervals(stderr ?? "");+}+export async function generateAutoCaptionsFromVideo(options: {videoPath: string;whisperExecutablePath?: string;@@ -234,8 +265,25 @@ export async function generateAutoCaptionsFromVideo(options: {throw new Error("Whisper completed, but no caption cues were produced.");}+ // Whisper cues span silence and don't break on pauses. Re-segment them against+ // ground-truth acoustic silence (ffmpeg `silencedetect`) so captions only cover+ // real speech. Failure here must not block caption generation — fall back to raw.+ let cuesToReturn = cues;+ try {+ const silences = await detectSilenceIntervals({ ffmpegPath, wavPath });+ const resegmented = resegmentCuesBySilence(cues, silences);+ if (resegmented.length > 0) {+ cuesToReturn = resegmented;+ }+ } catch (error) {+ console.warn(+ "[auto-captions] Silence-aware re-segmentation failed, using raw cues:",+ error,+ );+ }+return {- cues,+ cues: cuesToReturn,audioSourceLabel: audioSource.label,};} finally {diff --git a/electron/ipc/captions/silence.test.ts b/electron/ipc/captions/silence.test.tsnew file mode 100644index 00000000..941d1076--- /dev/null+++ b/electron/ipc/captions/silence.test.ts@@ -0,0 +1,141 @@+import { describe, expect, it } from "vitest";+import type { CaptionCuePayload } from "../types";+import { parseSilenceIntervals, resegmentCuesBySilence } from "./silence";++describe("parseSilenceIntervals", () => {+ it("pairs silence_start / silence_end lines into ms intervals", () => {+ const stderr = [+ "[silencedetect @ 0x1] silence_start: 0",+ "[silencedetect @ 0x1] silence_end: 3.2 | silence_duration: 3.2",+ "[silencedetect @ 0x1] silence_start: 8.5",+ "[silencedetect @ 0x1] silence_end: 9.3 | silence_duration: 0.8",+ ].join("\n");+ expect(parseSilenceIntervals(stderr)).toEqual([+ { startMs: 0, endMs: 3_200 },+ { startMs: 8_500, endMs: 9_300 },+ ]);+ });++ it("treats a trailing silence_start with no end as running to end-of-audio", () => {+ const stderr = "[silencedetect] silence_start: 12.0";+ expect(parseSilenceIntervals(stderr)).toEqual([+ { startMs: 12_000, endMs: Number.POSITIVE_INFINITY },+ ]);+ });++ it("returns an empty array when there is no silence output", () => {+ expect(parseSilenceIntervals("some unrelated ffmpeg log\n")).toEqual([]);+ });+});++describe("resegmentCuesBySilence", () => {+ it("trims leading silence off a word-less cue", () => {+ const cues: CaptionCuePayload[] = [+ { id: "caption-1", startMs: 0, endMs: 13_920, text: "Hello, this is a test" },+ ];+ const result = resegmentCuesBySilence(cues, [{ startMs: 0, endMs: 3_000 }]);+ expect(result).toHaveLength(1);+ // speech span [3000, 13920]; only an 80ms pad re-enters the silence+ expect(result[0].startMs).toBe(3_000 - 80);+ expect(result[0].endMs).toBe(13_920 + 80);+ expect(result[0].text).toBe("Hello, this is a test");+ });++ it("does NOT split on a short pause (slowing down stays one phrase)", () => {+ const cues: CaptionCuePayload[] = [+ { id: "caption-1", startMs: 0, endMs: 8_000, text: "one two three four" },+ ];+ // 800ms pause is below the 1500ms split threshold -> absorbed, stays one phrase+ const result = resegmentCuesBySilence(cues, [{ startMs: 3_000, endMs: 3_800 }]);+ expect(result).toHaveLength(1);+ expect(result[0].text).toBe("one two three four");+ });++ it("merges adjacent cues that are not separated by a long pause", () => {+ const cues: CaptionCuePayload[] = [+ { id: "a", startMs: 0, endMs: 3_000, text: "hello world" },+ { id: "b", startMs: 3_000, endMs: 6_000, text: "how are you" },+ ];+ // continuous speech, no silence -> one merged phrase+ const result = resegmentCuesBySilence(cues, []);+ expect(result).toHaveLength(1);+ expect(result[0].text).toBe("hello world how are you");+ });++ it("splits only at a long pause (>= 1500ms)", () => {+ const cues: CaptionCuePayload[] = [+ { id: "caption-1", startMs: 0, endMs: 10_000, text: "one two three four" },+ ];+ // 2000ms pause -> a real boundary+ const result = resegmentCuesBySilence(cues, [{ startMs: 4_000, endMs: 6_000 }]);+ expect(result).toHaveLength(2);+ expect(result[0].endMs).toBeLessThanOrEqual(result[1].startMs);+ expect(`${result[0].text} ${result[1].text}`).toBe("one two three four");+ });++ it("does not over-split: two short pauses inside a sentence stay one phrase", () => {+ const cues: CaptionCuePayload[] = [+ { id: "caption-1", startMs: 0, endMs: 9_000, text: "so this is a slow sentence" },+ ];+ const result = resegmentCuesBySilence(cues, [+ { startMs: 2_000, endMs: 2_700 }, // 700ms+ { startMs: 5_000, endMs: 5_900 }, // 900ms+ ]);+ expect(result).toHaveLength(1);+ expect(result[0].text).toBe("so this is a slow sentence");+ });++ it("drops a cue that falls entirely inside silence (hallucination)", () => {+ const cues: CaptionCuePayload[] = [+ { id: "caption-1", startMs: 5_000, endMs: 7_000, text: "Thank you." },+ ];+ const result = resegmentCuesBySilence(cues, [{ startMs: 4_000, endMs: 9_000 }]);+ expect(result).toHaveLength(0);+ });++ it("keeps a continuous cue (no silence) as a single phrase", () => {+ const cues: CaptionCuePayload[] = [+ { id: "caption-1", startMs: 1_000, endMs: 4_000, text: "continuous speech" },+ ];+ const result = resegmentCuesBySilence(cues, []);+ expect(result).toHaveLength(1);+ expect(result[0].text).toBe("continuous speech");+ });++ it("splits a word-timed cue by assigning each word to its speech region", () => {+ const cues: CaptionCuePayload[] = [+ {+ id: "caption-1",+ startMs: 0,+ endMs: 10_000,+ text: "alpha beta gamma",+ words: [+ { text: "alpha", startMs: 500, endMs: 1_000 },+ { text: "beta", startMs: 1_000, endMs: 1_500, leadingSpace: true },+ { text: "gamma", startMs: 7_000, endMs: 7_500, leadingSpace: true },+ ],+ },+ ];+ // 4000ms pause puts alpha+beta in region 1, gamma in region 2+ const result = resegmentCuesBySilence(cues, [{ startMs: 2_000, endMs: 6_000 }]);+ expect(result).toHaveLength(2);+ expect(result[0].text).toBe("alpha beta");+ expect(result[1].text).toBe("gamma");+ expect(result[0].endMs).toBeLessThanOrEqual(result[1].startMs);+ });++ it("produces sorted, non-overlapping cues with sequential ids", () => {+ const cues: CaptionCuePayload[] = [+ { id: "x", startMs: 0, endMs: 6_000, text: "first phrase here" },+ { id: "y", startMs: 6_000, endMs: 14_000, text: "second phrase here" },+ ];+ const result = resegmentCuesBySilence(cues, [{ startMs: 6_000, endMs: 8_000 }]);+ for (let index = 0; index < result.length - 1; index += 1) {+ expect(result[index].startMs).toBeLessThanOrEqual(result[index + 1].startMs);+ expect(result[index].endMs).toBeLessThanOrEqual(result[index + 1].startMs);+ }+ expect(result.map((cue) => cue.id)).toEqual(+ result.map((_, index) => `caption-${index + 1}`),+ );+ });+});diff --git a/electron/ipc/captions/silence.ts b/electron/ipc/captions/silence.tsnew file mode 100644index 00000000..2ddb7faa--- /dev/null+++ b/electron/ipc/captions/silence.ts@@ -0,0 +1,323 @@+import type { CaptionCuePayload, CaptionWordPayload } from "../types";+import { buildCaptionTextFromWords } from "./parser";++/**+ * Silence-aware caption re-segmentation.+ *+ * Whisper breaks speech on its own internal boundaries, not on real pauses, so cues can+ * cover silence and continuous speech gets chopped arbitrarily. We re-segment against+ * ground-truth silence from ffmpeg `silencedetect` instead:+ * ffmpeg silencedetect (stderr) -> parseSilenceIntervals -> resegmentCuesBySilence+ *+ * A phrase only breaks at a long pause (>= splitSilenceMs) or at silence touching the+ * transcript edges (leading/trailing trim). Short pauses are absorbed; speech not separated+ * by a long pause is merged; a region with no speech is dropped (drops hallucinations).+ */++/** ffmpeg `silencedetect` noise floor. Quieter than this counts as silence. */+export const SILENCE_NOISE_DB = -30;+/** ffmpeg `silencedetect` minimum silence duration (seconds) it should report at all. */+export const SILENCE_DETECT_MIN_S = 0.5;++/** A pause must be at least this long (ms) to break one phrase into two. The "sensitivity" knob. */+const DEFAULT_SPLIT_SILENCE_MS = 1_500;+/** Padding (ms) kept around each phrase so captions don't feel clipped. */+const DEFAULT_EDGE_PAD_MS = 80;+/** Speech regions shorter than this (ms) are dropped as artifacts. */+const DEFAULT_MIN_SPEECH_MS = 150;++export interface SilenceInterval {+ startMs: number;+ /** `Number.POSITIVE_INFINITY` for a trailing silence that runs to end-of-audio. */+ endMs: number;+}++export interface ResegmentOptions {+ /** Minimum pause (ms) that splits a phrase. Higher = fewer splits. */+ splitSilenceMs?: number;+ /** Padding (ms) kept around each phrase. */+ edgePadMs?: number;+ /** Speech regions shorter than this (ms) are dropped. */+ minSpeechMs?: number;+}++interface Span {+ startMs: number;+ endMs: number;+}++interface CaptionPiece extends Span {+ text: string;+ words?: CaptionWordPayload[];+}++/**+ * Parse ffmpeg `silencedetect` stderr into ordered, non-overlapping silence intervals.+ * Lines look like:+ * [silencedetect @ 0x..] silence_start: 12.34+ * [silencedetect @ 0x..] silence_end: 15.67 | silence_duration: 3.33+ */+export function parseSilenceIntervals(stderr: string): SilenceInterval[] {+ const intervals: SilenceInterval[] = [];+ let pendingStartMs: number | null = null;++ for (const line of stderr.split(/\r?\n/)) {+ const startMatch = line.match(/silence_start:\s*(-?[\d.]+)/);+ if (startMatch) {+ pendingStartMs = Math.max(0, Math.round(Number.parseFloat(startMatch[1]) * 1000));+ continue;+ }++ const endMatch = line.match(/silence_end:\s*(-?[\d.]+)/);+ if (endMatch && pendingStartMs !== null) {+ const endMs = Math.round(Number.parseFloat(endMatch[1]) * 1000);+ if (endMs > pendingStartMs) {+ intervals.push({ startMs: pendingStartMs, endMs });+ }+ pendingStartMs = null;+ }+ }++ // A trailing silence_start with no matching end runs to the end of the audio.+ if (pendingStartMs !== null) {+ intervals.push({ startMs: pendingStartMs, endMs: Number.POSITIVE_INFINITY });+ }++ return intervals.sort((left, right) => left.startMs - right.startMs);+}++/** Subtract a set of (sorted, non-overlapping) intervals from [startMs, endMs]. */+function subtractIntervals(startMs: number, endMs: number, intervals: Span[]): Span[] {+ const spans: Span[] = [];+ let cursor = startMs;++ for (const interval of intervals) {+ if (interval.endMs <= startMs || interval.startMs >= endMs) {+ continue;+ }+ const clippedStart = Math.max(interval.startMs, startMs);+ if (clippedStart > cursor) {+ spans.push({ startMs: cursor, endMs: clippedStart });+ }+ cursor = Math.max(cursor, Math.min(interval.endMs, endMs));+ }++ if (cursor < endMs) {+ spans.push({ startMs: cursor, endMs });+ }++ return spans;+}++function nearestRegionIndex(regions: Span[], timeMs: number): number {+ let bestIndex = 0;+ let bestDistance = Number.POSITIVE_INFINITY;+ regions.forEach((region, index) => {+ const distance =+ timeMs < region.startMs+ ? region.startMs - timeMs+ : timeMs > region.endMs+ ? timeMs - region.endMs+ : 0;+ if (distance < bestDistance) {+ bestDistance = distance;+ bestIndex = index;+ }+ });+ return bestIndex;+}++function joinText(existing: string, addition: string): string {+ return [existing, addition]+ .map((part) => part.trim())+ .filter(Boolean)+ .join(" ");+}++/**+ * Distribute a cue's plain text across the regions it overlaps, proportionally to how much+ * of the cue's duration falls in each region. Used only when a single Whisper cue straddles+ * a long pause (rare) and has no word timing.+ */+function splitTextProportionally(text: string, overlaps: Span[]): string[] {+ const tokens = text.trim().split(/\s+/).filter(Boolean);+ if (tokens.length === 0) {+ return overlaps.map(() => "");+ }++ const totalDurationMs =+ overlaps.reduce((sum, span) => sum + (span.endMs - span.startMs), 0) || 1;+ const result: string[] = [];+ let cursor = 0;++ overlaps.forEach((span, index) => {+ const isLast = index === overlaps.length - 1;+ const share = (span.endMs - span.startMs) / totalDurationMs;+ const count = isLast+ ? tokens.length - cursor+ : Math.min(tokens.length - cursor, Math.round(tokens.length * share));+ result.push(tokens.slice(cursor, cursor + count).join(" "));+ cursor += count;+ });++ if (cursor < tokens.length) {+ result[result.length - 1] = joinText(+ result[result.length - 1],+ tokens.slice(cursor).join(" "),+ );+ }++ return result;+}++/** Pad spans toward neighbors by at most half the silent gap, so cues never overlap. */+function padSpans(spans: Span[], edgePadMs: number): void {+ for (let index = 0; index < spans.length; index += 1) {+ const prevEndMs = index > 0 ? spans[index - 1].endMs : null;+ const nextStartMs = index < spans.length - 1 ? spans[index + 1].startMs : null;+ const rawStart = spans[index].startMs;+ const rawEnd = spans[index].endMs;++ const leftPad =+ prevEndMs === null+ ? Math.min(edgePadMs, rawStart)+ : Math.min(edgePadMs, Math.floor(Math.max(0, rawStart - prevEndMs) / 2));+ const rightPad =+ nextStartMs === null+ ? edgePadMs+ : Math.min(edgePadMs, Math.floor(Math.max(0, nextStartMs - rawEnd) / 2));++ spans[index].startMs = Math.max(0, Math.round(rawStart - leftPad));+ spans[index].endMs = Math.max(spans[index].startMs + 1, Math.round(rawEnd + rightPad));+ }+}++/**+ * Re-segment Whisper cues against detected silence into phrases that only break on long+ * pauses. Returns sorted, non-overlapping cues with fresh ids. If no silence is detected+ * the cues are merged into a single trimmed phrase (continuous speech).+ */+export function resegmentCuesBySilence(+ cues: CaptionCuePayload[],+ silences: SilenceInterval[],+ options: ResegmentOptions = {},+): CaptionCuePayload[] {+ const splitSilenceMs = options.splitSilenceMs ?? DEFAULT_SPLIT_SILENCE_MS;+ const edgePadMs = options.edgePadMs ?? DEFAULT_EDGE_PAD_MS;+ const minSpeechMs = options.minSpeechMs ?? DEFAULT_MIN_SPEECH_MS;++ if (cues.length === 0) {+ return [];+ }++ const sortedCues = [...cues].sort(+ (left, right) => left.startMs - right.startMs || left.endMs - right.endMs,+ );+ const transcriptStartMs = sortedCues[0].startMs;+ const transcriptEndMs = sortedCues.reduce(+ (max, cue) => Math.max(max, cue.endMs),+ transcriptStartMs,+ );++ // Boundaries = long pauses, plus any silence touching the transcript edges (so leading+ // and trailing silence is always trimmed even if short).+ const boundaries: Span[] = silences+ .map((silence) => ({+ startMs: Math.max(silence.startMs, transcriptStartMs),+ endMs: Math.min(silence.endMs, transcriptEndMs),+ }))+ .filter((span) => span.endMs > span.startMs)+ .filter(+ (span) =>+ span.endMs - span.startMs >= splitSilenceMs ||+ span.startMs <= transcriptStartMs ||+ span.endMs >= transcriptEndMs,+ )+ .sort((left, right) => left.startMs - right.startMs);++ const regions = subtractIntervals(transcriptStartMs, transcriptEndMs, boundaries).filter(+ (region) => region.endMs - region.startMs >= minSpeechMs,+ );+ if (regions.length === 0) {+ return [];+ }++ const regionTexts = regions.map(() => "");+ const regionWords: CaptionWordPayload[][] = regions.map(() => []);++ for (const cue of sortedCues) {+ const overlapping = regions+ .map((region, index) => ({ region, index }))+ .filter(({ region }) => cue.startMs < region.endMs && cue.endMs > region.startMs);+ if (overlapping.length === 0) {+ continue;+ }++ const words = Array.isArray(cue.words) ? (cue.words as CaptionWordPayload[]) : [];+ if (words.length > 0) {+ for (const word of words) {+ const center = (word.startMs + word.endMs) / 2;+ const match = overlapping.find(+ ({ region }) => center >= region.startMs && center <= region.endMs,+ );+ const targetIndex =+ match?.index ??+ overlapping[+ nearestRegionIndex(+ overlapping.map((entry) => entry.region),+ center,+ )+ ].index;+ regionWords[targetIndex].push(word);+ }+ continue;+ }++ if (overlapping.length === 1) {+ const { index } = overlapping[0];+ regionTexts[index] = joinText(regionTexts[index], cue.text);+ continue;+ }++ const overlaps = overlapping.map(({ region }) => ({+ startMs: Math.max(cue.startMs, region.startMs),+ endMs: Math.min(cue.endMs, region.endMs),+ }));+ const texts = splitTextProportionally(cue.text, overlaps);+ overlapping.forEach(({ index }, position) => {+ regionTexts[index] = joinText(regionTexts[index], texts[position]);+ });+ }++ const pieces = regions+ .map((region, index): CaptionPiece => {+ const words = regionWords[index]+ .slice()+ .sort((left, right) => left.startMs - right.startMs || left.endMs - right.endMs);+ if (words.length > 0) {+ return {+ startMs: words[0].startMs,+ endMs: words[words.length - 1].endMs,+ text: buildCaptionTextFromWords(words),+ words,+ };+ }+ return {+ startMs: region.startMs,+ endMs: region.endMs,+ text: regionTexts[index].trim(),+ };+ })+ .filter((piece) => piece.text.trim().length > 0);++ pieces.sort((left, right) => left.startMs - right.startMs || left.endMs - right.endMs);+ padSpans(pieces, edgePadMs);++ return pieces.map((piece, index) => ({+ id: `caption-${index + 1}`,+ startMs: piece.startMs,+ endMs: piece.endMs,+ text: piece.text,+ ...(piece.words && piece.words.length > 0 ? { words: piece.words } : {}),+ }));+}