Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(stt): end each phrase on its speech offset, and look past its kept tail
EtienneLescot4260bb757fff
diff --git a/electron/stt/snapWordBoundaries.test.ts b/electron/stt/snapWordBoundaries.test.tsindex 46a6b4bf8..8a62d1c25 100644--- a/electron/stt/snapWordBoundaries.test.ts+++ b/electron/stt/snapWordBoundaries.test.ts@@ -83,10 +83,10 @@ describe("snapWordBoundariesToAudio with speech intervals", () => {const times = (words: SttWordSegment[]) =>words.map((w) => [w.word, ms(w.startSec), ms(w.endSec)]);- it("pulls each phrase's first word onto its onset, past the punctuation DTW dropped there", () => {+ it("puts each phrase's first word on its onset and its closing punctuation on its end", () => {// The shape of a real French take: whisper put "Salut" 0.58 s and "Bah"- // 0.25 s after the speech started, and the "!" closing the first phrase- // just after the second one began.+ // 0.25 s after the speech started, ran "Salut" on through the pause, and+ // dropped the "!" closing the first phrase just after the second began.const words = [word({ word: "Salut", startSec: 2.15, endSec: 3.37 }),word({ word: "!", startSec: 3.37, endSec: 3.39 }),@@ -98,16 +98,17 @@ describe("snapWordBoundariesToAudio with speech intervals", () => {{ startSec: 3.36, endSec: 4.35 },];expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([- ["Salut", 1.57, 3.36],- ["!", 3.36, 3.36],+ ["Salut", 1.57, 2.56],+ ["!", 2.56, 2.56],["Bah", 3.36, 4.01],["voilà", 4.01, 4.35],]);});- it("ends each phrase's last word where its speech stops, taking trailing punctuation along", () => {- // The end of the same take: whisper closed the last segment at 2.79 while- // "tac" ran to 3.04, and dropped the "!" on the word itself.+ it("ends each phrase's last word where its speech stops, stretched or cut back", () => {+ // The end of the same take: "tic," ran on into the pause, whisper closed+ // the last segment at 2.79 while "tac" ran to 3.04, and dropped the "!" on+ // the word itself.const words = [word({ word: "tic,", startSec: 1.95, endSec: 2.59 }),word({ word: "tac", startSec: 2.59, endSec: 2.79 }),@@ -118,14 +119,14 @@ describe("snapWordBoundariesToAudio with speech intervals", () => {{ startSec: 2.59, endSec: 3.04 },];expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([- ["tic,", 1.95, 2.59],+ ["tic,", 1.95, 2.37],["tac", 2.59, 3.04],["!", 3.04, 3.04],]);});it("leaves a phrase alone when its words are on time, or too far off to be its edges", () => {- const onTime = [word({ startSec: 0.95, endSec: 3.1 })];+ const onTime = [word({ startSec: 0.95, endSec: 3 })];expect(times(snapWordBoundariesToAudio(onTime, flat, [{ startSec: 1, endSec: 3 }]))).toEqual(times(onTime),);@@ -149,6 +150,29 @@ describe("snapWordBoundariesToAudio with speech intervals", () => {{ startSec: 0.5, endSec: 1 },{ startSec: 2, endSec: 3 },];- expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual(times(words));+ expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([+ ["x", 0.5, 1],+ ["y", 1.95, 2.4],+ ["z", 2.4, 3],+ ]);+ });++ it("does not take a word in the previous phrase's tail for the next one's first", () => {+ // The helper keeps 0.1 s past each offset. "b" was reported in the first+ // phrase's tail, and "c" still has to reach the second onset.+ const words = [+ word({ word: "a", startSec: 1, endSec: 2.05 }),+ word({ word: "b", startSec: 2.05, endSec: 3.4 }),+ word({ word: "c", startSec: 3.4, endSec: 4 }),+ ];+ const speech = [+ { startSec: 1, endSec: 2 },+ { startSec: 3, endSec: 4 },+ ];+ expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([+ ["a", 1, 2.05],+ ["b", 2.05, 2.07],+ ["c", 3, 4],+ ]);});});diff --git a/electron/stt/snapWordBoundaries.ts b/electron/stt/snapWordBoundaries.tsindex 2e61d465e..862de6518 100644--- a/electron/stt/snapWordBoundaries.ts+++ b/electron/stt/snapWordBoundaries.ts@@ -20,9 +20,10 @@//// The edges of a phrase are the exception. Its first word, DTW reports 0.1–0.6 s// late (measured against Silero VAD on a real French recording), far past what-// the lookback can reach; its last word ends where whisper's segment ends, which-// can stop short of the speech. When the helper sent its speech intervals, both-// are pulled onto the edges of the stretch of speech they belong to.+// the lookback can reach. Its last word ends where whisper's segment ends, which+// can stop short of the speech, or where the next word starts, which runs on+// through the whole pause. When the helper sent its speech intervals, both are+// put on the edges of the stretch of speech they belong to.import type { SttVadSegment, SttWordSegment } from "./transcriptionContract";@@ -45,13 +46,20 @@ const MIN_WORD_SEC = 0.02;/*** How far a phrase's first word may start after its speech, or its last word- * end before it, and still be pulled onto that edge — the calibration knob of+ * end before it, and still be stretched onto that edge — the calibration knob of* the anchoring step, sized above the measured 0.1–0.6 s. Past it, the word is* more likely a neighbour of one whisper dropped, and stretching it over that- * audio would be a guess.+ * audio would be a guess. Cutting a word back to where the speech stops needs+ * no such bound: the VAD says nothing is said past it.*/const MAX_ANCHOR_SEC = 1;+/**+ * Audio the helper keeps past each speech offset, as whisper.cpp does, so a soft+ * ending survives. A word DTW reports in there belongs to the stretch it trails.+ */+const TAIL_SEC = 0.1;+/** French puts a space before `!`, `?`, `:` and `;`, so whisper emits them as words. */const isPunctuation = (word: string) => /^[\p{P}\p{S}]+$/u.test(word);@@ -73,20 +81,22 @@ function rmsEnvelope(samples: Float32Array): Float32Array {}/**- * Pull the first and last word of every speech stretch onto the stretch's- * edges. The onset already carries the VAD's 30 ms pad, so a cut there lands- * just before the attack. What sits past an edge moves with it: the previous- * word's shared boundary, and any punctuation DTW dropped inside the speech,- * which collapses to a point on the edge.+ * Put the first and last word of every speech stretch on the stretch's edges.+ * The onset already carries the VAD's 30 ms pad, so a cut there lands just+ * before the attack. Words own the speech and no pause: the last word ends on+ * the offset, and the punctuation closing the phrase collapses to a point+ * there, wherever DTW dropped it.*/function anchorOnSpeech(words: SttWordSegment[], speech: SttVadSegment[]): SttWordSegment[] {const out = words.map((w) => ({ ...w }));let k = 0;for (let i = 0; i < speech.length; i++) {const { startSec: onset, endSec: offset } = speech[i];- const previousEnd = i > 0 ? speech[i - 1].endSec : Number.NEGATIVE_INFINITY;- // The phrase's first word: the first real word reported after the previous stretch ended.- while (k < out.length && (out[k].startSec < previousEnd || isPunctuation(out[k].word))) k++;+ const previousTail =+ i > 0 ? Math.min(speech[i - 1].endSec + TAIL_SEC, onset) : Number.NEGATIVE_INFINITY;+ const tail = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);+ // The phrase's first word: the first real word reported past the previous stretch's tail.+ while (k < out.length && (out[k].startSec < previousTail || isPunctuation(out[k].word))) k++;if (k === out.length) break;const late = out[k].startSec - onset;if (late > 0 && late <= MAX_ANCHOR_SEC && out[k].startSec < offset) {@@ -96,18 +106,16 @@ function anchorOnSpeech(words: SttWordSegment[], speech: SttVadSegment[]): SttWoout[j].startSec = Math.min(out[j].startSec, onset);}}- // Its last word: the last real word reported before the speech stops.+ // Its last word: the last real word reported before its tail ends.let m = -1;- for (let j = k; j < out.length && out[j].startSec < offset; j++) {+ for (let j = k; j < out.length && out[j].startSec < tail; j++) {if (!isPunctuation(out[j].word)) m = j;}- const early = m < 0 ? 0 : offset - out[m].endSec;- if (early > 0 && early <= MAX_ANCHOR_SEC) {- out[m].endSec = offset;- for (let j = m + 1; j < out.length && out[j].startSec < offset; j++) {- out[j].startSec = offset;- out[j].endSec = Math.max(out[j].endSec, offset);- }+ if (m < 0 || offset - out[m].endSec > MAX_ANCHOR_SEC) continue;+ out[m].endSec = Math.max(offset, out[m].startSec + MIN_WORD_SEC);+ for (let j = m + 1; j < out.length && isPunctuation(out[j].word); j++) {+ out[j].startSec = out[m].endSec;+ out[j].endSec = out[m].endSec;}}return out;diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.mdindex 026350743..0e42ed1b4 100644--- a/technical-documentation/architecture/transcription-and-captions.md+++ b/technical-documentation/architecture/transcription-and-captions.md@@ -233,12 +233,16 @@ it verbatim in the response.The edges of a phrase are past that reach. Its first word, DTW reports0.1–0.6 s after the speech starts (measured against the VAD on a real- French take); its last word ends on whisper's segment end, which can stop- short of the speech. With `speech`, the first real word of each stretch- starts on the stretch's onset and its last real word ends on its offset,- punctuation between collapsing onto the edge. `MAX_ANCHOR_SEC` is that- step's knob: a word further off than 1 s is more likely the neighbour of- one whisper dropped, and is left alone.+ French take). Its last word ends on whisper's segment end, which can stop+ short of the speech, or on the next word's start, which runs on through the+ pause. With `speech`, the first real word of each stretch starts on the+ stretch's onset and its last real word ends on its offset; the punctuation+ closing the phrase collapses to a point there. Words own the speech, never+ the pause: deleting a phrase's last word keeps the pause after it. A word+ reported in the 0.1 s tail the helper keeps past an offset belongs to that+ stretch. `MAX_ANCHOR_SEC` bounds the stretching only: a word further off+ than 1 s is more likely the neighbour of one whisper dropped, and is left+ alone.> Why this matters beyond caption timing: the transcript editor turns a word> selection into a trim of exactly `[firstWord.startSec, lastWord.endSec]`,