Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(stt): tell which phrase a word belongs to by whisper's own start
EtienneLescot46c304e1547d
diff --git a/electron/stt/snapWordBoundaries.test.ts b/electron/stt/snapWordBoundaries.test.tsindex 8a62d1c25..4af4d908e 100644--- a/electron/stt/snapWordBoundaries.test.ts+++ b/electron/stt/snapWordBoundaries.test.ts@@ -175,4 +175,25 @@ describe("snapWordBoundariesToAudio with speech intervals", () => {["c", 3, 4],]);});++ it("keeps a word with the stretch whisper heard it in, once the snap pulls it before the onset", () => {+ // 80 ms between two stretches: the snap moves "b" from 2.10 to the silence+ // at 2.07, inside what would be the first stretch's tail. It still opens+ // the second stretch and keeps its audio.+ const samples = audioWithSilences(6, [[2, 2.08]]);+ const words = [+ word({ word: "a", startSec: 1, endSec: 2.1 }),+ word({ word: "b", startSec: 2.1, endSec: 2.4 }),+ word({ word: "c", startSec: 2.4, endSec: 3 }),+ ];+ const speech = [+ { startSec: 1, endSec: 2 },+ { startSec: 2.08, endSec: 3 },+ ];+ expect(times(snapWordBoundariesToAudio(words, samples, speech))).toEqual([+ ["a", 1, 2],+ ["b", 2.07, 2.4],+ ["c", 2.4, 3],+ ]);+ });});diff --git a/electron/stt/snapWordBoundaries.ts b/electron/stt/snapWordBoundaries.tsindex 862de6518..70e8eada6 100644--- a/electron/stt/snapWordBoundaries.ts+++ b/electron/stt/snapWordBoundaries.ts@@ -86,36 +86,44 @@ function rmsEnvelope(samples: Float32Array): Float32Array {* before the attack. Words own the speech and no pause: the last word ends on* the offset, and the punctuation closing the phrase collapses to a point* there, wherever DTW dropped it.+ *+ * Which stretch a word belongs to is read off `reportedStarts`, whisper's own+ * times: the RMS snap pulls a phrase's first word back into the silence before+ * it, which can lie in the previous stretch's tail.*/-function anchorOnSpeech(words: SttWordSegment[], speech: SttVadSegment[]): SttWordSegment[] {+function anchorOnSpeech(+ words: SttWordSegment[],+ reportedStarts: number[],+ speech: SttVadSegment[],+): SttWordSegment[] {const out = words.map((w) => ({ ...w }));let k = 0;for (let i = 0; i < speech.length; i++) {const { startSec: onset, endSec: offset } = speech[i];- const previousTail =- i > 0 ? Math.min(speech[i - 1].endSec + TAIL_SEC, onset) : Number.NEGATIVE_INFINITY;+ // A stretch owns the words reported before the end of the audio the helper+ // kept for it: its speech, plus a tail that stops short of the next one.const tail = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);- // The phrase's first word: the first real word reported past the previous stretch's tail.- while (k < out.length && (out[k].startSec < previousTail || isPunctuation(out[k].word))) k++;- if (k === out.length) break;- const late = out[k].startSec - onset;- if (late > 0 && late <= MAX_ANCHOR_SEC && out[k].startSec < offset) {- out[k].startSec = onset;- for (let j = k - 1; j >= 0 && out[j].endSec > onset; j--) {+ let first = -1;+ let last = -1;+ for (; k < out.length && reportedStarts[k] < tail; k++) {+ if (isPunctuation(out[k].word)) continue;+ if (first < 0) first = k;+ last = k;+ }+ if (first < 0) continue;+ const late = out[first].startSec - onset;+ if (late > 0 && late <= MAX_ANCHOR_SEC) {+ out[first].startSec = onset;+ for (let j = first - 1; j >= 0 && out[j].endSec > onset; j--) {out[j].endSec = onset;out[j].startSec = Math.min(out[j].startSec, onset);}}- // Its last word: the last real word reported before its tail ends.- let m = -1;- for (let j = k; j < out.length && out[j].startSec < tail; j++) {- if (!isPunctuation(out[j].word)) m = j;- }- if (m < 0 || offset - out[m].endSec > MAX_ANCHOR_SEC) continue;- out[m].endSec = Math.max(offset, out[m].startSec + MIN_WORD_SEC);- for (let j = m + 1; j < out.length && isPunctuation(out[j].word); j++) {- out[j].startSec = out[m].endSec;- out[j].endSec = out[m].endSec;+ if (offset - out[last].endSec > MAX_ANCHOR_SEC) continue;+ out[last].endSec = Math.max(offset, out[last].startSec + MIN_WORD_SEC);+ for (let j = last + 1; j < out.length && isPunctuation(out[j].word); j++) {+ out[j].startSec = out[last].endSec;+ out[j].endSec = out[last].endSec;}}return out;@@ -167,5 +175,11 @@ export function snapWordBoundariesToAudio(endSec: Math.max(snap(w.endSec), startSec + MIN_WORD_SEC),};});- return speech ? anchorOnSpeech(snapped, speech) : snapped;+ return speech+ ? anchorOnSpeech(+ snapped,+ words.map((w) => w.startSec),+ speech,+ )+ : snapped;}