Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(stt): keep a sentence's last word in the stretch it closes
EtienneLescot899372c2726d
diff --git a/electron/stt/ctcAlign.test.ts b/electron/stt/ctcAlign.test.tsindex 91cc15d4c..579f97a37 100644--- a/electron/stt/ctcAlign.test.ts+++ b/electron/stt/ctcAlign.test.ts@@ -185,6 +185,22 @@ describe("alignWordsOnEmissions", () => {expect(r(out[0].startSec)).toBe(r(30 * STRIDE + PAUSE_START_OFFSET_SEC));});+ it("aligns a sentence's last word in the stretch it closes, even anchored in the pause", () => {+ // "b." is said on frame 25 (0.5 s), but DTW anchored it at 0.75 s, past the+ // first stretch's tail (issue #948). It is aligned there, not in "c"'s audio.+ const tokens = [...gap(15), "a", "|", ...gap(8), "b", ...gap(49), "c", ...gap(24)];+ const out = alignWordsOnEmissions(+ [word("a", 0.25, 0.35, 0.3), word("b.", 0.35, 0.75, 0.75), word("c", 0.75, 1.6, 1.55)],+ [+ { startSec: 0.2, endSec: 0.6 },+ { startSec: 1.4, endSec: 1.8 },+ ],+ emissions(tokens),+ );+ expect(r(out[1].startSec)).toBe(r(25 * STRIDE + PAUSE_START_OFFSET_SEC));+ expect(r(out[2].startSec)).toBe(r(75 * STRIDE + PAUSE_START_OFFSET_SEC));+ });+it("keeps the helper's times where it cannot align", () => {const words = [word("ab", 0.3, 0.5, 0.35), word("c", 1.5, 1.7, 1.6)];// The second word is outside every stretch; the first one's letters do not fit.diff --git a/electron/stt/ctcAlign.ts b/electron/stt/ctcAlign.tsindex 335b6c196..4e142671e 100644--- a/electron/stt/ctcAlign.ts+++ b/electron/stt/ctcAlign.ts@@ -18,6 +18,7 @@// Self-contained on purpose: tools/stt-eval/word-timing imports this file// straight into Node, which resolves no extensionless import.+import { ownWords } from "./snapWordBoundaries.ts";import type { SttVadSegment } from "./transcriptionContract";/** What `/emissions` answers, decoded. */@@ -58,7 +59,7 @@ export const PAUSE_END_OFFSET_SEC = 0.15;/** Audio kept on each side of a stretch of speech, so its first and last letters have context. */export const REGION_MARGIN_SEC = 0.3;-/** As in snapWordBoundaries.ts: a stretch owns the words anchored before its end plus this tail. */+/** As in snapWordBoundaries.ts: the audio the helper keeps past each speech offset. */const TAIL_SEC = 0.1;const WILDCARD = -1;@@ -175,8 +176,8 @@ export function ctcViterbi(}/**- * Re-time `words` on `emissions`. Each speech stretch owns the words anchored- * in it (the rule of snapWordBoundaries.ts); they are aligned on the frames+ * Re-time `words` on `emissions`. Each speech stretch owns the words+ * `ownWords` (snapWordBoundaries.ts) gives it; they are aligned on the frames* between the neighbouring stretches, with the vocabulary's word delimiter* between words when it has one. Words outside every stretch, stretches the* model cannot fit, and a vocabulary with no word in it keep the helper's@@ -192,12 +193,11 @@ export function alignWordsOnEmissions<W extends AlignableWord>(const delimiter = index.get("|");const V = emissions.vocab.length;const { strideSec } = emissions;- let k = 0;+ const ranges = ownWords(words, speech);for (let i = 0; i < speech.length; i++) {const { startSec: onset, endSec: offset } = speech[i];- const until = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);- const owned: number[] = [];- for (; k < out.length && words[k].anchorSec < until; k++) owned.push(k);+ const [from, to] = ranges[i];+ const owned = Array.from({ length: to - from }, (_, n) => from + n);// A region's frames stop up to one receptive field (25 ms) short of the audio// it was cut from, so a stretch that runs to the end of the upload ends// past them; two frames of slack keep it. `hi` below stays on the frames.diff --git a/electron/stt/snapWordBoundaries.test.ts b/electron/stt/snapWordBoundaries.test.tsindex dcd260f92..62d48cf12 100644--- a/electron/stt/snapWordBoundaries.test.ts+++ b/electron/stt/snapWordBoundaries.test.ts@@ -1,5 +1,5 @@import { describe, expect, it } from "vitest";-import { anchorWordsOnSpeech, type HelperWord } from "./snapWordBoundaries";+import { anchorWordsOnSpeech, type HelperWord, ownWords } from "./snapWordBoundaries";import type { SttWordSegment } from "./transcriptionContract";/** A helper word; its anchor defaults to its start, as for a request's first word. */@@ -95,4 +95,83 @@ describe("anchorWordsOnSpeech", () => {["d", 3.5, 4],]);});++ // Issue #948, measured on a Zira TTS clip: "…as if nothing happened. The+ // transcript…" with speech stopping at 30.59 and resuming at 31.33. DTW+ // anchored "happened." at 30.69, past the first stretch's tail.+ const pause = [+ { startSec: 28.13, endSec: 30.59 },+ { startSec: 31.33, endSec: 35.23 },+ ];++ it("keeps a sentence's last word in the stretch it closes when DTW anchors it in the pause", () => {+ const words = [+ word("and", 27.97, 28.35, 28.35),+ word("nothing", 29.65, 30.03, 30.03),+ word("happened.", 30.03, 30.69, 30.69),+ word("The", 30.69, 31.47, 31.47),+ word("transcript", 31.47, 32.13, 32.13),+ ];+ expect(times(anchorWordsOnSpeech(words, pause))).toEqual([+ ["and", 28.13, 28.35],+ ["nothing", 29.65, 30.03],+ ["happened.", 30.03, 30.59],+ ["The", 31.33, 31.47],+ ["transcript", 31.47, 32.13],+ ]);+ expect(ownWords(words, pause)).toEqual([+ [0, 3],+ [3, 5],+ ]);+ });++ it("keeps the punctuation whisper split off a sentence's last word with it", () => {+ const words = [+ word("et", 28.2, 28.4, 28.4),+ word("vraiment", 28.4, 30.69, 30.69),+ word("?", 30.69, 30.7, 30.7),+ word("Oui", 30.7, 31.6, 31.5),+ ];+ expect(ownWords(words, pause)).toEqual([+ [0, 3],+ [3, 4],+ ]);+ });++ it("lets a one-word sentence open a stretch when it is anchored in its speech", () => {+ const words = [+ word("Agreed?", 28.13, 30.5, 30.4),+ word("Yes.", 30.5, 31.6, 31.5),+ word("Good", 31.6, 32, 32),+ ];+ expect(ownWords(words, pause)).toEqual([+ [0, 1],+ [1, 3],+ ]);+ });++ it("gives a sentence's last word to the next stretch when the one before has no words", () => {+ const words = [word("Done.", 31, 31.6, 31.2), word("Next", 31.6, 32, 32)];+ expect(ownWords(words, pause)).toEqual([+ [0, 0],+ [0, 2],+ ]);+ });++ it("puts the words in order, none inverted, whatever stretch owns them", () => {+ // The aligner-on shape of the report, as the aligner left it when the next+ // stretch owned "happened": a 20 ms word after the pause and "The" earlier.+ const aligned = [+ word("nothing", 29.68, 30.59, 30.03),+ word("happened", 31.33, 31.35, 30.69),+ word("The", 30.69, 31.48, 31.47),+ word("transcript", 31.48, 32.12, 32.13),+ ];+ const out = anchorWordsOnSpeech(aligned, pause);+ for (const [j, w] of out.entries()) {+ expect(w.endSec).toBeGreaterThanOrEqual(w.startSec);+ if (j > 0) expect(w.startSec).toBeGreaterThanOrEqual(out[j - 1].startSec);+ }+ expect(out.map((w) => w.word)).toEqual(["nothing", "happened", "The", "transcript"]);+ });});diff --git a/electron/stt/snapWordBoundaries.ts b/electron/stt/snapWordBoundaries.tsindex 247eac6ad..7998a0f8e 100644--- a/electron/stt/snapWordBoundaries.ts+++ b/electron/stt/snapWordBoundaries.ts@@ -49,6 +49,61 @@ const TAIL_SEC = 0.1;/** French puts a space before `!`, `?`, `:` and `;`, so whisper emits them as words. */const isPunctuation = (word: string) => /^[\p{P}\p{S}]+$/u.test(word);+/** `.`, `!`, `?`, `…` or a full-width form, maybe inside a closing quote or bracket. */+const endsSentence = (text: string) => /[.!?…。!?.。][\p{Pe}\p{Pf}"']*$/u.test(text);++/** End of the audio the helper kept for stretch `i`: its speech, plus a tail short of the next. */+const keptUntil = (speech: SttVadSegment[], i: number) =>+ Math.min(speech[i].endSec + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);++/**+ * The words each speech stretch owns, as `[from, to)` ranges of `words`: those+ * anchored before the end of the audio the helper kept for it. Both the aligner+ * (ctcAlign.ts) and the edge anchoring below read this, so a word is aligned in+ * the stretch whose edges it is put on.+ *+ * One exception. DTW can anchor a sentence's last word in the pause after it,+ * past the kept tail: the next stretch then put it on its onset, after the+ * words that follow it, and the aligner looked for it in the wrong audio (issue+ * #948). So a word that ends a sentence and is anchored in the pause, before the+ * next stretch's speech, closes the stretch before it, with its punctuation,+ * when that stretch has words. One anchored inside the next speech still opens+ * it: on LibriSpeech, nine one-word sentences ("Heaven!", "What?") open a+ * stretch that way, and giving them to the stretch before cost phrase deletes.+ */+export function ownWords(+ words: ReadonlyArray<{ word: string; anchorSec: number }>,+ speech: SttVadSegment[],+): Array<[number, number]> {+ const out: Array<[number, number]> = [];+ let k = 0;+ for (let i = 0; i < speech.length; i++) {+ const from = k;+ const until = keptUntil(speech, i);+ while (k < words.length && words[k].anchorSec < until) k++;+ // The word that would open the next stretch, and the punctuation after it.+ let first = k;+ while (first < words.length && isPunctuation(words[first].word)) first++;+ let end = first + 1;+ while (end < words.length && isPunctuation(words[end].word)) end++;+ if (+ i + 1 < speech.length &&+ first < words.length &&+ words[first].anchorSec < speech[i + 1].startSec &&+ words.slice(from, k).some((w) => !isPunctuation(w.word)) &&+ endsSentence(+ words+ .slice(first, end)+ .map((w) => w.word)+ .join(""),+ )+ )+ k = end;+ out.push([from, k]);+ }+ return out;+}+/*** Put the first and last word of every speech stretch on the stretch's edges.* The onset already carries the VAD's 30 ms pad, so a cut there lands just@@ -65,16 +120,25 @@ export function anchorWordsOnSpeech(...w,endSec: Math.max(w.endSec, w.startSec + MIN_WORD_SEC),}));- if (!speech) return out;- let k = 0;+ if (speech) anchorEdges(out, words, speech);+ // Whatever the stretches did, words come out in order and never inverted. A+ // start past the next word's comes back to it rather than pushing the next+ // one: the onset clamp is what overshoots (a word whisper invented, put on+ // the onset after the real one the aligner found earlier).+ for (let j = out.length - 1; j >= 0; j--) {+ if (j + 1 < out.length) out[j].startSec = Math.min(out[j].startSec, out[j + 1].startSec);+ out[j].endSec = Math.max(out[j].endSec, out[j].startSec);+ }+ return out;+}++function anchorEdges(out: SttWordSegment[], words: HelperWord[], speech: SttVadSegment[]) {+ const owned = ownWords(words, speech);for (let i = 0; i < speech.length; i++) {const { startSec: onset, endSec: offset } = speech[i];- // A stretch owns the words anchored before the end of the audio the helper- // kept for it: its speech, plus a tail that stops short of the next one.- const tail = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);let first = -1;let last = -1;- for (; k < out.length && words[k].anchorSec < tail; k++) {+ for (let k = owned[i][0]; k < owned[i][1]; k++) {if (isPunctuation(out[k].word)) continue;if (first < 0) first = k;last = k;@@ -95,5 +159,4 @@ export function anchorWordsOnSpeech(out[j].endSec = out[last].endSec;}}- return out;}diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.mdindex d3b1dfcaf..0f16828cb 100644--- a/technical-documentation/architecture/transcription-and-captions.md+++ b/technical-documentation/architecture/transcription-and-captions.md@@ -309,7 +309,17 @@ it verbatim in the response.- its last real word ends on the offset, and the punctuation closing thephrase collapses to a point there;- a stretch owns the words whose `anchor` falls in it or in the 0.1 s tail- the helper keeps past its offset.+ the helper keeps past its offset (`ownWords`, which step 5 reads too);+ - except a word that ends a sentence (`.`, `!`, `?`, `…`, full-width forms+ included) anchored in the pause, before the next stretch's speech: it+ closes the stretch before it, with its punctuation. DTW put "happened."+ of "…as if nothing happened. The transcript…" there on an rc.6 take, so+ the next stretch put it on its onset, after "The", as a 20 ms word, and+ the aligner looked for it in "The"'s audio (issue #948). A one-word+ sentence anchored inside the next speech ("Heaven!") still opens it;+ moving those too cost phrase deletes on LibriSpeech;+ - last, starts are made monotonic and no word ends before it starts: a+ start past the next word's comes back to it.Words own the speech, never the pause: deleting a phrase's last word keepsthe pause after it. `MAX_ANCHOR_SEC` bounds the stretching only: a word more