Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(stt): keep a sentence's last word in the stretch it closes

EtienneLescot899372c2726d
diff --git a/electron/stt/ctcAlign.test.ts b/electron/stt/ctcAlign.test.ts
index 91cc15d4c..579f97a37 100644
--- a/electron/stt/ctcAlign.test.ts
+++ b/electron/stt/ctcAlign.test.ts
@@ -185,6 +185,22 @@ describe("alignWordsOnEmissions", () => {
expect(r(out[0].startSec)).toBe(r(30 * STRIDE + PAUSE_START_OFFSET_SEC));
});
+ it("aligns a sentence's last word in the stretch it closes, even anchored in the pause", () => {
+ // "b." is said on frame 25 (0.5 s), but DTW anchored it at 0.75 s, past the
+ // first stretch's tail (issue #948). It is aligned there, not in "c"'s audio.
+ const tokens = [...gap(15), "a", "|", ...gap(8), "b", ...gap(49), "c", ...gap(24)];
+ const out = alignWordsOnEmissions(
+ [word("a", 0.25, 0.35, 0.3), word("b.", 0.35, 0.75, 0.75), word("c", 0.75, 1.6, 1.55)],
+ [
+ { startSec: 0.2, endSec: 0.6 },
+ { startSec: 1.4, endSec: 1.8 },
+ ],
+ emissions(tokens),
+ );
+ expect(r(out[1].startSec)).toBe(r(25 * STRIDE + PAUSE_START_OFFSET_SEC));
+ expect(r(out[2].startSec)).toBe(r(75 * STRIDE + PAUSE_START_OFFSET_SEC));
+ });
+
it("keeps the helper's times where it cannot align", () => {
const words = [word("ab", 0.3, 0.5, 0.35), word("c", 1.5, 1.7, 1.6)];
// The second word is outside every stretch; the first one's letters do not fit.
diff --git a/electron/stt/ctcAlign.ts b/electron/stt/ctcAlign.ts
index 335b6c196..4e142671e 100644
--- a/electron/stt/ctcAlign.ts
+++ b/electron/stt/ctcAlign.ts
@@ -18,6 +18,7 @@
// Self-contained on purpose: tools/stt-eval/word-timing imports this file
// straight into Node, which resolves no extensionless import.
+import { ownWords } from "./snapWordBoundaries.ts";
import type { SttVadSegment } from "./transcriptionContract";
/** What `/emissions` answers, decoded. */
@@ -58,7 +59,7 @@ export const PAUSE_END_OFFSET_SEC = 0.15;
/** Audio kept on each side of a stretch of speech, so its first and last letters have context. */
export const REGION_MARGIN_SEC = 0.3;
-/** As in snapWordBoundaries.ts: a stretch owns the words anchored before its end plus this tail. */
+/** As in snapWordBoundaries.ts: the audio the helper keeps past each speech offset. */
const TAIL_SEC = 0.1;
const WILDCARD = -1;
@@ -175,8 +176,8 @@ export function ctcViterbi(
}
/**
- * Re-time `words` on `emissions`. Each speech stretch owns the words anchored
- * in it (the rule of snapWordBoundaries.ts); they are aligned on the frames
+ * Re-time `words` on `emissions`. Each speech stretch owns the words
+ * `ownWords` (snapWordBoundaries.ts) gives it; they are aligned on the frames
* between the neighbouring stretches, with the vocabulary's word delimiter
* between words when it has one. Words outside every stretch, stretches the
* model cannot fit, and a vocabulary with no word in it keep the helper's
@@ -192,12 +193,11 @@ export function alignWordsOnEmissions<W extends AlignableWord>(
const delimiter = index.get("|");
const V = emissions.vocab.length;
const { strideSec } = emissions;
- let k = 0;
+ const ranges = ownWords(words, speech);
for (let i = 0; i < speech.length; i++) {
const { startSec: onset, endSec: offset } = speech[i];
- const until = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);
- const owned: number[] = [];
- for (; k < out.length && words[k].anchorSec < until; k++) owned.push(k);
+ const [from, to] = ranges[i];
+ const owned = Array.from({ length: to - from }, (_, n) => from + n);
// A region's frames stop up to one receptive field (25 ms) short of the audio
// it was cut from, so a stretch that runs to the end of the upload ends
// past them; two frames of slack keep it. `hi` below stays on the frames.
diff --git a/electron/stt/snapWordBoundaries.test.ts b/electron/stt/snapWordBoundaries.test.ts
index dcd260f92..62d48cf12 100644
--- a/electron/stt/snapWordBoundaries.test.ts
+++ b/electron/stt/snapWordBoundaries.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
-import { anchorWordsOnSpeech, type HelperWord } from "./snapWordBoundaries";
+import { anchorWordsOnSpeech, type HelperWord, ownWords } from "./snapWordBoundaries";
import type { SttWordSegment } from "./transcriptionContract";
/** A helper word; its anchor defaults to its start, as for a request's first word. */
@@ -95,4 +95,83 @@ describe("anchorWordsOnSpeech", () => {
["d", 3.5, 4],
]);
});
+
+ // Issue #948, measured on a Zira TTS clip: "…as if nothing happened. The
+ // transcript…" with speech stopping at 30.59 and resuming at 31.33. DTW
+ // anchored "happened." at 30.69, past the first stretch's tail.
+ const pause = [
+ { startSec: 28.13, endSec: 30.59 },
+ { startSec: 31.33, endSec: 35.23 },
+ ];
+
+ it("keeps a sentence's last word in the stretch it closes when DTW anchors it in the pause", () => {
+ const words = [
+ word("and", 27.97, 28.35, 28.35),
+ word("nothing", 29.65, 30.03, 30.03),
+ word("happened.", 30.03, 30.69, 30.69),
+ word("The", 30.69, 31.47, 31.47),
+ word("transcript", 31.47, 32.13, 32.13),
+ ];
+ expect(times(anchorWordsOnSpeech(words, pause))).toEqual([
+ ["and", 28.13, 28.35],
+ ["nothing", 29.65, 30.03],
+ ["happened.", 30.03, 30.59],
+ ["The", 31.33, 31.47],
+ ["transcript", 31.47, 32.13],
+ ]);
+ expect(ownWords(words, pause)).toEqual([
+ [0, 3],
+ [3, 5],
+ ]);
+ });
+
+ it("keeps the punctuation whisper split off a sentence's last word with it", () => {
+ const words = [
+ word("et", 28.2, 28.4, 28.4),
+ word("vraiment", 28.4, 30.69, 30.69),
+ word("?", 30.69, 30.7, 30.7),
+ word("Oui", 30.7, 31.6, 31.5),
+ ];
+ expect(ownWords(words, pause)).toEqual([
+ [0, 3],
+ [3, 4],
+ ]);
+ });
+
+ it("lets a one-word sentence open a stretch when it is anchored in its speech", () => {
+ const words = [
+ word("Agreed?", 28.13, 30.5, 30.4),
+ word("Yes.", 30.5, 31.6, 31.5),
+ word("Good", 31.6, 32, 32),
+ ];
+ expect(ownWords(words, pause)).toEqual([
+ [0, 1],
+ [1, 3],
+ ]);
+ });
+
+ it("gives a sentence's last word to the next stretch when the one before has no words", () => {
+ const words = [word("Done.", 31, 31.6, 31.2), word("Next", 31.6, 32, 32)];
+ expect(ownWords(words, pause)).toEqual([
+ [0, 0],
+ [0, 2],
+ ]);
+ });
+
+ it("puts the words in order, none inverted, whatever stretch owns them", () => {
+ // The aligner-on shape of the report, as the aligner left it when the next
+ // stretch owned "happened": a 20 ms word after the pause and "The" earlier.
+ const aligned = [
+ word("nothing", 29.68, 30.59, 30.03),
+ word("happened", 31.33, 31.35, 30.69),
+ word("The", 30.69, 31.48, 31.47),
+ word("transcript", 31.48, 32.12, 32.13),
+ ];
+ const out = anchorWordsOnSpeech(aligned, pause);
+ for (const [j, w] of out.entries()) {
+ expect(w.endSec).toBeGreaterThanOrEqual(w.startSec);
+ if (j > 0) expect(w.startSec).toBeGreaterThanOrEqual(out[j - 1].startSec);
+ }
+ expect(out.map((w) => w.word)).toEqual(["nothing", "happened", "The", "transcript"]);
+ });
});
diff --git a/electron/stt/snapWordBoundaries.ts b/electron/stt/snapWordBoundaries.ts
index 247eac6ad..7998a0f8e 100644
--- a/electron/stt/snapWordBoundaries.ts
+++ b/electron/stt/snapWordBoundaries.ts
@@ -49,6 +49,61 @@ const TAIL_SEC = 0.1;
/** French puts a space before `!`, `?`, `:` and `;`, so whisper emits them as words. */
const isPunctuation = (word: string) => /^[\p{P}\p{S}]+$/u.test(word);
+/** `.`, `!`, `?`, `…` or a full-width form, maybe inside a closing quote or bracket. */
+const endsSentence = (text: string) => /[.!?…。!?.。][\p{Pe}\p{Pf}"']*$/u.test(text);
+
+/** End of the audio the helper kept for stretch `i`: its speech, plus a tail short of the next. */
+const keptUntil = (speech: SttVadSegment[], i: number) =>
+ Math.min(speech[i].endSec + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);
+
+/**
+ * The words each speech stretch owns, as `[from, to)` ranges of `words`: those
+ * anchored before the end of the audio the helper kept for it. Both the aligner
+ * (ctcAlign.ts) and the edge anchoring below read this, so a word is aligned in
+ * the stretch whose edges it is put on.
+ *
+ * One exception. DTW can anchor a sentence's last word in the pause after it,
+ * past the kept tail: the next stretch then put it on its onset, after the
+ * words that follow it, and the aligner looked for it in the wrong audio (issue
+ * #948). So a word that ends a sentence and is anchored in the pause, before the
+ * next stretch's speech, closes the stretch before it, with its punctuation,
+ * when that stretch has words. One anchored inside the next speech still opens
+ * it: on LibriSpeech, nine one-word sentences ("Heaven!", "What?") open a
+ * stretch that way, and giving them to the stretch before cost phrase deletes.
+ */
+export function ownWords(
+ words: ReadonlyArray<{ word: string; anchorSec: number }>,
+ speech: SttVadSegment[],
+): Array<[number, number]> {
+ const out: Array<[number, number]> = [];
+ let k = 0;
+ for (let i = 0; i < speech.length; i++) {
+ const from = k;
+ const until = keptUntil(speech, i);
+ while (k < words.length && words[k].anchorSec < until) k++;
+ // The word that would open the next stretch, and the punctuation after it.
+ let first = k;
+ while (first < words.length && isPunctuation(words[first].word)) first++;
+ let end = first + 1;
+ while (end < words.length && isPunctuation(words[end].word)) end++;
+ if (
+ i + 1 < speech.length &&
+ first < words.length &&
+ words[first].anchorSec < speech[i + 1].startSec &&
+ words.slice(from, k).some((w) => !isPunctuation(w.word)) &&
+ endsSentence(
+ words
+ .slice(first, end)
+ .map((w) => w.word)
+ .join(""),
+ )
+ )
+ k = end;
+ out.push([from, k]);
+ }
+ return out;
+}
+
/**
* Put the first and last word of every speech stretch on the stretch's edges.
* The onset already carries the VAD's 30 ms pad, so a cut there lands just
@@ -65,16 +120,25 @@ export function anchorWordsOnSpeech(
...w,
endSec: Math.max(w.endSec, w.startSec + MIN_WORD_SEC),
}));
- if (!speech) return out;
- let k = 0;
+ if (speech) anchorEdges(out, words, speech);
+ // Whatever the stretches did, words come out in order and never inverted. A
+ // start past the next word's comes back to it rather than pushing the next
+ // one: the onset clamp is what overshoots (a word whisper invented, put on
+ // the onset after the real one the aligner found earlier).
+ for (let j = out.length - 1; j >= 0; j--) {
+ if (j + 1 < out.length) out[j].startSec = Math.min(out[j].startSec, out[j + 1].startSec);
+ out[j].endSec = Math.max(out[j].endSec, out[j].startSec);
+ }
+ return out;
+}
+
+function anchorEdges(out: SttWordSegment[], words: HelperWord[], speech: SttVadSegment[]) {
+ const owned = ownWords(words, speech);
for (let i = 0; i < speech.length; i++) {
const { startSec: onset, endSec: offset } = speech[i];
- // A stretch owns the words anchored before the end of the audio the helper
- // kept for it: its speech, plus a tail that stops short of the next one.
- const tail = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);
let first = -1;
let last = -1;
- for (; k < out.length && words[k].anchorSec < tail; k++) {
+ for (let k = owned[i][0]; k < owned[i][1]; k++) {
if (isPunctuation(out[k].word)) continue;
if (first < 0) first = k;
last = k;
@@ -95,5 +159,4 @@ export function anchorWordsOnSpeech(
out[j].endSec = out[last].endSec;
}
}
- return out;
}
diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md
index d3b1dfcaf..0f16828cb 100644
--- a/technical-documentation/architecture/transcription-and-captions.md
+++ b/technical-documentation/architecture/transcription-and-captions.md
@@ -309,7 +309,17 @@ it verbatim in the response.
- its last real word ends on the offset, and the punctuation closing the
phrase collapses to a point there;
- a stretch owns the words whose `anchor` falls in it or in the 0.1 s tail
- the helper keeps past its offset.
+ the helper keeps past its offset (`ownWords`, which step 5 reads too);
+ - except a word that ends a sentence (`.`, `!`, `?`, `…`, full-width forms
+ included) anchored in the pause, before the next stretch's speech: it
+ closes the stretch before it, with its punctuation. DTW put "happened."
+ of "…as if nothing happened. The transcript…" there on an rc.6 take, so
+ the next stretch put it on its onset, after "The", as a 20 ms word, and
+ the aligner looked for it in "The"'s audio (issue #948). A one-word
+ sentence anchored inside the next speech ("Heaven!") still opens it;
+ moving those too cost phrase deletes on LibriSpeech;
+ - last, starts are made monotonic and no word ends before it starts: a
+ start past the next word's comes back to it.
Words own the speech, never the pause: deleting a phrase's last word keeps
the pause after it. `MAX_ANCHOR_SEC` bounds the stretching only: a word more