Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(stt): start each word at the end of the token before it
EtienneLescotc9c5642da329
diff --git a/.gitignore b/.gitignoreindex 24ad425eb..8b2682b6e 100644--- a/.gitignore+++ b/.gitignore@@ -95,6 +95,9 @@ result-*/tools/stt-eval/whispercpp-dtw-poc/fixtures/*.wav/tools/stt-eval/whispercpp-dtw-poc/results/+# Word-timing harness (tools/stt-eval/word-timing): TTS corpus and results.+/tools/stt-eval/word-timing/data/+opencode.jsonopencode.jsondiff --git a/electron/native/whisper-stt/src/main.cpp b/electron/native/whisper-stt/src/main.cppindex 61303f3c3..037230a57 100644--- a/electron/native/whisper-stt/src/main.cpp+++ b/electron/native/whisper-stt/src/main.cpp@@ -223,9 +223,12 @@ std::string detect_active_backend() {}struct Word {- double start = 0.0;- double end = 0.0;- double prob = 0.0;+ double start = 0.0;+ double end = 0.0;+ // t_dtw of the word's first token: the end of that token, so always inside+ // the word. Only used to tell which stretch of speech a word belongs to.+ double anchor = 0.0;+ double prob = 0.0;std::string text;};@@ -517,6 +520,16 @@ int main(int argc, char** argv) {};std::vector<Segment> segments;+ // whisper.cpp writes a token's t_dtw when the DTW path enters the decoder+ // row that PREDICTS the next token (whisper_exp_compute_token_level_+ // timestamps_dtw, v1.9.1): it marks the END of the token, not its start.+ // So a word runs from the t_dtw of the text token before it to the t_dtw+ // of its own last token. Taking its first token's t_dtw as the start put+ // every word one token late (+175 ms median, tools/stt-eval/word-timing).+ // The previous token carries across segments; the request's very first+ // word has none and starts on its first token's time (the Node side+ // anchors it on the speech onset anyway).+ double prev_tok_end = -1.0;const int n_segments = input.empty() ? 0 : whisper_full_n_segments(ctx);for (int si = 0; si < n_segments; ++si) {Segment seg;@@ -524,11 +537,11 @@ int main(int argc, char** argv) {seg.end = to_original_sec(whisper_full_get_segment_t1(ctx, si), kept);if (const char* t = whisper_full_get_segment_text(ctx, si)) seg.text = t;- struct W { double t_dtw_first; double p_sum; int p_n; std::string text; };+ struct W { double start; double end; double anchor; double p_sum; int p_n; std::string text; };std::vector<W> word_buf;std::string cur_text;bool in_word = false;- double w_first_t_dtw = 0;+ double w_start = 0, w_end = 0, w_anchor = 0;double w_p_sum = 0; int w_p_n = 0;const int n_tokens = whisper_full_n_tokens(ctx, si);@@ -555,30 +568,32 @@ int main(int argc, char** argv) {const double td_dtw = to_original_sec(td.t_dtw >= 0 ? td.t_dtw : 0, kept);if (starts_word && in_word) {- word_buf.push_back({ w_first_t_dtw, w_p_sum, w_p_n, cur_text });+ word_buf.push_back({ w_start, w_end, w_anchor, w_p_sum, w_p_n, cur_text });w_p_sum = 0; w_p_n = 0; cur_text.clear();}if (starts_word) {in_word = true;- w_first_t_dtw = td_dtw;+ w_start = prev_tok_end >= 0 ? prev_tok_end : td_dtw;+ w_anchor = td_dtw;cur_text = (!raw.empty() && raw[0] == ' ') ? raw.substr(1) : raw;} else {cur_text += raw;}+ w_end = td_dtw;+ prev_tok_end = td_dtw;w_p_sum += td.p;w_p_n += 1;}if (in_word) {- word_buf.push_back({ w_first_t_dtw, w_p_sum, w_p_n, cur_text });+ word_buf.push_back({ w_start, w_end, w_anchor, w_p_sum, w_p_n, cur_text });}- for (size_t wi = 0; wi < word_buf.size(); ++wi) {+ for (const W& b : word_buf) {Word w;- w.start = word_buf[wi].t_dtw_first;- w.end = (wi + 1 < word_buf.size())- ? word_buf[wi + 1].t_dtw_first- : seg.end;- w.prob = word_buf[wi].p_sum / std::max(1, word_buf[wi].p_n);- w.text = word_buf[wi].text;+ w.start = b.start;+ w.end = b.end;+ w.anchor = b.anchor;+ w.prob = b.p_sum / std::max(1, b.p_n);+ w.text = b.text;seg.words.push_back(w);}segments.push_back(std::move(seg));@@ -642,6 +657,7 @@ int main(int argc, char** argv) {{"word", w.text},{"start", w.start},{"end", w.end},+ {"anchor", w.anchor},{"probability", w.prob},});}diff --git a/electron/stt/snapWordBoundaries.test.ts b/electron/stt/snapWordBoundaries.test.tsindex 4af4d908e..dcd260f92 100644--- a/electron/stt/snapWordBoundaries.test.ts+++ b/electron/stt/snapWordBoundaries.test.ts@@ -1,199 +1,98 @@import { describe, expect, it } from "vitest";-import { snapWordBoundariesToAudio } from "./snapWordBoundaries";+import { anchorWordsOnSpeech, type HelperWord } from "./snapWordBoundaries";import type { SttWordSegment } from "./transcriptionContract";-const SAMPLE_RATE = 16_000;--/** Mono 16 kHz buffer that is loud everywhere except the given silent spans. */-function audioWithSilences(durationSec: number, silences: Array<[number, number]>): Float32Array {- const samples = new Float32Array(Math.round(durationSec * SAMPLE_RATE));- for (let i = 0; i < samples.length; i++) {- const t = i / SAMPLE_RATE;- const silent = silences.some(([from, to]) => t >= from && t < to);- // Alternating ±0.5 gives a flat, non-zero RMS without needing a real tone.- samples[i] = silent ? 0 : i % 2 === 0 ? 0.5 : -0.5;- }- return samples;-}--const word = (w: Partial<SttWordSegment> = {}): SttWordSegment => ({- word: "w",- startSec: 0,- endSec: 0.1,- ...w,+/** A helper word; its anchor defaults to its start, as for a request's first word. */+const word = (+ text: string,+ startSec: number,+ endSec: number,+ anchorSec = startSec,+): HelperWord => ({+ word: text,+ startSec,+ endSec,+ anchorSec,});--describe("snapWordBoundariesToAudio", () => {- it("pulls a late boundary back into the silence that precedes it", () => {- // Speech stops at 1.0 and resumes at 1.2; whisper reports the next word- // starting at 1.3 — 100 ms after the audio actually resumed.- const samples = audioWithSilences(3, [[1.0, 1.2]]);- const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.8 })], samples);- expect(snapped.startSec).toBeGreaterThanOrEqual(1.0);- expect(snapped.startSec).toBeLessThan(1.2);- });-- it("leaves a boundary alone when nothing quieter precedes it", () => {- // A word ending a phrase: whisper is already right, the frames before the- // boundary are all speech, so the quietest frame in the window is the- // boundary itself and it must not drift.- const samples = audioWithSilences(3, [[1.5, 2.0]]);- const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.0, endSec: 1.5 })], samples);- expect(snapped.endSec).toBeCloseTo(1.5, 2);- });-- it("never moves a boundary more than the lookback window", () => {- const samples = audioWithSilences(3, [[0.0, 1.0]]);- const [snapped] = snapWordBoundariesToAudio([word({ startSec: 2.0, endSec: 2.5 })], samples);- expect(snapped.startSec).toBeGreaterThanOrEqual(2.0 - 0.15);- });-- it("keeps a boundary shared by two words shared", () => {- const samples = audioWithSilences(3, [[1.0, 1.2]]);- const [first, second] = snapWordBoundariesToAudio(- [word({ startSec: 0.5, endSec: 1.3 }), word({ startSec: 1.3, endSec: 1.8 })],- samples,- );- expect(first.endSec).toBeCloseTo(second.startSec, 6);- });-- it("leaves boundaries that fall outside the decoded audio alone", () => {- // Clamping these into range would collapse every boundary onto the end of- // the buffer instead of leaving the unmeasurable ones untouched.- const samples = audioWithSilences(0.1, []);- const words = [word({ startSec: 5.51, endSec: 6.85 })];- expect(snapWordBoundariesToAudio(words, samples)).toEqual(words);- });-- it("keeps degenerate words non-empty and passes words through without audio", () => {- const samples = audioWithSilences(3, [[1.0, 1.2]]);- const [degenerate] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.3 })], samples);- expect(degenerate.endSec).toBeGreaterThan(degenerate.startSec);-- const untouched = [word({ startSec: 1.3, endSec: 1.8 })];- expect(snapWordBoundariesToAudio(untouched, new Float32Array(0))).toEqual(untouched);+const ms = (sec: number) => Math.round(sec * 1000) / 1000;+const times = (words: SttWordSegment[]) => words.map((w) => [w.word, ms(w.startSec), ms(w.endSec)]);++describe("anchorWordsOnSpeech", () => {+ it("returns the helper's times without speech intervals, minus the anchor", () => {+ const out = anchorWordsOnSpeech([word("a", 1, 1.4, 1.2), word("b", 1.4, 1.4, 1.4)]);+ expect(out).toEqual([+ { word: "a", startSec: 1, endSec: 1.4 },+ { word: "b", startSec: 1.4, endSec: 1.42 },+ ]);});-});-describe("snapWordBoundariesToAudio with speech intervals", () => {- // Loud and flat everywhere, so the RMS snap moves nothing and only the- // anchoring on `speech` is under test.- const flat = audioWithSilences(6, []);- const ms = (sec: number) => Math.round(sec * 1000) / 1000;- const times = (words: SttWordSegment[]) =>- words.map((w) => [w.word, ms(w.startSec), ms(w.endSec)]);-- it("puts each phrase's first word on its onset and its closing punctuation on its end", () => {- // The shape of a real French take: whisper put "Salut" 0.58 s and "Bah"- // 0.25 s after the speech started, ran "Salut" on through the pause, and- // dropped the "!" closing the first phrase just after the second began.+ it("puts a phrase's first word on its onset whether DTW put it late or early", () => {+ // "Salut" opens the request, so it has no previous token and starts late,+ // on its own first token. "Bah" starts where the previous token ended, at+ // the end of the first phrase: in the pause, before its own speech.const words = [- word({ word: "Salut", startSec: 2.15, endSec: 3.37 }),- word({ word: "!", startSec: 3.37, endSec: 3.39 }),- word({ word: "Bah", startSec: 3.61, endSec: 4.01 }),- word({ word: "voilà", startSec: 4.01, endSec: 4.35 }),+ word("Salut", 2.15, 2.5),+ word("!", 2.5, 2.6, 2.6),+ word("Bah", 2.6, 3.8, 3.61),+ word("voilà", 3.8, 4.35, 4.01),];const speech = [{ startSec: 1.57, endSec: 2.56 },{ startSec: 3.36, endSec: 4.35 },];- expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([+ expect(times(anchorWordsOnSpeech(words, speech))).toEqual([["Salut", 1.57, 2.56],["!", 2.56, 2.56],- ["Bah", 3.36, 4.01],- ["voilà", 4.01, 4.35],+ ["Bah", 3.36, 3.8],+ ["voilà", 3.8, 4.35],]);});it("ends each phrase's last word where its speech stops, stretched or cut back", () => {- // The end of the same take: "tic," ran on into the pause, whisper closed- // the last segment at 2.79 while "tac" ran to 3.04, and dropped the "!" on- // the word itself.const words = [- word({ word: "tic,", startSec: 1.95, endSec: 2.59 }),- word({ word: "tac", startSec: 2.59, endSec: 2.79 }),- word({ word: "!", startSec: 2.62, endSec: 2.79 }),+ word("tic,", 1.95, 2.59),+ word("tac", 2.59, 2.79, 2.7),+ word("!", 2.79, 2.9, 2.9),];const speech = [{ startSec: 1.95, endSec: 2.37 },{ startSec: 2.59, endSec: 3.04 },];- expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([+ expect(times(anchorWordsOnSpeech(words, speech))).toEqual([["tic,", 1.95, 2.37],["tac", 2.59, 3.04],["!", 3.04, 3.04],]);});- it("leaves a phrase alone when its words are on time, or too far off to be its edges", () => {- const onTime = [word({ startSec: 0.95, endSec: 3 })];- expect(times(snapWordBoundariesToAudio(onTime, flat, [{ startSec: 1, endSec: 3 }]))).toEqual(- times(onTime),- );+ it("leaves an edge alone when the word is too far past it to be that edge", () => {// 1.2 s after the onset and 1.1 s before the offset: more likely a neighbour// of words whisper dropped than the phrase's own edges.- const tooFar = [word({ startSec: 2.2, endSec: 2.5 })];- expect(times(snapWordBoundariesToAudio(tooFar, flat, [{ startSec: 1, endSec: 3.6 }]))).toEqual(+ const tooFar = [word("w", 2.2, 2.5)];+ expect(times(anchorWordsOnSpeech(tooFar, [{ startSec: 1, endSec: 3.6 }]))).toEqual(times(tooFar),);});- it("never mistakes a phrase's second word for its first", () => {- // "y" opens the second phrase but was reported just before its onset;- // "z" must not be dragged back over it.- const words = [- word({ word: "x", startSec: 0.5, endSec: 1.95 }),- word({ word: "y", startSec: 1.95, endSec: 2.4 }),- word({ word: "z", startSec: 2.4, endSec: 3 }),- ];- const speech = [- { startSec: 0.5, endSec: 1 },- { startSec: 2, endSec: 3 },- ];- expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([- ["x", 0.5, 1],- ["y", 1.95, 2.4],- ["z", 2.4, 3],- ]);- });-- it("does not take a word in the previous phrase's tail for the next one's first", () => {- // The helper keeps 0.1 s past each offset. "b" was reported in the first- // phrase's tail, and "c" still has to reach the second onset.+ it("gives a word to the stretch its anchor falls in, the kept tail included", () => {+ // "b" is anchored in the first stretch's 0.1 s tail: it closes that phrase.+ // "c" starts in the first stretch (the previous token's end) but is+ // anchored in the second, so it opens the second.const words = [- word({ word: "a", startSec: 1, endSec: 2.05 }),- word({ word: "b", startSec: 2.05, endSec: 3.4 }),- word({ word: "c", startSec: 3.4, endSec: 4 }),+ word("a", 1, 1.9, 1.5),+ word("b", 1.9, 2.05, 2.05),+ word("c", 2.05, 3.5, 3.2),+ word("d", 3.5, 4, 3.8),];const speech = [{ startSec: 1, endSec: 2 },{ startSec: 3, endSec: 4 },];- expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([- ["a", 1, 2.05],- ["b", 2.05, 2.07],- ["c", 3, 4],- ]);- });-- it("keeps a word with the stretch whisper heard it in, once the snap pulls it before the onset", () => {- // 80 ms between two stretches: the snap moves "b" from 2.10 to the silence- // at 2.07, inside what would be the first stretch's tail. It still opens- // the second stretch and keeps its audio.- const samples = audioWithSilences(6, [[2, 2.08]]);- const words = [- word({ word: "a", startSec: 1, endSec: 2.1 }),- word({ word: "b", startSec: 2.1, endSec: 2.4 }),- word({ word: "c", startSec: 2.4, endSec: 3 }),- ];- const speech = [- { startSec: 1, endSec: 2 },- { startSec: 2.08, endSec: 3 },- ];- expect(times(snapWordBoundariesToAudio(words, samples, speech))).toEqual([- ["a", 1, 2],- ["b", 2.07, 2.4],- ["c", 2.4, 3],+ expect(times(anchorWordsOnSpeech(words, speech))).toEqual([+ ["a", 1, 1.9],+ ["b", 1.9, 2],+ ["c", 3, 3.5],+ ["d", 3.5, 4],]);});});diff --git a/electron/stt/snapWordBoundaries.ts b/electron/stt/snapWordBoundaries.tsindex 70e8eada6..247eac6ad 100644--- a/electron/stt/snapWordBoundaries.ts+++ b/electron/stt/snapWordBoundaries.ts@@ -1,119 +1,88 @@-// Pulls whisper.cpp's DTW word boundaries back onto the audio they describe.+// Puts the edges of every phrase on the speech the helper's VAD found.//-// whisper.cpp reports one time per token and derives word spans from it, so a-// word's `start` is the point where the decoder *emitted* the token, not where-// the speaker started saying it. Measured on real recordings, that lands-// consistently 80–150 ms late, and because consecutive words share a boundary-// (`word[i].end === word[i+1].start`) the whole transcript is dragged right by-// roughly a syllable.+// The helper times a word from the end of the token before it to the end of its+// own last token (whisper.cpp's t_dtw marks where a token ENDS, see+// whisper-stt/src/main.cpp). Inside continuous speech that lands within ~30 ms+// of the audio (measured on a TTS corpus with exact word times,+// tools/stt-eval/word-timing), so inner boundaries are left alone: an RMS snap+// on top of them would drag correct boundaries early.//-// That is invisible in captions but not in the transcript editor: deleting-// words there turns the selection into a trim of exactly-// `[firstWord.startSec, lastWord.endSec]`, so a late boundary leaves the attack-// of the first removed word audible and bites into the following kept word.+// The edges of a phrase are the exception. The token before a phrase's first+// word is the previous phrase's last one, so that word "starts" at the end of+// the previous phrase, in the pause, or DTW puts it late when the helper+// starts a new decode window there. Its last word ends on whatever token DTW+// last aligned, short of the speech or past it. When the helper sent its speech+// intervals, both are put on the edges of the stretch of speech they belong to.//-// The correction is to look at the audio instead of guessing an offset: each-// boundary moves back to the quietest 10 ms frame within the preceding-// `LOOKBACK_SEC`. It is self-limiting — on a decaying tail (a word that ends a-// phrase, where whisper is already right) the quietest frame IS the reported-// one, so the boundary doesn't move at all.-//-// The edges of a phrase are the exception. Its first word, DTW reports 0.1–0.6 s-// late (measured against Silero VAD on a real French recording), far past what-// the lookback can reach. Its last word ends where whisper's segment ends, which-// can stop short of the speech, or where the next word starts, which runs on-// through the whole pause. When the helper sent its speech intervals, both are-// put on the edges of the stretch of speech they belong to.+// That matters because deleting words in the transcript editor trims exactly+// `[firstWord.startSec, lastWord.endSec]`: a late boundary leaves the attack of+// the first removed word audible, an early one bites into the kept word.import type { SttVadSegment, SttWordSegment } from "./transcriptionContract";-/** Matches `writeSamplesAsWav` — the samples handed to whisper are mono 16 kHz. */-const SAMPLE_RATE = 16_000;--/** RMS envelope resolution; finer than any boundary error worth correcting. */-const FRAME_SEC = 0.01;--/**- * How far back a boundary may travel — the calibration knob for this- * correction. Sized to the measured DTW lag (~80–150 ms). Widen it and- * boundaries inside continuous speech start snapping onto the *previous*- * syllable's trough; narrow it and the lag survives.- */-const LOOKBACK_SEC = 0.15;+/** A word as the helper reports it: `anchorSec` is always inside the word. */+export interface HelperWord extends SttWordSegment {+ /** End of the word's first token. Only tells which stretch of speech owns the word. */+ anchorSec: number;+}-/** Keep degenerate words (whisper sometimes reports end <= start) non-empty. */+/** Keep degenerate words non-empty. */const MIN_WORD_SEC = 0.02;/*** How far a phrase's first word may start after its speech, or its last word* end before it, and still be stretched onto that edge — the calibration knob of- * the anchoring step, sized above the measured 0.1–0.6 s. Past it, the word is- * more likely a neighbour of one whisper dropped, and stretching it over that- * audio would be a guess. Cutting a word back to where the speech stops needs- * no such bound: the VAD says nothing is said past it.+ * the anchoring step. Past it, the word is more likely a neighbour of one+ * whisper dropped, and stretching it over that audio would be a guess. Pulling+ * a word forward to the onset, or cutting it back to where the speech stops,+ * needs no such bound: the VAD says nothing is said outside it.*/const MAX_ANCHOR_SEC = 1;/*** Audio the helper keeps past each speech offset, as whisper.cpp does, so a soft- * ending survives. A word DTW reports in there belongs to the stretch it trails.+ * ending survives. A word whose anchor falls in there belongs to the stretch it+ * trails.*/const TAIL_SEC = 0.1;/** French puts a space before `!`, `?`, `:` and `;`, so whisper emits them as words. */const isPunctuation = (word: string) => /^[\p{P}\p{S}]+$/u.test(word);-/** Per-frame RMS of the mono signal — the cheapest usable "is this speech" proxy. */-function rmsEnvelope(samples: Float32Array): Float32Array {- const frameLength = Math.round(SAMPLE_RATE * FRAME_SEC);- const frameCount = Math.floor(samples.length / frameLength);- const envelope = new Float32Array(frameCount);- for (let f = 0; f < frameCount; f++) {- const start = f * frameLength;- let sum = 0;- for (let i = start; i < start + frameLength; i++) {- const v = samples[i] ?? 0;- sum += v * v;- }- envelope[f] = Math.sqrt(sum / frameLength);- }- return envelope;-}-/*** Put the first and last word of every speech stretch on the stretch's edges.* The onset already carries the VAD's 30 ms pad, so a cut there lands just* before the attack. Words own the speech and no pause: the last word ends on* the offset, and the punctuation closing the phrase collapses to a point- * there, wherever DTW dropped it.- *- * Which stretch a word belongs to is read off `reportedStarts`, whisper's own- * times: the RMS snap pulls a phrase's first word back into the silence before- * it, which can lie in the previous stretch's tail.+ * there, wherever DTW dropped it. Without `speech` (the helper ran without its+ * VAD model), the words come back as the helper timed them.*/-function anchorOnSpeech(- words: SttWordSegment[],- reportedStarts: number[],- speech: SttVadSegment[],+export function anchorWordsOnSpeech(+ words: HelperWord[],+ speech?: SttVadSegment[],): SttWordSegment[] {- const out = words.map((w) => ({ ...w }));+ const out: SttWordSegment[] = words.map(({ anchorSec: _, ...w }) => ({+ ...w,+ endSec: Math.max(w.endSec, w.startSec + MIN_WORD_SEC),+ }));+ if (!speech) return out;let k = 0;for (let i = 0; i < speech.length; i++) {const { startSec: onset, endSec: offset } = speech[i];- // A stretch owns the words reported before the end of the audio the helper+ // A stretch owns the words anchored before the end of the audio the helper// kept for it: its speech, plus a tail that stops short of the next one.const tail = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);let first = -1;let last = -1;- for (; k < out.length && reportedStarts[k] < tail; k++) {+ for (; k < out.length && words[k].anchorSec < tail; k++) {if (isPunctuation(out[k].word)) continue;if (first < 0) first = k;last = k;}if (first < 0) continue;- const late = out[first].startSec - onset;- if (late > 0 && late <= MAX_ANCHOR_SEC) {+ if (out[first].startSec - onset <= MAX_ANCHOR_SEC) {out[first].startSec = onset;+ out[first].endSec = Math.max(out[first].endSec, onset + MIN_WORD_SEC);for (let j = first - 1; j >= 0 && out[j].endSec > onset; j--) {out[j].endSec = onset;out[j].startSec = Math.min(out[j].startSec, onset);@@ -128,58 +97,3 @@ function anchorOnSpeech(}return out;}--/**- * Move every word boundary back to the quietest frame in the `LOOKBACK_SEC`- * preceding it. Boundaries shared by two words snap identically (same input- * time), so the transcript stays gap-free where whisper made it gap-free.- * With `speech` (the helper's VAD intervals), phrase edges are then anchored- * on the speech. Returns the words unchanged when there is no audio to- * measure against.- */-export function snapWordBoundariesToAudio(- words: SttWordSegment[],- samples: Float32Array,- speech?: SttVadSegment[],-): SttWordSegment[] {- const envelope = rmsEnvelope(samples);- if (envelope.length === 0) return words;-- const lookbackFrames = Math.round(LOOKBACK_SEC / FRAME_SEC);- const snap = (timeSec: number): number => {- const hi = Math.round(timeSec / FRAME_SEC);- // A boundary outside the audio we can measure (whisper occasionally reports- // times past the end of the samples) has nothing to snap to — clamping it- // into range would drag it to the end of the buffer instead.- if (hi <= 0 || hi >= envelope.length) return timeSec;- const lo = Math.max(0, hi - lookbackFrames);- let bestFrame = hi;- let bestRms = envelope[hi];- // Strict `<` while walking backwards keeps the frame CLOSEST to the- // reported boundary when a whole stretch is equally quiet, so digital- // silence can't drag a boundary the full window.- for (let f = hi - 1; f >= lo; f--) {- if (envelope[f] < bestRms) {- bestRms = envelope[f];- bestFrame = f;- }- }- return bestFrame * FRAME_SEC;- };-- const snapped = words.map((w) => {- const startSec = snap(w.startSec);- return {- ...w,- startSec,- endSec: Math.max(snap(w.endSec), startSec + MIN_WORD_SEC),- };- });- return speech- ? anchorOnSpeech(- snapped,- words.map((w) => w.startSec),- speech,- )- : snapped;-}diff --git a/electron/stt/whisperServer.test.ts b/electron/stt/whisperServer.test.tsindex 7f52d5a6e..62dd7442d 100644--- a/electron/stt/whisperServer.test.ts+++ b/electron/stt/whisperServer.test.ts@@ -349,17 +349,22 @@ describe("WhisperServerManager", () => {}});- it("anchors a phrase's first word on the speech onset the helper reports", async () => {+ it("anchors a phrase's first word on the onset of the speech its anchor falls in", async () => {+ // "Salut" starts where the previous token ended, in the first stretch's+ // tail, but its anchor puts it in the second stretch.const fakeJson = {segments: [{text: " Salut",start: 1.57,end: 2.56,- words: [{ word: " Salut", start: 2.15, end: 2.56 }],+ words: [{ word: " Salut", start: 1.05, end: 2.56, anchor: 1.8 }],},],- speech: [{ start: 1.57, end: 2.56 }],+ speech: [+ { start: 0.2, end: 1 },+ { start: 1.57, end: 2.56 },+ ],backend: "whispercpp-cpu",};vi.stubGlobal(@@ -371,7 +376,7 @@ describe("WhisperServerManager", () => {(mgr as unknown as { process: unknown; port: number }).process = {};(mgr as unknown as { process: unknown; port: number }).port = 9999;const result = await mgr.transcribe({ samples: new Float32Array(16_000 * 3) });- expect(result.wordSegments[0].startSec).toBeCloseTo(1.57, 6);+ expect(result.wordSegments).toEqual([{ word: "Salut", startSec: 1.57, endSec: 2.56 }]);} finally {vi.unstubAllGlobals();}diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.tsindex 9a27c6dee..a03c05449 100644--- a/electron/stt/whisperServer.ts+++ b/electron/stt/whisperServer.ts@@ -7,7 +7,7 @@ import path from "node:path";import type { Readable } from "node:stream";import { resolveBinaryPath } from "./gpuDetector";-import { snapWordBoundariesToAudio } from "./snapWordBoundaries";+import { anchorWordsOnSpeech } from "./snapWordBoundaries";import type {SttBackend,SttPhraseSegment,@@ -81,6 +81,8 @@ interface WhisperJsonWord {word?: string;start?: number;end?: number;+ /** End of the word's first token: always inside the word (see snapWordBoundaries.ts). */+ anchor?: number;probability?: number;}@@ -576,23 +578,28 @@ export class WhisperServerManager {startSec: this.toSec(s.start, 0),endSec: this.toSec(s.end, 0),}));- // whisper.cpp's DTW boundaries run ~80–150 ms behind the audio, and a- // phrase's first word up to 0.6 s, which the transcript editor turns into- // imprecise trims (see snapWordBoundaries.ts). Re-anchor them on the same- // samples whisper was given, and on the helper's speech intervals.- const wordSegments: SttWordSegment[] = snapWordBoundariesToAudio(+ // The helper's word times are right inside a phrase but not on its edges,+ // which the transcript editor turns into imprecise trims: put those on the+ // helper's speech intervals (see snapWordBoundaries.ts).+ const wordSegments: SttWordSegment[] = anchorWordsOnSpeech(raw.flatMap((seg) =>(seg.words ?? []).map((w) => {const word = (w.word ?? "").trim();const startSec = this.toSec(w.start, 0);const endSec = this.toSec(w.end, startSec + 0.05);+ const anchorSec = this.toSec(w.anchor, startSec);const confidence = typeof w.probability === "number" ? w.probability : undefined;- return { word, startSec, endSec: Math.max(startSec + 0.02, endSec), confidence };+ return {+ word,+ startSec,+ endSec: Math.max(startSec + 0.02, endSec),+ anchorSec,+ confidence,+ };}),).filter((w) => w.word.length > 0),- opts.samples,speech,);const detectedLanguage = json.detected_language ?? json.language ?? "auto";diff --git a/scripts/test-whisper-stt.mjs b/scripts/test-whisper-stt.mjsindex 7cef820bb..2a0e11223 100644--- a/scripts/test-whisper-stt.mjs+++ b/scripts/test-whisper-stt.mjs@@ -345,23 +345,16 @@ async function main() {);}- // NOT a failure. whisper.cpp gives the last word of a segment the segment's- // own t1 as its end, while the word's DTW start runs 80–150 ms late — so a- // token emitted near the boundary (typically standalone punctuation) can land- // after it and invert. The contract's [startSec, endSec) guarantee is enforced- // one layer up, deliberately: whisperServer.ts clamps to- // `Math.max(startSec + 0.02, endSec)` and snapWordBoundaries.ts keeps- // "degenerate words (whisper sometimes reports end <= start) non-empty". This- // harness speaks to the raw helper, below that clamp, so it reports the count- // as information instead of asserting on it.+ // A word runs from the end of the token before it to the end of its own last+ // token, and the helper rejects a non-monotonic DTW path, so it cannot end+ // before it starts. It can be empty (end == start): whisperServer.ts widens+ // those to 20 ms before they reach the document.const inverted = words.filter((w) => w.end < w.start);- if (inverted.length > 0) {- console.log(- ` info ${inverted.length}/${words.length} raw word(s) have end < start ` +- `(${inverted.map((w) => JSON.stringify(w.word)).join(", ")}) — expected at segment ` +- "boundaries; whisperServer.ts clamps these before they reach the document.",- );- }+ check(+ inverted.length === 0,+ "no word ends before it starts",+ inverted.map((w) => JSON.stringify(w.word)).join(", "),+ );if (refText) {const rate = wer(refText, text);diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.mdindex 0e42ed1b4..65c058b2b 100644--- a/technical-documentation/architecture/transcription-and-captions.md+++ b/technical-documentation/architecture/transcription-and-captions.md@@ -215,41 +215,64 @@ it verbatim in the response.3. **Word grouping** — BPE tokens join into a single word whenever thedetokenized text begins with a space, or at the first token of thesegment.-4. **Word range** — `word.start = t_dtw of the word's first token`, and- `word.end = t_dtw of the next word's first token` (or the segment's `t1`- for the segment's last word). The result is a monotonic, gap-free timeline- of word ranges that downstream code can use without rebasing.-5. **Re-anchor on the audio** — `t_dtw` marks where the decoder *emitted* a- token, not where the speaker started saying it, so every boundary lands- 80–150 ms late (measured across real recordings; the mean correction on- the reference clip is 83 ms). Because consecutive words share a boundary,- the whole transcript is dragged right by roughly a syllable.+4. **Word range** — a token's `t_dtw` marks where it **ends**, not where it+ starts. In `whisper_exp_compute_token_level_timestamps_dtw` (whisper.cpp+ 1.9.1) the alignment rows start at `<|notimestamps|>`, whose output is text+ token 0, and `t_dtw[k]` is written when the DTW path enters row *k+1*: the+ row that predicts token *k+1*. So:+ - `word.start` = `t_dtw` of the text token **before** the word's first+ token. The previous token carries across segments; the request's very+ first word has none and falls back to its own first token.+ - `word.end` = `t_dtw` of the word's **last** token.+ - `word.anchor` = `t_dtw` of the word's first token. It always lies inside+ the word, and only decides which stretch of speech owns it (step 5).++ The result is a monotonic, gap-free timeline of word ranges on the upload's+ clock. Taking the first token's `t_dtw` as the start, as the helper did+ before, put every word one token late: +175 ms median on the TTS corpus+ below, and a median 200 ms on a real French take. The repo's DTW POC saw+ the same thing as whisper.cpp's `t_dtw` matching faster-whisper's word+ **end** (`tools/stt-eval/whispercpp-dtw-poc/REPORT.md`).+5. **Anchor phrase edges on the speech** —[`electron/stt/snapWordBoundaries.ts`](../../electron/stt/snapWordBoundaries.ts)- pulls each boundary back to the quietest 10 ms frame in the preceding- 150 ms of the same samples whisper was given. It is self-limiting: on a- decaying tail — a word ending a phrase, where whisper is already right —- the quietest frame *is* the reported one and nothing moves. Boundaries- past the end of the decoded audio are left untouched.-- The edges of a phrase are past that reach. Its first word, DTW reports- 0.1–0.6 s after the speech starts (measured against the VAD on a real- French take). Its last word ends on whisper's segment end, which can stop- short of the speech, or on the next word's start, which runs on through the- pause. With `speech`, the first real word of each stretch starts on the- stretch's onset and its last real word ends on its offset; the punctuation- closing the phrase collapses to a point there. Words own the speech, never- the pause: deleting a phrase's last word keeps the pause after it. A word- reported in the 0.1 s tail the helper keeps past an offset belongs to that- stretch. `MAX_ANCHOR_SEC` bounds the stretching only: a word further off- than 1 s is more likely the neighbour of one whisper dropped, and is left- alone.+ leaves boundaries inside a phrase alone: they are within ~30 ms of the audio+ already, and an energy snap on top of them drags correct boundaries early.+ The edges of a phrase are different. The token before a phrase's first word+ is the previous phrase's last one, so that word starts in the pause, as+ early as the previous phrase's end; a word that opens a decode window starts+ late on its own first token. Its last word ends on the last token DTW+ aligned, short of the speech or past it into the pause. With `speech`:+ - the first real word of each stretch starts on the stretch's onset, pulled+ back when it is late and clamped forward when it is early;+ - its last real word ends on the offset, and the punctuation closing the+ phrase collapses to a point there;+ - a stretch owns the words whose `anchor` falls in it or in the 0.1 s tail+ the helper keeps past its offset.++ Words own the speech, never the pause: deleting a phrase's last word keeps+ the pause after it. `MAX_ANCHOR_SEC` bounds the stretching only: a word more+ than 1 s past an edge is more likely the neighbour of one whisper dropped,+ and is left alone.> Why this matters beyond caption timing: the transcript editor turns a word> selection into a trim of exactly `[firstWord.startSec, lastWord.endSec]`,> so a late boundary leaves the attack of the first removed word audible and-> bites into the following kept word. `LOOKBACK_SEC` in that module is the-> calibration knob — widen it and boundaries inside continuous speech start-> snapping onto the previous syllable's trough.+> an early one bites into the kept word next to it.++**Measuring it** — [`tools/stt-eval/word-timing/`](../../tools/stt-eval/word-timing/README.md)+scores the helper and this post-pass against a Windows-TTS corpus with exact+word times (27 min of French and English narration, clean and noisy):+boundary error by position, and the audible residue and clipping of deleting+one word or one phrase. Issue #948 has the method, the baseline and the plan.++| Pipeline (clean + noisy) | Inner start: median / P90 / within 50 ms | Phrase-initial start within 50 ms | One-word delete: clean cuts | Phrase delete: clean |+|---|---|---|---|---|+| First-token start + 150 ms RMS snap + VAD edges (before #948) | 105 / 275 ms / 32% | 83% | 5% | 84% |+| Previous-token start + two-way VAD edges | 31 / 125 ms / 64% | 89% | 19% | 88% |++A +15 ms calibration offset on every boundary gained 4 points of inner+boundaries within 50 ms but dropped noisy phrase deletes to 82%, so it is not+applied.The recognition call returns **both** the phrase segments and the per-wordsegments in one pass@@ -302,7 +325,7 @@ and the digest together.| Module | Role ||---|---|| `electron/stt/whisperServer.ts` | Server lifecycle; `POST /inference` client; verbose_json parser. |-| `electron/stt/snapWordBoundaries.ts` | Re-anchors DTW word boundaries on the audio's RMS envelope, and phrase edges on the VAD's speech (see step 5 above). |+| `electron/stt/snapWordBoundaries.ts` | Anchors phrase edges on the VAD's speech (see step 5 above). || `electron/stt/wav.ts` | WAV write + temp-file cleanup helpers. || `electron/stt/gpuDetector.ts` | Per-platform binary resolver (no GPU probing). || `electron/stt/modelManager.ts` | Single GGML file download, SHA-256 verify, atomic write. |@@ -769,7 +792,7 @@ it deletes datain `electron/native/whisper-stt/src/main.cpp` are exercised only atruntime.- **Word timing inside a phrase.** Phrase edges sit on the VAD (step 5), but- a word in the middle of continuous speech still starts on its DTW time- pulled back by at most 150 ms of RMS lookback. A dedicated forced aligner- is the upgrade path (issue #626); measure against the VAD-anchored edges- before adopting one.+ a word in the middle of continuous speech is only as good as whisper-small's+ DTW: about 30 ms median and 125 ms P90, so one single-word delete in five is+ clean. Character-level DTW, then a CTC forced aligner, are the upgrade path+ (issue #948 Phases 2 and 3); `tools/stt-eval/word-timing` measures them.diff --git a/tools/stt-eval/word-timing/README.md b/tools/stt-eval/word-timing/README.mdnew file mode 100644index 000000000..b12ec4649--- /dev/null+++ b/tools/stt-eval/word-timing/README.md@@ -0,0 +1,69 @@+# Word-timing harness++Scores transcript word times against exact ground truth: how far each word+boundary is from the audio, and what deleting one word or one phrase in the+transcript editor would leave audible or clip. Method, baseline and plan:+issue #948. How the app times words:+[transcription-and-captions.md § Word-level alignment](../../../technical-documentation/architecture/transcription-and-captions.md).++Node 22+, no dependencies. The corpus is synthesized with Windows speech+synthesis, so generating it needs Windows; scoring runs anywhere.++## Data++Everything generated goes to `data/` next to these scripts (gitignored), or+wherever `OSC_WORD_TIMING_DATA` points:++- `clips/`: `<id>.wav` (clean), `<id>.noisy.wav`, `<id>.ref.json` (reference words)+- `corpus-manifest.json`+- `results/raw/<tag>/`: the helper's raw responses; `results/<name>.{txt,json}`: scores++## Generate the corpus (Windows, once, about 5 min)++```sh+node make-corpus.mjs # 48 TTS clips + a noisy copy of each + reference times+node validate-ref.mjs # checks the reference against the audio's energy+```++- Voices: OneCore Hortense, Paul and Julie (French), SAPI Zira (English).+ Scripts are in `corpus-texts.mjs`, synthesis in `tts.ps1`.+- ffmpeg: `electron/native/bin/win32-x64/ffmpeg.exe`, or set `FFMPEG`.++## Run and score++```sh+node run-helper.mjs <tag> <path/to/whisper-stt-server.exe> [--cpu]+node evaluate.mjs <tag> [--snap <post-pass.ts>] [--out <name>]+node summarize.mjs "Before=<baseline>" "After=<name>"+```++- `run-helper.mjs` starts its own helper on a port in 20500-20599 with the+ app's models (`%APPDATA%/openscreen/stt-models/whisper-ggml`: `ggml-small-q8_0.bin`+ and `ggml-silero-v6.2.0.bin`), sends every clip like the app does, and stops+ it. Vulkan takes about 2 min for the 55 min of audio, CPU about 17.+- `evaluate.mjs` parses the responses as `whisperServer.ts` does and runs the+ post-pass: the repo's `electron/stt/snapWordBoundaries.ts` by default, or the+ file given with `--snap`. It reports three stages: `raw`, `post` (no speech+ intervals) and `post+vad` (what the app ships).+- A baseline is the same two commands on an older helper and post-pass:+ build the helper at that revision, and pass+ `git show <rev>:electron/stt/snapWordBoundaries.ts` saved to a file as `--snap`.++## Real speech++```sh+node real-check.mjs <whisper-stt-server.exe> take.wav # 16 kHz mono s16+```++No ground truth there: it prints how far each word's first-token time lies after+its start, and where each phrase's first word lands against the VAD onset, raw+and after the post-pass.++## Reading the numbers++- Positions come from the reference: *phrase-initial* has a pause of at least+ 100 ms before it, *phrase-final* after it, *inner* is everything else.+- A delete is *clean* when at most 20 ms of the deleted word stays audible and+ at most 20 ms of its neighbours is cut.+- Synthetic speech flatters every method. Use the harness to rank approaches,+ and check a winner on real speech.diff --git a/tools/stt-eval/word-timing/corpus-texts.mjs b/tools/stt-eval/word-timing/corpus-texts.mjsnew file mode 100644index 000000000..294517fe3--- /dev/null+++ b/tools/stt-eval/word-timing/corpus-texts.mjs@@ -0,0 +1,40 @@+// Screen-recording narration scripts. `{N}` is an explicit pause of N ms+// (SSML <break>). Hesitations, restarts, numbers, UI and product names, and+// sentences run together with no pause are deliberate.+export const TEXTS = {+ fr: [+ `Bonjour à tous, {400} aujourd'hui je vais vous montrer comment enregistrer votre écran avec OpenScreen. {700} Alors, {250} la première chose à faire, c'est d'ouvrir la barre d'enregistrement en bas de l'écran. {500} Vous voyez ici le bouton Écran, {200} je clique dessus et je choisis mon moniteur principal, celui en 1920 par 1080. {900} Ensuite, euh, {300} j'active le micro, {150} et la caméra si vous voulez apparaître dans la vidéo. {600} Et voilà, il ne reste plus qu'à appuyer sur le bouton rouge pour lancer l'enregistrement. {1200} Un compte à rebours de trois secondes s'affiche, puis ça démarre.`,+ `Dans cette vidéo on va parler du montage. {500} Quand l'enregistrement se termine, l'éditeur s'ouvre tout seul avec la vidéo, la caméra en incrustation et les zooms déjà placés là où j'ai cliqué. {800} Si un zoom ne vous plaît pas, {200} vous le sélectionnez sur la timeline et vous appuyez sur Supprimer. {400} Pour en ajouter un, c'est la touche Z. {1000} Bon, {300} on peut aussi régler la durée du zoom, par exemple 1,5 seconde au lieu de 2 secondes, et changer le niveau d'agrandissement à 150 pour cent.`,+ `Passons maintenant à la transcription. {600} OpenScreen transcrit l'audio directement sur votre ordinateur, sans rien envoyer sur internet. {500} Le modèle fait environ 264 mégaoctets et il se télécharge une seule fois. {900} Une fois le texte affiché, hum, {400} vous pouvez supprimer des mots comme dans un traitement de texte. {300} Je sélectionne cette phrase, là, {200} je la supprime, et la vidéo est coupée exactement au même endroit. {1500} C'est très pratique pour enlever les hésitations et les silences trop longs.`,+ `Alors attention, {300} il y a un petit piège avec les sous-titres. {600} Par défaut ils utilisent la police Inter en taille 48, et si votre vidéo est en format vertical, en 1080 par 1920, le texte risque de déborder. {800} Dans ce cas, euh, {250} ouvrez l'onglet Sous-titres dans le panneau de droite et réduisez la taille à 36. {500} Vous pouvez aussi activer le mode karaoké qui surligne chaque mot au moment où il est prononcé {700} et c'est justement là que la précision des horodatages devient importante.`,+ `Je vais exporter la vidéo. {400} Je clique sur Exporter en haut à droite, {300} je choisis le format MP4, la résolution 1080p et 60 images par seconde. {900} Le rendu se fait avec le compositeur natif, qui utilise la carte graphique quand elle est disponible, sinon le processeur. {600} Pour une vidéo de cinq minutes, ça prend environ quarante secondes sur ma machine. {1100} Si vous préférez un GIF pour une documentation, {200} choisissez GIF dans la liste, avec une largeur de 800 pixels, c'est un bon compromis.`,+ `Petite astuce pour finir. {500} Si vous enregistrez souvent la même fenêtre, par exemple Visual Studio Code ou votre navigateur, {300} OpenScreen garde votre dernier choix en mémoire. {700} Donc la prochaine fois, un seul clic suffit. {400} Et si vous vous trompez pendant l'enregistrement, pas de panique, {250} le bouton Recommencer efface la prise en cours et relance le compte à rebours. {1300} Voilà, c'est tout pour aujourd'hui, merci de m'avoir suivi et à bientôt pour la version 2.1.`,+ `Euh, {300} donc là on est sur le tableau de bord de l'application, {400} vous voyez trois colonnes: {200} les projets récents, les modèles et les paramètres. {800} Je vais créer un nouveau projet que je vais appeler Démo client, {300} voilà, {500} et je vais importer une vidéo existante, un fichier MKV de douze minutes environ. {1000} Pendant l'import, la barre de progression indique 35 pour cent, puis 70, {300} puis c'est terminé. {600} On peut maintenant ajouter la musique de fond, avec un volume à moins 18 décibels pour qu'elle ne couvre pas la voix.`,+ `Question fréquente: {400} est-ce que ça marche sur Linux? {600} Oui, {200} OpenScreen fonctionne sous Windows, macOS et Linux, {300} et sur Linux c'est le portail du bureau qui vous demande quelle fenêtre partager au moment de lancer l'enregistrement. {900} Autre question, {300} est-ce que c'est payant? {500} Non, c'est gratuit et open source, et ça le restera. {1200} Dernière chose, si vous avez un bug, hum, {300} utilisez le menu Enregistrer le diagnostic dans l'icône de la barre des tâches et joignez le fichier à votre ticket GitHub.`,+ ],+ en: [+ `Hi everyone, {400} today I'm going to show you how to record your screen with OpenScreen. {700} So, {250} the first thing to do is open the recording bar at the bottom of the screen. {500} You can see the Screen button right here, {200} I click it and pick my main monitor, the 1920 by 1080 one. {900} Then, uh, {300} I turn on the microphone, {150} and the camera if you want to appear in the video. {600} And that's it, all that's left is to hit the red button to start recording. {1200} A three second countdown shows up, and then it starts.`,+ `In this video we're going to talk about editing. {500} When the recording stops, the editor opens by itself with the video, the webcam overlay and the zooms already placed where I clicked. {800} If you don't like a zoom, {200} select it on the timeline and press Delete. {400} To add one, press the Z key. {1000} Okay, {300} you can also change how long the zoom lasts, say 1.5 seconds instead of 2 seconds, and set the magnification to 150 percent.`,+ `Now let's move on to transcription. {600} OpenScreen transcribes the audio right on your computer, without sending anything to the internet. {500} The model is about 264 megabytes and it only downloads once. {900} Once the text shows up, um, {400} you can delete words just like in a word processor. {300} I select this sentence here, {200} I delete it, and the video is cut at exactly the same spot. {1500} It's really handy for removing hesitations and long silences.`,+ `Now watch out, {300} there's a small catch with captions. {600} By default they use the Inter font at size 48, and if your video is vertical, 1080 by 1920, the text can overflow. {800} In that case, uh, {250} open the Captions tab in the right panel and bring the size down to 36. {500} You can also turn on karaoke mode which highlights each word as it's spoken {700} and that's exactly where timestamp precision starts to matter.`,+ `I'm going to export the video. {400} I click Export at the top right, {300} I choose MP4, 1080p resolution and 60 frames per second. {900} Rendering uses the native compositor, which runs on the graphics card when there is one, and on the processor otherwise. {600} For a five minute video, that takes about forty seconds on my machine. {1100} If you'd rather have a GIF for your docs, {200} pick GIF in the list, with a width of 800 pixels, that's a good trade-off.`,+ `One last tip. {500} If you often record the same window, like Visual Studio Code or your browser, {300} OpenScreen remembers your last choice. {700} So next time, a single click is enough. {400} And if you make a mistake while recording, don't panic, {250} the Restart button throws away the current take and restarts the countdown. {1300} That's all for today, thanks for watching and see you soon for version 2.1.`,+ `Um, {300} so here we're on the app dashboard, {400} you can see three columns: {200} recent projects, templates and settings. {800} I'm going to create a new project and call it Client demo, {300} there we go, {500} and I'll import an existing video, an MKV file of about twelve minutes. {1000} While it imports, the progress bar says 35 percent, then 70, {300} then it's done. {600} Now we can add background music, with the volume at minus 18 decibels so it doesn't drown out the voice.`,+ `Frequently asked question: {400} does it work on Linux? {600} Yes, {200} OpenScreen runs on Windows, macOS and Linux, {300} and on Linux it's the desktop portal that asks you which window to share when you start recording. {900} Another question, {300} does it cost anything? {500} No, it's free and open source, and it will stay that way. {1200} One last thing, if you hit a bug, hmm, {300} use the Save Diagnostics menu in the tray icon and attach the file to your GitHub issue.`,+ ],+};++// Three renders per script (every voice/rate below). OneCore has three French+// voices; English only has SAPI Zira installed on this machine.+export const RENDERS = {+ fr: [+ { engine: "onecore", voice: "Microsoft Hortense", rate: 1.0 },+ { engine: "onecore", voice: "Microsoft Paul", rate: 1.25 },+ { engine: "onecore", voice: "Microsoft Julie", rate: 0.9 },+ ],+ en: [+ { engine: "sapi", voice: "Microsoft Zira Desktop", rate: 1.0 },+ { engine: "sapi", voice: "Microsoft Zira Desktop", rate: 1.3 },+ { engine: "sapi", voice: "Microsoft Zira Desktop", rate: 0.85 },+ ],+};diff --git a/tools/stt-eval/word-timing/evaluate.mjs b/tools/stt-eval/word-timing/evaluate.mjsnew file mode 100644index 000000000..b923c5d6a--- /dev/null+++ b/tools/stt-eval/word-timing/evaluate.mjs@@ -0,0 +1,301 @@+// Scores helper output against the TTS reference, for each pipeline stage.+// Usage: node evaluate.mjs <tag> [--snap <post-pass.ts>] [--out <name>] [--quiet]+// reads <data>/results/raw/<tag>/*.json, writes <data>/results/<out>.json and+// <out>.txt, and prints the table.+// Stages:+// raw helper words, parsed as whisperServer.ts transcribeImpl does+// post the post-pass without speech intervals+// post+vad the post-pass with the helper's `speech`, as the app runs it+// `--snap` swaps the post-pass (default: the repo's snapWordBoundaries.ts), so a+// candidate is scored exactly like the shipped code. It takes the current+// `anchorWordsOnSpeech(words, speech)` or the pre-#948+// `snapWordBoundariesToAudio(words, samples, speech)`, so the old pipeline can+// be scored too (`git show <rev>:electron/stt/snapWordBoundaries.ts`).+import { readdirSync, readFileSync, writeFileSync } from "node:fs";+import path from "node:path";+import { pathToFileURL } from "node:url";+import { align, CLIPS, envelopeDb, norm, REPO, RESULTS, readWav, stats } from "./lib.mjs";++const [tag, ...rest] = process.argv.slice(2);+const flag = (name, dflt) => {+ const i = rest.indexOf(name);+ return i >= 0 ? rest[i + 1] : dflt;+};+const snapPath = path.resolve(+ flag("--snap", path.join(REPO, "electron/stt/snapWordBoundaries.ts")),+);+const outName = flag("--out", tag);+const quiet = rest.includes("--quiet");+const mod = await import(pathToFileURL(snapPath).href);+const postPass = mod.anchorWordsOnSpeech+ ? (words, _samples, speech) => mod.anchorWordsOnSpeech(words, speech)+ : mod.snapWordBoundariesToAudio;++const PAUSE = 0.1; // reference gap that makes a word phrase-initial / phrase-final+const AUDIBLE_DB = -50; // clean-audio frame energy that counts as audible speech+const CLEAN_MS = 20; // a cut is clean when both audible residue and clipping stay under this+const FRAME = 0.005;+const MS = 1000;++/** Mirrors whisperServer.ts transcribeImpl. */+function parse(json) {+ const toSec = (v, d) => {+ const n = typeof v === "string" ? Number(v) : v;+ return Number.isFinite(n) ? n : d;+ };+ const words = (json.segments ?? [])+ .flatMap((seg) =>+ (seg.words ?? []).map((w) => {+ const word = (w.word ?? "").trim();+ const startSec = toSec(w.start, 0);+ const endSec = toSec(w.end, startSec + 0.05);+ return {+ word,+ startSec,+ endSec: Math.max(startSec + 0.02, endSec),+ anchorSec: toSec(w.anchor, startSec),+ };+ }),+ )+ .filter((w) => w.word.length > 0);+ const speech = json.speech?.map((s) => ({+ startSec: toSec(s.start, 0),+ endSec: toSec(s.end, 0),+ }));+ return { words, speech };+}++const overlap = (a0, a1, b0, b1) => Math.max(0, Math.min(a1, b1) - Math.max(a0, b0));+/** Audible seconds of [a0, a1]: 5 ms frames of the CLEAN audio above AUDIBLE_DB. */+const audible = (env, a0, a1) => {+ if (a1 <= a0) return 0;+ let n = 0;+ for (+ let f = Math.max(0, Math.floor(a0 / FRAME));+ f < Math.min(env.length, Math.ceil(a1 / FRAME));+ f+++ ) {+ if (env[f] > AUDIBLE_DB) n += overlap(a0, a1, f * FRAME, (f + 1) * FRAME);+ }+ return n;+};+/** Seconds of [r0, r1] outside the cut [c0, c1], and the audible part of it. */+const outside = (env, r0, r1, c0, c1) => {+ const segs = [+ [r0, Math.min(r1, c0)],+ [Math.max(r0, c1), r1],+ ].filter(([a, b]) => b > a);+ return {+ sec: segs.reduce((s, [a, b]) => s + b - a, 0),+ aud: segs.reduce((s, [a, b]) => s + audible(env, a, b), 0),+ };+};+const inside = (env, r0, r1, c0, c1) => {+ const a = Math.max(r0, c0),+ b = Math.min(r1, c1);+ return b > a ? { sec: b - a, aud: audible(env, a, b) } : { sec: 0, aud: 0 };+};++const STAGES = ["raw", "post", "post+vad"];+const acc = {};+const push = (k, v) => {+ acc[k] ??= [];+ acc[k].push(v);+};+const counts = {};+const inc = (k, v = 1) => (counts[k] = (counts[k] ?? 0) + v);+const perClip = [];++const rawDir = path.join(RESULTS, "raw", tag);+for (const file of readdirSync(rawDir)+ .filter((f) => f.endsWith(".json"))+ .sort()) {+ const [id, cond] = file.replace(/\.json$/, "").split(".");+ const json = JSON.parse(readFileSync(path.join(rawDir, file), "utf8"));+ const ref = JSON.parse(readFileSync(path.join(CLIPS, `${id}.ref.json`), "utf8"));+ const samples = readWav(path.join(CLIPS, cond === "clean" ? `${id}.wav` : `${id}.noisy.wav`));+ const env = envelopeDb(readWav(path.join(CLIPS, `${id}.wav`)), FRAME);+ const { words: rawWords, speech } = parse(json);+ const t0 = performance.now();+ const post = postPass(rawWords, samples);+ const full = postPass(rawWords, samples, speech);+ const tsMs = performance.now() - t0;+ const variants = { raw: rawWords, post, "post+vad": full };++ const R = ref.words.map((w, i) => ({ ...w, i, n: norm(w.text) })).filter((w) => w.n);+ const H = rawWords.map((w, j) => ({ j, n: norm(w.word) })).filter((w) => w.n);+ const { pairs, sub, ins, del } = align(+ R.map((w) => w.n),+ H.map((w) => w.n),+ );+ const groups = ["all", cond, `${cond}.${ref.lang}`];+ for (const g of groups) {+ inc(`${g}.refWords`, R.length);+ inc(`${g}.matched`, pairs.length);+ inc(`${g}.sub`, sub);+ inc(`${g}.ins`, ins);+ inc(`${g}.del`, del);+ }+ perClip.push({+ id,+ cond,+ lang: ref.lang,+ audioSec: ref.durationSec,+ helperSec: json.timing?.elapsed_s,+ wallMs: json.wallMs,+ tsMs,+ refWords: R.length,+ matched: pairs.length,+ sub,+ ins,+ del,+ });++ // Phrase position from the reference: a pause (>= PAUSE) before / after the word.+ const pos = R.map((w, k) => ({+ initial: k === 0 || w.start - R[k - 1].end >= PAUSE,+ final: k === R.length - 1 || R[k + 1].start - w.end >= PAUSE,+ }));+ const matchOf = new Map(pairs.map(([ri, hj]) => [ri, H[hj].j]));+ for (const stage of STAGES) {+ const V = variants[stage];+ for (const [ri, hj] of pairs) {+ const r = R[ri];+ const h = V[H[hj].j];+ const ds = h.startSec - r.start;+ const de = h.endSec - r.end;+ const sCls = pos[ri].initial ? "initial" : "inner";+ const eCls = pos[ri].final ? "final" : "inner";+ for (const g of groups) {+ push(`${g}|${stage}|start|all`, Math.abs(ds));+ push(`${g}|${stage}|start|${sCls}`, Math.abs(ds));+ push(`${g}|${stage}|end|all`, Math.abs(de));+ push(`${g}|${stage}|end|${eCls}`, Math.abs(de));+ push(`${g}|${stage}|bias_start|${sCls}`, ds);+ push(`${g}|${stage}|bias_end|${eCls}`, de);+ }+ // Trim: this word deleted alone -> cut [h.start, h.end].+ const res = outside(env, r.start, r.end, h.startSec, h.endSec);+ let clipSec = 0,+ clipAud = 0,+ clipShare = 0;+ for (const k of [ri - 1, ri + 1]) {+ const nb = R[k];+ if (!nb) continue;+ const c = inside(env, nb.start, nb.end, h.startSec, h.endSec);+ clipSec += c.sec;+ clipAud += c.aud;+ clipShare = Math.max(clipShare, c.sec / Math.max(1e-3, nb.end - nb.start));+ }+ for (const g of groups) {+ push(`${g}|${stage}|trim|residueMs`, res.sec * MS);+ push(`${g}|${stage}|trim|residueShare`, res.sec / Math.max(1e-3, r.end - r.start));+ push(`${g}|${stage}|trim|residueAudMs`, res.aud * MS);+ push(`${g}|${stage}|trim|clipMs`, clipSec * MS);+ push(`${g}|${stage}|trim|clipShare`, clipShare);+ push(`${g}|${stage}|trim|clipAudMs`, clipAud * MS);+ push(+ `${g}|${stage}|trim|clean`,+ res.aud * MS <= CLEAN_MS && clipAud * MS <= CLEAN_MS ? 1 : 0,+ );+ }+ }+ // Phrase deletion: every reference phrase (words between two pauses) whose+ // first and last words are matched -> cut [first.start, last.end].+ let k = 0;+ while (k < R.length) {+ let e = k;+ while (!pos[e].final) e++;+ if (matchOf.has(k) && matchOf.has(e)) {+ const c0 = V[matchOf.get(k)].startSec,+ c1 = V[matchOf.get(e)].endSec;+ const res = outside(env, R[k].start, R[e].end, c0, c1);+ let clipAud = 0;+ for (const nb of [R[k - 1], R[e + 1]])+ if (nb) clipAud += inside(env, nb.start, nb.end, c0, c1).aud;+ for (const g of groups) {+ push(`${g}|${stage}|phrase|residueAudMs`, res.aud * MS);+ push(`${g}|${stage}|phrase|clipAudMs`, clipAud * MS);+ push(+ `${g}|${stage}|phrase|clean`,+ res.aud * MS <= CLEAN_MS && clipAud * MS <= CLEAN_MS ? 1 : 0,+ );+ }+ }+ k = e + 1;+ }+ }+}++const summary = {};+for (const [key, xs] of Object.entries(acc)) {+ if (key.includes("|trim|") || key.includes("|phrase|") || key.includes("|bias_")) {+ const a = [...xs].sort((p, q) => p - q);+ summary[key] = {+ n: a.length,+ mean: a.reduce((s, x) => s + x, 0) / a.length,+ median: a[Math.floor(a.length / 2)],+ p90: a[Math.floor(a.length * 0.9)],+ };+ } else summary[key] = stats(xs);+}+writeFileSync(+ path.join(RESULTS, `${outName}.json`),+ JSON.stringify({ tag, snapPath, counts, summary, perClip }, null, 1),+);++// ---- table ----+const f = (x) => (x * MS).toFixed(0).padStart(4);+const g0 = (x) => x.toFixed(0).padStart(4);+const pc = (x) => `${(x * 100).toFixed(0)}%`.padStart(4);+const line = (g, st, what) => {+ const s = summary[`${g}|${st}|${what}`];+ return s?.n+ ? `${f(s.mean)} ${f(s.median)} ${f(s.p90)} ${pc(s.w20)} ${pc(s.w50)} ${pc(s.w100)} n=${s.n}`+ : "-";+};+const lines = [`# ${outName} (post-pass ${path.basename(snapPath)})`];+for (const g of ["all", "clean", "noisy", "clean.fr", "clean.en", "noisy.fr", "noisy.en"]) {+ if (!counts[`${g}.refWords`]) continue;+ const wer =+ (counts[`${g}.sub`] + counts[`${g}.ins`] + counts[`${g}.del`]) / counts[`${g}.refWords`];+ lines.push(+ `\n=== ${g}: ${counts[`${g}.refWords`]} ref words, matched ${pc(counts[`${g}.matched`] / counts[`${g}.refWords`])}, WER ${pc(wer)} (sub ${counts[`${g}.sub`]} ins ${counts[`${g}.ins`]} del ${counts[`${g}.del`]})`,+ );+ lines.push(`${"".padEnd(28)}mean med p90 <20 <50 <100 (ms, |error|)`);+ for (const st of STAGES) {+ const t = (k) => summary[`${g}|${st}|${k}`];+ for (const what of [+ "start|initial",+ "start|inner",+ "end|inner",+ "end|final",+ "start|all",+ "end|all",+ ])+ lines.push(`${st.padEnd(10)}${what.padEnd(18)}${line(g, st, what)}`);+ lines.push(+ `${st.padEnd(10)}bias (median signed ms): start initial ${f(t("bias_start|initial").median)} inner ${f(t("bias_start|inner").median)} | end inner ${f(t("bias_end|inner").median)} final ${f(t("bias_end|final").median)}`,+ );+ lines.push(+ `${st.padEnd(10)}1-word delete: residue ${g0(t("trim|residueMs").mean)} ms (audible ${g0(t("trim|residueAudMs").mean)}, p90 ${g0(t("trim|residueAudMs").p90)}), share ${pc(t("trim|residueShare").mean)} | clipping ${g0(t("trim|clipMs").mean)} ms (audible ${g0(t("trim|clipAudMs").mean)}, p90 ${g0(t("trim|clipAudMs").p90)}), share ${pc(t("trim|clipShare").mean)} | clean cuts ${pc(t("trim|clean").mean)}`,+ );+ lines.push(+ `${st.padEnd(10)}phrase delete: audible residue ${g0(t("phrase|residueAudMs").mean)} ms (p90 ${g0(t("phrase|residueAudMs").p90)}), audible clipping ${g0(t("phrase|clipAudMs").mean)} ms (p90 ${g0(t("phrase|clipAudMs").p90)}), clean ${pc(t("phrase|clean").mean)} n=${t("phrase|clean").n}`,+ );+ }+}+const rt = perClip.reduce(+ (a, c) => ({+ audio: a.audio + c.audioSec,+ helper: a.helper + (c.helperSec ?? 0),+ wall: a.wall + c.wallMs / MS,+ ts: a.ts + c.tsMs / MS,+ }),+ { audio: 0, helper: 0, wall: 0, ts: 0 },+);+lines.push(+ `\nruntime: ${perClip.length} clips, ${(rt.audio / 60).toFixed(1)} min audio; helper ${rt.helper.toFixed(1)} s (RTF ${(rt.helper / rt.audio).toFixed(3)}), per clip mean ${(rt.helper / perClip.length).toFixed(2)} s; TS post-pass ${(rt.ts * MS).toFixed(0)} ms total (${((rt.ts * MS) / perClip.length).toFixed(1)} ms/clip, both stages)`,+);+writeFileSync(path.join(RESULTS, `${outName}.txt`), lines.join("\n") + "\n");+if (!quiet) console.log(lines.join("\n"));diff --git a/tools/stt-eval/word-timing/lib.mjs b/tools/stt-eval/word-timing/lib.mjsnew file mode 100644index 000000000..71391e1a3--- /dev/null+++ b/tools/stt-eval/word-timing/lib.mjs@@ -0,0 +1,243 @@+// Shared helpers for the word-timing harness: paths, WAV I/O, energy envelope,+// silence runs, reference ends, word normalization and alignment, stats.+// Node stdlib only.+import { spawn } from "node:child_process";+import { readFileSync } from "node:fs";+import path from "node:path";+import { fileURLToPath } from "node:url";++export const HERE = path.dirname(fileURLToPath(import.meta.url));+export const REPO = path.resolve(HERE, "../../..");+/** Corpus, raw helper output and results. Gitignored; OSC_WORD_TIMING_DATA moves it. */+export const DATA = path.resolve(process.env.OSC_WORD_TIMING_DATA ?? path.join(HERE, "data"));+export const CLIPS = path.join(DATA, "clips");+export const RESULTS = path.join(DATA, "results");+export const FFMPEG =+ process.env.FFMPEG ?? path.join(REPO, "electron/native/bin/win32-x64/ffmpeg.exe");+export const MODELS = path.join(+ process.env.APPDATA ?? "",+ "openscreen",+ "stt-models",+ "whisper-ggml",+);+export const SR = 16000;++/**+ * Starts whisper-stt-server with the app's models on a random port in+ * 20500-20599 and waits until it answers. The process is killed on exit.+ */+export async function startHelper(exe, { cpu = false, env = process.env } = {}) {+ const port = 20500 + Math.floor(Math.random() * 100);+ const args = [+ "--model",+ path.join(MODELS, "ggml-small-q8_0.bin"),+ "--vad-model",+ path.join(MODELS, "ggml-silero-v6.2.0.bin"),+ "--port",+ String(port),+ "--host",+ "127.0.0.1",+ "--threads",+ "16",+ ...(cpu ? ["--cpu"] : []),+ ];+ const child = spawn(exe, args, { stdio: ["ignore", "ignore", "pipe"], env });+ let stderr = "";+ child.stderr.on("data", (d) => {+ stderr = (stderr + d).slice(-4000);+ });+ const stop = () => child.kill();+ process.on("exit", stop);+ process.on("SIGINT", () => process.exit(130));+ const base = `http://127.0.0.1:${port}`;+ for (let t = 0; ; t++) {+ if (child.exitCode !== null) throw new Error(`helper exited ${child.exitCode}: ${stderr}`);+ try {+ if ((await fetch(base)).ok) return { base, stop };+ } catch {+ // not listening yet+ }+ if (t > 600) {+ stop();+ throw new Error(`helper not ready: ${stderr}`);+ }+ await new Promise((r) => setTimeout(r, 200));+ }+}++/** POSTs a WAV to /inference the way the app does (verbose_json, language "auto"). */+export async function transcribe(base, wav, language = "auto") {+ const form = new FormData();+ form.set("file", new Blob([readFileSync(wav)], { type: "audio/wav" }), path.basename(wav));+ form.set("response_format", "verbose_json");+ form.set("language", language);+ const t0 = performance.now();+ const res = await fetch(`${base}/inference`, { method: "POST", body: form });+ const wallMs = performance.now() - t0;+ const json = await res.json();+ if (!res.ok) throw new Error(`${wav}: HTTP ${res.status} ${JSON.stringify(json)}`);+ return { json, wallMs };+}++/** 16-bit PCM mono 16 kHz WAV -> Float32Array in [-1, 1]. Walks chunks. */+export function readWav(file) {+ const b = readFileSync(file);+ let off = 12;+ let fmt = null;+ while (off + 8 <= b.length) {+ const id = b.toString("ascii", off, off + 4);+ const size = b.readUInt32LE(off + 4);+ if (id === "fmt ")+ fmt = {+ channels: b.readUInt16LE(off + 10),+ rate: b.readUInt32LE(off + 12),+ bits: b.readUInt16LE(off + 22),+ };+ if (id === "data") {+ if (!fmt || fmt.channels !== 1 || fmt.bits !== 16 || fmt.rate !== SR)+ throw new Error(`${file}: need 16 kHz mono s16, got ${JSON.stringify(fmt)}`);+ const n = Math.min(size, b.length - off - 8) >> 1;+ const out = new Float32Array(n);+ for (let i = 0; i < n; i++) out[i] = b.readInt16LE(off + 8 + i * 2) / 32768;+ return out;+ }+ off += 8 + size + (size & 1);+ }+ throw new Error(`${file}: no data chunk`);+}++/** RMS in dBFS per `frameSec` frame. */+export function envelopeDb(samples, frameSec = 0.005) {+ const L = Math.round(SR * frameSec);+ const n = Math.floor(samples.length / L);+ const out = new Float32Array(n);+ for (let f = 0; f < n; f++) {+ let s = 0;+ for (let i = f * L; i < (f + 1) * L; i++) s += samples[i] * samples[i];+ out[f] = 10 * Math.log10(s / L + 1e-12);+ }+ return out;+}++/** Runs of frames below `thrDb` lasting at least `minSec`: [{ start, end }] seconds. */+export function silenceRuns(env, frameSec, thrDb, minSec) {+ const runs = [];+ let s = -1;+ for (let f = 0; f <= env.length; f++) {+ const quiet = f < env.length && env[f] < thrDb;+ if (quiet && s < 0) s = f;+ if (!quiet && s >= 0) {+ if ((f - s) * frameSec >= minSec) runs.push({ start: s * frameSec, end: f * frameSec });+ s = -1;+ }+ }+ return runs;+}++// Reference ends. SAPI gives them (last non-silent phoneme). OneCore gives only+// starts: a word runs to the next word's start unless a pause (>= MIN_PAUSE of+// energy below THR_DB) sits between them, in which case it ends where the pause+// starts. Checked against SAPI's phoneme ends in validate-ref.mjs.+export const THR_DB = -50;+export const MIN_PAUSE = 0.1;+export function referenceWords(tts, samples) {+ const FRAME = 0.005;+ const runs = silenceRuns(envelopeDb(samples, FRAME), FRAME, THR_DB, MIN_PAUSE);+ const dur = samples.length / SR;+ return tts.words.map((w, i) => {+ const next = tts.words[i + 1]?.start ?? dur;+ const pause = runs.find((r) => r.start > w.start + 0.03 && r.start < next);+ const derivedEnd = pause ? pause.start : next;+ return {+ text: w.text,+ start: w.start,+ end: w.end ?? derivedEnd,+ derivedEnd,+ ttsEnd: w.end ?? null,+ };+ });+}++/** Lowercase, keep letters, digits and inner apostrophes only. */+export function norm(w) {+ return w+ .toLowerCase()+ .normalize("NFC")+ .replace(/[’']/g, "'")+ .replace(/[^\p{L}\p{N}']/gu, "")+ .replace(/^'+|'+$/g, "");+}++/** Levenshtein alignment of two token lists; returns matched pairs [i, j] and sub/ins/del counts. */+export function align(ref, hyp) {+ const n = ref.length;+ const m = hyp.length;+ const W = m + 1;+ const D = new Uint32Array((n + 1) * W);+ const B = new Uint8Array((n + 1) * W); // 0 diag, 1 up (del), 2 left (ins)+ for (let i = 1; i <= n; i++) {+ D[i * W] = i;+ B[i * W] = 1;+ }+ for (let j = 1; j <= m; j++) {+ D[j] = j;+ B[j] = 2;+ }+ for (let i = 1; i <= n; i++) {+ for (let j = 1; j <= m; j++) {+ const c = D[(i - 1) * W + j - 1] + (ref[i - 1] === hyp[j - 1] ? 0 : 1);+ const u = D[(i - 1) * W + j] + 1;+ const l = D[i * W + j - 1] + 1;+ if (c <= u && c <= l) {+ D[i * W + j] = c;+ B[i * W + j] = 0;+ } else if (u <= l) {+ D[i * W + j] = u;+ B[i * W + j] = 1;+ } else {+ D[i * W + j] = l;+ B[i * W + j] = 2;+ }+ }+ }+ const pairs = [];+ let sub = 0,+ ins = 0,+ del = 0;+ let i = n,+ j = m;+ while (i > 0 || j > 0) {+ const b = B[i * W + j];+ if (i > 0 && j > 0 && b === 0) {+ if (ref[i - 1] === hyp[j - 1]) pairs.push([i - 1, j - 1]);+ else sub++;+ i--;+ j--;+ } else if (i > 0 && (j === 0 || b === 1)) {+ del++;+ i--;+ } else {+ ins++;+ j--;+ }+ }+ pairs.reverse();+ return { pairs, sub, ins, del };+}++export function stats(xs) {+ if (xs.length === 0) return { n: 0 };+ const a = [...xs].sort((p, q) => p - q);+ const q = (p) => a[Math.min(a.length - 1, Math.floor(p * a.length))];+ const mean = a.reduce((s, x) => s + x, 0) / a.length;+ const within = (t) => a.filter((x) => x <= t).length / a.length;+ return {+ n: a.length,+ mean,+ median: q(0.5),+ p90: q(0.9),+ w20: within(0.02),+ w50: within(0.05),+ w100: within(0.1),+ };+}diff --git a/tools/stt-eval/word-timing/make-corpus.mjs b/tools/stt-eval/word-timing/make-corpus.mjsnew file mode 100644index 000000000..5bc031ed3--- /dev/null+++ b/tools/stt-eval/word-timing/make-corpus.mjs@@ -0,0 +1,137 @@+// Builds the synthetic corpus: SSML -> Windows TTS (tts.ps1) -> 16 kHz mono WAV+// + reference word times, plus a degraded copy (pink noise + light reverb).+// Usage: node make-corpus.mjs (skips clips already on disk)+import { execFileSync } from "node:child_process";+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import path from "node:path";+import { RENDERS, TEXTS } from "./corpus-texts.mjs";+import { CLIPS, DATA, FFMPEG, HERE, readWav, referenceWords } from "./lib.mjs";++mkdirSync(CLIPS, { recursive: true });++const esc = (s) => s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");+function ssml(text, lang, rate) {+ const body = text+ .split(/(\{\d+\})/)+ .map((part) => {+ const m = /^\{(\d+)\}$/.exec(part);+ return m ? `<break time="${m[1]}ms"/>` : esc(part);+ })+ .join("");+ // OneCore rejects a <prosody> without attributes: always write the rate.+ const pct = Math.round((rate - 1) * 100);+ const r = ` rate="${pct >= 0 ? "+" : ""}${pct}%"`;+ return `<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xml:lang="${lang === "fr" ? "fr-FR" : "en-US"}"><prosody${r}>${body}</prosody></speak>`;+}++function render(id, text, lang, r) {+ const base = path.join(CLIPS, id);+ if (existsSync(`${base}.ref.json`)) return JSON.parse(readFileSync(`${base}.ref.json`, "utf8"));+ const ssmlFile = `${base}.ssml`;+ writeFileSync(ssmlFile, ssml(text, lang, r.rate));+ execFileSync(+ "powershell",+ [+ "-NoProfile",+ "-ExecutionPolicy",+ "Bypass",+ "-File",+ path.join(HERE, "tts.ps1"),+ "-Engine",+ r.engine,+ "-Voice",+ r.voice,+ "-SsmlFile",+ ssmlFile,+ "-Out",+ base,+ ],+ { stdio: "inherit" },+ );+ // Canonical 16 kHz mono s16 with a plain header (both engines already emit+ // 16 kHz mono, so this is a container rewrite, no resampling).+ execFileSync(FFMPEG, [+ "-hide_banner",+ "-loglevel",+ "error",+ "-y",+ "-i",+ `${base}.raw.wav`,+ "-ar",+ "16000",+ "-ac",+ "1",+ "-c:a",+ "pcm_s16le",+ "-map_metadata",+ "-1",+ "-fflags",+ "+bitexact",+ `${base}.wav`,+ ]);+ rmSync(`${base}.raw.wav`);+ // Degraded copy: pink noise ~25 dB under the speech, two early reflections.+ execFileSync(FFMPEG, [+ "-hide_banner",+ "-loglevel",+ "error",+ "-y",+ "-i",+ `${base}.wav`,+ "-filter_complex",+ "[0:a]aecho=0.9:0.6:23|41:0.25|0.15[r];anoisesrc=color=pink:amplitude=0.012:sample_rate=16000:seed=7[n];[r][n]amix=inputs=2:duration=first:normalize=0[o]",+ "-map",+ "[o]",+ "-ar",+ "16000",+ "-ac",+ "1",+ "-c:a",+ "pcm_s16le",+ "-fflags",+ "+bitexact",+ `${base}.noisy.wav`,+ ]);+ const tts = JSON.parse(readFileSync(`${base}.tts.json`, "utf8").replace(/^/, ""));+ const samples = readWav(`${base}.wav`);+ const ref = {+ id,+ lang,+ ...r,+ durationSec: samples.length / 16000,+ words: referenceWords(tts, samples),+ };+ writeFileSync(`${base}.ref.json`, JSON.stringify(ref, null, 1));+ return ref;+}++const manifest = [];+for (const lang of ["fr", "en"]) {+ TEXTS[lang].forEach((text, i) => {+ for (const r of RENDERS[lang]) {+ const id = `${lang}${i + 1}-${r.voice.split(" ")[1].toLowerCase()}-${Math.round(r.rate * 100)}`;+ const ref = render(id, text, lang, r);+ manifest.push({+ id,+ lang,+ engine: r.engine,+ voice: r.voice,+ rate: r.rate,+ script: i + 1,+ durationSec: ref.durationSec,+ words: ref.words.length,+ });+ console.log(id, ref.durationSec.toFixed(1), "s", ref.words.length, "words");+ }+ });+}+const total = manifest.reduce((s, c) => s + c.durationSec, 0);+writeFileSync(+ path.join(DATA, "corpus-manifest.json"),+ JSON.stringify(+ { totalSec: total, degraded: "pink noise amplitude 0.012 + aecho 23/41 ms", clips: manifest },+ null,+ 1,+ ),+);+console.log(`total ${(total / 60).toFixed(1)} min clean (+ the same again degraded)`);diff --git a/tools/stt-eval/word-timing/real-check.mjs b/tools/stt-eval/word-timing/real-check.mjsnew file mode 100644index 000000000..09a3d7865--- /dev/null+++ b/tools/stt-eval/word-timing/real-check.mjs@@ -0,0 +1,65 @@+// Sanity check on real speech, which has no ground truth: the VAD onset is the+// only boundary we can trust there.+// Usage: node real-check.mjs <whisper-stt-server.exe> <take.wav> [--cpu] [--snap <post-pass.ts>]+// (16 kHz mono s16 WAV, e.g. ffmpeg -i take.webm -ar 16000 -ac 1 take.wav)+// Prints how far each word's first-token time (`anchor`) lies after its start+// (the one-token lag the helper corrects), and, per VAD stretch, where its first+// word starts relative to the onset: as the helper reports it, and after the+// post-pass. Writes the raw response to <data>/results/real-<take>.json.+import { mkdirSync, writeFileSync } from "node:fs";+import path from "node:path";+import { pathToFileURL } from "node:url";+import { REPO, RESULTS, startHelper, transcribe } from "./lib.mjs";++const [exe, wav, ...rest] = process.argv.slice(2);+if (!exe || !wav)+ throw new Error(+ "usage: node real-check.mjs <whisper-stt-server.exe> <take.wav> [--cpu] [--snap <post-pass.ts>]",+ );+const snapAt = rest.indexOf("--snap");+const snapPath = path.resolve(+ snapAt >= 0 ? rest[snapAt + 1] : path.join(REPO, "electron/stt/snapWordBoundaries.ts"),+);+const { anchorWordsOnSpeech } = await import(pathToFileURL(snapPath).href);++const { base, stop } = await startHelper(exe, { cpu: rest.includes("--cpu") });+let json;+try {+ ({ json } = await transcribe(base, wav));+} finally {+ stop();+}+mkdirSync(RESULTS, { recursive: true });+writeFileSync(path.join(RESULTS, `real-${path.basename(wav, ".wav")}.json`), JSON.stringify(json));++const isP = (w) => /^[\p{P}\p{S}]+$/u.test(w.word);+const raw = json.segments+ .flatMap((s) => s.words)+ .map((w) => ({+ word: w.word.trim(),+ startSec: w.start,+ endSec: Math.max(w.start + 0.02, w.end),+ anchorSec: w.anchor ?? w.start,+ }))+ .filter((w) => w.word);+const speech = json.speech.map((s) => ({ startSec: s.start, endSec: s.end }));+const post = anchorWordsOnSpeech(raw, speech);+const lag = raw+ .filter((w) => !isP(w))+ .map((w) => w.anchorSec - w.startSec)+ .sort((a, b) => a - b);+const q = (p) => (lag[Math.floor(lag.length * p)] * 1000).toFixed(0);+console.log(+ `${json.detected_language}, ${raw.length} words; first-token time minus start: median ${q(0.5)} ms, p10 ${q(0.1)}, p90 ${q(0.9)}`,+);+const ms = (x) => `${x >= 0 ? "+" : ""}${(x * 1000).toFixed(0)} ms`;+let k = 0;+for (const [i, s] of speech.entries()) {+ const tail = Math.min(s.endSec + 0.1, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);+ while (k < raw.length && isP(raw[k])) k++;+ if (k >= raw.length || raw[k].anchorSec >= tail) continue;+ console.log(+ `stretch ${s.startSec.toFixed(2)}-${s.endSec.toFixed(2)}: "${raw[k].word}" starts ${ms(raw[k].startSec - s.startSec)} from the onset raw, ${ms(post[k].startSec - s.startSec)} after the post-pass`,+ );+ while (k < raw.length && raw[k].anchorSec < tail) k++;+}diff --git a/tools/stt-eval/word-timing/run-helper.mjs b/tools/stt-eval/word-timing/run-helper.mjsnew file mode 100644index 000000000..291acd5d4--- /dev/null+++ b/tools/stt-eval/word-timing/run-helper.mjs@@ -0,0 +1,53 @@+// Runs whisper-stt-server over the corpus and saves its raw verbose_json.+// Usage: node run-helper.mjs <tag> <whisper-stt-server.exe> [--cpu] [--condition clean|noisy|both]+// [--language auto|fr|en] [--only <id substring>] [--env KEY=VAL]+// -> <data>/results/raw/<tag>/<clipId>.<condition>.json (+ wallMs)+// Clips already done are skipped, so an interrupted run resumes.+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";+import path from "node:path";+import { CLIPS, DATA, RESULTS, startHelper, transcribe } from "./lib.mjs";++const [tag, exe, ...rest] = process.argv.slice(2);+if (!tag || !exe)+ throw new Error(+ "usage: node run-helper.mjs <tag> <whisper-stt-server.exe> [--cpu] [--condition both] [--language auto] [--only <substr>] [--env K=V]",+ );+const flag = (name, dflt) => {+ const i = rest.indexOf(name);+ return i >= 0 ? rest[i + 1] : dflt;+};+const condition = flag("--condition", "both");+const language = flag("--language", "auto"); // the app sends "auto"+const only = flag("--only", "");+const env = { ...process.env };+for (let i = 0; i < rest.length; i++)+ if (rest[i] === "--env") {+ const [k, v] = rest[i + 1].split("=");+ env[k] = v;+ }++const { base, stop } = await startHelper(exe, { cpu: rest.includes("--cpu"), env });+try {+ const outDir = path.join(RESULTS, "raw", tag);+ mkdirSync(outDir, { recursive: true });+ const manifest = JSON.parse(readFileSync(path.join(DATA, "corpus-manifest.json"), "utf8"));+ const conds = condition === "both" ? ["clean", "noisy"] : [condition];+ for (const c of manifest.clips) {+ if (only && !c.id.includes(only)) continue;+ for (const cond of conds) {+ const out = path.join(outDir, `${c.id}.${cond}.json`);+ if (existsSync(out)) continue;+ const wav = path.join(CLIPS, cond === "clean" ? `${c.id}.wav` : `${c.id}.noisy.wav`);+ const { json, wallMs } = await transcribe(base, wav, language);+ writeFileSync(out, JSON.stringify({ ...json, wallMs }));+ console.log(+ `${c.id}.${cond}`,+ `${(wallMs / 1000).toFixed(2)} s`,+ json.detected_language,+ json.backend,+ );+ }+ }+} finally {+ stop();+}diff --git a/tools/stt-eval/word-timing/summarize.mjs b/tools/stt-eval/word-timing/summarize.mjsnew file mode 100644index 000000000..849673c27--- /dev/null+++ b/tools/stt-eval/word-timing/summarize.mjs@@ -0,0 +1,39 @@+// Markdown table of evaluated configurations, from <data>/results/<name>.json.+// Usage: node summarize.mjs [<label>=<name>[:<stage>] ...] > summary.md+// default: the pre-#948 baseline (vk-stock) against the current pipeline (vk-phase1).+// <stage> is raw, post or post+vad (default).+import { readFileSync } from "node:fs";+import path from "node:path";+import { RESULTS } from "./lib.mjs";++const args = process.argv.slice(2);+const CONFIGS = (args.length ? args : ["Before (shipped)=vk-stock", "Phase 1=vk-phase1"]).map(+ (a) => {+ const [label, spec] = a.split("=");+ const [file, stage = "post+vad"] = spec.split(":");+ return [label, file, stage];+ },+);+const ms = (x) => (x == null ? "-" : (x * 1000).toFixed(0));+const pc = (x) => (x == null ? "-" : `${(x * 100).toFixed(0)}%`);+for (const group of ["all", "clean", "noisy", "clean.fr", "clean.en"]) {+ console.log(`\n**${group}**\n`);+ console.log(+ "| Config | inner start med / P90 / <50 ms | phrase-initial start med / <50 ms | inner end med / <50 ms | phrase-final end med | 1-word delete: clean cuts / audible residue / audible clipping | phrase delete: clean | WER |",+ );+ console.log("|---|---|---|---|---|---|---|---|");+ for (const [label, file, stage] of CONFIGS) {+ let j;+ try {+ j = JSON.parse(readFileSync(path.join(RESULTS, `${file}.json`), "utf8"));+ } catch {+ continue;+ }+ const s = (k) => j.summary[`${group}|${stage}|${k}`];+ const c = (k) => j.counts[`${group}.${k}`];+ const wer = (c("sub") + c("ins") + c("del")) / c("refWords");+ console.log(+ `| ${label} | ${ms(s("start|inner").median)} / ${ms(s("start|inner").p90)} / ${pc(s("start|inner").w50)} | ${ms(s("start|initial").median)} / ${pc(s("start|initial").w50)} | ${ms(s("end|inner").median)} / ${pc(s("end|inner").w50)} | ${ms(s("bias_end|final").median)} (signed) | ${pc(s("trim|clean").mean)} / ${s("trim|residueAudMs").mean.toFixed(0)} ms / ${s("trim|clipAudMs").mean.toFixed(0)} ms | ${pc(s("phrase|clean").mean)} | ${pc(wer)} |`,+ );+ }+}diff --git a/tools/stt-eval/word-timing/tts.ps1 b/tools/stt-eval/word-timing/tts.ps1new file mode 100644index 000000000..e379580e7--- /dev/null+++ b/tools/stt-eval/word-timing/tts.ps1@@ -0,0 +1,81 @@+# Synthesizes one clip with exact word positions.+# -Engine onecore : WinRT Windows.Media.SpeechSynthesis (word-boundary TimedMetadataTrack)+# -Engine sapi : System.Speech (SpeakProgress + PhonemeReached for word ends)+# Writes <Out>.raw.wav (engine's own format) and <Out>.tts.json:+# { engine, voice, words: [{ text, start, end|null, charPos }] } — seconds.+# Input is an SSML <speak> document read from -SsmlFile (UTF-8).+param(+ [Parameter(Mandatory)] [ValidateSet("onecore", "sapi")] [string]$Engine,+ [Parameter(Mandatory)] [string]$Voice,+ [Parameter(Mandatory)] [string]$SsmlFile,+ [Parameter(Mandatory)] [string]$Out+)+$ErrorActionPreference = "Stop"+$ssml = [System.IO.File]::ReadAllText($SsmlFile, [System.Text.Encoding]::UTF8)+$words = New-Object System.Collections.ArrayList++if ($Engine -eq "onecore") {+ Add-Type -AssemblyName System.Runtime.WindowsRuntime+ [void][Windows.Media.SpeechSynthesis.SpeechSynthesizer, Windows.Media.SpeechSynthesis, ContentType = WindowsRuntime]+ [void][Windows.Media.SpeechSynthesis.SpeechSynthesisStream, Windows.Media.SpeechSynthesis, ContentType = WindowsRuntime]+ $asTask = [System.WindowsRuntimeSystemExtensions].GetMethods() | Where-Object {+ $_.Name -eq "AsTask" -and $_.GetParameters().Count -eq 1 -and+ $_.GetParameters()[0].ParameterType.Name -eq 'IAsyncOperation`1'+ } | Select-Object -First 1+ $synth = New-Object Windows.Media.SpeechSynthesis.SpeechSynthesizer+ $synth.Voice = [Windows.Media.SpeechSynthesis.SpeechSynthesizer]::AllVoices | Where-Object { $_.DisplayName -eq $Voice } | Select-Object -First 1+ if (-not $synth.Voice) { throw "voice not found: $Voice" }+ $synth.Options.IncludeWordBoundaryMetadata = $true+ $task = $asTask.MakeGenericMethod([Windows.Media.SpeechSynthesis.SpeechSynthesisStream]).Invoke($null, @($synth.SynthesizeSsmlToStreamAsync($ssml)))+ $task.Wait()+ $stream = $task.Result+ # OneCore word cues carry a start only (Duration is 0): ends are derived later.+ foreach ($track in $stream.TimedMetadataTracks) {+ if ($track.Id -ne "SpeechWord") { continue }+ foreach ($cue in $track.Cues) {+ [void]$words.Add([ordered]@{+ text = $cue.Text+ start = $cue.StartTime.TotalSeconds+ end = if ($cue.Duration.TotalSeconds -gt 0) { $cue.StartTime.TotalSeconds + $cue.Duration.TotalSeconds } else { $null }+ charPos = $cue.StartPositionInInput+ })+ }+ }+ $net = [System.IO.WindowsRuntimeStreamExtensions]::AsStreamForRead($stream)+ $fs = [System.IO.File]::Create("$Out.raw.wav")+ $net.CopyTo($fs)+ $fs.Close(); $net.Close()+} else {+ Add-Type -AssemblyName System.Speech+ $synth = New-Object System.Speech.Synthesis.SpeechSynthesizer+ $synth.SelectVoice($Voice)+ $fmt = New-Object System.Speech.AudioFormat.SpeechAudioFormatInfo(16000, [System.Speech.AudioFormat.AudioBitsPerSample]::Sixteen, [System.Speech.AudioFormat.AudioChannel]::Mono)+ $synth.SetOutputToWaveFile("$Out.raw.wav", $fmt)+ $phonemes = New-Object System.Collections.ArrayList+ $synth.add_SpeakProgress({ param($s, $e)+ [void]$words.Add([ordered]@{ text = $e.Text; start = $e.AudioPosition.TotalSeconds; end = $null; charPos = $e.CharacterPosition })+ })+ $synth.add_PhonemeReached({ param($s, $e)+ [void]$phonemes.Add(@($e.AudioPosition.TotalSeconds, $e.Duration.TotalSeconds, $e.Phoneme))+ })+ $synth.SpeakSsml($ssml)+ $synth.SetOutputToNull()+ $synth.Dispose()+ # A word ends where its last non-silence phoneme ends (SAPI phoneme 7 = silence);+ # phonemes are attributed to the word whose start precedes them.+ for ($i = 0; $i -lt $words.Count; $i++) {+ $from = $words[$i].start+ $to = if ($i + 1 -lt $words.Count) { $words[$i + 1].start } else { [double]::MaxValue }+ $end = $null+ foreach ($p in $phonemes) {+ if ($p[0] -ge $from -and $p[0] -lt $to -and ($p[2].Length -eq 0 -or [int]$p[2][0] -ne 7)) {+ $e = $p[0] + $p[1]+ if ($null -eq $end -or $e -gt $end) { $end = $e }+ }+ }+ $words[$i].end = $end+ }+}++$doc = [ordered]@{ engine = $Engine; voice = $Voice; words = @($words) }+[System.IO.File]::WriteAllText("$Out.tts.json", (ConvertTo-Json -Depth 5 $doc), (New-Object System.Text.UTF8Encoding($false)))diff --git a/tools/stt-eval/word-timing/validate-ref.mjs b/tools/stt-eval/word-timing/validate-ref.mjsnew file mode 100644index 000000000..b1ee063d9--- /dev/null+++ b/tools/stt-eval/word-timing/validate-ref.mjs@@ -0,0 +1,63 @@+// How good is the TTS reference? Checks it against the audio it describes.+// 1. Onsets: after every pause (>= 150 ms below -50 dBFS), the next reference+// word should start where the energy comes back. Signed: negative = the+// reference starts before the audible onset (a stop closure, silent by nature).+// 2. Offsets (SAPI only, where ends come from phonemes, independent of energy):+// the word before a pause should end where the energy drops.+// 3. OneCore ends are derived (next start, or pause start). The same derivation+// applied to SAPI clips is compared with SAPI's phoneme ends.+// Usage: node validate-ref.mjs -> results/ref-validation.json + summary on stdout+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";+import path from "node:path";+import { CLIPS, DATA, envelopeDb, RESULTS, readWav, silenceRuns, stats } from "./lib.mjs";++const manifest = JSON.parse(readFileSync(path.join(DATA, "corpus-manifest.json"), "utf8"));+const out = {+ onset: { onecore: [], sapi: [] },+ offsetSapi: [],+ derivedVsPhonemeSapi: [],+ derivedVsPhonemeSapiAtPause: [],+};+for (const c of manifest.clips) {+ const ref = JSON.parse(readFileSync(path.join(CLIPS, `${c.id}.ref.json`), "utf8"));+ const env = envelopeDb(readWav(path.join(CLIPS, `${c.id}.wav`)), 0.005);+ const runs = silenceRuns(env, 0.005, -50, 0.15);+ for (const r of runs) {+ const after = ref.words.find((w) => w.start >= r.start - 0.05);+ if (after && after.start < r.end + 0.2) out.onset[c.engine].push(after.start - r.end);+ const before = [...ref.words].reverse().find((w) => w.start < r.start);+ if (c.engine === "sapi" && before?.ttsEnd != null && r.start > 0.01)+ out.offsetSapi.push(before.ttsEnd - r.start);+ }+ if (c.engine === "sapi") {+ for (const w of ref.words) {+ if (w.ttsEnd == null) continue;+ out.derivedVsPhonemeSapi.push(w.derivedEnd - w.ttsEnd);+ if (runs.some((r) => r.start >= w.start && r.start <= w.derivedEnd + 0.01))+ out.derivedVsPhonemeSapiAtPause.push(w.derivedEnd - w.ttsEnd);+ }+ }+}+const signed = (xs) => {+ const a = [...xs].sort((p, q) => p - q);+ const q = (p) => +a[Math.min(a.length - 1, Math.floor(p * a.length))].toFixed(3);+ return { n: a.length, p10: q(0.1), median: q(0.5), p90: q(0.9), abs: stats(xs.map(Math.abs)) };+};+const summary = {+ onset_onecore: signed(out.onset.onecore),+ onset_sapi: signed(out.onset.sapi),+ offset_sapi: signed(out.offsetSapi),+ derivedEnd_minus_phonemeEnd_sapi_all: signed(out.derivedVsPhonemeSapi),+ derivedEnd_minus_phonemeEnd_sapi_at_pause: signed(out.derivedVsPhonemeSapiAtPause),+};+mkdirSync(RESULTS, { recursive: true });+writeFileSync(+ path.join(RESULTS, "ref-validation.json"),+ JSON.stringify({ summary, raw: out }, null, 1),+);+for (const [k, v] of Object.entries(summary)) {+ console.log(+ k.padEnd(44),+ `n=${v.n} p10=${v.p10} med=${v.median} p90=${v.p90} |abs| med=${v.abs.median?.toFixed(3)} w20=${(v.abs.w20 * 100).toFixed(0)}% w50=${(v.abs.w50 * 100).toFixed(0)}%`,+ );+}