Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(stt): keep transcript words on the audio's clock when the VAD cuts silence
EtienneLescot0f92f62ea09c
diff --git a/electron/native/whisper-stt/src/main.cpp b/electron/native/whisper-stt/src/main.cppindex f287eed2e..874398c93 100644--- a/electron/native/whisper-stt/src/main.cpp+++ b/electron/native/whisper-stt/src/main.cpp@@ -27,7 +27,8 @@// "start": 0.0, "end": 11.0,// "words": [ { "word": "...", "start": 0.5, "end": 0.9, "probability": 0.9 }, ... ]// }, ...-// ]+// ],+// "speech": [ { "start": 0.4, "end": 2.1 }, ... ] // only when the VAD model loaded// }//// Concurrency: whisper contexts are not thread-safe. /inference is serialized@@ -228,6 +229,33 @@ struct Word {std::string text;};+// One stretch of speech copied from the upload into the buffer whisper decodes.+struct Kept {+ int64_t at; // first sample in that buffer+ int64_t from; // first sample in the upload+ int64_t len;+};++// A whisper time (centiseconds on the speech-only buffer) back on the upload's+// clock, in seconds. Inside a kept stretch it moves with the stretch; in the+// silence inserted between two stretches it snaps to the nearer edge, so a+// word can never land in audio that was cut out. No stretches: VAD was off.+double to_original_sec(int64_t cs, const std::vector<Kept>& kept) {+ if (kept.empty()) return cs / 100.0;+ const int64_t x = cs * 160; // 16 kHz: 160 samples per centisecond+ for (size_t i = 0; i < kept.size(); ++i) {+ const Kept& k = kept[i];+ if (x < k.at) {+ if (i == 0) return k.from / 16000.0;+ const Kept& p = kept[i - 1];+ const bool nearer_prev = x - (p.at + p.len) <= k.at - x;+ return (nearer_prev ? p.from + p.len : k.from) / 16000.0;+ }+ if (x < k.at + k.len) return (k.from + x - k.at) / 16000.0;+ }+ return (kept.back().from + kept.back().len) / 16000.0;+}+} // namespaceint main(int argc, char** argv) {@@ -312,17 +340,17 @@ int main(int argc, char** argv) {log("model loaded; backend=" + active_backend);// ---- Init VAD context (Silero VAD v6.2.0) ----+ // On the CPU, always. Asked for the GPU on Vulkan, whisper.cpp 1.9.1 puts the+ // VAD weights in a Vulkan buffer, then finds no GPU for the VAD's own backend+ // and aborts inside ggml (0xC0000409) — so the app relaunched the helper with+ // --cpu and every transcription lost its GPU. Silero is 0.9 MB: the CPU is+ // not where the time goes.struct whisper_vad_context* vctx = nullptr;if (!vad_model_path.empty()) {struct whisper_vad_context_params vad_ctx_params = whisper_vad_default_context_params();vad_ctx_params.n_threads = threads;- vad_ctx_params.use_gpu = !force_cpu;+ vad_ctx_params.use_gpu = false;vctx = whisper_vad_init_from_file_with_params(vad_model_path.c_str(), vad_ctx_params);- if (!vctx && vad_ctx_params.use_gpu) {- log("GPU VAD initialization failed; retrying with CPU inference");- vad_ctx_params.use_gpu = false;- vctx = whisper_vad_init_from_file_with_params(vad_model_path.c_str(), vad_ctx_params);- }if (vctx) {log("VAD model loaded; path=" + vad_model_path);} else {@@ -407,14 +435,52 @@ int main(int argc, char** argv) {wparams.print_realtime = false;wparams.print_timestamps = false;wparams.n_threads = threads;- if (vctx && !vad_model_path.empty()) {- wparams.vad = true;- wparams.vad_model_path = vad_model_path.c_str();- wparams.vad_params = whisper_vad_default_params();++ // ---- Speech only (Silero VAD), cut here rather than by whisper_full ----+ // whisper_full's own `vad` param maps SEGMENT times back onto the upload+ // but not token times, so every word (t_dtw included) came back early by+ // all the silence removed before it: 13 s into a 25 s clip. Cutting here+ // keeps the map. The stretches also go out as `speech`, which the Node+ // side anchors phrase edges on.+ const auto t0 = std::chrono::steady_clock::now(); // VAD time is part of the run+ std::vector<float> speech;+ std::vector<Kept> kept;+ nlohmann::json speech_json = nlohmann::json::array();+ if (vctx) {+ whisper_vad_segments* vs = whisper_vad_segments_from_samples(+ vctx, whisper_vad_default_params(), pcm.data(), static_cast<int>(pcm.size()));+ if (!vs) {+ res.status = 500;+ res.set_content(R"({"error":"VAD failed"})", "application/json");+ return;+ }+ const int64_t n_pcm = static_cast<int64_t>(pcm.size());+ const auto to_sample = [&](float cs) {+ return std::clamp<int64_t>(std::llround(cs * 160.0), 0, n_pcm);+ };+ const int n_vs = whisper_vad_segments_n_segments(vs);+ for (int i = 0; i < n_vs; ++i) {+ const int64_t from = to_sample(whisper_vad_segments_get_segment_t0(vs, i));+ const int64_t end = to_sample(whisper_vad_segments_get_segment_t1(vs, i));+ if (end <= from) continue;+ speech_json.push_back({{"start", from / 16000.0}, {"end", end / 16000.0}});+ // As whisper.cpp does: 0.1 s past the detected end so a soft ending+ // survives, and 0.1 s of silence between stretches. Unlike it, the+ // tail never runs into the next stretch and decodes it twice.+ const int64_t next = i + 1 < n_vs+ ? to_sample(whisper_vad_segments_get_segment_t0(vs, i + 1))+ : n_pcm;+ const int64_t to = std::max(end, std::min(end + 1600, next));+ if (!kept.empty()) speech.insert(speech.end(), 1600, 0.0f);+ kept.push_back({static_cast<int64_t>(speech.size()), from, to - from});+ speech.insert(speech.end(), pcm.begin() + from, pcm.begin() + to);+ }+ whisper_vad_free_segments(vs);}+ // VAD on and no speech found: nothing to decode, so nothing to report.+ const std::vector<float>& input = vctx ? speech : pcm;- const auto t0 = std::chrono::steady_clock::now();- const int rc = whisper_full(ctx, wparams, pcm.data(), static_cast<int>(pcm.size()));+ const int rc = input.empty() ? 0 : whisper_full(ctx, wparams, input.data(), static_cast<int>(input.size()));const auto t1 = std::chrono::steady_clock::now();if (rc != 0) {log("whisper_full returned " + std::to_string(rc));@@ -451,11 +517,11 @@ int main(int argc, char** argv) {};std::vector<Segment> segments;- const int n_segments = whisper_full_n_segments(ctx);+ const int n_segments = input.empty() ? 0 : whisper_full_n_segments(ctx);for (int si = 0; si < n_segments; ++si) {Segment seg;- seg.start = whisper_full_get_segment_t0(ctx, si) / 100.0;- seg.end = whisper_full_get_segment_t1(ctx, si) / 100.0;+ seg.start = to_original_sec(whisper_full_get_segment_t0(ctx, si), kept);+ seg.end = to_original_sec(whisper_full_get_segment_t1(ctx, si), kept);if (const char* t = whisper_full_get_segment_text(ctx, si)) seg.text = t;struct W { double t_dtw_first; double p_sum; int p_n; std::string text; };@@ -486,7 +552,7 @@ int main(int argc, char** argv) {++non_special_tokens;const bool starts_word = (!in_word) || (!raw.empty() && raw[0] == ' ');- const double td_dtw = (td.t_dtw >= 0 ? td.t_dtw : 0) / 100.0;+ const double td_dtw = to_original_sec(td.t_dtw >= 0 ? td.t_dtw : 0, kept);if (starts_word && in_word) {word_buf.push_back({ w_first_t_dtw, w_p_sum, w_p_n, cur_text });@@ -547,7 +613,7 @@ int main(int argc, char** argv) {// whisper resolved — the detected one under "auto", and the forced one// otherwise, which is correct for both paths.std::string resolved_language = language;- const int lang_id = whisper_full_lang_id(ctx);+ const int lang_id = input.empty() ? -1 : whisper_full_lang_id(ctx);if (lang_id >= 0) {if (const char* lang_str = whisper_lang_str(lang_id)) {resolved_language = lang_str;@@ -583,6 +649,7 @@ int main(int argc, char** argv) {segs.push_back(std::move(seg));}reply["segments"] = std::move(segs);+ if (vctx) reply["speech"] = std::move(speech_json);res.set_content(reply.dump(), "application/json");});diff --git a/electron/stt/modelManager.ts b/electron/stt/modelManager.tsindex 959551237..bc37a0359 100644--- a/electron/stt/modelManager.ts+++ b/electron/stt/modelManager.ts@@ -20,8 +20,9 @@ import { pipeline } from "node:stream/promises";* The file is verified by SHA-256 and written atomically (via .partial rename)* to prevent partial downloads from being treated as complete.*- * Word timestamps come from whisper.cpp's native DTW token timestamps, so no- * separate VAD model is required. See `technical-documentation/architecture/transcription-and-captions.md`.+ * Word timestamps come from whisper.cpp's native DTW token timestamps. The+ * Silero VAD model only decides which audio whisper decodes, and gives the+ * speech edges phrases are anchored on. See `technical-documentation/architecture/transcription-and-captions.md`.*/export type SttModelId = "whisper" | "silero-vad";diff --git a/electron/stt/snapWordBoundaries.test.ts b/electron/stt/snapWordBoundaries.test.tsindex 2668c6966..46a6b4bf8 100644--- a/electron/stt/snapWordBoundaries.test.ts+++ b/electron/stt/snapWordBoundaries.test.ts@@ -74,3 +74,81 @@ describe("snapWordBoundariesToAudio", () => {expect(snapWordBoundariesToAudio(untouched, new Float32Array(0))).toEqual(untouched);});});++describe("snapWordBoundariesToAudio with speech intervals", () => {+ // Loud and flat everywhere, so the RMS snap moves nothing and only the+ // anchoring on `speech` is under test.+ const flat = audioWithSilences(6, []);+ const ms = (sec: number) => Math.round(sec * 1000) / 1000;+ const times = (words: SttWordSegment[]) =>+ words.map((w) => [w.word, ms(w.startSec), ms(w.endSec)]);++ it("pulls each phrase's first word onto its onset, past the punctuation DTW dropped there", () => {+ // The shape of a real French take: whisper put "Salut" 0.58 s and "Bah"+ // 0.25 s after the speech started, and the "!" closing the first phrase+ // just after the second one began.+ const words = [+ word({ word: "Salut", startSec: 2.15, endSec: 3.37 }),+ word({ word: "!", startSec: 3.37, endSec: 3.39 }),+ word({ word: "Bah", startSec: 3.61, endSec: 4.01 }),+ word({ word: "voilà", startSec: 4.01, endSec: 4.35 }),+ ];+ const speech = [+ { startSec: 1.57, endSec: 2.56 },+ { startSec: 3.36, endSec: 4.35 },+ ];+ expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([+ ["Salut", 1.57, 3.36],+ ["!", 3.36, 3.36],+ ["Bah", 3.36, 4.01],+ ["voilà", 4.01, 4.35],+ ]);+ });++ it("ends each phrase's last word where its speech stops, taking trailing punctuation along", () => {+ // The end of the same take: whisper closed the last segment at 2.79 while+ // "tac" ran to 3.04, and dropped the "!" on the word itself.+ const words = [+ word({ word: "tic,", startSec: 1.95, endSec: 2.59 }),+ word({ word: "tac", startSec: 2.59, endSec: 2.79 }),+ word({ word: "!", startSec: 2.62, endSec: 2.79 }),+ ];+ const speech = [+ { startSec: 1.95, endSec: 2.37 },+ { startSec: 2.59, endSec: 3.04 },+ ];+ expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([+ ["tic,", 1.95, 2.59],+ ["tac", 2.59, 3.04],+ ["!", 3.04, 3.04],+ ]);+ });++ it("leaves a phrase alone when its words are on time, or too far off to be its edges", () => {+ const onTime = [word({ startSec: 0.95, endSec: 3.1 })];+ expect(times(snapWordBoundariesToAudio(onTime, flat, [{ startSec: 1, endSec: 3 }]))).toEqual(+ times(onTime),+ );+ // 1.2 s after the onset and 1.1 s before the offset: more likely a neighbour+ // of words whisper dropped than the phrase's own edges.+ const tooFar = [word({ startSec: 2.2, endSec: 2.5 })];+ expect(times(snapWordBoundariesToAudio(tooFar, flat, [{ startSec: 1, endSec: 3.6 }]))).toEqual(+ times(tooFar),+ );+ });++ it("never mistakes a phrase's second word for its first", () => {+ // "y" opens the second phrase but was reported just before its onset;+ // "z" must not be dragged back over it.+ const words = [+ word({ word: "x", startSec: 0.5, endSec: 1.95 }),+ word({ word: "y", startSec: 1.95, endSec: 2.4 }),+ word({ word: "z", startSec: 2.4, endSec: 3 }),+ ];+ const speech = [+ { startSec: 0.5, endSec: 1 },+ { startSec: 2, endSec: 3 },+ ];+ expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual(times(words));+ });+});diff --git a/electron/stt/snapWordBoundaries.ts b/electron/stt/snapWordBoundaries.tsindex d5d0b6f10..2e61d465e 100644--- a/electron/stt/snapWordBoundaries.ts+++ b/electron/stt/snapWordBoundaries.ts@@ -17,8 +17,14 @@// `LOOKBACK_SEC`. It is self-limiting — on a decaying tail (a word that ends a// phrase, where whisper is already right) the quietest frame IS the reported// one, so the boundary doesn't move at all.+//+// The edges of a phrase are the exception. Its first word, DTW reports 0.1–0.6 s+// late (measured against Silero VAD on a real French recording), far past what+// the lookback can reach; its last word ends where whisper's segment ends, which+// can stop short of the speech. When the helper sent its speech intervals, both+// are pulled onto the edges of the stretch of speech they belong to.-import type { SttWordSegment } from "./transcriptionContract";+import type { SttVadSegment, SttWordSegment } from "./transcriptionContract";/** Matches `writeSamplesAsWav` — the samples handed to whisper are mono 16 kHz. */const SAMPLE_RATE = 16_000;@@ -37,6 +43,18 @@ const LOOKBACK_SEC = 0.15;/** Keep degenerate words (whisper sometimes reports end <= start) non-empty. */const MIN_WORD_SEC = 0.02;+/**+ * How far a phrase's first word may start after its speech, or its last word+ * end before it, and still be pulled onto that edge — the calibration knob of+ * the anchoring step, sized above the measured 0.1–0.6 s. Past it, the word is+ * more likely a neighbour of one whisper dropped, and stretching it over that+ * audio would be a guess.+ */+const MAX_ANCHOR_SEC = 1;++/** French puts a space before `!`, `?`, `:` and `;`, so whisper emits them as words. */+const isPunctuation = (word: string) => /^[\p{P}\p{S}]+$/u.test(word);+/** Per-frame RMS of the mono signal — the cheapest usable "is this speech" proxy. */function rmsEnvelope(samples: Float32Array): Float32Array {const frameLength = Math.round(SAMPLE_RATE * FRAME_SEC);@@ -54,15 +72,59 @@ function rmsEnvelope(samples: Float32Array): Float32Array {return envelope;}+/**+ * Pull the first and last word of every speech stretch onto the stretch's+ * edges. The onset already carries the VAD's 30 ms pad, so a cut there lands+ * just before the attack. What sits past an edge moves with it: the previous+ * word's shared boundary, and any punctuation DTW dropped inside the speech,+ * which collapses to a point on the edge.+ */+function anchorOnSpeech(words: SttWordSegment[], speech: SttVadSegment[]): SttWordSegment[] {+ const out = words.map((w) => ({ ...w }));+ let k = 0;+ for (let i = 0; i < speech.length; i++) {+ const { startSec: onset, endSec: offset } = speech[i];+ const previousEnd = i > 0 ? speech[i - 1].endSec : Number.NEGATIVE_INFINITY;+ // The phrase's first word: the first real word reported after the previous stretch ended.+ while (k < out.length && (out[k].startSec < previousEnd || isPunctuation(out[k].word))) k++;+ if (k === out.length) break;+ const late = out[k].startSec - onset;+ if (late > 0 && late <= MAX_ANCHOR_SEC && out[k].startSec < offset) {+ out[k].startSec = onset;+ for (let j = k - 1; j >= 0 && out[j].endSec > onset; j--) {+ out[j].endSec = onset;+ out[j].startSec = Math.min(out[j].startSec, onset);+ }+ }+ // Its last word: the last real word reported before the speech stops.+ let m = -1;+ for (let j = k; j < out.length && out[j].startSec < offset; j++) {+ if (!isPunctuation(out[j].word)) m = j;+ }+ const early = m < 0 ? 0 : offset - out[m].endSec;+ if (early > 0 && early <= MAX_ANCHOR_SEC) {+ out[m].endSec = offset;+ for (let j = m + 1; j < out.length && out[j].startSec < offset; j++) {+ out[j].startSec = offset;+ out[j].endSec = Math.max(out[j].endSec, offset);+ }+ }+ }+ return out;+}+/*** Move every word boundary back to the quietest frame in the `LOOKBACK_SEC`* preceding it. Boundaries shared by two words snap identically (same input* time), so the transcript stays gap-free where whisper made it gap-free.- * Returns the words unchanged when there is no audio to measure against.+ * With `speech` (the helper's VAD intervals), phrase edges are then anchored+ * on the speech. Returns the words unchanged when there is no audio to+ * measure against.*/export function snapWordBoundariesToAudio(words: SttWordSegment[],samples: Float32Array,+ speech?: SttVadSegment[],): SttWordSegment[] {const envelope = rmsEnvelope(samples);if (envelope.length === 0) return words;@@ -89,7 +151,7 @@ export function snapWordBoundariesToAudio(return bestFrame * FRAME_SEC;};- return words.map((w) => {+ const snapped = words.map((w) => {const startSec = snap(w.startSec);return {...w,@@ -97,4 +159,5 @@ export function snapWordBoundariesToAudio(endSec: Math.max(snap(w.endSec), startSec + MIN_WORD_SEC),};});+ return speech ? anchorOnSpeech(snapped, speech) : snapped;}diff --git a/electron/stt/whisperServer.test.ts b/electron/stt/whisperServer.test.tsindex 314926e58..1d4c3e41b 100644--- a/electron/stt/whisperServer.test.ts+++ b/electron/stt/whisperServer.test.ts@@ -350,6 +350,34 @@ describe("WhisperServerManager", () => {}});+ it("anchors a phrase's first word on the speech onset the helper reports", async () => {+ const fakeJson = {+ segments: [+ {+ text: " Salut",+ start: 1.57,+ end: 2.56,+ words: [{ word: " Salut", start: 2.15, end: 2.56 }],+ },+ ],+ speech: [{ start: 1.57, end: 2.56 }],+ backend: "whispercpp-cpu",+ };+ vi.stubGlobal(+ "fetch",+ vi.fn(async () => new Response(JSON.stringify(fakeJson), { status: 200 })),+ );+ try {+ const mgr = new WhisperServerManager();+ (mgr as unknown as { process: unknown; port: number }).process = {};+ (mgr as unknown as { process: unknown; port: number }).port = 9999;+ const result = await mgr.transcribe({ samples: new Float32Array(16_000 * 3) });+ expect(result.wordSegments[0].startSec).toBeCloseTo(1.57, 6);+ } finally {+ vi.unstubAllGlobals();+ }+ });+it("spawns whisper-stt-server with --model", async () => {const fs = await import("node:fs/promises");const { spawn } = await import("node:child_process");diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.tsindex a9698674f..6ffa887f7 100644--- a/electron/stt/whisperServer.ts+++ b/electron/stt/whisperServer.ts@@ -115,6 +115,8 @@ interface WhisperJsonResponse {detected_language?: string;backend?: string;timing?: WhisperJsonTiming;+ /** Silero VAD's speech intervals; absent when the helper runs without the VAD model. */+ speech?: Array<{ start?: number | string; end?: number | string }>;}export class WhisperServerManager {@@ -597,9 +599,14 @@ export class WhisperServerManager {return { text, startSec, endSec: Math.max(endSec, startSec + 0.05) };}).filter((s) => s.text.length > 0);- // whisper.cpp's DTW boundaries run ~80–150 ms behind the audio, which the- // transcript editor turns into imprecise trims (see snapWordBoundaries.ts).- // Re-anchor them on the same samples whisper was given.+ const speech = json.speech?.map((s) => ({+ startSec: this.toSec(s.start, 0),+ endSec: this.toSec(s.end, 0),+ }));+ // whisper.cpp's DTW boundaries run ~80–150 ms behind the audio, and a+ // phrase's first word up to 0.6 s, which the transcript editor turns into+ // imprecise trims (see snapWordBoundaries.ts). Re-anchor them on the same+ // samples whisper was given, and on the helper's speech intervals.const wordSegments: SttWordSegment[] = snapWordBoundariesToAudio(raw.flatMap((seg) =>@@ -613,6 +620,7 @@ export class WhisperServerManager {).filter((w) => w.word.length > 0),opts.samples,+ speech,);const detectedLanguage = json.detected_language ?? json.language ?? "auto";const backend = this.toBackend(json.backend);diff --git a/scripts/test-whisper-stt.mjs b/scripts/test-whisper-stt.mjsindex 7e52bac2b..7cef820bb 100644--- a/scripts/test-whisper-stt.mjs+++ b/scripts/test-whisper-stt.mjs@@ -19,6 +19,8 @@// Env overrides:// OPENSCREEN_WHISPER_SERVER_EXE helper binary (default: the staged one for this host)// OPENSCREEN_WHISPER_MODEL GGML model (default: the userData cache location)+// OPENSCREEN_VAD_MODEL Silero VAD model (default: next to the GGML model;+// the speech checks run only when it is there, as in the app)import { spawn, spawnSync } from "node:child_process";import fs from "node:fs";@@ -47,6 +49,9 @@ const BIN =);const MODEL = process.env.OPENSCREEN_WHISPER_MODEL ?? defaultModelPath();+const VAD_MODEL =+ process.env.OPENSCREEN_VAD_MODEL ?? path.join(path.dirname(MODEL), "ggml-silero-v6.2.0.bin");+const WITH_VAD = fs.existsSync(VAD_MODEL);const LANGUAGE = argOf("--language") ?? "auto";const WAV_ARG = argOf("--wav");const REF_ARG = argOf("--ref");@@ -208,7 +213,8 @@ async function main() {`${meta.durationSec.toFixed(2)}s`,);console.log(`helper : ${BIN}`);- console.log(`model : ${MODEL}\n`);+ console.log(`model : ${MODEL}`);+ console.log(`vad : ${WITH_VAD ? VAD_MODEL : "(none, speech checks skipped)"}\n`);const port = 20100 + Math.floor(process.pid % 400);const child = spawn(@@ -222,6 +228,7 @@ async function main() {"127.0.0.1","--threads",String(Math.max(1, os.cpus().length)),+ ...(WITH_VAD ? ["--vad-model", VAD_MODEL] : []),],{ stdio: ["ignore", "pipe", "pipe"] },);@@ -320,6 +327,24 @@ async function main() {"word starts are monotonic non-decreasing",);+ // With the VAD model, whisper decodes only the speech and every time has to+ // be mapped back onto the clip. whisper.cpp's own VAD mapped segment times+ // but not word times, which put words seconds early (13 s into a 25 s clip)+ // while every check above still passed. A word starting in what the VAD+ // called silence is that bug; 0.15 s covers the 0.1 s tail kept past each stretch.+ if (WITH_VAD) {+ const speech = json.speech ?? [];+ check(Array.isArray(json.speech), "response carries the speech intervals", `${speech.length}`);+ const stray = words.filter(+ (w) => !speech.some((s) => w.start >= s.start - 0.15 && w.start <= s.end + 0.15),+ );+ check(+ stray.length === 0,+ "every word starts inside a stretch of speech",+ stray.map((w) => `${JSON.stringify(w.word)}@${w.start}`).join(", "),+ );+ }+// NOT a failure. whisper.cpp gives the last word of a segment the segment's// own t1 as its end, while the word's DTW start runs 80–150 ms late — so a// token emitted near the boundary (typically standalone punctuation) can landdiff --git a/src/lib/captioning/transcribe.ts b/src/lib/captioning/transcribe.tsindex 7b8473608..82aaad14c 100644--- a/src/lib/captioning/transcribe.ts+++ b/src/lib/captioning/transcribe.ts@@ -128,13 +128,12 @@ function runTranscription(options?.signal?.addEventListener("abort", onAbort, { once: true });const forcedLanguage =options?.language && options.language !== "auto" ? options.language : undefined;- // ponytail: word timestamps come back already absolute from whisper.cpp- // because its built-in Silero VAD (started on the server with- // `--vad --vad-model`) splits audio into speech regions *before* the ASR- // decoder runs and offsets each region's timestamps to its position in the- // original audio. No trim + offset math here on purpose: an earlier- // iteration trimmed leading silence with a peak detector and got false- // positives on quiet music intros / room tone. VAD or nothing.+ // ponytail: word timestamps come back already absolute. The helper cuts the+ // silence out itself (Silero VAD, `--vad-model`) and maps every word back+ // onto the original audio: whisper.cpp's own VAD maps segment times but not+ // word times. No trim + offset math here on purpose: an earlier iteration+ // trimmed leading silence with a peak detector and got false positives on+ // quiet music intros / room tone. VAD or nothing.return api.transcribe({ ...payload, language: forcedLanguage }).then((result) => {diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.mdindex d2a499302..026350743 100644--- a/technical-documentation/architecture/transcription-and-captions.md+++ b/technical-documentation/architecture/transcription-and-captions.md@@ -18,7 +18,7 @@ flowchart LRA -- "extract mono 16 kHz" --> B["transcribeAsset<br/>(src/lib/ai-edition/document/transcribe.ts)"]B -- "IPC: Float32Array + language" --> C["SttManager<br/>(electron/stt/index.ts)"]C -- "POST /inference (WAV)" --> D["whisper-stt-server<br/>(electron/native/whisper-stt/)"]- D -- "whisper_full() + DTW" --> E["SttTranscribeResponse<br/>(segments + wordSegments)"]+ D -- "VAD cut + whisper_full() + DTW" --> E["SttTranscribeResponse<br/>(segments + wordSegments)"]E --> CC -- "IPC return" --> BB --> F["AxcutTranscript<br/>on document.transcripts[]"]@@ -186,6 +186,23 @@ it verbatim in the response.### Word-level alignment (DTW token timestamps)+0. **Speech only** — when the Silero VAD model is on disk (`ggml-silero-v6.2.0.bin`,+ downloaded next to the whisper model), the helper cuts the silence out of+ the upload before whisper sees it, the way whisper.cpp's own VAD does: each+ speech stretch plus 0.1 s of tail, 0.1 s of silence between stretches. It+ keeps the map, and every time below goes back through it onto the upload's+ clock. The stretches are returned as `speech` for step 5.+ > **Not whisper_full's `vad` param.** whisper.cpp 1.9.1 maps its *segment*+ > times back onto the original audio but not its token times, `t_dtw`+ > included, so every word came back early by all the silence removed before+ > it: 13 s into a 25 s clip. That shipped in 2.0.0-rc.1, and the harness+ > missed it (times were monotonic and inside the clip, just wrong); it now+ > checks that every word starts inside a stretch of speech.+ >+ > **The VAD runs on the CPU, always.** Asked for the GPU on Vulkan, 1.9.1+ > puts the VAD weights in a Vulkan buffer, finds no GPU for the VAD's own+ > backend and aborts in ggml (`0xC0000409`), so `SttManager` relaunched the+ > helper with `--cpu` and every transcription lost its GPU.1. **Decode** — `whisper_full()` runs with the default sampling parameters(`WHISPER_SAMPLING_GREEDY`), returning phrase segments plus a per-tokenarray.@@ -214,6 +231,15 @@ it verbatim in the response.the quietest frame *is* the reported one and nothing moves. Boundariespast the end of the decoded audio are left untouched.+ The edges of a phrase are past that reach. Its first word, DTW reports+ 0.1–0.6 s after the speech starts (measured against the VAD on a real+ French take); its last word ends on whisper's segment end, which can stop+ short of the speech. With `speech`, the first real word of each stretch+ starts on the stretch's onset and its last real word ends on its offset,+ punctuation between collapsing onto the edge. `MAX_ANCHOR_SEC` is that+ step's knob: a word further off than 1 s is more likely the neighbour of+ one whisper dropped, and is left alone.+> Why this matters beyond caption timing: the transcript editor turns a word> selection into a trim of exactly `[firstWord.startSec, lastWord.endSec]`,> so a late boundary leaves the attack of the first removed word audible and@@ -239,9 +265,11 @@ linked above).### Model-The single shipped artifact is `ggml-small-q8_0.bin` from+The recogniser is `ggml-small-q8_0.bin` from`ggerganov/whisper.cpp` on HuggingFace: Whisper `small`, multilingual (~99-languages), q8_0 quantised, ~264 MB. Precision is baked into the GGML file —+languages), q8_0 quantised, ~264 MB. Beside it, the same code path fetches+Silero VAD v6.2.0 (`ggml-org/whisper-vad`, 0.9 MB) for step 0 of the+alignment. Precision is baked into the GGML file —there is no runtime `--int8` flag. `electron/stt/modelManager.ts` downloadsthe file once into the user-data cache and writes it through an atomic`.partial` rename, so a half-downloaded file can never be picked up as a@@ -270,13 +298,13 @@ and the digest together.| Module | Role ||---|---|| `electron/stt/whisperServer.ts` | Server lifecycle; `POST /inference` client; verbose_json parser. |-| `electron/stt/snapWordBoundaries.ts` | Re-anchors DTW word boundaries on the audio's RMS envelope (see step 5 above). |+| `electron/stt/snapWordBoundaries.ts` | Re-anchors DTW word boundaries on the audio's RMS envelope, and phrase edges on the VAD's speech (see step 5 above). || `electron/stt/wav.ts` | WAV write + temp-file cleanup helpers. || `electron/stt/gpuDetector.ts` | Per-platform binary resolver (no GPU probing). || `electron/stt/modelManager.ts` | Single GGML file download, SHA-256 verify, atomic write. || `electron/stt/transcriptionContract.ts` | Shared IPC types (`SttBackend`, `SttWordSegment`, `SttPhraseSegment`, `SttStatusEvent`). || `electron/stt/index.ts` | `SttManager` — IPC entry point; wires the pieces together. |-| `electron/native/whisper-stt/src/main.cpp` | httplib HTTP server; calls `whisper_full()` with DTW; reports the device it bound via `ggml_backend_dev_name()`. |+| `electron/native/whisper-stt/src/main.cpp` | httplib HTTP server; cuts the silence out with Silero VAD and maps times back (step 0); calls `whisper_full()` with DTW; reports the device it bound via `ggml_backend_dev_name()`. || `electron/native/whisper-stt/CMakeLists.txt` | Pulls whisper.cpp via FetchContent; enables Metal (macOS arm64), Vulkan (Windows/Linux x64), CPU fallback everywhere; static backend linking into `whisper.dll`/`ggml.dll`. |The helper is one executable per platform; backends are baked in at build@@ -736,9 +764,8 @@ it deletes data- **No C++ unit tests.** The WAV reader and the DTW-inactive guardrailin `electron/native/whisper-stt/src/main.cpp` are exercised only atruntime.-- **Caption sync fine-tuning.** DTW `t_dtw` is a *word-end* (the moment- the ASR committed the token), so a caption can look a touch late on- screen for the first second of a word. A future tweak could blend- with the previous word's `t_dtw` for the start bound or mix with- `token.t0`; record the final choice here once verified against real- recordings.+- **Word timing inside a phrase.** Phrase edges sit on the VAD (step 5), but+ a word in the middle of continuous speech still starts on its DTW time+ pulled back by at most 150 ms of RMS lookback. A dedicated forced aligner+ is the upgrade path (issue #626); measure against the VAD-anchored edges+ before adopting one.