Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(stt): keep transcript words on the audio's clock when the VAD cuts silence

EtienneLescot0f92f62ea09c
diff --git a/electron/native/whisper-stt/src/main.cpp b/electron/native/whisper-stt/src/main.cpp
index f287eed2e..874398c93 100644
--- a/electron/native/whisper-stt/src/main.cpp
+++ b/electron/native/whisper-stt/src/main.cpp
@@ -27,7 +27,8 @@
// "start": 0.0, "end": 11.0,
// "words": [ { "word": "...", "start": 0.5, "end": 0.9, "probability": 0.9 }, ... ]
// }, ...
-// ]
+// ],
+// "speech": [ { "start": 0.4, "end": 2.1 }, ... ] // only when the VAD model loaded
// }
//
// Concurrency: whisper contexts are not thread-safe. /inference is serialized
@@ -228,6 +229,33 @@ struct Word {
std::string text;
};
+// One stretch of speech copied from the upload into the buffer whisper decodes.
+struct Kept {
+ int64_t at; // first sample in that buffer
+ int64_t from; // first sample in the upload
+ int64_t len;
+};
+
+// A whisper time (centiseconds on the speech-only buffer) back on the upload's
+// clock, in seconds. Inside a kept stretch it moves with the stretch; in the
+// silence inserted between two stretches it snaps to the nearer edge, so a
+// word can never land in audio that was cut out. No stretches: VAD was off.
+double to_original_sec(int64_t cs, const std::vector<Kept>& kept) {
+ if (kept.empty()) return cs / 100.0;
+ const int64_t x = cs * 160; // 16 kHz: 160 samples per centisecond
+ for (size_t i = 0; i < kept.size(); ++i) {
+ const Kept& k = kept[i];
+ if (x < k.at) {
+ if (i == 0) return k.from / 16000.0;
+ const Kept& p = kept[i - 1];
+ const bool nearer_prev = x - (p.at + p.len) <= k.at - x;
+ return (nearer_prev ? p.from + p.len : k.from) / 16000.0;
+ }
+ if (x < k.at + k.len) return (k.from + x - k.at) / 16000.0;
+ }
+ return (kept.back().from + kept.back().len) / 16000.0;
+}
+
} // namespace
int main(int argc, char** argv) {
@@ -312,17 +340,17 @@ int main(int argc, char** argv) {
log("model loaded; backend=" + active_backend);
// ---- Init VAD context (Silero VAD v6.2.0) ----
+ // On the CPU, always. Asked for the GPU on Vulkan, whisper.cpp 1.9.1 puts the
+ // VAD weights in a Vulkan buffer, then finds no GPU for the VAD's own backend
+ // and aborts inside ggml (0xC0000409) — so the app relaunched the helper with
+ // --cpu and every transcription lost its GPU. Silero is 0.9 MB: the CPU is
+ // not where the time goes.
struct whisper_vad_context* vctx = nullptr;
if (!vad_model_path.empty()) {
struct whisper_vad_context_params vad_ctx_params = whisper_vad_default_context_params();
vad_ctx_params.n_threads = threads;
- vad_ctx_params.use_gpu = !force_cpu;
+ vad_ctx_params.use_gpu = false;
vctx = whisper_vad_init_from_file_with_params(vad_model_path.c_str(), vad_ctx_params);
- if (!vctx && vad_ctx_params.use_gpu) {
- log("GPU VAD initialization failed; retrying with CPU inference");
- vad_ctx_params.use_gpu = false;
- vctx = whisper_vad_init_from_file_with_params(vad_model_path.c_str(), vad_ctx_params);
- }
if (vctx) {
log("VAD model loaded; path=" + vad_model_path);
} else {
@@ -407,14 +435,52 @@ int main(int argc, char** argv) {
wparams.print_realtime = false;
wparams.print_timestamps = false;
wparams.n_threads = threads;
- if (vctx && !vad_model_path.empty()) {
- wparams.vad = true;
- wparams.vad_model_path = vad_model_path.c_str();
- wparams.vad_params = whisper_vad_default_params();
+
+ // ---- Speech only (Silero VAD), cut here rather than by whisper_full ----
+ // whisper_full's own `vad` param maps SEGMENT times back onto the upload
+ // but not token times, so every word (t_dtw included) came back early by
+ // all the silence removed before it: 13 s into a 25 s clip. Cutting here
+ // keeps the map. The stretches also go out as `speech`, which the Node
+ // side anchors phrase edges on.
+ const auto t0 = std::chrono::steady_clock::now(); // VAD time is part of the run
+ std::vector<float> speech;
+ std::vector<Kept> kept;
+ nlohmann::json speech_json = nlohmann::json::array();
+ if (vctx) {
+ whisper_vad_segments* vs = whisper_vad_segments_from_samples(
+ vctx, whisper_vad_default_params(), pcm.data(), static_cast<int>(pcm.size()));
+ if (!vs) {
+ res.status = 500;
+ res.set_content(R"({"error":"VAD failed"})", "application/json");
+ return;
+ }
+ const int64_t n_pcm = static_cast<int64_t>(pcm.size());
+ const auto to_sample = [&](float cs) {
+ return std::clamp<int64_t>(std::llround(cs * 160.0), 0, n_pcm);
+ };
+ const int n_vs = whisper_vad_segments_n_segments(vs);
+ for (int i = 0; i < n_vs; ++i) {
+ const int64_t from = to_sample(whisper_vad_segments_get_segment_t0(vs, i));
+ const int64_t end = to_sample(whisper_vad_segments_get_segment_t1(vs, i));
+ if (end <= from) continue;
+ speech_json.push_back({{"start", from / 16000.0}, {"end", end / 16000.0}});
+ // As whisper.cpp does: 0.1 s past the detected end so a soft ending
+ // survives, and 0.1 s of silence between stretches. Unlike it, the
+ // tail never runs into the next stretch and decodes it twice.
+ const int64_t next = i + 1 < n_vs
+ ? to_sample(whisper_vad_segments_get_segment_t0(vs, i + 1))
+ : n_pcm;
+ const int64_t to = std::max(end, std::min(end + 1600, next));
+ if (!kept.empty()) speech.insert(speech.end(), 1600, 0.0f);
+ kept.push_back({static_cast<int64_t>(speech.size()), from, to - from});
+ speech.insert(speech.end(), pcm.begin() + from, pcm.begin() + to);
+ }
+ whisper_vad_free_segments(vs);
}
+ // VAD on and no speech found: nothing to decode, so nothing to report.
+ const std::vector<float>& input = vctx ? speech : pcm;
- const auto t0 = std::chrono::steady_clock::now();
- const int rc = whisper_full(ctx, wparams, pcm.data(), static_cast<int>(pcm.size()));
+ const int rc = input.empty() ? 0 : whisper_full(ctx, wparams, input.data(), static_cast<int>(input.size()));
const auto t1 = std::chrono::steady_clock::now();
if (rc != 0) {
log("whisper_full returned " + std::to_string(rc));
@@ -451,11 +517,11 @@ int main(int argc, char** argv) {
};
std::vector<Segment> segments;
- const int n_segments = whisper_full_n_segments(ctx);
+ const int n_segments = input.empty() ? 0 : whisper_full_n_segments(ctx);
for (int si = 0; si < n_segments; ++si) {
Segment seg;
- seg.start = whisper_full_get_segment_t0(ctx, si) / 100.0;
- seg.end = whisper_full_get_segment_t1(ctx, si) / 100.0;
+ seg.start = to_original_sec(whisper_full_get_segment_t0(ctx, si), kept);
+ seg.end = to_original_sec(whisper_full_get_segment_t1(ctx, si), kept);
if (const char* t = whisper_full_get_segment_text(ctx, si)) seg.text = t;
struct W { double t_dtw_first; double p_sum; int p_n; std::string text; };
@@ -486,7 +552,7 @@ int main(int argc, char** argv) {
++non_special_tokens;
const bool starts_word = (!in_word) || (!raw.empty() && raw[0] == ' ');
- const double td_dtw = (td.t_dtw >= 0 ? td.t_dtw : 0) / 100.0;
+ const double td_dtw = to_original_sec(td.t_dtw >= 0 ? td.t_dtw : 0, kept);
if (starts_word && in_word) {
word_buf.push_back({ w_first_t_dtw, w_p_sum, w_p_n, cur_text });
@@ -547,7 +613,7 @@ int main(int argc, char** argv) {
// whisper resolved — the detected one under "auto", and the forced one
// otherwise, which is correct for both paths.
std::string resolved_language = language;
- const int lang_id = whisper_full_lang_id(ctx);
+ const int lang_id = input.empty() ? -1 : whisper_full_lang_id(ctx);
if (lang_id >= 0) {
if (const char* lang_str = whisper_lang_str(lang_id)) {
resolved_language = lang_str;
@@ -583,6 +649,7 @@ int main(int argc, char** argv) {
segs.push_back(std::move(seg));
}
reply["segments"] = std::move(segs);
+ if (vctx) reply["speech"] = std::move(speech_json);
res.set_content(reply.dump(), "application/json");
});
diff --git a/electron/stt/modelManager.ts b/electron/stt/modelManager.ts
index 959551237..bc37a0359 100644
--- a/electron/stt/modelManager.ts
+++ b/electron/stt/modelManager.ts
@@ -20,8 +20,9 @@ import { pipeline } from "node:stream/promises";
* The file is verified by SHA-256 and written atomically (via .partial rename)
* to prevent partial downloads from being treated as complete.
*
- * Word timestamps come from whisper.cpp's native DTW token timestamps, so no
- * separate VAD model is required. See `technical-documentation/architecture/transcription-and-captions.md`.
+ * Word timestamps come from whisper.cpp's native DTW token timestamps. The
+ * Silero VAD model only decides which audio whisper decodes, and gives the
+ * speech edges phrases are anchored on. See `technical-documentation/architecture/transcription-and-captions.md`.
*/
export type SttModelId = "whisper" | "silero-vad";
diff --git a/electron/stt/snapWordBoundaries.test.ts b/electron/stt/snapWordBoundaries.test.ts
index 2668c6966..46a6b4bf8 100644
--- a/electron/stt/snapWordBoundaries.test.ts
+++ b/electron/stt/snapWordBoundaries.test.ts
@@ -74,3 +74,81 @@ describe("snapWordBoundariesToAudio", () => {
expect(snapWordBoundariesToAudio(untouched, new Float32Array(0))).toEqual(untouched);
});
});
+
+describe("snapWordBoundariesToAudio with speech intervals", () => {
+ // Loud and flat everywhere, so the RMS snap moves nothing and only the
+ // anchoring on `speech` is under test.
+ const flat = audioWithSilences(6, []);
+ const ms = (sec: number) => Math.round(sec * 1000) / 1000;
+ const times = (words: SttWordSegment[]) =>
+ words.map((w) => [w.word, ms(w.startSec), ms(w.endSec)]);
+
+ it("pulls each phrase's first word onto its onset, past the punctuation DTW dropped there", () => {
+ // The shape of a real French take: whisper put "Salut" 0.58 s and "Bah"
+ // 0.25 s after the speech started, and the "!" closing the first phrase
+ // just after the second one began.
+ const words = [
+ word({ word: "Salut", startSec: 2.15, endSec: 3.37 }),
+ word({ word: "!", startSec: 3.37, endSec: 3.39 }),
+ word({ word: "Bah", startSec: 3.61, endSec: 4.01 }),
+ word({ word: "voilà", startSec: 4.01, endSec: 4.35 }),
+ ];
+ const speech = [
+ { startSec: 1.57, endSec: 2.56 },
+ { startSec: 3.36, endSec: 4.35 },
+ ];
+ expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([
+ ["Salut", 1.57, 3.36],
+ ["!", 3.36, 3.36],
+ ["Bah", 3.36, 4.01],
+ ["voilà", 4.01, 4.35],
+ ]);
+ });
+
+ it("ends each phrase's last word where its speech stops, taking trailing punctuation along", () => {
+ // The end of the same take: whisper closed the last segment at 2.79 while
+ // "tac" ran to 3.04, and dropped the "!" on the word itself.
+ const words = [
+ word({ word: "tic,", startSec: 1.95, endSec: 2.59 }),
+ word({ word: "tac", startSec: 2.59, endSec: 2.79 }),
+ word({ word: "!", startSec: 2.62, endSec: 2.79 }),
+ ];
+ const speech = [
+ { startSec: 1.95, endSec: 2.37 },
+ { startSec: 2.59, endSec: 3.04 },
+ ];
+ expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([
+ ["tic,", 1.95, 2.59],
+ ["tac", 2.59, 3.04],
+ ["!", 3.04, 3.04],
+ ]);
+ });
+
+ it("leaves a phrase alone when its words are on time, or too far off to be its edges", () => {
+ const onTime = [word({ startSec: 0.95, endSec: 3.1 })];
+ expect(times(snapWordBoundariesToAudio(onTime, flat, [{ startSec: 1, endSec: 3 }]))).toEqual(
+ times(onTime),
+ );
+ // 1.2 s after the onset and 1.1 s before the offset: more likely a neighbour
+ // of words whisper dropped than the phrase's own edges.
+ const tooFar = [word({ startSec: 2.2, endSec: 2.5 })];
+ expect(times(snapWordBoundariesToAudio(tooFar, flat, [{ startSec: 1, endSec: 3.6 }]))).toEqual(
+ times(tooFar),
+ );
+ });
+
+ it("never mistakes a phrase's second word for its first", () => {
+ // "y" opens the second phrase but was reported just before its onset;
+ // "z" must not be dragged back over it.
+ const words = [
+ word({ word: "x", startSec: 0.5, endSec: 1.95 }),
+ word({ word: "y", startSec: 1.95, endSec: 2.4 }),
+ word({ word: "z", startSec: 2.4, endSec: 3 }),
+ ];
+ const speech = [
+ { startSec: 0.5, endSec: 1 },
+ { startSec: 2, endSec: 3 },
+ ];
+ expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual(times(words));
+ });
+});
diff --git a/electron/stt/snapWordBoundaries.ts b/electron/stt/snapWordBoundaries.ts
index d5d0b6f10..2e61d465e 100644
--- a/electron/stt/snapWordBoundaries.ts
+++ b/electron/stt/snapWordBoundaries.ts
@@ -17,8 +17,14 @@
// `LOOKBACK_SEC`. It is self-limiting — on a decaying tail (a word that ends a
// phrase, where whisper is already right) the quietest frame IS the reported
// one, so the boundary doesn't move at all.
+//
+// The edges of a phrase are the exception. Its first word, DTW reports 0.1–0.6 s
+// late (measured against Silero VAD on a real French recording), far past what
+// the lookback can reach; its last word ends where whisper's segment ends, which
+// can stop short of the speech. When the helper sent its speech intervals, both
+// are pulled onto the edges of the stretch of speech they belong to.
-import type { SttWordSegment } from "./transcriptionContract";
+import type { SttVadSegment, SttWordSegment } from "./transcriptionContract";
/** Matches `writeSamplesAsWav` — the samples handed to whisper are mono 16 kHz. */
const SAMPLE_RATE = 16_000;
@@ -37,6 +43,18 @@ const LOOKBACK_SEC = 0.15;
/** Keep degenerate words (whisper sometimes reports end <= start) non-empty. */
const MIN_WORD_SEC = 0.02;
+/**
+ * How far a phrase's first word may start after its speech, or its last word
+ * end before it, and still be pulled onto that edge — the calibration knob of
+ * the anchoring step, sized above the measured 0.1–0.6 s. Past it, the word is
+ * more likely a neighbour of one whisper dropped, and stretching it over that
+ * audio would be a guess.
+ */
+const MAX_ANCHOR_SEC = 1;
+
+/** French puts a space before `!`, `?`, `:` and `;`, so whisper emits them as words. */
+const isPunctuation = (word: string) => /^[\p{P}\p{S}]+$/u.test(word);
+
/** Per-frame RMS of the mono signal — the cheapest usable "is this speech" proxy. */
function rmsEnvelope(samples: Float32Array): Float32Array {
const frameLength = Math.round(SAMPLE_RATE * FRAME_SEC);
@@ -54,15 +72,59 @@ function rmsEnvelope(samples: Float32Array): Float32Array {
return envelope;
}
+/**
+ * Pull the first and last word of every speech stretch onto the stretch's
+ * edges. The onset already carries the VAD's 30 ms pad, so a cut there lands
+ * just before the attack. What sits past an edge moves with it: the previous
+ * word's shared boundary, and any punctuation DTW dropped inside the speech,
+ * which collapses to a point on the edge.
+ */
+function anchorOnSpeech(words: SttWordSegment[], speech: SttVadSegment[]): SttWordSegment[] {
+ const out = words.map((w) => ({ ...w }));
+ let k = 0;
+ for (let i = 0; i < speech.length; i++) {
+ const { startSec: onset, endSec: offset } = speech[i];
+ const previousEnd = i > 0 ? speech[i - 1].endSec : Number.NEGATIVE_INFINITY;
+ // The phrase's first word: the first real word reported after the previous stretch ended.
+ while (k < out.length && (out[k].startSec < previousEnd || isPunctuation(out[k].word))) k++;
+ if (k === out.length) break;
+ const late = out[k].startSec - onset;
+ if (late > 0 && late <= MAX_ANCHOR_SEC && out[k].startSec < offset) {
+ out[k].startSec = onset;
+ for (let j = k - 1; j >= 0 && out[j].endSec > onset; j--) {
+ out[j].endSec = onset;
+ out[j].startSec = Math.min(out[j].startSec, onset);
+ }
+ }
+ // Its last word: the last real word reported before the speech stops.
+ let m = -1;
+ for (let j = k; j < out.length && out[j].startSec < offset; j++) {
+ if (!isPunctuation(out[j].word)) m = j;
+ }
+ const early = m < 0 ? 0 : offset - out[m].endSec;
+ if (early > 0 && early <= MAX_ANCHOR_SEC) {
+ out[m].endSec = offset;
+ for (let j = m + 1; j < out.length && out[j].startSec < offset; j++) {
+ out[j].startSec = offset;
+ out[j].endSec = Math.max(out[j].endSec, offset);
+ }
+ }
+ }
+ return out;
+}
+
/**
* Move every word boundary back to the quietest frame in the `LOOKBACK_SEC`
* preceding it. Boundaries shared by two words snap identically (same input
* time), so the transcript stays gap-free where whisper made it gap-free.
- * Returns the words unchanged when there is no audio to measure against.
+ * With `speech` (the helper's VAD intervals), phrase edges are then anchored
+ * on the speech. Returns the words unchanged when there is no audio to
+ * measure against.
*/
export function snapWordBoundariesToAudio(
words: SttWordSegment[],
samples: Float32Array,
+ speech?: SttVadSegment[],
): SttWordSegment[] {
const envelope = rmsEnvelope(samples);
if (envelope.length === 0) return words;
@@ -89,7 +151,7 @@ export function snapWordBoundariesToAudio(
return bestFrame * FRAME_SEC;
};
- return words.map((w) => {
+ const snapped = words.map((w) => {
const startSec = snap(w.startSec);
return {
...w,
@@ -97,4 +159,5 @@ export function snapWordBoundariesToAudio(
endSec: Math.max(snap(w.endSec), startSec + MIN_WORD_SEC),
};
});
+ return speech ? anchorOnSpeech(snapped, speech) : snapped;
}
diff --git a/electron/stt/whisperServer.test.ts b/electron/stt/whisperServer.test.ts
index 314926e58..1d4c3e41b 100644
--- a/electron/stt/whisperServer.test.ts
+++ b/electron/stt/whisperServer.test.ts
@@ -350,6 +350,34 @@ describe("WhisperServerManager", () => {
}
});
+ it("anchors a phrase's first word on the speech onset the helper reports", async () => {
+ const fakeJson = {
+ segments: [
+ {
+ text: " Salut",
+ start: 1.57,
+ end: 2.56,
+ words: [{ word: " Salut", start: 2.15, end: 2.56 }],
+ },
+ ],
+ speech: [{ start: 1.57, end: 2.56 }],
+ backend: "whispercpp-cpu",
+ };
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async () => new Response(JSON.stringify(fakeJson), { status: 200 })),
+ );
+ try {
+ const mgr = new WhisperServerManager();
+ (mgr as unknown as { process: unknown; port: number }).process = {};
+ (mgr as unknown as { process: unknown; port: number }).port = 9999;
+ const result = await mgr.transcribe({ samples: new Float32Array(16_000 * 3) });
+ expect(result.wordSegments[0].startSec).toBeCloseTo(1.57, 6);
+ } finally {
+ vi.unstubAllGlobals();
+ }
+ });
+
it("spawns whisper-stt-server with --model", async () => {
const fs = await import("node:fs/promises");
const { spawn } = await import("node:child_process");
diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.ts
index a9698674f..6ffa887f7 100644
--- a/electron/stt/whisperServer.ts
+++ b/electron/stt/whisperServer.ts
@@ -115,6 +115,8 @@ interface WhisperJsonResponse {
detected_language?: string;
backend?: string;
timing?: WhisperJsonTiming;
+ /** Silero VAD's speech intervals; absent when the helper runs without the VAD model. */
+ speech?: Array<{ start?: number | string; end?: number | string }>;
}
export class WhisperServerManager {
@@ -597,9 +599,14 @@ export class WhisperServerManager {
return { text, startSec, endSec: Math.max(endSec, startSec + 0.05) };
})
.filter((s) => s.text.length > 0);
- // whisper.cpp's DTW boundaries run ~80–150 ms behind the audio, which the
- // transcript editor turns into imprecise trims (see snapWordBoundaries.ts).
- // Re-anchor them on the same samples whisper was given.
+ const speech = json.speech?.map((s) => ({
+ startSec: this.toSec(s.start, 0),
+ endSec: this.toSec(s.end, 0),
+ }));
+ // whisper.cpp's DTW boundaries run ~80–150 ms behind the audio, and a
+ // phrase's first word up to 0.6 s, which the transcript editor turns into
+ // imprecise trims (see snapWordBoundaries.ts). Re-anchor them on the same
+ // samples whisper was given, and on the helper's speech intervals.
const wordSegments: SttWordSegment[] = snapWordBoundariesToAudio(
raw
.flatMap((seg) =>
@@ -613,6 +620,7 @@ export class WhisperServerManager {
)
.filter((w) => w.word.length > 0),
opts.samples,
+ speech,
);
const detectedLanguage = json.detected_language ?? json.language ?? "auto";
const backend = this.toBackend(json.backend);
diff --git a/scripts/test-whisper-stt.mjs b/scripts/test-whisper-stt.mjs
index 7e52bac2b..7cef820bb 100644
--- a/scripts/test-whisper-stt.mjs
+++ b/scripts/test-whisper-stt.mjs
@@ -19,6 +19,8 @@
// Env overrides:
// OPENSCREEN_WHISPER_SERVER_EXE helper binary (default: the staged one for this host)
// OPENSCREEN_WHISPER_MODEL GGML model (default: the userData cache location)
+// OPENSCREEN_VAD_MODEL Silero VAD model (default: next to the GGML model;
+// the speech checks run only when it is there, as in the app)
import { spawn, spawnSync } from "node:child_process";
import fs from "node:fs";
@@ -47,6 +49,9 @@ const BIN =
);
const MODEL = process.env.OPENSCREEN_WHISPER_MODEL ?? defaultModelPath();
+const VAD_MODEL =
+ process.env.OPENSCREEN_VAD_MODEL ?? path.join(path.dirname(MODEL), "ggml-silero-v6.2.0.bin");
+const WITH_VAD = fs.existsSync(VAD_MODEL);
const LANGUAGE = argOf("--language") ?? "auto";
const WAV_ARG = argOf("--wav");
const REF_ARG = argOf("--ref");
@@ -208,7 +213,8 @@ async function main() {
`${meta.durationSec.toFixed(2)}s`,
);
console.log(`helper : ${BIN}`);
- console.log(`model : ${MODEL}\n`);
+ console.log(`model : ${MODEL}`);
+ console.log(`vad : ${WITH_VAD ? VAD_MODEL : "(none, speech checks skipped)"}\n`);
const port = 20100 + Math.floor(process.pid % 400);
const child = spawn(
@@ -222,6 +228,7 @@ async function main() {
"127.0.0.1",
"--threads",
String(Math.max(1, os.cpus().length)),
+ ...(WITH_VAD ? ["--vad-model", VAD_MODEL] : []),
],
{ stdio: ["ignore", "pipe", "pipe"] },
);
@@ -320,6 +327,24 @@ async function main() {
"word starts are monotonic non-decreasing",
);
+ // With the VAD model, whisper decodes only the speech and every time has to
+ // be mapped back onto the clip. whisper.cpp's own VAD mapped segment times
+ // but not word times, which put words seconds early (13 s into a 25 s clip)
+ // while every check above still passed. A word starting in what the VAD
+ // called silence is that bug; 0.15 s covers the 0.1 s tail kept past each stretch.
+ if (WITH_VAD) {
+ const speech = json.speech ?? [];
+ check(Array.isArray(json.speech), "response carries the speech intervals", `${speech.length}`);
+ const stray = words.filter(
+ (w) => !speech.some((s) => w.start >= s.start - 0.15 && w.start <= s.end + 0.15),
+ );
+ check(
+ stray.length === 0,
+ "every word starts inside a stretch of speech",
+ stray.map((w) => `${JSON.stringify(w.word)}@${w.start}`).join(", "),
+ );
+ }
+
// NOT a failure. whisper.cpp gives the last word of a segment the segment's
// own t1 as its end, while the word's DTW start runs 80–150 ms late — so a
// token emitted near the boundary (typically standalone punctuation) can land
diff --git a/src/lib/captioning/transcribe.ts b/src/lib/captioning/transcribe.ts
index 7b8473608..82aaad14c 100644
--- a/src/lib/captioning/transcribe.ts
+++ b/src/lib/captioning/transcribe.ts
@@ -128,13 +128,12 @@ function runTranscription(
options?.signal?.addEventListener("abort", onAbort, { once: true });
const forcedLanguage =
options?.language && options.language !== "auto" ? options.language : undefined;
- // ponytail: word timestamps come back already absolute from whisper.cpp
- // because its built-in Silero VAD (started on the server with
- // `--vad --vad-model`) splits audio into speech regions *before* the ASR
- // decoder runs and offsets each region's timestamps to its position in the
- // original audio. No trim + offset math here on purpose: an earlier
- // iteration trimmed leading silence with a peak detector and got false
- // positives on quiet music intros / room tone. VAD or nothing.
+ // ponytail: word timestamps come back already absolute. The helper cuts the
+ // silence out itself (Silero VAD, `--vad-model`) and maps every word back
+ // onto the original audio: whisper.cpp's own VAD maps segment times but not
+ // word times. No trim + offset math here on purpose: an earlier iteration
+ // trimmed leading silence with a peak detector and got false positives on
+ // quiet music intros / room tone. VAD or nothing.
return api
.transcribe({ ...payload, language: forcedLanguage })
.then((result) => {
diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md
index d2a499302..026350743 100644
--- a/technical-documentation/architecture/transcription-and-captions.md
+++ b/technical-documentation/architecture/transcription-and-captions.md
@@ -18,7 +18,7 @@ flowchart LR
A -- "extract mono 16 kHz" --> B["transcribeAsset<br/>(src/lib/ai-edition/document/transcribe.ts)"]
B -- "IPC: Float32Array + language" --> C["SttManager<br/>(electron/stt/index.ts)"]
C -- "POST /inference (WAV)" --> D["whisper-stt-server<br/>(electron/native/whisper-stt/)"]
- D -- "whisper_full() + DTW" --> E["SttTranscribeResponse<br/>(segments + wordSegments)"]
+ D -- "VAD cut + whisper_full() + DTW" --> E["SttTranscribeResponse<br/>(segments + wordSegments)"]
E --> C
C -- "IPC return" --> B
B --> F["AxcutTranscript<br/>on document.transcripts[]"]
@@ -186,6 +186,23 @@ it verbatim in the response.
### Word-level alignment (DTW token timestamps)
+0. **Speech only** — when the Silero VAD model is on disk (`ggml-silero-v6.2.0.bin`,
+ downloaded next to the whisper model), the helper cuts the silence out of
+ the upload before whisper sees it, the way whisper.cpp's own VAD does: each
+ speech stretch plus 0.1 s of tail, 0.1 s of silence between stretches. It
+ keeps the map, and every time below goes back through it onto the upload's
+ clock. The stretches are returned as `speech` for step 5.
+ > **Not whisper_full's `vad` param.** whisper.cpp 1.9.1 maps its *segment*
+ > times back onto the original audio but not its token times, `t_dtw`
+ > included, so every word came back early by all the silence removed before
+ > it: 13 s into a 25 s clip. That shipped in 2.0.0-rc.1, and the harness
+ > missed it (times were monotonic and inside the clip, just wrong); it now
+ > checks that every word starts inside a stretch of speech.
+ >
+ > **The VAD runs on the CPU, always.** Asked for the GPU on Vulkan, 1.9.1
+ > puts the VAD weights in a Vulkan buffer, finds no GPU for the VAD's own
+ > backend and aborts in ggml (`0xC0000409`), so `SttManager` relaunched the
+ > helper with `--cpu` and every transcription lost its GPU.
1. **Decode** — `whisper_full()` runs with the default sampling parameters
(`WHISPER_SAMPLING_GREEDY`), returning phrase segments plus a per-token
array.
@@ -214,6 +231,15 @@ it verbatim in the response.
the quietest frame *is* the reported one and nothing moves. Boundaries
past the end of the decoded audio are left untouched.
+ The edges of a phrase are past that reach. Its first word, DTW reports
+ 0.1–0.6 s after the speech starts (measured against the VAD on a real
+ French take); its last word ends on whisper's segment end, which can stop
+ short of the speech. With `speech`, the first real word of each stretch
+ starts on the stretch's onset and its last real word ends on its offset,
+ punctuation between collapsing onto the edge. `MAX_ANCHOR_SEC` is that
+ step's knob: a word further off than 1 s is more likely the neighbour of
+ one whisper dropped, and is left alone.
+
> Why this matters beyond caption timing: the transcript editor turns a word
> selection into a trim of exactly `[firstWord.startSec, lastWord.endSec]`,
> so a late boundary leaves the attack of the first removed word audible and
@@ -239,9 +265,11 @@ linked above).
### Model
-The single shipped artifact is `ggml-small-q8_0.bin` from
+The recogniser is `ggml-small-q8_0.bin` from
`ggerganov/whisper.cpp` on HuggingFace: Whisper `small`, multilingual (~99
-languages), q8_0 quantised, ~264 MB. Precision is baked into the GGML file —
+languages), q8_0 quantised, ~264 MB. Beside it, the same code path fetches
+Silero VAD v6.2.0 (`ggml-org/whisper-vad`, 0.9 MB) for step 0 of the
+alignment. Precision is baked into the GGML file —
there is no runtime `--int8` flag. `electron/stt/modelManager.ts` downloads
the file once into the user-data cache and writes it through an atomic
`.partial` rename, so a half-downloaded file can never be picked up as a
@@ -270,13 +298,13 @@ and the digest together.
| Module | Role |
|---|---|
| `electron/stt/whisperServer.ts` | Server lifecycle; `POST /inference` client; verbose_json parser. |
-| `electron/stt/snapWordBoundaries.ts` | Re-anchors DTW word boundaries on the audio's RMS envelope (see step 5 above). |
+| `electron/stt/snapWordBoundaries.ts` | Re-anchors DTW word boundaries on the audio's RMS envelope, and phrase edges on the VAD's speech (see step 5 above). |
| `electron/stt/wav.ts` | WAV write + temp-file cleanup helpers. |
| `electron/stt/gpuDetector.ts` | Per-platform binary resolver (no GPU probing). |
| `electron/stt/modelManager.ts` | Single GGML file download, SHA-256 verify, atomic write. |
| `electron/stt/transcriptionContract.ts` | Shared IPC types (`SttBackend`, `SttWordSegment`, `SttPhraseSegment`, `SttStatusEvent`). |
| `electron/stt/index.ts` | `SttManager` — IPC entry point; wires the pieces together. |
-| `electron/native/whisper-stt/src/main.cpp` | httplib HTTP server; calls `whisper_full()` with DTW; reports the device it bound via `ggml_backend_dev_name()`. |
+| `electron/native/whisper-stt/src/main.cpp` | httplib HTTP server; cuts the silence out with Silero VAD and maps times back (step 0); calls `whisper_full()` with DTW; reports the device it bound via `ggml_backend_dev_name()`. |
| `electron/native/whisper-stt/CMakeLists.txt` | Pulls whisper.cpp via FetchContent; enables Metal (macOS arm64), Vulkan (Windows/Linux x64), CPU fallback everywhere; static backend linking into `whisper.dll`/`ggml.dll`. |
The helper is one executable per platform; backends are baked in at build
@@ -736,9 +764,8 @@ it deletes data
- **No C++ unit tests.** The WAV reader and the DTW-inactive guardrail
in `electron/native/whisper-stt/src/main.cpp` are exercised only at
runtime.
-- **Caption sync fine-tuning.** DTW `t_dtw` is a *word-end* (the moment
- the ASR committed the token), so a caption can look a touch late on
- screen for the first second of a word. A future tweak could blend
- with the previous word's `t_dtw` for the start bound or mix with
- `token.t0`; record the final choice here once verified against real
- recordings.
+- **Word timing inside a phrase.** Phrase edges sit on the VAD (step 5), but
+ a word in the middle of continuous speech still starts on its DTW time
+ pulled back by at most 150 ms of RMS lookback. A dedicated forced aligner
+ is the upgrade path (issue #626); measure against the VAD-anchored edges
+ before adopting one.