Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(stt): address CodeRabbit review feedback on Silero VAD pre-pass

CyberSparkxc8ffaee6f93a
diff --git a/electron/native/whisper-stt/src/main.cpp b/electron/native/whisper-stt/src/main.cpp
index 83a53f086..f287eed2e 100644
--- a/electron/native/whisper-stt/src/main.cpp
+++ b/electron/native/whisper-stt/src/main.cpp
@@ -126,8 +126,16 @@ bool read_wav_pcm16(const std::string& path, std::vector<float>& pcm,
if (f.gcount() != 4) break;
const uint32_t chunk_size = read_u32();
if (std::memcmp(chunk_tag, "fmt ", 4) == 0) {
+ if (chunk_size < 16) {
+ log("invalid fmt chunk size: " + std::to_string(chunk_size));
+ return false;
+ }
fmt_format = read_u16();
fmt_channels = read_u16();
+ if (fmt_channels == 0) {
+ log("invalid fmt_channels: 0");
+ return false;
+ }
fmt_sample_rate = read_u32();
(void)read_u32();
(void)read_u16();
@@ -136,9 +144,10 @@ bool read_wav_pcm16(const std::string& path, std::vector<float>& pcm,
if (fmt_extra) f.seekg(fmt_extra, std::ios::cur);
got_fmt = true;
} else if (std::memcmp(chunk_tag, "data", 4) == 0) {
- if (!got_fmt || fmt_format != 1 || fmt_bits != 16) {
+ if (!got_fmt || fmt_format != 1 || fmt_bits != 16 || fmt_channels == 0) {
log("expected PCM16, got format=" + std::to_string(fmt_format) +
- " bits=" + std::to_string(fmt_bits));
+ " bits=" + std::to_string(fmt_bits) +
+ " channels=" + std::to_string(fmt_channels));
return false;
}
sample_rate_out = static_cast<int>(fmt_sample_rate);
@@ -328,8 +337,12 @@ int main(int argc, char** argv) {
// GET / — readiness probe. The Node wrapper polls this until 200 to know
// the model is loaded and the GPU is bound (a Vulkan/D3D driver bug can
// make whisper_init succeed but the first /inference still segfault).
- svr.Get("/", [](const httplib::Request&, httplib::Response& res) {
- res.set_content("ok", "text/plain");
+ svr.Get("/", [&](const httplib::Request&, httplib::Response& res) {
+ nlohmann::json probe = {
+ {"status", "ok"},
+ {"vad", vctx != nullptr}
+ };
+ res.set_content(probe.dump(), "application/json");
});
// POST /inference — multipart form with `file` (WAV) + `language` + `response_format`.
@@ -577,7 +590,7 @@ int main(int argc, char** argv) {
// Runs Silero VAD segmentation and returns speech intervals [start, end] in seconds.
svr.Post("/vad", [&](const httplib::Request& req, httplib::Response& res) {
if (!vctx) {
- res.status = 400;
+ res.status = 503;
res.set_content(R"({"error":"VAD model was not loaded on server startup"})", "application/json");
return;
}
@@ -631,8 +644,8 @@ int main(int argc, char** argv) {
const float t0 = whisper_vad_segments_get_segment_t0(vad_segments, i);
const float t1 = whisper_vad_segments_get_segment_t1(vad_segments, i);
reply["segments"].push_back({
- {"start", static_cast<double>(t0)},
- {"end", static_cast<double>(t1)}
+ {"start", static_cast<double>(t0) / 100.0},
+ {"end", static_cast<double>(t1) / 100.0}
});
}
whisper_vad_free_segments(vad_segments);
diff --git a/electron/stt/index.ts b/electron/stt/index.ts
index e463baa16..656d7bf6d 100644
--- a/electron/stt/index.ts
+++ b/electron/stt/index.ts
@@ -3,15 +3,16 @@ import { app, type IpcMain } from "electron";
import { planChunks } from "./chunking";
import { extractMono16kPcm } from "./extractAudio";
import { ensureModels, modelPaths } from "./modelManager";
-import type {
- SttPhraseSegment,
- SttStatusEvent,
- SttTiming,
- SttTranscribeRequest,
- SttTranscribeResponse,
- SttVadResponse,
- SttVadSegment,
- SttWordSegment,
+import {
+ STT_VAD_UNAVAILABLE,
+ type SttPhraseSegment,
+ type SttStatusEvent,
+ type SttTiming,
+ type SttTranscribeRequest,
+ type SttTranscribeResponse,
+ type SttVadResponse,
+ type SttVadSegment,
+ type SttWordSegment,
} from "./transcriptionContract";
import { WhisperServerManager } from "./whisperServer";
@@ -443,12 +444,20 @@ export class SttManager {
await this.server.shutdown();
}
+ /** True when the helper reported Silero VAD loaded and ready. */
+ isVadAvailable(): boolean {
+ return this.server.status.vadAvailable;
+ }
+
/**
* Run Voice Activity Detection (Silero VAD) to detect speech segments in samples.
*/
async detectSpeech(samples: Float32Array): Promise<SttVadSegment[]> {
if (this.shuttingDown) throw cancelledError();
await this.init();
+ if (!this.server.status.vadAvailable) {
+ throw new Error(STT_VAD_UNAVAILABLE);
+ }
return this.server.detectVadSegments({ samples });
}
}
diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.ts
index b990ce9cf..c9b7d4850 100644
--- a/electron/stt/transcriptionContract.ts
+++ b/electron/stt/transcriptionContract.ts
@@ -141,6 +141,7 @@ export interface SttTranscribeRequest {
* a literal typed twice is a fallback that silently stops working.
*/
export const STT_NATIVE_EXTRACTION_UNAVAILABLE = "stt:native-extraction-unavailable";
+export const STT_VAD_UNAVAILABLE = "stt:vad-unavailable";
/** IPC response: main → renderer. */
export interface SttTranscribeResponse {
diff --git a/electron/stt/whisperServer.test.ts b/electron/stt/whisperServer.test.ts
index a79b93093..bcd035733 100644
--- a/electron/stt/whisperServer.test.ts
+++ b/electron/stt/whisperServer.test.ts
@@ -3,8 +3,9 @@ import { mkdtemp, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { STT_VAD_UNAVAILABLE } from "./transcriptionContract";
import { writeSamplesAsWav } from "./wav";
-import { WhisperServerManager } from "./whisperServer";
+import { MAX_VAD_CHUNK_SAMPLES, mergeVadIntervals, WhisperServerManager } from "./whisperServer";
vi.mock("node:child_process", async (importOriginal) => {
// `node:child_process` is CJS, so the ESM namespace also carries a `default`
@@ -680,7 +681,9 @@ describe("WhisperServerManager", () => {
const fs = await import("node:fs/promises");
const { spawn } = await import("node:child_process");
const dir = await mkdtemp(path.join(tmpdir(), "whisper-vad-args-"));
+ const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform");
try {
+ Object.defineProperty(process, "platform", { value: "linux", configurable: true });
const modelPath = path.join(dir, "ggml-small-q8_0.bin");
const vadModelPath = path.join(dir, "ggml-silero-v6.2.0.bin");
const fakeBinaryPath = path.join(dir, "whisper-stt-server");
@@ -698,7 +701,13 @@ describe("WhisperServerManager", () => {
}),
});
vi.mocked(spawn).mockImplementationOnce(() => child as never);
- vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true } as Response));
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockResolvedValue({
+ ok: true,
+ json: async () => ({ status: "ok", vad: true }),
+ } as unknown as Response),
+ );
const mgr = new WhisperServerManager();
await mgr.start({
@@ -715,6 +724,7 @@ describe("WhisperServerManager", () => {
expect(args[vadIdx + 1]).toBe(vadModelPath);
await mgr.stop();
} finally {
+ if (originalPlatform) Object.defineProperty(process, "platform", originalPlatform);
vi.unstubAllGlobals();
await rm(dir, { recursive: true, force: true });
}
@@ -733,8 +743,10 @@ describe("WhisperServerManager", () => {
);
try {
const mgr = new WhisperServerManager();
- (mgr as unknown as { process: unknown; port: number }).process = {};
- (mgr as unknown as { process: unknown; port: number }).port = 9999;
+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {};
+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;
+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =
+ true;
const segments = await mgr.detectVadSegments({ samples: new Float32Array(1600) });
expect(segments).toEqual([
@@ -753,8 +765,10 @@ describe("WhisperServerManager", () => {
);
try {
const mgr = new WhisperServerManager();
- (mgr as unknown as { process: unknown; port: number }).process = {};
- (mgr as unknown as { process: unknown; port: number }).port = 9999;
+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {};
+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;
+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =
+ true;
await expect(mgr.detectVadSegments({ samples: new Float32Array(1600) })).rejects.toThrow(
/whisper-stt-server \/vad HTTP 400/,
@@ -763,4 +777,98 @@ describe("WhisperServerManager", () => {
vi.unstubAllGlobals();
}
});
+
+ it("handles malformed WAV /vad HTTP 400 without crashing helper availability", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(
+ async () => new Response(JSON.stringify({ error: "failed to parse WAV" }), { status: 400 }),
+ ),
+ );
+ try {
+ const mgr = new WhisperServerManager();
+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {
+ pid: 999,
+ };
+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;
+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =
+ true;
+
+ await expect(mgr.detectVadSegments({ samples: new Float32Array(100) })).rejects.toThrow(
+ /whisper-stt-server \/vad HTTP 400/,
+ );
+ expect(mgr.status.running).toBe(true);
+ } finally {
+ vi.unstubAllGlobals();
+ }
+ });
+
+ it("rejects detectVadSegments when VAD is unavailable", async () => {
+ const mgr = new WhisperServerManager();
+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {};
+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;
+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =
+ false;
+
+ await expect(mgr.detectVadSegments({ samples: new Float32Array(1600) })).rejects.toThrow(
+ STT_VAD_UNAVAILABLE,
+ );
+ });
+
+ describe("mergeVadIntervals", () => {
+ it("returns empty array for empty input", () => {
+ expect(mergeVadIntervals([])).toEqual([]);
+ });
+
+ it("returns single interval as-is", () => {
+ expect(mergeVadIntervals([{ startSec: 1, endSec: 2 }])).toEqual([{ startSec: 1, endSec: 2 }]);
+ });
+
+ it("merges overlapping and adjacent intervals", () => {
+ const input = [
+ { startSec: 1, endSec: 3 },
+ { startSec: 2.5, endSec: 5 },
+ { startSec: 5, endSec: 6.2 },
+ { startSec: 8, endSec: 10 },
+ ];
+ expect(mergeVadIntervals(input)).toEqual([
+ { startSec: 1, endSec: 6.2 },
+ { startSec: 8, endSec: 10 },
+ ]);
+ });
+
+ it("drops inverted intervals where endSec <= startSec", () => {
+ const input = [
+ { startSec: 2, endSec: 1 },
+ { startSec: 3, endSec: 4 },
+ ];
+ expect(mergeVadIntervals(input)).toEqual([{ startSec: 3, endSec: 4 }]);
+ });
+ });
+
+ it("chunks long audio across MAX_VAD_CHUNK_SAMPLES and offsets timestamps", async () => {
+ const fetchMock = vi.fn(async () => {
+ const callIndex = fetchMock.mock.calls.length - 1;
+ const segments = callIndex === 0 ? [{ start: 10, end: 180 }] : [{ start: 0, end: 20 }];
+ return new Response(JSON.stringify({ segments }), { status: 200 });
+ });
+ vi.stubGlobal("fetch", fetchMock);
+ try {
+ const mgr = new WhisperServerManager();
+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {};
+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;
+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =
+ true;
+
+ // Total duration = 200 seconds @ 16 kHz = 3,200,000 samples (> MAX_VAD_CHUNK_SAMPLES = 2,880,000)
+ const samples = new Float32Array(MAX_VAD_CHUNK_SAMPLES + 16_000 * 20);
+ const segments = await mgr.detectVadSegments({ samples });
+
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ // 180s in chunk 0 + 20s in chunk 1 with offset 180s -> merged [10, 200]
+ expect(segments).toEqual([{ startSec: 10, endSec: 200 }]);
+ } finally {
+ vi.unstubAllGlobals();
+ }
+ });
});
diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.ts
index 1673717f6..fc58aec51 100644
--- a/electron/stt/whisperServer.ts
+++ b/electron/stt/whisperServer.ts
@@ -8,12 +8,13 @@ import type { Readable } from "node:stream";
import { resolveBinaryPath } from "./gpuDetector";
import { snapWordBoundariesToAudio } from "./snapWordBoundaries";
-import type {
- SttBackend,
- SttPhraseSegment,
- SttTiming,
- SttVadSegment,
- SttWordSegment,
+import {
+ STT_VAD_UNAVAILABLE,
+ type SttBackend,
+ type SttPhraseSegment,
+ type SttTiming,
+ type SttVadSegment,
+ type SttWordSegment,
} from "./transcriptionContract";
import { cleanupWav, writeSamplesAsWav } from "./wav";
@@ -75,6 +76,7 @@ export interface WhisperServerStatus {
backend: SttBackend | null;
startedAtMs: number | null;
lastError: string | null;
+ vadAvailable: boolean;
}
/** Per-word entry inside a whisper-stt-server `/inference` JSON segment. */
@@ -123,7 +125,11 @@ export class WhisperServerManager {
private lastError: string | null = null;
private startedAtMs: number | null = null;
private inFlight: Promise<unknown> = Promise.resolve();
- private starting: Promise<{ port: number; backend: SttBackend }> | null = null;
+ private starting: Promise<{
+ port: number;
+ backend: SttBackend;
+ vadAvailable: boolean;
+ }> | null = null;
/** Buffered stderr from the helper; surfaced on shutdown + poll failures. */
private stderrTail = "";
@@ -148,12 +154,12 @@ export class WhisperServerManager {
});
}
- /** Check the server's HTTP root for a 200; resolves once responsive. */
+ /** Check the server's HTTP root for a 200; resolves once responsive. Returns VAD availability. */
private static async pollUntilReady(
baseUrl: string,
timeoutMs = 30_000,
shouldContinue: () => boolean = () => true,
- ): Promise<void> {
+ ): Promise<{ vad: boolean }> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (!shouldContinue()) throw new Error("whisper-stt-server exited before readiness");
@@ -164,7 +170,10 @@ export class WhisperServerManager {
);
try {
const res = await fetch(baseUrl, { method: "GET", signal: controller.signal });
- if (res.ok) return;
+ if (res.ok) {
+ const body = typeof res.json === "function" ? await res.json().catch(() => null) : null;
+ return { vad: Boolean((body as { vad?: boolean } | null)?.vad) };
+ }
} catch {
// not up yet
} finally {
@@ -199,6 +208,8 @@ export class WhisperServerManager {
if (log) console.error(`[stt] ${message}`);
}
+ private vadAvailable = false;
+
/** True when a process is alive and a model is loaded. */
get status(): WhisperServerStatus {
return {
@@ -208,15 +219,22 @@ export class WhisperServerManager {
backend: this.backend,
startedAtMs: this.startedAtMs,
lastError: this.lastError,
+ vadAvailable: this.vadAvailable,
};
}
+ get isVadAvailable(): boolean {
+ return this.vadAvailable;
+ }
+
/**
* Spawn the helper if not running and return once `/` returns 200. Idempotent —
* if a server is already up we just return its port so the caller never pays
* the cold-start cost twice.
*/
- async start(options: WhisperServerStartOptions): Promise<{ port: number; backend: SttBackend }> {
+ async start(
+ options: WhisperServerStartOptions,
+ ): Promise<{ port: number; backend: SttBackend; vadAvailable: boolean }> {
if (this.starting) return this.starting;
this.starting = this.startImpl(options).finally(() => {
this.starting = null;
@@ -226,12 +244,16 @@ export class WhisperServerManager {
private async startImpl(
options: WhisperServerStartOptions,
- ): Promise<{ port: number; backend: SttBackend }> {
+ ): Promise<{ port: number; backend: SttBackend; vadAvailable: boolean }> {
if (this.shuttingDown) {
throw new Error("whisper-stt-server manager is shutting down");
}
if (this.process && this.port) {
- return { port: this.port, backend: this.backend ?? options.backend ?? "whispercpp-cpu" };
+ return {
+ port: this.port,
+ backend: this.backend ?? options.backend ?? "whispercpp-cpu",
+ vadAvailable: this.vadAvailable,
+ };
}
const resolved = options.binaryPath
@@ -259,7 +281,9 @@ export class WhisperServerManager {
throw new Error(`Whisper GGML model not found at ${options.modelPath}`);
}
- const launch = async (forceCpu: boolean): Promise<{ port: number; backend: SttBackend }> => {
+ const launch = async (
+ forceCpu: boolean,
+ ): Promise<{ port: number; backend: SttBackend; vadAvailable: boolean }> => {
const port = await WhisperServerManager.pickFreePort();
if (this.shuttingDown) {
throw new Error("whisper-stt-server manager is shutting down");
@@ -307,6 +331,7 @@ export class WhisperServerManager {
this.process = null;
this.port = null;
this.startedAtMs = null;
+ this.vadAvailable = false;
}
});
child.once("error", (err) => {
@@ -322,6 +347,7 @@ export class WhisperServerManager {
this.process = null;
this.port = null;
this.startedAtMs = null;
+ this.vadAvailable = false;
}
});
@@ -336,8 +362,9 @@ export class WhisperServerManager {
});
child.once("error", reject);
});
+ let readyInfo: { vad: boolean };
try {
- await Promise.race([
+ readyInfo = await Promise.race([
WhisperServerManager.pollUntilReady(
`http://127.0.0.1:${port}`,
30_000,
@@ -357,7 +384,8 @@ export class WhisperServerManager {
this.recordError(message, { log: !alreadyLogged });
throw new Error(message);
}
- return { port, backend: activeBackend };
+ this.vadAvailable = Boolean(readyInfo?.vad);
+ return { port, backend: activeBackend, vadAvailable: this.vadAvailable };
};
try {
@@ -383,6 +411,7 @@ export class WhisperServerManager {
/** Send SIGTERM and wait for the helper to exit. Resolves even if it was already down. */
async stop(): Promise<void> {
+ this.vadAvailable = false;
if (!this.process) {
this.port = null;
this.startedAtMs = null;
@@ -601,9 +630,8 @@ export class WhisperServerManager {
return task;
}
- private async detectVadSegmentsImpl(opts: { samples: Float32Array }): Promise<SttVadSegment[]> {
- await this.ensureReady();
- const wavPath = await writeSamplesAsWav(opts.samples);
+ private async detectVadChunk(samples: Float32Array): Promise<SttVadSegment[]> {
+ const wavPath = await writeSamplesAsWav(samples);
try {
const url = `${this.baseUrl()}/vad`;
const form = new FormData();
@@ -654,4 +682,57 @@ export class WhisperServerManager {
await cleanupWav(wavPath);
}
}
+
+ private async detectVadSegmentsImpl(opts: { samples: Float32Array }): Promise<SttVadSegment[]> {
+ await this.ensureReady();
+ if (!this.vadAvailable) {
+ throw new Error(STT_VAD_UNAVAILABLE);
+ }
+ if (opts.samples.length === 0) return [];
+
+ const allSegments: SttVadSegment[] = [];
+ const totalSamples = opts.samples.length;
+ for (let offset = 0; offset < totalSamples; offset += MAX_VAD_CHUNK_SAMPLES) {
+ const chunkSamples = opts.samples.subarray(
+ offset,
+ Math.min(totalSamples, offset + MAX_VAD_CHUNK_SAMPLES),
+ );
+ const chunkSegments = await this.detectVadChunk(chunkSamples);
+ const offsetSec = offset / 16_000;
+ for (const seg of chunkSegments) {
+ allSegments.push({
+ startSec: Math.round((seg.startSec + offsetSec) * 1000) / 1000,
+ endSec: Math.round((seg.endSec + offsetSec) * 1000) / 1000,
+ });
+ }
+ }
+
+ return mergeVadIntervals(allSegments);
+ }
+}
+
+/**
+ * Silero VAD processes audio quickly, but buffering hours of 16 kHz audio as
+ * single WAV blobs can exhaust Node memory. We bound individual VAD passes to
+ * chunks of at most 3 minutes (180s = 2,880,000 samples @ 16 kHz), keeping
+ * memory under ~6 MB per WAV, and merge the resulting intervals.
+ */
+export const MAX_VAD_CHUNK_SAMPLES = 16_000 * 180;
+
+/**
+ * Merges contiguous or overlapping speech segments into non-overlapping intervals.
+ */
+export function mergeVadIntervals(segments: SttVadSegment[]): SttVadSegment[] {
+ if (segments.length <= 1) return segments;
+ const merged: SttVadSegment[] = [];
+ for (const seg of segments) {
+ if (seg.endSec <= seg.startSec) continue;
+ const last = merged[merged.length - 1];
+ if (last && seg.startSec <= last.endSec) {
+ last.endSec = Math.max(last.endSec, seg.endSec);
+ } else {
+ merged.push({ startSec: seg.startSec, endSec: seg.endSec });
+ }
+ }
+ return merged;
}