Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(stt): address CodeRabbit review feedback on Silero VAD pre-pass
CyberSparkxc8ffaee6f93a
diff --git a/electron/native/whisper-stt/src/main.cpp b/electron/native/whisper-stt/src/main.cppindex 83a53f086..f287eed2e 100644--- a/electron/native/whisper-stt/src/main.cpp+++ b/electron/native/whisper-stt/src/main.cpp@@ -126,8 +126,16 @@ bool read_wav_pcm16(const std::string& path, std::vector<float>& pcm,if (f.gcount() != 4) break;const uint32_t chunk_size = read_u32();if (std::memcmp(chunk_tag, "fmt ", 4) == 0) {+ if (chunk_size < 16) {+ log("invalid fmt chunk size: " + std::to_string(chunk_size));+ return false;+ }fmt_format = read_u16();fmt_channels = read_u16();+ if (fmt_channels == 0) {+ log("invalid fmt_channels: 0");+ return false;+ }fmt_sample_rate = read_u32();(void)read_u32();(void)read_u16();@@ -136,9 +144,10 @@ bool read_wav_pcm16(const std::string& path, std::vector<float>& pcm,if (fmt_extra) f.seekg(fmt_extra, std::ios::cur);got_fmt = true;} else if (std::memcmp(chunk_tag, "data", 4) == 0) {- if (!got_fmt || fmt_format != 1 || fmt_bits != 16) {+ if (!got_fmt || fmt_format != 1 || fmt_bits != 16 || fmt_channels == 0) {log("expected PCM16, got format=" + std::to_string(fmt_format) +- " bits=" + std::to_string(fmt_bits));+ " bits=" + std::to_string(fmt_bits) ++ " channels=" + std::to_string(fmt_channels));return false;}sample_rate_out = static_cast<int>(fmt_sample_rate);@@ -328,8 +337,12 @@ int main(int argc, char** argv) {// GET / — readiness probe. The Node wrapper polls this until 200 to know// the model is loaded and the GPU is bound (a Vulkan/D3D driver bug can// make whisper_init succeed but the first /inference still segfault).- svr.Get("/", [](const httplib::Request&, httplib::Response& res) {- res.set_content("ok", "text/plain");+ svr.Get("/", [&](const httplib::Request&, httplib::Response& res) {+ nlohmann::json probe = {+ {"status", "ok"},+ {"vad", vctx != nullptr}+ };+ res.set_content(probe.dump(), "application/json");});// POST /inference — multipart form with `file` (WAV) + `language` + `response_format`.@@ -577,7 +590,7 @@ int main(int argc, char** argv) {// Runs Silero VAD segmentation and returns speech intervals [start, end] in seconds.svr.Post("/vad", [&](const httplib::Request& req, httplib::Response& res) {if (!vctx) {- res.status = 400;+ res.status = 503;res.set_content(R"({"error":"VAD model was not loaded on server startup"})", "application/json");return;}@@ -631,8 +644,8 @@ int main(int argc, char** argv) {const float t0 = whisper_vad_segments_get_segment_t0(vad_segments, i);const float t1 = whisper_vad_segments_get_segment_t1(vad_segments, i);reply["segments"].push_back({- {"start", static_cast<double>(t0)},- {"end", static_cast<double>(t1)}+ {"start", static_cast<double>(t0) / 100.0},+ {"end", static_cast<double>(t1) / 100.0}});}whisper_vad_free_segments(vad_segments);diff --git a/electron/stt/index.ts b/electron/stt/index.tsindex e463baa16..656d7bf6d 100644--- a/electron/stt/index.ts+++ b/electron/stt/index.ts@@ -3,15 +3,16 @@ import { app, type IpcMain } from "electron";import { planChunks } from "./chunking";import { extractMono16kPcm } from "./extractAudio";import { ensureModels, modelPaths } from "./modelManager";-import type {- SttPhraseSegment,- SttStatusEvent,- SttTiming,- SttTranscribeRequest,- SttTranscribeResponse,- SttVadResponse,- SttVadSegment,- SttWordSegment,+import {+ STT_VAD_UNAVAILABLE,+ type SttPhraseSegment,+ type SttStatusEvent,+ type SttTiming,+ type SttTranscribeRequest,+ type SttTranscribeResponse,+ type SttVadResponse,+ type SttVadSegment,+ type SttWordSegment,} from "./transcriptionContract";import { WhisperServerManager } from "./whisperServer";@@ -443,12 +444,20 @@ export class SttManager {await this.server.shutdown();}+ /** True when the helper reported Silero VAD loaded and ready. */+ isVadAvailable(): boolean {+ return this.server.status.vadAvailable;+ }+/*** Run Voice Activity Detection (Silero VAD) to detect speech segments in samples.*/async detectSpeech(samples: Float32Array): Promise<SttVadSegment[]> {if (this.shuttingDown) throw cancelledError();await this.init();+ if (!this.server.status.vadAvailable) {+ throw new Error(STT_VAD_UNAVAILABLE);+ }return this.server.detectVadSegments({ samples });}}diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.tsindex b990ce9cf..c9b7d4850 100644--- a/electron/stt/transcriptionContract.ts+++ b/electron/stt/transcriptionContract.ts@@ -141,6 +141,7 @@ export interface SttTranscribeRequest {* a literal typed twice is a fallback that silently stops working.*/export const STT_NATIVE_EXTRACTION_UNAVAILABLE = "stt:native-extraction-unavailable";+export const STT_VAD_UNAVAILABLE = "stt:vad-unavailable";/** IPC response: main → renderer. */export interface SttTranscribeResponse {diff --git a/electron/stt/whisperServer.test.ts b/electron/stt/whisperServer.test.tsindex a79b93093..bcd035733 100644--- a/electron/stt/whisperServer.test.ts+++ b/electron/stt/whisperServer.test.ts@@ -3,8 +3,9 @@ import { mkdtemp, rm, stat, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import path from "node:path";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { STT_VAD_UNAVAILABLE } from "./transcriptionContract";import { writeSamplesAsWav } from "./wav";-import { WhisperServerManager } from "./whisperServer";+import { MAX_VAD_CHUNK_SAMPLES, mergeVadIntervals, WhisperServerManager } from "./whisperServer";vi.mock("node:child_process", async (importOriginal) => {// `node:child_process` is CJS, so the ESM namespace also carries a `default`@@ -680,7 +681,9 @@ describe("WhisperServerManager", () => {const fs = await import("node:fs/promises");const { spawn } = await import("node:child_process");const dir = await mkdtemp(path.join(tmpdir(), "whisper-vad-args-"));+ const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform");try {+ Object.defineProperty(process, "platform", { value: "linux", configurable: true });const modelPath = path.join(dir, "ggml-small-q8_0.bin");const vadModelPath = path.join(dir, "ggml-silero-v6.2.0.bin");const fakeBinaryPath = path.join(dir, "whisper-stt-server");@@ -698,7 +701,13 @@ describe("WhisperServerManager", () => {}),});vi.mocked(spawn).mockImplementationOnce(() => child as never);- vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true } as Response));+ vi.stubGlobal(+ "fetch",+ vi.fn().mockResolvedValue({+ ok: true,+ json: async () => ({ status: "ok", vad: true }),+ } as unknown as Response),+ );const mgr = new WhisperServerManager();await mgr.start({@@ -715,6 +724,7 @@ describe("WhisperServerManager", () => {expect(args[vadIdx + 1]).toBe(vadModelPath);await mgr.stop();} finally {+ if (originalPlatform) Object.defineProperty(process, "platform", originalPlatform);vi.unstubAllGlobals();await rm(dir, { recursive: true, force: true });}@@ -733,8 +743,10 @@ describe("WhisperServerManager", () => {);try {const mgr = new WhisperServerManager();- (mgr as unknown as { process: unknown; port: number }).process = {};- (mgr as unknown as { process: unknown; port: number }).port = 9999;+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {};+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =+ true;const segments = await mgr.detectVadSegments({ samples: new Float32Array(1600) });expect(segments).toEqual([@@ -753,8 +765,10 @@ describe("WhisperServerManager", () => {);try {const mgr = new WhisperServerManager();- (mgr as unknown as { process: unknown; port: number }).process = {};- (mgr as unknown as { process: unknown; port: number }).port = 9999;+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {};+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =+ true;await expect(mgr.detectVadSegments({ samples: new Float32Array(1600) })).rejects.toThrow(/whisper-stt-server \/vad HTTP 400/,@@ -763,4 +777,98 @@ describe("WhisperServerManager", () => {vi.unstubAllGlobals();}});++ it("handles malformed WAV /vad HTTP 400 without crashing helper availability", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn(+ async () => new Response(JSON.stringify({ error: "failed to parse WAV" }), { status: 400 }),+ ),+ );+ try {+ const mgr = new WhisperServerManager();+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {+ pid: 999,+ };+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =+ true;++ await expect(mgr.detectVadSegments({ samples: new Float32Array(100) })).rejects.toThrow(+ /whisper-stt-server \/vad HTTP 400/,+ );+ expect(mgr.status.running).toBe(true);+ } finally {+ vi.unstubAllGlobals();+ }+ });++ it("rejects detectVadSegments when VAD is unavailable", async () => {+ const mgr = new WhisperServerManager();+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {};+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =+ false;++ await expect(mgr.detectVadSegments({ samples: new Float32Array(1600) })).rejects.toThrow(+ STT_VAD_UNAVAILABLE,+ );+ });++ describe("mergeVadIntervals", () => {+ it("returns empty array for empty input", () => {+ expect(mergeVadIntervals([])).toEqual([]);+ });++ it("returns single interval as-is", () => {+ expect(mergeVadIntervals([{ startSec: 1, endSec: 2 }])).toEqual([{ startSec: 1, endSec: 2 }]);+ });++ it("merges overlapping and adjacent intervals", () => {+ const input = [+ { startSec: 1, endSec: 3 },+ { startSec: 2.5, endSec: 5 },+ { startSec: 5, endSec: 6.2 },+ { startSec: 8, endSec: 10 },+ ];+ expect(mergeVadIntervals(input)).toEqual([+ { startSec: 1, endSec: 6.2 },+ { startSec: 8, endSec: 10 },+ ]);+ });++ it("drops inverted intervals where endSec <= startSec", () => {+ const input = [+ { startSec: 2, endSec: 1 },+ { startSec: 3, endSec: 4 },+ ];+ expect(mergeVadIntervals(input)).toEqual([{ startSec: 3, endSec: 4 }]);+ });+ });++ it("chunks long audio across MAX_VAD_CHUNK_SAMPLES and offsets timestamps", async () => {+ const fetchMock = vi.fn(async () => {+ const callIndex = fetchMock.mock.calls.length - 1;+ const segments = callIndex === 0 ? [{ start: 10, end: 180 }] : [{ start: 0, end: 20 }];+ return new Response(JSON.stringify({ segments }), { status: 200 });+ });+ vi.stubGlobal("fetch", fetchMock);+ try {+ const mgr = new WhisperServerManager();+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {};+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;+ (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =+ true;++ // Total duration = 200 seconds @ 16 kHz = 3,200,000 samples (> MAX_VAD_CHUNK_SAMPLES = 2,880,000)+ const samples = new Float32Array(MAX_VAD_CHUNK_SAMPLES + 16_000 * 20);+ const segments = await mgr.detectVadSegments({ samples });++ expect(fetchMock).toHaveBeenCalledTimes(2);+ // 180s in chunk 0 + 20s in chunk 1 with offset 180s -> merged [10, 200]+ expect(segments).toEqual([{ startSec: 10, endSec: 200 }]);+ } finally {+ vi.unstubAllGlobals();+ }+ });});diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.tsindex 1673717f6..fc58aec51 100644--- a/electron/stt/whisperServer.ts+++ b/electron/stt/whisperServer.ts@@ -8,12 +8,13 @@ import type { Readable } from "node:stream";import { resolveBinaryPath } from "./gpuDetector";import { snapWordBoundariesToAudio } from "./snapWordBoundaries";-import type {- SttBackend,- SttPhraseSegment,- SttTiming,- SttVadSegment,- SttWordSegment,+import {+ STT_VAD_UNAVAILABLE,+ type SttBackend,+ type SttPhraseSegment,+ type SttTiming,+ type SttVadSegment,+ type SttWordSegment,} from "./transcriptionContract";import { cleanupWav, writeSamplesAsWav } from "./wav";@@ -75,6 +76,7 @@ export interface WhisperServerStatus {backend: SttBackend | null;startedAtMs: number | null;lastError: string | null;+ vadAvailable: boolean;}/** Per-word entry inside a whisper-stt-server `/inference` JSON segment. */@@ -123,7 +125,11 @@ export class WhisperServerManager {private lastError: string | null = null;private startedAtMs: number | null = null;private inFlight: Promise<unknown> = Promise.resolve();- private starting: Promise<{ port: number; backend: SttBackend }> | null = null;+ private starting: Promise<{+ port: number;+ backend: SttBackend;+ vadAvailable: boolean;+ }> | null = null;/** Buffered stderr from the helper; surfaced on shutdown + poll failures. */private stderrTail = "";@@ -148,12 +154,12 @@ export class WhisperServerManager {});}- /** Check the server's HTTP root for a 200; resolves once responsive. */+ /** Check the server's HTTP root for a 200; resolves once responsive. Returns VAD availability. */private static async pollUntilReady(baseUrl: string,timeoutMs = 30_000,shouldContinue: () => boolean = () => true,- ): Promise<void> {+ ): Promise<{ vad: boolean }> {const deadline = Date.now() + timeoutMs;while (Date.now() < deadline) {if (!shouldContinue()) throw new Error("whisper-stt-server exited before readiness");@@ -164,7 +170,10 @@ export class WhisperServerManager {);try {const res = await fetch(baseUrl, { method: "GET", signal: controller.signal });- if (res.ok) return;+ if (res.ok) {+ const body = typeof res.json === "function" ? await res.json().catch(() => null) : null;+ return { vad: Boolean((body as { vad?: boolean } | null)?.vad) };+ }} catch {// not up yet} finally {@@ -199,6 +208,8 @@ export class WhisperServerManager {if (log) console.error(`[stt] ${message}`);}+ private vadAvailable = false;+/** True when a process is alive and a model is loaded. */get status(): WhisperServerStatus {return {@@ -208,15 +219,22 @@ export class WhisperServerManager {backend: this.backend,startedAtMs: this.startedAtMs,lastError: this.lastError,+ vadAvailable: this.vadAvailable,};}+ get isVadAvailable(): boolean {+ return this.vadAvailable;+ }+/*** Spawn the helper if not running and return once `/` returns 200. Idempotent —* if a server is already up we just return its port so the caller never pays* the cold-start cost twice.*/- async start(options: WhisperServerStartOptions): Promise<{ port: number; backend: SttBackend }> {+ async start(+ options: WhisperServerStartOptions,+ ): Promise<{ port: number; backend: SttBackend; vadAvailable: boolean }> {if (this.starting) return this.starting;this.starting = this.startImpl(options).finally(() => {this.starting = null;@@ -226,12 +244,16 @@ export class WhisperServerManager {private async startImpl(options: WhisperServerStartOptions,- ): Promise<{ port: number; backend: SttBackend }> {+ ): Promise<{ port: number; backend: SttBackend; vadAvailable: boolean }> {if (this.shuttingDown) {throw new Error("whisper-stt-server manager is shutting down");}if (this.process && this.port) {- return { port: this.port, backend: this.backend ?? options.backend ?? "whispercpp-cpu" };+ return {+ port: this.port,+ backend: this.backend ?? options.backend ?? "whispercpp-cpu",+ vadAvailable: this.vadAvailable,+ };}const resolved = options.binaryPath@@ -259,7 +281,9 @@ export class WhisperServerManager {throw new Error(`Whisper GGML model not found at ${options.modelPath}`);}- const launch = async (forceCpu: boolean): Promise<{ port: number; backend: SttBackend }> => {+ const launch = async (+ forceCpu: boolean,+ ): Promise<{ port: number; backend: SttBackend; vadAvailable: boolean }> => {const port = await WhisperServerManager.pickFreePort();if (this.shuttingDown) {throw new Error("whisper-stt-server manager is shutting down");@@ -307,6 +331,7 @@ export class WhisperServerManager {this.process = null;this.port = null;this.startedAtMs = null;+ this.vadAvailable = false;}});child.once("error", (err) => {@@ -322,6 +347,7 @@ export class WhisperServerManager {this.process = null;this.port = null;this.startedAtMs = null;+ this.vadAvailable = false;}});@@ -336,8 +362,9 @@ export class WhisperServerManager {});child.once("error", reject);});+ let readyInfo: { vad: boolean };try {- await Promise.race([+ readyInfo = await Promise.race([WhisperServerManager.pollUntilReady(`http://127.0.0.1:${port}`,30_000,@@ -357,7 +384,8 @@ export class WhisperServerManager {this.recordError(message, { log: !alreadyLogged });throw new Error(message);}- return { port, backend: activeBackend };+ this.vadAvailable = Boolean(readyInfo?.vad);+ return { port, backend: activeBackend, vadAvailable: this.vadAvailable };};try {@@ -383,6 +411,7 @@ export class WhisperServerManager {/** Send SIGTERM and wait for the helper to exit. Resolves even if it was already down. */async stop(): Promise<void> {+ this.vadAvailable = false;if (!this.process) {this.port = null;this.startedAtMs = null;@@ -601,9 +630,8 @@ export class WhisperServerManager {return task;}- private async detectVadSegmentsImpl(opts: { samples: Float32Array }): Promise<SttVadSegment[]> {- await this.ensureReady();- const wavPath = await writeSamplesAsWav(opts.samples);+ private async detectVadChunk(samples: Float32Array): Promise<SttVadSegment[]> {+ const wavPath = await writeSamplesAsWav(samples);try {const url = `${this.baseUrl()}/vad`;const form = new FormData();@@ -654,4 +682,57 @@ export class WhisperServerManager {await cleanupWav(wavPath);}}++ private async detectVadSegmentsImpl(opts: { samples: Float32Array }): Promise<SttVadSegment[]> {+ await this.ensureReady();+ if (!this.vadAvailable) {+ throw new Error(STT_VAD_UNAVAILABLE);+ }+ if (opts.samples.length === 0) return [];++ const allSegments: SttVadSegment[] = [];+ const totalSamples = opts.samples.length;+ for (let offset = 0; offset < totalSamples; offset += MAX_VAD_CHUNK_SAMPLES) {+ const chunkSamples = opts.samples.subarray(+ offset,+ Math.min(totalSamples, offset + MAX_VAD_CHUNK_SAMPLES),+ );+ const chunkSegments = await this.detectVadChunk(chunkSamples);+ const offsetSec = offset / 16_000;+ for (const seg of chunkSegments) {+ allSegments.push({+ startSec: Math.round((seg.startSec + offsetSec) * 1000) / 1000,+ endSec: Math.round((seg.endSec + offsetSec) * 1000) / 1000,+ });+ }+ }++ return mergeVadIntervals(allSegments);+ }+}++/**+ * Silero VAD processes audio quickly, but buffering hours of 16 kHz audio as+ * single WAV blobs can exhaust Node memory. We bound individual VAD passes to+ * chunks of at most 3 minutes (180s = 2,880,000 samples @ 16 kHz), keeping+ * memory under ~6 MB per WAV, and merge the resulting intervals.+ */+export const MAX_VAD_CHUNK_SAMPLES = 16_000 * 180;++/**+ * Merges contiguous or overlapping speech segments into non-overlapping intervals.+ */+export function mergeVadIntervals(segments: SttVadSegment[]): SttVadSegment[] {+ if (segments.length <= 1) return segments;+ const merged: SttVadSegment[] = [];+ for (const seg of segments) {+ if (seg.endSec <= seg.startSec) continue;+ const last = merged[merged.length - 1];+ if (last && seg.startSec <= last.endSec) {+ last.endSec = Math.max(last.endSec, seg.endSec);+ } else {+ merged.push({ startSec: seg.startSec, endSec: seg.endSec });+ }+ }+ return merged;}