Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
feat(stt): add cross-platform Silero VAD pre-pass via whisper.cpp
CyberSparkx53956cc4b9f1
diff --git a/electron/native/whisper-stt/src/main.cpp b/electron/native/whisper-stt/src/main.cppindex 5762de8fa..83a53f086 100644--- a/electron/native/whisper-stt/src/main.cpp+++ b/electron/native/whisper-stt/src/main.cpp@@ -44,6 +44,7 @@#include <cstdint>#include <cstdlib>#include <cstring>+#include <filesystem>#include <fstream>#include <iostream>#include <mutex>@@ -222,6 +223,7 @@ struct Word {int main(int argc, char** argv) {std::string model_path;+ std::string vad_model_path;std::string host = "127.0.0.1";bool host_from_flag = false;bool force_cpu = false;@@ -231,6 +233,7 @@ int main(int argc, char** argv) {for (int i = 1; i < argc; ++i) {const std::string a = argv[i];if (a == "--model" && i + 1 < argc) model_path = argv[++i];+ else if (a == "--vad-model" && i + 1 < argc) vad_model_path = argv[++i];else if (a == "--host" && i + 1 < argc) { host = argv[++i]; host_from_flag = true; }else if (a == "--port" && i + 1 < argc) port = std::atoi(argv[++i]);else if (a == "--threads" && i + 1 < argc) threads = std::atoi(argv[++i]);@@ -241,6 +244,10 @@ int main(int argc, char** argv) {if (model_path.empty()) {if (const char* p = std::getenv("OPENSCREEN_WHISPER_MODEL")) model_path = p;}+ if (vad_model_path.empty()) {+ if (const char* p = std::getenv("OPENSCREEN_VAD_MODEL")) vad_model_path = p;+ else if (const char* p = std::getenv("OPENSCREEN_WHISPER_VAD_MODEL")) vad_model_path = p;+ }if (port == 0) {if (const char* p = std::getenv("OPENSCREEN_WHISPER_PORT")) port = std::atoi(p);}@@ -264,7 +271,9 @@ int main(int argc, char** argv) {"OPENSCREEN_WHISPER_MODEL is required" << std::endl;return 2;}- log("boot: model=" + model_path + " host=" + host ++ log("boot: model=" + model_path ++ (!vad_model_path.empty() ? (" vad_model=" + vad_model_path) : "") ++ " host=" + host +" port=" + (port > 0 ? std::to_string(port) : "(any)") +" threads=" + std::to_string(threads));@@ -293,6 +302,25 @@ int main(int argc, char** argv) {const std::string active_backend = cparams.use_gpu ? detect_active_backend() : "whispercpp-cpu";log("model loaded; backend=" + active_backend);+ // ---- Init VAD context (Silero VAD v6.2.0) ----+ struct whisper_vad_context* vctx = nullptr;+ if (!vad_model_path.empty()) {+ struct whisper_vad_context_params vad_ctx_params = whisper_vad_default_context_params();+ vad_ctx_params.n_threads = threads;+ vad_ctx_params.use_gpu = !force_cpu;+ vctx = whisper_vad_init_from_file_with_params(vad_model_path.c_str(), vad_ctx_params);+ if (!vctx && vad_ctx_params.use_gpu) {+ log("GPU VAD initialization failed; retrying with CPU inference");+ vad_ctx_params.use_gpu = false;+ vctx = whisper_vad_init_from_file_with_params(vad_model_path.c_str(), vad_ctx_params);+ }+ if (vctx) {+ log("VAD model loaded; path=" + vad_model_path);+ } else {+ log("WARNING: failed to load VAD model from " + vad_model_path);+ }+ }+// ---- HTTP server ----httplib::Server svr;std::mutex infer_mu; // whisper contexts are not thread-safe@@ -366,6 +394,11 @@ int main(int argc, char** argv) {wparams.print_realtime = false;wparams.print_timestamps = false;wparams.n_threads = threads;+ if (vctx && !vad_model_path.empty()) {+ wparams.vad = true;+ wparams.vad_model_path = vad_model_path.c_str();+ wparams.vad_params = whisper_vad_default_params();+ }const auto t0 = std::chrono::steady_clock::now();const int rc = whisper_full(ctx, wparams, pcm.data(), static_cast<int>(pcm.size()));@@ -540,12 +573,80 @@ int main(int argc, char** argv) {res.set_content(reply.dump(), "application/json");});+ // POST /vad — multipart form with `file` (WAV: 16 kHz mono PCM16).+ // Runs Silero VAD segmentation and returns speech intervals [start, end] in seconds.+ svr.Post("/vad", [&](const httplib::Request& req, httplib::Response& res) {+ if (!vctx) {+ res.status = 400;+ res.set_content(R"({"error":"VAD model was not loaded on server startup"})", "application/json");+ return;+ }+ auto it = req.files.find("file");+ if (it == req.files.end()) {+ res.status = 400;+ res.set_content(R"({"error":"missing 'file' form field"})", "application/json");+ return;+ }+ const auto& file_entry = it->second;++ static std::atomic<uint64_t> tmp_vad_counter{0};+ const auto tmp_vad_id = tmp_vad_counter.fetch_add(1, std::memory_order_relaxed);+ const auto tmp_vad_ts = std::chrono::high_resolution_clock::now().time_since_epoch().count();+ const std::string tmp_wav = (std::filesystem::temp_directory_path() /+ ("openscreen-vad-" + std::to_string(tmp_vad_ts) ++ "-" + std::to_string(tmp_vad_id) + ".wav")).string();+ {+ std::ofstream out(tmp_wav, std::ios::binary);+ out.write(file_entry.content.data(),+ static_cast<std::streamsize>(file_entry.content.size()));+ }+ std::vector<float> pcm;+ int sample_rate = 0, channels = 0;+ const bool ok = read_wav_pcm16(tmp_wav, pcm, sample_rate, channels);+ std::error_code ec;+ std::filesystem::remove(tmp_wav, ec);+ if (!ok) {+ res.status = 400;+ res.set_content(R"({"error":"failed to parse WAV"})", "application/json");+ return;+ }+ if (sample_rate != 16000 || channels != 1) {+ res.status = 400;+ res.set_content(+ R"({"error":"expected 16 kHz mono PCM16 WAV"})",+ "application/json");+ return;+ }++ const std::lock_guard<std::mutex> lk(infer_mu);+ struct whisper_vad_params vad_params = whisper_vad_default_params();+ struct whisper_vad_segments* vad_segments =+ whisper_vad_segments_from_samples(vctx, vad_params, pcm.data(), static_cast<int>(pcm.size()));++ nlohmann::json reply;+ reply["segments"] = nlohmann::json::array();+ if (vad_segments) {+ const int n_segs = whisper_vad_segments_n_segments(vad_segments);+ for (int i = 0; i < n_segs; ++i) {+ const float t0 = whisper_vad_segments_get_segment_t0(vad_segments, i);+ const float t1 = whisper_vad_segments_get_segment_t1(vad_segments, i);+ reply["segments"].push_back({+ {"start", static_cast<double>(t0)},+ {"end", static_cast<double>(t1)}+ });+ }+ whisper_vad_free_segments(vad_segments);+ }+ res.set_content(reply.dump(), "application/json");+ });+// ---- bind + listen ----int bound_port = port;if (bound_port == 0) {bound_port = svr.bind_to_any_port(host);} else if (!svr.bind_to_port(host, bound_port)) {std::cerr << "FATAL: bind_to_port(" << host << ":" << bound_port << ") failed" << std::endl;+ if (vctx) whisper_vad_free(vctx);whisper_free(ctx);return 4;}@@ -554,6 +655,7 @@ int main(int argc, char** argv) {if (rc != 0) {std::cerr << "FATAL: listen_after_bind failed" << std::endl;}+ if (vctx) whisper_vad_free(vctx);whisper_free(ctx);return rc;}diff --git a/electron/stt/index.ts b/electron/stt/index.tsindex a20f4e577..e463baa16 100644--- a/electron/stt/index.ts+++ b/electron/stt/index.ts@@ -9,6 +9,8 @@ import type {SttTiming,SttTranscribeRequest,SttTranscribeResponse,+ SttVadResponse,+ SttVadSegment,SttWordSegment,} from "./transcriptionContract";import { WhisperServerManager } from "./whisperServer";@@ -97,6 +99,7 @@ export class SttManager {private initPromise: Promise<void> | null = null;/** Kept from `prepare()` so a chunk retry can respawn a helper that died mid-run. */private modelPath: string | null = null;+ private vadModelPath: string | null = null;/*** Bumped by `cancel()`. The chunk loop compares it against the value it* captured on entry, so a cancel that lands after a new run started cannot@@ -194,8 +197,12 @@ export class SttManager {const paths = modelPaths(modelsDir);this.modelPath = paths.whisper;+ this.vadModelPath = paths["silero-vad"];try {- await this.server.start({ modelPath: paths.whisper });+ await this.server.start({+ modelPath: paths.whisper,+ vadModelPath: paths["silero-vad"],+ });} catch (error) {if (this.shuttingDown) throw cancelledError();throw error;@@ -236,7 +243,9 @@ export class SttManager {if (this.shuttingDown) throw cancelledError();if (attempt === CHUNK_ATTEMPTS) break;if (this.modelPath) {- await this.server.start({ modelPath: this.modelPath }).catch(() => undefined);+ await this.server+ .start({ modelPath: this.modelPath, vadModelPath: this.vadModelPath })+ .catch(() => undefined);}if (this.shuttingDown) throw cancelledError();await new Promise((resolve) => setTimeout(resolve, 500 * attempt));@@ -433,6 +442,15 @@ export class SttManager {this.cancelEpoch++;await this.server.shutdown();}++ /**+ * Run Voice Activity Detection (Silero VAD) to detect speech segments in samples.+ */+ async detectSpeech(samples: Float32Array): Promise<SttVadSegment[]> {+ if (this.shuttingDown) throw cancelledError();+ await this.init();+ return this.server.detectVadSegments({ samples });+ }}let singleton: SttManager | null = null;@@ -491,6 +509,13 @@ export function registerSttIpc(ipcMain: IpcMain): void {}},);+ ipcMain.handle(+ "stt:vad",+ async (_event, req: { samples: Float32Array }): Promise<SttVadResponse> => {+ const segments = await manager.detectSpeech(req.samples);+ return { segments };+ },+ );ipcMain.handle("stt:cancel", () => {manager.cancel();});diff --git a/electron/stt/modelManager.test.ts b/electron/stt/modelManager.test.tsindex e5f186fbd..441906234 100644--- a/electron/stt/modelManager.test.ts+++ b/electron/stt/modelManager.test.ts@@ -30,20 +30,39 @@ describe("modelManager", () => {}});+ it("exposes the silero-vad model descriptor with a single GGML file", () => {+ expect(STT_MODELS["silero-vad"].cacheDir).toBe("whisper-ggml");+ expect(STT_MODELS["silero-vad"].repoId).toBe("ggml-org/whisper-vad");+ expect(STT_MODELS["silero-vad"].files.length).toBe(1);+ expect(STT_MODELS["silero-vad"].files[0].name).toBe("ggml-silero-v6.2.0.bin");+ expect(STT_MODELS["silero-vad"].files[0].expectedSha256).toBe(+ "2aa269b785eeb53a82983a20501ddf7c1d9c48e33ab63a41391ac6c9f7fb6987",+ );+ for (const f of STT_MODELS["silero-vad"].files) {+ expect(f.approximateBytes).toBeGreaterThan(0);+ expect(f.url).toContain("huggingface.co");+ expect(f.url).toMatch(/\/resolve\/[0-9a-f]{40}\//);+ }+ });+it("modelPaths places the GGML file under the cache directory", () => {const paths = modelPaths(dir);expect(paths.whisper).toBe(path.join(dir, "whisper-ggml", "ggml-small-q8_0.bin"));+ expect(paths["silero-vad"]).toBe(path.join(dir, "whisper-ggml", "ggml-silero-v6.2.0.bin"));});it("areModelsPresent returns false when the model file is missing", async () => {expect(await areModelsPresent(dir)).toBe(false);});- it("areModelsPresent returns true once the GGML file is present", async () => {+ it("areModelsPresent returns true once the GGML files are present", async () => {const paths = modelPaths(dir);await mkdir(path.dirname(paths.whisper), { recursive: true });expect(await areModelsPresent(dir)).toBe(false);await writeFile(paths.whisper, "dummy-ggml");+ expect(await areModelsPresent(dir, ["whisper"])).toBe(true);+ expect(await areModelsPresent(dir)).toBe(false);+ await writeFile(paths["silero-vad"], "dummy-vad");expect(await areModelsPresent(dir)).toBe(true);});diff --git a/electron/stt/modelManager.ts b/electron/stt/modelManager.tsindex cb13fd31f..959551237 100644--- a/electron/stt/modelManager.ts+++ b/electron/stt/modelManager.ts@@ -24,7 +24,7 @@ import { pipeline } from "node:stream/promises";* separate VAD model is required. See `technical-documentation/architecture/transcription-and-captions.md`.*/-export type SttModelId = "whisper";+export type SttModelId = "whisper" | "silero-vad";export interface SttModelFile {/** Relative path within the model directory (e.g. "ggml-small-q8_0.bin"). */@@ -62,6 +62,10 @@ const MODEL_FILE = "ggml-small-q8_0.bin";// LFS oid for MODEL_FILE is exactly the digest below.const MODEL_REVISION = "5359861c739e955e79d9a303bcbc70fb988958b1";+const VAD_REPO = "ggml-org/whisper-vad";+const VAD_FILE = "ggml-silero-v6.2.0.bin";+const VAD_REVISION = "9ffd54a1e1ee413ddf265af9913beaf518d1639b";+export const STT_MODELS: Record<SttModelId, SttModelDescriptor> = {whisper: {cacheDir: "whisper-ggml",@@ -75,22 +79,43 @@ export const STT_MODELS: Record<SttModelId, SttModelDescriptor> = {},],},+ "silero-vad": {+ cacheDir: "whisper-ggml",+ repoId: VAD_REPO,+ files: [+ {+ name: VAD_FILE,+ url: `${MODEL_BASE}/${VAD_REPO}/resolve/${VAD_REVISION}/${VAD_FILE}`,+ expectedSha256: "2aa269b785eeb53a82983a20501ddf7c1d9c48e33ab63a41391ac6c9f7fb6987",+ approximateBytes: 885_098,+ },+ ],+ },};export function modelPaths(baseDir: string): Record<SttModelId, string> {return {whisper: path.join(baseDir, STT_MODELS.whisper.cacheDir, MODEL_FILE),+ "silero-vad": path.join(baseDir, STT_MODELS["silero-vad"].cacheDir, VAD_FILE),};}/**- * True when the GGML model file exists and is non-empty.+ * True when the GGML model files exist and are non-empty.*/-export async function areModelsPresent(baseDir: string): Promise<boolean> {+export async function areModelsPresent(+ baseDir: string,+ only: SttModelId[] = ["whisper", "silero-vad"],+): Promise<boolean> {const paths = modelPaths(baseDir);try {- const s = await stat(paths.whisper);- return s.isFile() && s.size > 0;+ const results = await Promise.all(+ only.map(async (id) => {+ const s = await stat(paths[id]);+ return s.isFile() && s.size > 0;+ }),+ );+ return results.every(Boolean);} catch {return false;}@@ -217,7 +242,7 @@ async function ensureFile(export interface EnsureModelsOptions {baseDir: string;- /** Models to ensure; defaults to all (currently just `whisper`). */+ /** Models to ensure; defaults to all (`whisper`, `silero-vad`). */only?: SttModelId[];onProgress?: (event: {id: SttModelId;@@ -228,9 +253,9 @@ export interface EnsureModelsOptions {fetcher?: typeof fetch;}-/** Ensure the GGML model file is present locally; downloads with progress + retry. */+/** Ensure the GGML model files are present locally; downloads with progress + retry. */export async function ensureModels(opts: EnsureModelsOptions): Promise<void> {- const targets = (opts.only ?? (["whisper"] as SttModelId[])).map((id) => ({+ const targets = (opts.only ?? (["whisper", "silero-vad"] as SttModelId[])).map((id) => ({id,descriptor: STT_MODELS[id],filePath: modelPaths(opts.baseDir)[id],diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.tsindex d2e67ae11..b990ce9cf 100644--- a/electron/stt/transcriptionContract.ts+++ b/electron/stt/transcriptionContract.ts@@ -71,7 +71,7 @@ export interface SttStatusEvent {/** Total bytes for the in-flight download. */totalBytes?: number;/** Which model is downloading. */- model?: "whisper";+ model?: "whisper" | "silero-vad";/*** Seconds of audio transcribed so far, and the total for this request. Only* when `phase === "transcribe"`. Progress is reported per CHUNK (see@@ -158,3 +158,14 @@ export interface SttTranscribeResponse {/** IPC success envelope; thrown errors cross as a rejection. */export type SttTranscribeResult = SttTranscribeResponse;++/** A detected speech segment with timestamps in seconds. */+export interface SttVadSegment {+ startSec: number;+ endSec: number;+}++/** Response from the VAD speech detection pass. */+export interface SttVadResponse {+ segments: SttVadSegment[];+}diff --git a/electron/stt/whisperServer.test.ts b/electron/stt/whisperServer.test.tsindex 28ddd2105..a79b93093 100644--- a/electron/stt/whisperServer.test.ts+++ b/electron/stt/whisperServer.test.ts@@ -675,4 +675,92 @@ describe("WhisperServerManager", () => {await rm(dir, { recursive: true, force: true });}});++ it("passes --vad-model flag to spawned process when vadModelPath exists", async () => {+ const fs = await import("node:fs/promises");+ const { spawn } = await import("node:child_process");+ const dir = await mkdtemp(path.join(tmpdir(), "whisper-vad-args-"));+ try {+ const modelPath = path.join(dir, "ggml-small-q8_0.bin");+ const vadModelPath = path.join(dir, "ggml-silero-v6.2.0.bin");+ const fakeBinaryPath = path.join(dir, "whisper-stt-server");+ await fs.writeFile(modelPath, "dummy-ggml");+ await fs.writeFile(vadModelPath, "dummy-vad");+ await fs.writeFile(fakeBinaryPath, "x", { mode: 0o755 });++ const child = Object.assign(new EventEmitter(), {+ stdout: new EventEmitter(),+ stderr: new EventEmitter(),+ pid: 1234,+ kill: vi.fn(() => {+ queueMicrotask(() => child.emit("exit", 0));+ return true;+ }),+ });+ vi.mocked(spawn).mockImplementationOnce(() => child as never);+ vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true } as Response));++ const mgr = new WhisperServerManager();+ await mgr.start({+ modelPath,+ vadModelPath,+ binaryPath: fakeBinaryPath,+ backend: "whispercpp-cpu",+ });++ expect(spawn).toHaveBeenCalledTimes(1);+ const args = vi.mocked(spawn).mock.calls[0][1];+ expect(args).toContain("--vad-model");+ const vadIdx = args.indexOf("--vad-model");+ expect(args[vadIdx + 1]).toBe(vadModelPath);+ await mgr.stop();+ } finally {+ vi.unstubAllGlobals();+ await rm(dir, { recursive: true, force: true });+ }+ });++ it("detectVadSegments extracts segments from /vad response", async () => {+ const fakeJson = {+ segments: [+ { start: 0.25, end: 1.5 },+ { start: 2.1, end: 4.8 },+ ],+ };+ vi.stubGlobal(+ "fetch",+ vi.fn(async () => new Response(JSON.stringify(fakeJson), { status: 200 })),+ );+ try {+ const mgr = new WhisperServerManager();+ (mgr as unknown as { process: unknown; port: number }).process = {};+ (mgr as unknown as { process: unknown; port: number }).port = 9999;++ const segments = await mgr.detectVadSegments({ samples: new Float32Array(1600) });+ expect(segments).toEqual([+ { startSec: 0.25, endSec: 1.5 },+ { startSec: 2.1, endSec: 4.8 },+ ]);+ } finally {+ vi.unstubAllGlobals();+ }+ });++ it("detectVadSegments surfaces server error from /vad endpoint", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn(async () => new Response("VAD model was not loaded", { status: 400 })),+ );+ try {+ const mgr = new WhisperServerManager();+ (mgr as unknown as { process: unknown; port: number }).process = {};+ (mgr as unknown as { process: unknown; port: number }).port = 9999;++ await expect(mgr.detectVadSegments({ samples: new Float32Array(1600) })).rejects.toThrow(+ /whisper-stt-server \/vad HTTP 400/,+ );+ } finally {+ vi.unstubAllGlobals();+ }+ });});diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.tsindex 950dc879c..1673717f6 100644--- a/electron/stt/whisperServer.ts+++ b/electron/stt/whisperServer.ts@@ -12,6 +12,7 @@ import type {SttBackend,SttPhraseSegment,SttTiming,+ SttVadSegment,SttWordSegment,} from "./transcriptionContract";import { cleanupWav, writeSamplesAsWav } from "./wav";@@ -59,6 +60,8 @@ const REQUEST_TIMEOUT_MS = 280_000;export interface WhisperServerStartOptions {/** Absolute path to the GGML model file (e.g. ggml-small-q8_0.bin). */modelPath: string;+ /** Absolute path to the Silero VAD GGML model file (e.g. ggml-silero-v6.2.0.bin). */+ vadModelPath?: string | null;/** Externally-resolved binary path (skips gpuDetector on startup); null = auto. */binaryPath?: string | null;/** Externally-resolved backend (logs only); null = auto. */@@ -271,6 +274,9 @@ export class WhisperServerManager {"--threads",String(Math.max(1, os.cpus().length)),];+ if (options.vadModelPath && existsSync(options.vadModelPath)) {+ args.push("--vad-model", options.vadModelPath);+ }if (forceCpu) args.push("--cpu");const child = spawn(binaryPath, args, { stdio: ["ignore", "pipe", "pipe"] });const activeBackend: SttBackend = forceCpu ? "whispercpp-cpu" : resolved.backend;@@ -587,4 +593,65 @@ export class WhisperServerManager {await cleanupWav(wavPath);}}++ /** Run Voice Activity Detection (Silero VAD) to detect speech intervals. */+ async detectVadSegments(opts: { samples: Float32Array }): Promise<SttVadSegment[]> {+ const task = this.inFlight.then(() => this.detectVadSegmentsImpl(opts));+ this.inFlight = task.catch(() => undefined);+ return task;+ }++ private async detectVadSegmentsImpl(opts: { samples: Float32Array }): Promise<SttVadSegment[]> {+ await this.ensureReady();+ const wavPath = await writeSamplesAsWav(opts.samples);+ try {+ const url = `${this.baseUrl()}/vad`;+ const form = new FormData();+ const fileBuffer = await readFile(wavPath);+ const blob = new Blob([fileBuffer], { type: "audio/wav" });+ form.set("file", blob, path.basename(wavPath));++ let res: Response;+ try {+ res = await fetch(url, {+ method: "POST",+ body: form,+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),+ });+ } catch (error) {+ throw Object.assign(+ new Error(+ `whisper-stt-server /vad failed: ` ++ `${error instanceof Error ? error.message : String(error)}; ` ++ `stderr=${this.stderrTail.slice(-256)}`,+ ),+ { cause: error },+ );+ }++ if (!res.ok) {+ const text = await res.text().catch(() => "");+ throw new Error(`whisper-stt-server /vad HTTP ${res.status}: ${text.slice(0, 512)}`);+ }++ const json = (await res.json().catch((error: unknown) => {+ throw Object.assign(+ new Error(+ `whisper-stt-server /vad response was unreadable: ` ++ `${error instanceof Error ? error.message : String(error)}`,+ ),+ { cause: error },+ );+ })) as { segments?: Array<{ start?: number | string; end?: number | string }> };++ const raw = json.segments ?? [];+ return raw.map((seg) => {+ const startSec = this.toSec(seg.start, 0);+ const endSec = this.toSec(seg.end, startSec);+ return { startSec, endSec };+ });+ } finally {+ await cleanupWav(wavPath);+ }+ }}