Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
refactor(stt): drop the unreachable /vad endpoint and stt:vad IPC
EtienneLescot6fd9fb772bc5
diff --git a/electron/native/whisper-stt/src/main.cpp b/electron/native/whisper-stt/src/main.cppindex 874398c93..61303f3c3 100644--- a/electron/native/whisper-stt/src/main.cpp+++ b/electron/native/whisper-stt/src/main.cpp@@ -653,73 +653,6 @@ int main(int argc, char** argv) {res.set_content(reply.dump(), "application/json");});- // POST /vad — multipart form with `file` (WAV: 16 kHz mono PCM16).- // Runs Silero VAD segmentation and returns speech intervals [start, end] in seconds.- svr.Post("/vad", [&](const httplib::Request& req, httplib::Response& res) {- if (!vctx) {- res.status = 503;- res.set_content(R"({"error":"VAD model was not loaded on server startup"})", "application/json");- return;- }- auto it = req.files.find("file");- if (it == req.files.end()) {- res.status = 400;- res.set_content(R"({"error":"missing 'file' form field"})", "application/json");- return;- }- const auto& file_entry = it->second;-- static std::atomic<uint64_t> tmp_vad_counter{0};- const auto tmp_vad_id = tmp_vad_counter.fetch_add(1, std::memory_order_relaxed);- const auto tmp_vad_ts = std::chrono::high_resolution_clock::now().time_since_epoch().count();- const std::string tmp_wav = (std::filesystem::temp_directory_path() /- ("openscreen-vad-" + std::to_string(tmp_vad_ts) +- "-" + std::to_string(tmp_vad_id) + ".wav")).string();- {- std::ofstream out(tmp_wav, std::ios::binary);- out.write(file_entry.content.data(),- static_cast<std::streamsize>(file_entry.content.size()));- }- std::vector<float> pcm;- int sample_rate = 0, channels = 0;- const bool ok = read_wav_pcm16(tmp_wav, pcm, sample_rate, channels);- std::error_code ec;- std::filesystem::remove(tmp_wav, ec);- if (!ok) {- res.status = 400;- res.set_content(R"({"error":"failed to parse WAV"})", "application/json");- return;- }- if (sample_rate != 16000 || channels != 1) {- res.status = 400;- res.set_content(- R"({"error":"expected 16 kHz mono PCM16 WAV"})",- "application/json");- return;- }-- const std::lock_guard<std::mutex> lk(infer_mu);- struct whisper_vad_params vad_params = whisper_vad_default_params();- struct whisper_vad_segments* vad_segments =- whisper_vad_segments_from_samples(vctx, vad_params, pcm.data(), static_cast<int>(pcm.size()));-- nlohmann::json reply;- reply["segments"] = nlohmann::json::array();- if (vad_segments) {- const int n_segs = whisper_vad_segments_n_segments(vad_segments);- for (int i = 0; i < n_segs; ++i) {- const float t0 = whisper_vad_segments_get_segment_t0(vad_segments, i);- const float t1 = whisper_vad_segments_get_segment_t1(vad_segments, i);- reply["segments"].push_back({- {"start", static_cast<double>(t0) / 100.0},- {"end", static_cast<double>(t1) / 100.0}- });- }- whisper_vad_free_segments(vad_segments);- }- res.set_content(reply.dump(), "application/json");- });-// ---- bind + listen ----int bound_port = port;if (bound_port == 0) {diff --git a/electron/stt/index.ts b/electron/stt/index.tsindex 656d7bf6d..4ee2b24c9 100644--- a/electron/stt/index.ts+++ b/electron/stt/index.ts@@ -3,16 +3,13 @@ import { app, type IpcMain } from "electron";import { planChunks } from "./chunking";import { extractMono16kPcm } from "./extractAudio";import { ensureModels, modelPaths } from "./modelManager";-import {- STT_VAD_UNAVAILABLE,- type SttPhraseSegment,- type SttStatusEvent,- type SttTiming,- type SttTranscribeRequest,- type SttTranscribeResponse,- type SttVadResponse,- type SttVadSegment,- type SttWordSegment,+import type {+ SttPhraseSegment,+ SttStatusEvent,+ SttTiming,+ SttTranscribeRequest,+ SttTranscribeResponse,+ SttWordSegment,} from "./transcriptionContract";import { WhisperServerManager } from "./whisperServer";@@ -443,23 +440,6 @@ export class SttManager {this.cancelEpoch++;await this.server.shutdown();}-- /** True when the helper reported Silero VAD loaded and ready. */- isVadAvailable(): boolean {- return this.server.status.vadAvailable;- }-- /**- * Run Voice Activity Detection (Silero VAD) to detect speech segments in samples.- */- async detectSpeech(samples: Float32Array): Promise<SttVadSegment[]> {- if (this.shuttingDown) throw cancelledError();- await this.init();- if (!this.server.status.vadAvailable) {- throw new Error(STT_VAD_UNAVAILABLE);- }- return this.server.detectVadSegments({ samples });- }}let singleton: SttManager | null = null;@@ -518,13 +498,6 @@ export function registerSttIpc(ipcMain: IpcMain): void {}},);- ipcMain.handle(- "stt:vad",- async (_event, req: { samples: Float32Array }): Promise<SttVadResponse> => {- const segments = await manager.detectSpeech(req.samples);- return { segments };- },- );ipcMain.handle("stt:cancel", () => {manager.cancel();});diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.tsindex c9b7d4850..5433698ac 100644--- a/electron/stt/transcriptionContract.ts+++ b/electron/stt/transcriptionContract.ts@@ -141,7 +141,6 @@ export interface SttTranscribeRequest {* a literal typed twice is a fallback that silently stops working.*/export const STT_NATIVE_EXTRACTION_UNAVAILABLE = "stt:native-extraction-unavailable";-export const STT_VAD_UNAVAILABLE = "stt:vad-unavailable";/** IPC response: main → renderer. */export interface SttTranscribeResponse {@@ -165,8 +164,3 @@ export interface SttVadSegment {startSec: number;endSec: number;}--/** Response from the VAD speech detection pass. */-export interface SttVadResponse {- segments: SttVadSegment[];-}diff --git a/electron/stt/whisperServer.test.ts b/electron/stt/whisperServer.test.tsindex 1d4c3e41b..7f52d5a6e 100644--- a/electron/stt/whisperServer.test.ts+++ b/electron/stt/whisperServer.test.ts@@ -3,9 +3,8 @@ import { mkdtemp, rm, stat, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import path from "node:path";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";-import { STT_VAD_UNAVAILABLE } from "./transcriptionContract";import { writeSamplesAsWav } from "./wav";-import { MAX_VAD_CHUNK_SAMPLES, mergeVadIntervals, WhisperServerManager } from "./whisperServer";+import { WhisperServerManager } from "./whisperServer";vi.mock("node:child_process", async (importOriginal) => {// `node:child_process` is CJS, so the ESM namespace also carries a `default`@@ -757,175 +756,4 @@ describe("WhisperServerManager", () => {await rm(dir, { recursive: true, force: true });}});-- it("detectVadSegments extracts segments from /vad response", async () => {- const fakeJson = {- segments: [- { start: 0.25, end: 1.5 },- { start: 2.1, end: 4.8 },- ],- };- vi.stubGlobal(- "fetch",- vi.fn(async () => new Response(JSON.stringify(fakeJson), { status: 200 })),- );- try {- const mgr = new WhisperServerManager();- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {};- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =- true;-- const segments = await mgr.detectVadSegments({ samples: new Float32Array(1600) });- expect(segments).toEqual([- { startSec: 0.25, endSec: 1.5 },- { startSec: 2.1, endSec: 4.8 },- ]);- } finally {- vi.unstubAllGlobals();- }- });-- it("detectVadSegments surfaces server error from /vad endpoint", async () => {- vi.stubGlobal(- "fetch",- vi.fn(async () => new Response("VAD model was not loaded", { status: 400 })),- );- try {- const mgr = new WhisperServerManager();- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {};- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =- true;-- await expect(mgr.detectVadSegments({ samples: new Float32Array(1600) })).rejects.toThrow(- /whisper-stt-server \/vad HTTP 400/,- );- } finally {- vi.unstubAllGlobals();- }- });-- it("filters out /vad segments missing end that map to zero-length intervals", async () => {- const fakeJson = {- segments: [- { start: 1.5 }, // missing end -> maps to startSec: 1.5, endSec: 1.5- ],- };- vi.stubGlobal(- "fetch",- vi.fn(async () => new Response(JSON.stringify(fakeJson), { status: 200 })),- );- try {- const mgr = new WhisperServerManager();- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {};- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =- true;-- const segments = await mgr.detectVadSegments({ samples: new Float32Array(1600) });- expect(segments).toEqual([]);- } finally {- vi.unstubAllGlobals();- }- });-- it("handles malformed WAV /vad HTTP 400 without crashing helper availability", async () => {- vi.stubGlobal(- "fetch",- vi.fn(- async () => new Response(JSON.stringify({ error: "failed to parse WAV" }), { status: 400 }),- ),- );- try {- const mgr = new WhisperServerManager();- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {- pid: 999,- };- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =- true;-- await expect(mgr.detectVadSegments({ samples: new Float32Array(100) })).rejects.toThrow(- /whisper-stt-server \/vad HTTP 400/,- );- expect(mgr.status.running).toBe(true);- } finally {- vi.unstubAllGlobals();- }- });-- it("rejects detectVadSegments when VAD is unavailable", async () => {- const mgr = new WhisperServerManager();- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {};- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =- false;-- await expect(mgr.detectVadSegments({ samples: new Float32Array(1600) })).rejects.toThrow(- STT_VAD_UNAVAILABLE,- );- });-- describe("mergeVadIntervals", () => {- it("returns empty array for empty input", () => {- expect(mergeVadIntervals([])).toEqual([]);- });-- it("returns single interval as-is", () => {- expect(mergeVadIntervals([{ startSec: 1, endSec: 2 }])).toEqual([{ startSec: 1, endSec: 2 }]);- });-- it("merges overlapping and adjacent intervals", () => {- const input = [- { startSec: 1, endSec: 3 },- { startSec: 2.5, endSec: 5 },- { startSec: 5, endSec: 6.2 },- { startSec: 8, endSec: 10 },- ];- expect(mergeVadIntervals(input)).toEqual([- { startSec: 1, endSec: 6.2 },- { startSec: 8, endSec: 10 },- ]);- });-- it("drops inverted intervals where endSec <= startSec", () => {- const input = [- { startSec: 2, endSec: 1 },- { startSec: 3, endSec: 4 },- ];- expect(mergeVadIntervals(input)).toEqual([{ startSec: 3, endSec: 4 }]);- });-- it("drops singleton intervals where endSec <= startSec", () => {- expect(mergeVadIntervals([{ startSec: 2, endSec: 1 }])).toEqual([]);- expect(mergeVadIntervals([{ startSec: 2, endSec: 2 }])).toEqual([]);- });- });-- it("chunks long audio across MAX_VAD_CHUNK_SAMPLES and offsets timestamps", async () => {- const fetchMock = vi.fn(async () => {- const callIndex = fetchMock.mock.calls.length - 1;- const segments = callIndex === 0 ? [{ start: 10, end: 180 }] : [{ start: 0, end: 20 }];- return new Response(JSON.stringify({ segments }), { status: 200 });- });- vi.stubGlobal("fetch", fetchMock);- try {- const mgr = new WhisperServerManager();- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {};- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999;- (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable =- true;-- // Total duration = 200 seconds @ 16 kHz = 3,200,000 samples (> MAX_VAD_CHUNK_SAMPLES = 2,880,000)- const samples = new Float32Array(MAX_VAD_CHUNK_SAMPLES + 16_000 * 20);- const segments = await mgr.detectVadSegments({ samples });-- expect(fetchMock).toHaveBeenCalledTimes(2);- // 180s in chunk 0 + 20s in chunk 1 with offset 180s -> merged [10, 200]- expect(segments).toEqual([{ startSec: 10, endSec: 200 }]);- } finally {- vi.unstubAllGlobals();- }- });});diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.tsindex 6ffa887f7..9a27c6dee 100644--- a/electron/stt/whisperServer.ts+++ b/electron/stt/whisperServer.ts@@ -8,13 +8,11 @@ import type { Readable } from "node:stream";import { resolveBinaryPath } from "./gpuDetector";import { snapWordBoundariesToAudio } from "./snapWordBoundaries";-import {- STT_VAD_UNAVAILABLE,- type SttBackend,- type SttPhraseSegment,- type SttTiming,- type SttVadSegment,- type SttWordSegment,+import type {+ SttBackend,+ SttPhraseSegment,+ SttTiming,+ SttWordSegment,} from "./transcriptionContract";import { cleanupWav, writeSamplesAsWav } from "./wav";@@ -76,7 +74,6 @@ export interface WhisperServerStatus {backend: SttBackend | null;startedAtMs: number | null;lastError: string | null;- vadAvailable: boolean;}/** Per-word entry inside a whisper-stt-server `/inference` JSON segment. */@@ -127,11 +124,7 @@ export class WhisperServerManager {private lastError: string | null = null;private startedAtMs: number | null = null;private inFlight: Promise<unknown> = Promise.resolve();- private starting: Promise<{- port: number;- backend: SttBackend;- vadAvailable: boolean;- }> | null = null;+ private starting: Promise<{ port: number; backend: SttBackend }> | null = null;/** Buffered stderr from the helper; surfaced on shutdown + poll failures. */private stderrTail = "";@@ -156,12 +149,12 @@ export class WhisperServerManager {});}- /** Check the server's HTTP root for a 200; resolves once responsive. Returns VAD availability. */+ /** Check the server's HTTP root for a 200; resolves once responsive. */private static async pollUntilReady(baseUrl: string,timeoutMs = 30_000,shouldContinue: () => boolean = () => true,- ): Promise<{ vad: boolean }> {+ ): Promise<void> {const deadline = Date.now() + timeoutMs;while (Date.now() < deadline) {if (!shouldContinue()) throw new Error("whisper-stt-server exited before readiness");@@ -172,10 +165,7 @@ export class WhisperServerManager {);try {const res = await fetch(baseUrl, { method: "GET", signal: controller.signal });- if (res.ok) {- const body = typeof res.json === "function" ? await res.json().catch(() => null) : null;- return { vad: Boolean((body as { vad?: boolean } | null)?.vad) };- }+ if (res.ok) return;} catch {// not up yet} finally {@@ -210,8 +200,6 @@ export class WhisperServerManager {if (log) console.error(`[stt] ${message}`);}- private vadAvailable = false;-/** True when a process is alive and a model is loaded. */get status(): WhisperServerStatus {return {@@ -221,22 +209,15 @@ export class WhisperServerManager {backend: this.backend,startedAtMs: this.startedAtMs,lastError: this.lastError,- vadAvailable: this.vadAvailable,};}- get isVadAvailable(): boolean {- return this.vadAvailable;- }-/*** Spawn the helper if not running and return once `/` returns 200. Idempotent —* if a server is already up we just return its port so the caller never pays* the cold-start cost twice.*/- async start(- options: WhisperServerStartOptions,- ): Promise<{ port: number; backend: SttBackend; vadAvailable: boolean }> {+ async start(options: WhisperServerStartOptions): Promise<{ port: number; backend: SttBackend }> {if (this.starting) return this.starting;this.starting = this.startImpl(options).finally(() => {this.starting = null;@@ -246,7 +227,7 @@ export class WhisperServerManager {private async startImpl(options: WhisperServerStartOptions,- ): Promise<{ port: number; backend: SttBackend; vadAvailable: boolean }> {+ ): Promise<{ port: number; backend: SttBackend }> {if (this.shuttingDown) {throw new Error("whisper-stt-server manager is shutting down");}@@ -254,7 +235,6 @@ export class WhisperServerManager {return {port: this.port,backend: this.backend ?? options.backend ?? "whispercpp-cpu",- vadAvailable: this.vadAvailable,};}@@ -283,9 +263,7 @@ export class WhisperServerManager {throw new Error(`Whisper GGML model not found at ${options.modelPath}`);}- const launch = async (- forceCpu: boolean,- ): Promise<{ port: number; backend: SttBackend; vadAvailable: boolean }> => {+ const launch = async (forceCpu: boolean): Promise<{ port: number; backend: SttBackend }> => {const port = await WhisperServerManager.pickFreePort();if (this.shuttingDown) {throw new Error("whisper-stt-server manager is shutting down");@@ -333,7 +311,6 @@ export class WhisperServerManager {this.process = null;this.port = null;this.startedAtMs = null;- this.vadAvailable = false;}});child.once("error", (err) => {@@ -349,7 +326,6 @@ export class WhisperServerManager {this.process = null;this.port = null;this.startedAtMs = null;- this.vadAvailable = false;}});@@ -364,9 +340,8 @@ export class WhisperServerManager {});child.once("error", reject);});- let readyInfo: { vad: boolean };try {- readyInfo = await Promise.race([+ await Promise.race([WhisperServerManager.pollUntilReady(`http://127.0.0.1:${port}`,30_000,@@ -386,8 +361,7 @@ export class WhisperServerManager {this.recordError(message, { log: !alreadyLogged });throw new Error(message);}- this.vadAvailable = Boolean(readyInfo?.vad);- return { port, backend: activeBackend, vadAvailable: this.vadAvailable };+ return { port, backend: activeBackend };};try {@@ -413,7 +387,6 @@ export class WhisperServerManager {/** Send SIGTERM and wait for the helper to exit. Resolves even if it was already down. */async stop(): Promise<void> {- this.vadAvailable = false;if (!this.process) {this.port = null;this.startedAtMs = null;@@ -630,117 +603,4 @@ export class WhisperServerManager {await cleanupWav(wavPath);}}-- /** Run Voice Activity Detection (Silero VAD) to detect speech intervals. */- async detectVadSegments(opts: { samples: Float32Array }): Promise<SttVadSegment[]> {- const task = this.inFlight.then(() => this.detectVadSegmentsImpl(opts));- this.inFlight = task.catch(() => undefined);- return task;- }-- private async detectVadChunk(samples: Float32Array): Promise<SttVadSegment[]> {- const wavPath = await writeSamplesAsWav(samples);- try {- const url = `${this.baseUrl()}/vad`;- const form = new FormData();- const fileBuffer = await readFile(wavPath);- const blob = new Blob([fileBuffer], { type: "audio/wav" });- form.set("file", blob, path.basename(wavPath));-- let res: Response;- try {- res = await fetch(url, {- method: "POST",- body: form,- signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),- });- } catch (error) {- throw Object.assign(- new Error(- `whisper-stt-server /vad failed: ` +- `${error instanceof Error ? error.message : String(error)}; ` +- `stderr=${this.stderrTail.slice(-256)}`,- ),- { cause: error },- );- }-- if (!res.ok) {- const text = await res.text().catch(() => "");- throw new Error(`whisper-stt-server /vad HTTP ${res.status}: ${text.slice(0, 512)}`);- }-- const json = (await res.json().catch((error: unknown) => {- throw Object.assign(- new Error(- `whisper-stt-server /vad response was unreadable: ` +- `${error instanceof Error ? error.message : String(error)}`,- ),- { cause: error },- );- })) as { segments?: Array<{ start?: number | string; end?: number | string }> };-- const raw = json.segments ?? [];- return raw.map((seg) => {- const startSec = this.toSec(seg.start, 0);- const endSec = this.toSec(seg.end, startSec);- return { startSec, endSec };- });- } finally {- await cleanupWav(wavPath);- }- }-- private async detectVadSegmentsImpl(opts: { samples: Float32Array }): Promise<SttVadSegment[]> {- await this.ensureReady();- if (!this.vadAvailable) {- throw new Error(STT_VAD_UNAVAILABLE);- }- if (opts.samples.length === 0) return [];-- const allSegments: SttVadSegment[] = [];- const totalSamples = opts.samples.length;- for (let offset = 0; offset < totalSamples; offset += MAX_VAD_CHUNK_SAMPLES) {- const chunkSamples = opts.samples.subarray(- offset,- Math.min(totalSamples, offset + MAX_VAD_CHUNK_SAMPLES),- );- const chunkSegments = await this.detectVadChunk(chunkSamples);- const offsetSec = offset / 16_000;- for (const seg of chunkSegments) {- allSegments.push({- startSec: Math.round((seg.startSec + offsetSec) * 1000) / 1000,- endSec: Math.round((seg.endSec + offsetSec) * 1000) / 1000,- });- }- }-- return mergeVadIntervals(allSegments);- }-}--/**- * Silero VAD processes audio quickly, but buffering hours of 16 kHz audio as- * single WAV blobs can exhaust Node memory. We bound individual VAD passes to- * chunks of at most 3 minutes (180s = 2,880,000 samples @ 16 kHz), keeping- * memory under ~6 MB per WAV, and merge the resulting intervals.- */-export const MAX_VAD_CHUNK_SAMPLES = 16_000 * 180;--/**- * Merges contiguous or overlapping speech segments into non-overlapping intervals.- */-export function mergeVadIntervals(segments: SttVadSegment[]): SttVadSegment[] {- if (segments.length <= 1) return segments.filter((seg) => seg.endSec > seg.startSec);- const merged: SttVadSegment[] = [];- for (const seg of segments) {- if (seg.endSec <= seg.startSec) continue;- const last = merged[merged.length - 1];- if (last && seg.startSec <= last.endSec) {- last.endSec = Math.max(last.endSec, seg.endSec);- } else {- merged.push({ startSec: seg.startSec, endSec: seg.endSec });- }- }- return merged;}