A free, open source, and extensible speech-to-text application that works completely offline.
transcribe.cpp vad (#2267)
CJ Pais3985508ae7cf
diff --git a/AGENTS.md b/AGENTS.mdindex eda5426b..88ae7b52 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -44,7 +44,7 @@ bun run format:backend # cargo fmt only```bashmkdir -p src-tauri/resources/models-curl -o src-tauri/resources/models/silero_vad_v4.onnx https://blob.handy.computer/silero_vad_v4.onnx+curl -L -o src-tauri/resources/models/silero-vad-v6.2-F32.gguf https://huggingface.co/handy-computer/silero-vad-v6.2-gguf/resolve/main/silero-vad-v6.2-F32.gguf```For detailed platform-specific build setup, see [BUILD.md](BUILD.md).@@ -105,7 +105,7 @@ Handy is a cross-platform desktop speech-to-text application built with Tauri 2.- `transcribe-cpp` - Local Whisper-family inference (GGML/GGUF) with GPU acceleration- `transcribe-rs` - ONNX speech recognition (Parakeet, Moonshine, SenseVoice, etc.)- `handy-recorder` - Microphone capture: device selection, resampling to 16 kHz, failure detection-- `vad-rs` - Voice Activity Detection+- `transcribe-cpp` (Silero VAD v6, CPU-only, in-process) / `earshot` - Voice Activity Detection- `rdev` - Global keyboard shortcuts- `rodio` - Audio playback for feedback soundsdiff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 9c14c8bc..31c59fc0 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -54,7 +54,7 @@ Before you begin, ensure you have the following installed:```bashmkdir -p src-tauri/resources/models- curl -o src-tauri/resources/models/silero_vad_v4.onnx https://blob.handy.computer/silero_vad_v4.onnx+ curl -L -o src-tauri/resources/models/silero-vad-v6.2-F32.gguf https://huggingface.co/handy-computer/silero-vad-v6.2-gguf/resolve/main/silero-vad-v6.2-F32.gguf```6. **Run in development mode**:diff --git a/CRUSH.md b/CRUSH.mddeleted file mode 100644index ab03ffff..00000000--- a/CRUSH.md+++ /dev/null@@ -1,65 +0,0 @@-# Development Commands--**Environment Setup:**--```bash-bun install # Install dependencies-mkdir -p src-tauri/resources/models-curl -o src-tauri/resources/models/silero_vad_v4.onnx https://blob.handy.computer/silero_vad_v4.onnx-```--**Development:**--```bash-bun run tauri dev # Full app development-CMAKE_POLICY_VERSION_MINIMUM=3.5 bun run tauri dev # macOS with cmake fix-bun run dev # Frontend only (Vite)-bun run build # Build frontend-bun run tauri build # Production build-```--**Type Check & Build:**--```bash-bunx tsc --noEmit # Type checking-bun run build # Build and validate-```--# Code Style Guidelines--**Rust (Backend):**--- Use `anyhow::Error` for error handling with descriptive messages-- Prefer `Arc<Mutex<T>>` for shared state in managers-- Log with appropriate levels: `debug!`, `info!`, `eprintln!` for errors-- Builder pattern for initialization chains-- Snake_case for functions and variables, PascalCase for types-- Separate logical sections with comment blocks: `/* ─────────── */`--**TypeScript/React (Frontend):**--- Functional components with TypeScript interfaces-- Zod schemas for type validation and inference-- `useCallback` hooks for stable function references-- Destructure props with defaults: `disabled = false`-- Prefer interface aliases over type aliases for objects-- React.FC for explicit component typing-- PascalCase for components, camelCase for variables/functions--**Imports:**--- Group imports: external libs, internal modules, relative imports-- Use type imports for TypeScript: `import type { Settings }`-- Named imports preferred over default exports--**Error Handling:**--- Frontend: Try/catch with user feedback, rollback optimistic updates-- Backend: `?` operator with anyhow context messages-- Log errors appropriately for debugging level--**Component Patterns:**--- Container component pattern for layout-- Composition over inheritance-- Prop drilling minimized with context where appropriatediff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lockindex a96062c6..d3cabb77 100644--- a/src-tauri/Cargo.lock+++ b/src-tauri/Cargo.lock@@ -1815,16 +1815,6 @@ version = "0.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"-[[package]]-name = "eyre"-version = "0.6.12"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec"-dependencies = [- "indenter",- "once_cell",-]-[[package]]name = "fallible-iterator"version = "0.3.0"@@ -2611,7 +2601,6 @@ dependencies = ["tokio","transcribe-cpp","transcribe-rs",- "vad-rs","webview2-com","whatlang","windows 0.61.3",@@ -3108,12 +3097,6 @@ dependencies = ["tiff",]-[[package]]-name = "indenter"-version = "0.3.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5"-[[package]]name = "indexmap"version = "1.9.3"@@ -5384,12 +5367,6 @@ dependencies = ["windows-sys 0.52.0",]-[[package]]-name = "ringbuffer"-version = "0.15.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3df6368f71f205ff9c33c076d170dd56ebf68e8161c733c0caa07a7a5509ed53"-[[package]]name = "rkyv"version = "0.7.46"@@ -7753,17 +7730,6 @@ dependencies = ["wasm-bindgen",]-[[package]]-name = "vad-rs"-version = "0.1.6"-source = "git+https://github.com/cjpais/vad-rs#2a412ed858695b9251f3f5a1a20d95b59fa7c498"-dependencies = [- "eyre",- "ndarray",- "ort",- "ringbuffer",-]-[[package]]name = "value-bag"version = "1.12.0"diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.tomlindex 9ec8e8f2..9def144f 100644--- a/src-tauri/Cargo.toml+++ b/src-tauri/Cargo.toml@@ -49,7 +49,6 @@ hound = "3.5.1"log = "0.4.25"env_filter = "0.1.0"tokio = "1.43.0"-vad-rs = { git = "https://github.com/cjpais/vad-rs", default-features = false }earshot = "1.2.2"enigo = "0.6.1"# Feedback sounds. No `recording`: microphone capture is handy-recorder's.diff --git a/src-tauri/resources/models/silero-vad-v6.2-F32.gguf b/src-tauri/resources/models/silero-vad-v6.2-F32.ggufnew file mode 100644index 00000000..1c85efc4Binary files /dev/null and b/src-tauri/resources/models/silero-vad-v6.2-F32.gguf differdiff --git a/src-tauri/resources/models/silero_vad_v4.onnx b/src-tauri/resources/models/silero_vad_v4.onnxdeleted file mode 100644index e6db48d6..00000000Binary files a/src-tauri/resources/models/silero_vad_v4.onnx and /dev/null differdiff --git a/src-tauri/src/audio_toolkit/vad/mod.rs b/src-tauri/src/audio_toolkit/vad/mod.rsindex f1abc698..f1bdbd1b 100644--- a/src-tauri/src/audio_toolkit/vad/mod.rs+++ b/src-tauri/src/audio_toolkit/vad/mod.rs@@ -85,10 +85,10 @@ mod tests {#[test]fn duration_profiles_preserve_silero_timings() {- assert_eq!(frames_for_duration_ms(VAD_PREFILL_MS, 480), 15);- assert_eq!(frames_for_duration_ms(VAD_OFFLINE_HANGOVER_MS, 480), 15);- assert_eq!(frames_for_duration_ms(VAD_STREAMING_HANGOVER_MS, 480), 55);- assert_eq!(frames_for_duration_ms(VAD_ONSET_MS, 480), 2);+ assert_eq!(frames_for_duration_ms(VAD_PREFILL_MS, 512), 15);+ assert_eq!(frames_for_duration_ms(VAD_OFFLINE_HANGOVER_MS, 512), 15);+ assert_eq!(frames_for_duration_ms(VAD_STREAMING_HANGOVER_MS, 512), 52);+ assert_eq!(frames_for_duration_ms(VAD_ONSET_MS, 512), 2);}#[test]diff --git a/src-tauri/src/audio_toolkit/vad/silero.rs b/src-tauri/src/audio_toolkit/vad/silero.rsindex 4fc64eae..1785e5f0 100644--- a/src-tauri/src/audio_toolkit/vad/silero.rs+++ b/src-tauri/src/audio_toolkit/vad/silero.rs@@ -1,29 +1,70 @@use anyhow::Result;use std::path::Path;+use std::sync::{Mutex, OnceLock, PoisonError};-use vad_rs::Vad;+use transcribe_cpp::{Backend, BackendMask, Model, ModelOptions, VadSession, VadSessionOptions};use super::{VadFrame, VoiceActivityDetector};use crate::audio_toolkit::constants;-const SILERO_FRAME_MS: u32 = 30;-const SILERO_FRAME_SAMPLES: usize =- (constants::WHISPER_SAMPLE_RATE * SILERO_FRAME_MS / 1000) as usize;-pub struct SileroVad {- engine: Vad,+ // `VadSession` is Send but not Sync; the trait requires both. Only ever+ // reached through `get_mut`, so it is never locked and never poisoned.+ session: Mutex<VadSession>,+ frame_samples: usize,threshold: f32,}+/// Register transcribe.cpp's CPU backends once per process. The allowed mask+/// is fixed at first registration, so it must never include a GPU here.+fn init_cpu_backends() -> Result<()> {+ static INIT: OnceLock<Result<(), String>> = OnceLock::new();+ INIT.get_or_init(|| {+ // Route native/ggml diagnostics into Handy's log; must precede any+ // native call.+ transcribe_cpp::init_logging();+ transcribe_cpp::init_backends_with(None::<&Path>, BackendMask::CPU)+ .map_err(|e| e.to_string())+ })+ .clone()+ .map_err(|e| anyhow::anyhow!("Failed to initialize transcribe.cpp CPU backend: {e}"))+}+impl SileroVad {pub fn new<P: AsRef<Path>>(model_path: P, threshold: f32) -> Result<Self> {if !(0.0..=1.0).contains(&threshold) {anyhow::bail!("threshold must be between 0.0 and 1.0");}+ init_cpu_backends()?;+ let model = Model::load_with(+ model_path.as_ref(),+ &ModelOptions {+ backend: Backend::Cpu,+ device: None,+ },+ )+ .map_err(|e| anyhow::anyhow!("Failed to load VAD model: {e}"))?;++ let info = model+ .vad_info()+ .map_err(|e| anyhow::anyhow!("Model does not support VAD: {e}"))?;+ if info.sample_rate != constants::WHISPER_SAMPLE_RATE as i32 || info.frame_samples <= 0 {+ anyhow::bail!(+ "unsupported VAD model: {} Hz, {} samples per frame",+ info.sample_rate,+ info.frame_samples+ );+ }++ // The session keeps the native model alive on its own.+ let session = model+ .vad_session_with(&VadSessionOptions { n_threads: 1 })+ .map_err(|e| anyhow::anyhow!("Failed to create VAD session: {e}"))?;+Ok(Self {- engine: Vad::new(&model_path, constants::WHISPER_SAMPLE_RATE as usize)- .map_err(|e| anyhow::anyhow!("Failed to create VAD: {e}"))?,+ session: Mutex::new(session),+ frame_samples: info.frame_samples as usize,threshold,})}@@ -31,19 +72,31 @@ impl SileroVad {impl VoiceActivityDetector for SileroVad {fn push_frame<'a>(&'a mut self, frame: &'a [f32]) -> Result<VadFrame<'a>> {- if frame.len() != SILERO_FRAME_SAMPLES {+ if frame.len() != self.frame_samples {anyhow::bail!(- "expected {SILERO_FRAME_SAMPLES} samples, got {}",+ "expected {} samples, got {}",+ self.frame_samples,frame.len());}let result = self- .engine- .compute(frame)+ .session+ .get_mut()+ .unwrap_or_else(PoisonError::into_inner)+ .stream_feed(frame).map_err(|e| anyhow::anyhow!("Silero VAD error: {e}"))?;- if result.prob > self.threshold {+ // A whole frame in must score exactly one frame; anything else means+ // the stream is out of step with the recorder's framing.+ let [prob] = result.probs[..] else {+ anyhow::bail!(+ "Silero VAD returned {} probabilities for one frame",+ result.probs.len()+ );+ };++ if prob > self.threshold {Ok(VadFrame::Speech(frame))} else {Ok(VadFrame::Noise)@@ -51,12 +104,15 @@ impl VoiceActivityDetector for SileroVad {}fn frame_samples(&self) -> usize {- SILERO_FRAME_SAMPLES+ self.frame_samples}fn reset(&mut self) {- // Clear the Silero LSTM hidden/cell state so a new session doesn't- // inherit recurrent context from the previous recording.- self.engine.reset();+ // Clear the Silero LSTM state and frame context so a new session+ // doesn't inherit recurrent context from the previous recording.+ self.session+ .get_mut()+ .unwrap_or_else(PoisonError::into_inner)+ .stream_reset();}}diff --git a/src-tauri/src/managers/audio.rs b/src-tauri/src/managers/audio.rsindex 6e811b9e..c2e8db65 100644--- a/src-tauri/src/managers/audio.rs+++ b/src-tauri/src/managers/audio.rs@@ -330,7 +330,7 @@ fn create_capture_vad(let vad_path = app_handle.path().resolve(- "resources/models/silero_vad_v4.onnx",+ "resources/models/silero-vad-v6.2-F32.gguf",tauri::path::BaseDirectory::Resource,).map_err(|e| anyhow::anyhow!("Failed to resolve VAD path: {e}"))?;@@ -345,7 +345,7 @@ fn create_capture_vad(),};- // Earshot uses 16 ms frames while Silero uses 30 ms. Convert the existing+ // Earshot uses 16 ms frames while Silero uses 32 ms. Convert the existing// time-based capture profile to each detector's frame size so selecting a// backend does not shorten pre-roll, onset, or post-speech audio.let frame_samples = detector.frame_samples();