Skip to content

armgit / Handy

Public Imported from GitHub
0

A free, open source, and extensible speech-to-text application that works completely offline.

transcribe.cpp vad (#2267)

CJ Pais3985508ae7cf
diff --git a/AGENTS.md b/AGENTS.md
index eda5426b..88ae7b52 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -44,7 +44,7 @@ bun run format:backend # cargo fmt only
```bash
mkdir -p src-tauri/resources/models
-curl -o src-tauri/resources/models/silero_vad_v4.onnx https://blob.handy.computer/silero_vad_v4.onnx
+curl -L -o src-tauri/resources/models/silero-vad-v6.2-F32.gguf https://huggingface.co/handy-computer/silero-vad-v6.2-gguf/resolve/main/silero-vad-v6.2-F32.gguf
```
For detailed platform-specific build setup, see [BUILD.md](BUILD.md).
@@ -105,7 +105,7 @@ Handy is a cross-platform desktop speech-to-text application built with Tauri 2.
- `transcribe-cpp` - Local Whisper-family inference (GGML/GGUF) with GPU acceleration
- `transcribe-rs` - ONNX speech recognition (Parakeet, Moonshine, SenseVoice, etc.)
- `handy-recorder` - Microphone capture: device selection, resampling to 16 kHz, failure detection
-- `vad-rs` - Voice Activity Detection
+- `transcribe-cpp` (Silero VAD v6, CPU-only, in-process) / `earshot` - Voice Activity Detection
- `rdev` - Global keyboard shortcuts
- `rodio` - Audio playback for feedback sounds
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 9c14c8bc..31c59fc0 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -54,7 +54,7 @@ Before you begin, ensure you have the following installed:
```bash
mkdir -p src-tauri/resources/models
- curl -o src-tauri/resources/models/silero_vad_v4.onnx https://blob.handy.computer/silero_vad_v4.onnx
+ curl -L -o src-tauri/resources/models/silero-vad-v6.2-F32.gguf https://huggingface.co/handy-computer/silero-vad-v6.2-gguf/resolve/main/silero-vad-v6.2-F32.gguf
```
6. **Run in development mode**:
diff --git a/CRUSH.md b/CRUSH.md
deleted file mode 100644
index ab03ffff..00000000
--- a/CRUSH.md
+++ /dev/null
@@ -1,65 +0,0 @@
-# Development Commands
-
-**Environment Setup:**
-
-```bash
-bun install # Install dependencies
-mkdir -p src-tauri/resources/models
-curl -o src-tauri/resources/models/silero_vad_v4.onnx https://blob.handy.computer/silero_vad_v4.onnx
-```
-
-**Development:**
-
-```bash
-bun run tauri dev # Full app development
-CMAKE_POLICY_VERSION_MINIMUM=3.5 bun run tauri dev # macOS with cmake fix
-bun run dev # Frontend only (Vite)
-bun run build # Build frontend
-bun run tauri build # Production build
-```
-
-**Type Check & Build:**
-
-```bash
-bunx tsc --noEmit # Type checking
-bun run build # Build and validate
-```
-
-# Code Style Guidelines
-
-**Rust (Backend):**
-
-- Use `anyhow::Error` for error handling with descriptive messages
-- Prefer `Arc<Mutex<T>>` for shared state in managers
-- Log with appropriate levels: `debug!`, `info!`, `eprintln!` for errors
-- Builder pattern for initialization chains
-- Snake_case for functions and variables, PascalCase for types
-- Separate logical sections with comment blocks: `/* ─────────── */`
-
-**TypeScript/React (Frontend):**
-
-- Functional components with TypeScript interfaces
-- Zod schemas for type validation and inference
-- `useCallback` hooks for stable function references
-- Destructure props with defaults: `disabled = false`
-- Prefer interface aliases over type aliases for objects
-- React.FC for explicit component typing
-- PascalCase for components, camelCase for variables/functions
-
-**Imports:**
-
-- Group imports: external libs, internal modules, relative imports
-- Use type imports for TypeScript: `import type { Settings }`
-- Named imports preferred over default exports
-
-**Error Handling:**
-
-- Frontend: Try/catch with user feedback, rollback optimistic updates
-- Backend: `?` operator with anyhow context messages
-- Log errors appropriately for debugging level
-
-**Component Patterns:**
-
-- Container component pattern for layout
-- Composition over inheritance
-- Prop drilling minimized with context where appropriate
diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock
index a96062c6..d3cabb77 100644
--- a/src-tauri/Cargo.lock
+++ b/src-tauri/Cargo.lock
@@ -1815,16 +1815,6 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
-[[package]]
-name = "eyre"
-version = "0.6.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec"
-dependencies = [
- "indenter",
- "once_cell",
-]
-
[[package]]
name = "fallible-iterator"
version = "0.3.0"
@@ -2611,7 +2601,6 @@ dependencies = [
"tokio",
"transcribe-cpp",
"transcribe-rs",
- "vad-rs",
"webview2-com",
"whatlang",
"windows 0.61.3",
@@ -3108,12 +3097,6 @@ dependencies = [
"tiff",
]
-[[package]]
-name = "indenter"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5"
-
[[package]]
name = "indexmap"
version = "1.9.3"
@@ -5384,12 +5367,6 @@ dependencies = [
"windows-sys 0.52.0",
]
-[[package]]
-name = "ringbuffer"
-version = "0.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3df6368f71f205ff9c33c076d170dd56ebf68e8161c733c0caa07a7a5509ed53"
-
[[package]]
name = "rkyv"
version = "0.7.46"
@@ -7753,17 +7730,6 @@ dependencies = [
"wasm-bindgen",
]
-[[package]]
-name = "vad-rs"
-version = "0.1.6"
-source = "git+https://github.com/cjpais/vad-rs#2a412ed858695b9251f3f5a1a20d95b59fa7c498"
-dependencies = [
- "eyre",
- "ndarray",
- "ort",
- "ringbuffer",
-]
-
[[package]]
name = "value-bag"
version = "1.12.0"
diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml
index 9ec8e8f2..9def144f 100644
--- a/src-tauri/Cargo.toml
+++ b/src-tauri/Cargo.toml
@@ -49,7 +49,6 @@ hound = "3.5.1"
log = "0.4.25"
env_filter = "0.1.0"
tokio = "1.43.0"
-vad-rs = { git = "https://github.com/cjpais/vad-rs", default-features = false }
earshot = "1.2.2"
enigo = "0.6.1"
# Feedback sounds. No `recording`: microphone capture is handy-recorder's.
diff --git a/src-tauri/resources/models/silero-vad-v6.2-F32.gguf b/src-tauri/resources/models/silero-vad-v6.2-F32.gguf
new file mode 100644
index 00000000..1c85efc4
Binary files /dev/null and b/src-tauri/resources/models/silero-vad-v6.2-F32.gguf differ
diff --git a/src-tauri/resources/models/silero_vad_v4.onnx b/src-tauri/resources/models/silero_vad_v4.onnx
deleted file mode 100644
index e6db48d6..00000000
Binary files a/src-tauri/resources/models/silero_vad_v4.onnx and /dev/null differ
diff --git a/src-tauri/src/audio_toolkit/vad/mod.rs b/src-tauri/src/audio_toolkit/vad/mod.rs
index f1abc698..f1bdbd1b 100644
--- a/src-tauri/src/audio_toolkit/vad/mod.rs
+++ b/src-tauri/src/audio_toolkit/vad/mod.rs
@@ -85,10 +85,10 @@ mod tests {
#[test]
fn duration_profiles_preserve_silero_timings() {
- assert_eq!(frames_for_duration_ms(VAD_PREFILL_MS, 480), 15);
- assert_eq!(frames_for_duration_ms(VAD_OFFLINE_HANGOVER_MS, 480), 15);
- assert_eq!(frames_for_duration_ms(VAD_STREAMING_HANGOVER_MS, 480), 55);
- assert_eq!(frames_for_duration_ms(VAD_ONSET_MS, 480), 2);
+ assert_eq!(frames_for_duration_ms(VAD_PREFILL_MS, 512), 15);
+ assert_eq!(frames_for_duration_ms(VAD_OFFLINE_HANGOVER_MS, 512), 15);
+ assert_eq!(frames_for_duration_ms(VAD_STREAMING_HANGOVER_MS, 512), 52);
+ assert_eq!(frames_for_duration_ms(VAD_ONSET_MS, 512), 2);
}
#[test]
diff --git a/src-tauri/src/audio_toolkit/vad/silero.rs b/src-tauri/src/audio_toolkit/vad/silero.rs
index 4fc64eae..1785e5f0 100644
--- a/src-tauri/src/audio_toolkit/vad/silero.rs
+++ b/src-tauri/src/audio_toolkit/vad/silero.rs
@@ -1,29 +1,70 @@
use anyhow::Result;
use std::path::Path;
+use std::sync::{Mutex, OnceLock, PoisonError};
-use vad_rs::Vad;
+use transcribe_cpp::{Backend, BackendMask, Model, ModelOptions, VadSession, VadSessionOptions};
use super::{VadFrame, VoiceActivityDetector};
use crate::audio_toolkit::constants;
-const SILERO_FRAME_MS: u32 = 30;
-const SILERO_FRAME_SAMPLES: usize =
- (constants::WHISPER_SAMPLE_RATE * SILERO_FRAME_MS / 1000) as usize;
-
pub struct SileroVad {
- engine: Vad,
+ // `VadSession` is Send but not Sync; the trait requires both. Only ever
+ // reached through `get_mut`, so it is never locked and never poisoned.
+ session: Mutex<VadSession>,
+ frame_samples: usize,
threshold: f32,
}
+/// Register transcribe.cpp's CPU backends once per process. The allowed mask
+/// is fixed at first registration, so it must never include a GPU here.
+fn init_cpu_backends() -> Result<()> {
+ static INIT: OnceLock<Result<(), String>> = OnceLock::new();
+ INIT.get_or_init(|| {
+ // Route native/ggml diagnostics into Handy's log; must precede any
+ // native call.
+ transcribe_cpp::init_logging();
+ transcribe_cpp::init_backends_with(None::<&Path>, BackendMask::CPU)
+ .map_err(|e| e.to_string())
+ })
+ .clone()
+ .map_err(|e| anyhow::anyhow!("Failed to initialize transcribe.cpp CPU backend: {e}"))
+}
+
impl SileroVad {
pub fn new<P: AsRef<Path>>(model_path: P, threshold: f32) -> Result<Self> {
if !(0.0..=1.0).contains(&threshold) {
anyhow::bail!("threshold must be between 0.0 and 1.0");
}
+ init_cpu_backends()?;
+ let model = Model::load_with(
+ model_path.as_ref(),
+ &ModelOptions {
+ backend: Backend::Cpu,
+ device: None,
+ },
+ )
+ .map_err(|e| anyhow::anyhow!("Failed to load VAD model: {e}"))?;
+
+ let info = model
+ .vad_info()
+ .map_err(|e| anyhow::anyhow!("Model does not support VAD: {e}"))?;
+ if info.sample_rate != constants::WHISPER_SAMPLE_RATE as i32 || info.frame_samples <= 0 {
+ anyhow::bail!(
+ "unsupported VAD model: {} Hz, {} samples per frame",
+ info.sample_rate,
+ info.frame_samples
+ );
+ }
+
+ // The session keeps the native model alive on its own.
+ let session = model
+ .vad_session_with(&VadSessionOptions { n_threads: 1 })
+ .map_err(|e| anyhow::anyhow!("Failed to create VAD session: {e}"))?;
+
Ok(Self {
- engine: Vad::new(&model_path, constants::WHISPER_SAMPLE_RATE as usize)
- .map_err(|e| anyhow::anyhow!("Failed to create VAD: {e}"))?,
+ session: Mutex::new(session),
+ frame_samples: info.frame_samples as usize,
threshold,
})
}
@@ -31,19 +72,31 @@ impl SileroVad {
impl VoiceActivityDetector for SileroVad {
fn push_frame<'a>(&'a mut self, frame: &'a [f32]) -> Result<VadFrame<'a>> {
- if frame.len() != SILERO_FRAME_SAMPLES {
+ if frame.len() != self.frame_samples {
anyhow::bail!(
- "expected {SILERO_FRAME_SAMPLES} samples, got {}",
+ "expected {} samples, got {}",
+ self.frame_samples,
frame.len()
);
}
let result = self
- .engine
- .compute(frame)
+ .session
+ .get_mut()
+ .unwrap_or_else(PoisonError::into_inner)
+ .stream_feed(frame)
.map_err(|e| anyhow::anyhow!("Silero VAD error: {e}"))?;
- if result.prob > self.threshold {
+ // A whole frame in must score exactly one frame; anything else means
+ // the stream is out of step with the recorder's framing.
+ let [prob] = result.probs[..] else {
+ anyhow::bail!(
+ "Silero VAD returned {} probabilities for one frame",
+ result.probs.len()
+ );
+ };
+
+ if prob > self.threshold {
Ok(VadFrame::Speech(frame))
} else {
Ok(VadFrame::Noise)
@@ -51,12 +104,15 @@ impl VoiceActivityDetector for SileroVad {
}
fn frame_samples(&self) -> usize {
- SILERO_FRAME_SAMPLES
+ self.frame_samples
}
fn reset(&mut self) {
- // Clear the Silero LSTM hidden/cell state so a new session doesn't
- // inherit recurrent context from the previous recording.
- self.engine.reset();
+ // Clear the Silero LSTM state and frame context so a new session
+ // doesn't inherit recurrent context from the previous recording.
+ self.session
+ .get_mut()
+ .unwrap_or_else(PoisonError::into_inner)
+ .stream_reset();
}
}
diff --git a/src-tauri/src/managers/audio.rs b/src-tauri/src/managers/audio.rs
index 6e811b9e..c2e8db65 100644
--- a/src-tauri/src/managers/audio.rs
+++ b/src-tauri/src/managers/audio.rs
@@ -330,7 +330,7 @@ fn create_capture_vad(
let vad_path = app_handle
.path()
.resolve(
- "resources/models/silero_vad_v4.onnx",
+ "resources/models/silero-vad-v6.2-F32.gguf",
tauri::path::BaseDirectory::Resource,
)
.map_err(|e| anyhow::anyhow!("Failed to resolve VAD path: {e}"))?;
@@ -345,7 +345,7 @@ fn create_capture_vad(
),
};
- // Earshot uses 16 ms frames while Silero uses 30 ms. Convert the existing
+ // Earshot uses 16 ms frames while Silero uses 32 ms. Convert the existing
// time-based capture profile to each detector's frame size so selecting a
// backend does not shorten pre-roll, onset, or post-speech audio.
let frame_samples = detector.frame_samples();