A free, open source, and extensible speech-to-text application that works completely offline.
use anyhow::Result;
use std::path::Path;
use std::sync::{Mutex, PoisonError};
use transcribe_cpp::{Backend, Model, ModelOptions, VadSession, VadSessionOptions};
use super::{check_whole_frames, SpeechScorer, VadFrame, VoiceActivityDetector};
use crate::audio_toolkit::constants;
use crate::audio_toolkit::cpu_engine::init_cpu_backends;
pub struct SileroVad {
// `VadSession` is Send but not Sync; the trait requires both. Only ever
// reached through `get_mut`, so it is never locked and never poisoned.
session: Mutex<VadSession>,
frame_samples: usize,
threshold: f32,
}
impl SileroVad {
pub fn new<P: AsRef<Path>>(model_path: P, threshold: f32) -> Result<Self> {
if !(0.0..=1.0).contains(&threshold) {
anyhow::bail!("threshold must be between 0.0 and 1.0");
}
init_cpu_backends()?;
let model = Model::load_with(
model_path.as_ref(),
&ModelOptions {
backend: Backend::Cpu,
device: None,
},
)
.map_err(|e| anyhow::anyhow!("Failed to load VAD model: {e}"))?;
let info = model
.vad_info()
.map_err(|e| anyhow::anyhow!("Model does not support VAD: {e}"))?;
if info.sample_rate != constants::WHISPER_SAMPLE_RATE as i32 || info.frame_samples <= 0 {
anyhow::bail!(
"unsupported VAD model: {} Hz, {} samples per frame",
info.sample_rate,
info.frame_samples
);
}
// The session keeps the native model alive on its own.
let session = model
.vad_session_with(&VadSessionOptions { n_threads: 1 })
.map_err(|e| anyhow::anyhow!("Failed to create VAD session: {e}"))?;
Ok(Self {
session: Mutex::new(session),
frame_samples: info.frame_samples as usize,
threshold,
})
}
}
impl VoiceActivityDetector for SileroVad {
fn push_frame<'a>(&'a mut self, frame: &'a [f32]) -> Result<VadFrame<'a>> {
if frame.len() != self.frame_samples {
anyhow::bail!(
"expected {} samples, got {}",
self.frame_samples,
frame.len()
);
}
if self.score(frame)?[0] > self.threshold {
Ok(VadFrame::Speech(frame))
} else {
Ok(VadFrame::Noise)
}
}
fn frame_samples(&self) -> usize {
self.frame_samples
}
fn reset(&mut self) {
// Clear the Silero LSTM state and frame context so a new session
// doesn't inherit recurrent context from the previous recording.
self.session
.get_mut()
.unwrap_or_else(PoisonError::into_inner)
.stream_reset();
}
}
impl SpeechScorer for SileroVad {
fn score(&mut self, frames: &[f32]) -> Result<Vec<f32>> {
check_whole_frames(frames, self.frame_samples)?;
let result = self
.session
.get_mut()
.unwrap_or_else(PoisonError::into_inner)
.stream_feed(frames)
.map_err(|e| anyhow::anyhow!("Silero VAD error: {e}"))?;
// Whole frames in must score exactly that many frames; anything else
// means the stream is out of step with the caller's framing.
let expected = frames.len() / self.frame_samples;
if result.probs.len() != expected {
anyhow::bail!(
"Silero VAD returned {} probabilities for {expected} frames",
result.probs.len()
);
}
Ok(result.probs)
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Needs the bundled model (see AGENTS.md); run with `--ignored`.
#[test]
#[ignore]
fn scores_the_same_in_pieces_and_frame_by_frame() {
let path = concat!(
env!("CARGO_MANIFEST_DIR"),
"/resources/models/silero-vad-v6.2-F32.gguf"
);
let mut vad = SileroVad::new(path, 0.3).unwrap();
let frame = vad.frame_samples();
let pcm: Vec<f32> = (0..frame * 600)
.map(|i| (i as f32 * 0.013).sin() * (i as f32 * 0.0007).cos() * 0.2)
.collect();
let whole = vad.score(&pcm).unwrap();
assert_eq!(whole.len(), 600);
let mut pieces = SileroVad::new(path, 0.3).unwrap();
let mut scored = pieces.score(&pcm[..frame * 250]).unwrap();
scored.extend(pieces.score(&pcm[frame * 250..]).unwrap());
assert_eq!(scored, whole);
let mut by_frame = SileroVad::new(path, 0.3).unwrap();
let fed: Vec<f32> = pcm
.chunks(frame)
.flat_map(|chunk| by_frame.score(chunk).unwrap())
.collect();
assert_eq!(fed, whole);
assert!(vad.score(&pcm[..frame + 1]).is_err());
}
}