A free, open source, and extensible speech-to-text application that works completely offline.
use new backend
CJ Paisa60cf5787428
diff --git a/src-tauri/src/managers/audio.rs b/src-tauri/src/managers/audio.rsindex d0d2f27a..e5ce3465 100644--- a/src-tauri/src/managers/audio.rs+++ b/src-tauri/src/managers/audio.rs@@ -1,16 +1,14 @@use crate::settings::get_settings;-use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};-use cpal::SampleFormat;+use audio_toolkit::{vad::SmoothedVad, AudioRecorder, SileroVad};use log::{debug, info};-use rubato::{FftFixedIn, Resampler};use std::sync::{Arc, Mutex};use std::time::Instant;-use std::vec::Vec;use tauri::{App, Manager};-use vad_rs::Vad;const WHISPER_SAMPLE_RATE: usize = 16000;+/* ──────────────────────────────────────────────────────────────── */+#[derive(Clone, Debug)]pub enum RecordingState {Idle,@@ -23,46 +21,34 @@ pub enum MicrophoneMode {OnDemand,}+/* ──────────────────────────────────────────────────────────────── */++fn create_audio_recorder(vad_path: &str) -> Result<AudioRecorder, anyhow::Error> {+ let silero = SileroVad::new(vad_path, 0.5)+ .map_err(|e| anyhow::anyhow!("Failed to create SileroVad: {}", e))?;+ let smoothed_vad = SmoothedVad::new(Box::new(silero), 15, 15);+ let recorder = AudioRecorder::new()+ .map_err(|e| anyhow::anyhow!("Failed to create AudioRecorder: {}", e))?+ .with_vad(Box::new(smoothed_vad));+ Ok(recorder)+}++/* ──────────────────────────────────────────────────────────────── */+#[derive(Clone)]pub struct AudioRecordingManager {state: Arc<Mutex<RecordingState>>,- buffer: Arc<Mutex<Vec<f32>>>,mode: Arc<Mutex<MicrophoneMode>>,app_handle: tauri::AppHandle,- // For on-demand mode- stream_handle: Arc<Mutex<Option<std::thread::JoinHandle<()>>>>,- stream_active: Arc<Mutex<bool>>,-}-trait SampleToF32 {- fn to_f32(&self) -> f32;-}--impl SampleToF32 for i8 {- fn to_f32(&self) -> f32 {- *self as f32 / 128.0- }-}--impl SampleToF32 for i16 {- fn to_f32(&self) -> f32 {- *self as f32 / 32768.0- }-}--impl SampleToF32 for i32 {- fn to_f32(&self) -> f32 {- *self as f32 / 2147483648.0- }-}--impl SampleToF32 for f32 {- fn to_f32(&self) -> f32 {- *self- }+ recorder: Arc<Mutex<Option<AudioRecorder>>>,+ is_open: Arc<Mutex<bool>>,+ is_recording: Arc<Mutex<bool>>,}impl AudioRecordingManager {+ /* ---------- construction ------------------------------------------------ */+pub fn new(app: &App) -> Result<Self, anyhow::Error> {let settings = get_settings(&app.handle());let mode = if settings.always_on_microphone {@@ -71,23 +57,17 @@ impl AudioRecordingManager {MicrophoneMode::OnDemand};- let state = Arc::new(Mutex::new(RecordingState::Idle));- let buffer = Arc::new(Mutex::new(Vec::new()));- let mode_arc = Arc::new(Mutex::new(mode.clone()));- let app_handle = app.handle().clone();- let stream_handle = Arc::new(Mutex::new(None));- let stream_active = Arc::new(Mutex::new(false));-let manager = Self {- state: state.clone(),- buffer: buffer.clone(),- mode: mode_arc,- app_handle,- stream_handle,- stream_active,+ state: Arc::new(Mutex::new(RecordingState::Idle)),+ mode: Arc::new(Mutex::new(mode.clone())),+ app_handle: app.handle().clone(),++ recorder: Arc::new(Mutex::new(None)),+ is_open: Arc::new(Mutex::new(false)),+ is_recording: Arc::new(Mutex::new(false)),};- // If always-on mode, start the stream immediately+ // Always-on? Open immediately.if matches!(mode, MicrophoneMode::AlwaysOn) {manager.start_microphone_stream()?;}@@ -95,393 +75,159 @@ impl AudioRecordingManager {Ok(manager)}- pub fn start_microphone_stream(&self) -> Result<(), anyhow::Error> {- let start_time = Instant::now();- debug!("Starting microphone stream initialization...");+ /* ---------- microphone life-cycle -------------------------------------- */- let mut stream_active = self.stream_active.lock().unwrap();- if *stream_active {- debug!("Microphone stream already active, skipping initialization");- return Ok(()); // Stream already active+ pub fn start_microphone_stream(&self) -> Result<(), anyhow::Error> {+ let mut open_flag = self.is_open.lock().unwrap();+ if *open_flag {+ debug!("Microphone stream already active");+ return Ok(());}- let vad_path = self.app_handle.path().resolve(- "resources/models/silero_vad_v4.onnx",- tauri::path::BaseDirectory::Resource,- )?;- debug!("VAD model path resolved: {:?}", vad_path);-- let host = cpal::default_host();- debug!("Audio host initialized");-- let device = host- .default_input_device()- .ok_or_else(|| anyhow::Error::msg("No input device available"))?;- debug!("Default input device acquired: {:?}", device.name());-- let config = device.default_input_config()?;- let sample_rate = config.sample_rate().0;- debug!(- "Audio config - Sample rate: {}, Channels: {}, Format: {:?}",- sample_rate,- config.channels(),- config.sample_format()- );-- // Configure the resampler - keeping 1024 as input size for FFT efficiency- let resampler_start = Instant::now();- let resampler = FftFixedIn::new(sample_rate as usize, WHISPER_SAMPLE_RATE, 1024, 2, 1)?;- debug!("Resampler initialized in {:?}", resampler_start.elapsed());-- let vad_start = Instant::now();- let vad = Arc::new(Mutex::new(Vad::new(vad_path, WHISPER_SAMPLE_RATE).unwrap()));- debug!("VAD initialized in {:?}", vad_start.elapsed());- let vad_clone = Arc::clone(&vad);-- let state_clone = Arc::clone(&self.state);- let buffer_clone = Arc::clone(&self.buffer);- let resampler = Arc::new(Mutex::new(resampler));- let resampler_clone = Arc::clone(&resampler);- let stream_active_clone = Arc::clone(&self.stream_active);-- // Create a temporary buffer to accumulate samples- let temp_buffer = Arc::new(Mutex::new(Vec::new()));- let temp_buffer_clone = Arc::clone(&temp_buffer);-- // Create a buffer for resampled chunks waiting for VAD processing- let vad_buffer = Arc::new(Mutex::new(Vec::new()));- let vad_buffer_clone = Arc::clone(&vad_buffer);-- // Generic function to process audio data- fn process_audio<T: SampleToF32 + Send + 'static>(- data: &[T],- channels: usize,- state_clone: Arc<Mutex<RecordingState>>,- temp_buffer_clone: Arc<Mutex<Vec<f32>>>,- resampler_clone: Arc<Mutex<FftFixedIn<f32>>>,- vad_buffer_clone: Arc<Mutex<Vec<f32>>>,- buffer_clone: Arc<Mutex<Vec<f32>>>,- vad_clone: Arc<Mutex<Vad>>,- stream_active_clone: Arc<Mutex<bool>>,- ) {- // Check if stream should still be active- let stream_active = stream_active_clone.lock().unwrap();- if !*stream_active {- return;- }- drop(stream_active);-- let state_guard = state_clone.lock().unwrap();- if let RecordingState::Recording { .. } = *state_guard {- let mut temp_buffer = temp_buffer_clone.lock().unwrap();-- // Handle multichannel audio by mixing down to mono- if channels > 1 {- // Process chunks of `channels` samples at a time (each chunk is one audio frame)- for chunk in data.chunks(channels) {- // Average the channels to create a mono sample- let mono_sample: f32 =- chunk.iter().map(|s| s.to_f32()).sum::<f32>() / channels as f32;- temp_buffer.push(mono_sample);- }- } else {- // Single channel processing- let f32_data: Vec<f32> = data.iter().map(|sample| sample.to_f32()).collect();- temp_buffer.extend_from_slice(&f32_data);- }+ let start_time = Instant::now();- // Process when we have enough samples- while temp_buffer.len() >= 1024 {- // Take the first 1024 samples for processing- let chunk: Vec<f32> = temp_buffer.drain(..1024).collect();-- // Convert input data to the format expected by Rubato- let input_frames = vec![chunk];-- // Process the audio chunk through the resampler- let mut resampler = resampler_clone.lock().unwrap();- if let Ok(resampled) = resampler.process(&input_frames, None) {- // Add resampled data to VAD buffer- let mut vad_buffer = vad_buffer_clone.lock().unwrap();- vad_buffer.extend_from_slice(&resampled[0]);-- // Process 30ms chunks (480 samples) for VAD- while vad_buffer.len() >= 480 {- let chunk = vad_buffer.drain(..480).collect::<Vec<f32>>();-- // Use VAD to detect speech- if let Ok(mut vad) = vad_clone.lock() {- match vad.compute(&chunk) {- Ok(result) => {- if result.prob > 0.3 {- let mut buffer = buffer_clone.lock().unwrap();- buffer.extend_from_slice(&chunk);- }- }- Err(error) => {- eprintln!("Error computing VAD: {:?}", error)- }- }- }- }- }- }- }+ let vad_path = self+ .app_handle+ .path()+ .resolve(+ "resources/models/silero_vad_v4.onnx",+ tauri::path::BaseDirectory::Resource,+ )+ .map_err(|e| anyhow::anyhow!("Failed to resolve VAD path: {}", e))?;+ let mut recorder_opt = self.recorder.lock().unwrap();++ if recorder_opt.is_none() {+ *recorder_opt = Some(create_audio_recorder(vad_path.to_str().unwrap())?);}- let handle = std::thread::spawn(move || {- let err_fn = |err| eprintln!("Error in stream: {}", err);-- // Build the appropriate stream based on the sample format- // Store the number of channels for use in the closure- let channels = config.channels() as usize;- debug!(- "Audio stream thread started - Using {} channel(s) for recording",- channels- );-- let stream = match config.sample_format() {- SampleFormat::I8 => {- let stream_active_i8 = Arc::clone(&stream_active_clone);- device.build_input_stream(- &config.into(),- move |data: &[i8], _| {- process_audio(- data,- channels,- Arc::clone(&state_clone),- Arc::clone(&temp_buffer_clone),- Arc::clone(&resampler_clone),- Arc::clone(&vad_buffer_clone),- Arc::clone(&buffer_clone),- Arc::clone(&vad_clone),- Arc::clone(&stream_active_i8),- )- },- err_fn,- None,- )- }-- SampleFormat::I16 => {- let stream_active_i16 = Arc::clone(&stream_active_clone);- device.build_input_stream(- &config.into(),- move |data: &[i16], _| {- process_audio(- data,- channels,- Arc::clone(&state_clone),- Arc::clone(&temp_buffer_clone),- Arc::clone(&resampler_clone),- Arc::clone(&vad_buffer_clone),- Arc::clone(&buffer_clone),- Arc::clone(&vad_clone),- Arc::clone(&stream_active_i16),- )- },- err_fn,- None,- )- }-- SampleFormat::I32 => {- let stream_active_i32 = Arc::clone(&stream_active_clone);- device.build_input_stream(- &config.into(),- move |data: &[i32], _| {- process_audio(- data,- channels,- Arc::clone(&state_clone),- Arc::clone(&temp_buffer_clone),- Arc::clone(&resampler_clone),- Arc::clone(&vad_buffer_clone),- Arc::clone(&buffer_clone),- Arc::clone(&vad_clone),- Arc::clone(&stream_active_i32),- )- },- err_fn,- None,- )- }-- SampleFormat::F32 => {- let stream_active_f32 = Arc::clone(&stream_active_clone);- device.build_input_stream(- &config.into(),- move |data: &[f32], _| {- process_audio(- data,- channels,- Arc::clone(&state_clone),- Arc::clone(&temp_buffer_clone),- Arc::clone(&resampler_clone),- Arc::clone(&vad_buffer_clone),- Arc::clone(&buffer_clone),- Arc::clone(&vad_clone),- Arc::clone(&stream_active_f32),- )- },- err_fn,- None,- )- }-- sample_format => {- panic!("Unsupported sample format: {:?}", sample_format);- }- }- .expect("Failed to build input stream");-- debug!("Audio input stream built successfully");- stream.play().expect("Failed to play stream");- debug!("Audio stream started playing");-- // Keep the thread alive until stream_active becomes false- let stream_active_for_loop = Arc::clone(&stream_active_clone);- loop {- std::thread::sleep(std::time::Duration::from_millis(100));- let active = {- let guard = stream_active_for_loop.lock().unwrap();- *guard- };- if !active {- break;- }- }- });-- *self.stream_handle.lock().unwrap() = Some(handle);- *stream_active = true;+ if let Some(rec) = recorder_opt.as_mut() {+ rec.open(None)+ .map_err(|e| anyhow::anyhow!("Failed to open recorder: {}", e))?;+ }- let total_time = start_time.elapsed();+ *open_flag = true;info!(- "Microphone stream initialization completed in {:?}",- total_time+ "Microphone stream initialized in {:?}",+ start_time.elapsed());Ok(())}pub fn stop_microphone_stream(&self) {- let mut stream_active = self.stream_active.lock().unwrap();- if !*stream_active {- return; // Stream already stopped+ let mut open_flag = self.is_open.lock().unwrap();+ if !*open_flag {+ return;}- *stream_active = false;- drop(stream_active);-- // Wait for the stream thread to finish- if let Some(handle) = self.stream_handle.lock().unwrap().take() {- let _ = handle.join();+ if let Some(rec) = self.recorder.lock().unwrap().as_mut() {+ // If still recording, stop first.+ if *self.is_recording.lock().unwrap() {+ let _ = rec.stop();+ *self.is_recording.lock().unwrap() = false;+ }+ let _ = rec.close();}++ *open_flag = false;+ debug!("Microphone stream stopped");}+ /* ---------- mode switching --------------------------------------------- */+pub fn update_mode(&self, new_mode: MicrophoneMode) -> Result<(), anyhow::Error> {- let mut mode = self.mode.lock().unwrap();- let current_mode = mode.clone();-- if matches!(current_mode, MicrophoneMode::AlwaysOn)- && matches!(new_mode, MicrophoneMode::OnDemand)- {- // Switching from always-on to on-demand- // Stop the stream if not currently recording- let state = self.state.lock().unwrap();- if matches!(*state, RecordingState::Idle) {- drop(state);- drop(mode);- self.stop_microphone_stream();- mode = self.mode.lock().unwrap();+ let mode_guard = self.mode.lock().unwrap();+ let cur_mode = mode_guard.clone();++ match (cur_mode, &new_mode) {+ (MicrophoneMode::AlwaysOn, MicrophoneMode::OnDemand) => {+ if matches!(*self.state.lock().unwrap(), RecordingState::Idle) {+ drop(mode_guard);+ self.stop_microphone_stream();+ }+ }+ (MicrophoneMode::OnDemand, MicrophoneMode::AlwaysOn) => {+ drop(mode_guard);+ self.start_microphone_stream()?;}- } else if matches!(current_mode, MicrophoneMode::OnDemand)- && matches!(new_mode, MicrophoneMode::AlwaysOn)- {- // Switching from on-demand to always-on- drop(mode);- self.start_microphone_stream()?;- mode = self.mode.lock().unwrap();+ _ => {}}- *mode = new_mode;+ *self.mode.lock().unwrap() = new_mode;Ok(())}+ /* ---------- recording --------------------------------------------------- */+pub fn try_start_recording(&self, binding_id: &str) -> bool {let mut state = self.state.lock().unwrap();- match *state {- RecordingState::Idle => {- // For on-demand mode, start the microphone stream now- let mode = self.mode.lock().unwrap();- if matches!(*mode, MicrophoneMode::OnDemand) {- debug!("On-demand mode: Starting microphone stream for recording");- drop(mode);- drop(state);- if let Err(e) = self.start_microphone_stream() {- eprintln!("Failed to start microphone stream: {}", e);- return false;- }- state = self.state.lock().unwrap();- }- // Clear the buffer before starting new recording- self.buffer.lock().unwrap().clear();- *state = RecordingState::Recording {- binding_id: binding_id.to_string(),- };- println!("Started recording for binding {}", binding_id);- true+ if let RecordingState::Idle = *state {+ // Ensure microphone is open in on-demand mode+ if matches!(*self.mode.lock().unwrap(), MicrophoneMode::OnDemand) {+ if let Err(e) = self.start_microphone_stream() {+ eprintln!("Failed to open microphone stream: {e}");+ return false;+ }}- RecordingState::Recording {- binding_id: ref active_id,- } => {- println!(- "Cannot start recording: already recording for binding {}",- active_id- );- false++ if let Some(rec) = self.recorder.lock().unwrap().as_ref() {+ if rec.start().is_ok() {+ *self.is_recording.lock().unwrap() = true;+ *state = RecordingState::Recording {+ binding_id: binding_id.to_string(),+ };+ debug!("Recording started for binding {binding_id}");+ return true;+ }}+ eprintln!("Recorder not available");+ false+ } else {+ false}}pub fn stop_recording(&self, binding_id: &str) -> Option<Vec<f32>> {let mut state = self.state.lock().unwrap();- println!("Stop recording called from binding {}", binding_id);+match *state {RecordingState::Recording {- binding_id: ref active_id,- } if active_id == binding_id => {+ binding_id: ref active,+ } if active == binding_id => {*state = RecordingState::Idle;- println!("Stopped recording for binding {}", binding_id);-- // For on-demand mode, stop the microphone stream- let mode = self.mode.lock().unwrap();- if matches!(*mode, MicrophoneMode::OnDemand) {- debug!("On-demand mode: Stopping microphone stream after recording");- drop(mode);- drop(state);- self.stop_microphone_stream();- }+ drop(state);++ let samples = if let Some(rec) = self.recorder.lock().unwrap().as_ref() {+ match rec.stop() {+ Ok(buf) => buf,+ Err(e) => {+ eprintln!("stop() failed: {e}");+ Vec::new()+ }+ }+ } else {+ eprintln!("Recorder not available");+ Vec::new()+ };- let mut buffer = self.buffer.lock().unwrap();- let audio_data: Vec<f32> = buffer.drain(..).collect();+ *self.is_recording.lock().unwrap() = false;- let samples = audio_data.len();+ // In on-demand mode turn the mic off again+ if matches!(*self.mode.lock().unwrap(), MicrophoneMode::OnDemand) {+ self.stop_microphone_stream();+ }- if samples < WHISPER_SAMPLE_RATE && samples > 1000 {- let target_samples = WHISPER_SAMPLE_RATE * 5 / 4; // sample rate * 1.25- let mut padded_audio = audio_data;- padded_audio.resize(target_samples, 0.0); // Pad with silence (zeros)- Some(padded_audio)+ // Pad if very short+ let s_len = samples.len();+ if s_len < WHISPER_SAMPLE_RATE && s_len > 1000 {+ let mut padded = samples;+ padded.resize(WHISPER_SAMPLE_RATE * 5 / 4, 0.0);+ Some(padded)} else {- Some(audio_data)+ Some(samples)}}- _ => {- // println!("Cannot stop recording: not recording or wrong binding");- None- }+ _ => None,}}}