A free, open source, and extensible speech-to-text application that works completely offline.
activate the microphone on demand
CJ Pais6ad9534ab73c
diff --git a/src-tauri/src/actions.rs b/src-tauri/src/actions.rsindex 56ccd619..a131417a 100644--- a/src-tauri/src/actions.rs+++ b/src-tauri/src/actions.rs@@ -1,13 +1,16 @@use crate::managers::audio::AudioRecordingManager;use crate::managers::transcription::TranscriptionManager;+use crate::settings::get_settings;use crate::utils;use crate::utils::change_tray_icon;use crate::utils::play_recording_start_sound;use crate::utils::play_recording_stop_sound;use crate::utils::TrayIconState;+use log::debug;use once_cell::sync::Lazy;use std::collections::HashMap;use std::sync::Arc;+use std::time::Instant;use tauri::AppHandle;use tauri::Manager;@@ -22,17 +25,54 @@ struct TranscribeAction;impl ShortcutAction for TranscribeAction {fn start(&self, app: &AppHandle, binding_id: &str, _shortcut_str: &str) {+ let start_time = Instant::now();+ debug!("TranscribeAction::start called for binding: {}", binding_id);+let binding_id = binding_id.to_string();change_tray_icon(app, TrayIconState::Recording);- // Play audio feedback for recording start- play_recording_start_sound(app);-let rm = app.state::<Arc<AudioRecordingManager>>();- rm.try_start_recording(&binding_id);++ // Get the microphone mode to determine audio feedback timing+ let settings = get_settings(app);+ let is_always_on = settings.always_on_microphone;+ debug!("Microphone mode - always_on: {}", is_always_on);++ if is_always_on {+ // Always-on mode: Play audio feedback immediately+ debug!("Always-on mode: Playing audio feedback immediately");+ play_recording_start_sound(app);+ let recording_started = rm.try_start_recording(&binding_id);+ debug!("Recording started: {}", recording_started);+ } else {+ // On-demand mode: Start recording first, then play audio feedback+ // This allows the microphone to be activated before playing the sound+ debug!("On-demand mode: Starting recording first, then audio feedback");+ let recording_start_time = Instant::now();+ if rm.try_start_recording(&binding_id) {+ debug!("Recording started in {:?}", recording_start_time.elapsed());+ // Small delay to ensure microphone stream is active+ let app_clone = app.clone();+ std::thread::spawn(move || {+ std::thread::sleep(std::time::Duration::from_millis(100));+ debug!("Playing delayed audio feedback");+ play_recording_start_sound(&app_clone);+ });+ } else {+ debug!("Failed to start recording");+ }+ }++ debug!(+ "TranscribeAction::start completed in {:?}",+ start_time.elapsed()+ );}fn stop(&self, app: &AppHandle, binding_id: &str, _shortcut_str: &str) {+ let stop_time = Instant::now();+ debug!("TranscribeAction::stop called for binding: {}", binding_id);+let ah = app.clone();let rm = Arc::clone(&app.state::<Arc<AudioRecordingManager>>());let tm = Arc::clone(&app.state::<Arc<TranscriptionManager>>());@@ -46,16 +86,37 @@ impl ShortcutAction for TranscribeAction {tauri::async_runtime::spawn(async move {let binding_id = binding_id.clone(); // Clone for the inner async task+ debug!(+ "Starting async transcription task for binding: {}",+ binding_id+ );++ let stop_recording_time = Instant::now();if let Some(samples) = rm.stop_recording(&binding_id) {+ debug!(+ "Recording stopped and samples retrieved in {:?}, sample count: {}",+ stop_recording_time.elapsed(),+ samples.len()+ );++ let transcription_time = Instant::now();match tm.transcribe(samples) {Ok(transcription) => {- println!("Transcription Result: {}", transcription);+ debug!(+ "Transcription completed in {:?}: '{}'",+ transcription_time.elapsed(),+ transcription+ );if !transcription.is_empty() {let transcription_clone = transcription.clone();let ah_clone = ah.clone();+ let paste_time = Instant::now();ah.run_on_main_thread(move || {match utils::paste(transcription_clone, ah_clone) {- Ok(()) => println!("Text pasted successfully"),+ Ok(()) => debug!(+ "Text pasted successfully in {:?}",+ paste_time.elapsed()+ ),Err(e) => eprintln!("Failed to paste transcription: {}", e),}})@@ -64,10 +125,17 @@ impl ShortcutAction for TranscribeAction {});}}- Err(err) => println!("Global Shortcut Transcription error: {}", err),+ Err(err) => debug!("Global Shortcut Transcription error: {}", err),}+ } else {+ debug!("No samples retrieved from recording stop");}});++ debug!(+ "TranscribeAction::stop completed in {:?}",+ stop_time.elapsed()+ );}}diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rsnew file mode 100644index 00000000..1842b38e--- /dev/null+++ b/src-tauri/src/commands/audio.rs@@ -0,0 +1,29 @@+use crate::managers::audio::{AudioRecordingManager, MicrophoneMode};+use crate::settings::{get_settings, write_settings};+use std::sync::Arc;+use tauri::{AppHandle, Manager};++#[tauri::command]+pub fn update_microphone_mode(app: AppHandle, always_on: bool) -> Result<(), String> {+ // Update settings+ let mut settings = get_settings(&app);+ settings.always_on_microphone = always_on;+ write_settings(&app, settings);++ // Update the audio manager mode+ let rm = app.state::<Arc<AudioRecordingManager>>();+ let new_mode = if always_on {+ MicrophoneMode::AlwaysOn+ } else {+ MicrophoneMode::OnDemand+ };++ rm.update_mode(new_mode)+ .map_err(|e| format!("Failed to update microphone mode: {}", e))+}++#[tauri::command]+pub fn get_microphone_mode(app: AppHandle) -> Result<bool, String> {+ let settings = get_settings(&app);+ Ok(settings.always_on_microphone)+}diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rsindex c446ac88..2d8ee0a9 100644--- a/src-tauri/src/commands/mod.rs+++ b/src-tauri/src/commands/mod.rs@@ -1 +1,2 @@+pub mod audio;pub mod models;diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rsindex 4d350cdf..bbcd9af3 100644--- a/src-tauri/src/lib.rs+++ b/src-tauri/src/lib.rs@@ -185,7 +185,9 @@ pub fn run() {commands::models::is_model_loading,commands::models::has_any_models_available,commands::models::has_any_models_or_downloads,- commands::models::get_recommended_first_model+ commands::models::get_recommended_first_model,+ commands::audio::update_microphone_mode,+ commands::audio::get_microphone_mode]).run(tauri::generate_context!()).expect("error while running tauri application");diff --git a/src-tauri/src/managers/audio.rs b/src-tauri/src/managers/audio.rsindex f49a823f..8ba40cb8 100644--- a/src-tauri/src/managers/audio.rs+++ b/src-tauri/src/managers/audio.rs@@ -1,7 +1,10 @@+use crate::settings::get_settings;use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};use cpal::SampleFormat;+use log::{debug, info};use rubato::{FftFixedIn, Resampler};use std::sync::{Arc, Mutex};+use std::time::Instant;use std::vec::Vec;use tauri::{App, Manager};use vad_rs::Vad;@@ -14,10 +17,21 @@ pub enum RecordingState {Recording { binding_id: String },}+#[derive(Clone, Debug)]+pub enum MicrophoneMode {+ AlwaysOn,+ OnDemand,+}+#[derive(Clone)]pub struct AudioRecordingManager {state: Arc<Mutex<RecordingState>>,buffer: Arc<Mutex<Vec<f32>>>,+ mode: Arc<Mutex<MicrophoneMode>>,+ app_handle: tauri::AppHandle,+ // For on-demand mode+ stream_handle: Arc<Mutex<Option<std::thread::JoinHandle<()>>>>,+ stream_active: Arc<Mutex<bool>>,}trait SampleToF32 {@@ -50,31 +64,85 @@ impl SampleToF32 for f32 {impl AudioRecordingManager {pub fn new(app: &App) -> Result<Self, anyhow::Error> {- let vad_path = app.path().resolve(+ let settings = get_settings(&app.handle());+ let mode = if settings.always_on_microphone {+ MicrophoneMode::AlwaysOn+ } else {+ MicrophoneMode::OnDemand+ };++ let state = Arc::new(Mutex::new(RecordingState::Idle));+ let buffer = Arc::new(Mutex::new(Vec::new()));+ let mode_arc = Arc::new(Mutex::new(mode.clone()));+ let app_handle = app.handle().clone();+ let stream_handle = Arc::new(Mutex::new(None));+ let stream_active = Arc::new(Mutex::new(false));++ let manager = Self {+ state: state.clone(),+ buffer: buffer.clone(),+ mode: mode_arc,+ app_handle,+ stream_handle,+ stream_active,+ };++ // If always-on mode, start the stream immediately+ if matches!(mode, MicrophoneMode::AlwaysOn) {+ manager.start_microphone_stream()?;+ }++ Ok(manager)+ }++ pub fn start_microphone_stream(&self) -> Result<(), anyhow::Error> {+ let start_time = Instant::now();+ debug!("Starting microphone stream initialization...");++ let mut stream_active = self.stream_active.lock().unwrap();+ if *stream_active {+ debug!("Microphone stream already active, skipping initialization");+ return Ok(()); // Stream already active+ }++ let vad_path = self.app_handle.path().resolve("resources/models/silero_vad_v4.onnx",tauri::path::BaseDirectory::Resource,)?;+ debug!("VAD model path resolved: {:?}", vad_path);+let host = cpal::default_host();+ debug!("Audio host initialized");+let device = host.default_input_device().ok_or_else(|| anyhow::Error::msg("No input device available"))?;+ debug!("Default input device acquired: {:?}", device.name());let config = device.default_input_config()?;let sample_rate = config.sample_rate().0;+ debug!(+ "Audio config - Sample rate: {}, Channels: {}, Format: {:?}",+ sample_rate,+ config.channels(),+ config.sample_format()+ );// Configure the resampler - keeping 1024 as input size for FFT efficiency+ let resampler_start = Instant::now();let resampler = FftFixedIn::new(sample_rate as usize, WHISPER_SAMPLE_RATE, 1024, 2, 1)?;+ debug!("Resampler initialized in {:?}", resampler_start.elapsed());+ let vad_start = Instant::now();let vad = Arc::new(Mutex::new(Vad::new(vad_path, WHISPER_SAMPLE_RATE).unwrap()));+ debug!("VAD initialized in {:?}", vad_start.elapsed());let vad_clone = Arc::clone(&vad);- let state = Arc::new(Mutex::new(RecordingState::Idle));- let buffer = Arc::new(Mutex::new(Vec::new()));+ let state_clone = Arc::clone(&self.state);+ let buffer_clone = Arc::clone(&self.buffer);let resampler = Arc::new(Mutex::new(resampler));-- let state_clone = Arc::clone(&state);- let buffer_clone = Arc::clone(&buffer);let resampler_clone = Arc::clone(&resampler);+ let stream_active_clone = Arc::clone(&self.stream_active);// Create a temporary buffer to accumulate sampleslet temp_buffer = Arc::new(Mutex::new(Vec::new()));@@ -94,7 +162,15 @@ impl AudioRecordingManager {vad_buffer_clone: Arc<Mutex<Vec<f32>>>,buffer_clone: Arc<Mutex<Vec<f32>>>,vad_clone: Arc<Mutex<Vad>>,+ stream_active_clone: Arc<Mutex<bool>>,) {+ // Check if stream should still be active+ let stream_active = stream_active_clone.lock().unwrap();+ if !*stream_active {+ return;+ }+ drop(stream_active);+let state_guard = state_clone.lock().unwrap();if let RecordingState::Recording { .. } = *state_guard {let mut temp_buffer = temp_buffer_clone.lock().unwrap();@@ -153,101 +229,202 @@ impl AudioRecordingManager {}}- std::thread::spawn(move || {+ let handle = std::thread::spawn(move || {let err_fn = |err| eprintln!("Error in stream: {}", err);// Build the appropriate stream based on the sample format// Store the number of channels for use in the closurelet channels = config.channels() as usize;- println!("Using {} channel(s) for recording", channels);+ debug!(+ "Audio stream thread started - Using {} channel(s) for recording",+ channels+ );let stream = match config.sample_format() {- SampleFormat::I8 => device.build_input_stream(- &config.into(),- move |data: &[i8], _| {- process_audio(- data,- channels,- Arc::clone(&state_clone),- Arc::clone(&temp_buffer_clone),- Arc::clone(&resampler_clone),- Arc::clone(&vad_buffer_clone),- Arc::clone(&buffer_clone),- Arc::clone(&vad_clone),- )- },- err_fn,- None,- ),- SampleFormat::I16 => device.build_input_stream(- &config.into(),- move |data: &[i16], _| {- process_audio(- data,- channels,- Arc::clone(&state_clone),- Arc::clone(&temp_buffer_clone),- Arc::clone(&resampler_clone),- Arc::clone(&vad_buffer_clone),- Arc::clone(&buffer_clone),- Arc::clone(&vad_clone),- )- },- err_fn,- None,- ),- SampleFormat::I32 => device.build_input_stream(- &config.into(),- move |data: &[i32], _| {- process_audio(- data,- channels,- Arc::clone(&state_clone),- Arc::clone(&temp_buffer_clone),- Arc::clone(&resampler_clone),- Arc::clone(&vad_buffer_clone),- Arc::clone(&buffer_clone),- Arc::clone(&vad_clone),- )- },- err_fn,- None,- ),- SampleFormat::F32 => device.build_input_stream(- &config.into(),- move |data: &[f32], _| {- process_audio(- data,- channels,- Arc::clone(&state_clone),- Arc::clone(&temp_buffer_clone),- Arc::clone(&resampler_clone),- Arc::clone(&vad_buffer_clone),- Arc::clone(&buffer_clone),- Arc::clone(&vad_clone),- )- },- err_fn,- None,- ),+ SampleFormat::I8 => {+ let stream_active_i8 = Arc::clone(&stream_active_clone);+ device.build_input_stream(+ &config.into(),+ move |data: &[i8], _| {+ process_audio(+ data,+ channels,+ Arc::clone(&state_clone),+ Arc::clone(&temp_buffer_clone),+ Arc::clone(&resampler_clone),+ Arc::clone(&vad_buffer_clone),+ Arc::clone(&buffer_clone),+ Arc::clone(&vad_clone),+ Arc::clone(&stream_active_i8),+ )+ },+ err_fn,+ None,+ )+ }++ SampleFormat::I16 => {+ let stream_active_i16 = Arc::clone(&stream_active_clone);+ device.build_input_stream(+ &config.into(),+ move |data: &[i16], _| {+ process_audio(+ data,+ channels,+ Arc::clone(&state_clone),+ Arc::clone(&temp_buffer_clone),+ Arc::clone(&resampler_clone),+ Arc::clone(&vad_buffer_clone),+ Arc::clone(&buffer_clone),+ Arc::clone(&vad_clone),+ Arc::clone(&stream_active_i16),+ )+ },+ err_fn,+ None,+ )+ }++ SampleFormat::I32 => {+ let stream_active_i32 = Arc::clone(&stream_active_clone);+ device.build_input_stream(+ &config.into(),+ move |data: &[i32], _| {+ process_audio(+ data,+ channels,+ Arc::clone(&state_clone),+ Arc::clone(&temp_buffer_clone),+ Arc::clone(&resampler_clone),+ Arc::clone(&vad_buffer_clone),+ Arc::clone(&buffer_clone),+ Arc::clone(&vad_clone),+ Arc::clone(&stream_active_i32),+ )+ },+ err_fn,+ None,+ )+ }++ SampleFormat::F32 => {+ let stream_active_f32 = Arc::clone(&stream_active_clone);+ device.build_input_stream(+ &config.into(),+ move |data: &[f32], _| {+ process_audio(+ data,+ channels,+ Arc::clone(&state_clone),+ Arc::clone(&temp_buffer_clone),+ Arc::clone(&resampler_clone),+ Arc::clone(&vad_buffer_clone),+ Arc::clone(&buffer_clone),+ Arc::clone(&vad_clone),+ Arc::clone(&stream_active_f32),+ )+ },+ err_fn,+ None,+ )+ }+sample_format => {- // Use anyhow to return a proper error instead of panickingpanic!("Unsupported sample format: {:?}", sample_format);}}.expect("Failed to build input stream");+ debug!("Audio input stream built successfully");stream.play().expect("Failed to play stream");- std::thread::park();+ debug!("Audio stream started playing");++ // Keep the thread alive until stream_active becomes false+ let stream_active_for_loop = Arc::clone(&stream_active_clone);+ loop {+ std::thread::sleep(std::time::Duration::from_millis(100));+ let active = {+ let guard = stream_active_for_loop.lock().unwrap();+ *guard+ };+ if !active {+ break;+ }+ }});- Ok(Self { state, buffer })+ *self.stream_handle.lock().unwrap() = Some(handle);+ *stream_active = true;++ let total_time = start_time.elapsed();+ info!(+ "Microphone stream initialization completed in {:?}",+ total_time+ );+ Ok(())+ }++ pub fn stop_microphone_stream(&self) {+ let mut stream_active = self.stream_active.lock().unwrap();+ if !*stream_active {+ return; // Stream already stopped+ }++ *stream_active = false;+ drop(stream_active);++ // Wait for the stream thread to finish+ if let Some(handle) = self.stream_handle.lock().unwrap().take() {+ let _ = handle.join();+ }+ }++ pub fn update_mode(&self, new_mode: MicrophoneMode) -> Result<(), anyhow::Error> {+ let mut mode = self.mode.lock().unwrap();+ let current_mode = mode.clone();++ if matches!(current_mode, MicrophoneMode::AlwaysOn)+ && matches!(new_mode, MicrophoneMode::OnDemand)+ {+ // Switching from always-on to on-demand+ // Stop the stream if not currently recording+ let state = self.state.lock().unwrap();+ if matches!(*state, RecordingState::Idle) {+ drop(state);+ drop(mode);+ self.stop_microphone_stream();+ mode = self.mode.lock().unwrap();+ }+ } else if matches!(current_mode, MicrophoneMode::OnDemand)+ && matches!(new_mode, MicrophoneMode::AlwaysOn)+ {+ // Switching from on-demand to always-on+ drop(mode);+ self.start_microphone_stream()?;+ mode = self.mode.lock().unwrap();+ }++ *mode = new_mode;+ Ok(())}pub fn try_start_recording(&self, binding_id: &str) -> bool {let mut state = self.state.lock().unwrap();match *state {RecordingState::Idle => {+ // For on-demand mode, start the microphone stream now+ let mode = self.mode.lock().unwrap();+ if matches!(*mode, MicrophoneMode::OnDemand) {+ debug!("On-demand mode: Starting microphone stream for recording");+ drop(mode);+ drop(state);+ if let Err(e) = self.start_microphone_stream() {+ eprintln!("Failed to start microphone stream: {}", e);+ return false;+ }+ state = self.state.lock().unwrap();+ }+// Clear the buffer before starting new recordingself.buffer.lock().unwrap().clear();*state = RecordingState::Recording {@@ -278,6 +455,15 @@ impl AudioRecordingManager {*state = RecordingState::Idle;println!("Stopped recording for binding {}", binding_id);+ // For on-demand mode, stop the microphone stream+ let mode = self.mode.lock().unwrap();+ if matches!(*mode, MicrophoneMode::OnDemand) {+ debug!("On-demand mode: Stopping microphone stream after recording");+ drop(mode);+ drop(state);+ self.stop_microphone_stream();+ }+let mut buffer = self.buffer.lock().unwrap();let audio_data: Vec<f32> = buffer.drain(..).collect();diff --git a/src-tauri/src/settings.rs b/src-tauri/src/settings.rsindex 5a92cd96..cce0d678 100644--- a/src-tauri/src/settings.rs+++ b/src-tauri/src/settings.rs@@ -20,6 +20,8 @@ pub struct AppSettings {pub audio_feedback: bool,#[serde(default = "default_model")]pub selected_model: String,+ #[serde(default = "default_always_on_microphone")]+ pub always_on_microphone: bool,}fn default_model() -> String {@@ -28,6 +30,12 @@ fn default_model() -> String {"".to_string()}+fn default_always_on_microphone() -> bool {+ // Default to false for better user experience+ // True would be the old behavior (always-on for low latency)+ false+}+pub const SETTINGS_STORE_PATH: &str = "settings_store.json";pub fn get_default_settings() -> AppSettings {@@ -58,6 +66,7 @@ pub fn get_default_settings() -> AppSettings {push_to_talk: true,audio_feedback: false,selected_model: "".to_string(),+ always_on_microphone: false,}}