Skip to content

armgit / Handy

Public Imported from GitHub
0

A free, open source, and extensible speech-to-text application that works completely offline.

activate the microphone on demand

CJ Pais6ad9534ab73c
diff --git a/src-tauri/src/actions.rs b/src-tauri/src/actions.rs
index 56ccd619..a131417a 100644
--- a/src-tauri/src/actions.rs
+++ b/src-tauri/src/actions.rs
@@ -1,13 +1,16 @@
use crate::managers::audio::AudioRecordingManager;
use crate::managers::transcription::TranscriptionManager;
+use crate::settings::get_settings;
use crate::utils;
use crate::utils::change_tray_icon;
use crate::utils::play_recording_start_sound;
use crate::utils::play_recording_stop_sound;
use crate::utils::TrayIconState;
+use log::debug;
use once_cell::sync::Lazy;
use std::collections::HashMap;
use std::sync::Arc;
+use std::time::Instant;
use tauri::AppHandle;
use tauri::Manager;
@@ -22,17 +25,54 @@ struct TranscribeAction;
impl ShortcutAction for TranscribeAction {
fn start(&self, app: &AppHandle, binding_id: &str, _shortcut_str: &str) {
+ let start_time = Instant::now();
+ debug!("TranscribeAction::start called for binding: {}", binding_id);
+
let binding_id = binding_id.to_string();
change_tray_icon(app, TrayIconState::Recording);
- // Play audio feedback for recording start
- play_recording_start_sound(app);
-
let rm = app.state::<Arc<AudioRecordingManager>>();
- rm.try_start_recording(&binding_id);
+
+ // Get the microphone mode to determine audio feedback timing
+ let settings = get_settings(app);
+ let is_always_on = settings.always_on_microphone;
+ debug!("Microphone mode - always_on: {}", is_always_on);
+
+ if is_always_on {
+ // Always-on mode: Play audio feedback immediately
+ debug!("Always-on mode: Playing audio feedback immediately");
+ play_recording_start_sound(app);
+ let recording_started = rm.try_start_recording(&binding_id);
+ debug!("Recording started: {}", recording_started);
+ } else {
+ // On-demand mode: Start recording first, then play audio feedback
+ // This allows the microphone to be activated before playing the sound
+ debug!("On-demand mode: Starting recording first, then audio feedback");
+ let recording_start_time = Instant::now();
+ if rm.try_start_recording(&binding_id) {
+ debug!("Recording started in {:?}", recording_start_time.elapsed());
+ // Small delay to ensure microphone stream is active
+ let app_clone = app.clone();
+ std::thread::spawn(move || {
+ std::thread::sleep(std::time::Duration::from_millis(100));
+ debug!("Playing delayed audio feedback");
+ play_recording_start_sound(&app_clone);
+ });
+ } else {
+ debug!("Failed to start recording");
+ }
+ }
+
+ debug!(
+ "TranscribeAction::start completed in {:?}",
+ start_time.elapsed()
+ );
}
fn stop(&self, app: &AppHandle, binding_id: &str, _shortcut_str: &str) {
+ let stop_time = Instant::now();
+ debug!("TranscribeAction::stop called for binding: {}", binding_id);
+
let ah = app.clone();
let rm = Arc::clone(&app.state::<Arc<AudioRecordingManager>>());
let tm = Arc::clone(&app.state::<Arc<TranscriptionManager>>());
@@ -46,16 +86,37 @@ impl ShortcutAction for TranscribeAction {
tauri::async_runtime::spawn(async move {
let binding_id = binding_id.clone(); // Clone for the inner async task
+ debug!(
+ "Starting async transcription task for binding: {}",
+ binding_id
+ );
+
+ let stop_recording_time = Instant::now();
if let Some(samples) = rm.stop_recording(&binding_id) {
+ debug!(
+ "Recording stopped and samples retrieved in {:?}, sample count: {}",
+ stop_recording_time.elapsed(),
+ samples.len()
+ );
+
+ let transcription_time = Instant::now();
match tm.transcribe(samples) {
Ok(transcription) => {
- println!("Transcription Result: {}", transcription);
+ debug!(
+ "Transcription completed in {:?}: '{}'",
+ transcription_time.elapsed(),
+ transcription
+ );
if !transcription.is_empty() {
let transcription_clone = transcription.clone();
let ah_clone = ah.clone();
+ let paste_time = Instant::now();
ah.run_on_main_thread(move || {
match utils::paste(transcription_clone, ah_clone) {
- Ok(()) => println!("Text pasted successfully"),
+ Ok(()) => debug!(
+ "Text pasted successfully in {:?}",
+ paste_time.elapsed()
+ ),
Err(e) => eprintln!("Failed to paste transcription: {}", e),
}
})
@@ -64,10 +125,17 @@ impl ShortcutAction for TranscribeAction {
});
}
}
- Err(err) => println!("Global Shortcut Transcription error: {}", err),
+ Err(err) => debug!("Global Shortcut Transcription error: {}", err),
}
+ } else {
+ debug!("No samples retrieved from recording stop");
}
});
+
+ debug!(
+ "TranscribeAction::stop completed in {:?}",
+ stop_time.elapsed()
+ );
}
}
diff --git a/src-tauri/src/commands/audio.rs b/src-tauri/src/commands/audio.rs
new file mode 100644
index 00000000..1842b38e
--- /dev/null
+++ b/src-tauri/src/commands/audio.rs
@@ -0,0 +1,29 @@
+use crate::managers::audio::{AudioRecordingManager, MicrophoneMode};
+use crate::settings::{get_settings, write_settings};
+use std::sync::Arc;
+use tauri::{AppHandle, Manager};
+
+#[tauri::command]
+pub fn update_microphone_mode(app: AppHandle, always_on: bool) -> Result<(), String> {
+ // Update settings
+ let mut settings = get_settings(&app);
+ settings.always_on_microphone = always_on;
+ write_settings(&app, settings);
+
+ // Update the audio manager mode
+ let rm = app.state::<Arc<AudioRecordingManager>>();
+ let new_mode = if always_on {
+ MicrophoneMode::AlwaysOn
+ } else {
+ MicrophoneMode::OnDemand
+ };
+
+ rm.update_mode(new_mode)
+ .map_err(|e| format!("Failed to update microphone mode: {}", e))
+}
+
+#[tauri::command]
+pub fn get_microphone_mode(app: AppHandle) -> Result<bool, String> {
+ let settings = get_settings(&app);
+ Ok(settings.always_on_microphone)
+}
diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs
index c446ac88..2d8ee0a9 100644
--- a/src-tauri/src/commands/mod.rs
+++ b/src-tauri/src/commands/mod.rs
@@ -1 +1,2 @@
+pub mod audio;
pub mod models;
diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs
index 4d350cdf..bbcd9af3 100644
--- a/src-tauri/src/lib.rs
+++ b/src-tauri/src/lib.rs
@@ -185,7 +185,9 @@ pub fn run() {
commands::models::is_model_loading,
commands::models::has_any_models_available,
commands::models::has_any_models_or_downloads,
- commands::models::get_recommended_first_model
+ commands::models::get_recommended_first_model,
+ commands::audio::update_microphone_mode,
+ commands::audio::get_microphone_mode
])
.run(tauri::generate_context!())
.expect("error while running tauri application");
diff --git a/src-tauri/src/managers/audio.rs b/src-tauri/src/managers/audio.rs
index f49a823f..8ba40cb8 100644
--- a/src-tauri/src/managers/audio.rs
+++ b/src-tauri/src/managers/audio.rs
@@ -1,7 +1,10 @@
+use crate::settings::get_settings;
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use cpal::SampleFormat;
+use log::{debug, info};
use rubato::{FftFixedIn, Resampler};
use std::sync::{Arc, Mutex};
+use std::time::Instant;
use std::vec::Vec;
use tauri::{App, Manager};
use vad_rs::Vad;
@@ -14,10 +17,21 @@ pub enum RecordingState {
Recording { binding_id: String },
}
+#[derive(Clone, Debug)]
+pub enum MicrophoneMode {
+ AlwaysOn,
+ OnDemand,
+}
+
#[derive(Clone)]
pub struct AudioRecordingManager {
state: Arc<Mutex<RecordingState>>,
buffer: Arc<Mutex<Vec<f32>>>,
+ mode: Arc<Mutex<MicrophoneMode>>,
+ app_handle: tauri::AppHandle,
+ // For on-demand mode
+ stream_handle: Arc<Mutex<Option<std::thread::JoinHandle<()>>>>,
+ stream_active: Arc<Mutex<bool>>,
}
trait SampleToF32 {
@@ -50,31 +64,85 @@ impl SampleToF32 for f32 {
impl AudioRecordingManager {
pub fn new(app: &App) -> Result<Self, anyhow::Error> {
- let vad_path = app.path().resolve(
+ let settings = get_settings(&app.handle());
+ let mode = if settings.always_on_microphone {
+ MicrophoneMode::AlwaysOn
+ } else {
+ MicrophoneMode::OnDemand
+ };
+
+ let state = Arc::new(Mutex::new(RecordingState::Idle));
+ let buffer = Arc::new(Mutex::new(Vec::new()));
+ let mode_arc = Arc::new(Mutex::new(mode.clone()));
+ let app_handle = app.handle().clone();
+ let stream_handle = Arc::new(Mutex::new(None));
+ let stream_active = Arc::new(Mutex::new(false));
+
+ let manager = Self {
+ state: state.clone(),
+ buffer: buffer.clone(),
+ mode: mode_arc,
+ app_handle,
+ stream_handle,
+ stream_active,
+ };
+
+ // If always-on mode, start the stream immediately
+ if matches!(mode, MicrophoneMode::AlwaysOn) {
+ manager.start_microphone_stream()?;
+ }
+
+ Ok(manager)
+ }
+
+ pub fn start_microphone_stream(&self) -> Result<(), anyhow::Error> {
+ let start_time = Instant::now();
+ debug!("Starting microphone stream initialization...");
+
+ let mut stream_active = self.stream_active.lock().unwrap();
+ if *stream_active {
+ debug!("Microphone stream already active, skipping initialization");
+ return Ok(()); // Stream already active
+ }
+
+ let vad_path = self.app_handle.path().resolve(
"resources/models/silero_vad_v4.onnx",
tauri::path::BaseDirectory::Resource,
)?;
+ debug!("VAD model path resolved: {:?}", vad_path);
+
let host = cpal::default_host();
+ debug!("Audio host initialized");
+
let device = host
.default_input_device()
.ok_or_else(|| anyhow::Error::msg("No input device available"))?;
+ debug!("Default input device acquired: {:?}", device.name());
let config = device.default_input_config()?;
let sample_rate = config.sample_rate().0;
+ debug!(
+ "Audio config - Sample rate: {}, Channels: {}, Format: {:?}",
+ sample_rate,
+ config.channels(),
+ config.sample_format()
+ );
// Configure the resampler - keeping 1024 as input size for FFT efficiency
+ let resampler_start = Instant::now();
let resampler = FftFixedIn::new(sample_rate as usize, WHISPER_SAMPLE_RATE, 1024, 2, 1)?;
+ debug!("Resampler initialized in {:?}", resampler_start.elapsed());
+ let vad_start = Instant::now();
let vad = Arc::new(Mutex::new(Vad::new(vad_path, WHISPER_SAMPLE_RATE).unwrap()));
+ debug!("VAD initialized in {:?}", vad_start.elapsed());
let vad_clone = Arc::clone(&vad);
- let state = Arc::new(Mutex::new(RecordingState::Idle));
- let buffer = Arc::new(Mutex::new(Vec::new()));
+ let state_clone = Arc::clone(&self.state);
+ let buffer_clone = Arc::clone(&self.buffer);
let resampler = Arc::new(Mutex::new(resampler));
-
- let state_clone = Arc::clone(&state);
- let buffer_clone = Arc::clone(&buffer);
let resampler_clone = Arc::clone(&resampler);
+ let stream_active_clone = Arc::clone(&self.stream_active);
// Create a temporary buffer to accumulate samples
let temp_buffer = Arc::new(Mutex::new(Vec::new()));
@@ -94,7 +162,15 @@ impl AudioRecordingManager {
vad_buffer_clone: Arc<Mutex<Vec<f32>>>,
buffer_clone: Arc<Mutex<Vec<f32>>>,
vad_clone: Arc<Mutex<Vad>>,
+ stream_active_clone: Arc<Mutex<bool>>,
) {
+ // Check if stream should still be active
+ let stream_active = stream_active_clone.lock().unwrap();
+ if !*stream_active {
+ return;
+ }
+ drop(stream_active);
+
let state_guard = state_clone.lock().unwrap();
if let RecordingState::Recording { .. } = *state_guard {
let mut temp_buffer = temp_buffer_clone.lock().unwrap();
@@ -153,101 +229,202 @@ impl AudioRecordingManager {
}
}
- std::thread::spawn(move || {
+ let handle = std::thread::spawn(move || {
let err_fn = |err| eprintln!("Error in stream: {}", err);
// Build the appropriate stream based on the sample format
// Store the number of channels for use in the closure
let channels = config.channels() as usize;
- println!("Using {} channel(s) for recording", channels);
+ debug!(
+ "Audio stream thread started - Using {} channel(s) for recording",
+ channels
+ );
let stream = match config.sample_format() {
- SampleFormat::I8 => device.build_input_stream(
- &config.into(),
- move |data: &[i8], _| {
- process_audio(
- data,
- channels,
- Arc::clone(&state_clone),
- Arc::clone(&temp_buffer_clone),
- Arc::clone(&resampler_clone),
- Arc::clone(&vad_buffer_clone),
- Arc::clone(&buffer_clone),
- Arc::clone(&vad_clone),
- )
- },
- err_fn,
- None,
- ),
- SampleFormat::I16 => device.build_input_stream(
- &config.into(),
- move |data: &[i16], _| {
- process_audio(
- data,
- channels,
- Arc::clone(&state_clone),
- Arc::clone(&temp_buffer_clone),
- Arc::clone(&resampler_clone),
- Arc::clone(&vad_buffer_clone),
- Arc::clone(&buffer_clone),
- Arc::clone(&vad_clone),
- )
- },
- err_fn,
- None,
- ),
- SampleFormat::I32 => device.build_input_stream(
- &config.into(),
- move |data: &[i32], _| {
- process_audio(
- data,
- channels,
- Arc::clone(&state_clone),
- Arc::clone(&temp_buffer_clone),
- Arc::clone(&resampler_clone),
- Arc::clone(&vad_buffer_clone),
- Arc::clone(&buffer_clone),
- Arc::clone(&vad_clone),
- )
- },
- err_fn,
- None,
- ),
- SampleFormat::F32 => device.build_input_stream(
- &config.into(),
- move |data: &[f32], _| {
- process_audio(
- data,
- channels,
- Arc::clone(&state_clone),
- Arc::clone(&temp_buffer_clone),
- Arc::clone(&resampler_clone),
- Arc::clone(&vad_buffer_clone),
- Arc::clone(&buffer_clone),
- Arc::clone(&vad_clone),
- )
- },
- err_fn,
- None,
- ),
+ SampleFormat::I8 => {
+ let stream_active_i8 = Arc::clone(&stream_active_clone);
+ device.build_input_stream(
+ &config.into(),
+ move |data: &[i8], _| {
+ process_audio(
+ data,
+ channels,
+ Arc::clone(&state_clone),
+ Arc::clone(&temp_buffer_clone),
+ Arc::clone(&resampler_clone),
+ Arc::clone(&vad_buffer_clone),
+ Arc::clone(&buffer_clone),
+ Arc::clone(&vad_clone),
+ Arc::clone(&stream_active_i8),
+ )
+ },
+ err_fn,
+ None,
+ )
+ }
+
+ SampleFormat::I16 => {
+ let stream_active_i16 = Arc::clone(&stream_active_clone);
+ device.build_input_stream(
+ &config.into(),
+ move |data: &[i16], _| {
+ process_audio(
+ data,
+ channels,
+ Arc::clone(&state_clone),
+ Arc::clone(&temp_buffer_clone),
+ Arc::clone(&resampler_clone),
+ Arc::clone(&vad_buffer_clone),
+ Arc::clone(&buffer_clone),
+ Arc::clone(&vad_clone),
+ Arc::clone(&stream_active_i16),
+ )
+ },
+ err_fn,
+ None,
+ )
+ }
+
+ SampleFormat::I32 => {
+ let stream_active_i32 = Arc::clone(&stream_active_clone);
+ device.build_input_stream(
+ &config.into(),
+ move |data: &[i32], _| {
+ process_audio(
+ data,
+ channels,
+ Arc::clone(&state_clone),
+ Arc::clone(&temp_buffer_clone),
+ Arc::clone(&resampler_clone),
+ Arc::clone(&vad_buffer_clone),
+ Arc::clone(&buffer_clone),
+ Arc::clone(&vad_clone),
+ Arc::clone(&stream_active_i32),
+ )
+ },
+ err_fn,
+ None,
+ )
+ }
+
+ SampleFormat::F32 => {
+ let stream_active_f32 = Arc::clone(&stream_active_clone);
+ device.build_input_stream(
+ &config.into(),
+ move |data: &[f32], _| {
+ process_audio(
+ data,
+ channels,
+ Arc::clone(&state_clone),
+ Arc::clone(&temp_buffer_clone),
+ Arc::clone(&resampler_clone),
+ Arc::clone(&vad_buffer_clone),
+ Arc::clone(&buffer_clone),
+ Arc::clone(&vad_clone),
+ Arc::clone(&stream_active_f32),
+ )
+ },
+ err_fn,
+ None,
+ )
+ }
+
sample_format => {
- // Use anyhow to return a proper error instead of panicking
panic!("Unsupported sample format: {:?}", sample_format);
}
}
.expect("Failed to build input stream");
+ debug!("Audio input stream built successfully");
stream.play().expect("Failed to play stream");
- std::thread::park();
+ debug!("Audio stream started playing");
+
+ // Keep the thread alive until stream_active becomes false
+ let stream_active_for_loop = Arc::clone(&stream_active_clone);
+ loop {
+ std::thread::sleep(std::time::Duration::from_millis(100));
+ let active = {
+ let guard = stream_active_for_loop.lock().unwrap();
+ *guard
+ };
+ if !active {
+ break;
+ }
+ }
});
- Ok(Self { state, buffer })
+ *self.stream_handle.lock().unwrap() = Some(handle);
+ *stream_active = true;
+
+ let total_time = start_time.elapsed();
+ info!(
+ "Microphone stream initialization completed in {:?}",
+ total_time
+ );
+ Ok(())
+ }
+
+ pub fn stop_microphone_stream(&self) {
+ let mut stream_active = self.stream_active.lock().unwrap();
+ if !*stream_active {
+ return; // Stream already stopped
+ }
+
+ *stream_active = false;
+ drop(stream_active);
+
+ // Wait for the stream thread to finish
+ if let Some(handle) = self.stream_handle.lock().unwrap().take() {
+ let _ = handle.join();
+ }
+ }
+
+ pub fn update_mode(&self, new_mode: MicrophoneMode) -> Result<(), anyhow::Error> {
+ let mut mode = self.mode.lock().unwrap();
+ let current_mode = mode.clone();
+
+ if matches!(current_mode, MicrophoneMode::AlwaysOn)
+ && matches!(new_mode, MicrophoneMode::OnDemand)
+ {
+ // Switching from always-on to on-demand
+ // Stop the stream if not currently recording
+ let state = self.state.lock().unwrap();
+ if matches!(*state, RecordingState::Idle) {
+ drop(state);
+ drop(mode);
+ self.stop_microphone_stream();
+ mode = self.mode.lock().unwrap();
+ }
+ } else if matches!(current_mode, MicrophoneMode::OnDemand)
+ && matches!(new_mode, MicrophoneMode::AlwaysOn)
+ {
+ // Switching from on-demand to always-on
+ drop(mode);
+ self.start_microphone_stream()?;
+ mode = self.mode.lock().unwrap();
+ }
+
+ *mode = new_mode;
+ Ok(())
}
pub fn try_start_recording(&self, binding_id: &str) -> bool {
let mut state = self.state.lock().unwrap();
match *state {
RecordingState::Idle => {
+ // For on-demand mode, start the microphone stream now
+ let mode = self.mode.lock().unwrap();
+ if matches!(*mode, MicrophoneMode::OnDemand) {
+ debug!("On-demand mode: Starting microphone stream for recording");
+ drop(mode);
+ drop(state);
+ if let Err(e) = self.start_microphone_stream() {
+ eprintln!("Failed to start microphone stream: {}", e);
+ return false;
+ }
+ state = self.state.lock().unwrap();
+ }
+
// Clear the buffer before starting new recording
self.buffer.lock().unwrap().clear();
*state = RecordingState::Recording {
@@ -278,6 +455,15 @@ impl AudioRecordingManager {
*state = RecordingState::Idle;
println!("Stopped recording for binding {}", binding_id);
+ // For on-demand mode, stop the microphone stream
+ let mode = self.mode.lock().unwrap();
+ if matches!(*mode, MicrophoneMode::OnDemand) {
+ debug!("On-demand mode: Stopping microphone stream after recording");
+ drop(mode);
+ drop(state);
+ self.stop_microphone_stream();
+ }
+
let mut buffer = self.buffer.lock().unwrap();
let audio_data: Vec<f32> = buffer.drain(..).collect();
diff --git a/src-tauri/src/settings.rs b/src-tauri/src/settings.rs
index 5a92cd96..cce0d678 100644
--- a/src-tauri/src/settings.rs
+++ b/src-tauri/src/settings.rs
@@ -20,6 +20,8 @@ pub struct AppSettings {
pub audio_feedback: bool,
#[serde(default = "default_model")]
pub selected_model: String,
+ #[serde(default = "default_always_on_microphone")]
+ pub always_on_microphone: bool,
}
fn default_model() -> String {
@@ -28,6 +30,12 @@ fn default_model() -> String {
"".to_string()
}
+fn default_always_on_microphone() -> bool {
+ // Default to false for better user experience
+ // True would be the old behavior (always-on for low latency)
+ false
+}
+
pub const SETTINGS_STORE_PATH: &str = "settings_store.json";
pub fn get_default_settings() -> AppSettings {
@@ -58,6 +66,7 @@ pub fn get_default_settings() -> AppSettings {
push_to_talk: true,
audio_feedback: false,
selected_model: "".to_string(),
+ always_on_microphone: false,
}
}