A free, open source, and extensible speech-to-text application that works completely offline.
faster mic initialization (#1582)
CJ Paiscb9654ca8874
diff --git a/src-tauri/src/actions.rs b/src-tauri/src/actions.rsindex dbc83baa..974b63e8 100644--- a/src-tauri/src/actions.rs+++ b/src-tauri/src/actions.rs@@ -443,6 +443,7 @@ impl ShortcutAction for TranscribeAction {let rm = app.state::<Arc<AudioRecordingManager>>();// Load ASR model and VAD model in parallel+ let kickoff_started = Instant::now();tm.initiate_model_load();let rm_clone = Arc::clone(&rm);std::thread::spawn(move || {@@ -450,11 +451,15 @@ impl ShortcutAction for TranscribeAction {debug!("VAD pre-load failed: {}", e);}});+ let kickoff_elapsed = kickoff_started.elapsed();let binding_id = binding_id.to_string();+ let tray_started = Instant::now();change_tray_icon(app, TrayIconState::Recording);+ let tray_elapsed = tray_started.elapsed();// Get the microphone mode to determine audio feedback timing+ let plan_started = Instant::now();let settings = get_settings(app);let is_always_on = settings.always_on_microphone;@@ -479,15 +484,26 @@ impl ShortcutAction for TranscribeAction {if model_supports_streaming {tm.start_stream();}+ let plan_elapsed = plan_started.elapsed();// Sizing the overlay follows the same advertised capability. A model that// doesn't stream (or whose capability is not known yet) gets the compact// pill instead of an oversized transparent live window.+ let overlay_started = Instant::now();match settings.overlay_style {OverlayStyle::Live if model_supports_streaming => utils::show_streaming_overlay(app),OverlayStyle::Live | OverlayStyle::Minimal => show_recording_overlay(app),OverlayStyle::None => {} // show_overlay_state no-ops on None anyway}+ // Everything above runs before capture can begin, so each span here is+ // added keypress->capture latency.+ debug!(+ "start-path pre-recording steps: model_kickoff={:?} tray={:?} settings+stream_plan={:?} overlay={:?}",+ kickoff_elapsed,+ tray_elapsed,+ plan_elapsed,+ overlay_started.elapsed()+ );debug!("Microphone mode - always_on: {}", is_always_on);let mut recording_error: Option<String> = None;diff --git a/src-tauri/src/audio_toolkit/audio/recorder.rs b/src-tauri/src/audio_toolkit/audio/recorder.rsindex c8808de2..4870fd34 100644--- a/src-tauri/src/audio_toolkit/audio/recorder.rs+++ b/src-tauri/src/audio_toolkit/audio/recorder.rs@@ -4,7 +4,7 @@ use std::{atomic::{AtomicBool, Ordering},mpsc, Arc, Mutex,},- time::Duration,+ time::{Duration, Instant},};use cpal::{@@ -20,7 +20,10 @@ use crate::audio_toolkit::{};enum Cmd {- Start(VadPolicy),+ /// Begin capturing. Carries the send timestamp so the consumer can log how+ /// long the command sat in the channel (and how much audio was dropped+ /// before it was seen).+ Start(VadPolicy, Instant),Stop(mpsc::Sender<Vec<f32>>),Shutdown,}@@ -74,6 +77,13 @@ pub struct AudioRecorder {vad: Option<VadConfig>,level_cb: Option<Arc<dyn Fn(Vec<f32>) + Send + Sync + 'static>>,audio_cb: Option<AudioFrameCallback>,+ /// Preferred stream config cached per device name. The two HAL property+ /// queries in `get_preferred_config` cost ~40-85ms per open (worse on+ /// USB/Bluetooth), which lands on the keypress->capture path in on-demand+ /// mode. Keyed by name so a system-default change misses naturally;+ /// cleared whenever an open fails so a stale rate/format self-heals on the+ /// caller's retry.+ config_cache: Arc<Mutex<Option<(String, cpal::SupportedStreamConfig)>>>,}impl AudioRecorder {@@ -85,6 +95,7 @@ impl AudioRecorder {vad: None,level_cb: None,audio_cb: None,+ config_cache: Arc::new(Mutex::new(None)),})}@@ -148,13 +159,27 @@ impl AudioRecorder {let level_cb = self.level_cb.clone();// Move the optional real-time audio frame callback into the worker threadlet audio_cb = self.audio_cb.clone();+ let config_cache = Arc::clone(&self.config_cache);let worker = std::thread::spawn(move || {let stop_flag = Arc::new(AtomicBool::new(false));let stop_flag_for_stream = stop_flag.clone();let init_result = (|| -> Result<(cpal::Stream, u32), String> {- let config = AudioRecorder::get_preferred_config(&thread_device)- .map_err(|e| format!("Failed to fetch preferred config: {e}"))?;+ let config_started = Instant::now();+ let device_name = thread_device.name().unwrap_or_default();+ let cached_config = config_cache+ .lock()+ .unwrap()+ .as_ref()+ .filter(|(name, _)| !device_name.is_empty() && *name == device_name)+ .map(|(_, cfg)| cfg.clone());+ let config_was_cached = cached_config.is_some();+ let config = match cached_config {+ Some(cfg) => cfg,+ None => AudioRecorder::get_preferred_config(&thread_device)+ .map_err(|e| format!("Failed to fetch preferred config: {e}"))?,+ };+ let config_elapsed = config_started.elapsed();let sample_rate = config.sample_rate().0;let channels = config.channels() as usize;@@ -167,6 +192,7 @@ impl AudioRecorder {config.sample_format());+ let build_started = Instant::now();let stream = match config.sample_format() {cpal::SampleFormat::U8 => AudioRecorder::build_stream::<u8>(&thread_device,@@ -212,10 +238,25 @@ impl AudioRecorder {return Err(format!("Unsupported sample format: {sample_format:?}"));}};+ let build_elapsed = build_started.elapsed();+ let play_started = Instant::now();stream.play().map_err(|e| format!("Failed to start microphone stream: {e}"))?;+ log::debug!(+ "mic worker init: fetch_config={:?} (cached={}) build_stream={:?} play={:?}",+ config_elapsed,+ config_was_cached,+ build_elapsed,+ play_started.elapsed()+ );++ // The device accepted this config; remember it so the next+ // open skips the HAL property queries entirely.+ if !config_was_cached && !device_name.is_empty() {+ *config_cache.lock().unwrap() = Some((device_name, config));+ }Ok((stream, sample_rate))})();@@ -223,6 +264,9 @@ impl AudioRecorder {match init_result {Ok((stream, sample_rate)) => {let _ = init_tx.send(Ok(()));+ // Timestamp for the play()-returned -> first-samples gap the+ // init handshake can't see (hardware dependent).+ let stream_running_at = Instant::now();// Keep the stream alive while we process samples.run_consumer(sample_rate,@@ -232,10 +276,15 @@ impl AudioRecorder {level_cb,audio_cb,stop_flag,+ stream_running_at,);drop(stream);}Err(error_message) => {+ // A failed open may mean the cached config went stale+ // (device re-plugged, rate/format changed in the OS).+ // Drop it so the next attempt re-queries the device.+ *config_cache.lock().unwrap() = None;log::error!("{error_message}");let _ = init_tx.send(Err(error_message));}@@ -269,7 +318,7 @@ impl AudioRecorder {pub fn start(&self, vad_policy: VadPolicy) -> Result<(), Box<dyn std::error::Error>> {if let Some(tx) = &self.cmd_tx {- tx.send(Cmd::Start(vad_policy))?;+ tx.send(Cmd::Start(vad_policy, Instant::now()))?;}Ok(())}@@ -473,6 +522,7 @@ fn run_consumer(level_cb: Option<Arc<dyn Fn(Vec<f32>) + Send + Sync + 'static>>,audio_cb: Option<AudioFrameCallback>,stop_flag: Arc<AtomicBool>,+ stream_running_at: Instant,) {let mut frame_resampler = FrameResampler::new(in_sample_rate as usize,@@ -484,6 +534,13 @@ fn run_consumer(let mut recording = false;let mut vad_policy = VadPolicy::Offline;+ // ---------- latency instrumentation ---------------------------------- //+ // First-chunk arrival exposes the play()->samples-flowing gap; the+ // first-captured log confirms capture begins with the chunk in flight+ // when Cmd::Start lands.+ let mut first_chunk_logged = false;+ let mut awaiting_first_captured_chunk: Option<Instant> = None;+// ---------- spectrum visualisation setup ---------------------------- //const BUCKETS: usize = 16;// Scale the FFT window to the device sample rate so the analysis window@@ -542,34 +599,19 @@ fn run_consumer(// Runs until the stream closes and `recv` returns `Err`.while let Ok(chunk) = sample_rx.recv() {- let raw = match chunk {- AudioChunk::Samples(s) => s,- AudioChunk::EndOfStream => continue,- };-- // ---------- spectrum processing ---------------------------------- //- if let Some(buckets) = visualizer.feed(&raw) {- if let Some(cb) = &level_cb {- cb(buckets);- }- }-- // ---------- existing pipeline ------------------------------------ //- frame_resampler.push(&raw, &mut |frame: &[f32]| {- handle_frame(- frame,- recording,- vad_policy,- &vad,- &audio_cb,- &mut processed_samples,- )- });-- // non-blocking check for a command+ // Handle pending commands BEFORE the in-flight chunk so a Start+ // captures it. Commands used to be polled after processing, which+ // silently dropped one buffer period of audio (~10ms built-in, up to+ // ~100ms on Bluetooth) at every recording start.+ let mut pending = Some(chunk);while let Ok(cmd) = cmd_rx.try_recv() {match cmd {- Cmd::Start(policy) => {+ Cmd::Start(policy, sent_at) => {+ log::debug!(+ "Cmd::Start processed {:?} after send; capture begins with the in-flight chunk",+ sent_at.elapsed()+ );+ awaiting_first_captured_chunk = Some(Instant::now());stop_flag.store(false, Ordering::Relaxed);vad_policy = policy;processed_samples.clear();@@ -590,6 +632,21 @@ fn run_consumer(recording = false;stop_flag.store(true, Ordering::Relaxed);+ // The chunk in hand arrived before the stop; it belongs to+ // the recording, so feed it ahead of the drain below.+ if let Some(AudioChunk::Samples(raw)) = pending.take() {+ frame_resampler.push(&raw, &mut |frame: &[f32]| {+ handle_frame(+ frame,+ true,+ vad_policy,+ &vad,+ &audio_cb,+ &mut processed_samples,+ )+ });+ }+// Drain all remaining audio until the producer confirms end-of-stream.// The cpal callback sees the stop flag, sends EndOfStream, and goes// silent — guaranteeing every captured sample is in the channel@@ -639,5 +696,50 @@ fn run_consumer(}}}++ let raw = match pending.take() {+ Some(AudioChunk::Samples(s)) => s,+ // EndOfStream, or the chunk was consumed by a Stop above.+ _ => continue,+ };++ let chunk_ms = raw.len() as f64 * 1000.0 / in_sample_rate as f64;+ if !first_chunk_logged {+ first_chunk_logged = true;+ log::debug!(+ "first audio chunk arrived {:?} after stream start ({:.1}ms of audio)",+ stream_running_at.elapsed(),+ chunk_ms+ );+ }++ // ---------- spectrum processing ---------------------------------- //+ if let Some(buckets) = visualizer.feed(&raw) {+ if let Some(cb) = &level_cb {+ cb(buckets);+ }+ }++ // ---------- existing pipeline ------------------------------------ //+ frame_resampler.push(&raw, &mut |frame: &[f32]| {+ handle_frame(+ frame,+ recording,+ vad_policy,+ &vad,+ &audio_cb,+ &mut processed_samples,+ )+ });++ if recording {+ if let Some(started) = awaiting_first_captured_chunk.take() {+ log::debug!(+ "first captured chunk ({:.1}ms of audio) processed {:?} after Cmd::Start",+ chunk_ms,+ started.elapsed()+ );+ }+ }}}diff --git a/src-tauri/src/audio_toolkit/audio/resampler.rs b/src-tauri/src/audio_toolkit/audio/resampler.rsindex 149d99ba..cfb4ad51 100644--- a/src-tauri/src/audio_toolkit/audio/resampler.rs+++ b/src-tauri/src/audio_toolkit/audio/resampler.rs@@ -72,6 +72,10 @@ impl FrameResampler {if let Ok(out) = resampler.process(&[&self.in_buf[..]], None) {self.emit_frames(&out[0], &mut emit);}+ // Drop the consumed input: a full in_buf would satisfy the+ // next push()'s chunk check immediately, re-processing this+ // padded tail into the following recording.+ self.in_buf.clear();}}@@ -97,3 +101,31 @@ impl FrameResampler {}}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn finish_does_not_leak_tail_into_next_session() {+ // 48kHz -> 16kHz, 30ms frames (480 output samples per frame).+ let mut rs = FrameResampler::new(48000, 16000, Duration::from_millis(30));++ // Leave a partial chunk buffered, then end the session.+ rs.push(&[0.5f32; 100], |_| {});+ rs.finish(|_| {});++ // One fresh chunk yields ~341 output samples — below one frame, so+ // nothing should be emitted yet. If finish() left its padded tail in+ // in_buf, that tail is re-processed first, the output crosses the+ // 480-sample frame boundary, and a stale frame is emitted here.+ let mut emitted = 0usize;+ rs.push(&[0.25f32; RESAMPLER_CHUNK_SIZE], |frame| {+ emitted += frame.len()+ });+ assert_eq!(+ emitted, 0,+ "stale resampler tail from finish() leaked into the next session"+ );+ }+}diff --git a/src-tauri/src/managers/audio.rs b/src-tauri/src/managers/audio.rsindex 30ca8794..75c43dc6 100644--- a/src-tauri/src/managers/audio.rs+++ b/src-tauri/src/managers/audio.rs@@ -10,7 +10,7 @@ use crate::helpers::clamshell;use crate::managers::transcription::StreamRouter;use crate::settings::{get_settings, AppSettings};use crate::utils;-use log::{debug, error, info};+use log::{debug, error, info, warn};use std::sync::atomic::{AtomicU64, Ordering};use std::sync::{Arc, Mutex};use std::time::{Duration, Instant};@@ -185,6 +185,13 @@ pub struct AudioRecordingManager {close_generation: Arc<AtomicU64>,cancel_generation: Arc<AtomicU64>,stream_router: Arc<StreamRouter>,+ /// Resolution of a *named* microphone (selected or clamshell) to its cpal+ /// device, cached so on-demand recording starts skip the full device+ /// enumeration (~40-110ms). Keyed by the resolved name, so a settings+ /// change misses naturally; cleared when an open fails (device unplugged)+ /// so the retry re-enumerates. The system-default case is never cached —+ /// the recorder resolves the current default itself, cheaply.+ cached_device: Arc<Mutex<Option<(String, cpal::Device)>>>,}impl AudioRecordingManager {@@ -213,6 +220,7 @@ impl AudioRecordingManager {close_generation: Arc::new(AtomicU64::new(0)),cancel_generation: Arc::new(AtomicU64::new(0)),stream_router,+ cached_device: Arc::new(Mutex::new(None)),};// Always-on? Open immediately.@@ -225,31 +233,68 @@ impl AudioRecordingManager {/* ---------- helper methods --------------------------------------------- */+ /// The microphone name the settings ask for, or `None` for the system+ /// default. Only runs the clamshell probe (an `ioreg` subprocess, ~10-20ms)+ /// when a clamshell microphone is actually configured.+ fn desired_device_name(&self, settings: &AppSettings) -> Option<String> {+ if settings.clamshell_microphone.is_some() {+ let clamshell_started = Instant::now();+ let is_clamshell = clamshell::is_clamshell().unwrap_or(false);+ debug!(+ "device resolve: clamshell_check={:?} (clamshell={})",+ clamshell_started.elapsed(),+ is_clamshell+ );+ if is_clamshell {+ return settings.clamshell_microphone.clone();+ }+ }+ settings.selected_microphone.clone()+ }++ pub fn invalidate_device_cache(&self) {+ *self.cached_device.lock().unwrap() = None;+ }+fn get_effective_microphone_device(&self, settings: &AppSettings) -> Option<cpal::Device> {- // Check if we're in clamshell mode and have a clamshell microphone configured- let use_clamshell_mic = if let Ok(is_clamshell) = clamshell::is_clamshell() {- is_clamshell && settings.clamshell_microphone.is_some()- } else {- false+ let device_name = match self.desired_device_name(settings) {+ Some(name) => name,+ None => {+ debug!("device resolve: no mic configured -> system default");+ return None;+ }};- let device_name = if use_clamshell_mic {- settings.clamshell_microphone.as_ref().unwrap()- } else {- settings.selected_microphone.as_ref()?- };+ // Cache hit: skip the full enumeration. A stale device (unplugged)+ // fails at open, where the caller invalidates and retries fresh.+ if let Some((cached_name, device)) = self.cached_device.lock().unwrap().as_ref() {+ if *cached_name == device_name {+ debug!("device resolve: cache hit for '{}'", device_name);+ return Some(device.clone());+ }+ }// Find the device by name- match list_input_devices() {+ let enumerate_started = Instant::now();+ let device = match list_input_devices() {Ok(devices) => devices.into_iter()- .find(|d| d.name == *device_name)+ .find(|d| d.name == device_name).map(|d| d.device),Err(e) => {debug!("Failed to list devices, using default: {}", e);None}+ };+ debug!(+ "device resolve: enumerate={:?} (found={})",+ enumerate_started.elapsed(),+ device.is_some()+ );+ if let Some(d) = &device {+ *self.cached_device.lock().unwrap() = Some((device_name, d.clone()));}+ device}fn schedule_lazy_close(&self) {@@ -333,30 +378,41 @@ impl AudioRecordingManager {let mut did_mute_guard = self.did_mute.lock().unwrap();*did_mute_guard = false;- // Get the selected device from settings, considering clamshell mode+ // Get the selected device from settings, considering clamshell mode.+ // No pre-flight enumeration here: when nothing is configured the+ // recorder resolves the system default itself, and a machine with no+ // input devices at all fails inside open() with the same+ // "No input device found" error this used to check for.let settings = get_settings(&self.app_handle);+ let resolve_started = Instant::now();let selected_device = self.get_effective_microphone_device(&settings);-- // Pre-flight check: if no device was selected/configured AND no devices- // exist at all, fail early with a clear error instead of letting cpal- // produce a cryptic backend-specific message.- if selected_device.is_none() {- let has_any_device = list_input_devices()- .map(|devices| !devices.is_empty())- .unwrap_or(false);- if !has_any_device {- return Err(anyhow::anyhow!("No input device found"));- }- }+ let resolve_elapsed = resolve_started.elapsed();// Ensure VAD is loaded if it wasn't for whatever reason+ let vad_started = Instant::now();self.preload_vad()?;+ let vad_elapsed = vad_started.elapsed();+ let open_started = Instant::now();let mut recorder_opt = self.recorder.lock().unwrap();if let Some(rec) = recorder_opt.as_mut() {- rec.open(selected_device)- .map_err(|e| anyhow::anyhow!("Failed to open recorder: {}", e))?;+ if let Err(first_err) = rec.open(selected_device.clone()) {+ // A cached device or config may have gone stale (unplugged,+ // rate/format changed). Re-resolve from a fresh enumeration and+ // retry once before surfacing the error.+ warn!("Recorder open failed ({first_err}); re-resolving device and retrying once");+ self.invalidate_device_cache();+ let fresh_device = self.get_effective_microphone_device(&settings);+ rec.open(fresh_device)+ .map_err(|e| anyhow::anyhow!("Failed to open recorder: {}", e))?;+ }}+ debug!(+ "mic stream breakdown: device_resolve={:?} vad_ensure={:?} open={:?}",+ resolve_elapsed,+ vad_elapsed,+ open_started.elapsed()+ );*open_flag = true;// This timing covers through cpal's stream.play() returning — i.e. the@@ -457,6 +513,10 @@ impl AudioRecordingManager {}pub fn update_selected_device(&self) -> Result<(), anyhow::Error> {+ // Device settings changed; drop the cached resolution so the next+ // open re-enumerates. (The name-keyed cache would miss anyway; this+ // just avoids holding a stale cpal::Device alive.)+ self.invalidate_device_cache();// If currently open, restart the microphone stream to use the new deviceif *self.is_open.lock().unwrap() {self.close_generation.fetch_add(1, Ordering::SeqCst);diff --git a/src-tauri/src/overlay.rs b/src-tauri/src/overlay.rsindex 35f6ef7d..ae03bd74 100644--- a/src-tauri/src/overlay.rs+++ b/src-tauri/src/overlay.rs@@ -369,19 +369,37 @@ fn show_overlay_state(app_handle: &AppHandle, state: &str) {#[cfg(target_os = "linux")]update_gtk_layer_shell_anchors(&overlay_window);+ let size_started = std::time::Instant::now();let _ = overlay_window.set_size(tauri::Size::Logical(tauri::LogicalSize { width, height }));+ let size_elapsed = size_started.elapsed();++ let pos_started = std::time::Instant::now();+ let mut set_pos_elapsed = std::time::Duration::ZERO;if let Some((x, y)) = calculate_overlay_position(app_handle, width, height) {+ let set_pos_started = std::time::Instant::now();let _ = overlay_window.set_position(tauri::Position::Logical(tauri::LogicalPosition { x, y }));+ set_pos_elapsed = set_pos_started.elapsed();}+ let pos_calc_elapsed = pos_started.elapsed() - set_pos_elapsed;+ let show_started = std::time::Instant::now();let _ = overlay_window.show();+ let show_elapsed = show_started.elapsed();// On Windows, aggressively re-assert "topmost" in the native Z-order after showing#[cfg(target_os = "windows")]force_overlay_topmost(&overlay_window);let _ = overlay_window.emit("show-overlay", state);+ log::debug!(+ "overlay '{}': set_size={:?} pos_calc={:?} set_pos={:?} show={:?}",+ state,+ size_elapsed,+ pos_calc_elapsed,+ set_pos_elapsed,+ show_elapsed+ );}}diff --git a/src-tauri/src/tray.rs b/src-tauri/src/tray.rsindex 52edbb1d..4daabea7 100644--- a/src-tauri/src/tray.rs+++ b/src-tauri/src/tray.rs@@ -3,7 +3,7 @@ use crate::managers::model::ModelManager;use crate::managers::transcription::TranscriptionManager;use crate::settings;use crate::tray_i18n::get_tray_translations;-use log::{error, info, warn};+use log::{debug, error, info, warn};use std::sync::Arc;use tauri::image::Image;use tauri::menu::{CheckMenuItem, Menu, MenuItem, PredefinedMenuItem, Submenu};@@ -68,6 +68,7 @@ pub fn change_tray_icon(app: &AppHandle, icon: TrayIconState) {let icon_path = get_icon_path(theme, icon.clone());+ let icon_started = std::time::Instant::now();let _ = tray.set_icon(Some(Image::from_path(app.path()@@ -76,9 +77,17 @@ pub fn change_tray_icon(app: &AppHandle, icon: TrayIconState) {).expect("failed to set icon"),));+ let icon_elapsed = icon_started.elapsed();// Update menu based on state+ let menu_started = std::time::Instant::now();update_tray_menu(app, &icon, None);+ debug!(+ "tray icon change ({:?}): set_icon={:?} menu={:?}",+ icon,+ icon_elapsed,+ menu_started.elapsed()+ );}pub fn tray_tooltip() -> String {