A free, open source, and extensible speech-to-text application that works completely offline.
stop losing tail audio when a recording ends (#1958)
CJ Paisdf2168326e36
diff --git a/src-tauri/src/audio_toolkit/audio/recorder.rs b/src-tauri/src/audio_toolkit/audio/recorder.rsindex c5a720b3..16cd87f6 100644--- a/src-tauri/src/audio_toolkit/audio/recorder.rs+++ b/src-tauri/src/audio_toolkit/audio/recorder.rs@@ -435,46 +435,15 @@ impl AudioRecorder {};let stream_cb = move |data: &[T], _: &cpal::InputCallbackInfo| {- if stop_flag.load(Ordering::Relaxed) {- if !eos_sent {- let _ = sample_tx.send(AudioChunk::EndOfStream);- eos_sent = true;- }- return;- }- eos_sent = false;-- output_buffer.clear();-- if channels == 1 {- output_buffer.extend(data.iter().map(|&sample| sample.to_sample::<f32>()));- } else {- let frame_count = data.len() / channels;- output_buffer.reserve(frame_count);-- if let Some(ch) = use_channel {- for frame in data.chunks_exact(channels) {- let mono_sample = frame[ch].to_sample::<f32>();- output_buffer.push(mono_sample);- }- } else {- for frame in data.chunks_exact(channels) {- let mono_sample = frame- .iter()- .map(|&sample| sample.to_sample::<f32>())- .sum::<f32>()- / channels as f32;- output_buffer.push(mono_sample);- }- }- }-- if sample_tx- .send(AudioChunk::Samples(output_buffer.clone()))- .is_err()- {- log::error!("Failed to send samples");- }+ handle_input_block(+ data,+ channels,+ use_channel,+ &stop_flag,+ &mut eos_sent,+ &mut output_buffer,+ &sample_tx,+ );};device.build_input_stream(@@ -550,6 +519,73 @@ impl AudioRecorder {}}+/// Body of the cpal input callback, extracted for testing without a device.+/// Converts the block to mono and forwards it. The block that first observes+/// the stop flag was captured before the stop, so it is still forwarded —+/// dropping it loses up to a callback period of tail audio (worst on+/// Bluetooth) — followed by the end-of-stream sentinel; later blocks are+/// dropped until the flag clears.+fn handle_input_block<T>(+ data: &[T],+ channels: usize,+ use_channel: Option<usize>,+ stop_flag: &AtomicBool,+ eos_sent: &mut bool,+ output_buffer: &mut Vec<f32>,+ sample_tx: &mpsc::Sender<AudioChunk>,+) where+ T: Sample,+ f32: cpal::FromSample<T>,+{+ let stopping = stop_flag.load(Ordering::Relaxed);+ if stopping && *eos_sent {+ return;+ }++ output_buffer.clear();++ if channels == 1 {+ output_buffer.extend(data.iter().map(|&sample| sample.to_sample::<f32>()));+ } else {+ let frame_count = data.len() / channels;+ output_buffer.reserve(frame_count);++ if let Some(ch) = use_channel {+ for frame in data.chunks_exact(channels) {+ let mono_sample = frame[ch].to_sample::<f32>();+ output_buffer.push(mono_sample);+ }+ } else {+ for frame in data.chunks_exact(channels) {+ let mono_sample = frame+ .iter()+ .map(|&sample| sample.to_sample::<f32>())+ .sum::<f32>()+ / channels as f32;+ output_buffer.push(mono_sample);+ }+ }+ }++ // A failed send means the consumer thread is gone. During shutdown that is+ // expected (the consumer exits before the stream is dropped), so only+ // report it when capture was supposed to be live.+ if sample_tx+ .send(AudioChunk::Samples(output_buffer.clone()))+ .is_err()+ && !stopping+ {+ log::error!("Failed to send samples");+ }++ if stopping {+ let _ = sample_tx.send(AudioChunk::EndOfStream);+ *eos_sent = true;+ } else {+ *eos_sent = false;+ }+}+pub fn is_microphone_access_denied(error_message: &str) -> bool {let normalized = error_message.to_lowercase();normalized.contains("access is denied")@@ -567,7 +603,8 @@ pub fn is_no_input_device_error(error_message: &str) -> bool {#[cfg(test)]mod tests {use super::{- is_microphone_access_denied, is_no_input_device_error, run_consumer, AudioRecorder, Cmd,+ handle_input_block, is_microphone_access_denied, is_no_input_device_error, run_consumer,+ AudioChunk, AudioRecorder, Cmd,};use std::{sync::{@@ -622,6 +659,39 @@ mod tests {assert!(stopped.is_ok(), "shutdown waited for an audio sample");}+ #[test]+ fn boundary_block_forwarded_before_eos() {+ let (tx, rx) = mpsc::channel();+ let stop_flag = AtomicBool::new(false);+ let mut eos_sent = false;+ let mut scratch = Vec::new();+ let mut push = |flag: &AtomicBool, eos: &mut bool, block: &[f32]| {+ handle_input_block::<f32>(block, 1, None, flag, eos, &mut scratch, &tx)+ };++ // Running: blocks forwarded, no sentinel.+ push(&stop_flag, &mut eos_sent, &[0.1]);+ assert!(matches!(rx.try_recv(), Ok(AudioChunk::Samples(_))));+ assert!(rx.try_recv().is_err());++ // The block observing the stop flag is still forwarded, then EOS.+ stop_flag.store(true, Ordering::Relaxed);+ push(&stop_flag, &mut eos_sent, &[0.5, 0.5]);+ match rx.try_recv() {+ Ok(AudioChunk::Samples(samples)) => assert_eq!(samples, vec![0.5, 0.5]),+ _ => panic!("boundary block must be forwarded, not dropped"),+ }+ assert!(matches!(rx.try_recv(), Ok(AudioChunk::EndOfStream)));++ // Later blocks are dropped until the flag clears, then capture resumes.+ push(&stop_flag, &mut eos_sent, &[0.9]);+ assert!(rx.try_recv().is_err(), "blocks after EOS must be dropped");+ stop_flag.store(false, Ordering::Relaxed);+ push(&stop_flag, &mut eos_sent, &[0.2]);+ assert!(matches!(rx.try_recv(), Ok(AudioChunk::Samples(_))));+ assert!(rx.try_recv().is_err(), "no sentinel while running");+ }+#[test]fn detects_access_is_denied() {assert!(is_microphone_access_denied("Access is denied"));@@ -850,6 +920,26 @@ fn run_consumer()});+ // Diagnostic only: evidence for whether the VAD was+ // still withholding tail audio when capture stopped.+ // Suggestive, not conclusive, in either direction.+ if vad_policy != VadPolicy::Disabled {+ if let Some(cfg) = &vad {+ let report = cfg.detector.lock().unwrap().tail_report();+ if let Some(report) = report {+ log::debug!(+ "VAD at stop: withheld tail {} frames (~{}ms, {} voiced), in_speech={}, onset_counter={}, hangover_counter={}",+ report.withheld_frames,+ report.withheld_frames * 30,+ report.withheld_voiced_frames,+ report.in_speech,+ report.onset_counter,+ report.hangover_counter+ );+ }+ }+ }+let _ = reply_tx.send(std::mem::take(&mut processed_samples));// Resume the audio callback so the consumer loop can continuediff --git a/src-tauri/src/audio_toolkit/audio/resampler.rs b/src-tauri/src/audio_toolkit/audio/resampler.rsindex 94538d9d..13fd5e40 100644--- a/src-tauri/src/audio_toolkit/audio/resampler.rs+++ b/src-tauri/src/audio_toolkit/audio/resampler.rs@@ -10,6 +10,12 @@ pub struct FrameResampler {in_buf: Vec<f32>,frame_samples: usize,pending: Vec<f32>,+ in_hz: usize,+ out_hz: usize,+ /// Samples in/out of the inner resampler; `finish()` uses the pair to+ /// know how much real audio (~10-30ms) its delay line still holds.+ in_count: usize,+ out_count: usize,}impl FrameResampler {@@ -31,6 +37,10 @@ impl FrameResampler {in_buf: Vec::with_capacity(chunk_in),frame_samples,pending: Vec::with_capacity(frame_samples),+ in_hz,+ out_hz,+ in_count: 0,+ out_count: 0,}}@@ -39,6 +49,7 @@ impl FrameResampler {self.emit_frames(src, &mut emit);return;}+ self.in_count += src.len();while !src.is_empty() {let space = self.chunk_in - self.in_buf.len();@@ -56,6 +67,7 @@ impl FrameResampler {{// let duration = start.elapsed();// log::debug!("Resampler took: {:?}", duration);+ self.out_count += out[0].len();self.emit_frames(&out[0], &mut emit);}self.in_buf.clear();@@ -64,12 +76,16 @@ impl FrameResampler {}pub fn finish(&mut self, mut emit: impl FnMut(&[f32])) {- // Process any remaining input samples- if let Some(ref mut resampler) = self.resampler {+ if self.resampler.is_some() {+ // Process any remaining input samples (padded internally).if !self.in_buf.is_empty() {- // Pad with zeros to reach chunk size- self.in_buf.resize(self.chunk_in, 0.0);- if let Ok(out) = resampler.process(&[&self.in_buf[..]], None) {+ let result = self+ .resampler+ .as_mut()+ .unwrap()+ .process_partial(Some(&[&self.in_buf[..]]), None);+ if let Ok(out) = result {+ self.out_count += out[0].len();self.emit_frames(&out[0], &mut emit);}// Drop the consumed input: a full in_buf would satisfy the@@ -77,6 +93,31 @@ impl FrameResampler {// padded tail into the following recording.self.in_buf.clear();}++ // Output lags input by output_delay() samples, so all real audio+ // has emerged only once in*ratio + delay samples are out. Feed+ // zero chunks until then, trimming the synthetic remainder.+ if self.in_count > 0 {+ let delay = self.resampler.as_ref().unwrap().output_delay();+ let expected = self.in_count * self.out_hz / self.in_hz + delay;+ let mut rounds = 0;+ while self.out_count < expected && rounds < 8 {+ rounds += 1;+ let result = self+ .resampler+ .as_mut()+ .unwrap()+ .process_partial::<&[f32]>(None, None);+ match result {+ Ok(out) => {+ let take = (expected - self.out_count).min(out[0].len());+ self.out_count += take;+ self.emit_frames(&out[0][..take], &mut emit);+ }+ Err(_) => break,+ }+ }+ }}// Emit any remaining pending frame (padded with zeros)@@ -94,6 +135,8 @@ impl FrameResampler {pub fn reset(&mut self) {self.in_buf.clear();self.pending.clear();+ self.in_count = 0;+ self.out_count = 0;if let Some(ref mut resampler) = self.resampler {resampler.reset();}@@ -242,6 +285,47 @@ mod tests {}}+ /// Push silence ending in a 200-sample 0.5 burst, then assert finish()+ /// recovers the burst and emits floor(input*ratio) + output_delay samples,+ /// padded to whole 480-sample frames.+ fn assert_tail_burst_flushed(in_hz: usize, input_len: usize, expected_out: usize) {+ let mut rs = FrameResampler::new(in_hz, 16000, Duration::from_millis(30));+ let mut input = vec![0.0f32; input_len];+ input[input_len - 200..].fill(0.5);++ let mut out = Vec::new();+ rs.push(&input, |frame| out.extend_from_slice(frame));+ rs.finish(|frame| out.extend_from_slice(frame));++ let max_abs = out.iter().map(|s| s.abs()).fold(0.0f32, f32::max);+ assert!(+ max_abs > 0.3,+ "tail burst was lost in the resampler, max_abs={max_abs}"+ );+ assert_eq!(out.len(), expected_out);+ }++ #[test]+ fn finish_flushes_resampler_delay() {+ // Exact chunks keep in_buf empty, so the burst survives only via the+ // delay-line drain. 4096 in -> 1365 real out + 171 delay -> 1920 framed.+ assert_tail_burst_flushed(48000, 4 * RESAMPLER_CHUNK_SIZE, 1920);+ }++ #[test]+ fn finish_flushes_resampler_delay_44100() {+ // fft_size_in (1323) exceeds the 1024 chunk, so the drain must survive+ // a zero-output round. 4096 in -> 1486 real out + 240 delay -> 1920.+ assert_tail_burst_flushed(44100, 4 * RESAMPLER_CHUNK_SIZE, 1920);+ }++ #[test]+ fn finish_flushes_unaligned_tail() {+ // Ends mid-chunk: partial-chunk path plus delay drain together.+ // 4396 in -> 1465 real out + 171 delay -> 1920 framed.+ assert_tail_burst_flushed(48000, 4 * RESAMPLER_CHUNK_SIZE + 300, 1920);+ }+#[test]fn finish_does_not_leak_tail_into_next_session() {// 48kHz -> 16kHz, 30ms frames (480 output samples per frame).diff --git a/src-tauri/src/audio_toolkit/vad/mod.rs b/src-tauri/src/audio_toolkit/vad/mod.rsindex 02485a4f..586eb7e7 100644--- a/src-tauri/src/audio_toolkit/vad/mod.rs+++ b/src-tauri/src/audio_toolkit/vad/mod.rs@@ -31,9 +31,32 @@ pub trait VoiceActivityDetector: Send + Sync {/// subsequent frames. Detectors without a smoothing tail can ignore this.fn set_hangover_frames(&mut self, _frames: usize) {}+ /// End-of-recording diagnostic snapshot, taken after the final frame.+ /// Purely observational — implementations must not change what they emit.+ /// Detectors without smoothing state return None.+ fn tail_report(&self) -> Option<VadTailReport> {+ None+ }+fn reset(&mut self) {}}+/// End-of-recording snapshot of a smoothing detector's state. Voiced frames+/// in the withheld tail suggest — but don't prove — a final word cut off at+/// the stop; a clean report doesn't rule VAD loss out either (soft trailing+/// speech can be classified as noise).+#[derive(Debug, Clone, Copy)]+pub struct VadTailReport {+ /// Trailing frames buffered but never emitted downstream.+ pub withheld_frames: usize,+ /// How many of those withheld frames the inner VAD classified as voiced.+ pub withheld_voiced_frames: usize,+ pub in_speech: bool,+ /// Voiced frames counted toward an unconfirmed speech onset.+ pub onset_counter: usize,+ pub hangover_counter: usize,+}+mod silero;mod smoothed;diff --git a/src-tauri/src/audio_toolkit/vad/smoothed.rs b/src-tauri/src/audio_toolkit/vad/smoothed.rsindex 52de4d74..218cd615 100644--- a/src-tauri/src/audio_toolkit/vad/smoothed.rs+++ b/src-tauri/src/audio_toolkit/vad/smoothed.rs@@ -1,14 +1,22 @@-use super::{VadFrame, VoiceActivityDetector};+use super::{VadFrame, VadTailReport, VoiceActivityDetector};use anyhow::Result;use std::collections::VecDeque;+/// One pre-roll buffer slot. `emitted` and `voiced` exist only to power the+/// end-of-recording `tail_report()` diagnostic; they never affect emission.+struct BufferedFrame {+ samples: Vec<f32>,+ emitted: bool,+ voiced: bool,+}+pub struct SmoothedVad {inner_vad: Box<dyn VoiceActivityDetector>,prefill_frames: usize,hangover_frames: usize,onset_frames: usize,- frame_buffer: VecDeque<Vec<f32>>,+ frame_buffer: VecDeque<BufferedFrame>,hangover_counter: usize,onset_counter: usize,in_speech: bool,@@ -35,18 +43,31 @@ impl SmoothedVad {temp_out: Vec::new(),}}++ fn mark_last_emitted(&mut self) {+ if let Some(frame) = self.frame_buffer.back_mut() {+ frame.emitted = true;+ }+ }}impl VoiceActivityDetector for SmoothedVad {fn push_frame<'a>(&'a mut self, frame: &'a [f32]) -> Result<VadFrame<'a>> {// 1. Buffer every incoming frame for possible pre-roll- self.frame_buffer.push_back(frame.to_vec());+ self.frame_buffer.push_back(BufferedFrame {+ samples: frame.to_vec(),+ emitted: false,+ voiced: false,+ });while self.frame_buffer.len() > self.prefill_frames + 1 {self.frame_buffer.pop_front();}// 2. Delegate to the wrapped boolean VADlet is_voice = self.inner_vad.is_voice(frame)?;+ if let Some(last) = self.frame_buffer.back_mut() {+ last.voiced = is_voice;+ }match (self.in_speech, is_voice) {// Potential start of speech - need to accumulate onset frames@@ -60,8 +81,9 @@ impl VoiceActivityDetector for SmoothedVad {// Collect prefill + current frameself.temp_out.clear();- for buf in &self.frame_buffer {- self.temp_out.extend(buf);+ for buffered in self.frame_buffer.iter_mut() {+ self.temp_out.extend(buffered.samples.iter());+ buffered.emitted = true;}Ok(VadFrame::Speech(&self.temp_out))} else {@@ -73,6 +95,7 @@ impl VoiceActivityDetector for SmoothedVad {// Ongoing Speech(true, true) => {self.hangover_counter = self.hangover_frames;+ self.mark_last_emitted();Ok(VadFrame::Speech(frame))}@@ -80,6 +103,7 @@ impl VoiceActivityDetector for SmoothedVad {(true, false) => {if self.hangover_counter > 0 {self.hangover_counter -= 1;+ self.mark_last_emitted();Ok(VadFrame::Speech(frame))} else {self.in_speech = false;@@ -99,6 +123,32 @@ impl VoiceActivityDetector for SmoothedVad {self.hangover_frames = frames;}+ /// Trailing run of withheld frames plus smoothing state. Interior+ /// withheld frames (before already-emitted speech) are not counted.+ fn tail_report(&self) -> Option<VadTailReport> {+ let mut withheld_frames = 0;+ let mut withheld_voiced_frames = 0;+ for frame in self+ .frame_buffer+ .iter()+ .rev()+ .take_while(|frame| !frame.emitted)+ {+ withheld_frames += 1;+ if frame.voiced {+ withheld_voiced_frames += 1;+ }+ }++ Some(VadTailReport {+ withheld_frames,+ withheld_voiced_frames,+ in_speech: self.in_speech,+ onset_counter: self.onset_counter,+ hangover_counter: self.hangover_counter,+ })+ }+fn reset(&mut self) {self.inner_vad.reset();self.frame_buffer.clear();@@ -108,3 +158,72 @@ impl VoiceActivityDetector for SmoothedVad {self.temp_out.clear();}}++#[cfg(test)]+mod tests {+ use super::*;++ /// Inner VAD that replays a scripted voice/no-voice sequence.+ struct ScriptedVad {+ script: VecDeque<bool>,+ }++ impl ScriptedVad {+ fn new(script: &[bool]) -> Self {+ Self {+ script: script.iter().copied().collect(),+ }+ }+ }++ impl VoiceActivityDetector for ScriptedVad {+ fn push_frame<'a>(&'a mut self, frame: &'a [f32]) -> Result<VadFrame<'a>> {+ if self.script.pop_front().unwrap_or(false) {+ Ok(VadFrame::Speech(frame))+ } else {+ Ok(VadFrame::Noise)+ }+ }+ }++ fn frame(value: f32) -> Vec<f32> {+ vec![value; 4]+ }++ fn smoothed(script: &[bool], onset_frames: usize) -> SmoothedVad {+ SmoothedVad::new(Box::new(ScriptedVad::new(script)), 3, 2, onset_frames)+ }++ #[test]+ fn tail_report_counts_withheld_onset_tail() {+ // One voiced frame at the end: onset (2 frames) never confirms, so+ // both trailing frames are withheld and one of them is voiced —+ // consistent with a final word cut off at the stop boundary.+ let mut vad = smoothed(&[false, true], 2);+ assert!(!vad.push_frame(&frame(0.1)).unwrap().is_speech());+ assert!(!vad.push_frame(&frame(0.9)).unwrap().is_speech());++ let report = vad.tail_report().expect("smoothed VAD always reports");+ assert_eq!(report.withheld_frames, 2);+ assert_eq!(report.withheld_voiced_frames, 1);+ assert_eq!(report.onset_counter, 1);+ assert!(!report.in_speech);+ }++ #[test]+ fn tail_report_counts_only_trailing_run() {+ // Speech emitted through hangover, then silence past the hangover is+ // withheld. Only the trailing run counts — frames older than an+ // emitted frame are excluded.+ let mut vad = smoothed(&[true, true, false, false, false, false], 2);+ for v in [0.1, 0.2, 0.3, 0.4] {+ let _ = vad.push_frame(&frame(v)).unwrap();+ }+ assert!(!vad.push_frame(&frame(0.5)).unwrap().is_speech()); // past hangover+ assert!(!vad.push_frame(&frame(0.6)).unwrap().is_speech());++ let report = vad.tail_report().unwrap();+ assert_eq!(report.withheld_frames, 2);+ assert_eq!(report.withheld_voiced_frames, 0);+ }+}