Skip to content

armgit / Handy

Public Imported from GitHub
0

A free, open source, and extensible speech-to-text application that works completely offline.

stop losing tail audio when a recording ends (#1958)

CJ Paisdf2168326e36
diff --git a/src-tauri/src/audio_toolkit/audio/recorder.rs b/src-tauri/src/audio_toolkit/audio/recorder.rs
index c5a720b3..16cd87f6 100644
--- a/src-tauri/src/audio_toolkit/audio/recorder.rs
+++ b/src-tauri/src/audio_toolkit/audio/recorder.rs
@@ -435,46 +435,15 @@ impl AudioRecorder {
};
let stream_cb = move |data: &[T], _: &cpal::InputCallbackInfo| {
- if stop_flag.load(Ordering::Relaxed) {
- if !eos_sent {
- let _ = sample_tx.send(AudioChunk::EndOfStream);
- eos_sent = true;
- }
- return;
- }
- eos_sent = false;
-
- output_buffer.clear();
-
- if channels == 1 {
- output_buffer.extend(data.iter().map(|&sample| sample.to_sample::<f32>()));
- } else {
- let frame_count = data.len() / channels;
- output_buffer.reserve(frame_count);
-
- if let Some(ch) = use_channel {
- for frame in data.chunks_exact(channels) {
- let mono_sample = frame[ch].to_sample::<f32>();
- output_buffer.push(mono_sample);
- }
- } else {
- for frame in data.chunks_exact(channels) {
- let mono_sample = frame
- .iter()
- .map(|&sample| sample.to_sample::<f32>())
- .sum::<f32>()
- / channels as f32;
- output_buffer.push(mono_sample);
- }
- }
- }
-
- if sample_tx
- .send(AudioChunk::Samples(output_buffer.clone()))
- .is_err()
- {
- log::error!("Failed to send samples");
- }
+ handle_input_block(
+ data,
+ channels,
+ use_channel,
+ &stop_flag,
+ &mut eos_sent,
+ &mut output_buffer,
+ &sample_tx,
+ );
};
device.build_input_stream(
@@ -550,6 +519,73 @@ impl AudioRecorder {
}
}
+/// Body of the cpal input callback, extracted for testing without a device.
+/// Converts the block to mono and forwards it. The block that first observes
+/// the stop flag was captured before the stop, so it is still forwarded —
+/// dropping it loses up to a callback period of tail audio (worst on
+/// Bluetooth) — followed by the end-of-stream sentinel; later blocks are
+/// dropped until the flag clears.
+fn handle_input_block<T>(
+ data: &[T],
+ channels: usize,
+ use_channel: Option<usize>,
+ stop_flag: &AtomicBool,
+ eos_sent: &mut bool,
+ output_buffer: &mut Vec<f32>,
+ sample_tx: &mpsc::Sender<AudioChunk>,
+) where
+ T: Sample,
+ f32: cpal::FromSample<T>,
+{
+ let stopping = stop_flag.load(Ordering::Relaxed);
+ if stopping && *eos_sent {
+ return;
+ }
+
+ output_buffer.clear();
+
+ if channels == 1 {
+ output_buffer.extend(data.iter().map(|&sample| sample.to_sample::<f32>()));
+ } else {
+ let frame_count = data.len() / channels;
+ output_buffer.reserve(frame_count);
+
+ if let Some(ch) = use_channel {
+ for frame in data.chunks_exact(channels) {
+ let mono_sample = frame[ch].to_sample::<f32>();
+ output_buffer.push(mono_sample);
+ }
+ } else {
+ for frame in data.chunks_exact(channels) {
+ let mono_sample = frame
+ .iter()
+ .map(|&sample| sample.to_sample::<f32>())
+ .sum::<f32>()
+ / channels as f32;
+ output_buffer.push(mono_sample);
+ }
+ }
+ }
+
+ // A failed send means the consumer thread is gone. During shutdown that is
+ // expected (the consumer exits before the stream is dropped), so only
+ // report it when capture was supposed to be live.
+ if sample_tx
+ .send(AudioChunk::Samples(output_buffer.clone()))
+ .is_err()
+ && !stopping
+ {
+ log::error!("Failed to send samples");
+ }
+
+ if stopping {
+ let _ = sample_tx.send(AudioChunk::EndOfStream);
+ *eos_sent = true;
+ } else {
+ *eos_sent = false;
+ }
+}
+
pub fn is_microphone_access_denied(error_message: &str) -> bool {
let normalized = error_message.to_lowercase();
normalized.contains("access is denied")
@@ -567,7 +603,8 @@ pub fn is_no_input_device_error(error_message: &str) -> bool {
#[cfg(test)]
mod tests {
use super::{
- is_microphone_access_denied, is_no_input_device_error, run_consumer, AudioRecorder, Cmd,
+ handle_input_block, is_microphone_access_denied, is_no_input_device_error, run_consumer,
+ AudioChunk, AudioRecorder, Cmd,
};
use std::{
sync::{
@@ -622,6 +659,39 @@ mod tests {
assert!(stopped.is_ok(), "shutdown waited for an audio sample");
}
+ #[test]
+ fn boundary_block_forwarded_before_eos() {
+ let (tx, rx) = mpsc::channel();
+ let stop_flag = AtomicBool::new(false);
+ let mut eos_sent = false;
+ let mut scratch = Vec::new();
+ let mut push = |flag: &AtomicBool, eos: &mut bool, block: &[f32]| {
+ handle_input_block::<f32>(block, 1, None, flag, eos, &mut scratch, &tx)
+ };
+
+ // Running: blocks forwarded, no sentinel.
+ push(&stop_flag, &mut eos_sent, &[0.1]);
+ assert!(matches!(rx.try_recv(), Ok(AudioChunk::Samples(_))));
+ assert!(rx.try_recv().is_err());
+
+ // The block observing the stop flag is still forwarded, then EOS.
+ stop_flag.store(true, Ordering::Relaxed);
+ push(&stop_flag, &mut eos_sent, &[0.5, 0.5]);
+ match rx.try_recv() {
+ Ok(AudioChunk::Samples(samples)) => assert_eq!(samples, vec![0.5, 0.5]),
+ _ => panic!("boundary block must be forwarded, not dropped"),
+ }
+ assert!(matches!(rx.try_recv(), Ok(AudioChunk::EndOfStream)));
+
+ // Later blocks are dropped until the flag clears, then capture resumes.
+ push(&stop_flag, &mut eos_sent, &[0.9]);
+ assert!(rx.try_recv().is_err(), "blocks after EOS must be dropped");
+ stop_flag.store(false, Ordering::Relaxed);
+ push(&stop_flag, &mut eos_sent, &[0.2]);
+ assert!(matches!(rx.try_recv(), Ok(AudioChunk::Samples(_))));
+ assert!(rx.try_recv().is_err(), "no sentinel while running");
+ }
+
#[test]
fn detects_access_is_denied() {
assert!(is_microphone_access_denied("Access is denied"));
@@ -850,6 +920,26 @@ fn run_consumer(
)
});
+ // Diagnostic only: evidence for whether the VAD was
+ // still withholding tail audio when capture stopped.
+ // Suggestive, not conclusive, in either direction.
+ if vad_policy != VadPolicy::Disabled {
+ if let Some(cfg) = &vad {
+ let report = cfg.detector.lock().unwrap().tail_report();
+ if let Some(report) = report {
+ log::debug!(
+ "VAD at stop: withheld tail {} frames (~{}ms, {} voiced), in_speech={}, onset_counter={}, hangover_counter={}",
+ report.withheld_frames,
+ report.withheld_frames * 30,
+ report.withheld_voiced_frames,
+ report.in_speech,
+ report.onset_counter,
+ report.hangover_counter
+ );
+ }
+ }
+ }
+
let _ = reply_tx.send(std::mem::take(&mut processed_samples));
// Resume the audio callback so the consumer loop can continue
diff --git a/src-tauri/src/audio_toolkit/audio/resampler.rs b/src-tauri/src/audio_toolkit/audio/resampler.rs
index 94538d9d..13fd5e40 100644
--- a/src-tauri/src/audio_toolkit/audio/resampler.rs
+++ b/src-tauri/src/audio_toolkit/audio/resampler.rs
@@ -10,6 +10,12 @@ pub struct FrameResampler {
in_buf: Vec<f32>,
frame_samples: usize,
pending: Vec<f32>,
+ in_hz: usize,
+ out_hz: usize,
+ /// Samples in/out of the inner resampler; `finish()` uses the pair to
+ /// know how much real audio (~10-30ms) its delay line still holds.
+ in_count: usize,
+ out_count: usize,
}
impl FrameResampler {
@@ -31,6 +37,10 @@ impl FrameResampler {
in_buf: Vec::with_capacity(chunk_in),
frame_samples,
pending: Vec::with_capacity(frame_samples),
+ in_hz,
+ out_hz,
+ in_count: 0,
+ out_count: 0,
}
}
@@ -39,6 +49,7 @@ impl FrameResampler {
self.emit_frames(src, &mut emit);
return;
}
+ self.in_count += src.len();
while !src.is_empty() {
let space = self.chunk_in - self.in_buf.len();
@@ -56,6 +67,7 @@ impl FrameResampler {
{
// let duration = start.elapsed();
// log::debug!("Resampler took: {:?}", duration);
+ self.out_count += out[0].len();
self.emit_frames(&out[0], &mut emit);
}
self.in_buf.clear();
@@ -64,12 +76,16 @@ impl FrameResampler {
}
pub fn finish(&mut self, mut emit: impl FnMut(&[f32])) {
- // Process any remaining input samples
- if let Some(ref mut resampler) = self.resampler {
+ if self.resampler.is_some() {
+ // Process any remaining input samples (padded internally).
if !self.in_buf.is_empty() {
- // Pad with zeros to reach chunk size
- self.in_buf.resize(self.chunk_in, 0.0);
- if let Ok(out) = resampler.process(&[&self.in_buf[..]], None) {
+ let result = self
+ .resampler
+ .as_mut()
+ .unwrap()
+ .process_partial(Some(&[&self.in_buf[..]]), None);
+ if let Ok(out) = result {
+ self.out_count += out[0].len();
self.emit_frames(&out[0], &mut emit);
}
// Drop the consumed input: a full in_buf would satisfy the
@@ -77,6 +93,31 @@ impl FrameResampler {
// padded tail into the following recording.
self.in_buf.clear();
}
+
+ // Output lags input by output_delay() samples, so all real audio
+ // has emerged only once in*ratio + delay samples are out. Feed
+ // zero chunks until then, trimming the synthetic remainder.
+ if self.in_count > 0 {
+ let delay = self.resampler.as_ref().unwrap().output_delay();
+ let expected = self.in_count * self.out_hz / self.in_hz + delay;
+ let mut rounds = 0;
+ while self.out_count < expected && rounds < 8 {
+ rounds += 1;
+ let result = self
+ .resampler
+ .as_mut()
+ .unwrap()
+ .process_partial::<&[f32]>(None, None);
+ match result {
+ Ok(out) => {
+ let take = (expected - self.out_count).min(out[0].len());
+ self.out_count += take;
+ self.emit_frames(&out[0][..take], &mut emit);
+ }
+ Err(_) => break,
+ }
+ }
+ }
}
// Emit any remaining pending frame (padded with zeros)
@@ -94,6 +135,8 @@ impl FrameResampler {
pub fn reset(&mut self) {
self.in_buf.clear();
self.pending.clear();
+ self.in_count = 0;
+ self.out_count = 0;
if let Some(ref mut resampler) = self.resampler {
resampler.reset();
}
@@ -242,6 +285,47 @@ mod tests {
}
}
+ /// Push silence ending in a 200-sample 0.5 burst, then assert finish()
+ /// recovers the burst and emits floor(input*ratio) + output_delay samples,
+ /// padded to whole 480-sample frames.
+ fn assert_tail_burst_flushed(in_hz: usize, input_len: usize, expected_out: usize) {
+ let mut rs = FrameResampler::new(in_hz, 16000, Duration::from_millis(30));
+ let mut input = vec![0.0f32; input_len];
+ input[input_len - 200..].fill(0.5);
+
+ let mut out = Vec::new();
+ rs.push(&input, |frame| out.extend_from_slice(frame));
+ rs.finish(|frame| out.extend_from_slice(frame));
+
+ let max_abs = out.iter().map(|s| s.abs()).fold(0.0f32, f32::max);
+ assert!(
+ max_abs > 0.3,
+ "tail burst was lost in the resampler, max_abs={max_abs}"
+ );
+ assert_eq!(out.len(), expected_out);
+ }
+
+ #[test]
+ fn finish_flushes_resampler_delay() {
+ // Exact chunks keep in_buf empty, so the burst survives only via the
+ // delay-line drain. 4096 in -> 1365 real out + 171 delay -> 1920 framed.
+ assert_tail_burst_flushed(48000, 4 * RESAMPLER_CHUNK_SIZE, 1920);
+ }
+
+ #[test]
+ fn finish_flushes_resampler_delay_44100() {
+ // fft_size_in (1323) exceeds the 1024 chunk, so the drain must survive
+ // a zero-output round. 4096 in -> 1486 real out + 240 delay -> 1920.
+ assert_tail_burst_flushed(44100, 4 * RESAMPLER_CHUNK_SIZE, 1920);
+ }
+
+ #[test]
+ fn finish_flushes_unaligned_tail() {
+ // Ends mid-chunk: partial-chunk path plus delay drain together.
+ // 4396 in -> 1465 real out + 171 delay -> 1920 framed.
+ assert_tail_burst_flushed(48000, 4 * RESAMPLER_CHUNK_SIZE + 300, 1920);
+ }
+
#[test]
fn finish_does_not_leak_tail_into_next_session() {
// 48kHz -> 16kHz, 30ms frames (480 output samples per frame).
diff --git a/src-tauri/src/audio_toolkit/vad/mod.rs b/src-tauri/src/audio_toolkit/vad/mod.rs
index 02485a4f..586eb7e7 100644
--- a/src-tauri/src/audio_toolkit/vad/mod.rs
+++ b/src-tauri/src/audio_toolkit/vad/mod.rs
@@ -31,9 +31,32 @@ pub trait VoiceActivityDetector: Send + Sync {
/// subsequent frames. Detectors without a smoothing tail can ignore this.
fn set_hangover_frames(&mut self, _frames: usize) {}
+ /// End-of-recording diagnostic snapshot, taken after the final frame.
+ /// Purely observational — implementations must not change what they emit.
+ /// Detectors without smoothing state return None.
+ fn tail_report(&self) -> Option<VadTailReport> {
+ None
+ }
+
fn reset(&mut self) {}
}
+/// End-of-recording snapshot of a smoothing detector's state. Voiced frames
+/// in the withheld tail suggest — but don't prove — a final word cut off at
+/// the stop; a clean report doesn't rule VAD loss out either (soft trailing
+/// speech can be classified as noise).
+#[derive(Debug, Clone, Copy)]
+pub struct VadTailReport {
+ /// Trailing frames buffered but never emitted downstream.
+ pub withheld_frames: usize,
+ /// How many of those withheld frames the inner VAD classified as voiced.
+ pub withheld_voiced_frames: usize,
+ pub in_speech: bool,
+ /// Voiced frames counted toward an unconfirmed speech onset.
+ pub onset_counter: usize,
+ pub hangover_counter: usize,
+}
+
mod silero;
mod smoothed;
diff --git a/src-tauri/src/audio_toolkit/vad/smoothed.rs b/src-tauri/src/audio_toolkit/vad/smoothed.rs
index 52de4d74..218cd615 100644
--- a/src-tauri/src/audio_toolkit/vad/smoothed.rs
+++ b/src-tauri/src/audio_toolkit/vad/smoothed.rs
@@ -1,14 +1,22 @@
-use super::{VadFrame, VoiceActivityDetector};
+use super::{VadFrame, VadTailReport, VoiceActivityDetector};
use anyhow::Result;
use std::collections::VecDeque;
+/// One pre-roll buffer slot. `emitted` and `voiced` exist only to power the
+/// end-of-recording `tail_report()` diagnostic; they never affect emission.
+struct BufferedFrame {
+ samples: Vec<f32>,
+ emitted: bool,
+ voiced: bool,
+}
+
pub struct SmoothedVad {
inner_vad: Box<dyn VoiceActivityDetector>,
prefill_frames: usize,
hangover_frames: usize,
onset_frames: usize,
- frame_buffer: VecDeque<Vec<f32>>,
+ frame_buffer: VecDeque<BufferedFrame>,
hangover_counter: usize,
onset_counter: usize,
in_speech: bool,
@@ -35,18 +43,31 @@ impl SmoothedVad {
temp_out: Vec::new(),
}
}
+
+ fn mark_last_emitted(&mut self) {
+ if let Some(frame) = self.frame_buffer.back_mut() {
+ frame.emitted = true;
+ }
+ }
}
impl VoiceActivityDetector for SmoothedVad {
fn push_frame<'a>(&'a mut self, frame: &'a [f32]) -> Result<VadFrame<'a>> {
// 1. Buffer every incoming frame for possible pre-roll
- self.frame_buffer.push_back(frame.to_vec());
+ self.frame_buffer.push_back(BufferedFrame {
+ samples: frame.to_vec(),
+ emitted: false,
+ voiced: false,
+ });
while self.frame_buffer.len() > self.prefill_frames + 1 {
self.frame_buffer.pop_front();
}
// 2. Delegate to the wrapped boolean VAD
let is_voice = self.inner_vad.is_voice(frame)?;
+ if let Some(last) = self.frame_buffer.back_mut() {
+ last.voiced = is_voice;
+ }
match (self.in_speech, is_voice) {
// Potential start of speech - need to accumulate onset frames
@@ -60,8 +81,9 @@ impl VoiceActivityDetector for SmoothedVad {
// Collect prefill + current frame
self.temp_out.clear();
- for buf in &self.frame_buffer {
- self.temp_out.extend(buf);
+ for buffered in self.frame_buffer.iter_mut() {
+ self.temp_out.extend(buffered.samples.iter());
+ buffered.emitted = true;
}
Ok(VadFrame::Speech(&self.temp_out))
} else {
@@ -73,6 +95,7 @@ impl VoiceActivityDetector for SmoothedVad {
// Ongoing Speech
(true, true) => {
self.hangover_counter = self.hangover_frames;
+ self.mark_last_emitted();
Ok(VadFrame::Speech(frame))
}
@@ -80,6 +103,7 @@ impl VoiceActivityDetector for SmoothedVad {
(true, false) => {
if self.hangover_counter > 0 {
self.hangover_counter -= 1;
+ self.mark_last_emitted();
Ok(VadFrame::Speech(frame))
} else {
self.in_speech = false;
@@ -99,6 +123,32 @@ impl VoiceActivityDetector for SmoothedVad {
self.hangover_frames = frames;
}
+ /// Trailing run of withheld frames plus smoothing state. Interior
+ /// withheld frames (before already-emitted speech) are not counted.
+ fn tail_report(&self) -> Option<VadTailReport> {
+ let mut withheld_frames = 0;
+ let mut withheld_voiced_frames = 0;
+ for frame in self
+ .frame_buffer
+ .iter()
+ .rev()
+ .take_while(|frame| !frame.emitted)
+ {
+ withheld_frames += 1;
+ if frame.voiced {
+ withheld_voiced_frames += 1;
+ }
+ }
+
+ Some(VadTailReport {
+ withheld_frames,
+ withheld_voiced_frames,
+ in_speech: self.in_speech,
+ onset_counter: self.onset_counter,
+ hangover_counter: self.hangover_counter,
+ })
+ }
+
fn reset(&mut self) {
self.inner_vad.reset();
self.frame_buffer.clear();
@@ -108,3 +158,72 @@ impl VoiceActivityDetector for SmoothedVad {
self.temp_out.clear();
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// Inner VAD that replays a scripted voice/no-voice sequence.
+ struct ScriptedVad {
+ script: VecDeque<bool>,
+ }
+
+ impl ScriptedVad {
+ fn new(script: &[bool]) -> Self {
+ Self {
+ script: script.iter().copied().collect(),
+ }
+ }
+ }
+
+ impl VoiceActivityDetector for ScriptedVad {
+ fn push_frame<'a>(&'a mut self, frame: &'a [f32]) -> Result<VadFrame<'a>> {
+ if self.script.pop_front().unwrap_or(false) {
+ Ok(VadFrame::Speech(frame))
+ } else {
+ Ok(VadFrame::Noise)
+ }
+ }
+ }
+
+ fn frame(value: f32) -> Vec<f32> {
+ vec![value; 4]
+ }
+
+ fn smoothed(script: &[bool], onset_frames: usize) -> SmoothedVad {
+ SmoothedVad::new(Box::new(ScriptedVad::new(script)), 3, 2, onset_frames)
+ }
+
+ #[test]
+ fn tail_report_counts_withheld_onset_tail() {
+ // One voiced frame at the end: onset (2 frames) never confirms, so
+ // both trailing frames are withheld and one of them is voiced —
+ // consistent with a final word cut off at the stop boundary.
+ let mut vad = smoothed(&[false, true], 2);
+ assert!(!vad.push_frame(&frame(0.1)).unwrap().is_speech());
+ assert!(!vad.push_frame(&frame(0.9)).unwrap().is_speech());
+
+ let report = vad.tail_report().expect("smoothed VAD always reports");
+ assert_eq!(report.withheld_frames, 2);
+ assert_eq!(report.withheld_voiced_frames, 1);
+ assert_eq!(report.onset_counter, 1);
+ assert!(!report.in_speech);
+ }
+
+ #[test]
+ fn tail_report_counts_only_trailing_run() {
+ // Speech emitted through hangover, then silence past the hangover is
+ // withheld. Only the trailing run counts — frames older than an
+ // emitted frame are excluded.
+ let mut vad = smoothed(&[true, true, false, false, false, false], 2);
+ for v in [0.1, 0.2, 0.3, 0.4] {
+ let _ = vad.push_frame(&frame(v)).unwrap();
+ }
+ assert!(!vad.push_frame(&frame(0.5)).unwrap().is_speech()); // past hangover
+ assert!(!vad.push_frame(&frame(0.6)).unwrap().is_speech());
+
+ let report = vad.tail_report().unwrap();
+ assert_eq!(report.withheld_frames, 2);
+ assert_eq!(report.withheld_voiced_frames, 0);
+ }
+}