Skip to content

armgit / Handy

Public Imported from GitHub
0

A free, open source, and extensible speech-to-text application that works completely offline.

theoretically have different sample formats working as expected.

CJ Pais113edbab5c98
diff --git a/src-tauri/src/managers/audio.rs b/src-tauri/src/managers/audio.rs
index 010beee7..e27d00f7 100644
--- a/src-tauri/src/managers/audio.rs
+++ b/src-tauri/src/managers/audio.rs
@@ -1,4 +1,5 @@
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
+use cpal::SampleFormat;
use rubato::{FftFixedIn, Resampler};
use std::sync::{Arc, Mutex};
use std::vec::Vec;
@@ -16,6 +17,34 @@ pub struct AudioRecordingManager {
buffer: Arc<Mutex<Vec<f32>>>,
}
+trait SampleToF32 {
+ fn to_f32(&self) -> f32;
+}
+
+impl SampleToF32 for i8 {
+ fn to_f32(&self) -> f32 {
+ *self as f32 / 128.0
+ }
+}
+
+impl SampleToF32 for i16 {
+ fn to_f32(&self) -> f32 {
+ *self as f32 / 32768.0
+ }
+}
+
+impl SampleToF32 for i32 {
+ fn to_f32(&self) -> f32 {
+ *self as f32 / 2147483648.0
+ }
+}
+
+impl SampleToF32 for f32 {
+ fn to_f32(&self) -> f32 {
+ *self
+ }
+}
+
impl AudioRecordingManager {
pub fn new(app: &App) -> Result<Self, anyhow::Error> {
let vad_path = app.path().resolve(
@@ -52,64 +81,140 @@ impl AudioRecordingManager {
let vad_buffer = Arc::new(Mutex::new(Vec::new()));
let vad_buffer_clone = Arc::clone(&vad_buffer);
- std::thread::spawn(move || {
- let stream = match config.sample_format() {
- cpal::SampleFormat::F32 => device.build_input_stream(
- &config.into(),
- move |data: &[f32], _: &cpal::InputCallbackInfo| {
- let state_guard = state_clone.lock().unwrap();
- if let RecordingState::Recording { .. } = *state_guard {
- let mut temp_buffer = temp_buffer_clone.lock().unwrap();
- temp_buffer.extend_from_slice(data);
-
- // Process when we have enough samples
- while temp_buffer.len() >= 1024 {
- // Take the first 1024 samples for processing
- let chunk: Vec<f32> = temp_buffer.drain(..1024).collect();
-
- // Convert input data to the format expected by Rubato
- let input_frames = vec![chunk];
-
- // Process the audio chunk through the resampler
- let mut resampler = resampler_clone.lock().unwrap();
- if let Ok(resampled) = resampler.process(&input_frames, None) {
- // Add resampled data to VAD buffer
- let mut vad_buffer = vad_buffer_clone.lock().unwrap();
- vad_buffer.extend_from_slice(&resampled[0]);
-
- // Process 30ms chunks (480 samples) for VAD
- while vad_buffer.len() >= 480 {
- let chunk = vad_buffer.drain(..480).collect::<Vec<f32>>();
-
- // Use VAD to detect speech
- if let Ok(mut vad) = vad_clone.lock() {
- // println!("VAD lock acquired");
- match vad.compute(&chunk) {
- Ok(result) => {
- if result.prob > 0.15 {
- let mut buffer =
- buffer_clone.lock().unwrap();
- buffer.extend_from_slice(&chunk);
- }
- }
- Err(error) => {
- eprintln!("Error computing VAD: {:?}", error)
- }
- }
+ // Generic function to process audio data
+ fn process_audio<T: SampleToF32 + Send + 'static>(
+ data: &[T],
+ state_clone: Arc<Mutex<RecordingState>>,
+ temp_buffer_clone: Arc<Mutex<Vec<f32>>>,
+ resampler_clone: Arc<Mutex<FftFixedIn<f32>>>,
+ vad_buffer_clone: Arc<Mutex<Vec<f32>>>,
+ buffer_clone: Arc<Mutex<Vec<f32>>>,
+ vad_clone: Arc<Mutex<Vad>>,
+ ) {
+ let state_guard = state_clone.lock().unwrap();
+ if let RecordingState::Recording { .. } = *state_guard {
+ let mut temp_buffer = temp_buffer_clone.lock().unwrap();
+
+ // Convert incoming data to f32
+ let f32_data: Vec<f32> = data.iter().map(|sample| sample.to_f32()).collect();
+ temp_buffer.extend_from_slice(&f32_data);
+
+ // Process when we have enough samples
+ while temp_buffer.len() >= 1024 {
+ // Take the first 1024 samples for processing
+ let chunk: Vec<f32> = temp_buffer.drain(..1024).collect();
+
+ // Convert input data to the format expected by Rubato
+ let input_frames = vec![chunk];
+
+ // Process the audio chunk through the resampler
+ let mut resampler = resampler_clone.lock().unwrap();
+ if let Ok(resampled) = resampler.process(&input_frames, None) {
+ // Add resampled data to VAD buffer
+ let mut vad_buffer = vad_buffer_clone.lock().unwrap();
+ vad_buffer.extend_from_slice(&resampled[0]);
+
+ // Process 30ms chunks (480 samples) for VAD
+ while vad_buffer.len() >= 480 {
+ let chunk = vad_buffer.drain(..480).collect::<Vec<f32>>();
+
+ // Use VAD to detect speech
+ if let Ok(mut vad) = vad_clone.lock() {
+ match vad.compute(&chunk) {
+ Ok(result) => {
+ if result.prob > 0.15 {
+ let mut buffer = buffer_clone.lock().unwrap();
+ buffer.extend_from_slice(&chunk);
}
}
+ Err(error) => {
+ eprintln!("Error computing VAD: {:?}", error)
+ }
}
}
}
+ }
+ }
+ }
+ }
+
+ std::thread::spawn(move || {
+ let err_fn = |err| eprintln!("Error in stream: {}", err);
+
+ // Build the appropriate stream based on the sample format
+ let stream = match config.sample_format() {
+ SampleFormat::I8 => device.build_input_stream(
+ &config.into(),
+ move |data: &[i8], _| {
+ process_audio(
+ data,
+ Arc::clone(&state_clone),
+ Arc::clone(&temp_buffer_clone),
+ Arc::clone(&resampler_clone),
+ Arc::clone(&vad_buffer_clone),
+ Arc::clone(&buffer_clone),
+ Arc::clone(&vad_clone),
+ )
},
- |err| eprintln!("Error in stream: {}", err),
+ err_fn,
None,
),
- sample_format => panic!("Unsupported sample format: {:?}", sample_format),
+ SampleFormat::I16 => device.build_input_stream(
+ &config.into(),
+ move |data: &[i16], _| {
+ process_audio(
+ data,
+ Arc::clone(&state_clone),
+ Arc::clone(&temp_buffer_clone),
+ Arc::clone(&resampler_clone),
+ Arc::clone(&vad_buffer_clone),
+ Arc::clone(&buffer_clone),
+ Arc::clone(&vad_clone),
+ )
+ },
+ err_fn,
+ None,
+ ),
+ SampleFormat::I32 => device.build_input_stream(
+ &config.into(),
+ move |data: &[i32], _| {
+ process_audio(
+ data,
+ Arc::clone(&state_clone),
+ Arc::clone(&temp_buffer_clone),
+ Arc::clone(&resampler_clone),
+ Arc::clone(&vad_buffer_clone),
+ Arc::clone(&buffer_clone),
+ Arc::clone(&vad_clone),
+ )
+ },
+ err_fn,
+ None,
+ ),
+ SampleFormat::F32 => device.build_input_stream(
+ &config.into(),
+ move |data: &[f32], _| {
+ process_audio(
+ data,
+ Arc::clone(&state_clone),
+ Arc::clone(&temp_buffer_clone),
+ Arc::clone(&resampler_clone),
+ Arc::clone(&vad_buffer_clone),
+ Arc::clone(&buffer_clone),
+ Arc::clone(&vad_clone),
+ )
+ },
+ err_fn,
+ None,
+ ),
+ sample_format => {
+ // Use anyhow to return a proper error instead of panicking
+ panic!("Unsupported sample format: {:?}", sample_format);
+ }
}
- .unwrap();
+ .expect("Failed to build input stream");
- stream.play().unwrap();
+ stream.play().expect("Failed to play stream");
std::thread::park();
});