Skip to content

A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.

TypeScript

fix(voice): give Apple Speech the microphone's own audio

Alan Buscaglia7a8773cabfe1
diff --git a/apps/desktop/src-tauri/src/voice/engine.rs b/apps/desktop/src-tauri/src/voice/engine.rs
index 8aa5a66..509ac10 100644
--- a/apps/desktop/src-tauri/src/voice/engine.rs
+++ b/apps/desktop/src-tauri/src/voice/engine.rs
@@ -76,7 +76,7 @@ impl Recognizer for Preferred {
#[cfg(test)]
mod tests {
use super::*;
- use crate::voice::fake::{FakeClock, FakeMicrophone, FakeRecognizer, Recorder};
+ use crate::voice::fake::{FakeClock, FakeMicrophone, FakeNative, FakeRecognizer, Recorder};
use crate::voice::parakeet::{Engine, Loader, Locate};
use crate::voice::{Authorization, Transcript, Voice, VoiceError, VoiceStatus};
use std::path::{Path, PathBuf};
@@ -138,6 +138,31 @@ mod tests {
assert!(rig.apple.log.lock().unwrap().locales.is_empty(), "Apple Speech is not used");
}
+ #[test]
+ fn without_the_model_apple_speech_gets_the_device_buffers() {
+ let mut apple = FakeRecognizer::granted();
+ apple.native = true;
+ let rig = rig(apple, true);
+ rig.voice.start(None, Recorder::default().events()).unwrap();
+ let buffer = FakeNative { rate: 48_000, frames: 960 };
+ rig.mic.sink().audio_native(&[0.0; 320], &buffer);
+ let log = rig.apple.log.lock().unwrap();
+ assert_eq!(log.native, vec![buffer]);
+ assert!(log.fed.is_empty());
+ }
+
+ #[test]
+ fn parakeet_still_receives_the_16k_mono_stream_when_device_buffers_come_along() {
+ let mut apple = FakeRecognizer::granted();
+ apple.native = true;
+ let rig = rig(apple, true);
+ rig.installed.store(true, Ordering::SeqCst);
+ rig.voice.start(None, Recorder::default().events()).unwrap();
+ rig.mic.sink().audio_native(&[0.0; 320], &FakeNative { rate: 48_000, frames: 960 });
+ assert_eq!(rig.voice.stop(), Ok(Transcript { text: "parakeet heard 320".into() }));
+ assert!(rig.apple.log.lock().unwrap().native.is_empty(), "Apple Speech is not used");
+ }
+
#[test]
fn parakeet_needs_only_the_microphone_permission() {
let rig = rig(FakeRecognizer::with(Authorization::Denied, Authorization::Denied), true);
diff --git a/apps/desktop/src-tauri/src/voice/fake.rs b/apps/desktop/src-tauri/src/voice/fake.rs
index c916fff..92c6d33 100644
--- a/apps/desktop/src-tauri/src/voice/fake.rs
+++ b/apps/desktop/src-tauri/src/voice/fake.rs
@@ -1,7 +1,7 @@
//! Fakes for the voice tests: a microphone and a recognizer that record what the machine asks
//! of them and let a test play the platform's side (audio, results, failures).
-use super::{Authorization, Events, Microphone, Recognition, Recognizer, Sink, Stream, VoiceError, VoiceEvent};
+use super::{Authorization, Events, Microphone, NativeAudio, Recognition, Recognizer, Sink, Stream, VoiceError, VoiceEvent};
use crate::computer::Clock;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Condvar, Mutex};
@@ -109,6 +109,8 @@ pub struct RecognitionLog {
pub requests: usize,
pub locales: Vec<String>,
pub fed: Vec<f32>,
+ /// Device buffers, for a recognition that takes them.
+ pub native: Vec<FakeNative>,
pub finished: usize,
pub cancelled: usize,
pub sink: Option<Sink>,
@@ -124,6 +126,8 @@ pub struct FakeRecognizer {
pub final_text: Mutex<Option<String>>,
/// Reported as a failure when the recognition is finished.
pub fail_on_finish: Mutex<Option<String>>,
+ /// Plays a recognizer that takes the device's own audio (as Apple Speech does).
+ pub native: bool,
pub log: Arc<Mutex<RecognitionLog>>,
}
@@ -136,6 +140,7 @@ impl FakeRecognizer {
supported: vec!["en-US".into(), "es-ES".into(), "es-MX".into()],
final_text: Mutex::new(None),
fail_on_finish: Mutex::new(None),
+ native: false,
log: Arc::default(),
}
}
@@ -178,7 +183,7 @@ impl Recognizer for FakeRecognizer {
log.sink = Some(sink.clone());
let final_text = self.final_text.lock().unwrap().clone();
let failure = self.fail_on_finish.lock().unwrap().clone();
- Ok(Box::new(FakeRecognition { log: self.log.clone(), sink, final_text, failure }))
+ Ok(Box::new(FakeRecognition { log: self.log.clone(), sink, final_text, failure, native: self.native }))
}
}
@@ -187,6 +192,7 @@ struct FakeRecognition {
sink: Sink,
final_text: Option<String>,
failure: Option<String>,
+ native: bool,
}
impl Recognition for FakeRecognition {
@@ -194,6 +200,15 @@ impl Recognition for FakeRecognition {
self.log.lock().unwrap().fed.extend_from_slice(samples);
}
+ fn wants_native(&self) -> bool {
+ self.native
+ }
+
+ fn feed_native(&mut self, audio: &NativeAudio) {
+ let buffer = audio.downcast_ref::<FakeNative>().expect("the fake microphone's buffer");
+ self.log.lock().unwrap().native.push(buffer.clone());
+ }
+
fn finish(&mut self) {
self.log.lock().unwrap().finished += 1;
if let Some(text) = &self.final_text {
@@ -209,6 +224,13 @@ impl Recognition for FakeRecognition {
}
}
+/// A device buffer, as a microphone that captures at its own rate would pass it along.
+#[derive(Debug, Clone, PartialEq)]
+pub struct FakeNative {
+ pub rate: u32,
+ pub frames: usize,
+}
+
/// Collects the events a webview would receive.
#[derive(Default, Clone)]
pub struct Recorder(pub Arc<Mutex<Vec<VoiceEvent>>>);
diff --git a/apps/desktop/src-tauri/src/voice/machine.rs b/apps/desktop/src-tauri/src/voice/machine.rs
index 1bcd935..cfe1617 100644
--- a/apps/desktop/src-tauri/src/voice/machine.rs
+++ b/apps/desktop/src-tauri/src/voice/machine.rs
@@ -5,7 +5,7 @@
//! `Sink` bound to one recording, so results from an earlier recording are ignored. No
//! platform call or event is made while the state lock is held.
-use super::{audio, locale, Authorization, Events, Microphone, Recognition, Recognizer, Stream, Transcript};
+use super::{audio, locale, Authorization, Events, Microphone, NativeAudio, Recognition, Recognizer, Stream, Transcript};
use super::{VoiceError, VoiceEvent, VoiceStatus};
use crate::computer::Clock;
use std::sync::{Arc, Condvar, Mutex, MutexGuard, Weak};
@@ -55,9 +55,15 @@ struct Feed {
}
impl Feed {
- fn push(&mut self, samples: &[f32]) {
- if self.open {
- self.recognition.feed(samples);
+ /// The device's own buffer when the recognition takes it, else the 16 kHz stream.
+ fn push(&mut self, samples: &[f32], native: Option<&NativeAudio>) {
+ if !self.open {
+ return;
+ }
+ match native {
+ Some(native) if self.recognition.wants_native() => self.recognition.feed_native(native),
+ _ if !samples.is_empty() => self.recognition.feed(samples),
+ _ => {}
}
}
@@ -309,6 +315,17 @@ pub struct Sink {
impl Sink {
/// A chunk of `audio::SAMPLE_RATE` mono PCM: fed to the recognizer, and metered.
pub fn audio(&self, samples: &[f32]) {
+ self.capture(samples, None);
+ }
+
+ /// The same, with the device's own buffer it came from: a recognition that wants it gets
+ /// the buffer instead of the samples (S30.6); the meter still follows the samples, which
+ /// may be empty while the resampler fills its first period.
+ pub fn audio_native(&self, samples: &[f32], native: &NativeAudio) {
+ self.capture(samples, Some(native));
+ }
+
+ fn capture(&self, samples: &[f32], native: Option<&NativeAudio>) {
let Some(inner) = self.inner.upgrade() else { return };
let (feed, level) = {
let mut state = lock(&inner.state);
@@ -329,7 +346,7 @@ impl Sink {
if let Some((events, level)) = level {
events(level);
}
- lock(&feed).push(samples);
+ lock(&feed).push(samples, native);
}
/// The text so far.
@@ -397,7 +414,7 @@ impl Sink {
#[cfg(test)]
mod tests {
use super::*;
- use crate::voice::fake::{FakeClock, FakeMicrophone, FakeRecognizer, Gate, Recorder};
+ use crate::voice::fake::{FakeClock, FakeMicrophone, FakeNative, FakeRecognizer, Gate, Recorder};
use crate::voice::{Authorization, Transcript, VoiceError, VoiceEvent, VoiceStatus};
use std::sync::atomic::AtomicU64;
use std::sync::Arc;
@@ -592,6 +609,64 @@ mod tests {
assert!(last.unwrap() > 0.5, "the meter shows the loudest chunk since the last level");
}
+ #[test]
+ fn a_recognition_that_wants_native_audio_gets_the_device_buffers_instead_of_the_16k_stream() {
+ let mut recognizer = FakeRecognizer::granted();
+ recognizer.native = true;
+ let rig = rig(FakeMicrophone::granted(), recognizer);
+ rig.start().unwrap();
+ let sink = rig.mic.sink();
+ let buffer = FakeNative { rate: 48_000, frames: 480 };
+ sink.audio_native(&[0.5; 160], &buffer);
+ sink.audio_native(&[0.5; 160], &buffer);
+ let log = rig.recognizer.log.lock().unwrap();
+ assert_eq!(log.native, vec![buffer.clone(), buffer]);
+ assert!(log.fed.is_empty(), "the 16 kHz stream is not fed as well");
+ assert_eq!(rig.recorder.levels(), 1, "the meter still follows the 16 kHz stream");
+ }
+
+ #[test]
+ fn the_16k_stream_feeds_a_recognition_that_does_not_want_native_audio() {
+ let rig = granted();
+ rig.start().unwrap();
+ rig.mic.sink().audio_native(&[0.25; 160], &FakeNative { rate: 48_000, frames: 480 });
+ let log = rig.recognizer.log.lock().unwrap();
+ assert_eq!(log.fed, vec![0.25; 160]);
+ assert!(log.native.is_empty());
+ }
+
+ #[test]
+ fn without_device_buffers_a_recognition_that_wants_them_gets_the_16k_stream() {
+ let mut recognizer = FakeRecognizer::granted();
+ recognizer.native = true;
+ let rig = rig(FakeMicrophone::granted(), recognizer);
+ rig.start().unwrap();
+ rig.mic.sink().audio(&[0.25; 160]);
+ assert_eq!(rig.recognizer.log.lock().unwrap().fed.len(), 160);
+ }
+
+ #[test]
+ fn a_device_buffer_with_no_16k_samples_yet_still_reaches_a_native_recognition() {
+ let mut recognizer = FakeRecognizer::granted();
+ recognizer.native = true;
+ let rig = rig(FakeMicrophone::granted(), recognizer);
+ rig.start().unwrap();
+ rig.mic.sink().audio_native(&[], &FakeNative { rate: 48_000, frames: 2 });
+ assert_eq!(rig.recognizer.log.lock().unwrap().native.len(), 1);
+ }
+
+ #[test]
+ fn no_device_buffers_reach_the_recognizer_after_stop() {
+ let mut recognizer = FakeRecognizer::granted();
+ recognizer.native = true;
+ let rig = rig(FakeMicrophone::granted(), recognizer);
+ rig.start().unwrap();
+ let sink = rig.mic.sink();
+ rig.voice.stop().unwrap();
+ sink.audio_native(&[0.5; 160], &FakeNative { rate: 48_000, frames: 480 });
+ assert!(rig.recognizer.log.lock().unwrap().native.is_empty());
+ }
+
#[test]
fn no_audio_reaches_the_recognizer_after_stop() {
let rig = granted();
diff --git a/apps/desktop/src-tauri/src/voice/macos.rs b/apps/desktop/src-tauri/src/voice/macos.rs
index 0a39da2..6ae9521 100644
--- a/apps/desktop/src-tauri/src/voice/macos.rs
+++ b/apps/desktop/src-tauri/src/voice/macos.rs
@@ -1,6 +1,12 @@
//! The macOS voice backends: the native microphone (AVAudioEngine's input node, resampled to
-//! 16 kHz mono) and Apple's Speech framework (`SFSpeechRecognizer`, on-device when the locale
-//! supports it, else Apple's servers). Nothing here decides policy; `machine` does.
+//! 16 kHz mono, with each tap buffer passed along as it came) and Apple's Speech framework
+//! (`SFSpeechRecognizer`, on-device when the locale supports it, else Apple's servers). Nothing
+//! here decides policy; `machine` does.
+//!
+//! Apple Speech is fed the input node's own buffers (S30.6), as Apple's documented pattern
+//! does: with our 16 kHz box-filtered stream it misheard speech that Parakeet decoded from the
+//! same capture (L101-L102). The 16 kHz path in `AppleRecognition::feed` stays only for a
+//! microphone without device buffers.
//!
//! Threads: permission handlers, the audio tap, the interruption notification, and recognition
//! results all arrive off the main thread (the recognizer gets an operation queue of its own),
@@ -8,7 +14,7 @@
//! and from those callbacks; AVAudioEngine and the Speech request are not tied to a thread.
use super::audio::{self, Resampler, SAMPLE_RATE};
-use super::{Authorization, Microphone, Recognition, Recognizer, Sink, Stream, VoiceError};
+use super::{Authorization, Microphone, NativeAudio, Recognition, Recognizer, Sink, Stream, VoiceError};
use block2::RcBlock;
use objc2::rc::Retained;
use objc2::runtime::{AnyObject, Bool, ProtocolObject};
@@ -104,9 +110,8 @@ impl Microphone for MacMicrophone {
.map(|c| unsafe { std::slice::from_raw_parts((*data.add(c)).as_ptr(), frames) })
.collect();
let pcm = resampler.borrow_mut().process(&audio::mono(&planes));
- if !pcm.is_empty() {
- audio_sink.audio(&pcm);
- }
+ // The buffer itself goes along for Apple Speech, which appends it as captured.
+ audio_sink.audio_native(&pcm, buffer as &NativeAudio);
});
let tap_block = &*tap as *const block2::DynBlock<_> as *mut block2::DynBlock<_>;
// SAFETY: the block matches AVAudioNodeTapBlock and stays alive in the stream.
@@ -278,6 +283,19 @@ impl Recognition for AppleRecognition {
}
}
+ fn wants_native(&self) -> bool {
+ true
+ }
+
+ /// The input node's tap buffer, in its own rate and channel layout. The request retains
+ /// what it needs, as with Apple's `request.append(buffer)` in the tap.
+ fn feed_native(&mut self, audio: &NativeAudio) {
+ if let Some(buffer) = audio.downcast_ref::<AVAudioPCMBuffer>() {
+ // SAFETY: a valid buffer from the engine's tap, used within the tap call.
+ unsafe { self.request.appendAudioPCMBuffer(buffer) };
+ }
+ }
+
fn finish(&mut self) {
// SAFETY: ending the audio of a live request; the final result follows on the queue.
unsafe { self.request.endAudio() };
diff --git a/apps/desktop/src-tauri/src/voice/mod.rs b/apps/desktop/src-tauri/src/voice/mod.rs
index 3836358..44a05ed 100644
--- a/apps/desktop/src-tauri/src/voice/mod.rs
+++ b/apps/desktop/src-tauri/src/voice/mod.rs
@@ -2,7 +2,9 @@
//! user's message. Two seams keep the backends swappable:
//!
//! - `Microphone` captures audio and delivers 16 kHz mono f32 PCM (`audio::SAMPLE_RATE`), the
-//! format offline models such as sherpa-onnx expect, so any recognizer can consume it.
+//! format offline models such as sherpa-onnx expect, so any recognizer can consume it. When
+//! it can, it also passes the device's own buffers along (`NativeAudio`), for a recognizer
+//! that does better with the audio as captured (Apple Speech, S30.6).
//! - `Recognizer` turns that PCM into partial and final text. Apple's Speech framework
//! (`macos::AppleSpeech`) is the default on macOS; the optional local model (S30.5,
//! `parakeet::Parakeet`, NVIDIA Parakeet through sherpa-onnx) is preferred once the user has
@@ -164,8 +166,14 @@ impl From<VoiceError> for String {
}
}
-/// Native audio capture. Delivers `audio::SAMPLE_RATE` mono f32 chunks to `Sink::audio` (on
-/// the audio thread) and reports interruptions with `Sink::failed`.
+/// The capture in the device's own format (on macOS, the input node's `AVAudioPCMBuffer`),
+/// valid for one call. Opaque to the machine: only the recognition that asks for it downcasts
+/// it, so it must come from the microphone that recognition was made for.
+pub type NativeAudio = dyn std::any::Any;
+
+/// Native audio capture. Delivers `audio::SAMPLE_RATE` mono f32 chunks to `Sink::audio`, or
+/// to `Sink::audio_native` along with the device's own buffer (on the audio thread), and
+/// reports interruptions with `Sink::failed`.
pub trait Microphone: Send + Sync {
/// The current permission, without prompting.
fn authorization(&self) -> Authorization;
@@ -202,6 +210,14 @@ pub trait Recognizer: Send + Sync {
pub trait Recognition: Send {
/// More audio. Runs on the audio thread, so it must not block (heavy decoders queue it).
fn feed(&mut self, samples: &[f32]);
+ /// Whether this recognition takes the device's own buffers (`feed_native`) instead of the
+ /// 16 kHz stream, when the microphone provides them.
+ fn wants_native(&self) -> bool {
+ false
+ }
+ /// The device's own buffer, instead of `feed`; only called when `wants_native`. Runs on
+ /// the audio thread.
+ fn feed_native(&mut self, _audio: &NativeAudio) {}
/// No more audio: the final text follows through `Sink::finished`.
fn finish(&mut self);
/// Discards the recognition.