A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.
TypeScript
fix(voice): give Apple Speech the microphone's own audio
Alan Buscaglia7a8773cabfe1
diff --git a/apps/desktop/src-tauri/src/voice/engine.rs b/apps/desktop/src-tauri/src/voice/engine.rsindex 8aa5a66..509ac10 100644--- a/apps/desktop/src-tauri/src/voice/engine.rs+++ b/apps/desktop/src-tauri/src/voice/engine.rs@@ -76,7 +76,7 @@ impl Recognizer for Preferred {#[cfg(test)]mod tests {use super::*;- use crate::voice::fake::{FakeClock, FakeMicrophone, FakeRecognizer, Recorder};+ use crate::voice::fake::{FakeClock, FakeMicrophone, FakeNative, FakeRecognizer, Recorder};use crate::voice::parakeet::{Engine, Loader, Locate};use crate::voice::{Authorization, Transcript, Voice, VoiceError, VoiceStatus};use std::path::{Path, PathBuf};@@ -138,6 +138,31 @@ mod tests {assert!(rig.apple.log.lock().unwrap().locales.is_empty(), "Apple Speech is not used");}+ #[test]+ fn without_the_model_apple_speech_gets_the_device_buffers() {+ let mut apple = FakeRecognizer::granted();+ apple.native = true;+ let rig = rig(apple, true);+ rig.voice.start(None, Recorder::default().events()).unwrap();+ let buffer = FakeNative { rate: 48_000, frames: 960 };+ rig.mic.sink().audio_native(&[0.0; 320], &buffer);+ let log = rig.apple.log.lock().unwrap();+ assert_eq!(log.native, vec![buffer]);+ assert!(log.fed.is_empty());+ }++ #[test]+ fn parakeet_still_receives_the_16k_mono_stream_when_device_buffers_come_along() {+ let mut apple = FakeRecognizer::granted();+ apple.native = true;+ let rig = rig(apple, true);+ rig.installed.store(true, Ordering::SeqCst);+ rig.voice.start(None, Recorder::default().events()).unwrap();+ rig.mic.sink().audio_native(&[0.0; 320], &FakeNative { rate: 48_000, frames: 960 });+ assert_eq!(rig.voice.stop(), Ok(Transcript { text: "parakeet heard 320".into() }));+ assert!(rig.apple.log.lock().unwrap().native.is_empty(), "Apple Speech is not used");+ }+#[test]fn parakeet_needs_only_the_microphone_permission() {let rig = rig(FakeRecognizer::with(Authorization::Denied, Authorization::Denied), true);diff --git a/apps/desktop/src-tauri/src/voice/fake.rs b/apps/desktop/src-tauri/src/voice/fake.rsindex c916fff..92c6d33 100644--- a/apps/desktop/src-tauri/src/voice/fake.rs+++ b/apps/desktop/src-tauri/src/voice/fake.rs@@ -1,7 +1,7 @@//! Fakes for the voice tests: a microphone and a recognizer that record what the machine asks//! of them and let a test play the platform's side (audio, results, failures).-use super::{Authorization, Events, Microphone, Recognition, Recognizer, Sink, Stream, VoiceError, VoiceEvent};+use super::{Authorization, Events, Microphone, NativeAudio, Recognition, Recognizer, Sink, Stream, VoiceError, VoiceEvent};use crate::computer::Clock;use std::sync::atomic::{AtomicU64, Ordering};use std::sync::{Arc, Condvar, Mutex};@@ -109,6 +109,8 @@ pub struct RecognitionLog {pub requests: usize,pub locales: Vec<String>,pub fed: Vec<f32>,+ /// Device buffers, for a recognition that takes them.+ pub native: Vec<FakeNative>,pub finished: usize,pub cancelled: usize,pub sink: Option<Sink>,@@ -124,6 +126,8 @@ pub struct FakeRecognizer {pub final_text: Mutex<Option<String>>,/// Reported as a failure when the recognition is finished.pub fail_on_finish: Mutex<Option<String>>,+ /// Plays a recognizer that takes the device's own audio (as Apple Speech does).+ pub native: bool,pub log: Arc<Mutex<RecognitionLog>>,}@@ -136,6 +140,7 @@ impl FakeRecognizer {supported: vec!["en-US".into(), "es-ES".into(), "es-MX".into()],final_text: Mutex::new(None),fail_on_finish: Mutex::new(None),+ native: false,log: Arc::default(),}}@@ -178,7 +183,7 @@ impl Recognizer for FakeRecognizer {log.sink = Some(sink.clone());let final_text = self.final_text.lock().unwrap().clone();let failure = self.fail_on_finish.lock().unwrap().clone();- Ok(Box::new(FakeRecognition { log: self.log.clone(), sink, final_text, failure }))+ Ok(Box::new(FakeRecognition { log: self.log.clone(), sink, final_text, failure, native: self.native }))}}@@ -187,6 +192,7 @@ struct FakeRecognition {sink: Sink,final_text: Option<String>,failure: Option<String>,+ native: bool,}impl Recognition for FakeRecognition {@@ -194,6 +200,15 @@ impl Recognition for FakeRecognition {self.log.lock().unwrap().fed.extend_from_slice(samples);}+ fn wants_native(&self) -> bool {+ self.native+ }++ fn feed_native(&mut self, audio: &NativeAudio) {+ let buffer = audio.downcast_ref::<FakeNative>().expect("the fake microphone's buffer");+ self.log.lock().unwrap().native.push(buffer.clone());+ }+fn finish(&mut self) {self.log.lock().unwrap().finished += 1;if let Some(text) = &self.final_text {@@ -209,6 +224,13 @@ impl Recognition for FakeRecognition {}}+/// A device buffer, as a microphone that captures at its own rate would pass it along.+#[derive(Debug, Clone, PartialEq)]+pub struct FakeNative {+ pub rate: u32,+ pub frames: usize,+}+/// Collects the events a webview would receive.#[derive(Default, Clone)]pub struct Recorder(pub Arc<Mutex<Vec<VoiceEvent>>>);diff --git a/apps/desktop/src-tauri/src/voice/machine.rs b/apps/desktop/src-tauri/src/voice/machine.rsindex 1bcd935..cfe1617 100644--- a/apps/desktop/src-tauri/src/voice/machine.rs+++ b/apps/desktop/src-tauri/src/voice/machine.rs@@ -5,7 +5,7 @@//! `Sink` bound to one recording, so results from an earlier recording are ignored. No//! platform call or event is made while the state lock is held.-use super::{audio, locale, Authorization, Events, Microphone, Recognition, Recognizer, Stream, Transcript};+use super::{audio, locale, Authorization, Events, Microphone, NativeAudio, Recognition, Recognizer, Stream, Transcript};use super::{VoiceError, VoiceEvent, VoiceStatus};use crate::computer::Clock;use std::sync::{Arc, Condvar, Mutex, MutexGuard, Weak};@@ -55,9 +55,15 @@ struct Feed {}impl Feed {- fn push(&mut self, samples: &[f32]) {- if self.open {- self.recognition.feed(samples);+ /// The device's own buffer when the recognition takes it, else the 16 kHz stream.+ fn push(&mut self, samples: &[f32], native: Option<&NativeAudio>) {+ if !self.open {+ return;+ }+ match native {+ Some(native) if self.recognition.wants_native() => self.recognition.feed_native(native),+ _ if !samples.is_empty() => self.recognition.feed(samples),+ _ => {}}}@@ -309,6 +315,17 @@ pub struct Sink {impl Sink {/// A chunk of `audio::SAMPLE_RATE` mono PCM: fed to the recognizer, and metered.pub fn audio(&self, samples: &[f32]) {+ self.capture(samples, None);+ }++ /// The same, with the device's own buffer it came from: a recognition that wants it gets+ /// the buffer instead of the samples (S30.6); the meter still follows the samples, which+ /// may be empty while the resampler fills its first period.+ pub fn audio_native(&self, samples: &[f32], native: &NativeAudio) {+ self.capture(samples, Some(native));+ }++ fn capture(&self, samples: &[f32], native: Option<&NativeAudio>) {let Some(inner) = self.inner.upgrade() else { return };let (feed, level) = {let mut state = lock(&inner.state);@@ -329,7 +346,7 @@ impl Sink {if let Some((events, level)) = level {events(level);}- lock(&feed).push(samples);+ lock(&feed).push(samples, native);}/// The text so far.@@ -397,7 +414,7 @@ impl Sink {#[cfg(test)]mod tests {use super::*;- use crate::voice::fake::{FakeClock, FakeMicrophone, FakeRecognizer, Gate, Recorder};+ use crate::voice::fake::{FakeClock, FakeMicrophone, FakeNative, FakeRecognizer, Gate, Recorder};use crate::voice::{Authorization, Transcript, VoiceError, VoiceEvent, VoiceStatus};use std::sync::atomic::AtomicU64;use std::sync::Arc;@@ -592,6 +609,64 @@ mod tests {assert!(last.unwrap() > 0.5, "the meter shows the loudest chunk since the last level");}+ #[test]+ fn a_recognition_that_wants_native_audio_gets_the_device_buffers_instead_of_the_16k_stream() {+ let mut recognizer = FakeRecognizer::granted();+ recognizer.native = true;+ let rig = rig(FakeMicrophone::granted(), recognizer);+ rig.start().unwrap();+ let sink = rig.mic.sink();+ let buffer = FakeNative { rate: 48_000, frames: 480 };+ sink.audio_native(&[0.5; 160], &buffer);+ sink.audio_native(&[0.5; 160], &buffer);+ let log = rig.recognizer.log.lock().unwrap();+ assert_eq!(log.native, vec![buffer.clone(), buffer]);+ assert!(log.fed.is_empty(), "the 16 kHz stream is not fed as well");+ assert_eq!(rig.recorder.levels(), 1, "the meter still follows the 16 kHz stream");+ }++ #[test]+ fn the_16k_stream_feeds_a_recognition_that_does_not_want_native_audio() {+ let rig = granted();+ rig.start().unwrap();+ rig.mic.sink().audio_native(&[0.25; 160], &FakeNative { rate: 48_000, frames: 480 });+ let log = rig.recognizer.log.lock().unwrap();+ assert_eq!(log.fed, vec![0.25; 160]);+ assert!(log.native.is_empty());+ }++ #[test]+ fn without_device_buffers_a_recognition_that_wants_them_gets_the_16k_stream() {+ let mut recognizer = FakeRecognizer::granted();+ recognizer.native = true;+ let rig = rig(FakeMicrophone::granted(), recognizer);+ rig.start().unwrap();+ rig.mic.sink().audio(&[0.25; 160]);+ assert_eq!(rig.recognizer.log.lock().unwrap().fed.len(), 160);+ }++ #[test]+ fn a_device_buffer_with_no_16k_samples_yet_still_reaches_a_native_recognition() {+ let mut recognizer = FakeRecognizer::granted();+ recognizer.native = true;+ let rig = rig(FakeMicrophone::granted(), recognizer);+ rig.start().unwrap();+ rig.mic.sink().audio_native(&[], &FakeNative { rate: 48_000, frames: 2 });+ assert_eq!(rig.recognizer.log.lock().unwrap().native.len(), 1);+ }++ #[test]+ fn no_device_buffers_reach_the_recognizer_after_stop() {+ let mut recognizer = FakeRecognizer::granted();+ recognizer.native = true;+ let rig = rig(FakeMicrophone::granted(), recognizer);+ rig.start().unwrap();+ let sink = rig.mic.sink();+ rig.voice.stop().unwrap();+ sink.audio_native(&[0.5; 160], &FakeNative { rate: 48_000, frames: 480 });+ assert!(rig.recognizer.log.lock().unwrap().native.is_empty());+ }+#[test]fn no_audio_reaches_the_recognizer_after_stop() {let rig = granted();diff --git a/apps/desktop/src-tauri/src/voice/macos.rs b/apps/desktop/src-tauri/src/voice/macos.rsindex 0a39da2..6ae9521 100644--- a/apps/desktop/src-tauri/src/voice/macos.rs+++ b/apps/desktop/src-tauri/src/voice/macos.rs@@ -1,6 +1,12 @@//! The macOS voice backends: the native microphone (AVAudioEngine's input node, resampled to-//! 16 kHz mono) and Apple's Speech framework (`SFSpeechRecognizer`, on-device when the locale-//! supports it, else Apple's servers). Nothing here decides policy; `machine` does.+//! 16 kHz mono, with each tap buffer passed along as it came) and Apple's Speech framework+//! (`SFSpeechRecognizer`, on-device when the locale supports it, else Apple's servers). Nothing+//! here decides policy; `machine` does.+//!+//! Apple Speech is fed the input node's own buffers (S30.6), as Apple's documented pattern+//! does: with our 16 kHz box-filtered stream it misheard speech that Parakeet decoded from the+//! same capture (L101-L102). The 16 kHz path in `AppleRecognition::feed` stays only for a+//! microphone without device buffers.//!//! Threads: permission handlers, the audio tap, the interruption notification, and recognition//! results all arrive off the main thread (the recognizer gets an operation queue of its own),@@ -8,7 +14,7 @@//! and from those callbacks; AVAudioEngine and the Speech request are not tied to a thread.use super::audio::{self, Resampler, SAMPLE_RATE};-use super::{Authorization, Microphone, Recognition, Recognizer, Sink, Stream, VoiceError};+use super::{Authorization, Microphone, NativeAudio, Recognition, Recognizer, Sink, Stream, VoiceError};use block2::RcBlock;use objc2::rc::Retained;use objc2::runtime::{AnyObject, Bool, ProtocolObject};@@ -104,9 +110,8 @@ impl Microphone for MacMicrophone {.map(|c| unsafe { std::slice::from_raw_parts((*data.add(c)).as_ptr(), frames) }).collect();let pcm = resampler.borrow_mut().process(&audio::mono(&planes));- if !pcm.is_empty() {- audio_sink.audio(&pcm);- }+ // The buffer itself goes along for Apple Speech, which appends it as captured.+ audio_sink.audio_native(&pcm, buffer as &NativeAudio);});let tap_block = &*tap as *const block2::DynBlock<_> as *mut block2::DynBlock<_>;// SAFETY: the block matches AVAudioNodeTapBlock and stays alive in the stream.@@ -278,6 +283,19 @@ impl Recognition for AppleRecognition {}}+ fn wants_native(&self) -> bool {+ true+ }++ /// The input node's tap buffer, in its own rate and channel layout. The request retains+ /// what it needs, as with Apple's `request.append(buffer)` in the tap.+ fn feed_native(&mut self, audio: &NativeAudio) {+ if let Some(buffer) = audio.downcast_ref::<AVAudioPCMBuffer>() {+ // SAFETY: a valid buffer from the engine's tap, used within the tap call.+ unsafe { self.request.appendAudioPCMBuffer(buffer) };+ }+ }+fn finish(&mut self) {// SAFETY: ending the audio of a live request; the final result follows on the queue.unsafe { self.request.endAudio() };diff --git a/apps/desktop/src-tauri/src/voice/mod.rs b/apps/desktop/src-tauri/src/voice/mod.rsindex 3836358..44a05ed 100644--- a/apps/desktop/src-tauri/src/voice/mod.rs+++ b/apps/desktop/src-tauri/src/voice/mod.rs@@ -2,7 +2,9 @@//! user's message. Two seams keep the backends swappable://!//! - `Microphone` captures audio and delivers 16 kHz mono f32 PCM (`audio::SAMPLE_RATE`), the-//! format offline models such as sherpa-onnx expect, so any recognizer can consume it.+//! format offline models such as sherpa-onnx expect, so any recognizer can consume it. When+//! it can, it also passes the device's own buffers along (`NativeAudio`), for a recognizer+//! that does better with the audio as captured (Apple Speech, S30.6).//! - `Recognizer` turns that PCM into partial and final text. Apple's Speech framework//! (`macos::AppleSpeech`) is the default on macOS; the optional local model (S30.5,//! `parakeet::Parakeet`, NVIDIA Parakeet through sherpa-onnx) is preferred once the user has@@ -164,8 +166,14 @@ impl From<VoiceError> for String {}}-/// Native audio capture. Delivers `audio::SAMPLE_RATE` mono f32 chunks to `Sink::audio` (on-/// the audio thread) and reports interruptions with `Sink::failed`.+/// The capture in the device's own format (on macOS, the input node's `AVAudioPCMBuffer`),+/// valid for one call. Opaque to the machine: only the recognition that asks for it downcasts+/// it, so it must come from the microphone that recognition was made for.+pub type NativeAudio = dyn std::any::Any;++/// Native audio capture. Delivers `audio::SAMPLE_RATE` mono f32 chunks to `Sink::audio`, or+/// to `Sink::audio_native` along with the device's own buffer (on the audio thread), and+/// reports interruptions with `Sink::failed`.pub trait Microphone: Send + Sync {/// The current permission, without prompting.fn authorization(&self) -> Authorization;@@ -202,6 +210,14 @@ pub trait Recognizer: Send + Sync {pub trait Recognition: Send {/// More audio. Runs on the audio thread, so it must not block (heavy decoders queue it).fn feed(&mut self, samples: &[f32]);+ /// Whether this recognition takes the device's own buffers (`feed_native`) instead of the+ /// 16 kHz stream, when the microphone provides them.+ fn wants_native(&self) -> bool {+ false+ }+ /// The device's own buffer, instead of `feed`; only called when `wants_native`. Runs on+ /// the audio thread.+ fn feed_native(&mut self, _audio: &NativeAudio) {}/// No more audio: the final text follows through `Sink::finished`.fn finish(&mut self);/// Discards the recognition.