A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.
TypeScript
feat(desktop): recognize speech natively on macOS
Alan Buscaglia93ee6372fbbc
diff --git a/apps/desktop/src-tauri/Cargo.lock b/apps/desktop/src-tauri/Cargo.lockindex f8d1ac0..e76731a 100644--- a/apps/desktop/src-tauri/Cargo.lock+++ b/apps/desktop/src-tauri/Cargo.lock@@ -1283,10 +1283,13 @@ dependencies = ["objc2","objc2-app-kit","objc2-application-services",+ "objc2-av-foundation",+ "objc2-avf-audio","objc2-core-foundation","objc2-core-graphics","objc2-foundation","objc2-screen-capture-kit",+ "objc2-speech","serde","serde_json","tauri",@@ -2329,6 +2332,29 @@ dependencies = ["objc2-core-foundation",]+[[package]]+name = "objc2-av-foundation"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0"+dependencies = [+ "bitflags 2.13.2",+ "block2",+ "objc2",+ "objc2-foundation",+]++[[package]]+name = "objc2-avf-audio"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be"+dependencies = [+ "block2",+ "objc2",+ "objc2-foundation",+]+[[package]]name = "objc2-cloud-kit"version = "0.3.2"@@ -2495,6 +2521,18 @@ dependencies = ["objc2-security",]+[[package]]+name = "objc2-speech"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f9eb609f9d2a25e0f8b76954f3acaa58348ce2a91285fe867643b2224ac4d263"+dependencies = [+ "block2",+ "objc2",+ "objc2-avf-audio",+ "objc2-foundation",+]+[[package]]name = "objc2-ui-kit"version = "0.3.2"diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.tomlindex 87ea1e0..f4d20d1 100644--- a/apps/desktop/src-tauri/Cargo.toml+++ b/apps/desktop/src-tauri/Cargo.toml@@ -31,12 +31,16 @@ image = { version = "0.25", default-features = false, features = ["jpeg"] }[target.'cfg(target_os = "macos")'.dependencies]objc2 = "0.6"block2 = "0.6"-objc2-foundation = { version = "0.3", default-features = false, features = ["std", "NSArray", "NSDictionary", "NSError", "NSString", "NSValue"] }+objc2-foundation = { version = "0.3", default-features = false, features = ["std", "block2", "NSArray", "NSDictionary", "NSError", "NSLocale", "NSNotification", "NSOperation", "NSSet", "NSString", "NSValue"] }objc2-core-foundation = { version = "0.3", default-features = false, features = ["std", "CFArray", "CFCGTypes", "CFData", "CFDictionary", "CFNumber", "CFString"] }objc2-core-graphics = { version = "0.3", default-features = false, features = ["std", "CGDataProvider", "CGDirectDisplay", "CGError", "CGEvent", "CGEventTypes", "CGImage", "CGRemoteOperation", "CGWindow"] }objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "libc", "NSRunningApplication", "NSWorkspace"] }objc2-application-services = { version = "0.3", default-features = false, features = ["std", "libc", "AXError", "AXUIElement", "AXValue", "HIServices"] }objc2-screen-capture-kit = { version = "0.3", default-features = false, features = ["std", "block2", "libc", "objc2-core-foundation", "objc2-core-graphics", "SCScreenshotManager", "SCShareableContent", "SCStream"] }+# Voice input (S30.1): the Speech framework, the audio engine, and the microphone permission.+objc2-speech = { version = "0.3", default-features = false, features = ["std", "block2", "objc2-avf-audio", "SFSpeechRecognizer", "SFSpeechRecognitionRequest", "SFSpeechRecognitionResult", "SFSpeechRecognitionTask", "SFTranscription"] }+objc2-avf-audio = { version = "0.3", default-features = false, features = ["std", "block2", "AVAudioBuffer", "AVAudioEngine", "AVAudioFormat", "AVAudioIONode", "AVAudioNode", "AVAudioTime", "AVAudioTypes"] }+objc2-av-foundation = { version = "0.3", default-features = false, features = ["std", "block2", "AVCaptureDevice", "AVMediaFormat"] }# Debug builds encode screenshots fast too (S24.8): `image` holds the JPEG encoder and `zune-*`# decode JPEGs (tests). The encoder is generic, so most of its code is compiled in this crate,diff --git a/apps/desktop/src-tauri/Info.plist b/apps/desktop/src-tauri/Info.plistindex d4b6455..4550482 100644--- a/apps/desktop/src-tauri/Info.plist+++ b/apps/desktop/src-tauri/Info.plist@@ -5,5 +5,10 @@<!-- Accessory app: no Dock icon, even before the activation policy is applied. --><key>LSUIElement</key><true/>+ <!-- Voice input (S30.1): shown by macOS the first time the mic button is used. -->+ <key>NSMicrophoneUsageDescription</key>+ <string>Gentle Dot listens to your voice when you press the microphone button.</string>+ <key>NSSpeechRecognitionUsageDescription</key>+ <string>Gentle Dot turns what you say into a message when you press the microphone button.</string></dict></plist>diff --git a/apps/desktop/src-tauri/build.rs b/apps/desktop/src-tauri/build.rsindex 3c85eee..bc80693 100644--- a/apps/desktop/src-tauri/build.rs+++ b/apps/desktop/src-tauri/build.rs@@ -16,6 +16,10 @@ const COMMANDS: &[&str] = &["computer_stop","computer_status","computer_set_yolo",+ "voice_status",+ "voice_start",+ "voice_stop",+ "voice_cancel",];fn find_in_path(program: &str, path: &str) -> Option<PathBuf> {diff --git a/apps/desktop/src-tauri/capabilities/default.json b/apps/desktop/src-tauri/capabilities/default.jsonindex 3f09df0..7c49a7f 100644--- a/apps/desktop/src-tauri/capabilities/default.json+++ b/apps/desktop/src-tauri/capabilities/default.json@@ -25,6 +25,17 @@"allow-computer-set-yolo"]},+ {+ "identifier": "voice",+ "description": "Voice input (S30.1): the panel's mic button records and transcribes natively; the Dot never records.",+ "windows": ["panel"],+ "permissions": [+ "allow-voice-status",+ "allow-voice-start",+ "allow-voice-stop",+ "allow-voice-cancel"+ ]+ },{"identifier": "glow","description": "What the computer-control glow overlay (S28) may call: listening to the glow and session events, nothing else.",diff --git a/apps/desktop/src-tauri/permissions/autogenerated/voice_cancel.toml b/apps/desktop/src-tauri/permissions/autogenerated/voice_cancel.tomlnew file mode 100644index 0000000..f670fba--- /dev/null+++ b/apps/desktop/src-tauri/permissions/autogenerated/voice_cancel.toml@@ -0,0 +1,11 @@+# Automatically generated - DO NOT EDIT!++[[permission]]+identifier = "allow-voice-cancel"+description = "Enables the voice_cancel command without any pre-configured scope."+commands.allow = ["voice_cancel"]++[[permission]]+identifier = "deny-voice-cancel"+description = "Denies the voice_cancel command without any pre-configured scope."+commands.deny = ["voice_cancel"]diff --git a/apps/desktop/src-tauri/permissions/autogenerated/voice_start.toml b/apps/desktop/src-tauri/permissions/autogenerated/voice_start.tomlnew file mode 100644index 0000000..8ef4ff7--- /dev/null+++ b/apps/desktop/src-tauri/permissions/autogenerated/voice_start.toml@@ -0,0 +1,11 @@+# Automatically generated - DO NOT EDIT!++[[permission]]+identifier = "allow-voice-start"+description = "Enables the voice_start command without any pre-configured scope."+commands.allow = ["voice_start"]++[[permission]]+identifier = "deny-voice-start"+description = "Denies the voice_start command without any pre-configured scope."+commands.deny = ["voice_start"]diff --git a/apps/desktop/src-tauri/permissions/autogenerated/voice_status.toml b/apps/desktop/src-tauri/permissions/autogenerated/voice_status.tomlnew file mode 100644index 0000000..ee57f12--- /dev/null+++ b/apps/desktop/src-tauri/permissions/autogenerated/voice_status.toml@@ -0,0 +1,11 @@+# Automatically generated - DO NOT EDIT!++[[permission]]+identifier = "allow-voice-status"+description = "Enables the voice_status command without any pre-configured scope."+commands.allow = ["voice_status"]++[[permission]]+identifier = "deny-voice-status"+description = "Denies the voice_status command without any pre-configured scope."+commands.deny = ["voice_status"]diff --git a/apps/desktop/src-tauri/permissions/autogenerated/voice_stop.toml b/apps/desktop/src-tauri/permissions/autogenerated/voice_stop.tomlnew file mode 100644index 0000000..b2daaf8--- /dev/null+++ b/apps/desktop/src-tauri/permissions/autogenerated/voice_stop.toml@@ -0,0 +1,11 @@+# Automatically generated - DO NOT EDIT!++[[permission]]+identifier = "allow-voice-stop"+description = "Enables the voice_stop command without any pre-configured scope."+commands.allow = ["voice_stop"]++[[permission]]+identifier = "deny-voice-stop"+description = "Denies the voice_stop command without any pre-configured scope."+commands.deny = ["voice_stop"]diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rsindex 4625c68..37d3440 100644--- a/apps/desktop/src-tauri/src/lib.rs+++ b/apps/desktop/src-tauri/src/lib.rs@@ -9,5 +9,6 @@ pub mod platform;pub mod position;mod shell;pub mod status;+pub mod voice;pub use shell::run;diff --git a/apps/desktop/src-tauri/src/shell.rs b/apps/desktop/src-tauri/src/shell.rsindex ab86660..deb7179 100644--- a/apps/desktop/src-tauri/src/shell.rs+++ b/apps/desktop/src-tauri/src/shell.rs@@ -1,5 +1,5 @@//! Tauri wiring: windows, commands, tray menu, global shortcut, Dot snapping,-//! hiding the rose, the full-screen panel, and the daemon lifecycle (design §9).+//! hiding the rose, the full-screen panel, voice input, and the daemon lifecycle (design §9).use crate::computer::app as computer;use crate::computer::session::Reason;@@ -9,6 +9,7 @@ use crate::geometry::{self, Rect};use crate::platform::{self, LaunchRequest, Os, DOT_TITLE, PANEL_TITLE};use crate::position::{self, DotPosition};use crate::status::{is_template, tray_status, TrayGlyph};+use crate::voice::app as voice;use std::path::PathBuf;use std::sync::atomic::{AtomicBool, Ordering};use std::sync::{mpsc, Arc, Mutex};@@ -614,7 +615,11 @@ pub fn run() {computer::computer_request_permission,computer::computer_stop,computer::computer_status,- computer::computer_set_yolo+ computer::computer_set_yolo,+ voice::voice_status,+ voice::voice_start,+ voice::voice_stop,+ voice::voice_cancel]).setup(move |app| {#[cfg(target_os = "macos")]@@ -640,6 +645,7 @@ pub fn run() {full_screen: Mutex::new(None),});app.manage(computer::start(&handle));+ app.manage(voice::start());#[cfg(target_os = "macos")]register_panic_shortcut(&handle);diff --git a/apps/desktop/src-tauri/src/voice/app.rs b/apps/desktop/src-tauri/src/voice/app.rsnew file mode 100644index 0000000..3e03f33--- /dev/null+++ b/apps/desktop/src-tauri/src/voice/app.rs@@ -0,0 +1,138 @@+//! The app side of voice input (S30.1, L88): the `voice_*` commands and the `voice://*` events,+//! sent only to the webview that started the recording. Off macOS `voice_status` answers+//! "unavailable" and the other commands fail. Every command runs on a blocking thread: starting+//! may wait on a permission prompt and stopping waits for the final transcript.++use super::{Events, Transcript, Voice, VoiceError, VoiceEvent, VoiceStatus, UNSUPPORTED};+use std::sync::Arc;+use tauri::{AppHandle, Emitter, EventTarget, Manager};++/// The commands, as registered in `build.rs` and granted to the panel.+pub const COMMANDS: [&str; 4] = ["voice_status", "voice_start", "voice_stop", "voice_cancel"];++/// Voice input, when this platform has a backend.+pub struct VoiceInput {+ voice: Option<Voice>,+}++impl VoiceInput {+ pub fn new(voice: Voice) -> Self {+ VoiceInput { voice: Some(voice) }+ }++ pub fn unsupported() -> Self {+ VoiceInput { voice: None }+ }++ fn voice(&self) -> Result<&Voice, VoiceError> {+ self.voice.as_ref().ok_or(VoiceError::Unsupported)+ }++ pub fn status(&self) -> VoiceStatus {+ self.voice().map_or_else(|_| VoiceStatus::unavailable(UNSUPPORTED), Voice::status)+ }++ pub fn start(&self, locale: Option<&str>, events: Events) -> Result<(), VoiceError> {+ self.voice()?.start(locale, events)+ }++ pub fn stop(&self) -> Result<Transcript, VoiceError> {+ self.voice()?.stop()+ }++ pub fn cancel(&self) -> Result<(), VoiceError> {+ self.voice()?.cancel()+ }+}++/// Apple's Speech framework over the native microphone on macOS; nothing elsewhere.+pub fn start() -> VoiceInput {+ #[cfg(target_os = "macos")]+ return VoiceInput::new(Voice::new(+ Arc::new(super::MacMicrophone),+ Arc::new(super::AppleSpeech),+ Arc::new(crate::computer::SystemClock),+ ));+ #[cfg(not(target_os = "macos"))]+ VoiceInput::unsupported()+}++async fn blocking<T: Send + 'static>(+ app: AppHandle,+ action: impl FnOnce(&VoiceInput) -> Result<T, VoiceError> + Send + 'static,+) -> Result<T, String> {+ tauri::async_runtime::spawn_blocking(move || action(&app.state::<VoiceInput>()))+ .await+ .map_err(|error| error.to_string())?+ .map_err(String::from)+}++#[tauri::command]+pub async fn voice_status(app: AppHandle) -> Result<VoiceStatus, String> {+ blocking(app, |input| Ok(input.status())).await+}++#[tauri::command]+pub async fn voice_start(app: AppHandle, webview: tauri::WebviewWindow, locale: Option<String>) -> Result<(), String> {+ let target = EventTarget::webview_window(webview.label());+ let emitter = app.clone();+ let events: Events = Arc::new(move |event: VoiceEvent| {+ if let Err(error) = emitter.emit_to(target.clone(), event.name(), event.payload()) {+ eprintln!("gentle-dot: cannot send a voice event: {error}");+ }+ });+ blocking(app, move |input| input.start(locale.as_deref(), events)).await+}++#[tauri::command]+pub async fn voice_stop(app: AppHandle) -> Result<Transcript, String> {+ blocking(app, VoiceInput::stop).await+}++#[tauri::command]+pub async fn voice_cancel(app: AppHandle) -> Result<(), String> {+ blocking(app, VoiceInput::cancel).await+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::voice::{VoiceError, VoiceStatus, UNSUPPORTED};+ use std::sync::Arc;++ #[test]+ fn without_a_backend_voice_is_unavailable_and_every_action_errors() {+ let input = VoiceInput::unsupported();+ assert_eq!(input.status(), VoiceStatus::unavailable(UNSUPPORTED));+ assert_eq!(input.start(None, Arc::new(|_| {})), Err(VoiceError::Unsupported));+ assert_eq!(input.stop(), Err(VoiceError::Unsupported));+ assert_eq!(input.cancel(), Err(VoiceError::Unsupported));+ assert_eq!(String::from(VoiceError::Unsupported), "Voice input is available on macOS.");+ }++ #[test]+ fn the_command_names_match_the_l88_contract() {+ assert_eq!(COMMANDS, ["voice_status", "voice_start", "voice_stop", "voice_cancel"]);+ }++ #[test]+ fn the_commands_are_registered_and_granted_to_the_panel_only() {+ let build = include_str!("../../build.rs");+ let capabilities: serde_json::Value = serde_json::from_str(include_str!("../../capabilities/default.json")).unwrap();+ let granted_to = |permission: &str| -> Vec<&str> {+ let all = capabilities["capabilities"].as_array().unwrap();+ let mut windows: Vec<&str> = all+ .iter()+ .filter(|c| c["permissions"].as_array().unwrap().iter().any(|p| p == permission))+ .flat_map(|c| c["windows"].as_array().unwrap().iter().map(|w| w.as_str().unwrap()))+ .collect();+ windows.sort();+ windows+ };+ for command in COMMANDS {+ assert!(build.contains(&format!("\"{command}\",")), "{command} is missing from build.rs");+ let permission = format!("allow-{}", command.replace('_', "-"));+ assert_eq!(granted_to(&permission), vec!["panel"], "{permission}");+ }+ }+}diff --git a/apps/desktop/src-tauri/src/voice/audio.rs b/apps/desktop/src-tauri/src/voice/audio.rsnew file mode 100644index 0000000..843d833--- /dev/null+++ b/apps/desktop/src-tauri/src/voice/audio.rs@@ -0,0 +1,127 @@+//! PCM helpers shared by every microphone and recognizer: the 16 kHz mono contract, the+//! level meter, and a streaming resampler from the device rate.++/// What every `Microphone` delivers and every `Recognizer` consumes: 16 kHz mono f32.+pub const SAMPLE_RATE: u32 = 16_000;+/// The meter's floor: quieter input shows as silence.+const FLOOR_DB: f32 = -50.0;++/// Root mean square of a chunk.+pub fn rms(samples: &[f32]) -> f32 {+ if samples.is_empty() {+ return 0.0;+ }+ (samples.iter().map(|s| s * s).sum::<f32>() / samples.len() as f32).sqrt()+}++/// Maps an RMS to the meter's 0..1 on a decibel scale: -50 dBFS and below is 0, 0 dBFS is 1.+/// Speech sits around -40 to -10 dBFS, so a linear RMS would barely move the meter.+pub fn level(rms: f32) -> f32 {+ if rms <= 0.0 {+ return 0.0;+ }+ ((20.0 * rms.log10() - FLOOR_DB) / -FLOOR_DB).clamp(0.0, 1.0)+}++/// Averages the device's channels (non-interleaved) into one.+pub fn mono(channels: &[&[f32]]) -> Vec<f32> {+ let Some(first) = channels.first() else { return Vec::new() };+ if channels.len() == 1 {+ return first.to_vec();+ }+ let scale = 1.0 / channels.len() as f32;+ (0..first.len()).map(|i| channels.iter().map(|c| c.get(i).copied().unwrap_or(0.0)).sum::<f32>() * scale).collect()+}++/// Converts a mono stream from the device rate to `SAMPLE_RATE`, chunk by chunk. Downsampling+/// averages the input that falls in each output period (a box filter, enough to keep speech+/// free of audible aliasing); lower rates are held.+pub struct Resampler {+ /// Input samples per output sample.+ step: f64,+ /// Input consumed, and where the current output period ends.+ position: f64,+ edge: f64,+ sum: f32,+ count: u32,+ last: f32,+}++impl Resampler {+ pub fn new(from_rate: f64) -> Self {+ let step = from_rate / f64::from(SAMPLE_RATE);+ Resampler { step, position: 0.0, edge: step, sum: 0.0, count: 0, last: 0.0 }+ }++ pub fn process(&mut self, input: &[f32]) -> Vec<f32> {+ let mut out = Vec::with_capacity((input.len() as f64 / self.step) as usize + 1);+ for &sample in input {+ self.sum += sample;+ self.count += 1;+ self.position += 1.0;+ while self.position >= self.edge - 1e-9 {+ if self.count > 0 {+ self.last = self.sum / self.count as f32;+ self.sum = 0.0;+ self.count = 0;+ }+ out.push(self.last);+ self.edge += self.step;+ }+ }+ out+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn rms_of_a_constant_signal_is_its_magnitude() {+ assert_eq!(rms(&[]), 0.0);+ assert!((rms(&[0.5, -0.5, 0.5, -0.5]) - 0.5).abs() < 1e-6);+ }++ #[test]+ fn the_level_maps_decibels_to_zero_through_one() {+ assert_eq!(level(0.0), 0.0);+ assert_eq!(level(0.001), 0.0, "-60 dBFS is silence on the meter");+ assert_eq!(level(1.0), 1.0);+ assert_eq!(level(2.0), 1.0);+ let (quiet, loud) = (level(0.01), level(0.1));+ assert!(0.0 < quiet && quiet < loud && loud < 1.0);+ }++ #[test]+ fn channels_mix_down_to_mono() {+ assert_eq!(mono(&[&[1.0, 0.0], &[0.0, 0.0]]), vec![0.5, 0.0]);+ assert_eq!(mono(&[&[0.25, 0.75]]), vec![0.25, 0.75]);+ assert!(mono(&[]).is_empty());+ }++ #[test]+ fn resampling_48k_to_16k_keeps_a_third_of_the_samples_and_the_level() {+ let mut resampler = Resampler::new(48_000.0);+ let out = resampler.process(&[0.3; 4_800]);+ assert_eq!(out.len(), 1_600);+ assert!(out.iter().all(|s| (s - 0.3).abs() < 1e-6));+ }++ #[test]+ fn resampling_is_the_same_in_chunks_as_in_one_go() {+ let input: Vec<f32> = (0..4_410).map(|i| (i as f32 * 0.01).sin()).collect();+ let whole = Resampler::new(44_100.0).process(&input);+ let mut chunked = Resampler::new(44_100.0);+ let pieces: Vec<f32> = input.chunks(512).flat_map(|chunk| chunked.process(chunk)).collect();+ assert_eq!(whole.len(), 1_600);+ assert_eq!(pieces, whole);+ }++ #[test]+ fn resampling_at_16k_is_the_identity_and_lower_rates_are_held() {+ let input = [0.1, 0.2, 0.3];+ assert_eq!(Resampler::new(16_000.0).process(&input), input.to_vec());+ assert_eq!(Resampler::new(8_000.0).process(&[0.1, 0.2]), vec![0.1, 0.1, 0.2, 0.2]);+ }+}diff --git a/apps/desktop/src-tauri/src/voice/fake.rs b/apps/desktop/src-tauri/src/voice/fake.rsnew file mode 100644index 0000000..a218f2a--- /dev/null+++ b/apps/desktop/src-tauri/src/voice/fake.rs@@ -0,0 +1,247 @@+//! Fakes for the voice tests: a microphone and a recognizer that record what the machine asks+//! of them and let a test play the platform's side (audio, results, failures).++use super::{Authorization, Events, Microphone, Recognition, Recognizer, Sink, Stream, VoiceError, VoiceEvent};+use crate::computer::Clock;+use std::sync::atomic::{AtomicU64, Ordering};+use std::sync::{Arc, Condvar, Mutex};++/// Blocks a permission prompt until the test answers it.+#[derive(Default)]+pub struct Gate {+ open: Mutex<bool>,+ changed: Condvar,+}++impl Gate {+ pub fn release(&self) {+ *self.open.lock().unwrap() = true;+ self.changed.notify_all();+ }++ fn wait(&self) {+ let mut open = self.open.lock().unwrap();+ while !*open {+ open = self.changed.wait(open).unwrap();+ }+ }+}++#[derive(Default)]+pub struct MicLog {+ pub requests: usize,+ pub opened: usize,+ pub closed: usize,+ pub sink: Option<Sink>,+}++pub struct FakeMicrophone {+ pub authorization: Mutex<Authorization>,+ /// What a prompt answers.+ pub answer: Mutex<Authorization>,+ pub gate: Option<Arc<Gate>>,+ pub open_error: Mutex<Option<VoiceError>>,+ pub log: Arc<Mutex<MicLog>>,+}++impl FakeMicrophone {+ pub fn granted() -> Self {+ FakeMicrophone {+ authorization: Mutex::new(Authorization::Authorized),+ answer: Mutex::new(Authorization::Authorized),+ gate: None,+ open_error: Mutex::new(None),+ log: Arc::default(),+ }+ }++ pub fn with(authorization: Authorization, answer: Authorization) -> Self {+ let mic = FakeMicrophone::granted();+ *mic.authorization.lock().unwrap() = authorization;+ *mic.answer.lock().unwrap() = answer;+ mic+ }++ /// The sink of the latest capture, to play the audio thread.+ pub fn sink(&self) -> Sink {+ self.log.lock().unwrap().sink.clone().expect("the microphone was opened")+ }+}++impl Microphone for FakeMicrophone {+ fn authorization(&self) -> Authorization {+ *self.authorization.lock().unwrap()+ }++ fn request_authorization(&self) -> Authorization {+ self.log.lock().unwrap().requests += 1;+ if let Some(gate) = &self.gate {+ gate.wait();+ }+ let answer = *self.answer.lock().unwrap();+ *self.authorization.lock().unwrap() = answer;+ answer+ }++ fn open(&self, sink: Sink) -> Result<Box<dyn Stream>, VoiceError> {+ if let Some(error) = self.open_error.lock().unwrap().clone() {+ return Err(error);+ }+ let mut log = self.log.lock().unwrap();+ log.opened += 1;+ log.sink = Some(sink);+ Ok(Box::new(FakeStream { log: self.log.clone() }))+ }+}++struct FakeStream {+ log: Arc<Mutex<MicLog>>,+}++impl Stream for FakeStream {+ fn close(&mut self) {+ self.log.lock().unwrap().closed += 1;+ }+}++#[derive(Default)]+pub struct RecognitionLog {+ pub requests: usize,+ pub locales: Vec<String>,+ pub fed: Vec<f32>,+ pub finished: usize,+ pub cancelled: usize,+ pub sink: Option<Sink>,+}++pub struct FakeRecognizer {+ pub authorization: Mutex<Authorization>,+ pub answer: Mutex<Authorization>,+ pub current: String,+ pub supported: Vec<String>,+ /// Delivered through the sink when the recognition is finished; none plays a recognizer+ /// that never answers.+ pub final_text: Mutex<Option<String>>,+ /// Reported as a failure when the recognition is finished.+ pub fail_on_finish: Mutex<Option<String>>,+ pub log: Arc<Mutex<RecognitionLog>>,+}++impl FakeRecognizer {+ pub fn granted() -> Self {+ FakeRecognizer {+ authorization: Mutex::new(Authorization::Authorized),+ answer: Mutex::new(Authorization::Authorized),+ current: "en_US".into(),+ supported: vec!["en-US".into(), "es-ES".into(), "es-MX".into()],+ final_text: Mutex::new(None),+ fail_on_finish: Mutex::new(None),+ log: Arc::default(),+ }+ }++ pub fn with(authorization: Authorization, answer: Authorization) -> Self {+ let recognizer = FakeRecognizer::granted();+ *recognizer.authorization.lock().unwrap() = authorization;+ *recognizer.answer.lock().unwrap() = answer;+ recognizer+ }++ pub fn sink(&self) -> Sink {+ self.log.lock().unwrap().sink.clone().expect("a recognition began")+ }+}++impl Recognizer for FakeRecognizer {+ fn authorization(&self) -> Authorization {+ *self.authorization.lock().unwrap()+ }++ fn request_authorization(&self) -> Authorization {+ self.log.lock().unwrap().requests += 1;+ let answer = *self.answer.lock().unwrap();+ *self.authorization.lock().unwrap() = answer;+ answer+ }++ fn current_locale(&self) -> String {+ self.current.clone()+ }++ fn supported_locales(&self) -> Vec<String> {+ self.supported.clone()+ }++ fn begin(&self, locale: &str, sink: Sink) -> Result<Box<dyn Recognition>, VoiceError> {+ let mut log = self.log.lock().unwrap();+ log.locales.push(locale.into());+ log.sink = Some(sink.clone());+ let final_text = self.final_text.lock().unwrap().clone();+ let failure = self.fail_on_finish.lock().unwrap().clone();+ Ok(Box::new(FakeRecognition { log: self.log.clone(), sink, final_text, failure }))+ }+}++struct FakeRecognition {+ log: Arc<Mutex<RecognitionLog>>,+ sink: Sink,+ final_text: Option<String>,+ failure: Option<String>,+}++impl Recognition for FakeRecognition {+ fn feed(&mut self, samples: &[f32]) {+ self.log.lock().unwrap().fed.extend_from_slice(samples);+ }++ fn finish(&mut self) {+ self.log.lock().unwrap().finished += 1;+ if let Some(text) = &self.final_text {+ self.sink.finished(text);+ }+ if let Some(message) = &self.failure {+ self.sink.failed(message);+ }+ }++ fn cancel(&mut self) {+ self.log.lock().unwrap().cancelled += 1;+ }+}++/// Collects the events a webview would receive.+#[derive(Default, Clone)]+pub struct Recorder(pub Arc<Mutex<Vec<VoiceEvent>>>);++impl Recorder {+ pub fn events(&self) -> Events {+ let seen = self.0.clone();+ Arc::new(move |event| seen.lock().unwrap().push(event))+ }++ pub fn seen(&self) -> Vec<VoiceEvent> {+ self.0.lock().unwrap().clone()+ }++ pub fn levels(&self) -> usize {+ self.seen().iter().filter(|event| matches!(event, VoiceEvent::Level(_))).count()+ }+}++pub struct FakeClock(pub AtomicU64);++impl FakeClock {+ pub fn advance(&self, ms: u64) {+ self.0.fetch_add(ms, Ordering::SeqCst);+ }+}++impl Clock for FakeClock {+ fn now_ms(&self) -> u64 {+ self.0.load(Ordering::SeqCst)+ }++ fn sleep_ms(&self, ms: u64) {+ self.advance(ms);+ }+}diff --git a/apps/desktop/src-tauri/src/voice/locale.rs b/apps/desktop/src-tauri/src/voice/locale.rsnew file mode 100644index 0000000..4915dfc--- /dev/null+++ b/apps/desktop/src-tauri/src/voice/locale.rs@@ -0,0 +1,116 @@+//! Which locale the recognizer gets: the requested one, else a variant of its language, else+//! the user's locale (or a variant of its language). The UI may pass a browser tag such as+//! `es-419`, and the system may report `es_AR`, which Speech does not list.++use super::VoiceError;++/// The variant used when a language's own region is not supported.+const MAIN_VARIANTS: &[(&str, &str)] = &[("en", "en-US"), ("pt", "pt-BR"), ("zh", "zh-CN"), ("ar", "ar-SA")];++/// `es_AR`, `en_US@rg=gbzzzz`, or ` pt-br ` → `es-AR`, `en-US`, `pt-BR`.+pub fn normalize(id: &str) -> String {+ let id = id.trim();+ let id = id.split('@').next().unwrap_or_default();+ let mut parts = id.split(['_', '-']).filter(|part| !part.is_empty());+ let Some(language) = parts.next() else { return String::new() };+ let mut out = language.to_ascii_lowercase();+ for part in parts {+ out.push('-');+ match part.len() {+ 2 | 3 if part.chars().all(|c| c.is_ascii_alphabetic()) => out.push_str(&part.to_ascii_uppercase()),+ 4 => {+ let mut chars = part.chars();+ out.extend(chars.next().map(|c| c.to_ascii_uppercase()));+ out.push_str(&chars.as_str().to_ascii_lowercase());+ }+ _ => out.push_str(part),+ }+ }+ out+}++fn language(tag: &str) -> &str {+ tag.split('-').next().unwrap_or_default()+}++/// The supported entry for `tag`, as the recognizer spells it.+fn pick(tag: &str, current: &str, supported: &[(String, &String)]) -> Option<String> {+ let find = |wanted: &str| supported.iter().find(|(normal, _)| normal == wanted).map(|(_, id)| (*id).clone());+ if tag.is_empty() {+ return None;+ }+ if let Some(found) = find(tag) {+ return Some(found);+ }+ let lang = language(tag);+ if language(current) == lang {+ if let Some(found) = find(current) {+ return Some(found);+ }+ }+ let main = MAIN_VARIANTS.iter().find(|(l, _)| *l == lang).map(|(_, v)| v.to_string());+ let main = main.unwrap_or_else(|| format!("{lang}-{}", lang.to_ascii_uppercase()));+ if let Some(found) = find(&main) {+ return Some(found);+ }+ let mut same: Vec<_> = supported.iter().filter(|(normal, _)| language(normal) == lang).collect();+ same.sort();+ same.first().map(|(_, id)| (*id).clone())+}++pub fn resolve(requested: Option<&str>, current: &str, supported: &[String]) -> Result<String, VoiceError> {+ let normalized: Vec<(String, &String)> = supported.iter().map(|id| (normalize(id), id)).collect();+ let requested = requested.map(normalize).unwrap_or_default();+ let current = normalize(current);+ pick(&requested, ¤t, &normalized)+ .or_else(|| pick(¤t, ¤t, &normalized))+ .ok_or(VoiceError::NoRecognizer(if requested.is_empty() { current } else { requested }))+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::voice::VoiceError;++ fn supported() -> Vec<String> {+ ["en-AU", "en-GB", "en-US", "es-CL", "es-ES", "es-MX", "pt-BR", "pt-PT"].map(String::from).to_vec()+ }++ #[test]+ fn identifiers_normalize_to_bcp47() {+ assert_eq!(normalize("es_AR"), "es-AR");+ assert_eq!(normalize("en_US@rg=gbzzzz"), "en-US");+ assert_eq!(normalize(" pt-br "), "pt-BR");+ assert_eq!(normalize("zh-Hant_TW"), "zh-Hant-TW");+ }++ #[test]+ fn an_exact_match_wins() {+ assert_eq!(resolve(Some("es_MX"), "en_US", &supported()), Ok("es-MX".into()));+ assert_eq!(resolve(None, "en_GB", &supported()), Ok("en-GB".into()));+ }++ #[test]+ fn a_missing_region_falls_back_to_the_main_variant_of_the_language() {+ assert_eq!(resolve(Some("es-AR"), "en_US", &supported()), Ok("es-ES".into()));+ assert_eq!(resolve(None, "en_AR", &supported()), Ok("en-US".into()));+ assert_eq!(resolve(Some("pt"), "en_US", &supported()), Ok("pt-BR".into()));+ }++ #[test]+ fn the_current_region_is_preferred_within_the_requested_language() {+ assert_eq!(resolve(Some("es"), "es_CL", &supported()), Ok("es-CL".into()));+ }++ #[test]+ fn an_unsupported_language_falls_back_to_the_current_locale() {+ assert_eq!(resolve(Some("ja-JP"), "es_MX", &supported()), Ok("es-MX".into()));+ assert_eq!(resolve(Some(""), "pt_PT", &supported()), Ok("pt-PT".into()));+ }++ #[test]+ fn nothing_supported_is_an_error_naming_the_locale() {+ assert_eq!(resolve(None, "ja_JP", &supported()), Err(VoiceError::NoRecognizer("ja-JP".into())));+ assert_eq!(resolve(Some("fr-FR"), "ja_JP", &[]), Err(VoiceError::NoRecognizer("fr-FR".into())));+ }+}diff --git a/apps/desktop/src-tauri/src/voice/machine.rs b/apps/desktop/src-tauri/src/voice/machine.rsnew file mode 100644index 0000000..1c3b8ec--- /dev/null+++ b/apps/desktop/src-tauri/src/voice/machine.rs@@ -0,0 +1,657 @@+//! The voice state machine: one recording at a time, between a `Microphone` and a `Recognizer`.+//!+//! Idle → Starting (permissions, locale, recognition, capture) → Recording → Stopping → Idle.+//! Cancel works in every state. Platform callbacks arrive on arbitrary threads through a+//! `Sink` bound to one recording, so results from an earlier recording are ignored. No+//! platform call or event is made while the state lock is held.++use super::{audio, locale, Authorization, Events, Microphone, Recognition, Recognizer, Stream, Transcript};+use super::{VoiceError, VoiceEvent, VoiceStatus};+use crate::computer::Clock;+use std::sync::{Arc, Condvar, Mutex, MutexGuard, Weak};+use std::time::{Duration, Instant};++/// About 15 levels a second.+pub const LEVEL_INTERVAL_MS: u64 = 1_000 / 15;+/// How long `stop` waits for the final transcript before using the latest partial.+pub const FINAL_TIMEOUT: Duration = Duration::from_secs(3);++pub struct Voice {+ inner: Arc<Inner>,+ final_timeout: Duration,+}++struct Inner {+ microphone: Arc<dyn Microphone>,+ recognizer: Arc<dyn Recognizer>,+ clock: Arc<dyn Clock>,+ state: Mutex<State>,+ changed: Condvar,+}++enum State {+ Idle { next: u64 },+ Starting { recording: u64, cancelled: bool, failure: Option<String> },+ Recording(Session),+ Stopping(Session),+}++struct Session {+ recording: u64,+ events: Events,+ stream: Option<Box<dyn Stream>>,+ feed: Arc<Mutex<Feed>>,+ transcript: String,+ final_text: Option<String>,+ last_level_ms: Option<u64>,+ /// The loudest chunk since the last level.+ peak: f32,+}++/// The recognition, closed once finished or cancelled so no audio follows `finish`.+struct Feed {+ recognition: Box<dyn Recognition>,+ open: bool,+}++impl Feed {+ fn push(&mut self, samples: &[f32]) {+ if self.open {+ self.recognition.feed(samples);+ }+ }++ fn finish(&mut self) {+ if self.open {+ self.open = false;+ self.recognition.finish();+ }+ }++ fn cancel(&mut self) {+ self.open = false;+ self.recognition.cancel();+ }+}++impl State {+ fn recording(&self) -> Option<u64> {+ match self {+ State::Idle { .. } => None,+ State::Starting { recording, .. } => Some(*recording),+ State::Recording(session) | State::Stopping(session) => Some(session.recording),+ }+ }++ fn idle(&self) -> State {+ State::Idle { next: self.recording().map_or(0, |n| n + 1) }+ }+}++/// Tears a session down outside the state lock.+fn discard(session: Session) {+ let Session { mut stream, feed, .. } = session;+ if let Some(stream) = stream.as_mut() {+ stream.close();+ }+ lock(&feed).cancel();+}++fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {+ mutex.lock().unwrap_or_else(|poisoned| poisoned.into_inner())+}++fn authorize(+ current: Authorization,+ request: impl FnOnce() -> Authorization,+ denied: VoiceError,+ restricted: VoiceError,+) -> Result<(), VoiceError> {+ let answer = if current == Authorization::NotDetermined { request() } else { current };+ match answer {+ Authorization::Authorized => Ok(()),+ Authorization::Restricted => Err(restricted),+ Authorization::Denied | Authorization::NotDetermined => Err(denied),+ }+}++/// The reason a permission blocks voice input, without prompting.+fn blocked(current: Authorization, denied: VoiceError, restricted: VoiceError) -> Option<VoiceError> {+ match current {+ Authorization::Denied => Some(denied),+ Authorization::Restricted => Some(restricted),+ Authorization::NotDetermined | Authorization::Authorized => None,+ }+}++impl Voice {+ pub fn new(microphone: Arc<dyn Microphone>, recognizer: Arc<dyn Recognizer>, clock: Arc<dyn Clock>) -> Self {+ let inner = Inner { microphone, recognizer, clock, state: Mutex::new(State::Idle { next: 0 }), changed: Condvar::new() };+ Voice { inner: Arc::new(inner), final_timeout: FINAL_TIMEOUT }+ }++ pub fn with_final_timeout(mut self, timeout: Duration) -> Self {+ self.final_timeout = timeout;+ self+ }++ /// Whether voice input can work, without prompting: a permission the user denied, or no+ /// recognizer for the user's locale, makes it unavailable. Unasked permissions are asked+ /// on the first `start`.+ pub fn status(&self) -> VoiceStatus {+ let (recognizer, microphone) = (&self.inner.recognizer, &self.inner.microphone);+ let reason = blocked(recognizer.authorization(), VoiceError::SpeechDenied, VoiceError::SpeechRestricted)+ .or_else(|| blocked(microphone.authorization(), VoiceError::MicrophoneDenied, VoiceError::MicrophoneRestricted))+ .or_else(|| locale::resolve(None, &recognizer.current_locale(), &recognizer.supported_locales()).err());+ match reason {+ Some(reason) => VoiceStatus::unavailable(reason.to_string()),+ None => VoiceStatus::available(),+ }+ }++ /// Starts recording, prompting for permissions first when they were never asked. Blocks+ /// while a prompt is open, so it runs off the main thread. `events` reach the caller.+ pub fn start(&self, requested: Option<&str>, events: Events) -> Result<(), VoiceError> {+ let recording = {+ let mut state = lock(&self.inner.state);+ let State::Idle { next } = *state else { return Err(VoiceError::Busy) };+ *state = State::Starting { recording: next, cancelled: false, failure: None };+ next+ };+ let sink = Sink { inner: Arc::downgrade(&self.inner), recording };+ let begun = self.prepare(requested, recording).and_then(|locale| {+ let mut recognition = self.inner.recognizer.begin(&locale, sink.clone())?;+ match self.inner.microphone.open(sink) {+ Ok(stream) => Ok((stream, recognition)),+ Err(error) => {+ recognition.cancel();+ Err(error)+ }+ }+ });+ let mut state = lock(&self.inner.state);+ let State::Starting { cancelled, failure, .. } = std::mem::replace(&mut *state, State::Idle { next: recording + 1 }) else {+ unreachable!("only start leaves Starting");+ };+ let (stream, recognition) = begun?;+ let feed = Arc::new(Mutex::new(Feed { recognition, open: true }));+ let session = Session {+ recording,+ events,+ stream: Some(stream),+ feed,+ transcript: String::new(),+ final_text: None,+ last_level_ms: None,+ peak: 0.0,+ };+ let refused = if cancelled { Some(VoiceError::Cancelled) } else { failure.map(VoiceError::Failed) };+ if let Some(error) = refused {+ drop(state);+ discard(session);+ return Err(error);+ }+ *state = State::Recording(session);+ Ok(())+ }++ /// Locale first (no point prompting without a recognizer), then Speech, then Microphone.+ fn prepare(&self, requested: Option<&str>, recording: u64) -> Result<String, VoiceError> {+ let (recognizer, microphone) = (&self.inner.recognizer, &self.inner.microphone);+ let locale = locale::resolve(requested, &recognizer.current_locale(), &recognizer.supported_locales())?;+ authorize(recognizer.authorization(), || recognizer.request_authorization(), VoiceError::SpeechDenied, VoiceError::SpeechRestricted)?;+ self.still_starting(recording)?;+ authorize(microphone.authorization(), || microphone.request_authorization(), VoiceError::MicrophoneDenied, VoiceError::MicrophoneRestricted)?;+ self.still_starting(recording)?;+ Ok(locale)+ }++ fn still_starting(&self, recording: u64) -> Result<(), VoiceError> {+ match *lock(&self.inner.state) {+ State::Starting { recording: r, cancelled: false, .. } if r == recording => Ok(()),+ _ => Err(VoiceError::Cancelled),+ }+ }++ /// Stops the microphone and resolves with the final transcript, or the latest partial when+ /// the recognizer does not answer within the timeout. Blocks, so it runs off the main thread.+ pub fn stop(&self) -> Result<Transcript, VoiceError> {+ let (recording, mut stream, feed) = {+ let mut state = lock(&self.inner.state);+ match std::mem::replace(&mut *state, State::Idle { next: 0 }) {+ State::Recording(mut session) => {+ let taken = (session.recording, session.stream.take(), session.feed.clone());+ *state = State::Stopping(session);+ taken+ }+ State::Starting { recording, failure, .. } => {+ // Stopping before the capture is up is a cancel with nothing said.+ *state = State::Starting { recording, cancelled: true, failure };+ return Ok(Transcript { text: String::new() });+ }+ other => {+ let error = if matches!(other, State::Stopping(_)) { VoiceError::Stopping } else { VoiceError::NotRecording };+ *state = other;+ return Err(error);+ }+ }+ };+ if let Some(stream) = stream.as_mut() {+ stream.close();+ }+ lock(&feed).finish();++ let deadline = Instant::now() + self.final_timeout;+ let mut state = lock(&self.inner.state);+ loop {+ let State::Stopping(session) = &*state else { return Err(VoiceError::Cancelled) };+ if session.recording != recording {+ return Err(VoiceError::Cancelled);+ }+ let now = Instant::now();+ if session.final_text.is_some() || now >= deadline {+ break;+ }+ state = self.inner.changed.wait_timeout(state, deadline - now).map(|(s, _)| s).unwrap_or_else(|p| p.into_inner().0);+ }+ let idle = state.idle();+ let State::Stopping(session) = std::mem::replace(&mut *state, idle) else { unreachable!() };+ drop(state);+ let answered = session.final_text.is_some();+ let text = session.final_text.unwrap_or(session.transcript).trim().to_string();+ if !answered {+ lock(&feed).cancel();+ }+ Ok(Transcript { text })+ }++ /// Stops and discards. Harmless when idle; during a permission prompt the start is aborted.+ pub fn cancel(&self) -> Result<(), VoiceError> {+ let mut state = lock(&self.inner.state);+ match &mut *state {+ State::Idle { .. } => Ok(()),+ State::Starting { cancelled, .. } => {+ *cancelled = true;+ Ok(())+ }+ State::Recording(_) | State::Stopping(_) => {+ let idle = state.idle();+ let (State::Recording(session) | State::Stopping(session)) = std::mem::replace(&mut *state, idle) else {+ unreachable!()+ };+ drop(state);+ self.inner.changed.notify_all();+ discard(session);+ Ok(())+ }+ }+ }+}++/// The platform's way back into one recording: audio from the microphone, results and failures+/// from the recognizer. Cheap to clone; calls for an earlier recording are ignored.+#[derive(Clone)]+pub struct Sink {+ inner: Weak<Inner>,+ recording: u64,+}++impl Sink {+ /// A chunk of `audio::SAMPLE_RATE` mono PCM: fed to the recognizer, and metered.+ pub fn audio(&self, samples: &[f32]) {+ let Some(inner) = self.inner.upgrade() else { return };+ let (feed, level) = {+ let mut state = lock(&inner.state);+ let State::Recording(session) = &mut *state else { return };+ if session.recording != self.recording {+ return;+ }+ session.peak = session.peak.max(audio::rms(samples));+ let now = inner.clock.now_ms();+ let due = session.last_level_ms.is_none_or(|last| now.saturating_sub(last) >= LEVEL_INTERVAL_MS);+ let level = due.then(|| {+ session.last_level_ms = Some(now);+ let level = audio::level(std::mem::take(&mut session.peak));+ (session.events.clone(), VoiceEvent::Level(level))+ });+ (session.feed.clone(), level)+ };+ if let Some((events, level)) = level {+ events(level);+ }+ lock(&feed).push(samples);+ }++ /// The text so far.+ pub fn partial(&self, text: &str) {+ self.update(|session| {+ session.transcript = text.to_string();+ Some(VoiceEvent::Partial(text.to_string()))+ });+ }++ /// The final text: it resolves `stop`, or waits for it.+ pub fn finished(&self, text: &str) {+ self.update(|session| {+ session.transcript = text.to_string();+ session.final_text = Some(text.to_string());+ None+ });+ }++ /// A failure. While recording it is reported and ends the recording; while stopping, `stop`+ /// resolves with the text so far (a recognizer that heard nothing reports an error).+ pub fn failed(&self, message: &str) {+ let Some(inner) = self.inner.upgrade() else { return };+ let mut state = lock(&inner.state);+ match &mut *state {+ State::Starting { recording, failure, .. } if *recording == self.recording => {+ failure.get_or_insert_with(|| message.to_string());+ }+ State::Stopping(session) if session.recording == self.recording => {+ if session.final_text.is_none() {+ session.final_text = Some(session.transcript.clone());+ }+ drop(state);+ inner.changed.notify_all();+ }+ State::Recording(session) if session.recording == self.recording => {+ let idle = state.idle();+ let State::Recording(session) = std::mem::replace(&mut *state, idle) else { unreachable!() };+ drop(state);+ inner.changed.notify_all();+ (session.events)(VoiceEvent::Error(message.to_string()));+ discard(session);+ }+ _ => {}+ }+ }++ fn update(&self, change: impl FnOnce(&mut Session) -> Option<VoiceEvent>) {+ let Some(inner) = self.inner.upgrade() else { return };+ let (events, event) = {+ let mut state = lock(&inner.state);+ let (State::Recording(session) | State::Stopping(session)) = &mut *state else { return };+ if session.recording != self.recording {+ return;+ }+ (session.events.clone(), change(session))+ };+ inner.changed.notify_all();+ if let Some(event) = event {+ events(event);+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::voice::fake::{FakeClock, FakeMicrophone, FakeRecognizer, Gate, Recorder};+ use crate::voice::{Authorization, Transcript, VoiceError, VoiceEvent, VoiceStatus};+ use std::sync::atomic::AtomicU64;+ use std::sync::Arc;+ use std::time::Duration;++ struct Rig {+ voice: Voice,+ mic: Arc<FakeMicrophone>,+ recognizer: Arc<FakeRecognizer>,+ clock: Arc<FakeClock>,+ recorder: Recorder,+ }++ fn rig(mic: FakeMicrophone, recognizer: FakeRecognizer) -> Rig {+ let (mic, recognizer) = (Arc::new(mic), Arc::new(recognizer));+ let clock = Arc::new(FakeClock(AtomicU64::new(1_000)));+ let voice = Voice::new(mic.clone(), recognizer.clone(), clock.clone()).with_final_timeout(Duration::from_millis(50));+ Rig { voice, mic, recognizer, clock, recorder: Recorder::default() }+ }++ fn granted() -> Rig {+ rig(FakeMicrophone::granted(), FakeRecognizer::granted())+ }++ impl Rig {+ fn start(&self) -> Result<(), VoiceError> {+ self.voice.start(None, self.recorder.events())+ }+ }++ #[test]+ fn status_is_available_when_permissions_are_granted_or_not_asked_yet() {+ assert_eq!(granted().voice.status(), VoiceStatus::available());+ let unasked = rig(+ FakeMicrophone::with(Authorization::NotDetermined, Authorization::Authorized),+ FakeRecognizer::with(Authorization::NotDetermined, Authorization::Authorized),+ );+ assert_eq!(unasked.voice.status(), VoiceStatus::available());+ assert_eq!(unasked.mic.log.lock().unwrap().requests + unasked.recognizer.log.lock().unwrap().requests, 0);+ }++ #[test]+ fn status_explains_a_denied_or_restricted_permission() {+ let speech = rig(FakeMicrophone::granted(), FakeRecognizer::with(Authorization::Denied, Authorization::Denied));+ assert_eq!(speech.voice.status(), VoiceStatus::unavailable(VoiceError::SpeechDenied.to_string()));+ let mic = rig(FakeMicrophone::with(Authorization::Restricted, Authorization::Restricted), FakeRecognizer::granted());+ assert_eq!(mic.voice.status(), VoiceStatus::unavailable(VoiceError::MicrophoneRestricted.to_string()));+ }++ #[test]+ fn status_explains_a_missing_recognizer_for_the_current_locale() {+ let mut recognizer = FakeRecognizer::granted();+ recognizer.current = "ja_JP".into();+ recognizer.supported = vec![];+ let rig = rig(FakeMicrophone::granted(), recognizer);+ assert_eq!(rig.voice.status(), VoiceStatus::unavailable("Speech recognition is not available for ja-JP."));+ }++ #[test]+ fn start_prompts_for_permissions_on_first_use_then_records() {+ let rig = rig(+ FakeMicrophone::with(Authorization::NotDetermined, Authorization::Authorized),+ FakeRecognizer::with(Authorization::NotDetermined, Authorization::Authorized),+ );+ rig.start().unwrap();+ assert_eq!(rig.recognizer.log.lock().unwrap().requests, 1);+ assert_eq!(rig.mic.log.lock().unwrap().requests, 1);+ assert_eq!(rig.mic.log.lock().unwrap().opened, 1);+ assert_eq!(rig.recognizer.log.lock().unwrap().locales, vec!["en-US".to_string()]);+ }++ #[test]+ fn a_denied_prompt_fails_the_start_and_frees_the_slot() {+ let rig = rig(+ FakeMicrophone::with(Authorization::NotDetermined, Authorization::Denied),+ FakeRecognizer::granted(),+ );+ assert_eq!(rig.start(), Err(VoiceError::MicrophoneDenied));+ assert_eq!(rig.mic.log.lock().unwrap().opened, 0);+ *rig.mic.answer.lock().unwrap() = Authorization::Authorized;+ *rig.mic.authorization.lock().unwrap() = Authorization::NotDetermined;+ assert_eq!(rig.start(), Ok(()));+ }++ #[test]+ fn the_requested_locale_reaches_the_recognizer() {+ let rig = granted();+ rig.voice.start(Some("es_AR"), rig.recorder.events()).unwrap();+ assert_eq!(rig.recognizer.log.lock().unwrap().locales, vec!["es-ES".to_string()]);+ }++ #[test]+ fn only_one_recording_at_a_time() {+ let rig = granted();+ rig.start().unwrap();+ assert_eq!(rig.start(), Err(VoiceError::Busy));+ assert_eq!(rig.mic.log.lock().unwrap().opened, 1);+ }++ #[test]+ fn partial_text_is_sent_while_recording() {+ let rig = granted();+ rig.start().unwrap();+ rig.recognizer.sink().partial("hello");+ rig.recognizer.sink().partial("hello there");+ assert_eq!(+ rig.recorder.seen(),+ vec![VoiceEvent::Partial("hello".into()), VoiceEvent::Partial("hello there".into())]+ );+ }++ #[test]+ fn stop_closes_the_microphone_and_resolves_with_the_final_transcript() {+ let rig = granted();+ *rig.recognizer.final_text.lock().unwrap() = Some(" Hello there. ".into());+ rig.start().unwrap();+ rig.recognizer.sink().partial("hello");+ assert_eq!(rig.voice.stop(), Ok(Transcript { text: "Hello there.".into() }));+ assert_eq!(rig.mic.log.lock().unwrap().closed, 1);+ assert_eq!(rig.recognizer.log.lock().unwrap().finished, 1);+ rig.start().unwrap();+ }++ #[test]+ fn stop_falls_back_to_the_latest_partial_when_no_final_arrives() {+ let rig = granted();+ rig.start().unwrap();+ rig.recognizer.sink().partial("almost done");+ assert_eq!(rig.voice.stop(), Ok(Transcript { text: "almost done".into() }));+ assert_eq!(rig.recognizer.log.lock().unwrap().cancelled, 1);+ }++ #[test]+ fn stop_without_a_recording_is_an_error() {+ assert_eq!(granted().voice.stop(), Err(VoiceError::NotRecording));+ }++ #[test]+ fn cancel_discards_the_recording_and_ignores_its_late_results() {+ let rig = granted();+ rig.start().unwrap();+ let old = rig.recognizer.sink();+ assert_eq!(rig.voice.cancel(), Ok(()));+ assert_eq!(rig.mic.log.lock().unwrap().closed, 1);+ assert_eq!(rig.recognizer.log.lock().unwrap().cancelled, 1);+ rig.start().unwrap();+ old.partial("from before");+ old.failed("canceled");+ assert!(rig.recorder.seen().is_empty());+ assert_eq!(rig.voice.cancel(), Ok(()));+ assert_eq!(rig.voice.cancel(), Ok(()), "cancelling when idle is harmless");+ }++ #[test]+ fn cancel_during_a_permission_prompt_aborts_the_start() {+ let gate = Arc::new(Gate::default());+ let mut mic = FakeMicrophone::with(Authorization::NotDetermined, Authorization::Authorized);+ mic.gate = Some(gate.clone());+ let rig = Arc::new(rig(mic, FakeRecognizer::granted()));+ let starting = {+ let rig = rig.clone();+ std::thread::spawn(move || rig.start())+ };+ while rig.mic.log.lock().unwrap().requests == 0 {+ std::thread::yield_now();+ }+ assert_eq!(rig.start(), Err(VoiceError::Busy));+ assert_eq!(rig.voice.cancel(), Ok(()));+ gate.release();+ assert_eq!(starting.join().unwrap(), Err(VoiceError::Cancelled));+ assert_eq!(rig.mic.log.lock().unwrap().opened, 0);+ rig.start().unwrap();+ }++ #[test]+ fn audio_reaches_the_recognizer_and_levels_are_throttled() {+ let rig = granted();+ rig.start().unwrap();+ let sink = rig.mic.sink();+ for _ in 0..10 {+ sink.audio(&[0.5; 160]);+ rig.clock.advance(10);+ }+ assert_eq!(rig.recognizer.log.lock().unwrap().fed.len(), 1_600);+ assert_eq!(rig.recorder.levels(), 2, "one level now, the next after 1/15 s");+ rig.clock.advance(LEVEL_INTERVAL_MS);+ sink.audio(&[0.0; 160]);+ let last = rig.recorder.seen().into_iter().rev().find_map(|event| match event {+ VoiceEvent::Level(level) => Some(level),+ _ => None,+ });+ assert!(last.unwrap() > 0.5, "the meter shows the loudest chunk since the last level");+ }++ #[test]+ fn no_audio_reaches_the_recognizer_after_stop() {+ let rig = granted();+ rig.start().unwrap();+ let sink = rig.mic.sink();+ rig.voice.stop().unwrap();+ sink.audio(&[0.5; 160]);+ assert!(rig.recognizer.log.lock().unwrap().fed.is_empty());+ }++ #[test]+ fn a_failure_while_recording_is_reported_and_ends_the_recording() {+ let rig = granted();+ rig.start().unwrap();+ rig.mic.sink().failed(&VoiceError::Interrupted.to_string());+ assert_eq!(rig.recorder.seen(), vec![VoiceEvent::Error("The microphone was interrupted.".into())]);+ assert_eq!(rig.mic.log.lock().unwrap().closed, 1);+ assert_eq!(rig.recognizer.log.lock().unwrap().cancelled, 1);+ assert_eq!(rig.voice.stop(), Err(VoiceError::NotRecording));+ rig.start().unwrap();+ }++ #[test]+ fn a_failure_while_stopping_resolves_with_the_text_so_far() {+ let rig = granted();+ *rig.recognizer.fail_on_finish.lock().unwrap() = Some("No speech detected".into());+ rig.start().unwrap();+ rig.recognizer.sink().partial("so far");+ assert_eq!(rig.voice.stop(), Ok(Transcript { text: "so far".into() }));+ assert!(rig.recorder.seen().iter().all(|event| !matches!(event, VoiceEvent::Error(_))));+ }++ #[test]+ fn a_microphone_that_cannot_open_cancels_the_recognition() {+ let rig = granted();+ *rig.mic.open_error.lock().unwrap() = Some(VoiceError::NoMicrophone);+ assert_eq!(rig.start(), Err(VoiceError::NoMicrophone));+ assert_eq!(rig.recognizer.log.lock().unwrap().cancelled, 1);+ *rig.mic.open_error.lock().unwrap() = None;+ rig.start().unwrap();+ }++ #[test]+ fn a_failure_reported_while_starting_fails_the_start() {+ let rig = granted();+ // The recognizer reports a failure before the capture is up (as Speech can, on its queue).+ let recognizer = rig.recognizer.clone();+ let mic = rig.mic.clone();+ *mic.open_error.lock().unwrap() = None;+ let voice = Voice::new(Arc::new(FailingOnOpen(mic, recognizer.clone())), recognizer.clone(), rig.clock.clone());+ assert_eq!(voice.start(None, rig.recorder.events()), Err(VoiceError::Failed("Siri and Dictation are disabled.".into())));+ assert_eq!(recognizer.log.lock().unwrap().cancelled, 1);+ assert_eq!(voice.stop(), Err(VoiceError::NotRecording));+ }++ /// Opens the fake microphone after the recognition reported a failure.+ struct FailingOnOpen(Arc<FakeMicrophone>, Arc<FakeRecognizer>);++ impl crate::voice::Microphone for FailingOnOpen {+ fn authorization(&self) -> Authorization {+ self.0.authorization()+ }++ fn request_authorization(&self) -> Authorization {+ self.0.request_authorization()+ }++ fn open(&self, sink: Sink) -> Result<Box<dyn crate::voice::Stream>, VoiceError> {+ self.1.sink().failed("Siri and Dictation are disabled.");+ self.0.open(sink)+ }+ }+}diff --git a/apps/desktop/src-tauri/src/voice/macos.rs b/apps/desktop/src-tauri/src/voice/macos.rsnew file mode 100644index 0000000..0a39da2--- /dev/null+++ b/apps/desktop/src-tauri/src/voice/macos.rs@@ -0,0 +1,290 @@+//! The macOS voice backends: the native microphone (AVAudioEngine's input node, resampled to+//! 16 kHz mono) and Apple's Speech framework (`SFSpeechRecognizer`, on-device when the locale+//! supports it, else Apple's servers). Nothing here decides policy; `machine` does.+//!+//! Threads: permission handlers, the audio tap, the interruption notification, and recognition+//! results all arrive off the main thread (the recognizer gets an operation queue of its own),+//! and they only call the `Sink`. The objects below are used from the command's blocking thread+//! and from those callbacks; AVAudioEngine and the Speech request are not tied to a thread.++use super::audio::{self, Resampler, SAMPLE_RATE};+use super::{Authorization, Microphone, Recognition, Recognizer, Sink, Stream, VoiceError};+use block2::RcBlock;+use objc2::rc::Retained;+use objc2::runtime::{AnyObject, Bool, ProtocolObject};+use objc2::{available, AnyThread};+use objc2_av_foundation::{AVAuthorizationStatus, AVCaptureDevice, AVMediaTypeAudio};+use objc2_avf_audio::{+ AVAudioEngine, AVAudioEngineConfigurationChangeNotification, AVAudioFormat, AVAudioPCMBuffer, AVAudioTime,+};+use objc2_foundation::{NSError, NSLocale, NSNotification, NSNotificationCenter, NSObjectProtocol, NSOperationQueue, NSString};+use objc2_speech::{+ SFSpeechAudioBufferRecognitionRequest, SFSpeechRecognitionResult, SFSpeechRecognitionTask, SFSpeechRecognizer,+ SFSpeechRecognizerAuthorizationStatus,+};+use std::cell::RefCell;+use std::ptr::NonNull;+use std::sync::mpsc;++/// Frames per tap callback at the device rate (about 21 ms at 48 kHz).+const TAP_FRAMES: u32 = 1_024;++fn microphone_status(status: AVAuthorizationStatus) -> Authorization {+ match status {+ AVAuthorizationStatus::Authorized => Authorization::Authorized,+ AVAuthorizationStatus::Denied => Authorization::Denied,+ AVAuthorizationStatus::Restricted => Authorization::Restricted,+ _ => Authorization::NotDetermined,+ }+}++fn speech_status(status: SFSpeechRecognizerAuthorizationStatus) -> Authorization {+ match status {+ SFSpeechRecognizerAuthorizationStatus::Authorized => Authorization::Authorized,+ SFSpeechRecognizerAuthorizationStatus::Denied => Authorization::Denied,+ SFSpeechRecognizerAuthorizationStatus::Restricted => Authorization::Restricted,+ _ => Authorization::NotDetermined,+ }+}++fn describe(error: &NSError) -> String {+ error.localizedDescription().to_string()+}++/// The Mac's default input device.+pub struct MacMicrophone;++impl Microphone for MacMicrophone {+ fn authorization(&self) -> Authorization {+ // SAFETY: a framework constant, and audio is a media type this call accepts.+ match unsafe { AVMediaTypeAudio } {+ Some(audio) => microphone_status(unsafe { AVCaptureDevice::authorizationStatusForMediaType(audio) }),+ None => Authorization::Restricted,+ }+ }++ fn request_authorization(&self) -> Authorization {+ // SAFETY: as above.+ let Some(audio) = (unsafe { AVMediaTypeAudio }) else { return Authorization::Restricted };+ let (tx, rx) = mpsc::channel();+ let handler = RcBlock::new(move |granted: Bool| {+ let _ = tx.send(granted.as_bool());+ });+ // SAFETY: the handler matches the documented signature; it runs on an arbitrary queue.+ unsafe { AVCaptureDevice::requestAccessForMediaType_completionHandler(audio, &handler) };+ match rx.recv() {+ Ok(true) => Authorization::Authorized,+ _ => self.authorization(),+ }+ }++ fn open(&self, sink: Sink) -> Result<Box<dyn Stream>, VoiceError> {+ // SAFETY: plain constructors and getters on a new engine.+ let engine = unsafe { AVAudioEngine::new() };+ let input = unsafe { engine.inputNode() };+ let format = unsafe { input.outputFormatForBus(0) };+ let (rate, channels) = unsafe { (format.sampleRate(), format.channelCount() as usize) };+ if rate <= 0.0 || channels == 0 {+ return Err(VoiceError::NoMicrophone);+ }++ let resampler = RefCell::new(Resampler::new(rate));+ let audio_sink = sink.clone();+ let tap = RcBlock::new(move |buffer: NonNull<AVAudioPCMBuffer>, _when: NonNull<AVAudioTime>| {+ // SAFETY: the engine passes a valid buffer in the input node's format (deinterleaved+ // float32 with `channels` channels), valid for the duration of the call.+ let buffer = unsafe { buffer.as_ref() };+ let frames = unsafe { buffer.frameLength() } as usize;+ let data = unsafe { buffer.floatChannelData() };+ if data.is_null() || frames == 0 {+ return;+ }+ let planes: Vec<&[f32]> = (0..channels)+ // SAFETY: `data` holds one pointer per channel, each to `frames` samples.+ .map(|c| unsafe { std::slice::from_raw_parts((*data.add(c)).as_ptr(), frames) })+ .collect();+ let pcm = resampler.borrow_mut().process(&audio::mono(&planes));+ if !pcm.is_empty() {+ audio_sink.audio(&pcm);+ }+ });+ let tap_block = &*tap as *const block2::DynBlock<_> as *mut block2::DynBlock<_>;+ // SAFETY: the block matches AVAudioNodeTapBlock and stays alive in the stream.+ unsafe { input.installTapOnBus_bufferSize_format_block(0, TAP_FRAMES, Some(&format), tap_block) };+ // SAFETY: the engine has its input connected through the tap.+ unsafe { engine.prepare() };+ if let Err(error) = unsafe { engine.startAndReturnError() } {+ unsafe { input.removeTapOnBus(0) };+ return Err(VoiceError::Failed(format!("The microphone could not start: {}", describe(&error))));+ }++ // A device change (headset unplugged, input switched) stops the engine.+ let interrupted = RcBlock::new(move |_note: NonNull<NSNotification>| {+ sink.failed(&VoiceError::Interrupted.to_string());+ });+ let engine_object: &AnyObject = &engine;+ // SAFETY: a framework constant name, observed for this engine only; the block posts+ // on the notifying thread and only calls the sink.+ let observer = unsafe {+ NSNotificationCenter::defaultCenter().addObserverForName_object_queue_usingBlock(+ Some(AVAudioEngineConfigurationChangeNotification),+ Some(engine_object),+ None,+ &interrupted,+ )+ };+ Ok(Box::new(MacStream { engine, observer, _tap: tap, open: true }))+ }+}++struct MacStream {+ engine: Retained<AVAudioEngine>,+ observer: Retained<ProtocolObject<dyn NSObjectProtocol>>,+ _tap: RcBlock<dyn Fn(NonNull<AVAudioPCMBuffer>, NonNull<AVAudioTime>)>,+ open: bool,+}++// SAFETY: AVAudioEngine and the observer token are not tied to a thread; the stream is only+// used behind the machine's locks, one thread at a time, and the tap block is only invoked by+// the engine.+unsafe impl Send for MacStream {}++impl Stream for MacStream {+ fn close(&mut self) {+ if !std::mem::take(&mut self.open) {+ return;+ }+ // SAFETY: stopping a live engine and removing what `open` installed.+ unsafe {+ NSNotificationCenter::defaultCenter().removeObserver(AsRef::<AnyObject>::as_ref(&*self.observer));+ self.engine.stop();+ self.engine.inputNode().removeTapOnBus(0);+ }+ }+}++impl Drop for MacStream {+ fn drop(&mut self) {+ self.close();+ }+}++/// Apple's Speech framework.+pub struct AppleSpeech;++impl Recognizer for AppleSpeech {+ fn authorization(&self) -> Authorization {+ // SAFETY: a class getter without side effects (it never prompts).+ speech_status(unsafe { SFSpeechRecognizer::authorizationStatus() })+ }++ fn request_authorization(&self) -> Authorization {+ let (tx, rx) = mpsc::channel();+ let handler = RcBlock::new(move |status: SFSpeechRecognizerAuthorizationStatus| {+ let _ = tx.send(status);+ });+ // SAFETY: the handler matches the documented signature; Info.plist carries+ // NSSpeechRecognitionUsageDescription (without it this call aborts the app).+ unsafe { SFSpeechRecognizer::requestAuthorization(&handler) };+ rx.recv().map(speech_status).unwrap_or_else(|_| self.authorization())+ }++ fn current_locale(&self) -> String {+ NSLocale::currentLocale().localeIdentifier().to_string()+ }++ fn supported_locales(&self) -> Vec<String> {+ // SAFETY: a class getter returning a set of locales.+ let locales = unsafe { SFSpeechRecognizer::supportedLocales() };+ locales.allObjects().iter().map(|locale| locale.localeIdentifier().to_string()).collect()+ }++ fn begin(&self, locale: &str, sink: Sink) -> Result<Box<dyn Recognition>, VoiceError> {+ let ns_locale = NSLocale::initWithLocaleIdentifier(NSLocale::alloc(), &NSString::from_str(locale));+ let handler = RcBlock::new(move |result: *mut SFSpeechRecognitionResult, error: *mut NSError| {+ // SAFETY: Speech passes a valid result or error (or null), valid for the call.+ if let Some(result) = unsafe { result.as_ref() } {+ let text = unsafe { result.bestTranscription().formattedString() }.to_string();+ if unsafe { result.isFinal() } {+ sink.finished(&text);+ } else {+ sink.partial(&text);+ }+ } else if let Some(error) = unsafe { error.as_ref() } {+ sink.failed(&describe(error));+ }+ });+ // SAFETY: plain initializers, setters, and getters on objects created here.+ unsafe {+ let recognizer = SFSpeechRecognizer::initWithLocale(SFSpeechRecognizer::alloc(), &ns_locale)+ .ok_or_else(|| VoiceError::NoRecognizer(locale.to_string()))?;+ if !recognizer.isAvailable() {+ return Err(VoiceError::Failed(+ "Speech recognition is not available right now. Check the internet connection and try again.".into(),+ ));+ }+ // Results default to the main queue; keep them off it.+ recognizer.setQueue(&NSOperationQueue::new());+ let request = SFSpeechAudioBufferRecognitionRequest::new();+ request.setShouldReportPartialResults(true);+ if recognizer.supportsOnDeviceRecognition() {+ request.setRequiresOnDeviceRecognition(true);+ }+ if available!(macos = 13.0) {+ request.setAddsPunctuation(true);+ }+ let format = AVAudioFormat::initStandardFormatWithSampleRate_channels(+ AVAudioFormat::alloc(),+ f64::from(SAMPLE_RATE),+ 1,+ )+ .ok_or_else(|| VoiceError::Failed("The audio format is not supported.".into()))?;+ let task = recognizer.recognitionTaskWithRequest_resultHandler(&request, &handler);+ Ok(Box::new(AppleRecognition { _recognizer: recognizer, request, task, format }))+ }+ }+}++struct AppleRecognition {+ /// Kept alive until the final result arrives.+ _recognizer: Retained<SFSpeechRecognizer>,+ request: Retained<SFSpeechAudioBufferRecognitionRequest>,+ task: Retained<SFSpeechRecognitionTask>,+ /// 16 kHz mono float32, deinterleaved.+ format: Retained<AVAudioFormat>,+}++// SAFETY: the request is meant to be fed from the audio thread and ended or cancelled from+// another; the machine serializes every call behind a lock.+unsafe impl Send for AppleRecognition {}++impl Recognition for AppleRecognition {+ fn feed(&mut self, samples: &[f32]) {+ let Ok(frames) = u32::try_from(samples.len()) else { return };+ // SAFETY: a new buffer in our mono format with room for `frames`; its single channel+ // receives exactly `frames` samples before the request copies it.+ unsafe {+ let Some(buffer) = AVAudioPCMBuffer::initWithPCMFormat_frameCapacity(AVAudioPCMBuffer::alloc(), &self.format, frames)+ else {+ return;+ };+ buffer.setFrameLength(frames);+ let data = buffer.floatChannelData();+ if data.is_null() {+ return;+ }+ std::ptr::copy_nonoverlapping(samples.as_ptr(), (*data).as_ptr(), samples.len());+ self.request.appendAudioPCMBuffer(&buffer);+ }+ }++ fn finish(&mut self) {+ // SAFETY: ending the audio of a live request; the final result follows on the queue.+ unsafe { self.request.endAudio() };+ }++ fn cancel(&mut self) {+ // SAFETY: cancelling a live task; its late "cancelled" error is ignored by the sink.+ unsafe { self.task.cancel() };+ }+}diff --git a/apps/desktop/src-tauri/src/voice/mod.rs b/apps/desktop/src-tauri/src/voice/mod.rsnew file mode 100644index 0000000..8496088--- /dev/null+++ b/apps/desktop/src-tauri/src/voice/mod.rs@@ -0,0 +1,217 @@+//! Voice input (S30.1): the panel's mic button records natively and the text comes back as the+//! user's message. Two seams keep the backends swappable:+//!+//! - `Microphone` captures audio and delivers 16 kHz mono f32 PCM (`audio::SAMPLE_RATE`), the+//! format offline models such as sherpa-onnx expect, so any recognizer can consume it.+//! - `Recognizer` turns that PCM into partial and final text. Apple's Speech framework is the+//! default (`macos::AppleSpeech`); a local model can be added as a second implementation.+//!+//! `Voice` is the pure state machine between them (idle → starting → recording → stopping,+//! single flight, cancel, error mapping, level throttling, locale fallback), tested on every OS+//! with fakes. The objc2 bindings live in `macos`; off macOS the commands answer "unavailable".+//!+//! SpeechAnalyzer/SpeechTranscriber (macOS 26) are Swift-only, with no Objective-C interface to+//! bind from Rust, and the app supports macOS 12.3, so `SFSpeechRecognizer` is the recognizer.++pub mod app;+pub mod audio;+#[cfg(test)]+mod fake;+pub mod locale;+mod machine;+#[cfg(target_os = "macos")]+mod macos;++pub use machine::{Sink, Voice};+#[cfg(target_os = "macos")]+pub use macos::{AppleSpeech, MacMicrophone};++use serde::Serialize;+use std::fmt;+use std::sync::Arc;++/// Live text while the user speaks: `{text}`.+pub const PARTIAL_EVENT: &str = "voice://partial";+/// The input level, `{level}` in 0..1, about 15 times a second.+pub const LEVEL_EVENT: &str = "voice://level";+/// A failure while recording (permission, recognizer, interruption): `{message}`.+pub const ERROR_EVENT: &str = "voice://error";+pub const UNSUPPORTED: &str = "Voice input is available on macOS.";++/// The `voice_status` reply: `{available, reason?}`.+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]+pub struct VoiceStatus {+ pub available: bool,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub reason: Option<String>,+}++impl VoiceStatus {+ pub fn available() -> Self {+ VoiceStatus { available: true, reason: None }+ }++ pub fn unavailable(reason: impl Into<String>) -> Self {+ VoiceStatus { available: false, reason: Some(reason.into()) }+ }+}++/// The `voice_stop` reply: `{text}`, the final transcript.+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]+pub struct Transcript {+ pub text: String,+}++/// What the webview that started the recording hears.+#[derive(Debug, Clone, PartialEq)]+pub enum VoiceEvent {+ Partial(String),+ Level(f32),+ Error(String),+}++impl VoiceEvent {+ pub fn name(&self) -> &'static str {+ match self {+ VoiceEvent::Partial(_) => PARTIAL_EVENT,+ VoiceEvent::Level(_) => LEVEL_EVENT,+ VoiceEvent::Error(_) => ERROR_EVENT,+ }+ }++ pub fn payload(&self) -> serde_json::Value {+ match self {+ VoiceEvent::Partial(text) => serde_json::json!({ "text": text }),+ VoiceEvent::Level(level) => serde_json::json!({ "level": level }),+ VoiceEvent::Error(message) => serde_json::json!({ "message": message }),+ }+ }+}++/// Delivers events to the webview that started the recording; called from any thread.+pub type Events = Arc<dyn Fn(VoiceEvent) + Send + Sync>;++/// A macOS privacy permission, as the system reports it.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Authorization {+ NotDetermined,+ Denied,+ Restricted,+ Authorized,+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub enum VoiceError {+ Unsupported,+ Busy,+ Stopping,+ NotRecording,+ Cancelled,+ SpeechDenied,+ SpeechRestricted,+ MicrophoneDenied,+ MicrophoneRestricted,+ NoRecognizer(String),+ NoMicrophone,+ Interrupted,+ Failed(String),+}++impl fmt::Display for VoiceError {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ match self {+ VoiceError::Unsupported => f.write_str(UNSUPPORTED),+ VoiceError::Busy => f.write_str("Voice input is already recording."),+ VoiceError::Stopping => f.write_str("Voice input is already stopping."),+ VoiceError::NotRecording => f.write_str("Voice input is not recording."),+ VoiceError::Cancelled => f.write_str("Voice input was cancelled."),+ VoiceError::SpeechDenied => f.write_str(+ "Speech Recognition is off for Gentle Dot. Turn it on in System Settings → Privacy & Security → Speech Recognition.",+ ),+ VoiceError::SpeechRestricted => f.write_str("Speech Recognition is restricted on this Mac."),+ VoiceError::MicrophoneDenied => f.write_str(+ "Microphone access is off for Gentle Dot. Turn it on in System Settings → Privacy & Security → Microphone.",+ ),+ VoiceError::MicrophoneRestricted => f.write_str("Microphone access is restricted on this Mac."),+ VoiceError::NoRecognizer(locale) => write!(f, "Speech recognition is not available for {locale}."),+ VoiceError::NoMicrophone => f.write_str("No microphone is available."),+ VoiceError::Interrupted => f.write_str("The microphone was interrupted."),+ VoiceError::Failed(message) => f.write_str(message),+ }+ }+}++impl From<VoiceError> for String {+ fn from(error: VoiceError) -> String {+ error.to_string()+ }+}++/// Native audio capture. Delivers `audio::SAMPLE_RATE` mono f32 chunks to `Sink::audio` (on+/// the audio thread) and reports interruptions with `Sink::failed`.+pub trait Microphone: Send + Sync {+ /// The current permission, without prompting.+ fn authorization(&self) -> Authorization;+ /// Prompts when not yet determined. Blocks until the user answers: never on the main thread.+ fn request_authorization(&self) -> Authorization;+ fn open(&self, sink: Sink) -> Result<Box<dyn Stream>, VoiceError>;+}++/// An open capture.+pub trait Stream: Send {+ fn close(&mut self);+}++/// Speech to text over the microphone's PCM. Results arrive through `Sink::partial`,+/// `Sink::finished`, and `Sink::failed`, from any thread, but never from inside `feed`,+/// `finish`, or `cancel` while reporting a failure.+pub trait Recognizer: Send + Sync {+ /// The current permission, without prompting (`Authorized` when none is needed).+ fn authorization(&self) -> Authorization;+ /// Prompts when not yet determined. Blocks until the user answers: never on the main thread.+ fn request_authorization(&self) -> Authorization;+ /// The user's locale identifier, as the system reports it.+ fn current_locale(&self) -> String;+ fn supported_locales(&self) -> Vec<String>;+ fn begin(&self, locale: &str, sink: Sink) -> Result<Box<dyn Recognition>, VoiceError>;+}++/// One recognition in progress.+pub trait Recognition: Send {+ /// More audio. Runs on the audio thread, so it must not block (heavy decoders queue it).+ fn feed(&mut self, samples: &[f32]);+ /// No more audio: the final text follows through `Sink::finished`.+ fn finish(&mut self);+ /// Discards the recognition.+ fn cancel(&mut self);+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn events_carry_the_contract_names_and_payloads() {+ let partial = VoiceEvent::Partial("hola".into());+ assert_eq!((partial.name(), partial.payload()), ("voice://partial", serde_json::json!({"text": "hola"})));+ let level = VoiceEvent::Level(0.5);+ assert_eq!((level.name(), level.payload()), ("voice://level", serde_json::json!({"level": 0.5})));+ let error = VoiceEvent::Error("no".into());+ assert_eq!((error.name(), error.payload()), ("voice://error", serde_json::json!({"message": "no"})));+ }++ #[test]+ fn status_omits_the_reason_when_available() {+ assert_eq!(serde_json::to_value(VoiceStatus::available()).unwrap(), serde_json::json!({"available": true}));+ assert_eq!(+ serde_json::to_value(VoiceStatus::unavailable(UNSUPPORTED)).unwrap(),+ serde_json::json!({"available": false, "reason": "Voice input is available on macOS."})+ );+ }++ #[test]+ fn the_stop_reply_is_the_text() {+ let reply = Transcript { text: "hello".into() };+ assert_eq!(serde_json::to_value(reply).unwrap(), serde_json::json!({"text": "hello"}));+ }+}