Skip to content

A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.

TypeScript

feat(desktop): recognize speech natively on macOS

Alan Buscaglia93ee6372fbbc
diff --git a/apps/desktop/src-tauri/Cargo.lock b/apps/desktop/src-tauri/Cargo.lock
index f8d1ac0..e76731a 100644
--- a/apps/desktop/src-tauri/Cargo.lock
+++ b/apps/desktop/src-tauri/Cargo.lock
@@ -1283,10 +1283,13 @@ dependencies = [
"objc2",
"objc2-app-kit",
"objc2-application-services",
+ "objc2-av-foundation",
+ "objc2-avf-audio",
"objc2-core-foundation",
"objc2-core-graphics",
"objc2-foundation",
"objc2-screen-capture-kit",
+ "objc2-speech",
"serde",
"serde_json",
"tauri",
@@ -2329,6 +2332,29 @@ dependencies = [
"objc2-core-foundation",
]
+[[package]]
+name = "objc2-av-foundation"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0"
+dependencies = [
+ "bitflags 2.13.2",
+ "block2",
+ "objc2",
+ "objc2-foundation",
+]
+
+[[package]]
+name = "objc2-avf-audio"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be"
+dependencies = [
+ "block2",
+ "objc2",
+ "objc2-foundation",
+]
+
[[package]]
name = "objc2-cloud-kit"
version = "0.3.2"
@@ -2495,6 +2521,18 @@ dependencies = [
"objc2-security",
]
+[[package]]
+name = "objc2-speech"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9eb609f9d2a25e0f8b76954f3acaa58348ce2a91285fe867643b2224ac4d263"
+dependencies = [
+ "block2",
+ "objc2",
+ "objc2-avf-audio",
+ "objc2-foundation",
+]
+
[[package]]
name = "objc2-ui-kit"
version = "0.3.2"
diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml
index 87ea1e0..f4d20d1 100644
--- a/apps/desktop/src-tauri/Cargo.toml
+++ b/apps/desktop/src-tauri/Cargo.toml
@@ -31,12 +31,16 @@ image = { version = "0.25", default-features = false, features = ["jpeg"] }
[target.'cfg(target_os = "macos")'.dependencies]
objc2 = "0.6"
block2 = "0.6"
-objc2-foundation = { version = "0.3", default-features = false, features = ["std", "NSArray", "NSDictionary", "NSError", "NSString", "NSValue"] }
+objc2-foundation = { version = "0.3", default-features = false, features = ["std", "block2", "NSArray", "NSDictionary", "NSError", "NSLocale", "NSNotification", "NSOperation", "NSSet", "NSString", "NSValue"] }
objc2-core-foundation = { version = "0.3", default-features = false, features = ["std", "CFArray", "CFCGTypes", "CFData", "CFDictionary", "CFNumber", "CFString"] }
objc2-core-graphics = { version = "0.3", default-features = false, features = ["std", "CGDataProvider", "CGDirectDisplay", "CGError", "CGEvent", "CGEventTypes", "CGImage", "CGRemoteOperation", "CGWindow"] }
objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "libc", "NSRunningApplication", "NSWorkspace"] }
objc2-application-services = { version = "0.3", default-features = false, features = ["std", "libc", "AXError", "AXUIElement", "AXValue", "HIServices"] }
objc2-screen-capture-kit = { version = "0.3", default-features = false, features = ["std", "block2", "libc", "objc2-core-foundation", "objc2-core-graphics", "SCScreenshotManager", "SCShareableContent", "SCStream"] }
+# Voice input (S30.1): the Speech framework, the audio engine, and the microphone permission.
+objc2-speech = { version = "0.3", default-features = false, features = ["std", "block2", "objc2-avf-audio", "SFSpeechRecognizer", "SFSpeechRecognitionRequest", "SFSpeechRecognitionResult", "SFSpeechRecognitionTask", "SFTranscription"] }
+objc2-avf-audio = { version = "0.3", default-features = false, features = ["std", "block2", "AVAudioBuffer", "AVAudioEngine", "AVAudioFormat", "AVAudioIONode", "AVAudioNode", "AVAudioTime", "AVAudioTypes"] }
+objc2-av-foundation = { version = "0.3", default-features = false, features = ["std", "block2", "AVCaptureDevice", "AVMediaFormat"] }
# Debug builds encode screenshots fast too (S24.8): `image` holds the JPEG encoder and `zune-*`
# decode JPEGs (tests). The encoder is generic, so most of its code is compiled in this crate,
diff --git a/apps/desktop/src-tauri/Info.plist b/apps/desktop/src-tauri/Info.plist
index d4b6455..4550482 100644
--- a/apps/desktop/src-tauri/Info.plist
+++ b/apps/desktop/src-tauri/Info.plist
@@ -5,5 +5,10 @@
<!-- Accessory app: no Dock icon, even before the activation policy is applied. -->
<key>LSUIElement</key>
<true/>
+ <!-- Voice input (S30.1): shown by macOS the first time the mic button is used. -->
+ <key>NSMicrophoneUsageDescription</key>
+ <string>Gentle Dot listens to your voice when you press the microphone button.</string>
+ <key>NSSpeechRecognitionUsageDescription</key>
+ <string>Gentle Dot turns what you say into a message when you press the microphone button.</string>
</dict>
</plist>
diff --git a/apps/desktop/src-tauri/build.rs b/apps/desktop/src-tauri/build.rs
index 3c85eee..bc80693 100644
--- a/apps/desktop/src-tauri/build.rs
+++ b/apps/desktop/src-tauri/build.rs
@@ -16,6 +16,10 @@ const COMMANDS: &[&str] = &[
"computer_stop",
"computer_status",
"computer_set_yolo",
+ "voice_status",
+ "voice_start",
+ "voice_stop",
+ "voice_cancel",
];
fn find_in_path(program: &str, path: &str) -> Option<PathBuf> {
diff --git a/apps/desktop/src-tauri/capabilities/default.json b/apps/desktop/src-tauri/capabilities/default.json
index 3f09df0..7c49a7f 100644
--- a/apps/desktop/src-tauri/capabilities/default.json
+++ b/apps/desktop/src-tauri/capabilities/default.json
@@ -25,6 +25,17 @@
"allow-computer-set-yolo"
]
},
+ {
+ "identifier": "voice",
+ "description": "Voice input (S30.1): the panel's mic button records and transcribes natively; the Dot never records.",
+ "windows": ["panel"],
+ "permissions": [
+ "allow-voice-status",
+ "allow-voice-start",
+ "allow-voice-stop",
+ "allow-voice-cancel"
+ ]
+ },
{
"identifier": "glow",
"description": "What the computer-control glow overlay (S28) may call: listening to the glow and session events, nothing else.",
diff --git a/apps/desktop/src-tauri/permissions/autogenerated/voice_cancel.toml b/apps/desktop/src-tauri/permissions/autogenerated/voice_cancel.toml
new file mode 100644
index 0000000..f670fba
--- /dev/null
+++ b/apps/desktop/src-tauri/permissions/autogenerated/voice_cancel.toml
@@ -0,0 +1,11 @@
+# Automatically generated - DO NOT EDIT!
+
+[[permission]]
+identifier = "allow-voice-cancel"
+description = "Enables the voice_cancel command without any pre-configured scope."
+commands.allow = ["voice_cancel"]
+
+[[permission]]
+identifier = "deny-voice-cancel"
+description = "Denies the voice_cancel command without any pre-configured scope."
+commands.deny = ["voice_cancel"]
diff --git a/apps/desktop/src-tauri/permissions/autogenerated/voice_start.toml b/apps/desktop/src-tauri/permissions/autogenerated/voice_start.toml
new file mode 100644
index 0000000..8ef4ff7
--- /dev/null
+++ b/apps/desktop/src-tauri/permissions/autogenerated/voice_start.toml
@@ -0,0 +1,11 @@
+# Automatically generated - DO NOT EDIT!
+
+[[permission]]
+identifier = "allow-voice-start"
+description = "Enables the voice_start command without any pre-configured scope."
+commands.allow = ["voice_start"]
+
+[[permission]]
+identifier = "deny-voice-start"
+description = "Denies the voice_start command without any pre-configured scope."
+commands.deny = ["voice_start"]
diff --git a/apps/desktop/src-tauri/permissions/autogenerated/voice_status.toml b/apps/desktop/src-tauri/permissions/autogenerated/voice_status.toml
new file mode 100644
index 0000000..ee57f12
--- /dev/null
+++ b/apps/desktop/src-tauri/permissions/autogenerated/voice_status.toml
@@ -0,0 +1,11 @@
+# Automatically generated - DO NOT EDIT!
+
+[[permission]]
+identifier = "allow-voice-status"
+description = "Enables the voice_status command without any pre-configured scope."
+commands.allow = ["voice_status"]
+
+[[permission]]
+identifier = "deny-voice-status"
+description = "Denies the voice_status command without any pre-configured scope."
+commands.deny = ["voice_status"]
diff --git a/apps/desktop/src-tauri/permissions/autogenerated/voice_stop.toml b/apps/desktop/src-tauri/permissions/autogenerated/voice_stop.toml
new file mode 100644
index 0000000..b2daaf8
--- /dev/null
+++ b/apps/desktop/src-tauri/permissions/autogenerated/voice_stop.toml
@@ -0,0 +1,11 @@
+# Automatically generated - DO NOT EDIT!
+
+[[permission]]
+identifier = "allow-voice-stop"
+description = "Enables the voice_stop command without any pre-configured scope."
+commands.allow = ["voice_stop"]
+
+[[permission]]
+identifier = "deny-voice-stop"
+description = "Denies the voice_stop command without any pre-configured scope."
+commands.deny = ["voice_stop"]
diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs
index 4625c68..37d3440 100644
--- a/apps/desktop/src-tauri/src/lib.rs
+++ b/apps/desktop/src-tauri/src/lib.rs
@@ -9,5 +9,6 @@ pub mod platform;
pub mod position;
mod shell;
pub mod status;
+pub mod voice;
pub use shell::run;
diff --git a/apps/desktop/src-tauri/src/shell.rs b/apps/desktop/src-tauri/src/shell.rs
index ab86660..deb7179 100644
--- a/apps/desktop/src-tauri/src/shell.rs
+++ b/apps/desktop/src-tauri/src/shell.rs
@@ -1,5 +1,5 @@
//! Tauri wiring: windows, commands, tray menu, global shortcut, Dot snapping,
-//! hiding the rose, the full-screen panel, and the daemon lifecycle (design §9).
+//! hiding the rose, the full-screen panel, voice input, and the daemon lifecycle (design §9).
use crate::computer::app as computer;
use crate::computer::session::Reason;
@@ -9,6 +9,7 @@ use crate::geometry::{self, Rect};
use crate::platform::{self, LaunchRequest, Os, DOT_TITLE, PANEL_TITLE};
use crate::position::{self, DotPosition};
use crate::status::{is_template, tray_status, TrayGlyph};
+use crate::voice::app as voice;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{mpsc, Arc, Mutex};
@@ -614,7 +615,11 @@ pub fn run() {
computer::computer_request_permission,
computer::computer_stop,
computer::computer_status,
- computer::computer_set_yolo
+ computer::computer_set_yolo,
+ voice::voice_status,
+ voice::voice_start,
+ voice::voice_stop,
+ voice::voice_cancel
])
.setup(move |app| {
#[cfg(target_os = "macos")]
@@ -640,6 +645,7 @@ pub fn run() {
full_screen: Mutex::new(None),
});
app.manage(computer::start(&handle));
+ app.manage(voice::start());
#[cfg(target_os = "macos")]
register_panic_shortcut(&handle);
diff --git a/apps/desktop/src-tauri/src/voice/app.rs b/apps/desktop/src-tauri/src/voice/app.rs
new file mode 100644
index 0000000..3e03f33
--- /dev/null
+++ b/apps/desktop/src-tauri/src/voice/app.rs
@@ -0,0 +1,138 @@
+//! The app side of voice input (S30.1, L88): the `voice_*` commands and the `voice://*` events,
+//! sent only to the webview that started the recording. Off macOS `voice_status` answers
+//! "unavailable" and the other commands fail. Every command runs on a blocking thread: starting
+//! may wait on a permission prompt and stopping waits for the final transcript.
+
+use super::{Events, Transcript, Voice, VoiceError, VoiceEvent, VoiceStatus, UNSUPPORTED};
+use std::sync::Arc;
+use tauri::{AppHandle, Emitter, EventTarget, Manager};
+
+/// The commands, as registered in `build.rs` and granted to the panel.
+pub const COMMANDS: [&str; 4] = ["voice_status", "voice_start", "voice_stop", "voice_cancel"];
+
+/// Voice input, when this platform has a backend.
+pub struct VoiceInput {
+ voice: Option<Voice>,
+}
+
+impl VoiceInput {
+ pub fn new(voice: Voice) -> Self {
+ VoiceInput { voice: Some(voice) }
+ }
+
+ pub fn unsupported() -> Self {
+ VoiceInput { voice: None }
+ }
+
+ fn voice(&self) -> Result<&Voice, VoiceError> {
+ self.voice.as_ref().ok_or(VoiceError::Unsupported)
+ }
+
+ pub fn status(&self) -> VoiceStatus {
+ self.voice().map_or_else(|_| VoiceStatus::unavailable(UNSUPPORTED), Voice::status)
+ }
+
+ pub fn start(&self, locale: Option<&str>, events: Events) -> Result<(), VoiceError> {
+ self.voice()?.start(locale, events)
+ }
+
+ pub fn stop(&self) -> Result<Transcript, VoiceError> {
+ self.voice()?.stop()
+ }
+
+ pub fn cancel(&self) -> Result<(), VoiceError> {
+ self.voice()?.cancel()
+ }
+}
+
+/// Apple's Speech framework over the native microphone on macOS; nothing elsewhere.
+pub fn start() -> VoiceInput {
+ #[cfg(target_os = "macos")]
+ return VoiceInput::new(Voice::new(
+ Arc::new(super::MacMicrophone),
+ Arc::new(super::AppleSpeech),
+ Arc::new(crate::computer::SystemClock),
+ ));
+ #[cfg(not(target_os = "macos"))]
+ VoiceInput::unsupported()
+}
+
+async fn blocking<T: Send + 'static>(
+ app: AppHandle,
+ action: impl FnOnce(&VoiceInput) -> Result<T, VoiceError> + Send + 'static,
+) -> Result<T, String> {
+ tauri::async_runtime::spawn_blocking(move || action(&app.state::<VoiceInput>()))
+ .await
+ .map_err(|error| error.to_string())?
+ .map_err(String::from)
+}
+
+#[tauri::command]
+pub async fn voice_status(app: AppHandle) -> Result<VoiceStatus, String> {
+ blocking(app, |input| Ok(input.status())).await
+}
+
+#[tauri::command]
+pub async fn voice_start(app: AppHandle, webview: tauri::WebviewWindow, locale: Option<String>) -> Result<(), String> {
+ let target = EventTarget::webview_window(webview.label());
+ let emitter = app.clone();
+ let events: Events = Arc::new(move |event: VoiceEvent| {
+ if let Err(error) = emitter.emit_to(target.clone(), event.name(), event.payload()) {
+ eprintln!("gentle-dot: cannot send a voice event: {error}");
+ }
+ });
+ blocking(app, move |input| input.start(locale.as_deref(), events)).await
+}
+
+#[tauri::command]
+pub async fn voice_stop(app: AppHandle) -> Result<Transcript, String> {
+ blocking(app, VoiceInput::stop).await
+}
+
+#[tauri::command]
+pub async fn voice_cancel(app: AppHandle) -> Result<(), String> {
+ blocking(app, VoiceInput::cancel).await
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::voice::{VoiceError, VoiceStatus, UNSUPPORTED};
+ use std::sync::Arc;
+
+ #[test]
+ fn without_a_backend_voice_is_unavailable_and_every_action_errors() {
+ let input = VoiceInput::unsupported();
+ assert_eq!(input.status(), VoiceStatus::unavailable(UNSUPPORTED));
+ assert_eq!(input.start(None, Arc::new(|_| {})), Err(VoiceError::Unsupported));
+ assert_eq!(input.stop(), Err(VoiceError::Unsupported));
+ assert_eq!(input.cancel(), Err(VoiceError::Unsupported));
+ assert_eq!(String::from(VoiceError::Unsupported), "Voice input is available on macOS.");
+ }
+
+ #[test]
+ fn the_command_names_match_the_l88_contract() {
+ assert_eq!(COMMANDS, ["voice_status", "voice_start", "voice_stop", "voice_cancel"]);
+ }
+
+ #[test]
+ fn the_commands_are_registered_and_granted_to_the_panel_only() {
+ let build = include_str!("../../build.rs");
+ let capabilities: serde_json::Value = serde_json::from_str(include_str!("../../capabilities/default.json")).unwrap();
+ let granted_to = |permission: &str| -> Vec<&str> {
+ let all = capabilities["capabilities"].as_array().unwrap();
+ let mut windows: Vec<&str> = all
+ .iter()
+ .filter(|c| c["permissions"].as_array().unwrap().iter().any(|p| p == permission))
+ .flat_map(|c| c["windows"].as_array().unwrap().iter().map(|w| w.as_str().unwrap()))
+ .collect();
+ windows.sort();
+ windows
+ };
+ for command in COMMANDS {
+ assert!(build.contains(&format!("\"{command}\",")), "{command} is missing from build.rs");
+ let permission = format!("allow-{}", command.replace('_', "-"));
+ assert_eq!(granted_to(&permission), vec!["panel"], "{permission}");
+ }
+ }
+}
diff --git a/apps/desktop/src-tauri/src/voice/audio.rs b/apps/desktop/src-tauri/src/voice/audio.rs
new file mode 100644
index 0000000..843d833
--- /dev/null
+++ b/apps/desktop/src-tauri/src/voice/audio.rs
@@ -0,0 +1,127 @@
+//! PCM helpers shared by every microphone and recognizer: the 16 kHz mono contract, the
+//! level meter, and a streaming resampler from the device rate.
+
+/// What every `Microphone` delivers and every `Recognizer` consumes: 16 kHz mono f32.
+pub const SAMPLE_RATE: u32 = 16_000;
+/// The meter's floor: quieter input shows as silence.
+const FLOOR_DB: f32 = -50.0;
+
+/// Root mean square of a chunk.
+pub fn rms(samples: &[f32]) -> f32 {
+ if samples.is_empty() {
+ return 0.0;
+ }
+ (samples.iter().map(|s| s * s).sum::<f32>() / samples.len() as f32).sqrt()
+}
+
+/// Maps an RMS to the meter's 0..1 on a decibel scale: -50 dBFS and below is 0, 0 dBFS is 1.
+/// Speech sits around -40 to -10 dBFS, so a linear RMS would barely move the meter.
+pub fn level(rms: f32) -> f32 {
+ if rms <= 0.0 {
+ return 0.0;
+ }
+ ((20.0 * rms.log10() - FLOOR_DB) / -FLOOR_DB).clamp(0.0, 1.0)
+}
+
+/// Averages the device's channels (non-interleaved) into one.
+pub fn mono(channels: &[&[f32]]) -> Vec<f32> {
+ let Some(first) = channels.first() else { return Vec::new() };
+ if channels.len() == 1 {
+ return first.to_vec();
+ }
+ let scale = 1.0 / channels.len() as f32;
+ (0..first.len()).map(|i| channels.iter().map(|c| c.get(i).copied().unwrap_or(0.0)).sum::<f32>() * scale).collect()
+}
+
+/// Converts a mono stream from the device rate to `SAMPLE_RATE`, chunk by chunk. Downsampling
+/// averages the input that falls in each output period (a box filter, enough to keep speech
+/// free of audible aliasing); lower rates are held.
+pub struct Resampler {
+ /// Input samples per output sample.
+ step: f64,
+ /// Input consumed, and where the current output period ends.
+ position: f64,
+ edge: f64,
+ sum: f32,
+ count: u32,
+ last: f32,
+}
+
+impl Resampler {
+ pub fn new(from_rate: f64) -> Self {
+ let step = from_rate / f64::from(SAMPLE_RATE);
+ Resampler { step, position: 0.0, edge: step, sum: 0.0, count: 0, last: 0.0 }
+ }
+
+ pub fn process(&mut self, input: &[f32]) -> Vec<f32> {
+ let mut out = Vec::with_capacity((input.len() as f64 / self.step) as usize + 1);
+ for &sample in input {
+ self.sum += sample;
+ self.count += 1;
+ self.position += 1.0;
+ while self.position >= self.edge - 1e-9 {
+ if self.count > 0 {
+ self.last = self.sum / self.count as f32;
+ self.sum = 0.0;
+ self.count = 0;
+ }
+ out.push(self.last);
+ self.edge += self.step;
+ }
+ }
+ out
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn rms_of_a_constant_signal_is_its_magnitude() {
+ assert_eq!(rms(&[]), 0.0);
+ assert!((rms(&[0.5, -0.5, 0.5, -0.5]) - 0.5).abs() < 1e-6);
+ }
+
+ #[test]
+ fn the_level_maps_decibels_to_zero_through_one() {
+ assert_eq!(level(0.0), 0.0);
+ assert_eq!(level(0.001), 0.0, "-60 dBFS is silence on the meter");
+ assert_eq!(level(1.0), 1.0);
+ assert_eq!(level(2.0), 1.0);
+ let (quiet, loud) = (level(0.01), level(0.1));
+ assert!(0.0 < quiet && quiet < loud && loud < 1.0);
+ }
+
+ #[test]
+ fn channels_mix_down_to_mono() {
+ assert_eq!(mono(&[&[1.0, 0.0], &[0.0, 0.0]]), vec![0.5, 0.0]);
+ assert_eq!(mono(&[&[0.25, 0.75]]), vec![0.25, 0.75]);
+ assert!(mono(&[]).is_empty());
+ }
+
+ #[test]
+ fn resampling_48k_to_16k_keeps_a_third_of_the_samples_and_the_level() {
+ let mut resampler = Resampler::new(48_000.0);
+ let out = resampler.process(&[0.3; 4_800]);
+ assert_eq!(out.len(), 1_600);
+ assert!(out.iter().all(|s| (s - 0.3).abs() < 1e-6));
+ }
+
+ #[test]
+ fn resampling_is_the_same_in_chunks_as_in_one_go() {
+ let input: Vec<f32> = (0..4_410).map(|i| (i as f32 * 0.01).sin()).collect();
+ let whole = Resampler::new(44_100.0).process(&input);
+ let mut chunked = Resampler::new(44_100.0);
+ let pieces: Vec<f32> = input.chunks(512).flat_map(|chunk| chunked.process(chunk)).collect();
+ assert_eq!(whole.len(), 1_600);
+ assert_eq!(pieces, whole);
+ }
+
+ #[test]
+ fn resampling_at_16k_is_the_identity_and_lower_rates_are_held() {
+ let input = [0.1, 0.2, 0.3];
+ assert_eq!(Resampler::new(16_000.0).process(&input), input.to_vec());
+ assert_eq!(Resampler::new(8_000.0).process(&[0.1, 0.2]), vec![0.1, 0.1, 0.2, 0.2]);
+ }
+}
diff --git a/apps/desktop/src-tauri/src/voice/fake.rs b/apps/desktop/src-tauri/src/voice/fake.rs
new file mode 100644
index 0000000..a218f2a
--- /dev/null
+++ b/apps/desktop/src-tauri/src/voice/fake.rs
@@ -0,0 +1,247 @@
+//! Fakes for the voice tests: a microphone and a recognizer that record what the machine asks
+//! of them and let a test play the platform's side (audio, results, failures).
+
+use super::{Authorization, Events, Microphone, Recognition, Recognizer, Sink, Stream, VoiceError, VoiceEvent};
+use crate::computer::Clock;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Condvar, Mutex};
+
+/// Blocks a permission prompt until the test answers it.
+#[derive(Default)]
+pub struct Gate {
+ open: Mutex<bool>,
+ changed: Condvar,
+}
+
+impl Gate {
+ pub fn release(&self) {
+ *self.open.lock().unwrap() = true;
+ self.changed.notify_all();
+ }
+
+ fn wait(&self) {
+ let mut open = self.open.lock().unwrap();
+ while !*open {
+ open = self.changed.wait(open).unwrap();
+ }
+ }
+}
+
+#[derive(Default)]
+pub struct MicLog {
+ pub requests: usize,
+ pub opened: usize,
+ pub closed: usize,
+ pub sink: Option<Sink>,
+}
+
+pub struct FakeMicrophone {
+ pub authorization: Mutex<Authorization>,
+ /// What a prompt answers.
+ pub answer: Mutex<Authorization>,
+ pub gate: Option<Arc<Gate>>,
+ pub open_error: Mutex<Option<VoiceError>>,
+ pub log: Arc<Mutex<MicLog>>,
+}
+
+impl FakeMicrophone {
+ pub fn granted() -> Self {
+ FakeMicrophone {
+ authorization: Mutex::new(Authorization::Authorized),
+ answer: Mutex::new(Authorization::Authorized),
+ gate: None,
+ open_error: Mutex::new(None),
+ log: Arc::default(),
+ }
+ }
+
+ pub fn with(authorization: Authorization, answer: Authorization) -> Self {
+ let mic = FakeMicrophone::granted();
+ *mic.authorization.lock().unwrap() = authorization;
+ *mic.answer.lock().unwrap() = answer;
+ mic
+ }
+
+ /// The sink of the latest capture, to play the audio thread.
+ pub fn sink(&self) -> Sink {
+ self.log.lock().unwrap().sink.clone().expect("the microphone was opened")
+ }
+}
+
+impl Microphone for FakeMicrophone {
+ fn authorization(&self) -> Authorization {
+ *self.authorization.lock().unwrap()
+ }
+
+ fn request_authorization(&self) -> Authorization {
+ self.log.lock().unwrap().requests += 1;
+ if let Some(gate) = &self.gate {
+ gate.wait();
+ }
+ let answer = *self.answer.lock().unwrap();
+ *self.authorization.lock().unwrap() = answer;
+ answer
+ }
+
+ fn open(&self, sink: Sink) -> Result<Box<dyn Stream>, VoiceError> {
+ if let Some(error) = self.open_error.lock().unwrap().clone() {
+ return Err(error);
+ }
+ let mut log = self.log.lock().unwrap();
+ log.opened += 1;
+ log.sink = Some(sink);
+ Ok(Box::new(FakeStream { log: self.log.clone() }))
+ }
+}
+
+struct FakeStream {
+ log: Arc<Mutex<MicLog>>,
+}
+
+impl Stream for FakeStream {
+ fn close(&mut self) {
+ self.log.lock().unwrap().closed += 1;
+ }
+}
+
+#[derive(Default)]
+pub struct RecognitionLog {
+ pub requests: usize,
+ pub locales: Vec<String>,
+ pub fed: Vec<f32>,
+ pub finished: usize,
+ pub cancelled: usize,
+ pub sink: Option<Sink>,
+}
+
+pub struct FakeRecognizer {
+ pub authorization: Mutex<Authorization>,
+ pub answer: Mutex<Authorization>,
+ pub current: String,
+ pub supported: Vec<String>,
+ /// Delivered through the sink when the recognition is finished; none plays a recognizer
+ /// that never answers.
+ pub final_text: Mutex<Option<String>>,
+ /// Reported as a failure when the recognition is finished.
+ pub fail_on_finish: Mutex<Option<String>>,
+ pub log: Arc<Mutex<RecognitionLog>>,
+}
+
+impl FakeRecognizer {
+ pub fn granted() -> Self {
+ FakeRecognizer {
+ authorization: Mutex::new(Authorization::Authorized),
+ answer: Mutex::new(Authorization::Authorized),
+ current: "en_US".into(),
+ supported: vec!["en-US".into(), "es-ES".into(), "es-MX".into()],
+ final_text: Mutex::new(None),
+ fail_on_finish: Mutex::new(None),
+ log: Arc::default(),
+ }
+ }
+
+ pub fn with(authorization: Authorization, answer: Authorization) -> Self {
+ let recognizer = FakeRecognizer::granted();
+ *recognizer.authorization.lock().unwrap() = authorization;
+ *recognizer.answer.lock().unwrap() = answer;
+ recognizer
+ }
+
+ pub fn sink(&self) -> Sink {
+ self.log.lock().unwrap().sink.clone().expect("a recognition began")
+ }
+}
+
+impl Recognizer for FakeRecognizer {
+ fn authorization(&self) -> Authorization {
+ *self.authorization.lock().unwrap()
+ }
+
+ fn request_authorization(&self) -> Authorization {
+ self.log.lock().unwrap().requests += 1;
+ let answer = *self.answer.lock().unwrap();
+ *self.authorization.lock().unwrap() = answer;
+ answer
+ }
+
+ fn current_locale(&self) -> String {
+ self.current.clone()
+ }
+
+ fn supported_locales(&self) -> Vec<String> {
+ self.supported.clone()
+ }
+
+ fn begin(&self, locale: &str, sink: Sink) -> Result<Box<dyn Recognition>, VoiceError> {
+ let mut log = self.log.lock().unwrap();
+ log.locales.push(locale.into());
+ log.sink = Some(sink.clone());
+ let final_text = self.final_text.lock().unwrap().clone();
+ let failure = self.fail_on_finish.lock().unwrap().clone();
+ Ok(Box::new(FakeRecognition { log: self.log.clone(), sink, final_text, failure }))
+ }
+}
+
+struct FakeRecognition {
+ log: Arc<Mutex<RecognitionLog>>,
+ sink: Sink,
+ final_text: Option<String>,
+ failure: Option<String>,
+}
+
+impl Recognition for FakeRecognition {
+ fn feed(&mut self, samples: &[f32]) {
+ self.log.lock().unwrap().fed.extend_from_slice(samples);
+ }
+
+ fn finish(&mut self) {
+ self.log.lock().unwrap().finished += 1;
+ if let Some(text) = &self.final_text {
+ self.sink.finished(text);
+ }
+ if let Some(message) = &self.failure {
+ self.sink.failed(message);
+ }
+ }
+
+ fn cancel(&mut self) {
+ self.log.lock().unwrap().cancelled += 1;
+ }
+}
+
+/// Collects the events a webview would receive.
+#[derive(Default, Clone)]
+pub struct Recorder(pub Arc<Mutex<Vec<VoiceEvent>>>);
+
+impl Recorder {
+ pub fn events(&self) -> Events {
+ let seen = self.0.clone();
+ Arc::new(move |event| seen.lock().unwrap().push(event))
+ }
+
+ pub fn seen(&self) -> Vec<VoiceEvent> {
+ self.0.lock().unwrap().clone()
+ }
+
+ pub fn levels(&self) -> usize {
+ self.seen().iter().filter(|event| matches!(event, VoiceEvent::Level(_))).count()
+ }
+}
+
+pub struct FakeClock(pub AtomicU64);
+
+impl FakeClock {
+ pub fn advance(&self, ms: u64) {
+ self.0.fetch_add(ms, Ordering::SeqCst);
+ }
+}
+
+impl Clock for FakeClock {
+ fn now_ms(&self) -> u64 {
+ self.0.load(Ordering::SeqCst)
+ }
+
+ fn sleep_ms(&self, ms: u64) {
+ self.advance(ms);
+ }
+}
diff --git a/apps/desktop/src-tauri/src/voice/locale.rs b/apps/desktop/src-tauri/src/voice/locale.rs
new file mode 100644
index 0000000..4915dfc
--- /dev/null
+++ b/apps/desktop/src-tauri/src/voice/locale.rs
@@ -0,0 +1,116 @@
+//! Which locale the recognizer gets: the requested one, else a variant of its language, else
+//! the user's locale (or a variant of its language). The UI may pass a browser tag such as
+//! `es-419`, and the system may report `es_AR`, which Speech does not list.
+
+use super::VoiceError;
+
+/// The variant used when a language's own region is not supported.
+const MAIN_VARIANTS: &[(&str, &str)] = &[("en", "en-US"), ("pt", "pt-BR"), ("zh", "zh-CN"), ("ar", "ar-SA")];
+
+/// `es_AR`, `en_US@rg=gbzzzz`, or ` pt-br ` → `es-AR`, `en-US`, `pt-BR`.
+pub fn normalize(id: &str) -> String {
+ let id = id.trim();
+ let id = id.split('@').next().unwrap_or_default();
+ let mut parts = id.split(['_', '-']).filter(|part| !part.is_empty());
+ let Some(language) = parts.next() else { return String::new() };
+ let mut out = language.to_ascii_lowercase();
+ for part in parts {
+ out.push('-');
+ match part.len() {
+ 2 | 3 if part.chars().all(|c| c.is_ascii_alphabetic()) => out.push_str(&part.to_ascii_uppercase()),
+ 4 => {
+ let mut chars = part.chars();
+ out.extend(chars.next().map(|c| c.to_ascii_uppercase()));
+ out.push_str(&chars.as_str().to_ascii_lowercase());
+ }
+ _ => out.push_str(part),
+ }
+ }
+ out
+}
+
+fn language(tag: &str) -> &str {
+ tag.split('-').next().unwrap_or_default()
+}
+
+/// The supported entry for `tag`, as the recognizer spells it.
+fn pick(tag: &str, current: &str, supported: &[(String, &String)]) -> Option<String> {
+ let find = |wanted: &str| supported.iter().find(|(normal, _)| normal == wanted).map(|(_, id)| (*id).clone());
+ if tag.is_empty() {
+ return None;
+ }
+ if let Some(found) = find(tag) {
+ return Some(found);
+ }
+ let lang = language(tag);
+ if language(current) == lang {
+ if let Some(found) = find(current) {
+ return Some(found);
+ }
+ }
+ let main = MAIN_VARIANTS.iter().find(|(l, _)| *l == lang).map(|(_, v)| v.to_string());
+ let main = main.unwrap_or_else(|| format!("{lang}-{}", lang.to_ascii_uppercase()));
+ if let Some(found) = find(&main) {
+ return Some(found);
+ }
+ let mut same: Vec<_> = supported.iter().filter(|(normal, _)| language(normal) == lang).collect();
+ same.sort();
+ same.first().map(|(_, id)| (*id).clone())
+}
+
+pub fn resolve(requested: Option<&str>, current: &str, supported: &[String]) -> Result<String, VoiceError> {
+ let normalized: Vec<(String, &String)> = supported.iter().map(|id| (normalize(id), id)).collect();
+ let requested = requested.map(normalize).unwrap_or_default();
+ let current = normalize(current);
+ pick(&requested, &current, &normalized)
+ .or_else(|| pick(&current, &current, &normalized))
+ .ok_or(VoiceError::NoRecognizer(if requested.is_empty() { current } else { requested }))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::voice::VoiceError;
+
+ fn supported() -> Vec<String> {
+ ["en-AU", "en-GB", "en-US", "es-CL", "es-ES", "es-MX", "pt-BR", "pt-PT"].map(String::from).to_vec()
+ }
+
+ #[test]
+ fn identifiers_normalize_to_bcp47() {
+ assert_eq!(normalize("es_AR"), "es-AR");
+ assert_eq!(normalize("en_US@rg=gbzzzz"), "en-US");
+ assert_eq!(normalize(" pt-br "), "pt-BR");
+ assert_eq!(normalize("zh-Hant_TW"), "zh-Hant-TW");
+ }
+
+ #[test]
+ fn an_exact_match_wins() {
+ assert_eq!(resolve(Some("es_MX"), "en_US", &supported()), Ok("es-MX".into()));
+ assert_eq!(resolve(None, "en_GB", &supported()), Ok("en-GB".into()));
+ }
+
+ #[test]
+ fn a_missing_region_falls_back_to_the_main_variant_of_the_language() {
+ assert_eq!(resolve(Some("es-AR"), "en_US", &supported()), Ok("es-ES".into()));
+ assert_eq!(resolve(None, "en_AR", &supported()), Ok("en-US".into()));
+ assert_eq!(resolve(Some("pt"), "en_US", &supported()), Ok("pt-BR".into()));
+ }
+
+ #[test]
+ fn the_current_region_is_preferred_within_the_requested_language() {
+ assert_eq!(resolve(Some("es"), "es_CL", &supported()), Ok("es-CL".into()));
+ }
+
+ #[test]
+ fn an_unsupported_language_falls_back_to_the_current_locale() {
+ assert_eq!(resolve(Some("ja-JP"), "es_MX", &supported()), Ok("es-MX".into()));
+ assert_eq!(resolve(Some(""), "pt_PT", &supported()), Ok("pt-PT".into()));
+ }
+
+ #[test]
+ fn nothing_supported_is_an_error_naming_the_locale() {
+ assert_eq!(resolve(None, "ja_JP", &supported()), Err(VoiceError::NoRecognizer("ja-JP".into())));
+ assert_eq!(resolve(Some("fr-FR"), "ja_JP", &[]), Err(VoiceError::NoRecognizer("fr-FR".into())));
+ }
+}
diff --git a/apps/desktop/src-tauri/src/voice/machine.rs b/apps/desktop/src-tauri/src/voice/machine.rs
new file mode 100644
index 0000000..1c3b8ec
--- /dev/null
+++ b/apps/desktop/src-tauri/src/voice/machine.rs
@@ -0,0 +1,657 @@
+//! The voice state machine: one recording at a time, between a `Microphone` and a `Recognizer`.
+//!
+//! Idle → Starting (permissions, locale, recognition, capture) → Recording → Stopping → Idle.
+//! Cancel works in every state. Platform callbacks arrive on arbitrary threads through a
+//! `Sink` bound to one recording, so results from an earlier recording are ignored. No
+//! platform call or event is made while the state lock is held.
+
+use super::{audio, locale, Authorization, Events, Microphone, Recognition, Recognizer, Stream, Transcript};
+use super::{VoiceError, VoiceEvent, VoiceStatus};
+use crate::computer::Clock;
+use std::sync::{Arc, Condvar, Mutex, MutexGuard, Weak};
+use std::time::{Duration, Instant};
+
+/// About 15 levels a second.
+pub const LEVEL_INTERVAL_MS: u64 = 1_000 / 15;
+/// How long `stop` waits for the final transcript before using the latest partial.
+pub const FINAL_TIMEOUT: Duration = Duration::from_secs(3);
+
+pub struct Voice {
+ inner: Arc<Inner>,
+ final_timeout: Duration,
+}
+
+struct Inner {
+ microphone: Arc<dyn Microphone>,
+ recognizer: Arc<dyn Recognizer>,
+ clock: Arc<dyn Clock>,
+ state: Mutex<State>,
+ changed: Condvar,
+}
+
+enum State {
+ Idle { next: u64 },
+ Starting { recording: u64, cancelled: bool, failure: Option<String> },
+ Recording(Session),
+ Stopping(Session),
+}
+
+struct Session {
+ recording: u64,
+ events: Events,
+ stream: Option<Box<dyn Stream>>,
+ feed: Arc<Mutex<Feed>>,
+ transcript: String,
+ final_text: Option<String>,
+ last_level_ms: Option<u64>,
+ /// The loudest chunk since the last level.
+ peak: f32,
+}
+
+/// The recognition, closed once finished or cancelled so no audio follows `finish`.
+struct Feed {
+ recognition: Box<dyn Recognition>,
+ open: bool,
+}
+
+impl Feed {
+ fn push(&mut self, samples: &[f32]) {
+ if self.open {
+ self.recognition.feed(samples);
+ }
+ }
+
+ fn finish(&mut self) {
+ if self.open {
+ self.open = false;
+ self.recognition.finish();
+ }
+ }
+
+ fn cancel(&mut self) {
+ self.open = false;
+ self.recognition.cancel();
+ }
+}
+
+impl State {
+ fn recording(&self) -> Option<u64> {
+ match self {
+ State::Idle { .. } => None,
+ State::Starting { recording, .. } => Some(*recording),
+ State::Recording(session) | State::Stopping(session) => Some(session.recording),
+ }
+ }
+
+ fn idle(&self) -> State {
+ State::Idle { next: self.recording().map_or(0, |n| n + 1) }
+ }
+}
+
+/// Tears a session down outside the state lock.
+fn discard(session: Session) {
+ let Session { mut stream, feed, .. } = session;
+ if let Some(stream) = stream.as_mut() {
+ stream.close();
+ }
+ lock(&feed).cancel();
+}
+
+fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
+ mutex.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
+}
+
+fn authorize(
+ current: Authorization,
+ request: impl FnOnce() -> Authorization,
+ denied: VoiceError,
+ restricted: VoiceError,
+) -> Result<(), VoiceError> {
+ let answer = if current == Authorization::NotDetermined { request() } else { current };
+ match answer {
+ Authorization::Authorized => Ok(()),
+ Authorization::Restricted => Err(restricted),
+ Authorization::Denied | Authorization::NotDetermined => Err(denied),
+ }
+}
+
+/// The reason a permission blocks voice input, without prompting.
+fn blocked(current: Authorization, denied: VoiceError, restricted: VoiceError) -> Option<VoiceError> {
+ match current {
+ Authorization::Denied => Some(denied),
+ Authorization::Restricted => Some(restricted),
+ Authorization::NotDetermined | Authorization::Authorized => None,
+ }
+}
+
+impl Voice {
+ pub fn new(microphone: Arc<dyn Microphone>, recognizer: Arc<dyn Recognizer>, clock: Arc<dyn Clock>) -> Self {
+ let inner = Inner { microphone, recognizer, clock, state: Mutex::new(State::Idle { next: 0 }), changed: Condvar::new() };
+ Voice { inner: Arc::new(inner), final_timeout: FINAL_TIMEOUT }
+ }
+
+ pub fn with_final_timeout(mut self, timeout: Duration) -> Self {
+ self.final_timeout = timeout;
+ self
+ }
+
+ /// Whether voice input can work, without prompting: a permission the user denied, or no
+ /// recognizer for the user's locale, makes it unavailable. Unasked permissions are asked
+ /// on the first `start`.
+ pub fn status(&self) -> VoiceStatus {
+ let (recognizer, microphone) = (&self.inner.recognizer, &self.inner.microphone);
+ let reason = blocked(recognizer.authorization(), VoiceError::SpeechDenied, VoiceError::SpeechRestricted)
+ .or_else(|| blocked(microphone.authorization(), VoiceError::MicrophoneDenied, VoiceError::MicrophoneRestricted))
+ .or_else(|| locale::resolve(None, &recognizer.current_locale(), &recognizer.supported_locales()).err());
+ match reason {
+ Some(reason) => VoiceStatus::unavailable(reason.to_string()),
+ None => VoiceStatus::available(),
+ }
+ }
+
+ /// Starts recording, prompting for permissions first when they were never asked. Blocks
+ /// while a prompt is open, so it runs off the main thread. `events` reach the caller.
+ pub fn start(&self, requested: Option<&str>, events: Events) -> Result<(), VoiceError> {
+ let recording = {
+ let mut state = lock(&self.inner.state);
+ let State::Idle { next } = *state else { return Err(VoiceError::Busy) };
+ *state = State::Starting { recording: next, cancelled: false, failure: None };
+ next
+ };
+ let sink = Sink { inner: Arc::downgrade(&self.inner), recording };
+ let begun = self.prepare(requested, recording).and_then(|locale| {
+ let mut recognition = self.inner.recognizer.begin(&locale, sink.clone())?;
+ match self.inner.microphone.open(sink) {
+ Ok(stream) => Ok((stream, recognition)),
+ Err(error) => {
+ recognition.cancel();
+ Err(error)
+ }
+ }
+ });
+ let mut state = lock(&self.inner.state);
+ let State::Starting { cancelled, failure, .. } = std::mem::replace(&mut *state, State::Idle { next: recording + 1 }) else {
+ unreachable!("only start leaves Starting");
+ };
+ let (stream, recognition) = begun?;
+ let feed = Arc::new(Mutex::new(Feed { recognition, open: true }));
+ let session = Session {
+ recording,
+ events,
+ stream: Some(stream),
+ feed,
+ transcript: String::new(),
+ final_text: None,
+ last_level_ms: None,
+ peak: 0.0,
+ };
+ let refused = if cancelled { Some(VoiceError::Cancelled) } else { failure.map(VoiceError::Failed) };
+ if let Some(error) = refused {
+ drop(state);
+ discard(session);
+ return Err(error);
+ }
+ *state = State::Recording(session);
+ Ok(())
+ }
+
+ /// Locale first (no point prompting without a recognizer), then Speech, then Microphone.
+ fn prepare(&self, requested: Option<&str>, recording: u64) -> Result<String, VoiceError> {
+ let (recognizer, microphone) = (&self.inner.recognizer, &self.inner.microphone);
+ let locale = locale::resolve(requested, &recognizer.current_locale(), &recognizer.supported_locales())?;
+ authorize(recognizer.authorization(), || recognizer.request_authorization(), VoiceError::SpeechDenied, VoiceError::SpeechRestricted)?;
+ self.still_starting(recording)?;
+ authorize(microphone.authorization(), || microphone.request_authorization(), VoiceError::MicrophoneDenied, VoiceError::MicrophoneRestricted)?;
+ self.still_starting(recording)?;
+ Ok(locale)
+ }
+
+ fn still_starting(&self, recording: u64) -> Result<(), VoiceError> {
+ match *lock(&self.inner.state) {
+ State::Starting { recording: r, cancelled: false, .. } if r == recording => Ok(()),
+ _ => Err(VoiceError::Cancelled),
+ }
+ }
+
+ /// Stops the microphone and resolves with the final transcript, or the latest partial when
+ /// the recognizer does not answer within the timeout. Blocks, so it runs off the main thread.
+ pub fn stop(&self) -> Result<Transcript, VoiceError> {
+ let (recording, mut stream, feed) = {
+ let mut state = lock(&self.inner.state);
+ match std::mem::replace(&mut *state, State::Idle { next: 0 }) {
+ State::Recording(mut session) => {
+ let taken = (session.recording, session.stream.take(), session.feed.clone());
+ *state = State::Stopping(session);
+ taken
+ }
+ State::Starting { recording, failure, .. } => {
+ // Stopping before the capture is up is a cancel with nothing said.
+ *state = State::Starting { recording, cancelled: true, failure };
+ return Ok(Transcript { text: String::new() });
+ }
+ other => {
+ let error = if matches!(other, State::Stopping(_)) { VoiceError::Stopping } else { VoiceError::NotRecording };
+ *state = other;
+ return Err(error);
+ }
+ }
+ };
+ if let Some(stream) = stream.as_mut() {
+ stream.close();
+ }
+ lock(&feed).finish();
+
+ let deadline = Instant::now() + self.final_timeout;
+ let mut state = lock(&self.inner.state);
+ loop {
+ let State::Stopping(session) = &*state else { return Err(VoiceError::Cancelled) };
+ if session.recording != recording {
+ return Err(VoiceError::Cancelled);
+ }
+ let now = Instant::now();
+ if session.final_text.is_some() || now >= deadline {
+ break;
+ }
+ state = self.inner.changed.wait_timeout(state, deadline - now).map(|(s, _)| s).unwrap_or_else(|p| p.into_inner().0);
+ }
+ let idle = state.idle();
+ let State::Stopping(session) = std::mem::replace(&mut *state, idle) else { unreachable!() };
+ drop(state);
+ let answered = session.final_text.is_some();
+ let text = session.final_text.unwrap_or(session.transcript).trim().to_string();
+ if !answered {
+ lock(&feed).cancel();
+ }
+ Ok(Transcript { text })
+ }
+
+ /// Stops and discards. Harmless when idle; during a permission prompt the start is aborted.
+ pub fn cancel(&self) -> Result<(), VoiceError> {
+ let mut state = lock(&self.inner.state);
+ match &mut *state {
+ State::Idle { .. } => Ok(()),
+ State::Starting { cancelled, .. } => {
+ *cancelled = true;
+ Ok(())
+ }
+ State::Recording(_) | State::Stopping(_) => {
+ let idle = state.idle();
+ let (State::Recording(session) | State::Stopping(session)) = std::mem::replace(&mut *state, idle) else {
+ unreachable!()
+ };
+ drop(state);
+ self.inner.changed.notify_all();
+ discard(session);
+ Ok(())
+ }
+ }
+ }
+}
+
+/// The platform's way back into one recording: audio from the microphone, results and failures
+/// from the recognizer. Cheap to clone; calls for an earlier recording are ignored.
+#[derive(Clone)]
+pub struct Sink {
+ inner: Weak<Inner>,
+ recording: u64,
+}
+
+impl Sink {
+ /// A chunk of `audio::SAMPLE_RATE` mono PCM: fed to the recognizer, and metered.
+ pub fn audio(&self, samples: &[f32]) {
+ let Some(inner) = self.inner.upgrade() else { return };
+ let (feed, level) = {
+ let mut state = lock(&inner.state);
+ let State::Recording(session) = &mut *state else { return };
+ if session.recording != self.recording {
+ return;
+ }
+ session.peak = session.peak.max(audio::rms(samples));
+ let now = inner.clock.now_ms();
+ let due = session.last_level_ms.is_none_or(|last| now.saturating_sub(last) >= LEVEL_INTERVAL_MS);
+ let level = due.then(|| {
+ session.last_level_ms = Some(now);
+ let level = audio::level(std::mem::take(&mut session.peak));
+ (session.events.clone(), VoiceEvent::Level(level))
+ });
+ (session.feed.clone(), level)
+ };
+ if let Some((events, level)) = level {
+ events(level);
+ }
+ lock(&feed).push(samples);
+ }
+
+ /// The text so far.
+ pub fn partial(&self, text: &str) {
+ self.update(|session| {
+ session.transcript = text.to_string();
+ Some(VoiceEvent::Partial(text.to_string()))
+ });
+ }
+
+ /// The final text: it resolves `stop`, or waits for it.
+ pub fn finished(&self, text: &str) {
+ self.update(|session| {
+ session.transcript = text.to_string();
+ session.final_text = Some(text.to_string());
+ None
+ });
+ }
+
+ /// A failure. While recording it is reported and ends the recording; while stopping, `stop`
+ /// resolves with the text so far (a recognizer that heard nothing reports an error).
+ pub fn failed(&self, message: &str) {
+ let Some(inner) = self.inner.upgrade() else { return };
+ let mut state = lock(&inner.state);
+ match &mut *state {
+ State::Starting { recording, failure, .. } if *recording == self.recording => {
+ failure.get_or_insert_with(|| message.to_string());
+ }
+ State::Stopping(session) if session.recording == self.recording => {
+ if session.final_text.is_none() {
+ session.final_text = Some(session.transcript.clone());
+ }
+ drop(state);
+ inner.changed.notify_all();
+ }
+ State::Recording(session) if session.recording == self.recording => {
+ let idle = state.idle();
+ let State::Recording(session) = std::mem::replace(&mut *state, idle) else { unreachable!() };
+ drop(state);
+ inner.changed.notify_all();
+ (session.events)(VoiceEvent::Error(message.to_string()));
+ discard(session);
+ }
+ _ => {}
+ }
+ }
+
+ fn update(&self, change: impl FnOnce(&mut Session) -> Option<VoiceEvent>) {
+ let Some(inner) = self.inner.upgrade() else { return };
+ let (events, event) = {
+ let mut state = lock(&inner.state);
+ let (State::Recording(session) | State::Stopping(session)) = &mut *state else { return };
+ if session.recording != self.recording {
+ return;
+ }
+ (session.events.clone(), change(session))
+ };
+ inner.changed.notify_all();
+ if let Some(event) = event {
+ events(event);
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::voice::fake::{FakeClock, FakeMicrophone, FakeRecognizer, Gate, Recorder};
+ use crate::voice::{Authorization, Transcript, VoiceError, VoiceEvent, VoiceStatus};
+ use std::sync::atomic::AtomicU64;
+ use std::sync::Arc;
+ use std::time::Duration;
+
+ struct Rig {
+ voice: Voice,
+ mic: Arc<FakeMicrophone>,
+ recognizer: Arc<FakeRecognizer>,
+ clock: Arc<FakeClock>,
+ recorder: Recorder,
+ }
+
+ fn rig(mic: FakeMicrophone, recognizer: FakeRecognizer) -> Rig {
+ let (mic, recognizer) = (Arc::new(mic), Arc::new(recognizer));
+ let clock = Arc::new(FakeClock(AtomicU64::new(1_000)));
+ let voice = Voice::new(mic.clone(), recognizer.clone(), clock.clone()).with_final_timeout(Duration::from_millis(50));
+ Rig { voice, mic, recognizer, clock, recorder: Recorder::default() }
+ }
+
+ fn granted() -> Rig {
+ rig(FakeMicrophone::granted(), FakeRecognizer::granted())
+ }
+
+ impl Rig {
+ fn start(&self) -> Result<(), VoiceError> {
+ self.voice.start(None, self.recorder.events())
+ }
+ }
+
+ #[test]
+ fn status_is_available_when_permissions_are_granted_or_not_asked_yet() {
+ assert_eq!(granted().voice.status(), VoiceStatus::available());
+ let unasked = rig(
+ FakeMicrophone::with(Authorization::NotDetermined, Authorization::Authorized),
+ FakeRecognizer::with(Authorization::NotDetermined, Authorization::Authorized),
+ );
+ assert_eq!(unasked.voice.status(), VoiceStatus::available());
+ assert_eq!(unasked.mic.log.lock().unwrap().requests + unasked.recognizer.log.lock().unwrap().requests, 0);
+ }
+
+ #[test]
+ fn status_explains_a_denied_or_restricted_permission() {
+ let speech = rig(FakeMicrophone::granted(), FakeRecognizer::with(Authorization::Denied, Authorization::Denied));
+ assert_eq!(speech.voice.status(), VoiceStatus::unavailable(VoiceError::SpeechDenied.to_string()));
+ let mic = rig(FakeMicrophone::with(Authorization::Restricted, Authorization::Restricted), FakeRecognizer::granted());
+ assert_eq!(mic.voice.status(), VoiceStatus::unavailable(VoiceError::MicrophoneRestricted.to_string()));
+ }
+
+ #[test]
+ fn status_explains_a_missing_recognizer_for_the_current_locale() {
+ let mut recognizer = FakeRecognizer::granted();
+ recognizer.current = "ja_JP".into();
+ recognizer.supported = vec![];
+ let rig = rig(FakeMicrophone::granted(), recognizer);
+ assert_eq!(rig.voice.status(), VoiceStatus::unavailable("Speech recognition is not available for ja-JP."));
+ }
+
+ #[test]
+ fn start_prompts_for_permissions_on_first_use_then_records() {
+ let rig = rig(
+ FakeMicrophone::with(Authorization::NotDetermined, Authorization::Authorized),
+ FakeRecognizer::with(Authorization::NotDetermined, Authorization::Authorized),
+ );
+ rig.start().unwrap();
+ assert_eq!(rig.recognizer.log.lock().unwrap().requests, 1);
+ assert_eq!(rig.mic.log.lock().unwrap().requests, 1);
+ assert_eq!(rig.mic.log.lock().unwrap().opened, 1);
+ assert_eq!(rig.recognizer.log.lock().unwrap().locales, vec!["en-US".to_string()]);
+ }
+
+ #[test]
+ fn a_denied_prompt_fails_the_start_and_frees_the_slot() {
+ let rig = rig(
+ FakeMicrophone::with(Authorization::NotDetermined, Authorization::Denied),
+ FakeRecognizer::granted(),
+ );
+ assert_eq!(rig.start(), Err(VoiceError::MicrophoneDenied));
+ assert_eq!(rig.mic.log.lock().unwrap().opened, 0);
+ *rig.mic.answer.lock().unwrap() = Authorization::Authorized;
+ *rig.mic.authorization.lock().unwrap() = Authorization::NotDetermined;
+ assert_eq!(rig.start(), Ok(()));
+ }
+
+ #[test]
+ fn the_requested_locale_reaches_the_recognizer() {
+ let rig = granted();
+ rig.voice.start(Some("es_AR"), rig.recorder.events()).unwrap();
+ assert_eq!(rig.recognizer.log.lock().unwrap().locales, vec!["es-ES".to_string()]);
+ }
+
+ #[test]
+ fn only_one_recording_at_a_time() {
+ let rig = granted();
+ rig.start().unwrap();
+ assert_eq!(rig.start(), Err(VoiceError::Busy));
+ assert_eq!(rig.mic.log.lock().unwrap().opened, 1);
+ }
+
+ #[test]
+ fn partial_text_is_sent_while_recording() {
+ let rig = granted();
+ rig.start().unwrap();
+ rig.recognizer.sink().partial("hello");
+ rig.recognizer.sink().partial("hello there");
+ assert_eq!(
+ rig.recorder.seen(),
+ vec![VoiceEvent::Partial("hello".into()), VoiceEvent::Partial("hello there".into())]
+ );
+ }
+
+ #[test]
+ fn stop_closes_the_microphone_and_resolves_with_the_final_transcript() {
+ let rig = granted();
+ *rig.recognizer.final_text.lock().unwrap() = Some(" Hello there. ".into());
+ rig.start().unwrap();
+ rig.recognizer.sink().partial("hello");
+ assert_eq!(rig.voice.stop(), Ok(Transcript { text: "Hello there.".into() }));
+ assert_eq!(rig.mic.log.lock().unwrap().closed, 1);
+ assert_eq!(rig.recognizer.log.lock().unwrap().finished, 1);
+ rig.start().unwrap();
+ }
+
+ #[test]
+ fn stop_falls_back_to_the_latest_partial_when_no_final_arrives() {
+ let rig = granted();
+ rig.start().unwrap();
+ rig.recognizer.sink().partial("almost done");
+ assert_eq!(rig.voice.stop(), Ok(Transcript { text: "almost done".into() }));
+ assert_eq!(rig.recognizer.log.lock().unwrap().cancelled, 1);
+ }
+
+ #[test]
+ fn stop_without_a_recording_is_an_error() {
+ assert_eq!(granted().voice.stop(), Err(VoiceError::NotRecording));
+ }
+
+ #[test]
+ fn cancel_discards_the_recording_and_ignores_its_late_results() {
+ let rig = granted();
+ rig.start().unwrap();
+ let old = rig.recognizer.sink();
+ assert_eq!(rig.voice.cancel(), Ok(()));
+ assert_eq!(rig.mic.log.lock().unwrap().closed, 1);
+ assert_eq!(rig.recognizer.log.lock().unwrap().cancelled, 1);
+ rig.start().unwrap();
+ old.partial("from before");
+ old.failed("canceled");
+ assert!(rig.recorder.seen().is_empty());
+ assert_eq!(rig.voice.cancel(), Ok(()));
+ assert_eq!(rig.voice.cancel(), Ok(()), "cancelling when idle is harmless");
+ }
+
+ #[test]
+ fn cancel_during_a_permission_prompt_aborts_the_start() {
+ let gate = Arc::new(Gate::default());
+ let mut mic = FakeMicrophone::with(Authorization::NotDetermined, Authorization::Authorized);
+ mic.gate = Some(gate.clone());
+ let rig = Arc::new(rig(mic, FakeRecognizer::granted()));
+ let starting = {
+ let rig = rig.clone();
+ std::thread::spawn(move || rig.start())
+ };
+ while rig.mic.log.lock().unwrap().requests == 0 {
+ std::thread::yield_now();
+ }
+ assert_eq!(rig.start(), Err(VoiceError::Busy));
+ assert_eq!(rig.voice.cancel(), Ok(()));
+ gate.release();
+ assert_eq!(starting.join().unwrap(), Err(VoiceError::Cancelled));
+ assert_eq!(rig.mic.log.lock().unwrap().opened, 0);
+ rig.start().unwrap();
+ }
+
+ #[test]
+ fn audio_reaches_the_recognizer_and_levels_are_throttled() {
+ let rig = granted();
+ rig.start().unwrap();
+ let sink = rig.mic.sink();
+ for _ in 0..10 {
+ sink.audio(&[0.5; 160]);
+ rig.clock.advance(10);
+ }
+ assert_eq!(rig.recognizer.log.lock().unwrap().fed.len(), 1_600);
+ assert_eq!(rig.recorder.levels(), 2, "one level now, the next after 1/15 s");
+ rig.clock.advance(LEVEL_INTERVAL_MS);
+ sink.audio(&[0.0; 160]);
+ let last = rig.recorder.seen().into_iter().rev().find_map(|event| match event {
+ VoiceEvent::Level(level) => Some(level),
+ _ => None,
+ });
+ assert!(last.unwrap() > 0.5, "the meter shows the loudest chunk since the last level");
+ }
+
+ #[test]
+ fn no_audio_reaches_the_recognizer_after_stop() {
+ let rig = granted();
+ rig.start().unwrap();
+ let sink = rig.mic.sink();
+ rig.voice.stop().unwrap();
+ sink.audio(&[0.5; 160]);
+ assert!(rig.recognizer.log.lock().unwrap().fed.is_empty());
+ }
+
+ #[test]
+ fn a_failure_while_recording_is_reported_and_ends_the_recording() {
+ let rig = granted();
+ rig.start().unwrap();
+ rig.mic.sink().failed(&VoiceError::Interrupted.to_string());
+ assert_eq!(rig.recorder.seen(), vec![VoiceEvent::Error("The microphone was interrupted.".into())]);
+ assert_eq!(rig.mic.log.lock().unwrap().closed, 1);
+ assert_eq!(rig.recognizer.log.lock().unwrap().cancelled, 1);
+ assert_eq!(rig.voice.stop(), Err(VoiceError::NotRecording));
+ rig.start().unwrap();
+ }
+
+ #[test]
+ fn a_failure_while_stopping_resolves_with_the_text_so_far() {
+ let rig = granted();
+ *rig.recognizer.fail_on_finish.lock().unwrap() = Some("No speech detected".into());
+ rig.start().unwrap();
+ rig.recognizer.sink().partial("so far");
+ assert_eq!(rig.voice.stop(), Ok(Transcript { text: "so far".into() }));
+ assert!(rig.recorder.seen().iter().all(|event| !matches!(event, VoiceEvent::Error(_))));
+ }
+
+ #[test]
+ fn a_microphone_that_cannot_open_cancels_the_recognition() {
+ let rig = granted();
+ *rig.mic.open_error.lock().unwrap() = Some(VoiceError::NoMicrophone);
+ assert_eq!(rig.start(), Err(VoiceError::NoMicrophone));
+ assert_eq!(rig.recognizer.log.lock().unwrap().cancelled, 1);
+ *rig.mic.open_error.lock().unwrap() = None;
+ rig.start().unwrap();
+ }
+
+ #[test]
+ fn a_failure_reported_while_starting_fails_the_start() {
+ let rig = granted();
+ // The recognizer reports a failure before the capture is up (as Speech can, on its queue).
+ let recognizer = rig.recognizer.clone();
+ let mic = rig.mic.clone();
+ *mic.open_error.lock().unwrap() = None;
+ let voice = Voice::new(Arc::new(FailingOnOpen(mic, recognizer.clone())), recognizer.clone(), rig.clock.clone());
+ assert_eq!(voice.start(None, rig.recorder.events()), Err(VoiceError::Failed("Siri and Dictation are disabled.".into())));
+ assert_eq!(recognizer.log.lock().unwrap().cancelled, 1);
+ assert_eq!(voice.stop(), Err(VoiceError::NotRecording));
+ }
+
+ /// Opens the fake microphone after the recognition reported a failure.
+ struct FailingOnOpen(Arc<FakeMicrophone>, Arc<FakeRecognizer>);
+
+ impl crate::voice::Microphone for FailingOnOpen {
+ fn authorization(&self) -> Authorization {
+ self.0.authorization()
+ }
+
+ fn request_authorization(&self) -> Authorization {
+ self.0.request_authorization()
+ }
+
+ fn open(&self, sink: Sink) -> Result<Box<dyn crate::voice::Stream>, VoiceError> {
+ self.1.sink().failed("Siri and Dictation are disabled.");
+ self.0.open(sink)
+ }
+ }
+}
diff --git a/apps/desktop/src-tauri/src/voice/macos.rs b/apps/desktop/src-tauri/src/voice/macos.rs
new file mode 100644
index 0000000..0a39da2
--- /dev/null
+++ b/apps/desktop/src-tauri/src/voice/macos.rs
@@ -0,0 +1,290 @@
+//! The macOS voice backends: the native microphone (AVAudioEngine's input node, resampled to
+//! 16 kHz mono) and Apple's Speech framework (`SFSpeechRecognizer`, on-device when the locale
+//! supports it, else Apple's servers). Nothing here decides policy; `machine` does.
+//!
+//! Threads: permission handlers, the audio tap, the interruption notification, and recognition
+//! results all arrive off the main thread (the recognizer gets an operation queue of its own),
+//! and they only call the `Sink`. The objects below are used from the command's blocking thread
+//! and from those callbacks; AVAudioEngine and the Speech request are not tied to a thread.
+
+use super::audio::{self, Resampler, SAMPLE_RATE};
+use super::{Authorization, Microphone, Recognition, Recognizer, Sink, Stream, VoiceError};
+use block2::RcBlock;
+use objc2::rc::Retained;
+use objc2::runtime::{AnyObject, Bool, ProtocolObject};
+use objc2::{available, AnyThread};
+use objc2_av_foundation::{AVAuthorizationStatus, AVCaptureDevice, AVMediaTypeAudio};
+use objc2_avf_audio::{
+ AVAudioEngine, AVAudioEngineConfigurationChangeNotification, AVAudioFormat, AVAudioPCMBuffer, AVAudioTime,
+};
+use objc2_foundation::{NSError, NSLocale, NSNotification, NSNotificationCenter, NSObjectProtocol, NSOperationQueue, NSString};
+use objc2_speech::{
+ SFSpeechAudioBufferRecognitionRequest, SFSpeechRecognitionResult, SFSpeechRecognitionTask, SFSpeechRecognizer,
+ SFSpeechRecognizerAuthorizationStatus,
+};
+use std::cell::RefCell;
+use std::ptr::NonNull;
+use std::sync::mpsc;
+
+/// Frames per tap callback at the device rate (about 21 ms at 48 kHz).
+const TAP_FRAMES: u32 = 1_024;
+
+fn microphone_status(status: AVAuthorizationStatus) -> Authorization {
+ match status {
+ AVAuthorizationStatus::Authorized => Authorization::Authorized,
+ AVAuthorizationStatus::Denied => Authorization::Denied,
+ AVAuthorizationStatus::Restricted => Authorization::Restricted,
+ _ => Authorization::NotDetermined,
+ }
+}
+
+fn speech_status(status: SFSpeechRecognizerAuthorizationStatus) -> Authorization {
+ match status {
+ SFSpeechRecognizerAuthorizationStatus::Authorized => Authorization::Authorized,
+ SFSpeechRecognizerAuthorizationStatus::Denied => Authorization::Denied,
+ SFSpeechRecognizerAuthorizationStatus::Restricted => Authorization::Restricted,
+ _ => Authorization::NotDetermined,
+ }
+}
+
+fn describe(error: &NSError) -> String {
+ error.localizedDescription().to_string()
+}
+
+/// The Mac's default input device.
+pub struct MacMicrophone;
+
+impl Microphone for MacMicrophone {
+ fn authorization(&self) -> Authorization {
+ // SAFETY: a framework constant, and audio is a media type this call accepts.
+ match unsafe { AVMediaTypeAudio } {
+ Some(audio) => microphone_status(unsafe { AVCaptureDevice::authorizationStatusForMediaType(audio) }),
+ None => Authorization::Restricted,
+ }
+ }
+
+ fn request_authorization(&self) -> Authorization {
+ // SAFETY: as above.
+ let Some(audio) = (unsafe { AVMediaTypeAudio }) else { return Authorization::Restricted };
+ let (tx, rx) = mpsc::channel();
+ let handler = RcBlock::new(move |granted: Bool| {
+ let _ = tx.send(granted.as_bool());
+ });
+ // SAFETY: the handler matches the documented signature; it runs on an arbitrary queue.
+ unsafe { AVCaptureDevice::requestAccessForMediaType_completionHandler(audio, &handler) };
+ match rx.recv() {
+ Ok(true) => Authorization::Authorized,
+ _ => self.authorization(),
+ }
+ }
+
+ fn open(&self, sink: Sink) -> Result<Box<dyn Stream>, VoiceError> {
+ // SAFETY: plain constructors and getters on a new engine.
+ let engine = unsafe { AVAudioEngine::new() };
+ let input = unsafe { engine.inputNode() };
+ let format = unsafe { input.outputFormatForBus(0) };
+ let (rate, channels) = unsafe { (format.sampleRate(), format.channelCount() as usize) };
+ if rate <= 0.0 || channels == 0 {
+ return Err(VoiceError::NoMicrophone);
+ }
+
+ let resampler = RefCell::new(Resampler::new(rate));
+ let audio_sink = sink.clone();
+ let tap = RcBlock::new(move |buffer: NonNull<AVAudioPCMBuffer>, _when: NonNull<AVAudioTime>| {
+ // SAFETY: the engine passes a valid buffer in the input node's format (deinterleaved
+ // float32 with `channels` channels), valid for the duration of the call.
+ let buffer = unsafe { buffer.as_ref() };
+ let frames = unsafe { buffer.frameLength() } as usize;
+ let data = unsafe { buffer.floatChannelData() };
+ if data.is_null() || frames == 0 {
+ return;
+ }
+ let planes: Vec<&[f32]> = (0..channels)
+ // SAFETY: `data` holds one pointer per channel, each to `frames` samples.
+ .map(|c| unsafe { std::slice::from_raw_parts((*data.add(c)).as_ptr(), frames) })
+ .collect();
+ let pcm = resampler.borrow_mut().process(&audio::mono(&planes));
+ if !pcm.is_empty() {
+ audio_sink.audio(&pcm);
+ }
+ });
+ let tap_block = &*tap as *const block2::DynBlock<_> as *mut block2::DynBlock<_>;
+ // SAFETY: the block matches AVAudioNodeTapBlock and stays alive in the stream.
+ unsafe { input.installTapOnBus_bufferSize_format_block(0, TAP_FRAMES, Some(&format), tap_block) };
+ // SAFETY: the engine has its input connected through the tap.
+ unsafe { engine.prepare() };
+ if let Err(error) = unsafe { engine.startAndReturnError() } {
+ unsafe { input.removeTapOnBus(0) };
+ return Err(VoiceError::Failed(format!("The microphone could not start: {}", describe(&error))));
+ }
+
+ // A device change (headset unplugged, input switched) stops the engine.
+ let interrupted = RcBlock::new(move |_note: NonNull<NSNotification>| {
+ sink.failed(&VoiceError::Interrupted.to_string());
+ });
+ let engine_object: &AnyObject = &engine;
+ // SAFETY: a framework constant name, observed for this engine only; the block posts
+ // on the notifying thread and only calls the sink.
+ let observer = unsafe {
+ NSNotificationCenter::defaultCenter().addObserverForName_object_queue_usingBlock(
+ Some(AVAudioEngineConfigurationChangeNotification),
+ Some(engine_object),
+ None,
+ &interrupted,
+ )
+ };
+ Ok(Box::new(MacStream { engine, observer, _tap: tap, open: true }))
+ }
+}
+
+struct MacStream {
+ engine: Retained<AVAudioEngine>,
+ observer: Retained<ProtocolObject<dyn NSObjectProtocol>>,
+ _tap: RcBlock<dyn Fn(NonNull<AVAudioPCMBuffer>, NonNull<AVAudioTime>)>,
+ open: bool,
+}
+
+// SAFETY: AVAudioEngine and the observer token are not tied to a thread; the stream is only
+// used behind the machine's locks, one thread at a time, and the tap block is only invoked by
+// the engine.
+unsafe impl Send for MacStream {}
+
+impl Stream for MacStream {
+ fn close(&mut self) {
+ if !std::mem::take(&mut self.open) {
+ return;
+ }
+ // SAFETY: stopping a live engine and removing what `open` installed.
+ unsafe {
+ NSNotificationCenter::defaultCenter().removeObserver(AsRef::<AnyObject>::as_ref(&*self.observer));
+ self.engine.stop();
+ self.engine.inputNode().removeTapOnBus(0);
+ }
+ }
+}
+
+impl Drop for MacStream {
+ fn drop(&mut self) {
+ self.close();
+ }
+}
+
+/// Apple's Speech framework.
+pub struct AppleSpeech;
+
+impl Recognizer for AppleSpeech {
+ fn authorization(&self) -> Authorization {
+ // SAFETY: a class getter without side effects (it never prompts).
+ speech_status(unsafe { SFSpeechRecognizer::authorizationStatus() })
+ }
+
+ fn request_authorization(&self) -> Authorization {
+ let (tx, rx) = mpsc::channel();
+ let handler = RcBlock::new(move |status: SFSpeechRecognizerAuthorizationStatus| {
+ let _ = tx.send(status);
+ });
+ // SAFETY: the handler matches the documented signature; Info.plist carries
+ // NSSpeechRecognitionUsageDescription (without it this call aborts the app).
+ unsafe { SFSpeechRecognizer::requestAuthorization(&handler) };
+ rx.recv().map(speech_status).unwrap_or_else(|_| self.authorization())
+ }
+
+ fn current_locale(&self) -> String {
+ NSLocale::currentLocale().localeIdentifier().to_string()
+ }
+
+ fn supported_locales(&self) -> Vec<String> {
+ // SAFETY: a class getter returning a set of locales.
+ let locales = unsafe { SFSpeechRecognizer::supportedLocales() };
+ locales.allObjects().iter().map(|locale| locale.localeIdentifier().to_string()).collect()
+ }
+
+ fn begin(&self, locale: &str, sink: Sink) -> Result<Box<dyn Recognition>, VoiceError> {
+ let ns_locale = NSLocale::initWithLocaleIdentifier(NSLocale::alloc(), &NSString::from_str(locale));
+ let handler = RcBlock::new(move |result: *mut SFSpeechRecognitionResult, error: *mut NSError| {
+ // SAFETY: Speech passes a valid result or error (or null), valid for the call.
+ if let Some(result) = unsafe { result.as_ref() } {
+ let text = unsafe { result.bestTranscription().formattedString() }.to_string();
+ if unsafe { result.isFinal() } {
+ sink.finished(&text);
+ } else {
+ sink.partial(&text);
+ }
+ } else if let Some(error) = unsafe { error.as_ref() } {
+ sink.failed(&describe(error));
+ }
+ });
+ // SAFETY: plain initializers, setters, and getters on objects created here.
+ unsafe {
+ let recognizer = SFSpeechRecognizer::initWithLocale(SFSpeechRecognizer::alloc(), &ns_locale)
+ .ok_or_else(|| VoiceError::NoRecognizer(locale.to_string()))?;
+ if !recognizer.isAvailable() {
+ return Err(VoiceError::Failed(
+ "Speech recognition is not available right now. Check the internet connection and try again.".into(),
+ ));
+ }
+ // Results default to the main queue; keep them off it.
+ recognizer.setQueue(&NSOperationQueue::new());
+ let request = SFSpeechAudioBufferRecognitionRequest::new();
+ request.setShouldReportPartialResults(true);
+ if recognizer.supportsOnDeviceRecognition() {
+ request.setRequiresOnDeviceRecognition(true);
+ }
+ if available!(macos = 13.0) {
+ request.setAddsPunctuation(true);
+ }
+ let format = AVAudioFormat::initStandardFormatWithSampleRate_channels(
+ AVAudioFormat::alloc(),
+ f64::from(SAMPLE_RATE),
+ 1,
+ )
+ .ok_or_else(|| VoiceError::Failed("The audio format is not supported.".into()))?;
+ let task = recognizer.recognitionTaskWithRequest_resultHandler(&request, &handler);
+ Ok(Box::new(AppleRecognition { _recognizer: recognizer, request, task, format }))
+ }
+ }
+}
+
+struct AppleRecognition {
+ /// Kept alive until the final result arrives.
+ _recognizer: Retained<SFSpeechRecognizer>,
+ request: Retained<SFSpeechAudioBufferRecognitionRequest>,
+ task: Retained<SFSpeechRecognitionTask>,
+ /// 16 kHz mono float32, deinterleaved.
+ format: Retained<AVAudioFormat>,
+}
+
+// SAFETY: the request is meant to be fed from the audio thread and ended or cancelled from
+// another; the machine serializes every call behind a lock.
+unsafe impl Send for AppleRecognition {}
+
+impl Recognition for AppleRecognition {
+ fn feed(&mut self, samples: &[f32]) {
+ let Ok(frames) = u32::try_from(samples.len()) else { return };
+ // SAFETY: a new buffer in our mono format with room for `frames`; its single channel
+ // receives exactly `frames` samples before the request copies it.
+ unsafe {
+ let Some(buffer) = AVAudioPCMBuffer::initWithPCMFormat_frameCapacity(AVAudioPCMBuffer::alloc(), &self.format, frames)
+ else {
+ return;
+ };
+ buffer.setFrameLength(frames);
+ let data = buffer.floatChannelData();
+ if data.is_null() {
+ return;
+ }
+ std::ptr::copy_nonoverlapping(samples.as_ptr(), (*data).as_ptr(), samples.len());
+ self.request.appendAudioPCMBuffer(&buffer);
+ }
+ }
+
+ fn finish(&mut self) {
+ // SAFETY: ending the audio of a live request; the final result follows on the queue.
+ unsafe { self.request.endAudio() };
+ }
+
+ fn cancel(&mut self) {
+ // SAFETY: cancelling a live task; its late "cancelled" error is ignored by the sink.
+ unsafe { self.task.cancel() };
+ }
+}
diff --git a/apps/desktop/src-tauri/src/voice/mod.rs b/apps/desktop/src-tauri/src/voice/mod.rs
new file mode 100644
index 0000000..8496088
--- /dev/null
+++ b/apps/desktop/src-tauri/src/voice/mod.rs
@@ -0,0 +1,217 @@
+//! Voice input (S30.1): the panel's mic button records natively and the text comes back as the
+//! user's message. Two seams keep the backends swappable:
+//!
+//! - `Microphone` captures audio and delivers 16 kHz mono f32 PCM (`audio::SAMPLE_RATE`), the
+//! format offline models such as sherpa-onnx expect, so any recognizer can consume it.
+//! - `Recognizer` turns that PCM into partial and final text. Apple's Speech framework is the
+//! default (`macos::AppleSpeech`); a local model can be added as a second implementation.
+//!
+//! `Voice` is the pure state machine between them (idle → starting → recording → stopping,
+//! single flight, cancel, error mapping, level throttling, locale fallback), tested on every OS
+//! with fakes. The objc2 bindings live in `macos`; off macOS the commands answer "unavailable".
+//!
+//! SpeechAnalyzer/SpeechTranscriber (macOS 26) are Swift-only, with no Objective-C interface to
+//! bind from Rust, and the app supports macOS 12.3, so `SFSpeechRecognizer` is the recognizer.
+
+pub mod app;
+pub mod audio;
+#[cfg(test)]
+mod fake;
+pub mod locale;
+mod machine;
+#[cfg(target_os = "macos")]
+mod macos;
+
+pub use machine::{Sink, Voice};
+#[cfg(target_os = "macos")]
+pub use macos::{AppleSpeech, MacMicrophone};
+
+use serde::Serialize;
+use std::fmt;
+use std::sync::Arc;
+
+/// Live text while the user speaks: `{text}`.
+pub const PARTIAL_EVENT: &str = "voice://partial";
+/// The input level, `{level}` in 0..1, about 15 times a second.
+pub const LEVEL_EVENT: &str = "voice://level";
+/// A failure while recording (permission, recognizer, interruption): `{message}`.
+pub const ERROR_EVENT: &str = "voice://error";
+pub const UNSUPPORTED: &str = "Voice input is available on macOS.";
+
+/// The `voice_status` reply: `{available, reason?}`.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct VoiceStatus {
+ pub available: bool,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub reason: Option<String>,
+}
+
+impl VoiceStatus {
+ pub fn available() -> Self {
+ VoiceStatus { available: true, reason: None }
+ }
+
+ pub fn unavailable(reason: impl Into<String>) -> Self {
+ VoiceStatus { available: false, reason: Some(reason.into()) }
+ }
+}
+
+/// The `voice_stop` reply: `{text}`, the final transcript.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct Transcript {
+ pub text: String,
+}
+
+/// What the webview that started the recording hears.
+#[derive(Debug, Clone, PartialEq)]
+pub enum VoiceEvent {
+ Partial(String),
+ Level(f32),
+ Error(String),
+}
+
+impl VoiceEvent {
+ pub fn name(&self) -> &'static str {
+ match self {
+ VoiceEvent::Partial(_) => PARTIAL_EVENT,
+ VoiceEvent::Level(_) => LEVEL_EVENT,
+ VoiceEvent::Error(_) => ERROR_EVENT,
+ }
+ }
+
+ pub fn payload(&self) -> serde_json::Value {
+ match self {
+ VoiceEvent::Partial(text) => serde_json::json!({ "text": text }),
+ VoiceEvent::Level(level) => serde_json::json!({ "level": level }),
+ VoiceEvent::Error(message) => serde_json::json!({ "message": message }),
+ }
+ }
+}
+
+/// Delivers events to the webview that started the recording; called from any thread.
+pub type Events = Arc<dyn Fn(VoiceEvent) + Send + Sync>;
+
+/// A macOS privacy permission, as the system reports it.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Authorization {
+ NotDetermined,
+ Denied,
+ Restricted,
+ Authorized,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum VoiceError {
+ Unsupported,
+ Busy,
+ Stopping,
+ NotRecording,
+ Cancelled,
+ SpeechDenied,
+ SpeechRestricted,
+ MicrophoneDenied,
+ MicrophoneRestricted,
+ NoRecognizer(String),
+ NoMicrophone,
+ Interrupted,
+ Failed(String),
+}
+
+impl fmt::Display for VoiceError {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ VoiceError::Unsupported => f.write_str(UNSUPPORTED),
+ VoiceError::Busy => f.write_str("Voice input is already recording."),
+ VoiceError::Stopping => f.write_str("Voice input is already stopping."),
+ VoiceError::NotRecording => f.write_str("Voice input is not recording."),
+ VoiceError::Cancelled => f.write_str("Voice input was cancelled."),
+ VoiceError::SpeechDenied => f.write_str(
+ "Speech Recognition is off for Gentle Dot. Turn it on in System Settings → Privacy & Security → Speech Recognition.",
+ ),
+ VoiceError::SpeechRestricted => f.write_str("Speech Recognition is restricted on this Mac."),
+ VoiceError::MicrophoneDenied => f.write_str(
+ "Microphone access is off for Gentle Dot. Turn it on in System Settings → Privacy & Security → Microphone.",
+ ),
+ VoiceError::MicrophoneRestricted => f.write_str("Microphone access is restricted on this Mac."),
+ VoiceError::NoRecognizer(locale) => write!(f, "Speech recognition is not available for {locale}."),
+ VoiceError::NoMicrophone => f.write_str("No microphone is available."),
+ VoiceError::Interrupted => f.write_str("The microphone was interrupted."),
+ VoiceError::Failed(message) => f.write_str(message),
+ }
+ }
+}
+
+impl From<VoiceError> for String {
+ fn from(error: VoiceError) -> String {
+ error.to_string()
+ }
+}
+
+/// Native audio capture. Delivers `audio::SAMPLE_RATE` mono f32 chunks to `Sink::audio` (on
+/// the audio thread) and reports interruptions with `Sink::failed`.
+pub trait Microphone: Send + Sync {
+ /// The current permission, without prompting.
+ fn authorization(&self) -> Authorization;
+ /// Prompts when not yet determined. Blocks until the user answers: never on the main thread.
+ fn request_authorization(&self) -> Authorization;
+ fn open(&self, sink: Sink) -> Result<Box<dyn Stream>, VoiceError>;
+}
+
+/// An open capture.
+pub trait Stream: Send {
+ fn close(&mut self);
+}
+
+/// Speech to text over the microphone's PCM. Results arrive through `Sink::partial`,
+/// `Sink::finished`, and `Sink::failed`, from any thread, but never from inside `feed`,
+/// `finish`, or `cancel` while reporting a failure.
+pub trait Recognizer: Send + Sync {
+ /// The current permission, without prompting (`Authorized` when none is needed).
+ fn authorization(&self) -> Authorization;
+ /// Prompts when not yet determined. Blocks until the user answers: never on the main thread.
+ fn request_authorization(&self) -> Authorization;
+ /// The user's locale identifier, as the system reports it.
+ fn current_locale(&self) -> String;
+ fn supported_locales(&self) -> Vec<String>;
+ fn begin(&self, locale: &str, sink: Sink) -> Result<Box<dyn Recognition>, VoiceError>;
+}
+
+/// One recognition in progress.
+pub trait Recognition: Send {
+ /// More audio. Runs on the audio thread, so it must not block (heavy decoders queue it).
+ fn feed(&mut self, samples: &[f32]);
+ /// No more audio: the final text follows through `Sink::finished`.
+ fn finish(&mut self);
+ /// Discards the recognition.
+ fn cancel(&mut self);
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn events_carry_the_contract_names_and_payloads() {
+ let partial = VoiceEvent::Partial("hola".into());
+ assert_eq!((partial.name(), partial.payload()), ("voice://partial", serde_json::json!({"text": "hola"})));
+ let level = VoiceEvent::Level(0.5);
+ assert_eq!((level.name(), level.payload()), ("voice://level", serde_json::json!({"level": 0.5})));
+ let error = VoiceEvent::Error("no".into());
+ assert_eq!((error.name(), error.payload()), ("voice://error", serde_json::json!({"message": "no"})));
+ }
+
+ #[test]
+ fn status_omits_the_reason_when_available() {
+ assert_eq!(serde_json::to_value(VoiceStatus::available()).unwrap(), serde_json::json!({"available": true}));
+ assert_eq!(
+ serde_json::to_value(VoiceStatus::unavailable(UNSUPPORTED)).unwrap(),
+ serde_json::json!({"available": false, "reason": "Voice input is available on macOS."})
+ );
+ }
+
+ #[test]
+ fn the_stop_reply_is_the_text() {
+ let reply = Transcript { text: "hello".into() };
+ assert_eq!(serde_json::to_value(reply).unwrap(), serde_json::json!({"text": "hello"}));
+ }
+}