A free, open source, and extensible speech-to-text application that works completely offline.
isolate transcribe.cpp backend in own worker (#2208)
CJ Paisc8262ab3ebe5
diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lockindex ac879abd..77db5fcf 100644--- a/src-tauri/Cargo.lock+++ b/src-tauri/Cargo.lock@@ -7285,20 +7285,21 @@ dependencies = [[[package]]name = "transcribe-cpp"-version = "0.3.0"+version = "0.3.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f5134e51fd0cd73fc024b8004c495d96ec1fa5d89c5e341482fd60d82483002a"+checksum = "2f06368bb32251b5e93ceb8cae2d6ba9adff6a51693c7ce108449e65522b05c6"dependencies = ["log",+ "serde","thiserror 2.0.18","transcribe-cpp-sys",][[package]]name = "transcribe-cpp-sys"-version = "0.3.0"+version = "0.3.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "98c3d32764dc7ae892274022d52d369e92e1e765e403e56daaa3fe8fdbe18443"+checksum = "fedfadb42719d92e58a2c340ccfb6ef327ac1b846fc1d4716b65384a330513a6"dependencies = ["cmake","serde_json",diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.tomlindex d669d29e..0a9fe242 100644--- a/src-tauri/Cargo.toml+++ b/src-tauri/Cargo.toml@@ -75,11 +75,15 @@ rusqlite = { version = "0.37", features = ["bundled"] }tar = "0.4.44"flate2 = "1.0"sha2 = "0.10"+libc = "0.2"# Whisper-family models run through transcribe-cpp (GGUF/ggml, native backends);# transcribe-rs is now ONNX-only (Parakeet, Moonshine, SenseVoice, GigaAM,-# Canary, Cohere). Per-platform backend features are added in the target tables.+# Canary, Cohere). Per-platform backend features are added in the target tables+# (Cargo unifies features, so `serde` here applies to every platform).transcribe-rs = { version = "0.3.8", features = ["onnx"] }-transcribe-cpp = { version = "0.3.0", default-features = false }+transcribe-cpp = { version = "0.3.1", default-features = false, features = [+ "serde",+] }handy-keys = "0.3.4"ferrous-opencc = "0.2.3"clap = { version = "4", features = ["derive"] }@@ -142,17 +146,17 @@ winreg = "0.55"# signtool verify entirely. CI pins -DGGML_NATIVE=OFF (see build.yml) so the# static build targets a portable ARM baseline, not the build runner's CPU.[target.'cfg(all(windows, target_arch = "x86_64"))'.dependencies]-transcribe-cpp = { version = "0.3.0", default-features = false, features = [+transcribe-cpp = { version = "0.3.1", default-features = false, features = ["dynamic-backends","vulkan",] }[target.'cfg(all(windows, target_arch = "aarch64"))'.dependencies]-transcribe-cpp = { version = "0.3.0", default-features = false }+transcribe-cpp = { version = "0.3.1", default-features = false }[target.'cfg(target_os = "macos")'.dependencies]tauri-nspanel = { git = "https://github.com/ahkohd/tauri-nspanel", branch = "v2.1" }-transcribe-cpp = { version = "0.3.0", default-features = false, features = ["metal"] }+transcribe-cpp = { version = "0.3.1", default-features = false, features = ["metal"] }# Used by the receipt-sequenced ("reliable") paste path for lazy NSPasteboard# promises. Versions track what Tauri already pulls in.objc2 = "0.6"@@ -163,8 +167,7 @@ objc2-service-management = "0.3.2"[target.'cfg(target_os = "linux")'.dependencies]gtk-layer-shell = { version = "0.8", features = ["v0_6"] }gtk = "0.18"-libc = "0.2"-transcribe-cpp = { version = "0.3.0", default-features = false, features = [+transcribe-cpp = { version = "0.3.1", default-features = false, features = ["dynamic-backends","vulkan",] }diff --git a/src-tauri/src/actions.rs b/src-tauri/src/actions.rsindex cce7cfe4..04d88644 100644--- a/src-tauri/src/actions.rs+++ b/src-tauri/src/actions.rs@@ -650,11 +650,10 @@ impl ShortcutAction for TranscribeAction {let transcription_time = Instant::now();let transcription_result = match tm.finalize_stream() {// A finalized stream with usable text wins. An empty result- // (no active stream, produced nothing, or a finalize error- // after the engine was returned) falls back to a full batch- // transcription of the same audio. A finalize timeout is- // surfaced instead — the worker may still hold the engine,- // so a batch fallback would contend with it.+ // (no active stream, produced nothing, or the stream failed+ // or its worker crashed) falls back to a full batch+ // transcription of the same audio. A cancelled finalize is+ // surfaced instead, so a cancel never starts a batch run.Ok(Some(text)) if !text.trim().is_empty() => Ok(text),Ok(_) => tm.transcribe(samples),Err(err) => Err(err),diff --git a/src-tauri/src/commands/models.rs b/src-tauri/src/commands/models.rsindex b1bb8949..e49a5cf1 100644--- a/src-tauri/src/commands/models.rs+++ b/src-tauri/src/commands/models.rs@@ -72,8 +72,12 @@ pub async fn delete_model(// If deleting the active model, unload it and clear the settinglet settings = get_settings(&app_handle);if settings.selected_model == model_id {- transcription_manager- .unload_model()+ // Waits for the worker to exit (behind any transcription in progress)+ // before the file is deleted, so keep it off the async workers.+ let tm = Arc::clone(&transcription_manager);+ tauri::async_runtime::spawn_blocking(move || tm.unload_model())+ .await+ .map_err(|e| format!("Failed to unload model: {}", e))?.map_err(|e| format!("Failed to unload model: {}", e))?;let mut settings = get_settings(&app_handle);diff --git a/src-tauri/src/commands/transcription.rs b/src-tauri/src/commands/transcription.rsindex 6cfa2e6a..d51b1673 100644--- a/src-tauri/src/commands/transcription.rs+++ b/src-tauri/src/commands/transcription.rs@@ -2,6 +2,7 @@ use crate::managers::transcription::TranscriptionManager;use crate::settings::{get_settings, write_settings, ModelUnloadTimeout};use serde::Serialize;use specta::Type;+use std::sync::Arc;use tauri::{AppHandle, State};#[derive(Serialize, Type)]@@ -21,7 +22,7 @@ pub fn set_model_unload_timeout(app: AppHandle, timeout: ModelUnloadTimeout) {#[tauri::command]#[specta::specta]pub fn get_model_load_status(- transcription_manager: State<TranscriptionManager>,+ transcription_manager: State<Arc<TranscriptionManager>>,) -> Result<ModelLoadStatus, String> {Ok(ModelLoadStatus {is_loaded: transcription_manager.is_model_loaded(),@@ -32,9 +33,8 @@ pub fn get_model_load_status(#[tauri::command]#[specta::specta]pub fn unload_model_manually(- transcription_manager: State<TranscriptionManager>,+ transcription_manager: State<Arc<TranscriptionManager>>,) -> Result<(), String> {- transcription_manager- .unload_model()- .map_err(|e| format!("Failed to unload model: {}", e))+ transcription_manager.request_unload();+ Ok(())}diff --git a/src-tauri/src/engine_supervisor/mod.rs b/src-tauri/src/engine_supervisor/mod.rsnew file mode 100644index 00000000..a88634a4--- /dev/null+++ b/src-tauri/src/engine_supervisor/mod.rs@@ -0,0 +1,52 @@+//! Process isolation for transcribe.cpp.+//!+//! All transcribe.cpp native code (backend init, device enumeration, model+//! load, inference, streaming) runs in a child process: the Handy executable+//! re-launched with [`WORKER_FLAG`]. A GPU driver fault, `GGML_ASSERT` abort,+//! or hang then kills only the worker.+//!+//! [`EngineSupervisor`] is the single owner of that worker. It loads and+//! unloads models, runs and streams transcriptions, cancels them, and+//! recovers from crashes and hangs (retrying in a CPU-only worker when the+//! GPU was at fault), so the rest of the app never sees processes, restarts+//! or fallback. At most one worker holds a model at a time, and it lives+//! exactly as long as that model, so unloading returns all of its CPU and GPU+//! memory to the OS. Listing compute devices may briefly run a second,+//! model-less worker.+//!+//! A CPU-only worker ([`CPU_ONLY_FLAG`]) restricts transcribe.cpp to its CPU+//! backends before any are registered, so no GPU driver code runs in it at+//! all: a driver that crashes or hangs while initializing can't take the CPU+//! fallback down with it.+//!+//! - `protocol`: the framed request/response protocol between the two.+//! - `supervisor`: the parent side ([`EngineSupervisor`]).+//! - `worker`: the child process.++mod protocol;+mod supervisor;+mod worker;++pub use protocol::{DeviceInfo, DeviceSelector, LoadedInfo};+pub use supervisor::{+ EngineError, EngineSupervisor, Finalized, LoadSpec, StreamHandle, StreamProgress, Unloading,+};++/// Hidden first argument that turns the executable into a worker.+pub const WORKER_FLAG: &str = "--transcribe-worker";+/// Worker argument: register only the CPU backends.+const CPU_ONLY_FLAG: &str = "--cpu-only";+const LOG_LEVEL_ENV: &str = "HANDY_TRANSCRIBE_WORKER_LOG";++/// Whether this process was launched as a transcription worker. Checked in+/// `main` before CLI parsing, Tauri, or single-instance handling.+pub fn is_worker_invocation() -> bool {+ std::env::args_os()+ .nth(1)+ .is_some_and(|arg| arg == WORKER_FLAG)+}++/// Run the worker until the parent closes its stdin. Returns the exit code.+pub fn run_worker() -> i32 {+ worker::run()+}diff --git a/src-tauri/src/engine_supervisor/protocol.rs b/src-tauri/src/engine_supervisor/protocol.rsnew file mode 100644index 00000000..9e353c95--- /dev/null+++ b/src-tauri/src/engine_supervisor/protocol.rs@@ -0,0 +1,310 @@+//! Wire protocol between Handy and its transcribe.cpp worker process.+//!+//! Strict request/response: every [`Request`] gets exactly one [`Response`].+//! Each message is one frame: `[u32 json_len][json][u32 pcm_len][pcm bytes]`+//! (lengths little-endian, PCM as little-endian f32). JSON (not bincode) is+//! deliberate: transcribe-cpp's serde impls round-trip NaN confidences through+//! `null`, which only works with a self-describing format.++use serde::{Deserialize, Serialize};+use std::io::{self, Read, Write};+use std::path::PathBuf;+use transcribe_cpp::{+ Backend, Capabilities, DeviceType, RunOptions, StreamOptions, StreamText, StreamUpdate,+ Transcript,+};++/// Refuse frames larger than this: a garbage length must not turn into a+/// multi-gigabyte allocation. One hour of 16 kHz f32 PCM is ~230 MB.+const MAX_SECTION_BYTES: usize = 1 << 30;+const PCM_BYTES_PER_MINUTE: usize = 16_000 * 4 * 60;++#[derive(Debug, Serialize, Deserialize)]+pub enum Request {+ /// First request to every worker; answered once backend init is done+ /// (with an error if it registered no compute device). With+ /// `list_devices`, the worker also enumerates its compute devices.+ /// CPU-only workers that hold a model don't list: they would only see+ /// the CPU.+ Hello {+ list_devices: bool,+ },+ Load {+ path: PathBuf,+ backend: Backend,+ device: DeviceSelector,+ },+ /// Followed by the PCM to transcribe.+ Run {+ options: RunOptions,+ },+ StreamBegin {+ run: RunOptions,+ stream: StreamOptions,+ },+ /// Followed by the PCM frame to feed.+ Feed,+ /// `want_language`: also report the model's detected language (costs a+ /// transcript snapshot, so only asked for when it can change the outcome).+ Finalize {+ want_language: bool,+ },+ StreamReset,+}++/// How the worker should pick the device for a [`Request::Load`]. Device+/// handles are process-local, so the parent names devices by stable key or by+/// registry index and the worker resolves them.+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub enum DeviceSelector {+ Auto,+ /// Persisted identity from [`device_key`]. Falls back to automatic+ /// selection when the device is no longer present.+ Key(String),+ /// Registry index from `--list-devices`. Fails if not a usable device.+ Index(usize),+}++#[derive(Debug, Serialize, Deserialize)]+pub enum Response {+ /// `devices` is present only when the hello asked for them.+ Hello {+ devices: Option<Vec<DeviceInfo>>,+ },+ Loaded(LoadedInfo),+ Transcript(Transcript),+ Fed {+ update: StreamUpdate,+ /// Present only when the committed or tentative text changed.+ text: Option<StreamText>,+ },+ Finalized {+ update: StreamUpdate,+ text: StreamText,+ language: Option<String>,+ },+ Ok,+ Error(String),+}++/// Serializable mirror of `transcribe_cpp::Device` (which holds a+/// process-local handle and so has no serde impl).+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct DeviceInfo {+ pub name: String,+ pub description: String,+ pub kind: String,+ pub device_type: DeviceType,+ pub device_id: Option<String>,+ pub memory_total: u64,+ pub index: Option<usize>,+ /// Stable identity for persisting a device choice; see [`device_key`].+ pub key: String,+}++impl DeviceInfo {+ pub fn from_device(device: &transcribe_cpp::Device) -> Self {+ Self {+ name: device.name.clone(),+ description: device.description.clone(),+ kind: device.kind.clone(),+ device_type: device.device_type,+ device_id: device.device_id.clone(),+ memory_total: device.memory_total,+ index: device.index,+ key: device_key(&device.kind, device.device_id.as_deref(), &device.name),+ }+ }++ pub fn label(&self) -> &str {+ if self.description.is_empty() {+ &self.name+ } else {+ &self.description+ }+ }++ pub fn is_gpu(&self) -> bool {+ matches!(self.device_type, DeviceType::Gpu | DeviceType::Igpu)+ }+}++/// Persistent device identity. Uses the backend's stable `device_id` where+/// available and its name otherwise (Metal reports no id). Must stay+/// byte-identical to what older versions stored in settings.+pub fn device_key(kind: &str, device_id: Option<&str>, name: &str) -> String {+ let (identity_kind, identity) = match device_id {+ Some(device_id) => ("id", device_id),+ None => ("name", name),+ };+ serde_json::to_string(&(kind, identity_kind, identity))+ .expect("transcribe device identity is always JSON serializable")+}++/// What the parent needs to know about a loaded model, read once at load.+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct LoadedInfo {+ pub arch: String,+ pub variant: String,+ pub backend: String,+ /// Label of the device the model actually bound to.+ pub device: String,+ /// Whether that device is a GPU (a crash there may be the GPU's fault).+ pub on_gpu: bool,+ pub capabilities: Capabilities,+ pub supports_initial_prompt: bool,+}++/// Encode one frame. Refuses a section the reader would refuse, so an+/// oversized request fails cleanly here instead of killing the worker (which+/// would read as a crash and be blamed on the GPU).+pub fn encode_message<T: Serialize>(message: &T, pcm: Option<&[f32]>) -> io::Result<Vec<u8>> {+ let json = serde_json::to_vec(message).map_err(io::Error::other)?;+ check_section_len(json.len())?;+ let pcm = pcm.unwrap_or(&[]);+ if check_section_len(pcm.len() * 4).is_err() {+ return Err(io::Error::other(format!(+ "audio too long for one transcription: {} minutes (limit {} minutes)",+ pcm.len() * 4 / PCM_BYTES_PER_MINUTE,+ MAX_SECTION_BYTES / PCM_BYTES_PER_MINUTE,+ )));+ }+ let mut frame = Vec::with_capacity(8 + json.len() + pcm.len() * 4);+ frame.extend_from_slice(&(json.len() as u32).to_le_bytes());+ frame.extend_from_slice(&json);+ frame.extend_from_slice(&((pcm.len() * 4) as u32).to_le_bytes());+ for sample in pcm {+ frame.extend_from_slice(&sample.to_le_bytes());+ }+ Ok(frame)+}++pub fn write_message<T: Serialize>(+ w: &mut impl Write,+ message: &T,+ pcm: Option<&[f32]>,+) -> io::Result<()> {+ w.write_all(&encode_message(message, pcm)?)?;+ w.flush()+}++/// Read one frame. `Ok(None)` only on a clean EOF before the first byte of a+/// frame; a frame cut off part-way is an error.+pub fn read_message<T: for<'de> Deserialize<'de>>(+ r: &mut impl Read,+) -> io::Result<Option<(T, Vec<f32>)>> {+ let mut len = [0u8; 4];+ let mut filled = 0;+ while filled < len.len() {+ match r.read(&mut len[filled..]) {+ Ok(0) if filled == 0 => return Ok(None),+ Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()),+ Ok(n) => filled += n,+ Err(e) if e.kind() == io::ErrorKind::Interrupted => {}+ Err(e) => return Err(e),+ }+ }+ let json = read_section_body(r, len)?;+ let message = serde_json::from_slice(&json).map_err(io::Error::other)?;+ let pcm_bytes = read_section(r)?;+ if pcm_bytes.len() % 4 != 0 {+ return Err(io::Error::other(+ "PCM section is not a whole number of f32s",+ ));+ }+ let pcm = pcm_bytes+ .as_chunks::<4>()+ .0+ .iter()+ .map(|b| f32::from_le_bytes(*b))+ .collect();+ Ok(Some((message, pcm)))+}++fn read_section(r: &mut impl Read) -> io::Result<Vec<u8>> {+ let mut len = [0u8; 4];+ r.read_exact(&mut len)?;+ read_section_body(r, len)+}++fn read_section_body(r: &mut impl Read, len: [u8; 4]) -> io::Result<Vec<u8>> {+ let len = u32::from_le_bytes(len) as usize;+ check_section_len(len)?;+ let mut buf = vec![0u8; len];+ r.read_exact(&mut buf)?;+ Ok(buf)+}++/// Also keeps every section length within the frame's `u32` length prefix.+fn check_section_len(len: usize) -> io::Result<()> {+ if len > MAX_SECTION_BYTES {+ return Err(io::Error::other(format!("frame section too large: {len}")));+ }+ Ok(())+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn round_trips_message_with_pcm() {+ let mut buf = Vec::new();+ let pcm = [0.0f32, -1.0, 0.5, f32::MIN_POSITIVE];+ write_message(+ &mut buf,+ &Request::Run {+ options: RunOptions::default(),+ },+ Some(&pcm),+ )+ .unwrap();+ let (req, got): (Request, Vec<f32>) = read_message(&mut buf.as_slice()).unwrap().unwrap();+ assert!(matches!(req, Request::Run { .. }));+ assert_eq!(got, pcm);+ }++ #[test]+ fn clean_eof_is_none() {+ let empty: &[u8] = &[];+ assert!(read_message::<Request>(&mut &*empty).unwrap().is_none());+ }++ #[test]+ fn truncated_frame_is_an_error() {+ let mut buf = Vec::new();+ write_message(+ &mut buf,+ &Request::Run {+ options: RunOptions::default(),+ },+ Some(&[0.5f32; 8]),+ )+ .unwrap();+ // Cut inside the length prefix, the JSON, the PCM length and the PCM.+ for cut in [1, 3, 6, buf.len() - 34, buf.len() - 1] {+ let partial = &buf[..cut];+ let err = read_message::<Request>(&mut &*partial).unwrap_err();+ assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof, "cut at {cut}");+ }+ }++ #[test]+ fn section_limit_is_inclusive_and_fits_the_length_prefix() {+ assert!(check_section_len(MAX_SECTION_BYTES).is_ok());+ assert!(check_section_len(MAX_SECTION_BYTES + 1).is_err());+ assert!(MAX_SECTION_BYTES <= u32::MAX as usize);+ }++ #[test]+ fn device_key_matches_legacy_format() {+ assert_eq!(+ device_key("metal", None, "Metal"),+ r#"["metal","name","Metal"]"#+ );+ assert_eq!(+ device_key("vulkan", Some("0000:01:00.0"), "Vulkan0"),+ r#"["vulkan","id","0000:01:00.0"]"#+ );+ }+}diff --git a/src-tauri/src/engine_supervisor/supervisor.rs b/src-tauri/src/engine_supervisor/supervisor.rsnew file mode 100644index 00000000..8fae4f0b--- /dev/null+++ b/src-tauri/src/engine_supervisor/supervisor.rs@@ -0,0 +1,1975 @@+//! Parent side: [`EngineSupervisor`], the single owner of the transcribe.cpp+//! worker process.+//!+//! One owner thread works through a command queue and holds the only worker+//! slot, the loaded model's spec and any active stream, so "at most one+//! model worker alive" holds by construction. Callers' commands wait their+//! turn; waiting in the queue (e.g. behind a model load) is never a timeout.+//! Only [`EngineSupervisor::cancel`] and device probes bypass the queue: a+//! probe runs its own short-lived, model-less worker on the caller's thread,+//! so a hung GPU driver can't hold up dictation behind it.+//!+//! A worker that will only use the CPU is started CPU-only, so it never+//! registers a GPU backend. One rule decides: the spec it loads asks for+//! [`Backend::Cpu`] (the user's choice, or a fallback after a GPU failure),+//! or the host must not use the GPU at all.++use super::protocol::{+ encode_message, read_message, DeviceInfo, DeviceSelector, LoadedInfo, Request, Response,+};+use super::worker::LOG_LINE_PREFIX;+use super::{CPU_ONLY_FLAG, LOG_LEVEL_ENV, WORKER_FLAG};+use log::{debug, error, info, warn, Level};+use std::collections::VecDeque;+use std::fmt;+use std::io::{self, BufRead, BufReader, Write};+use std::panic::{catch_unwind, AssertUnwindSafe};+use std::path::PathBuf;+use std::process::{Child, ChildStdin, ChildStdout, Command as ProcessCommand, ExitStatus, Stdio};+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};+use std::sync::{mpsc, Arc, Mutex, MutexGuard};+use std::thread;+use std::time::{Duration, Instant};+use transcribe_cpp::{Backend, RunOptions, StreamOptions, StreamText, StreamUpdate, Transcript};++/// Spawn, backend init and device listing. Listing first opens the GPU,+/// which on macOS compiles ggml's Metal library when the system shader cache+/// does not hold it yet (~15 s after an install or update).+const HELLO_TIMEOUT: Duration = Duration::from_secs(60);+/// Model loads stay generous: a slow load is slow, not hung (#1841).+const LOAD_TIMEOUT: Duration = Duration::from_secs(180);+/// Floor for stream calls that do no decoding (begin, reset).+const CALL_FLOOR: Duration = Duration::from_secs(30);+/// How long a worker gets to exit after its stdin closes before it's killed.+const SHUTDOWN_GRACE: Duration = Duration::from_secs(2);+/// Native stderr lines kept for crash reports (a `GGML_ASSERT` message lands+/// here right before an abort).+const STDERR_TAIL_LINES: usize = 64;+const SAMPLE_RATE: f64 = 16_000.0;++/// How long decoding work may take before its worker counts as hung, by the+/// device the model runs on. On a GPU a hang is recovered from (retried on+/// CPU), so it should be caught soon. On CPU a hang is never retried, so the+/// deadline only decides when to give up: a slow machine must not be+/// mistaken for a hung one.+#[derive(Debug, Clone, Copy, PartialEq)]+struct Deadlines {+ /// Floor for a stream feed or finalize, so sub-second chunks and+ /// first-use shader compiles are not mistaken for hangs.+ call_floor: Duration,+ /// Floor for a batch run. A run has a fixed cost however short the+ /// clip: Whisper always encodes a full 30 s window, which on CPU takes+ /// seconds even for a 1 s clip.+ run_floor: Duration,+ /// Work on N seconds of audio may take up to this many times N.+ audio_factor: f64,+}++const GPU_DEADLINES: Deadlines = Deadlines {+ call_floor: Duration::from_secs(30),+ run_floor: Duration::from_secs(60),+ audio_factor: 10.0,+};++/// A single CPU feed or finalize can take well over 10 s, and a large model+/// on an older CPU runs slower than real time.+const CPU_DEADLINES: Deadlines = Deadlines {+ call_floor: Duration::from_secs(120),+ run_floor: Duration::from_secs(300),+ audio_factor: 20.0,+};++impl Deadlines {+ fn for_device(on_gpu: bool) -> Self {+ if on_gpu {+ GPU_DEADLINES+ } else {+ CPU_DEADLINES+ }+ }++ /// Deadline for stream work on `samples` of 16 kHz audio.+ fn audio(&self, samples: usize) -> Duration {+ self.call_floor.max(Duration::from_secs_f64(+ samples as f64 / SAMPLE_RATE * self.audio_factor,+ ))+ }++ /// Deadline for a batch run on `samples` of 16 kHz audio.+ fn run(&self, samples: usize) -> Duration {+ self.run_floor.max(self.audio(samples))+ }+}++fn ms_to_samples(ms: i64) -> usize {+ (ms.max(0) as usize) * (SAMPLE_RATE as usize / 1000)+}++/// What to load. The supervisor keeps it so a dead worker can be replaced+/// transparently.+#[derive(Debug, Clone)]+pub struct LoadSpec {+ pub path: PathBuf,+ pub backend: Backend,+ pub device: DeviceSelector,+}++impl LoadSpec {+ /// An explicitly chosen device (`--device-index`) never falls back to+ /// CPU: its failures are reported, not hidden.+ fn pinned(&self) -> bool {+ matches!(self.device, DeviceSelector::Index(_))+ }++ fn cpu_only(&self) -> bool {+ self.backend == Backend::Cpu+ }++ fn cpu(&self) -> Self {+ Self {+ path: self.path.clone(),+ backend: Backend::Cpu,+ device: DeviceSelector::Auto,+ }+ }+}++#[derive(Debug)]+pub enum EngineError {+ /// [`EngineSupervisor::cancel`] stopped the work.+ Cancelled,+ /// No model, a transcribe.cpp error, or a worker failure that could not+ /// be recovered.+ Failed(String),+}++impl fmt::Display for EngineError {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ match self {+ EngineError::Cancelled => f.write_str("transcription was cancelled"),+ EngineError::Failed(message) => f.write_str(message),+ }+ }+}++impl std::error::Error for EngineError {}++/// The result of one stream feed, handed to the stream's callback.+pub struct StreamProgress {+ pub update: StreamUpdate,+ /// Present only when the committed or tentative text changed.+ pub text: Option<StreamText>,+ /// Time from sending the frame to the worker until its answer.+ pub elapsed: Duration,+}++pub struct Finalized {+ pub update: StreamUpdate,+ pub text: StreamText,+ /// The model's detected language, when asked for.+ pub language: Option<String>,+ pub elapsed: Duration,+}++type Reply<T> = mpsc::Sender<T>;+type OnProgress = Box<dyn FnMut(StreamProgress) + Send>;++enum Command {+ Load(LoadSpec, Reply<Result<LoadedInfo, EngineError>>),+ Unload(Reply<()>),+ Transcribe {+ pcm: Vec<f32>,+ run: RunOptions,+ epoch: u64,+ reply: Reply<Result<Transcript, EngineError>>,+ },+ StreamBegin {+ active: ActiveStream,+ run: RunOptions,+ stream: StreamOptions,+ reply: Reply<Result<(), EngineError>>,+ },+ Feed {+ id: u64,+ pcm: Vec<f32>,+ },+ Finalize {+ id: u64,+ want_language: bool,+ epoch: u64,+ reply: Reply<Result<Option<Finalized>, EngineError>>,+ },+ StreamReset {+ id: u64,+ },+}++/// State readable without going through the queue.+#[derive(Default)]+struct Shared {+ /// Whether this host may use a GPU at all. When not, every worker is+ /// CPU-only, device probes included.+ gpu_allowed: bool,+ /// A worker with GPU backends crashed or hung (in backend init, device+ /// listing, model load or inference). Models load in CPU-only workers,+ /// and devices are not listed again, until [`EngineSupervisor::retry_gpu`].+ gpu_unavailable: AtomicBool,+ /// The loaded model (logically: it stays loaded while its worker is+ /// replaced after a crash or cancel).+ loaded: Mutex<Option<LoadedInfo>>,+ /// The compute devices as last listed by a worker; `None` until one has.+ devices: Mutex<Option<Vec<DeviceInfo>>>,+ /// Held for the length of a device probe, so concurrent callers wait for+ /// the one probe instead of starting their own.+ probing: Mutex<()>,+ /// Bumped by every [`EngineSupervisor::cancel`]. Work queued before the+ /// bump counts as cancelled.+ cancel_epoch: AtomicU64,+ /// The call the owner is waiting on, if any. A cancel takes it and kills+ /// its worker.+ in_flight: Mutex<Option<InFlight>>,+ /// Bumped by every [`EngineSupervisor::unload`] as it is called, so the+ /// model reads as unloaded at once, even while the unload waits its turn.+ unloads_requested: AtomicU64,+ next_id: AtomicU64,+}++impl Shared {+ /// Start a model-less worker just to list devices. If it may use the GPU+ /// and crashes or hangs doing so, the GPU is marked unavailable.+ fn probe_devices(&self) {+ let cpu_only = !self.gpu_allowed;+ match self.hello(cpu_only, true) {+ Ok(worker) => drop(worker),+ Err(failure) => {+ error!(+ "Could not start a transcription worker to list compute devices: {}",+ failure+ );+ if failure.worker_lost() && !cpu_only {+ self.mark_gpu_unavailable(format!(+ "a worker with GPU backends {failure} while listing compute devices"+ ));+ }+ }+ }+ }++ /// Start a worker and say hello, publishing its device list if it lists+ /// one.+ fn hello(&self, cpu_only: bool, list_devices: bool) -> Result<Worker, Failure> {+ let mut worker = Worker::spawn(cpu_only).map_err(Failure::Spawn)?;+ match worker.call(&Request::Hello { list_devices }, None, HELLO_TIMEOUT)? {+ Response::Hello { devices } => {+ if let Some(devices) = devices {+ self.set_devices(devices);+ }+ Ok(worker)+ }+ other => Err(unexpected(&other)),+ }+ }++ fn set_devices(&self, new: Vec<DeviceInfo>) {+ let mut current = lock(&self.devices);+ if let Some(old) = &*current {+ let identity = |d: &DeviceInfo| (d.key.clone(), d.index);+ if !old.iter().map(identity).eq(new.iter().map(identity)) {+ info!(+ "transcribe-cpp compute devices changed: [{}]",+ new.iter()+ .map(|d| format!("{} ({})", d.name, d.kind))+ .collect::<Vec<_>>()+ .join(", ")+ );+ }+ }+ *current = Some(new);+ }++ fn gpu_unavailable(&self) -> bool {+ self.gpu_unavailable.load(Ordering::Acquire)+ }++ /// `reason` is logged when this changes the state.+ fn mark_gpu_unavailable(&self, reason: impl fmt::Display) {+ if !self.gpu_unavailable.swap(true, Ordering::AcqRel) {+ warn!("Transcription GPU marked unavailable: {}", reason);+ }+ }++ fn epoch(&self) -> u64 {+ self.cancel_epoch.load(Ordering::SeqCst)+ }++ fn cancelled_since(&self, epoch: u64) -> bool {+ self.epoch() != epoch+ }++ fn next_id(&self) -> u64 {+ self.next_id.fetch_add(1, Ordering::Relaxed) + 1+ }+}++struct InFlight {+ what: &'static str,+ control: Arc<Control>,+}++fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {+ mutex.lock().unwrap_or_else(|e| e.into_inner())+}++/// Owns the one transcribe.cpp worker slot. Cheap to clone.+#[derive(Clone)]+pub struct EngineSupervisor {+ commands: mpsc::Sender<Command>,+ shared: Arc<Shared>,+}++impl EngineSupervisor {+ /// Start the owner thread. No worker runs until something needs one.+ /// Without `gpu_allowed`, every worker is CPU-only, so no GPU driver is+ /// ever loaded.+ pub fn new(gpu_allowed: bool) -> Self {+ #[cfg(all(unix, not(test)))]+ record_launched_exe();+ let (commands, queue) = mpsc::channel();+ let shared = Arc::new(Shared {+ gpu_allowed,+ ..Shared::default()+ });+ let owner = Owner {+ shared: Arc::clone(&shared),+ worker: None,+ spec: None,+ stream: None,+ unloads_done: 0,+ };+ thread::Builder::new()+ .name("transcribe-engine".into())+ .spawn(move || owner.run(queue))+ .expect("failed to start the transcription engine thread");+ Self { commands, shared }+ }++ fn ask<T>(&self, command: impl FnOnce(Reply<T>) -> Command) -> Option<T> {+ let (reply, answer) = mpsc::channel();+ self.commands.send(command(reply)).ok()?;+ answer.recv().ok()+ }++ /// Load a model, replacing the current one. The old worker has fully+ /// exited before the new one starts, so two models are never held at+ /// once.+ pub fn load(&self, spec: LoadSpec) -> Result<LoadedInfo, EngineError> {+ self.ask(|reply| Command::Load(spec, reply))+ .unwrap_or_else(|| Err(engine_stopped()))+ }++ /// Unload the model, without waiting. It reads as unloaded at once and+ /// the unload is queued ahead of anything requested later, so a new+ /// dictation queues a fresh load behind it and is never affected by it.+ /// [`Unloading::wait`] returns once the worker has exited.+ pub fn unload(&self) -> Unloading {+ let (reply, exited) = mpsc::channel();+ {+ let mut loaded = lock(&self.shared.loaded);+ self.shared.unloads_requested.fetch_add(1, Ordering::SeqCst);+ *loaded = None;+ if self.commands.send(Command::Unload(reply)).is_err() {+ return Unloading(None);+ }+ }+ Unloading(Some(exited))+ }++ /// An explicit transcribe.cpp accelerator or GPU device change: forget+ /// the GPU failure, if any, so the next load may use (and list) the GPU+ /// again. A new failure sets the flag again.+ pub fn retry_gpu(&self, reason: &str) {+ if self.shared.gpu_unavailable.swap(false, Ordering::AcqRel) {+ info!("Transcription GPU no longer marked unavailable: {}", reason);+ }+ }++ /// The loaded model. A snapshot; never waits on the queue.+ pub fn loaded(&self) -> Option<LoadedInfo> {+ lock(&self.shared.loaded).clone()+ }++ /// Transcribe 16 kHz mono PCM with the loaded model. A worker crash or+ /// hang is retried once in a fresh worker (on CPU if it was on the GPU).+ pub fn transcribe(&self, pcm: Vec<f32>, run: RunOptions) -> Result<Transcript, EngineError> {+ let epoch = self.shared.epoch();+ self.ask(|reply| Command::Transcribe {+ pcm,+ run,+ epoch,+ reply,+ })+ .unwrap_or_else(|| Err(engine_stopped()))+ }++ /// Begin a live stream on the loaded model. `on_progress` runs on the+ /// owner thread after every feed, so it must not block or call back into+ /// the supervisor.+ pub fn start_stream(+ &self,+ run: RunOptions,+ stream: StreamOptions,+ on_progress: impl FnMut(StreamProgress) + Send + 'static,+ ) -> Result<StreamHandle, EngineError> {+ let id = self.shared.next_id();+ // The stream is one piece of cancellable work: a cancel from here on+ // stops its feeds and its finalize.+ let epoch = self.shared.epoch();+ let closed = Arc::new(AtomicBool::new(false));+ self.ask(|reply| Command::StreamBegin {+ active: ActiveStream {+ id,+ epoch,+ closed: Arc::clone(&closed),+ on_progress: Box::new(on_progress),+ pending_ms: 0,+ },+ run,+ stream,+ reply,+ })+ .unwrap_or_else(|| Err(engine_stopped()))?;+ Ok(StreamHandle {+ id,+ epoch,+ closed,+ commands: self.commands.clone(),+ finished: false,+ })+ }++ /// Stop the transcription or stream in progress, and any queued behind+ /// it. Bypasses the queue. The worker doing the work is killed, so the+ /// next use starts a fresh one and reloads the model. Model loads are+ /// never cancelled.+ pub fn cancel(&self) {+ self.shared.cancel_epoch.fetch_add(1, Ordering::SeqCst);+ // Taking the call tells the owner its worker was killed.+ if let Some(call) = lock(&self.shared.in_flight).take() {+ info!(+ "Cancelling the {} by stopping its worker (pid {})",+ call.what, call.control.pid+ );+ call.control.kill();+ }+ }++ /// The compute devices, as listed by the latest worker. If no worker has+ /// listed them yet, starts a short-lived worker on this thread to list+ /// them. It runs beside any model worker, which it leaves alone. Not+ /// while the GPU is marked unavailable: that probe would crash or hang+ /// the same way.+ pub fn devices(&self) -> Option<Vec<DeviceInfo>> {+ let _probing = lock(&self.shared.probing);+ if lock(&self.shared.devices).is_none() && !self.shared.gpu_unavailable() {+ self.shared.probe_devices();+ }+ lock(&self.shared.devices).clone()+ }+}++/// An unload in progress; see [`EngineSupervisor::unload`].+pub struct Unloading(Option<mpsc::Receiver<()>>);++impl Unloading {+ /// Wait until the worker has exited (e.g. before deleting the model file).+ pub fn wait(self) {+ if let Some(exited) = self.0 {+ let _ = exited.recv();+ }+ }+}++/// A live stream in the worker. Feeds are queued without waiting, in order+/// with finalize. Dropping the handle without finalizing resets the stream.+pub struct StreamHandle {+ id: u64,+ /// Cancel epoch when the stream started.+ epoch: u64,+ /// Set when the handle is dropped; shared with the owner.+ closed: Arc<AtomicBool>,+ commands: mpsc::Sender<Command>,+ finished: bool,+}++impl StreamHandle {+ pub fn feed(&self, pcm: Vec<f32>) {+ let _ = self.commands.send(Command::Feed { id: self.id, pcm });+ }++ /// Flush the stream and return its final text. `Ok(None)` if the stream+ /// was lost or failed; fall back to batch transcription then.+ /// `Err(Cancelled)` if [`EngineSupervisor::cancel`] stopped it.+ pub fn finalize(mut self, want_language: bool) -> Result<Option<Finalized>, EngineError> {+ self.finished = true;+ let (reply, answer) = mpsc::channel();+ let command = Command::Finalize {+ id: self.id,+ want_language,+ epoch: self.epoch,+ reply,+ };+ if self.commands.send(command).is_err() {+ return Ok(None);+ }+ answer.recv().unwrap_or(Ok(None))+ }+}++impl Drop for StreamHandle {+ fn drop(&mut self) {+ if !self.finished {+ // Queued feeds for this stream are skipped from here on.+ self.closed.store(true, Ordering::Release);+ let _ = self.commands.send(Command::StreamReset { id: self.id });+ }+ }+}++fn engine_stopped() -> EngineError {+ EngineError::Failed("the transcription engine has stopped".to_string())+}++fn not_loaded() -> EngineError {+ EngineError::Failed("no transcription model is loaded".to_string())+}++/// How a call to the worker failed.+#[derive(Debug)]+enum Failure {+ /// The worker process could not be started.+ Spawn(io::Error),+ /// transcribe.cpp returned an error; the worker is fine.+ Remote(String),+ /// The worker exited or crashed. Carries exit status and its last+ /// native output.+ Crashed(String),+ /// The worker did not answer within the deadline and was killed.+ Hung(Duration),+ Cancelled,+}++impl Failure {+ /// Whether the worker is gone.+ fn worker_lost(&self) -> bool {+ matches!(self, Failure::Crashed(_) | Failure::Hung(_))+ }+}++impl fmt::Display for Failure {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ match self {+ Failure::Spawn(e) => write!(f, "failed to start transcription worker: {e}"),+ Failure::Remote(message) => f.write_str(message),+ Failure::Crashed(detail) => write!(f, "transcription worker crashed ({detail})"),+ Failure::Hung(t) => write!(f, "transcription worker did not respond within {t:?}"),+ Failure::Cancelled => f.write_str("cancelled"),+ }+ }+}++impl From<Failure> for EngineError {+ fn from(failure: Failure) -> Self {+ match failure {+ Failure::Cancelled => EngineError::Cancelled,+ other => EngineError::Failed(other.to_string()),+ }+ }+}++fn unexpected(response: &Response) -> Failure {+ Failure::Remote(format!(+ "unexpected transcription worker response: {response:?}"+ ))+}++// --- Owner thread ---------------------------------------------------------++struct ActiveStream {+ id: u64,+ /// Cancel epoch when the stream started; a later cancel stops it.+ epoch: u64,+ /// Set when the caller drops its handle.+ closed: Arc<AtomicBool>,+ on_progress: OnProgress,+ /// Audio received but not yet committed, which finalize must still+ /// decode; sizes the feed and finalize deadlines.+ pending_ms: i64,+}++struct Owner {+ shared: Arc<Shared>,+ /// The single worker slot.+ worker: Option<Worker>,+ /// The model the app asked for. Outlives its worker: after a crash,+ /// hang or cancel the next use starts a fresh worker for it.+ spec: Option<LoadSpec>,+ stream: Option<ActiveStream>,+ /// Unload commands processed, against [`Shared::unloads_requested`].+ unloads_done: u64,+}++impl Owner {+ fn run(mut self, queue: mpsc::Receiver<Command>) {+ while let Ok(command) = queue.recv() {+ match command {+ Command::Load(spec, reply) => {+ let _ = reply.send(self.load(spec));+ }+ Command::Unload(reply) => {+ self.unload();+ self.unloads_done += 1;+ let _ = reply.send(());+ }+ Command::Transcribe {+ pcm,+ run,+ epoch,+ reply,+ } => {+ let _ = reply.send(self.transcribe(&pcm, &run, epoch));+ }+ Command::StreamBegin {+ active,+ run,+ stream,+ reply,+ } => {+ let _ = reply.send(self.stream_begin(active, &run, &stream));+ }+ Command::Feed { id, pcm } => self.feed(id, &pcm),+ Command::Finalize {+ id,+ want_language,+ epoch,+ reply,+ } => {+ let _ = reply.send(self.finalize(id, want_language, epoch));+ }+ Command::StreamReset { id } => {+ if self.stream.as_ref().is_some_and(|s| s.id == id) {+ self.end_stream();+ }+ }+ }+ }+ // Every handle is gone.+ self.unload();+ }++ fn load(&mut self, spec: LoadSpec) -> Result<LoadedInfo, EngineError> {+ self.unload();+ self.spec = Some(spec);+ let result = self.start();+ if result.is_err() {+ self.spec = None;+ }+ result+ }++ fn unload(&mut self) {+ // The worker's exit ends any stream; no need to reset it first.+ self.stream = None;+ if let Some(worker) = self.worker.take() {+ let started = Instant::now();+ let pid = worker.control.pid;+ drop(worker);+ debug!(+ "Transcription worker (pid {}) stopped in {}ms",+ pid,+ started.elapsed().as_millis()+ );+ }+ self.spec = None;+ *lock(&self.shared.loaded) = None;+ }++ /// The spec a fresh worker should load: CPU while the GPU is marked+ /// unavailable, unless the device was chosen explicitly.+ fn effective_spec(&self, spec: &LoadSpec) -> LoadSpec {+ if spec.pinned() || spec.cpu_only() || !self.shared.gpu_unavailable() {+ return spec.clone();+ }+ info!("Loading the transcription model on CPU: the GPU failed earlier");+ spec.cpu()+ }++ /// Whether a worker for `spec` registers only the CPU backends.+ fn cpu_only_worker(&self, spec: &LoadSpec) -> bool {+ spec.cpu_only() || !self.shared.gpu_allowed+ }++ /// Start a worker for `spec` and load it there. A worker that may use the+ /// GPU lists the devices in its hello, which keeps the list current (a+ /// GPU that comes back is just noticed). A CPU-only worker would only+ /// see the CPU, so it doesn't list.+ fn start_worker(&self, spec: &LoadSpec) -> Result<(Worker, LoadedInfo), Failure> {+ let cpu_only = self.cpu_only_worker(spec);+ let mut worker = self.shared.hello(cpu_only, !cpu_only)?;+ let info = worker.load(spec)?;+ Ok((worker, info))+ }++ /// Start a worker and load the current spec in it. If a worker with GPU+ /// backends is lost on the way (backend init, device listing or the+ /// load), mark the GPU unavailable and retry once in a CPU-only worker.+ fn start(&mut self) -> Result<LoadedInfo, EngineError> {+ let spec = self.spec.clone().ok_or_else(not_loaded)?;+ let started = Instant::now();+ let wanted = self.effective_spec(&spec);+ let (worker, info) = match self.start_worker(&wanted) {+ Ok(started) => started,+ Err(failure)+ if failure.worker_lost() && !self.cpu_only_worker(&wanted) && !wanted.pinned() =>+ {+ self.shared.mark_gpu_unavailable(format!(+ "a worker for '{}' with GPU backends {}",+ wanted.path.display(),+ failure+ ));+ warn!("Retrying in a CPU-only worker");+ self.start_worker(&wanted.cpu())?+ }+ Err(failure) => return Err(failure.into()),+ };+ debug!(+ "Worker (pid {}) spawned and loaded the model in {}ms",+ worker.control.pid,+ started.elapsed().as_millis()+ );+ self.publish_loaded(&info);+ self.worker = Some(worker);+ Ok(info)+ }++ /// Report the model loaded, unless an unload was requested since and is+ /// still waiting in the queue.+ fn publish_loaded(&self, info: &LoadedInfo) {+ let mut loaded = lock(&self.shared.loaded);+ if self.shared.unloads_requested.load(Ordering::SeqCst) == self.unloads_done {+ *loaded = Some(info.clone());+ }+ }++ /// Make sure the slot holds a worker with the model loaded, starting a+ /// fresh one if the last one died or was cancelled.+ fn ensure_worker(&mut self) -> Result<(), EngineError> {+ if self.worker.is_some() {+ return Ok(());+ }+ if self.spec.is_none() {+ return Err(not_loaded());+ }+ info!("Starting a fresh transcription worker for the loaded model");+ if let Err(e) = self.start() {+ // The model can't be brought back; report it unloaded so the+ // next use loads it from scratch (and reports why it can't).+ self.spec = None;+ *lock(&self.shared.loaded) = None;+ return Err(e);+ }+ Ok(())+ }++ /// Drop a worker that crashed or hung (its handle already logged the+ /// details and native output). If it was on a GPU, mark the GPU+ /// unavailable so the next worker uses CPU. Returns whether it was.+ fn worker_lost(&mut self, during: &str, failure: &Failure) -> bool {+ // Dropping the handle reaps the process.+ let info = self.worker.take().and_then(|worker| worker.info.clone());+ let pinned = self.spec.as_ref().is_some_and(LoadSpec::pinned);+ match info {+ Some(info) if info.on_gpu => {+ if !pinned {+ self.shared.mark_gpu_unavailable(format!(+ "{failure} during {during} on GPU '{}' ({})",+ info.device, info.backend+ ));+ }+ true+ }+ _ => false,+ }+ }++ /// Deadlines for the device the current worker's model runs on.+ fn deadlines(&self) -> Deadlines {+ let on_gpu = self+ .worker+ .as_ref()+ .and_then(|worker| worker.info.as_ref())+ .is_some_and(|info| info.on_gpu);+ Deadlines::for_device(on_gpu)+ }++ /// A call that [`EngineSupervisor::cancel`] can stop. Returns+ /// `Cancelled` if a cancel came after `epoch`, whatever the worker said.+ fn call_work(+ &mut self,+ request: &Request,+ pcm: Option<&[f32]>,+ deadline: Duration,+ epoch: u64,+ ) -> Result<Response, Failure> {+ let worker = self.worker.as_mut().expect("caller ensured a worker");+ let frame = encode_request(request, pcm)?;+ let sent = {+ // Check, send and publish under the lock cancel() takes after+ // bumping the epoch: either this sees the cancel and sends+ // nothing, or the cancel sees the call and kills its worker.+ let mut in_flight = lock(&self.shared.in_flight);+ if self.shared.cancelled_since(epoch) {+ return Err(Failure::Cancelled);+ }+ let sent = worker.control.send(frame);+ *in_flight = Some(InFlight {+ what: request_name(request),+ control: Arc::clone(&worker.control),+ });+ sent+ };+ let result = if sent {+ worker.wait(request, deadline)+ } else {+ Err(worker.died())+ };+ // Gone if cancel() took the call, which means it killed the worker+ // (possibly just after it answered).+ let killed = lock(&self.shared.in_flight).take().is_none();+ if killed || self.shared.cancelled_since(epoch) {+ if killed || result.as_ref().is_err_and(Failure::worker_lost) {+ // The next use starts a fresh worker.+ self.worker = None;+ }+ return Err(Failure::Cancelled);+ }+ result+ }++ /// Retry once after a crash or hang, on CPU if it happened on the GPU.+ /// Not after a hang on CPU, and never for an explicitly chosen device.+ fn transcribe(+ &mut self,+ pcm: &[f32],+ run: &RunOptions,+ epoch: u64,+ ) -> Result<Transcript, EngineError> {+ // A stream left open would make the worker refuse the run.+ self.end_stream();+ let request = Request::Run {+ options: run.clone(),+ };+ let mut retried = false;+ loop {+ if self.shared.cancelled_since(epoch) {+ return Err(EngineError::Cancelled);+ }+ // A respawn here runs under the load deadline; the run's own+ // deadline starts only once the model is loaded.+ self.ensure_worker()?;+ let deadline = self.deadlines().run(pcm.len());+ match self.call_work(&request, Some(pcm), deadline, epoch) {+ Ok(Response::Transcript(transcript)) => return Ok(transcript),+ Ok(other) => return Err(unexpected(&other).into()),+ Err(failure) if failure.worker_lost() => {+ let on_gpu = self.worker_lost("transcription", &failure);+ let pinned = self.spec.as_ref().is_some_and(LoadSpec::pinned);+ let hung_on_cpu = matches!(failure, Failure::Hung(_)) && !on_gpu;+ if retried || pinned || hung_on_cpu {+ return Err(failure.into());+ }+ warn!("Retrying the transcription in a fresh worker");+ retried = true;+ }+ Err(failure) => return Err(failure.into()),+ }+ }+ }++ fn stream_begin(+ &mut self,+ active: ActiveStream,+ run: &RunOptions,+ stream: &StreamOptions,+ ) -> Result<(), EngineError> {+ self.end_stream();+ if self.shared.cancelled_since(active.epoch) {+ return Err(EngineError::Cancelled);+ }+ self.ensure_worker()?;+ let worker = self.worker.as_mut().expect("worker was just ensured");+ let request = Request::StreamBegin {+ run: run.clone(),+ stream: stream.clone(),+ };+ match worker.call(&request, None, CALL_FLOOR) {+ Ok(Response::Ok) => {+ self.stream = Some(active);+ Ok(())+ }+ Ok(other) => Err(unexpected(&other).into()),+ Err(failure) => {+ if failure.worker_lost() {+ self.worker_lost("stream start", &failure);+ }+ Err(failure.into())+ }+ }+ }++ /// Feeds never respawn the worker (the stream state would be lost); a+ /// lost stream just drops the rest of its frames and finalizes to+ /// `None`, and the caller falls back to batch. Feeds are cancellable like+ /// any other dictation work.+ fn feed(&mut self, id: u64, pcm: &[f32]) {+ let Some(stream) = self.stream.as_ref().filter(|s| s.id == id) else {+ return;+ };+ // Skipped once the caller dropped its handle or cancelled.+ if stream.closed.load(Ordering::Acquire)+ || self.shared.cancelled_since(stream.epoch)+ || self.worker.is_none()+ {+ return;+ }+ let epoch = stream.epoch;+ let deadline = self+ .deadlines()+ .audio(pcm.len() + ms_to_samples(stream.pending_ms));+ let started = Instant::now();+ match self.call_work(&Request::Feed, Some(pcm), deadline, epoch) {+ Ok(Response::Fed { update, text }) => {+ let Some(stream) = self.stream.as_mut() else {+ return;+ };+ stream.pending_ms = pending_ms(&update);+ let progress = StreamProgress {+ update,+ text,+ elapsed: started.elapsed(),+ };+ if catch_unwind(AssertUnwindSafe(|| (stream.on_progress)(progress))).is_err() {+ error!("Live transcription callback panicked");+ }+ }+ Ok(other) => warn!("Stream feed failed: {}", unexpected(&other)),+ Err(Failure::Remote(message)) => warn!("Stream feed failed: {}", message),+ // Later feeds are skipped, and finalize reports the cancel.+ Err(Failure::Cancelled) => {}+ Err(failure) => {+ self.stream = None;+ self.worker_lost("stream feed", &failure);+ }+ }+ }++ fn finalize(+ &mut self,+ id: u64,+ want_language: bool,+ epoch: u64,+ ) -> Result<Option<Finalized>, EngineError> {+ // Checked first: a stream lost to a cancel must report the cancel,+ // not "no result", which would start a batch run.+ if self.shared.cancelled_since(epoch) {+ if self.stream.as_ref().is_some_and(|s| s.id == id) {+ self.end_stream();+ }+ return Err(EngineError::Cancelled);+ }+ let Some(stream) = self.stream.take_if(|s| s.id == id) else {+ // Lost earlier (or replaced): fall back to batch.+ return Ok(None);+ };+ if self.worker.is_none() {+ return Ok(None);+ }+ let deadline = self.deadlines().audio(ms_to_samples(stream.pending_ms));+ let started = Instant::now();+ match self.call_work(&Request::Finalize { want_language }, None, deadline, epoch) {+ Ok(Response::Finalized {+ update,+ text,+ language,+ }) => Ok(Some(Finalized {+ update,+ text,+ language,+ elapsed: started.elapsed(),+ })),+ Ok(other) => {+ error!("Stream finalize failed: {}", unexpected(&other));+ Ok(None)+ }+ Err(Failure::Cancelled) => {+ // The finalize may never have reached the worker.+ self.reset_worker_stream();+ Err(EngineError::Cancelled)+ }+ Err(Failure::Remote(message)) => {+ error!("Stream finalize failed: {}", message);+ Ok(None)+ }+ Err(failure) => {+ self.worker_lost("stream finalize", &failure);+ Ok(None)+ }+ }+ }++ fn end_stream(&mut self) {+ if self.stream.take().is_some() {+ self.reset_worker_stream();+ }+ }++ fn reset_worker_stream(&mut self) {+ let Some(worker) = self.worker.as_mut() else {+ return;+ };+ match worker.call(&Request::StreamReset, None, CALL_FLOOR) {+ Ok(_) => {}+ // Expected when the stream already ended in the worker.+ Err(Failure::Remote(message)) => debug!("Stream reset: {}", message),+ Err(failure) => {+ self.worker_lost("stream reset", &failure);+ }+ }+ }+}++fn pending_ms(update: &StreamUpdate) -> i64 {+ update+ .buffered_ms+ .max(update.input_received_ms - update.audio_committed_ms)+ .max(0)+}++// --- Worker process handle ------------------------------------------------++/// The parts of a worker that other threads touch: its stdin queue and the+/// process (to kill it). Every write goes through the one writer thread, so+/// a cancel can never interleave with a large frame.+struct Control {+ pid: u32,+ child: Mutex<Child>,+ stdin: Mutex<Option<mpsc::Sender<Vec<u8>>>>,+}++impl Control {+ fn send(&self, frame: Vec<u8>) -> bool {+ lock(&self.stdin)+ .as_ref()+ .is_some_and(|stdin| stdin.send(frame).is_ok())+ }++ /// Closing stdin tells the worker to exit (it ends on EOF).+ fn close_stdin(&self) {+ lock(&self.stdin).take();+ }++ fn kill(&self) {+ let _ = lock(&self.child).kill();+ }+}++/// One worker process, owned by the supervisor's thread. Strict+/// request/response. Dropping it closes the worker's stdin, gives it+/// [`SHUTDOWN_GRACE`] to exit, kills it otherwise, and reaps it: once+/// dropped, the worker is gone.+struct Worker {+ control: Arc<Control>,+ responses: mpsc::Receiver<Response>,+ stderr_done: mpsc::Receiver<()>,+ stderr_tail: Arc<Mutex<VecDeque<String>>>,+ /// Set once a model is loaded.+ info: Option<LoadedInfo>,+}++impl Worker {+ /// Start a worker process. A `cpu_only` worker never registers a GPU+ /// backend.+ fn spawn(cpu_only: bool) -> io::Result<Self> {+ // Under `cargo test` the current exe is the test harness; tests point+ // this at the built app instead.+ #[cfg(test)]+ let exe = PathBuf::from(+ std::env::var_os("HANDY_TRANSCRIBE_WORKER_EXE")+ .expect("set HANDY_TRANSCRIBE_WORKER_EXE"),+ );+ #[cfg(not(test))]+ let exe = worker_exe()?;+ let mut command = ProcessCommand::new(exe);+ #[cfg(target_os = "linux")]+ if let Some(arg0) = std::env::args_os().next() {+ // Keep `ps` showing Handy's own path rather than /proc/self/exe.+ use std::os::unix::process::CommandExt;+ command.arg0(arg0);+ }+ command.arg(WORKER_FLAG);+ if cpu_only {+ command.arg(CPU_ONLY_FLAG);+ }+ command+ .env(LOG_LEVEL_ENV, log::max_level().as_str())+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::piped());+ #[cfg(windows)]+ {+ use std::os::windows::process::CommandExt;+ const CREATE_NO_WINDOW: u32 = 0x0800_0000;+ command.creation_flags(CREATE_NO_WINDOW);+ }+ let mut child = command.spawn()?;+ let pid = child.id();+ let stdin = child.stdin.take().expect("stdin is piped");+ let stdout = child.stdout.take().expect("stdout is piped");+ let stderr = child.stderr.take().expect("stderr is piped");++ let (stdin_tx, frames) = mpsc::channel();+ let (response_tx, responses) = mpsc::channel();+ let (stderr_done_tx, stderr_done) = mpsc::channel();+ #[cfg(test)]+ LIVE_WORKERS.fetch_add(1, Ordering::SeqCst);+ // Built before the threads so a failure below still reaps the child.+ let worker = Worker {+ control: Arc::new(Control {+ pid,+ child: Mutex::new(child),+ stdin: Mutex::new(Some(stdin_tx)),+ }),+ responses,+ stderr_done,+ stderr_tail: Arc::new(Mutex::new(VecDeque::with_capacity(STDERR_TAIL_LINES))),+ info: None,+ };++ thread::Builder::new()+ .name("transcribe-worker-in".into())+ .spawn(move || write_frames(stdin, frames))?;+ thread::Builder::new()+ .name("transcribe-worker-out".into())+ .spawn(move || read_responses(stdout, response_tx))?;+ // Always drain stderr: a full pipe would block the worker.+ let tail = Arc::clone(&worker.stderr_tail);+ thread::Builder::new()+ .name("transcribe-worker-err".into())+ .spawn(move || {+ // Split on raw bytes: native code may write non-UTF-8 (e.g. a+ // path in the Windows ANSI code page), and `lines()` would end+ // the loop there, losing every later line, abort message included.+ for line in BufReader::new(stderr).split(b'\n') {+ let Ok(line) = line else { break };+ let line = String::from_utf8_lossy(&line);+ let line = line.strip_suffix('\r').unwrap_or(&line).to_string();+ forward_worker_log(&line);+ let mut tail = lock(&tail);+ if tail.len() == STDERR_TAIL_LINES {+ tail.pop_front();+ }+ tail.push_back(line);+ }+ let _ = stderr_done_tx.send(());+ })?;++ debug!(+ "Started {}transcription worker (pid {})",+ if cpu_only { "CPU-only " } else { "" },+ pid+ );+ Ok(worker)+ }++ fn load(&mut self, spec: &LoadSpec) -> Result<LoadedInfo, Failure> {+ let request = Request::Load {+ path: spec.path.clone(),+ backend: spec.backend,+ device: spec.device.clone(),+ };+ match self.call(&request, None, LOAD_TIMEOUT)? {+ Response::Loaded(info) => {+ self.info = Some(info.clone());+ Ok(info)+ }+ other => Err(unexpected(&other)),+ }+ }++ /// Send one request and wait up to `deadline` for its response. Hitting+ /// the deadline kills the worker; after any `Crashed`/`Hung` the handle+ /// is dead and should be dropped.+ fn call(+ &mut self,+ request: &Request,+ pcm: Option<&[f32]>,+ deadline: Duration,+ ) -> Result<Response, Failure> {+ if !self.control.send(encode_request(request, pcm)?) {+ return Err(self.died());+ }+ self.wait(request, deadline)+ }++ /// Wait for the response to `request`, already sent.+ fn wait(&mut self, request: &Request, deadline: Duration) -> Result<Response, Failure> {+ match self.responses.recv_timeout(deadline) {+ Ok(Response::Error(message)) => Err(Failure::Remote(message)),+ Ok(response) => Ok(response),+ Err(mpsc::RecvTimeoutError::Timeout) => {+ self.control.kill();+ let tail = self.native_tail();+ error!(+ "Transcription worker (pid {}) did not answer {} within {:?}; killed it{}",+ self.control.pid,+ request_name(request),+ deadline,+ if tail.is_empty() {+ String::new()+ } else {+ format!(". Native output before the hang:\n{}", tail.join("\n"))+ }+ );+ Err(Failure::Hung(deadline))+ }+ Err(mpsc::RecvTimeoutError::Disconnected) => Err(self.died()),+ }+ }++ /// The worker's stdout closed: collect its exit status and last output.+ fn died(&mut self) -> Failure {+ // Let the stderr thread catch the final lines (the abort message).+ let _ = self.stderr_done.recv_timeout(Duration::from_secs(1));+ let status = {+ let mut child = lock(&self.control.child);+ match wait_exit(&mut child, Duration::from_secs(1)) {+ Some(status) => status.to_string(),+ None => {+ let _ = child.kill();+ "unresponsive after closing its output; killed".to_string()+ }+ }+ };+ let tail = self.native_tail();+ if tail.is_empty() {+ error!(+ "Transcription worker (pid {}) died: {}",+ self.control.pid, status+ );+ } else {+ error!(+ "Transcription worker (pid {}) died: {}. Native output before exit:\n{}",+ self.control.pid,+ status,+ tail.join("\n")+ );+ }+ Failure::Crashed(match tail.last() {+ Some(last) => format!("{status}: {last}"),+ None => status,+ })+ }++ /// Recent raw native output (worker log lines are excluded; they were+ /// already re-logged).+ fn native_tail(&self) -> Vec<String> {+ lock(&self.stderr_tail)+ .iter()+ .filter(|line| !line.starts_with(LOG_LINE_PREFIX))+ .cloned()+ .collect()+ }+}++impl Drop for Worker {+ fn drop(&mut self) {+ self.control.close_stdin();+ let mut child = lock(&self.control.child);+ if wait_exit(&mut child, SHUTDOWN_GRACE).is_none() {+ warn!(+ "Transcription worker (pid {}) did not exit within {:?}; killing it",+ self.control.pid, SHUTDOWN_GRACE+ );+ let _ = child.kill();+ let _ = child.wait();+ }+ #[cfg(test)]+ LIVE_WORKERS.fetch_sub(1, Ordering::SeqCst);+ }+}++/// Wait up to `timeout` for `child` to exit. `None` if it is still running+/// (or its status can't be read).+fn wait_exit(child: &mut Child, timeout: Duration) -> Option<ExitStatus> {+ let deadline = Instant::now() + timeout;+ loop {+ match child.try_wait() {+ Ok(Some(status)) => return Some(status),+ Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(5)),+ _ => return None,+ }+ }+}++/// Worker processes started and not yet reaped. Every worker is reaped when+/// its handle drops, so this is exactly the worker processes still alive.+#[cfg(test)]+static LIVE_WORKERS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);++/// The file Handy was started from, with its identity (device and inode),+/// recorded at startup; see [`worker_exe`]. `None` if either couldn't be+/// read, which skips the check.+#[cfg(all(unix, not(test)))]+static LAUNCHED_EXE: std::sync::OnceLock<Option<(PathBuf, (u64, u64))>> =+ std::sync::OnceLock::new();++#[cfg(all(unix, not(test)))]+fn file_id(path: &std::path::Path) -> io::Result<(u64, u64)> {+ use std::os::unix::fs::MetadataExt;+ let metadata = std::fs::metadata(path)?;+ Ok((metadata.dev(), metadata.ino()))+}++/// Must run before an upgrade can replace the file, so at startup.+#[cfg(all(unix, not(test)))]+fn record_launched_exe() {+ LAUNCHED_EXE.get_or_init(|| {+ std::env::current_exe()+ .and_then(|path| {+ let id = file_id(&path)?;+ Ok((path, id))+ })+ .inspect_err(|e| warn!("Could not identify Handy's executable: {}", e))+ .ok()+ });+}++/// The executable a worker runs: always this very binary, so the two agree+/// on the protocol and on transcribe.cpp's backend libraries. Upgrading or+/// moving Handy while it runs replaces or removes the file it started from,+/// and a worker started from there would be the new version (macOS), or this+/// binary loading the new version's backend libraries (Linux). Only a+/// restart fixes that, so refuse to start one and say so. Windows locks a+/// running executable against replacement. On Linux the worker runs from+/// `/proc/self/exe`, which is this binary even once its file is replaced.+#[cfg(not(test))]+fn worker_exe() -> io::Result<PathBuf> {+ #[cfg(unix)]+ if let Some((path, launched)) = LAUNCHED_EXE.get().and_then(Option::as_ref) {+ if file_id(path).ok().as_ref() != Some(launched) {+ return Err(io::Error::other(+ "Handy was updated or moved while it was running; restart Handy",+ ));+ }+ }+ #[cfg(target_os = "linux")]+ return Ok(PathBuf::from("/proc/self/exe"));+ #[cfg(not(target_os = "linux"))]+ std::env::current_exe()+}++fn encode_request(request: &Request, pcm: Option<&[f32]>) -> Result<Vec<u8>, Failure> {+ encode_message(request, pcm)+ .map_err(|e| Failure::Remote(format!("failed to encode request: {e}")))+}++fn write_frames(mut stdin: ChildStdin, frames: mpsc::Receiver<Vec<u8>>) {+ // Ends when the handle closes stdin (dropping it here sends EOF) or the+ // worker is gone.+ for frame in frames {+ if stdin+ .write_all(&frame)+ .and_then(|()| stdin.flush())+ .is_err()+ {+ break;+ }+ }+}++fn read_responses(stdout: ChildStdout, responses: mpsc::Sender<Response>) {+ let mut stdout = BufReader::new(stdout);+ loop {+ match read_message::<Response>(&mut stdout) {+ Ok(Some((response, _))) => {+ if responses.send(response).is_err() {+ break;+ }+ }+ Ok(None) => break,+ Err(e) => {+ error!("Transcription worker protocol error: {}", e);+ break;+ }+ }+ }+}++fn request_name(request: &Request) -> &'static str {+ match request {+ Request::Hello { .. } => "hello",+ Request::Load { .. } => "model load",+ Request::Run { .. } => "transcription",+ Request::StreamBegin { .. } => "stream begin",+ Request::Feed => "stream feed",+ Request::Finalize { .. } => "stream finalize",+ Request::StreamReset => "stream reset",+ }+}++/// Re-log a worker stderr line in the parent. Structured lines keep their+/// level; anything else is raw native output.+fn forward_worker_log(line: &str) {+ const TARGET: &str = "transcribe_worker";+ if let Some(rest) = line.strip_prefix(LOG_LINE_PREFIX) {+ let mut parts = rest.splitn(3, '\t');+ if let (Some(level), Some(target), Some(message)) =+ (parts.next(), parts.next(), parts.next())+ {+ let level = level.parse::<Level>().unwrap_or(Level::Info);+ log::log!(target: TARGET, level, "[{}] {}", target, message);+ return;+ }+ }+ log::info!(target: TARGET, "{}", line);+}++#[cfg(test)]+mod unit_tests {+ use super::*;++ #[test]+ fn deadlines_scale_with_the_audio_above_a_floor() {+ let gpu = Deadlines::for_device(true);+ assert_eq!(gpu, GPU_DEADLINES);+ assert_eq!(gpu.audio(0), gpu.call_floor);+ assert_eq!(gpu.audio(16_000 / 2), gpu.call_floor);+ assert_eq!(gpu.audio(16_000 * 5), Duration::from_secs(50));+ assert_eq!(gpu.run(16_000), gpu.run_floor);+ assert_eq!(gpu.run(16_000 * 10), Duration::from_secs(100));++ let cpu = Deadlines::for_device(false);+ assert_eq!(cpu, CPU_DEADLINES);+ assert_eq!(cpu.audio(16_000), cpu.call_floor);+ assert_eq!(cpu.audio(16_000 * 10), Duration::from_secs(200));+ assert_eq!(cpu.run(16_000 * 5), cpu.run_floor);+ assert_eq!(cpu.run(16_000 * 60), Duration::from_secs(1200));+ // CPU is never stricter than GPU.+ for secs in [0, 1, 5, 30, 120, 600] {+ assert!(cpu.audio(secs * 16_000) >= gpu.audio(secs * 16_000));+ assert!(cpu.run(secs * 16_000) >= gpu.run(secs * 16_000));+ }++ assert_eq!(ms_to_samples(1500), 24_000);+ assert_eq!(ms_to_samples(-5), 0);+ }+}++/// End-to-end checks against a real worker and model. Crash and hang tests+/// need a debug worker exe (fault injection is compiled out of release), and+/// the stream tests a model that can stream.+/// Run with:+/// `HANDY_TRANSCRIBE_WORKER_EXE=target/debug/handy HANDY_TEST_MODEL=<gguf>+/// HANDY_TEST_WAV=<16 kHz mono wav> cargo test --lib engine_supervisor --+/// --ignored --nocapture --test-threads=1`+#[cfg(test)]+mod tests {+ use super::*;++ fn env_path(name: &str) -> PathBuf {+ PathBuf::from(std::env::var_os(name).unwrap_or_else(|| panic!("set {name}")))+ }++ fn test_pcm() -> Vec<f32> {+ crate::audio_toolkit::read_wav_samples(env_path("HANDY_TEST_WAV")).expect("read wav")+ }++ fn spec() -> LoadSpec {+ LoadSpec {+ path: env_path("HANDY_TEST_MODEL"),+ backend: Backend::Auto,+ device: DeviceSelector::Auto,+ }+ }++ /// Faults are read by each worker at spawn, so set them before loading.+ fn set_fault(fault: Option<&str>, gpu_only: bool) {+ let marker = std::env::temp_dir().join("handy-worker-fault-once");+ let _ = std::fs::remove_file(&marker);+ std::env::remove_var("HANDY_WORKER_FAULT_ONCE");+ std::env::remove_var("HANDY_WORKER_FAULT_GPU_ONLY");+ match fault {+ Some(fault) => {+ std::env::set_var("HANDY_WORKER_FAULT", fault);+ if gpu_only {+ std::env::set_var("HANDY_WORKER_FAULT_GPU_ONLY", "1");+ } else {+ std::env::set_var("HANDY_WORKER_FAULT_ONCE", marker);+ }+ }+ None => std::env::remove_var("HANDY_WORKER_FAULT"),+ }+ }++ /// The workers alive right now.+ fn worker_count() -> usize {+ LIVE_WORKERS.load(Ordering::SeqCst)+ }++ /// Feed 30 ms frames a little faster than a microphone would.+ fn feed_paced(stream: &StreamHandle, pcm: &[f32]) {+ for frame in pcm.chunks(480) {+ stream.feed(frame.to_vec());+ thread::sleep(Duration::from_millis(2));+ }+ }++ fn stream_all(+ engine: &EngineSupervisor,+ pcm: &[f32],+ ) -> Result<Option<Finalized>, EngineError> {+ let stream =+ engine.start_stream(RunOptions::default(), StreamOptions::default(), |_| {})?;+ feed_paced(&stream, pcm);+ stream.finalize(false)+ }++ /// Block until the owner is waiting on `what` (a [`request_name`]) in+ /// the worker, so a cancel lands on that work rather than before or+ /// after it, however fast the model is.+ fn wait_in_flight(engine: &EngineSupervisor, what: &str) {+ let started = Instant::now();+ while !lock(&engine.shared.in_flight)+ .as_ref()+ .is_some_and(|call| call.what == what)+ {+ assert!(+ started.elapsed() < Duration::from_secs(60),+ "{what} never started"+ );+ thread::sleep(Duration::from_millis(1));+ }+ }++ /// The test clip repeated to at least `secs` seconds.+ fn long_pcm(secs: usize) -> Vec<f32> {+ let clip = test_pcm();+ clip.repeat((secs * 16_000).div_ceil(clip.len()))+ }++ /// Never two workers: a reload or unload has fully stopped the old+ /// worker before anything else happens.+ #[test]+ #[ignore]+ fn one_worker_at_a_time() {+ set_fault(None, false);+ let engine = EngineSupervisor::new(true);+ for _ in 0..3 {+ engine.load(spec()).unwrap();+ assert_eq!(worker_count(), 1);+ }+ engine.unload().wait();+ assert_eq!(worker_count(), 0);+ assert!(engine.loaded().is_none());+ // A device probe with nothing loaded leaves no worker behind.+ assert!(engine.devices().is_some());+ assert_eq!(worker_count(), 0);+ }++ /// A worker crash mid-stream loses the stream; the batch fallback then+ /// runs in a fresh worker.+ #[test]+ #[ignore]+ fn crash_mid_stream_falls_back_to_batch() {+ let pcm = test_pcm();+ set_fault(Some("abort@feed"), false);+ let engine = EngineSupervisor::new(true);+ engine.load(spec()).unwrap();+ assert!(stream_all(&engine, &pcm).unwrap().is_none());+ let t = Instant::now();+ let transcript = engine.transcribe(pcm, RunOptions::default()).unwrap();+ println!(+ "batch fallback after crash {:?} on '{}': {}",+ t.elapsed(),+ engine.loaded().unwrap().backend,+ transcript.text+ );+ assert!(engine.loaded().is_some());+ set_fault(None, false);+ }++ /// A hung finalize is killed at its deadline (sized by the buffered+ /// audio, at least the device's call floor) and the batch fallback runs in a fresh+ /// worker.+ #[test]+ #[ignore]+ fn hung_finalize_is_killed_and_recovers() {+ let pcm = test_pcm();+ set_fault(Some("hang@finalize"), false);+ let engine = EngineSupervisor::new(true);+ engine.load(spec()).unwrap();+ let t = Instant::now();+ assert!(stream_all(&engine, &pcm).unwrap().is_none());+ println!("stream + hung finalize gave up after {:?}", t.elapsed());+ let t = Instant::now();+ let transcript = engine.transcribe(pcm, RunOptions::default()).unwrap();+ println!(+ "batch fallback after hang {:?}: {}",+ t.elapsed(),+ transcript.text+ );+ set_fault(None, false);+ }++ /// Cancelling a stream finalize reports `Cancelled`, never "no result"+ /// (which would start a batch run), and the next use works.+ #[test]+ #[ignore]+ fn cancel_during_finalize_is_cancelled() {+ let pcm = test_pcm();+ set_fault(Some("hang@finalize"), false);+ let engine = EngineSupervisor::new(true);+ engine.load(spec()).unwrap();+ let stream = engine+ .start_stream(RunOptions::default(), StreamOptions::default(), |_| {})+ .unwrap();+ feed_paced(&stream, &pcm);+ let finalize = thread::spawn(move || stream.finalize(false));+ wait_in_flight(&engine, "stream finalize");+ let t = Instant::now();+ engine.cancel();+ let result = finalize.join().unwrap();+ println!("finalize cancelled {:?} after cancel()", t.elapsed());+ assert!(matches!(result, Err(EngineError::Cancelled)));+ assert!(t.elapsed() < Duration::from_secs(2));+ assert!(!engine.shared.gpu_unavailable());+ engine.transcribe(pcm, RunOptions::default()).unwrap();+ set_fault(None, false);+ }++ /// Cancelling during a hung feed stops it at once (not at the feed+ /// deadline), skips the queued frames, and the finalize reports+ /// `Cancelled` even though the stream's worker is gone.+ #[test]+ #[ignore]+ fn cancel_during_hung_feed_is_cancelled() {+ let pcm = test_pcm();+ set_fault(Some("hang@feed"), false);+ let engine = EngineSupervisor::new(true);+ engine.load(spec()).unwrap();+ let stream = engine+ .start_stream(RunOptions::default(), StreamOptions::default(), |_| {})+ .unwrap();+ feed_paced(&stream, &pcm);+ wait_in_flight(&engine, "stream feed");+ let t = Instant::now();+ engine.cancel();+ let result = stream.finalize(false);+ println!("hung feed cancelled {:?} after cancel()", t.elapsed());+ assert!(matches!(result, Err(EngineError::Cancelled)));+ assert!(t.elapsed() < Duration::from_secs(2));+ assert!(!engine.shared.gpu_unavailable());+ engine.transcribe(pcm, RunOptions::default()).unwrap();+ set_fault(None, false);+ }++ /// An unload reads as unloaded at once, even while it waits behind a+ /// run, so a new dictation queues a fresh load instead of assuming the+ /// model is there.+ #[test]+ #[ignore]+ fn unload_reads_unloaded_at_once() {+ set_fault(None, false);+ let engine = EngineSupervisor::new(true);+ engine.load(spec()).unwrap();+ let run = {+ let engine = engine.clone();+ thread::spawn(move || engine.transcribe(long_pcm(60), RunOptions::default()))+ };+ wait_in_flight(&engine, "transcription");+ let unload = {+ let engine = engine.clone();+ thread::spawn(move || engine.unload().wait())+ };+ let t = Instant::now();+ while engine.loaded().is_some() {+ assert!(t.elapsed() < Duration::from_secs(1), "still reads loaded");+ thread::sleep(Duration::from_millis(1));+ }+ assert!(lock(&engine.shared.in_flight).is_some(), "run still going");+ run.join().unwrap().unwrap();+ unload.join().unwrap();+ assert!(engine.loaded().is_none());+ assert_eq!(worker_count(), 0);+ }++ /// An unload that is still waiting its turn can't affect a load+ /// requested after it (a new recording right after an "Immediately"+ /// unload): the newer model ends up loaded and usable.+ #[test]+ #[ignore]+ fn queued_unload_never_affects_a_later_load() {+ set_fault(None, false);+ let engine = EngineSupervisor::new(true);+ engine.load(spec()).unwrap();+ let run = {+ let engine = engine.clone();+ thread::spawn(move || engine.transcribe(long_pcm(60), RunOptions::default()))+ };+ wait_in_flight(&engine, "transcription");+ drop(engine.unload());+ assert!(+ engine.loaded().is_none(),+ "a new recording must see no model"+ );+ engine.load(spec()).unwrap();+ run.join().unwrap().unwrap();+ assert!(engine.loaded().is_some());+ assert_eq!(worker_count(), 1);+ engine+ .transcribe(test_pcm(), RunOptions::default())+ .unwrap();+ }++ /// A crash that only happens on the GPU: the dictation succeeds on CPU,+ /// the GPU is marked unavailable, and later loads use CPU.+ #[test]+ #[ignore]+ fn gpu_only_crash_recovers_on_cpu() {+ let pcm = test_pcm();+ set_fault(Some("abort@run"), true);+ let engine = EngineSupervisor::new(true);+ let info = engine.load(spec()).unwrap();+ assert!(info.on_gpu, "needs a GPU to test GPU-only faults");+ let transcript = engine+ .transcribe(pcm.clone(), RunOptions::default())+ .unwrap();+ println!("recovered on CPU: {}", transcript.text);+ assert!(engine.shared.gpu_unavailable());+ assert!(!engine.loaded().unwrap().on_gpu);+ let info = engine.load(spec()).unwrap();+ assert!(+ !info.on_gpu,+ "loads stay on CPU while the GPU is marked unavailable"+ );+ engine.transcribe(pcm, RunOptions::default()).unwrap();+ set_fault(None, false);+ }++ /// A GPU driver that crashes while its backend registers, before the+ /// hello: only a CPU-only worker survives it, so that is where the model+ /// loads. The GPU is marked unavailable, later loads stay CPU-only and devices+ /// aren't probed, until "try the GPU again" (a settings change).+ #[test]+ #[ignore]+ fn gpu_backend_init_crash_uses_cpu_only_worker() {+ let pcm = test_pcm();+ set_fault(Some("abort@init"), true);+ let engine = EngineSupervisor::new(true);+ let info = engine.load(spec()).unwrap();+ assert!(!info.on_gpu, "loaded on '{}'", info.backend);+ assert!(engine.shared.gpu_unavailable());+ assert!(engine.devices().is_none());+ engine.transcribe(pcm, RunOptions::default()).unwrap();+ // The fault still fires in any worker with GPU backends.+ assert!(!engine.load(spec()).unwrap().on_gpu);+ set_fault(None, false);+ engine.retry_gpu("test");+ assert!(engine.load(spec()).unwrap().on_gpu);+ assert!(engine.devices().is_some());+ }++ /// With the CPU accelerator, the model worker is CPU-only (a GPU-only+ /// init fault never fires), and a device probe runs beside it rather+ /// than stopping it.+ #[test]+ #[ignore]+ fn cpu_setting_uses_cpu_only_worker_and_probe_leaves_it_running() {+ let pcm = test_pcm();+ set_fault(Some("abort@init"), true);+ let engine = EngineSupervisor::new(true);+ let info = engine+ .load(LoadSpec {+ backend: Backend::Cpu,+ ..spec()+ })+ .unwrap();+ assert!(!info.on_gpu);+ assert!(!engine.shared.gpu_unavailable());+ set_fault(None, false);+ assert!(engine.devices().is_some());+ assert_eq!(worker_count(), 1);+ assert!(engine.loaded().is_some());+ engine.transcribe(pcm, RunOptions::default()).unwrap();+ }++ /// A host that must not use the GPU: even an `Auto` load and the device+ /// probe run CPU-only, so a GPU-only init fault never fires.+ #[test]+ #[ignore]+ fn gpu_disallowed_host_never_registers_a_gpu_backend() {+ set_fault(Some("abort@init"), true);+ let engine = EngineSupervisor::new(false);+ let devices = engine.devices().expect("devices not listed");+ assert!(devices.iter().all(|d| !d.is_gpu()), "{devices:?}");+ assert!(!engine.load(spec()).unwrap().on_gpu);+ assert!(!engine.shared.gpu_unavailable());+ set_fault(None, false);+ }++ /// The mask itself: a CPU-only worker registers no GPU device, including+ /// compiled-in ones such as Metal.+ #[test]+ #[ignore]+ fn cpu_only_worker_lists_no_gpu() {+ set_fault(None, false);+ let mut worker = Worker::spawn(true).unwrap();+ let Ok(Response::Hello {+ devices: Some(devices),+ }) = worker.call(&Request::Hello { list_devices: true }, None, HELLO_TIMEOUT)+ else {+ panic!("no device list");+ };+ println!("CPU-only worker devices: {devices:?}");+ assert!(!devices.is_empty());+ assert!(devices.iter().all(|d| !d.is_gpu()));+ }++ /// An explicit device never falls back to CPU.+ #[test]+ #[ignore]+ fn pinned_device_fails_loudly() {+ let pcm = test_pcm();+ set_fault(Some("abort@run"), true);+ let engine = EngineSupervisor::new(true);+ let devices = engine.devices().expect("devices not listed");+ let gpu = devices.iter().find(|d| d.is_gpu()).expect("needs a GPU");+ let info = engine+ .load(LoadSpec {+ device: DeviceSelector::Index(gpu.index.unwrap()),+ ..spec()+ })+ .unwrap();+ assert!(info.on_gpu);+ assert!(engine.transcribe(pcm, RunOptions::default()).is_err());+ assert!(!engine.shared.gpu_unavailable());+ set_fault(None, false);+ }++ /// Cancel mid-run: the run stops promptly, without a retry, the model+ /// still reads as loaded, and the next run works in a fresh worker.+ #[test]+ #[ignore]+ fn cancel_stops_a_run() {+ let pcm = long_pcm(60);+ set_fault(None, false);+ let engine = EngineSupervisor::new(true);+ engine.load(spec()).unwrap();+ let run = {+ let engine = engine.clone();+ let pcm = pcm.clone();+ thread::spawn(move || engine.transcribe(pcm, RunOptions::default()))+ };+ wait_in_flight(&engine, "transcription");+ let t = Instant::now();+ engine.cancel();+ let result = run.join().unwrap();+ println!("cancelled {:?} after cancel()", t.elapsed());+ assert!(matches!(result, Err(EngineError::Cancelled)));+ assert!(t.elapsed() < Duration::from_secs(2));+ assert!(engine.loaded().is_some());+ assert!(!engine.shared.gpu_unavailable());+ let t = Instant::now();+ engine+ .transcribe(test_pcm(), RunOptions::default())+ .unwrap();+ println!("next run took {:?}", t.elapsed());+ }++ /// Cancel a hung run: killed at once, no retry, no GPU+ /// flag, and the next run starts a fresh worker.+ #[test]+ #[ignore]+ fn cancel_kills_a_hung_run() {+ let pcm = test_pcm();+ set_fault(Some("hang@run"), false);+ let engine = EngineSupervisor::new(true);+ engine.load(spec()).unwrap();+ let run = {+ let engine = engine.clone();+ let pcm = pcm.clone();+ thread::spawn(move || engine.transcribe(pcm, RunOptions::default()))+ };+ wait_in_flight(&engine, "transcription");+ // Cancel only once the worker is actually hung, not on its way there.+ let marker = std::env::temp_dir().join("handy-worker-fault-once");+ while !marker.exists() {+ thread::sleep(Duration::from_millis(1));+ }+ let t = Instant::now();+ engine.cancel();+ let result = run.join().unwrap();+ println!("hung run cancelled {:?} after cancel()", t.elapsed());+ assert!(matches!(result, Err(EngineError::Cancelled)));+ assert!(t.elapsed() < Duration::from_secs(2));+ assert!(!engine.shared.gpu_unavailable());+ engine.transcribe(pcm, RunOptions::default()).unwrap();+ set_fault(None, false);+ }++ /// A worker hung in native code still exits when its parent goes away+ /// (stdin closes), without being killed.+ #[test]+ #[ignore]+ fn hung_worker_exits_when_stdin_closes() {+ set_fault(Some("hang@run"), false);+ let mut worker = Worker::spawn(false).unwrap();+ worker+ .call(+ &Request::Hello {+ list_devices: false,+ },+ None,+ HELLO_TIMEOUT,+ )+ .unwrap();+ let spec = spec();+ let load = Request::Load {+ path: spec.path,+ backend: spec.backend,+ device: spec.device,+ };+ worker.call(&load, None, LOAD_TIMEOUT).unwrap();+ let run = Request::Run {+ options: RunOptions::default(),+ };+ assert!(worker+ .control+ .send(encode_message(&run, Some(&test_pcm())).unwrap()));+ thread::sleep(Duration::from_secs(1));+ worker.control.close_stdin();+ let t = Instant::now();+ let status = loop {+ if let Some(status) = lock(&worker.control.child).try_wait().unwrap() {+ break status;+ }+ assert!(t.elapsed() < SHUTDOWN_GRACE, "worker outlived its stdin");+ thread::sleep(Duration::from_millis(10));+ };+ println!(+ "hung worker exited {:?} after stdin closed: {status}",+ t.elapsed()+ );+ assert!(status.success());+ set_fault(None, false);+ }+}diff --git a/src-tauri/src/engine_supervisor/worker.rs b/src-tauri/src/engine_supervisor/worker.rsnew file mode 100644index 00000000..1dd69c04--- /dev/null+++ b/src-tauri/src/engine_supervisor/worker.rs@@ -0,0 +1,475 @@+//! The worker process: the only place transcribe.cpp native code runs. Owns+//! the model, session and stream, and serves [`Request`]s until the parent+//! closes its stdin.+//!+//! A dedicated thread reads stdin so the parent's writes never block, and+//! the worker exits the moment the parent goes away, even if hung.++use super::protocol::{+ read_message, write_message, DeviceInfo, DeviceSelector, LoadedInfo, Request, Response,+};+use super::{CPU_ONLY_FLAG, LOG_LEVEL_ENV};+use log::{debug, error, warn, LevelFilter, Log, Metadata, Record};+use std::fs::File;+use std::io::{self, BufReader, Write};+use std::path::Path;+use std::sync::mpsc;+use std::thread;+use transcribe_cpp::{+ Backend, BackendMask, DeviceType, Feature, Model, ModelOptions, Session, Stream,+};++/// Prefix on worker log lines so the parent can tell them apart from raw+/// native output (e.g. a `GGML_ASSERT` message right before an abort).+pub(super) const LOG_LINE_PREFIX: &str = "\u{1}";++/// A request as handed from the stdin reader to the main loop.+struct Incoming {+ request: Request,+ pcm: Vec<f32>,+}++pub fn run() -> i32 {+ // The worker owns the per-run engine scratch, so it needs the same glibc+ // tuning as the app (#1792). Before anything allocates.+ crate::memory::init_allocator();+ #[cfg(unix)]+ ignore_app_signals();+ #[cfg(target_os = "linux")]+ set_process_name();+ // Take the real stdout for the protocol before any native code runs, and+ // point fd 1 at stderr: ggml writes to stdout in places, and any stray+ // byte there would corrupt the protocol stream.+ let protocol_out = match take_stdout_for_protocol() {+ Ok(file) => file,+ Err(e) => {+ eprintln!("transcribe worker: failed to claim stdout: {e}");+ return 2;+ }+ };+ init_logger();++ let (requests_tx, requests) = mpsc::channel();+ if let Err(e) = thread::Builder::new()+ .name("transcribe-worker-in".into())+ .spawn(move || read_requests(requests_tx))+ {+ error!("Failed to start the request reader: {}", e);+ return 2;+ }++ transcribe_cpp::init_logging();+ // Registering a GPU backend runs its driver code, so this is where a+ // broken driver crashes or hangs, before the hello is answered.+ let cpu_only = std::env::args_os().any(|arg| arg == CPU_ONLY_FLAG);+ inject_fault("init", !cpu_only);+ let init = if cpu_only {+ transcribe_cpp::init_backends_with(None::<&Path>, BackendMask::CPU)+ } else {+ transcribe_cpp::init_backends_default()+ };+ // Init only fails when no compute device registered at all, which no+ // other worker could fix either: the hello reports it.+ let init_error = init+ .err()+ .map(|e| format!("Failed to initialize transcribe.cpp backends: {e}"));+ debug!(+ "transcribe.cpp allowed backends: {:#x}",+ transcribe_cpp::allowed_backends().bits()+ );++ let mut output = protocol_out;+ let mut session: Option<(Session, LoadedInfo)> = None;++ while let Ok(Incoming { request, pcm }) = requests.recv() {+ let finished_run = matches!(request, Request::Run { .. });+ let response = match request {+ Request::Hello { list_devices } => match &init_error {+ Some(message) => Response::Error(message.clone()),+ None => Response::Hello {+ devices: list_devices.then(|| {+ inject_fault("list", !cpu_only);+ transcribe_cpp::devices()+ .iter()+ .map(DeviceInfo::from_device)+ .collect()+ }),+ },+ },+ Request::Load {+ path,+ backend,+ device,+ } => {+ // Free any previous model first to avoid holding two at once.+ session = None;+ match load(&path, backend, device) {+ Ok(loaded) => {+ let info = loaded.1.clone();+ session = Some(loaded);+ Response::Loaded(info)+ }+ Err(e) => Response::Error(e),+ }+ }+ Request::Run { options } => match session.as_mut() {+ Some((session, info)) => {+ inject_fault("run", info.on_gpu);+ match session.run(&pcm, &options) {+ Ok(transcript) => Response::Transcript(transcript),+ Err(e) => Response::Error(e.to_string()),+ }+ }+ None => not_loaded(),+ },+ Request::StreamBegin { run, stream } => match session.as_mut() {+ Some((session, info)) => match session.stream(&run, &stream) {+ Ok(stream) => {+ if write_message(&mut output, &Response::Ok, None).is_err() {+ return 1;+ }+ let on_gpu = info.on_gpu;+ if let Err(e) = serve_stream(stream, &requests, &mut output, on_gpu) {+ error!("Protocol write failed during stream: {}", e);+ return 1;+ }+ crate::memory::trim_freed_memory();+ continue;+ }+ Err(e) => Response::Error(e.to_string()),+ },+ None => not_loaded(),+ },+ Request::Feed | Request::Finalize { .. } | Request::StreamReset => {+ Response::Error("no active stream".to_string())+ }+ };+ if let Err(e) = write_message(&mut output, &response, None) {+ error!("Protocol write failed: {}", e);+ return 1;+ }+ // After the reply, so it never delays the transcript.+ if finished_run {+ drop(pcm);+ crate::memory::trim_freed_memory();+ }+ }+ // Unreachable in practice: the reader exits the process on EOF.+ 0+}++/// The stdin reader. Forwards requests to the main loop, and ends the+/// process when the parent closes stdin (unload, quit, or the parent died).+/// Cancels never reach the worker: the parent kills it instead. `_exit`+/// skips C++ static destructors, so a model still alive at that point can't+/// trip ggml-metal's teardown asserts, and the OS reclaims its CPU and GPU+/// memory.+fn read_requests(requests: mpsc::Sender<Incoming>) -> ! {+ let mut input = BufReader::new(io::stdin().lock());+ loop {+ match read_message::<Request>(&mut input) {+ Ok(Some((request, pcm))) => {+ if requests.send(Incoming { request, pcm }).is_err() {+ exit_now(1);+ }+ }+ Ok(None) => exit_now(0),+ Err(e) => {+ error!("Protocol read failed: {}", e);+ exit_now(1);+ }+ }+ }+}++fn exit_now(code: i32) -> ! {+ let _ = io::stderr().flush();+ // SAFETY: `_exit` only ends the process; nothing runs afterwards.+ unsafe { libc::_exit(code) }+}++/// Serve requests against an active stream until it is finalized or reset.+fn serve_stream(+ mut stream: Stream<'_>,+ requests: &mpsc::Receiver<Incoming>,+ output: &mut impl Write,+ on_gpu: bool,+) -> io::Result<()> {+ while let Ok(Incoming { request, pcm }) = requests.recv() {+ let (response, done) = match request {+ Request::Feed => {+ inject_fault("feed", on_gpu);+ match stream.feed(&pcm) {+ Ok(update) => {+ let text = (update.committed_changed || update.tentative_changed)+ .then(|| stream.text());+ (Response::Fed { update, text }, false)+ }+ Err(e) => (Response::Error(e.to_string()), false),+ }+ }+ Request::Finalize { want_language } => {+ inject_fault("finalize", on_gpu);+ match stream.finalize() {+ Ok(update) => {+ let language = if want_language {+ stream.snapshot().language+ } else {+ None+ };+ let text = stream.text();+ (+ Response::Finalized {+ update,+ text,+ language,+ },+ true,+ )+ }+ // Finalize ends the stream even when it fails.+ Err(e) => (Response::Error(e.to_string()), true),+ }+ }+ Request::StreamReset => {+ stream.reset();+ (Response::Ok, true)+ }+ _ => (+ Response::Error("a stream is active; finalize or reset it first".to_string()),+ false,+ ),+ };+ write_message(output, &response, None)?;+ if done {+ break;+ }+ }+ Ok(())+}++fn not_loaded() -> Response {+ Response::Error("no model loaded".to_string())+}++fn load(+ path: &Path,+ backend: Backend,+ selector: DeviceSelector,+) -> Result<(Session, LoadedInfo), String> {+ let device = match selector {+ DeviceSelector::Auto => None,+ DeviceSelector::Key(key) => {+ let found = transcribe_cpp::devices().into_iter().find(|d| {+ let info = DeviceInfo::from_device(d);+ info.is_gpu() && info.key == key+ });+ if found.is_none() {+ warn!(+ "Stored transcribe GPU device '{}' is no longer available; using automatic device selection",+ key+ );+ }+ found+ }+ DeviceSelector::Index(index) => {+ let device = transcribe_cpp::devices()+ .into_iter()+ .find(|d| d.index == Some(index))+ .ok_or_else(|| {+ format!("No compute device with index {index} (see --list-devices)")+ })?;+ if matches!(device.device_type, DeviceType::Accel | DeviceType::Unknown) {+ return Err(format!(+ "Device index {index} ({}) cannot host a model",+ device.kind+ ));+ }+ Some(device)+ }+ };++ inject_fault("load", backend != Backend::Cpu);+ let model = Model::load_with(path, &ModelOptions { backend, device })+ .map_err(|e| format!("Failed to load model: {e}"))?;+ let session = model+ .session()+ .map_err(|e| format!("Failed to create session: {e}"))?;+ let bound = model.device().ok().map(|d| DeviceInfo::from_device(&d));+ let backend_name = model.backend();+ let info = LoadedInfo {+ arch: model.arch(),+ variant: model.variant(),+ on_gpu: match &bound {+ Some(device) => device.is_gpu(),+ None => !backend_name.is_empty() && !backend_name.eq_ignore_ascii_case("cpu"),+ },+ backend: backend_name,+ device: bound+ .as_ref()+ .map(|d| d.label().to_string())+ .unwrap_or_else(|| "unknown".to_string()),+ capabilities: model.capabilities(),+ supports_initial_prompt: model.supports(Feature::InitialPrompt),+ };+ Ok((session, info))+}++/// Debug-build fault injection for exercising crash/hang recovery:+/// `HANDY_WORKER_FAULT=<abort|segv|hang>@<init|list|load|run|feed|finalize>`+/// (`abort` is what `GGML_ASSERT` ends in, `segv` a real invalid memory write)+/// (`init` is backend registration, before the hello). With `HANDY_WORKER_FAULT_ONCE=<marker+/// path>` it fires only in the first worker to reach that stage (the marker+/// file records that it fired). With `HANDY_WORKER_FAULT_GPU_ONLY=1` it fires+/// only on a GPU (backend init and device listing count as GPU work unless+/// the worker is CPU-only), so recovery on CPU can succeed.+#[cfg(debug_assertions)]+fn inject_fault(stage: &str, on_gpu: bool) {+ let Ok(spec) = std::env::var("HANDY_WORKER_FAULT") else {+ return;+ };+ let Some((kind, at)) = spec.split_once('@') else {+ return;+ };+ if at != stage {+ return;+ }+ if !on_gpu && std::env::var_os("HANDY_WORKER_FAULT_GPU_ONLY").is_some() {+ return;+ }+ if let Some(marker) = std::env::var_os("HANDY_WORKER_FAULT_ONCE") {+ if std::fs::OpenOptions::new()+ .write(true)+ .create_new(true)+ .open(&marker)+ .is_err()+ {+ return;+ }+ }+ eprintln!("injected fault: {kind} at {stage}");+ match kind {+ "abort" => std::process::abort(),+ // SAFETY: deliberately crash the worker with a real invalid memory+ // write, as a faulting driver does: SIGSEGV on Unix, an access+ // violation on Windows (where raising SIGSEGV only exits with code+ // 3). On Unix, restore the default disposition first so Rust's+ // stack-overflow handler doesn't get involved.+ "segv" => unsafe {+ #[cfg(unix)]+ libc::signal(libc::SIGSEGV, libc::SIG_DFL);+ std::ptr::write_volatile(std::ptr::null_mut::<u32>(), 1);+ },+ "hang" => loop {+ std::thread::sleep(std::time::Duration::from_secs(3600));+ },+ _ => {}+ }+}++#[cfg(not(debug_assertions))]+fn inject_fault(_stage: &str, _on_gpu: bool) {}++/// Logs go to stderr as `\x01LEVEL\ttarget\tmessage` lines; the parent+/// re-logs them at the same level.+struct StderrLogger;++impl Log for StderrLogger {+ fn enabled(&self, metadata: &Metadata) -> bool {+ metadata.level() <= log::max_level()+ }++ fn log(&self, record: &Record) {+ if !self.enabled(record.metadata()) {+ return;+ }+ let message = record.args().to_string();+ let mut stderr = io::stderr().lock();+ for line in message.lines() {+ let _ = writeln!(+ stderr,+ "{LOG_LINE_PREFIX}{}\t{}\t{}",+ record.level(),+ record.target(),+ line+ );+ }+ }++ fn flush(&self) {+ let _ = io::stderr().flush();+ }+}++fn init_logger() {+ static LOGGER: StderrLogger = StderrLogger;+ let level = std::env::var(LOG_LEVEL_ENV)+ .ok()+ .and_then(|v| v.parse::<LevelFilter>().ok())+ .unwrap_or(LevelFilter::Info);+ if log::set_logger(&LOGGER).is_ok() {+ log::set_max_level(level);+ }+}++/// SIGUSR2 toggles transcription in the app (and WebKitGTK uses SIGUSR1).+/// Aimed at Handy by name, e.g. the README's `pkill -USR2 -n handy`, one+/// can reach this worker instead, and its default action would kill it.+#[cfg(unix)]+fn ignore_app_signals() {+ // SAFETY: setting a signal's disposition to SIG_IGN has no preconditions.+ unsafe {+ libc::signal(libc::SIGUSR1, libc::SIG_IGN);+ libc::signal(libc::SIGUSR2, libc::SIG_IGN);+ }+}++/// Started from /proc/self/exe, the kernel names this process "exe". Name it+/// as Handy's, but not `handy`: `pkill`/`killall` match names case-sensitively,+/// so `pkill -USR2 -n handy` keeps reaching the app, never this newer process.+#[cfg(target_os = "linux")]+fn set_process_name() {+ // SAFETY: PR_SET_NAME copies a NUL-terminated name of up to 16 bytes.+ unsafe {+ libc::prctl(libc::PR_SET_NAME, c"Handy-worker".as_ptr());+ }+}++#[cfg(unix)]+fn take_stdout_for_protocol() -> io::Result<File> {+ use std::os::fd::FromRawFd;+ // SAFETY: plain fd duplication; the new fd is owned by the returned File.+ unsafe {+ let fd = libc::dup(libc::STDOUT_FILENO);+ if fd < 0 {+ return Err(io::Error::last_os_error());+ }+ if libc::dup2(libc::STDERR_FILENO, libc::STDOUT_FILENO) < 0 {+ return Err(io::Error::last_os_error());+ }+ Ok(File::from_raw_fd(fd))+ }+}++#[cfg(windows)]+fn take_stdout_for_protocol() -> io::Result<File> {+ use std::os::windows::io::FromRawHandle;+ // Native code writes through the CRT's fd 1, so redirect that one. The+ // protocol uses the OS handle behind a CRT duplicate of the original.+ // SAFETY: CRT fd duplication; the duplicate's handle is owned by the File+ // for the rest of the process.+ unsafe {+ let fd = libc::dup(1);+ if fd < 0 {+ return Err(io::Error::last_os_error());+ }+ if libc::dup2(2, 1) < 0 {+ return Err(io::Error::last_os_error());+ }+ let handle = libc::get_osfhandle(fd);+ if handle == -1 {+ return Err(io::Error::last_os_error());+ }+ Ok(File::from_raw_handle(handle as _))+ }+}diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rsindex 898770bc..a41d62c5 100644--- a/src-tauri/src/lib.rs+++ b/src-tauri/src/lib.rs@@ -9,6 +9,7 @@ mod chinese_script;pub mod cli;mod clipboard;mod commands;+pub mod engine_supervisor;mod helpers;mod input;mod llm_client;@@ -206,10 +207,6 @@ fn initialize_core_logic(app_handle: &AppHandle) {let history_manager =Arc::new(HistoryManager::new(app_handle).expect("Failed to initialize history manager"));- // Initialize the transcribe-cpp native backend (logging + backend module- // registration) once, before any whisper model is loaded.- managers::transcription::init_transcribe_backend();-// Apply accelerator preferences before any model loadsmanagers::transcription::apply_accelerator_settings(app_handle);@@ -309,10 +306,8 @@ fn initialize_core_logic(app_handle: &AppHandle) {log::warn!("No model is currently loaded.");return;}- match transcription_manager.unload_model() {- Ok(()) => log::info!("Model unloaded via tray."),- Err(e) => log::error!("Failed to unload model via tray: {}", e),- }+ transcription_manager.request_unload();+ log::info!("Model unloaded via tray.");}"cancel" => {use crate::utils::cancel_current_operation;@@ -438,7 +433,14 @@ fn run_headless_transcription(app: &AppHandle, args: &CliArgs) -> i32 {// --list-devices: print registered compute devices (with indices) and exit.// Useful on multi-GPU machines to discover the index for --device-index.if args.list_devices {- let devices = crate::managers::transcription::describe_compute_devices();+ let tm = app.state::<Arc<TranscriptionManager>>();+ let devices = match crate::managers::transcription::describe_compute_devices(&tm) {+ Ok(devices) => devices,+ Err(e) => {+ eprintln!("error: {}", e);+ return 1;+ }+ };if devices.is_empty() {println!("No transcribe-cpp compute devices registered.");} else {@@ -914,19 +916,17 @@ pub fn run(cli_args: CliArgs) {TranscriptionManager::new(&app_handle, model_manager.clone()).expect("Failed to initialize transcription manager"),);+ managers::transcription::report_compute_devices(&transcription_manager);app_handle.manage(model_manager);app_handle.manage(transcription_manager);- managers::transcription::init_transcribe_backend();- managers::transcription::report_compute_devices();managers::transcription::apply_accelerator_settings(&app_handle);let handle = app_handle.clone();let args = cli_args.clone();std::thread::spawn(move || {let code = run_headless_guarded(|| run_headless_transcription(&handle, &args));- // Drop the loaded engine before teardown: ggml-metal's global- // device free asserts (SIGABRT) if a model's Metal resources- // are still alive at C++ static-destructor time.+ // Drop the loaded engine before exit so its transcription+ // worker is told to shut down cleanly.if let Some(tm) = handle.try_state::<Arc<TranscriptionManager>>() {let _ = tm.unload_model();}@@ -1031,10 +1031,14 @@ pub fn run(cli_args: CliArgs) {// stays off the startup path. get_available_accelerators then enumerates// ORT execution providers and transcribe-cpp compute devices; without this// the cost is paid synchronously when the user first opens Advanced- // settings, freezing the UI. Result is cached in a OnceLock.- std::thread::spawn(|| {- crate::managers::transcription::report_compute_devices();- let _ = crate::managers::transcription::get_available_accelerators();+ // settings, freezing the UI. The device list is kept by the+ // transcription engine and refreshed by every worker it starts that may+ // use the GPU (CPU-only workers can't see it).+ let devices_app_handle = app_handle.clone();+ std::thread::spawn(move || {+ let tm = devices_app_handle.state::<Arc<TranscriptionManager>>();+ crate::managers::transcription::report_compute_devices(&tm);+ let _ = crate::managers::transcription::get_available_accelerators(&tm);});// Hide tray icon if --no-tray was passed@@ -1095,6 +1099,8 @@ pub fn run(cli_args: CliArgs) {#[cfg(target_os = "macos")]apply_startup_activation_policy(&mut app, headless_mode);+ // `app` is only used by the macOS arm.+ #[cfg_attr(not(target_os = "macos"), allow(unused_variables))]app.run(|app, event| match &event {#[cfg(target_os = "macos")]tauri::RunEvent::Reopen { .. } => {@@ -1113,12 +1119,10 @@ pub fn run(cli_args: CliArgs) {}show_main_window(app);}- // Teardown transcribe.cpp before exit- tauri::RunEvent::Exit => {- if let Some(tm) = app.try_state::<Arc<TranscriptionManager>>() {- let _ = tm.unload_model();- }- }+ // No transcription teardown on exit: transcribe.cpp runs only in the+ // worker process, which exits by itself as soon as this process's end+ // of its stdin closes, even if it is hung. Waiting on an unload here+ // could block quitting behind a hung model load._ => {}});}diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rsindex 5322eb5b..db502060 100644--- a/src-tauri/src/main.rs+++ b/src-tauri/src/main.rs@@ -5,8 +5,6 @@ use clap::Parser;use handy_app_lib::CliArgs;fn main() {- let cli_args = CliArgs::parse();-#[cfg(target_os = "linux")]{// DMABUF renderer causes crashes on various GPU/display server configurations@@ -25,5 +23,13 @@ fn main() {}}+ // Transcription worker: runs transcribe.cpp in isolation and never touches+ // Tauri, the CLI, or single-instance handling. Checked after the env setup+ // above so the worker inherits it (e.g. the Vulkan layer opt-out).+ if handy_app_lib::engine_supervisor::is_worker_invocation() {+ std::process::exit(handy_app_lib::engine_supervisor::run_worker());+ }++ let cli_args = CliArgs::parse();handy_app_lib::run(cli_args)}diff --git a/src-tauri/src/managers/transcription.rs b/src-tauri/src/managers/transcription.rsindex 32de0c59..e3907236 100644--- a/src-tauri/src/managers/transcription.rs+++ b/src-tauri/src/managers/transcription.rs@@ -3,6 +3,10 @@ use crate::audio_toolkit::{remove_filler_words, OutputLanguageEvidence,};use crate::chinese_script::{convert_chinese_script, ChineseVariety};+use crate::engine_supervisor::{+ DeviceInfo, DeviceSelector, EngineError, EngineSupervisor, LoadSpec, LoadedInfo,+ StreamProgress, Unloading,+};use crate::managers::audio::AudioRecordingManager;use crate::managers::model::{EngineType, ModelManager};use crate::settings::{@@ -15,15 +19,12 @@ use serde::{Deserialize, Serialize};use specta::Type;use std::panic::{catch_unwind, AssertUnwindSafe};use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};-use std::sync::{mpsc, Arc, Condvar, Mutex, MutexGuard, OnceLock};+use std::sync::{mpsc, Arc, Condvar, Mutex, MutexGuard};use std::thread;use std::time::{Duration, Instant, SystemTime};use tauri::{AppHandle, Emitter, Manager};use tauri_specta::Event;-use transcribe_cpp::{- Backend, Feature, Model, ModelOptions, RunExtension, RunOptions, Session, StreamOptions, Task,- WhisperRunOptions,-};+use transcribe_cpp::{Backend, RunExtension, RunOptions, StreamOptions, Task, WhisperRunOptions};use transcribe_rs::{onnx::{canary::CanaryModel,@@ -38,7 +39,6 @@ use transcribe_rs::{};const STREAM_PERF_LOG_INTERVAL: Duration = Duration::from_secs(5);-const STREAM_FINALIZE_REPLY_TIMEOUT: Duration = Duration::from_secs(30);fn panic_payload_message(payload: &(dyn std::any::Any + Send)) -> String {if let Some(message) = payload.downcast_ref::<&str>() {@@ -101,12 +101,15 @@ pub struct StreamPhaseEvent {/// is processed before finalize runs.enum StreamCmd {Feed(Vec<f32>),- /// Flush the stream and reply with the final text, or `None` if no stream- /// was ever active (caller should fall back to batch transcription).- Finalize(mpsc::Sender<Option<FinalizedStreamText>>),+ /// Flush the stream and reply with the final text, or `None` if no+ /// usable stream exists (caller should fall back to batch transcription).+ /// `Err(Cancelled)` if the finalize itself was cancelled.+ Finalize(mpsc::Sender<StreamFinalizeReply>),Cancel,}+type StreamFinalizeReply = Result<Option<FinalizedStreamText>, EngineError>;+struct FinalizedStreamText {text: String,output_language: OutputLanguageEvidence,@@ -177,11 +180,10 @@ impl StreamRouter {}}-enum LoadedEngine {- /// Whisper-family models (whisper, breeze-asr, custom .bin/.gguf) via- /// transcribe-cpp. Holds the live `Session`, which keeps its `Model` alive- /// internally, so repeated dictation reuses the session without reloading.- TranscribeCpp(Session),+/// In-process ONNX engines. transcribe-cpp models (whisper family, parakeet,+/// custom .bin/.gguf) live in the [`EngineSupervisor`]'s worker process+/// instead, so a native crash or hang there cannot take the app down.+enum OnnxEngine {Parakeet(ParakeetModel),Moonshine(MoonshineModel),MoonshineStreaming(StreamingModel),@@ -214,14 +216,13 @@ impl Drop for LoadingGuard {}}-/// RAII guard that clears the streaming worker/lease flags on any worker exit --/// normal return, early return, or a panic in an engine call that unwinds the-/// detached worker thread. Tokens prevent an older worker from clearing a newer-/// worker's state if a start/finalize race ever slips through.+/// RAII guard that clears the streaming worker flags on any worker exit -+/// normal return, early return, or a panic that unwinds the detached worker+/// thread. Tokens prevent an older worker from clearing a newer worker's+/// state if a start/finalize race ever slips through.struct StreamWorkerGuard {worker_id: u64,active_stream_worker: Arc<AtomicU64>,- active_engine_lease: Arc<AtomicU64>,stream_active: Arc<AtomicBool>,}@@ -230,12 +231,6 @@ impl Drop for StreamWorkerGuard {if self.active_stream_worker.load(Ordering::Acquire) == self.worker_id {self.stream_active.store(false, Ordering::Release);}- let _ = self.active_engine_lease.compare_exchange(- self.worker_id,- 0,- Ordering::AcqRel,- Ordering::Acquire,- );let _ = self.active_stream_worker.compare_exchange(self.worker_id,0,@@ -247,7 +242,11 @@ impl Drop for StreamWorkerGuard {#[derive(Clone)]pub struct TranscriptionManager {- engine: Arc<Mutex<Option<LoadedEngine>>>,+ /// The transcribe-cpp engine: owns the worker process, its model, and+ /// recovery from crashes and hangs.+ engine: EngineSupervisor,+ /// The loaded ONNX engine, if the current model is an ONNX one.+ onnx: Arc<Mutex<Option<OnnxEngine>>>,model_manager: Arc<ModelManager>,app_handle: AppHandle,current_model_id: Arc<Mutex<Option<String>>>,@@ -261,12 +260,12 @@ pub struct TranscriptionManager {/// [`StreamRouter`]. Shared with the audio recorder so per-frame feeds skip/// Tauri state and the manager lock.router: Arc<StreamRouter>,- /// True only while a transcribe-cpp `Stream` is actually in flight (set by- /// the worker once `stream()` succeeds). Used for overlay/UI decisions.+ /// True only while a transcribe-cpp stream is actually in flight (set by+ /// the worker once the stream begins). Used for overlay/UI decisions.stream_active: Arc<AtomicBool>,- /// Streaming uses four independent flags: router open = frames should route,- /// worker active = no second worker may start, engine lease = engine is out- /// of the mutex, stream active = UI should show a live session.+ /// Streaming uses three independent flags: router open = frames should+ /// route, worker active = no second worker may start, stream active = UI+ /// should show a live session.////// Monotonic id source for stream workers; zero means "no worker".next_stream_worker_id: Arc<AtomicU64>,@@ -274,16 +273,13 @@ pub struct TranscriptionManager {/// yet. This prevents a second worker from starting after finalize/cancel/// closes the router but before the first worker has fully exited.active_stream_worker: Arc<AtomicU64>,- /// Nonzero while the streaming worker has taken the engine out of `engine`.- /// `is_model_loaded()` consults this so the model still reports "loaded"- /// while the worker holds it.- active_engine_lease: Arc<AtomicU64>,}impl TranscriptionManager {pub fn new(app_handle: &AppHandle, model_manager: Arc<ModelManager>) -> Result<Self> {let manager = Self {- engine: Arc::new(Mutex::new(None)),+ engine: EngineSupervisor::new(!transcribe_gpu_disabled_for_host()),+ onnx: Arc::new(Mutex::new(None)),model_manager,app_handle: app_handle.clone(),current_model_id: Arc::new(Mutex::new(None)),@@ -297,7 +293,6 @@ impl TranscriptionManager {stream_active: Arc::new(AtomicBool::new(false)),next_stream_worker_id: Arc::new(AtomicU64::new(1)),active_stream_worker: Arc::new(AtomicU64::new(0)),- active_engine_lease: Arc::new(AtomicU64::new(0)),};// Start the idle watcher@@ -374,18 +369,18 @@ impl TranscriptionManager {Ok(manager)}- /// Lock the engine mutex, recovering from poison if a previous transcription panicked.- fn lock_engine(&self) -> MutexGuard<'_, Option<LoadedEngine>> {- self.engine.lock().unwrap_or_else(|poisoned| {+ /// Lock the ONNX engine mutex, recovering from poison if a previous transcription panicked.+ fn lock_onnx(&self) -> MutexGuard<'_, Option<OnnxEngine>> {+ self.onnx.lock().unwrap_or_else(|poisoned| {warn!("Engine mutex was poisoned by a previous panic, recovering");poisoned.into_inner()})}pub fn is_model_loaded(&self) -> bool {- // The engine may be leased out to the streaming worker (taken out of- // the mutex). It's still loaded, just in use, so report true.- self.lock_engine().is_some() || self.active_engine_lease.load(Ordering::Acquire) != 0+ // A transcribe-cpp model stays loaded while it is busy, so a batch+ // run or stream in progress never reads as "unloaded".+ self.engine.loaded().is_some() || self.lock_onnx().is_some()}/// Accelerator changes should not disturb the current transcription. Mark@@ -395,6 +390,44 @@ impl TranscriptionManager {self.reload_model_on_next_use.store(true, Ordering::Release);}+ /// An explicit transcribe.cpp accelerator or GPU device change is the+ /// user's way of trying the GPU again: forget every GPU failure so far,+ /// so the next load may use (and list) the GPU. The ONNX accelerator does+ /// not affect this.+ pub fn retry_transcribe_gpu(&self, reason: &str) {+ self.engine.retry_gpu(reason);+ }++ /// The engine gives up on a model by itself when its worker can't be+ /// restarted (e.g. the file is gone, or it crashes on CPU too). Keep the+ /// loaded model id and the UI in step, so the next use loads the model+ /// again and reports why it can't.+ fn forget_model_if_engine_dropped(&self, model_id: &str, error: &str) {+ if self.engine.loaded().is_some() {+ return;+ }+ {+ let mut current_model = self.current_model_id.lock().unwrap();+ if current_model.as_deref() != Some(model_id) {+ return;+ }+ *current_model = None;+ }+ warn!(+ "Transcription model '{}' is no longer loaded: {}",+ model_id, error+ );+ let _ = self.app_handle.emit(+ "model-state-changed",+ ModelStateEvent {+ event_type: "unloaded".to_string(),+ model_id: None,+ model_name: None,+ error: Some(error.to_string()),+ },+ );+ }+/// Atomically check whether a model load is in progress and, if not, mark/// one as starting. Returns a [`LoadingGuard`] whose [`Drop`] impl will/// clear the flag and wake waiters. Returns `None` if a load is already in@@ -411,15 +444,34 @@ impl TranscriptionManager {})}+ /// Unload the model. Returns once the transcribe-cpp worker has exited,+ /// which frees all of its memory (e.g. before its file is deleted).pub fn unload_model(&self) -> Result<()> {let unload_start = std::time::Instant::now();debug!("Starting to unload model");+ self.begin_unload().wait();+ debug!(+ "Model unloaded manually (took {}ms)",+ unload_start.elapsed().as_millis()+ );+ Ok(())+ }- {- let mut engine = self.lock_engine();- // Dropping the engine frees all resources- *engine = None;- }+ /// Unload the model without waiting for the transcribe-cpp worker to+ /// exit, so it never blocks the caller (e.g. the event loop) behind a+ /// transcription in progress. The model reads as unloaded at once.+ pub fn request_unload(&self) {+ drop(self.begin_unload());+ }++ /// Every state change of an unload, done now on the caller's thread: a+ /// dictation started right after sees no model and loads it again, and+ /// its load is queued behind this unload, so the unload can never affect+ /// it. Only the worker's exit is left to wait for.+ fn begin_unload(&self) -> Unloading {+ let unloading = self.engine.unload();+ // Dropping an ONNX engine frees its resources.+ *self.lock_onnx() = None;{let mut current_model = self.current_model_id.lock().unwrap();*current_model = None;@@ -435,13 +487,7 @@ impl TranscriptionManager {error: None,},);-- let unload_duration = unload_start.elapsed();- debug!(- "Model unloaded manually (took {}ms)",- unload_duration.as_millis()- );- Ok(())+ unloading}fn now_ms() -> u64 {@@ -456,16 +502,16 @@ impl TranscriptionManager {self.last_activity.store(Self::now_ms(), Ordering::Relaxed);}- /// Unloads the model immediately if the setting is enabled and the model is loaded+ /// Unloads the model immediately if the setting is enabled and the model+ /// is loaded. Never waits for the worker to exit, so it is safe on any+ /// thread, including the event loop.pub fn maybe_unload_immediately(&self, context: &str) {let settings = get_settings(&self.app_handle);if settings.model_unload_timeout == ModelUnloadTimeout::Immediately&& self.is_model_loaded(){info!("Immediately unloading model after {}", context);- if let Err(e) = self.unload_model() {- warn!("Failed to immediately unload model: {}", e);- }+ self.request_unload();}}@@ -474,7 +520,7 @@ impl TranscriptionManager {}/// Like [`load_model`](Self::load_model), but lets a caller hard-select the- /// compute device for this one load by its `transcribe_cpp::devices()`+ /// compute device for this one load by its transcribe-cpp device/// registry index (the index shown by `--list-devices`). `None` keeps the/// persisted accelerator setting (which may be Auto). Only affects/// transcribe-cpp (whisper-family) models; the selection is not persisted.@@ -541,13 +587,14 @@ impl TranscriptionManager {.get_model_path(model_id).inspect_err(|error| emit_loading_failed(&error.to_string()))?;- // Drop the current engine BEFORE building the new one so transcribe-cpp- // frees the previous native context first — avoids holding two models at- // once (peak memory on large GGUFs). Clear the id too: if the new load- // fails, status should read "no loaded model", not the dropped engine.- {- let mut engine = self.lock_engine();- *engine = None;+ // Drop the current engine BEFORE building the new one so the previous+ // model is freed first — avoids holding two models at once (peak memory+ // on large GGUFs). A transcribe-cpp load replaces its worker the same+ // way. Clear the id too: if the new load fails, status should read "no+ // loaded model", not the dropped engine.+ *self.lock_onnx() = None;+ if !matches!(model_info.engine_type, EngineType::TranscribeCpp) {+ self.engine.unload().wait();}{let mut current_model = self.current_model_id.lock().unwrap();@@ -556,61 +603,51 @@ impl TranscriptionManager {// Create appropriate engine based on model type- let loaded_engine = match model_info.engine_type {+ let loaded_onnx = match model_info.engine_type {EngineType::TranscribeCpp => {- // The whisper backend is chosen at load time (transcribe-cpp has- // no runtime global). With an explicit `device_index` (the- // --device-index flag) hard-select that registered device;- // otherwise re-read the persisted accelerator preference (so an- // accelerator change marked for reload takes effect here).+ // The backend is chosen at load time. With an explicit+ // `device_index` (the --device-index flag) hard-select that+ // registered device; otherwise re-read the persisted+ // accelerator preference (so an accelerator change marked for+ // reload takes effect here). The worker resolves the selector+ // to a device handle, and rejects an index that isn't a+ // loadable device (a host without GPU access lists none).let (backend, device) = match device_index {- Some(index) => resolve_device_index(index).inspect_err(|e| {- emit_loading_failed(&e.to_string());- })?,+ Some(index) => (Backend::Auto, DeviceSelector::Index(index)),None => {let settings = get_settings(&self.app_handle);let accelerator = settings.transcribe_accelerator;- let device = resolve_gpu_device(+ match resolve_gpu_device(accelerator,settings.transcribe_gpu_device.as_deref(),- );- // Backend::Auto accepts an exact GPU device. Without a- // valid exact device, backend selection handles the- // retired generic GPU state and host CPU guard.- let backend = if device.is_some() {- Backend::Auto- } else {- select_transcribe_backend(accelerator)- };- (backend, device)+ ) {+ // Backend::Auto accepts an exact GPU device.+ Some(key) => (Backend::Auto, DeviceSelector::Key(key)),+ // Without a valid exact device, backend selection+ // handles the retired generic GPU state and host+ // CPU guard.+ None => (select_transcribe_backend(accelerator), DeviceSelector::Auto),+ }}};- let requested_device = device- .as_ref()- .map(transcribe_device_label)- .unwrap_or_else(|| "automatic".to_string());- let model_options = ModelOptions { backend, device };- let model = Model::load_with(&model_path, &model_options).map_err(|e| {- let error_msg = format!("Failed to load whisper model {}: {}", model_id, e);- emit_loading_failed(&error_msg);- anyhow::anyhow!(error_msg)- })?;- // The bound backend may differ from the request (e.g. CPU- // fallback under Auto); log what actually loaded.- let bound_backend = model.backend();- let session = model.session().map_err(|e| {- let error_msg = format!(- "Failed to create session for whisper model {}: {}",- model_id, e- );- emit_loading_failed(&error_msg);- anyhow::anyhow!(error_msg)- })?;+ let requested_device = format!("{:?}", device);+ let info = self+ .engine+ .load(LoadSpec {+ path: model_path.clone(),+ backend,+ device,+ })+ .map_err(|e| {+ let error_msg = format!("Failed to load model {}: {}", model_id, e);+ emit_loading_failed(&error_msg);+ anyhow::anyhow!(error_msg)+ })?;// Reconcile the registry's advertised capabilities with the// loaded model's real ones (GGUF metadata) so badges/gating// reflect runtime truth, not the pre-download probe. The// load-completed event below triggers the frontend refresh.- let caps = session.model().capabilities();+ let caps = &info.capabilities;self.model_manager.set_runtime_capabilities(model_id,caps.supports_streaming,@@ -618,24 +655,22 @@ impl TranscriptionManager {caps.supports_language_detect,caps.languages.clone(),);- let bound_device = model- .device()- .map(|device| transcribe_device_label(&device))- .unwrap_or_else(|_| "unknown".to_string());+ // The bound backend may differ from the request (e.g. CPU+ // fallback under Auto or after a GPU crash); log what loaded.info!(- "Loaded whisper model '{}' (requested {:?}, requested device '{}', \+ "Loaded transcribe-cpp model '{}' (requested {:?}, requested device {}, \bound backend '{}', bound device '{}', supports_streaming={}, \supports_translate={}, supports_language_detect={})",model_id,backend,requested_device,- bound_backend,- bound_device,+ info.backend,+ info.device,caps.supports_streaming,caps.supports_translate,caps.supports_language_detect);- LoadedEngine::TranscribeCpp(session)+ None}EngineType::Parakeet => {let engine =@@ -645,7 +680,7 @@ impl TranscriptionManager {emit_loading_failed(&error_msg);anyhow::anyhow!(error_msg)})?;- LoadedEngine::Parakeet(engine)+ Some(OnnxEngine::Parakeet(engine))}EngineType::Moonshine => {let engine = MoonshineModel::load(@@ -658,7 +693,7 @@ impl TranscriptionManager {emit_loading_failed(&error_msg);anyhow::anyhow!(error_msg)})?;- LoadedEngine::Moonshine(engine)+ Some(OnnxEngine::Moonshine(engine))}EngineType::MoonshineStreaming => {let engine = StreamingModel::load(&model_path, 0, &Quantization::default())@@ -670,7 +705,7 @@ impl TranscriptionManager {emit_loading_failed(&error_msg);anyhow::anyhow!(error_msg)})?;- LoadedEngine::MoonshineStreaming(engine)+ Some(OnnxEngine::MoonshineStreaming(engine))}EngineType::SenseVoice => {let engine =@@ -680,7 +715,7 @@ impl TranscriptionManager {emit_loading_failed(&error_msg);anyhow::anyhow!(error_msg)})?;- LoadedEngine::SenseVoice(engine)+ Some(OnnxEngine::SenseVoice(engine))}EngineType::GigaAM => {let engine = GigaAMModel::load(&model_path, &Quantization::Int8).map_err(|e| {@@ -688,7 +723,7 @@ impl TranscriptionManager {emit_loading_failed(&error_msg);anyhow::anyhow!(error_msg)})?;- LoadedEngine::GigaAM(engine)+ Some(OnnxEngine::GigaAM(engine))}EngineType::Canary => {let engine = CanaryModel::load(&model_path, &Quantization::Int8).map_err(|e| {@@ -696,7 +731,7 @@ impl TranscriptionManager {emit_loading_failed(&error_msg);anyhow::anyhow!(error_msg)})?;- LoadedEngine::Canary(engine)+ Some(OnnxEngine::Canary(engine))}EngineType::Cohere => {let engine = CohereModel::load(&model_path, &Quantization::Int8).map_err(|e| {@@ -704,15 +739,12 @@ impl TranscriptionManager {emit_loading_failed(&error_msg);anyhow::anyhow!(error_msg)})?;- LoadedEngine::Cohere(engine)+ Some(OnnxEngine::Cohere(engine))}};// Update the current engine and model ID- {- let mut engine = self.lock_engine();- *engine = Some(loaded_engine);- }+ *self.lock_onnx() = loaded_onnx;{let mut current_model = self.current_model_id.lock().unwrap();*current_model = Some(model_id.to_string());@@ -782,13 +814,10 @@ impl TranscriptionManager {/// its real backend string; ONNX engines report "onnx"; `None` when no/// model is loaded.pub fn current_backend(&self) -> Option<String> {- match self.lock_engine().as_ref() {- Some(LoadedEngine::TranscribeCpp(session)) => {- Some(session.model().backend().to_string())- }- Some(_) => Some("onnx".to_string()),- None => None,+ if let Some(info) = self.engine.loaded() {+ return Some(info.backend);}+ self.lock_onnx().as_ref().map(|_| "onnx".to_string())}/// Whether a live streaming run is currently in flight.@@ -837,7 +866,6 @@ impl TranscriptionManager {let _worker = StreamWorkerGuard {worker_id,active_stream_worker: Arc::clone(&self.active_stream_worker),- active_engine_lease: Arc::clone(&self.active_engine_lease),stream_active: Arc::clone(&self.stream_active),};@@ -852,79 +880,36 @@ impl TranscriptionManager {let model_id = self.get_current_model().unwrap_or_default();- // Take the engine out of the mutex so we own it during streaming,- // structurally excluding any concurrent batch transcription (which- // transcribe-cpp's compute_lock would refuse anyway). Returned when the- // worker exits, or dropped if the model was switched/unloaded mid-stream.- if self- .active_engine_lease- .compare_exchange(0, worker_id, Ordering::AcqRel, Ordering::Acquire)- .is_err()- {- warn!("Live preview: another worker already holds the transcription engine");+ // Only transcribe-cpp models expose streaming; ONNX engines fall back to+ // batch. The loaded model (not the ModelManager copy) is the source of+ // truth for run-path capabilities.+ let Some(info) = self.engine.loaded() else {+ info!(+ "Live preview: model '{}' is not a loaded transcribe-cpp model; \+ streaming is unavailable, using batch transcription",+ model_id+ );self.router.clear();drain_until_finalize(rx);return;- }- let mut engine = match self.lock_engine().take() {- Some(e) => e,- None => {- info!(- "Live preview: model '{}' was unloaded before streaming could begin; \- falling back to batch transcription",- model_id- );- let _ = self.active_engine_lease.compare_exchange(- worker_id,- 0,- Ordering::AcqRel,- Ordering::Acquire,- );- self.router.clear();- drain_until_finalize(rx);- return;- }- };-- // Only transcribe-cpp models expose streaming; ONNX engines fall back to- // batch. The loaded session (not the ModelManager copy) is the source of- // truth for run-path capabilities.- let (supports_streaming, supports_translate, languages) = match &engine {- LoadedEngine::TranscribeCpp(session) => {- let model = session.model();- let caps = model.capabilities();- info!(- "Live preview: model '{}' arch='{}' variant='{}' supports_streaming={} \- supports_translate={} languages={:?}",- model_id,- model.arch(),- model.variant(),- caps.supports_streaming,- caps.supports_translate,- caps.languages,- );- (- caps.supports_streaming,- caps.supports_translate,- caps.languages,- )- }- _ => {- info!(- "Live preview: model '{}' is not a transcribe-cpp model; \- streaming is unavailable, using batch transcription",- model_id- );- (false, false, Vec::new())- }};-- if !supports_streaming {- self.return_engine(engine, &model_id);+ let caps = &info.capabilities;+ info!(+ "Live preview: model '{}' arch='{}' variant='{}' supports_streaming={} \+ supports_translate={} languages={:?}",+ model_id,+ info.arch,+ info.variant,+ caps.supports_streaming,+ caps.supports_translate,+ caps.languages,+ );+ if !caps.supports_streaming {self.router.clear();drain_until_finalize(rx);return;}+ let languages = caps.languages.clone();// Build run options mirroring the offline transcribe-cpp path: task +// language gated against what the model actually advertises.@@ -935,7 +920,7 @@ impl TranscriptionManager {settings.translate_to_english,&effective_language,&languages,- supports_translate,+ caps.supports_translate,);let output_language = resolve_output_language_evidence(&settings,@@ -950,160 +935,126 @@ impl TranscriptionManager {..Default::default()};- // Run the stream on the held session. The Stream borrows the session- // (and thus the engine) for its lifetime, so the feed/finalize loop- // lives in a labeled block — when it exits, the borrow is released and- // the engine can be moved into return_engine().- let mut preview_script = PreviewScript::new(settings.chinese_script, &output_language);- let mut finalize_reply: Option<mpsc::Sender<Option<FinalizedStreamText>>> = None;- let mut finalize_result: Option<Option<FinalizedStreamText>> = None;- let stream_started = 'stream: {- let session = match &mut engine {- LoadedEngine::TranscribeCpp(s) => s,- _ => break 'stream false,- };-- // Read the backend string before beginning the stream — the- // `Stream` borrows `session` mutably for its lifetime, so we can't- // call `session.model()` once it exists.- let backend = session.model().backend();+ // Feed results arrive on the engine's thread; this callback only+ // converts and emits preview text.+ let perf = Arc::new(Mutex::new(StreamPerf::new()));+ let on_progress = {+ let perf = Arc::clone(&perf);+ let app_handle = self.app_handle.clone();+ let languages = languages.clone();+ let mut preview_script = PreviewScript::new(settings.chinese_script, &output_language);+ move |progress: StreamProgress| {+ let mut perf = lock_perf(&perf);+ perf.record_compute(progress.elapsed);+ perf.record_update(&progress.update);+ if let Some(text) = progress.text {+ perf.record_emit();+ let (committed, tentative) =+ preview_script.convert(&text.committed, &text.tentative, &languages);+ emit_stream_text(&app_handle, &committed, &tentative);+ }+ perf.maybe_log();+ }+ };- // StreamOptions::default() uses CommitPolicy::Auto and lets the- // family pick its own streaming strategy (no family-specific ext).- let mut stream = match session.stream(&run_options, &StreamOptions::default()) {- Ok(s) => s,+ // Run the stream in the engine's worker process. Feeds are queued+ // without waiting; a crashed or hung worker makes finalize report no+ // result, and the caller falls back to batch transcription. StreamOptions::default()+ // uses CommitPolicy::Auto and lets the family pick its own streaming+ // strategy (no family-specific ext).+ let stream =+ match self+ .engine+ .start_stream(run_options, StreamOptions::default(), on_progress)+ {+ Ok(stream) => stream,Err(e) => {error!("Failed to begin stream: {}", e);- break 'stream false;+ self.forget_model_if_engine_dropped(&model_id, &e.to_string());+ drain_until_finalize(rx);+ return;}};- self.stream_active.store(true, Ordering::Release);- self.touch_activity();- info!(- "Live streaming transcription started (model '{}', backend '{}')",- model_id, backend- );+ self.stream_active.store(true, Ordering::Release);+ self.touch_activity();+ info!(+ "Live streaming transcription started (model '{}', backend '{}')",+ model_id, info.backend+ );- let mut perf = StreamPerf::new();- while let Ok(cmd) = rx.recv() {- match cmd {- StreamCmd::Feed(pcm) => {- self.touch_activity();- perf.record_feed(pcm.len());- let feed_start = Instant::now();- match stream.feed(&pcm) {- Ok(update) => {- perf.record_compute(feed_start.elapsed());- perf.record_update(- update.revision,- update.input_received_ms,- update.audio_committed_ms,- update.buffered_ms,- );- if update.committed_changed || update.tentative_changed {- let text = stream.text();- perf.record_emit();- let (committed, tentative) = preview_script.convert(- &text.committed,- &text.tentative,- &languages,- );- self.emit_stream_text(&committed, &tentative);- }- perf.maybe_log();- }- Err(e) => {- perf.record_compute(feed_start.elapsed());- warn!("stream feed failed: {}", e);- }+ while let Ok(cmd) = rx.recv() {+ match cmd {+ StreamCmd::Feed(pcm) => {+ self.touch_activity();+ lock_perf(&perf).record_feed(pcm.len());+ stream.feed(pcm);+ }+ StreamCmd::Finalize(reply) => {+ // In auto mode the model's own LID is the best remaining+ // evidence; the snapshot is only materialized when it can+ // change the outcome.+ let want_language = matches!(output_language, OutputLanguageEvidence::Unknown);+ let result = match stream.finalize(want_language) {+ // After finalize the committed prefix holds the full+ // text; display() = committed + tentative is the safe read.+ Ok(Some(finalized)) => {+ let mut perf = lock_perf(&perf);+ perf.record_compute(finalized.elapsed);+ perf.record_update(&finalized.update);+ let output_language = match &output_language {+ OutputLanguageEvidence::Unknown => with_model_detected_language(+ OutputLanguageEvidence::Unknown,+ finalized.language,+ ),+ resolved => resolved.clone(),+ };+ Ok(Some(FinalizedStreamText {+ text: finalized.text.full,+ output_language,+ supported_languages: languages.clone(),+ }))}- }- StreamCmd::Finalize(reply) => {- let finalize_start = Instant::now();- let result = match stream.finalize() {- // After finalize the committed prefix holds the full- // text; display() = committed + tentative is the safe read.- Ok(update) => {- perf.record_compute(finalize_start.elapsed());- perf.record_update(- update.revision,- update.input_received_ms,- update.audio_committed_ms,- update.buffered_ms,- );- // In auto mode the model's own LID is the best- // remaining evidence; the snapshot is only- // materialized when it can change the outcome.- let output_language = match &output_language {- OutputLanguageEvidence::Unknown => {- with_model_detected_language(- OutputLanguageEvidence::Unknown,- stream.snapshot().language,- )- }- resolved => resolved.clone(),- };- Some(FinalizedStreamText {- text: stream.text().full,- output_language,- supported_languages: languages.clone(),- })- }- Err(e) => {- perf.record_compute(finalize_start.elapsed());- error!(- "stream finalize failed: {}; falling back to batch transcription",- e- );- None- }- };- let chars = match &result {- Some(finalized) => finalized.text.len(),- _ => 0,- };- perf.log_finalized(chars);- finalize_reply = Some(reply);- finalize_result = Some(result);- break;- }- StreamCmd::Cancel => {- stream.reset();- break;- }+ Ok(None) => {+ error!(+ "Live transcription could not be finalized; \+ falling back to batch transcription"+ );+ self.forget_model_if_engine_dropped(+ &model_id,+ "live transcription failed and the model could not be restarted",+ );+ Ok(None)+ }+ Err(e) => {+ info!("Live transcription finalize stopped: {}", e);+ Err(e)+ }+ };+ let chars = match &result {+ Ok(Some(finalized)) => finalized.text.len(),+ _ => 0,+ };+ lock_perf(&perf).log_finalized(chars);+ let _ = reply.send(result);+ return;}+ // Dropping the stream handle resets the worker's stream.+ StreamCmd::Cancel => return,}-- true- };- // `stream` + the `&mut engine` borrow are released here.-- if !stream_started {- // Stream never began (model doesn't support streaming or begin- // failed); drain so the finalize handshake still completes and the- // caller falls back to batch transcription. Return the engine first- // so the fallback can immediately use it.- self.return_engine(engine, &model_id);- drain_until_finalize(rx);- return;}-- self.return_engine(engine, &model_id);- if let (Some(reply), Some(result)) = (finalize_reply, finalize_result) {- let _ = reply.send(result);- }- // `_worker` drops here, clearing this worker's active/lease flags after- // the engine has been returned to the pool.+ // The channel closed without finalize/cancel; dropping the stream+ // handle resets the worker's stream. `_worker` drops after, clearing+ // this worker's flags.}- /// Return the leased engine to the mutex, unless the model was switched or- /// unloaded during transcription (in which case the stale engine is dropped).- fn return_engine(&self, engine: LoadedEngine, expected_model_id: &str) {+ /// Return the taken ONNX engine to the mutex, unless the model was switched+ /// or unloaded during transcription (in which case the stale engine is dropped).+ fn return_engine(&self, engine: OnnxEngine, expected_model_id: &str) {let still_current =self.current_model_id.lock().unwrap().as_deref() == Some(expected_model_id);if still_current {- *self.lock_engine() = Some(engine);+ *self.lock_onnx() = Some(engine);} else {info!("Model changed/unloaded during transcription; dropping stale engine (was '{}')",@@ -1115,10 +1066,11 @@ impl TranscriptionManager {/// Flush the active stream and return its final, post-filtered text.///- /// `Ok(None)` means no usable stream was active and the caller may fall back- /// to batch transcription. `Err` means finalize itself failed or timed out.- /// A timeout may still leave the worker holding the engine, so callers- /// should surface it instead of immediately starting a batch fallback.+ /// `Ok(None)` means no usable stream result exists and the caller should+ /// fall back to batch transcription. `Err` means the transcription was+ /// cancelled. There is no timeout here: waiting behind a model load is a+ /// queue position, not a failure (#1841), and the engine bounds the+ /// finalize itself by the audio it still has to decode.pub fn finalize_stream(&self) -> Result<Option<String>> {let Some(tx) = self.router.take() else {return Ok(None);@@ -1127,17 +1079,10 @@ impl TranscriptionManager {if tx.send(StreamCmd::Finalize(reply_tx)).is_err() {return Ok(None);}- let finalized = match reply_rx.recv_timeout(STREAM_FINALIZE_REPLY_TIMEOUT) {- Ok(Some(finalized)) => finalized,- Ok(None) => return Ok(None),- Err(mpsc::RecvTimeoutError::Disconnected) => return Ok(None),- Err(mpsc::RecvTimeoutError::Timeout) => {- self.stream_active.store(false, Ordering::Release);- return Err(anyhow::anyhow!(- "Timed out waiting {:?} for live transcription to finalize",- STREAM_FINALIZE_REPLY_TIMEOUT- ));- }+ let finalized = match reply_rx.recv() {+ Ok(Ok(Some(finalized))) => finalized,+ Ok(Ok(None)) | Err(_) => return Ok(None),+ Ok(Err(e)) => return Err(e.into()),};let settings = get_settings(&self.app_handle);@@ -1163,6 +1108,14 @@ impl TranscriptionManager {self.stream_active.store(false, Ordering::Release);}+ /// Stop the transcription in progress (a batch run or stream finalize),+ /// and any queued behind it, for an explicit user cancel. The worker doing+ /// it is stopped; the model stays loaded and the next use starts a fresh+ /// worker for it.+ pub fn cancel_transcription(&self) {+ self.engine.cancel();+ }+/// Emit a working-phase event to the streaming overlay (spinner + label).pub fn emit_stream_working(&self, kind: StreamWorkKind) {let _ = StreamPhaseEvent {@@ -1172,14 +1125,6 @@ impl TranscriptionManager {.emit(&self.app_handle);}- fn emit_stream_text(&self, committed: &str, tentative: &str) {- let _ = StreamTextEvent {- committed: committed.to_string(),- tentative: tentative.to_string(),- }- .emit(&self.app_handle);- }-pub fn transcribe(&self, audio: Vec<f32>) -> Result<String> {#[cfg(debug_assertions)]if std::env::var("HANDY_FORCE_TRANSCRIPTION_FAILURE").is_ok() {@@ -1210,8 +1155,7 @@ impl TranscriptionManager {is_loading = self.loading_condvar.wait(is_loading).unwrap();}- let engine_guard = self.lock_engine();- if engine_guard.is_none() {+ if !self.is_model_loaded() {return Err(anyhow::anyhow!("Model is not loaded for transcription."));}}@@ -1241,271 +1185,36 @@ impl TranscriptionManager {);}- // Whether the loaded transcribe-cpp model advertises- // Feature::InitialPrompt. Informational (logged below); the whisper- // run extension and the fuzzy-correction skip are gated on- // `model_is_whisper` instead, since non-whisper archs can advertise- // the feature while rejecting the whisper-kind extension.- let mut model_takes_initial_prompt = false;- // Whether the loaded model is actually whisper-family (arch string).- // Non-whisper archs (e.g. Voxtral Small) can advertise- // Feature::InitialPrompt yet reject the whisper-kind run extension- // with INVALID_ARG, so the whisper extension must be gated on the- // arch, not on the feature (see #1601).- let mut model_is_whisper = false;-// Perform transcription with the appropriate engine.- // We use catch_unwind to prevent engine panics from poisoning the mutex,- // which would make the app hang indefinitely on subsequent operations.- let (result, output_language, model_languages) = {- let mut engine_guard = self.lock_engine();-- // Take the engine out so we own it during transcription.- // If the engine panics, we simply don't put it back (effectively unloading it)- // instead of poisoning the mutex.- let mut engine = match engine_guard.take() {- Some(e) => e,- None => {- return Err(anyhow::anyhow!(- "Model failed to load after auto-load attempt. Please check your model settings."- ));- }- };-- // Release the lock before transcribing — no mutex held during the engine call- drop(engine_guard);-- // Probe live transcribe-cpp capabilities once (cheap GGUF-metadata- // reads); the loaded session is the source of truth, not the- // ModelManager copy. The whisper run extension is kind-tagged, so- // non-whisper archs (parakeet, voxtral, …) reject it with- // INVALID_ARG; attach it — and translate — only where supported.- let mut model_supports_translate = false;- let mut model_languages = self- .model_manager- .get_model_info(&active_model)- .map(|info| info.supported_languages)- .unwrap_or_default();- let mut output_was_translated = false;- let mut applied_language_hint: Option<String> = None;- let mut model_detected_language: Option<String> = None;- if let LoadedEngine::TranscribeCpp(session) = &engine {- let model = session.model();- let caps = model.capabilities();- model_takes_initial_prompt = model.supports(Feature::InitialPrompt);- model_is_whisper = model.arch() == "whisper";- model_supports_translate = caps.supports_translate;- model_languages = caps.languages;- debug!(- "transcribe-cpp model '{}' on '{}': initial_prompt={}, translate={}, languages={:?}",- settings.selected_model,- model.backend(),- model_takes_initial_prompt,- model_supports_translate,- model_languages- );+ let run = match self.engine.loaded() {+ Some(info) => {+ self.transcribe_cpp(info, audio, &settings, &validated_language, &active_model)?}-- let transcribe_result = catch_unwind(AssertUnwindSafe(|| -> Result<String> {- match &mut engine {- LoadedEngine::TranscribeCpp(session) => {- // Custom words become the initial prompt ONLY for models- // that accept one (whisper family). Attaching the- // whisper run extension to a non-whisper arch is rejected- // with INVALID_ARG, so skip it there and let the fuzzy- // post-correction handle custom words instead.- let family = if settings.custom_words.is_empty() || !model_is_whisper {- None- } else {- Some(RunExtension::Whisper(WhisperRunOptions {- initial_prompt: Some(settings.custom_words.join(", ")),- ..Default::default()- }))- };-- let run_plan = transcribe_cpp_run_plan(- settings.translate_to_english,- &validated_language,- &model_languages,- model_supports_translate,- );- output_was_translated = run_plan.target_language.as_deref() == Some("en");- applied_language_hint = run_plan.language.clone();-- let run_options = RunOptions {- task: run_plan.task,- language: run_plan.language,- target_language: run_plan.target_language,- family,- ..Default::default()- };-- debug!(- "transcribe-cpp run: task={:?}, language={:?}, initial_prompt={}",- run_options.task,- run_options.language,- run_options.family.is_some()- );-- session- .run(&audio, &run_options)- .map(|t| {- // Whisper's audio-based LID (auto mode only;- // `None` when a language hint was passed).- model_detected_language = t.language;- t.text- })- .map_err(|e| {- anyhow::anyhow!("transcribe-cpp transcription failed: {}", e)- })- }- LoadedEngine::Parakeet(parakeet_engine) => {- let params = ParakeetParams {- timestamp_granularity: Some(TimestampGranularity::Segment),- ..Default::default()- };- parakeet_engine- .transcribe_with(&audio, ¶ms)- .map(|r| r.text)- .map_err(|e| anyhow::anyhow!("Parakeet transcription failed: {}", e))- }- LoadedEngine::Moonshine(moonshine_engine) => moonshine_engine- .transcribe(&audio, &TranscribeOptions::default())- .map(|r| r.text)- .map_err(|e| anyhow::anyhow!("Moonshine transcription failed: {}", e)),- LoadedEngine::MoonshineStreaming(streaming_engine) => streaming_engine- .transcribe(&audio, &TranscribeOptions::default())- .map(|r| r.text)- .map_err(|e| {- anyhow::anyhow!("Moonshine streaming transcription failed: {}", e)- }),- LoadedEngine::SenseVoice(sense_voice_engine) => {- let language = match validated_language.as_str() {- "zh" => Some("zh".to_string()),- "en" => Some("en".to_string()),- "ja" => Some("ja".to_string()),- "ko" => Some("ko".to_string()),- "yue" => Some("yue".to_string()),- _ => None,- };- applied_language_hint = language.clone();- let params = SenseVoiceParams {- language,- use_itn: Some(true),- };- sense_voice_engine- .transcribe_with(&audio, ¶ms)- .map(|r| r.text)- .map_err(|e| anyhow::anyhow!("SenseVoice transcription failed: {}", e))- }- LoadedEngine::GigaAM(gigaam_engine) => gigaam_engine- .transcribe(&audio, &TranscribeOptions::default())- .map(|r| r.text)- .map_err(|e| anyhow::anyhow!("GigaAM transcription failed: {}", e)),- LoadedEngine::Canary(canary_engine) => {- output_was_translated = settings.translate_to_english;- let lang = if validated_language == "auto" {- None- } else {- Some(validated_language.clone())- };- applied_language_hint = lang.clone();- let options = TranscribeOptions {- language: lang,- translate: settings.translate_to_english,- ..Default::default()- };- canary_engine- .transcribe(&audio, &options)- .map(|r| r.text)- .map_err(|e| anyhow::anyhow!("Canary transcription failed: {}", e))- }- LoadedEngine::Cohere(cohere_engine) => {- let lang = if validated_language == "auto" {- None- } else {- Some(validated_language.clone())- };- applied_language_hint = lang.clone();- let options = TranscribeOptions {- language: lang,- ..Default::default()- };- cohere_engine- .transcribe(&audio, &options)- .map(|r| r.text)- .map_err(|e| anyhow::anyhow!("Cohere transcription failed: {}", e))- }- }- }));-- let text = match transcribe_result {- Ok(inner_result) => {- // Success or normal error: return the engine unless a model- // switch/unload invalidated it while it was in use.- self.return_engine(engine, &active_model);- inner_result?- }- Err(panic_payload) => {- // Engine panicked — do NOT put it back (it's in an unknown state).- // The engine is dropped here, effectively unloading it.- let panic_msg = panic_payload_message(panic_payload.as_ref());- error!(- "Transcription engine panicked: {}. Model has been unloaded.",- panic_msg- );-- // Clear the model ID so it will be reloaded on next attempt- {- let mut current_model = self- .current_model_id- .lock()- .unwrap_or_else(|e| e.into_inner());- *current_model = None;- }-- let _ = self.app_handle.emit(- "model-state-changed",- ModelStateEvent {- event_type: "unloaded".to_string(),- model_id: None,- model_name: None,- error: Some(format!("Engine panicked: {}", panic_msg)),- },- );-- return Err(anyhow::anyhow!(- "Transcription engine panicked: {}. The model has been unloaded and will reload on next attempt.",- panic_msg- ));- }- };-- let output_language = with_model_detected_language(- resolve_output_language_evidence(- &settings,- applied_language_hint.as_deref(),- &model_languages,- output_was_translated,- ),- model_detected_language,- );- debug!("Output language evidence: {:?}", output_language);-- (text, output_language, model_languages)+ None => self.transcribe_onnx(&audio, &settings, &validated_language, &active_model)?,};+ let output_language = with_model_detected_language(+ resolve_output_language_evidence(+ &settings,+ run.applied_language_hint.as_deref(),+ &run.languages,+ run.output_was_translated,+ ),+ run.model_detected_language,+ );+ debug!("Output language evidence: {:?}", output_language);+// Apply fuzzy word correction if custom words are configured — UNLESS the// words were already handed to the model as an initial prompt (whisper// family). We don't pass a prompt to non-whisper models (it requires the// whisper-kind run extension), so they still get fuzzy correction here,// same as the ONNX engines.let filtered_result = post_process_transcription_text(- result,+ run.text,&settings,- model_is_whisper,+ run.model_is_whisper,&output_language,- &model_languages,+ &run.languages,);let et = std::time::Instant::now();@@ -1540,6 +1249,293 @@ impl TranscriptionManager {Ok(final_result)}++ /// transcribe-cpp: the model runs in the engine's worker process, which+ /// recovers from crashes and hangs itself. The loaded model's live+ /// capabilities (cheap GGUF-metadata reads) are the source of truth, not+ /// the ModelManager copy. The whisper run extension is kind-tagged, so+ /// non-whisper archs (parakeet, voxtral, …) reject it with INVALID_ARG;+ /// attach it — and translate — only where supported.+ fn transcribe_cpp(+ &self,+ info: LoadedInfo,+ audio: Vec<f32>,+ settings: &AppSettings,+ validated_language: &str,+ active_model: &str,+ ) -> Result<RunOutcome> {+ // Whether the model advertises Feature::InitialPrompt. Informational+ // (logged below); the whisper run extension and the fuzzy-correction+ // skip are gated on `model_is_whisper` instead, since non-whisper+ // archs can advertise the feature while rejecting the whisper-kind+ // extension.+ let model_takes_initial_prompt = info.supports_initial_prompt;+ let model_is_whisper = info.arch == "whisper";+ let model_supports_translate = info.capabilities.supports_translate;+ let languages = info.capabilities.languages;+ debug!(+ "transcribe-cpp model '{}' on '{}': initial_prompt={}, translate={}, languages={:?}",+ settings.selected_model,+ info.backend,+ model_takes_initial_prompt,+ model_supports_translate,+ languages+ );++ // Custom words become the initial prompt ONLY for models that accept+ // one (whisper family). Attaching the whisper run extension to a+ // non-whisper arch is rejected with INVALID_ARG, so skip it there and+ // let the fuzzy post-correction handle custom words instead.+ let family = if settings.custom_words.is_empty() || !model_is_whisper {+ None+ } else {+ Some(RunExtension::Whisper(WhisperRunOptions {+ initial_prompt: Some(settings.custom_words.join(", ")),+ ..Default::default()+ }))+ };++ let run_plan = transcribe_cpp_run_plan(+ settings.translate_to_english,+ validated_language,+ &languages,+ model_supports_translate,+ );+ let output_was_translated = run_plan.target_language.as_deref() == Some("en");+ let applied_language_hint = run_plan.language.clone();++ let run_options = RunOptions {+ task: run_plan.task,+ language: run_plan.language,+ target_language: run_plan.target_language,+ family,+ ..Default::default()+ };++ debug!(+ "transcribe-cpp run: task={:?}, language={:?}, initial_prompt={}",+ run_options.task,+ run_options.language,+ run_options.family.is_some()+ );++ let transcript = match self.engine.transcribe(audio, run_options) {+ Ok(transcript) => transcript,+ Err(EngineError::Cancelled) => return Err(EngineError::Cancelled.into()),+ Err(e) => {+ self.forget_model_if_engine_dropped(active_model, &e.to_string());+ return Err(anyhow::anyhow!(+ "transcribe-cpp transcription failed: {}",+ e+ ));+ }+ };+ Ok(RunOutcome {+ text: transcript.text,+ languages,+ applied_language_hint,+ output_was_translated,+ // Whisper's audio-based LID (auto mode only; `None` when a+ // language hint was passed).+ model_detected_language: transcript.language,+ model_is_whisper,+ })+ }++ /// In-process ONNX engines.+ fn transcribe_onnx(+ &self,+ audio: &[f32],+ settings: &AppSettings,+ validated_language: &str,+ active_model: &str,+ ) -> Result<RunOutcome> {+ let languages = self+ .model_manager+ .get_model_info(active_model)+ .map(|info| info.supported_languages)+ .unwrap_or_default();+ let mut output_was_translated = false;+ let mut applied_language_hint: Option<String> = None;++ // Take the engine out so we own it during transcription. If the+ // engine panics, we simply don't put it back (effectively unloading+ // it) instead of poisoning the mutex. No lock is held during the+ // engine call.+ let mut engine = match self.lock_onnx().take() {+ Some(e) => e,+ None => {+ return Err(anyhow::anyhow!(+ "Model failed to load after auto-load attempt. Please check your model settings."+ ));+ }+ };++ // We use catch_unwind to prevent engine panics from poisoning the+ // mutex, which would make the app hang indefinitely on subsequent+ // operations.+ let transcribe_result = catch_unwind(AssertUnwindSafe(|| -> Result<String> {+ match &mut engine {+ OnnxEngine::Parakeet(parakeet_engine) => {+ let params = ParakeetParams {+ timestamp_granularity: Some(TimestampGranularity::Segment),+ ..Default::default()+ };+ parakeet_engine+ .transcribe_with(audio, ¶ms)+ .map(|r| r.text)+ .map_err(|e| anyhow::anyhow!("Parakeet transcription failed: {}", e))+ }+ OnnxEngine::Moonshine(moonshine_engine) => moonshine_engine+ .transcribe(audio, &TranscribeOptions::default())+ .map(|r| r.text)+ .map_err(|e| anyhow::anyhow!("Moonshine transcription failed: {}", e)),+ OnnxEngine::MoonshineStreaming(streaming_engine) => streaming_engine+ .transcribe(audio, &TranscribeOptions::default())+ .map(|r| r.text)+ .map_err(|e| {+ anyhow::anyhow!("Moonshine streaming transcription failed: {}", e)+ }),+ OnnxEngine::SenseVoice(sense_voice_engine) => {+ let language = match validated_language {+ "zh" => Some("zh".to_string()),+ "en" => Some("en".to_string()),+ "ja" => Some("ja".to_string()),+ "ko" => Some("ko".to_string()),+ "yue" => Some("yue".to_string()),+ _ => None,+ };+ applied_language_hint = language.clone();+ let params = SenseVoiceParams {+ language,+ use_itn: Some(true),+ };+ sense_voice_engine+ .transcribe_with(audio, ¶ms)+ .map(|r| r.text)+ .map_err(|e| anyhow::anyhow!("SenseVoice transcription failed: {}", e))+ }+ OnnxEngine::GigaAM(gigaam_engine) => gigaam_engine+ .transcribe(audio, &TranscribeOptions::default())+ .map(|r| r.text)+ .map_err(|e| anyhow::anyhow!("GigaAM transcription failed: {}", e)),+ OnnxEngine::Canary(canary_engine) => {+ output_was_translated = settings.translate_to_english;+ let lang = if validated_language == "auto" {+ None+ } else {+ Some(validated_language.to_string())+ };+ applied_language_hint = lang.clone();+ let options = TranscribeOptions {+ language: lang,+ translate: settings.translate_to_english,+ ..Default::default()+ };+ canary_engine+ .transcribe(audio, &options)+ .map(|r| r.text)+ .map_err(|e| anyhow::anyhow!("Canary transcription failed: {}", e))+ }+ OnnxEngine::Cohere(cohere_engine) => {+ let lang = if validated_language == "auto" {+ None+ } else {+ Some(validated_language.to_string())+ };+ applied_language_hint = lang.clone();+ let options = TranscribeOptions {+ language: lang,+ ..Default::default()+ };+ cohere_engine+ .transcribe(audio, &options)+ .map(|r| r.text)+ .map_err(|e| anyhow::anyhow!("Cohere transcription failed: {}", e))+ }+ }+ }));++ let text = match transcribe_result {+ Ok(inner_result) => {+ // Success or normal error: return the engine unless a model+ // switch/unload invalidated it while it was in use.+ self.return_engine(engine, active_model);+ inner_result?+ }+ Err(panic_payload) => {+ // Engine panicked — do NOT put it back (it's in an unknown state).+ // The engine is dropped here, effectively unloading it.+ let panic_msg = panic_payload_message(panic_payload.as_ref());+ error!(+ "Transcription engine panicked: {}. Model has been unloaded.",+ panic_msg+ );++ // Clear the model ID so it will be reloaded on next attempt+ {+ let mut current_model = self+ .current_model_id+ .lock()+ .unwrap_or_else(|e| e.into_inner());+ *current_model = None;+ }++ let _ = self.app_handle.emit(+ "model-state-changed",+ ModelStateEvent {+ event_type: "unloaded".to_string(),+ model_id: None,+ model_name: None,+ error: Some(format!("Engine panicked: {}", panic_msg)),+ },+ );++ return Err(anyhow::anyhow!(+ "Transcription engine panicked: {}. The model has been unloaded and will reload on next attempt.",+ panic_msg+ ));+ }+ };+ Ok(RunOutcome {+ text,+ languages,+ applied_language_hint,+ output_was_translated,+ model_detected_language: None,+ model_is_whisper: false,+ })+ }+}++/// What a batch transcription produced, plus what post-processing needs to+/// know about how it was produced.+struct RunOutcome {+ text: String,+ /// The model's supported languages.+ languages: Vec<String>,+ applied_language_hint: Option<String>,+ output_was_translated: bool,+ /// The language the model itself detected, if it reported one.+ model_detected_language: Option<String>,+ /// Whether the loaded model is actually whisper-family (arch string).+ /// Non-whisper archs (e.g. Voxtral Small) can advertise+ /// Feature::InitialPrompt yet reject the whisper-kind run extension with+ /// INVALID_ARG, so the whisper extension must be gated on the arch, not+ /// on the feature (see #1601).+ model_is_whisper: bool,+}++fn emit_stream_text(app_handle: &AppHandle, committed: &str, tentative: &str) {+ let _ = StreamTextEvent {+ committed: committed.to_string(),+ tentative: tentative.to_string(),+ }+ .emit(app_handle);+}++fn lock_perf(perf: &Mutex<StreamPerf>) -> MutexGuard<'_, StreamPerf> {+ perf.lock().unwrap_or_else(|e| e.into_inner())}struct StreamPerf {@@ -1578,17 +1574,11 @@ impl StreamPerf {self.stream_compute_elapsed += elapsed;}- fn record_update(- &mut self,- revision: i32,- input_received_ms: i64,- audio_committed_ms: i64,- buffered_ms: i64,- ) {- self.latest_revision = revision;- self.latest_input_received_ms = input_received_ms;- self.latest_audio_committed_ms = audio_committed_ms;- self.latest_buffered_ms = buffered_ms;+ fn record_update(&mut self, update: &transcribe_cpp::StreamUpdate) {+ self.latest_revision = update.revision;+ self.latest_input_received_ms = update.input_received_ms;+ self.latest_audio_committed_ms = update.audio_committed_ms;+ self.latest_buffered_ms = update.buffered_ms;}fn record_emit(&mut self) {@@ -1943,7 +1933,7 @@ fn drain_until_finalize(rx: mpsc::Receiver<StreamCmd>) {match cmd {StreamCmd::Feed(_) => {}StreamCmd::Finalize(reply) => {- let _ = reply.send(None);+ let _ = reply.send(Ok(None));break;}StreamCmd::Cancel => break,@@ -1951,97 +1941,56 @@ fn drain_until_finalize(rx: mpsc::Receiver<StreamCmd>) {}}-/// Initialize the transcribe-cpp native backend once at startup: route native +-/// ggml diagnostics into the `log` facade and register compute backend modules.-/// In a static build (macOS Metal) `init_backends_default` is a harmless no-op;-/// in a `dynamic-backends` build it loads the per-ISA CPU / GPU modules. Must run-/// before the first model load.-pub fn init_transcribe_backend() {- transcribe_cpp::init_logging();- match transcribe_cpp::init_backends_default() {- Ok(()) => {- if transcribe_gpu_disabled_for_host() {- warn!(- "Windows x64 build is running under emulation on an ARM64 host; \- disabling transcribe.cpp GPU acceleration and using CPU"- );- }- }- Err(e) => warn!("Failed to initialize transcribe-cpp backends: {}", e),- }-}--/// Log the compute devices [`init_transcribe_backend`] registered.+/// Log the compute devices transcribe-cpp registers.///-/// Listing devices is what first opens the GPU. On macOS that loads ggml's-/// Metal library, which is compiled from source whenever the system's shader-/// cache does not hold it yet (the first launch after an install or update),-/// so the app calls this from a background thread instead of its startup-/// path. A model load that comes first waits on the same one-time compile.-pub fn report_compute_devices() {- let devices = transcribe_compute_devices();- info!(- "transcribe-cpp initialized with {} compute device(s): [{}]",- devices.len(),- devices- .iter()- .map(|d| format!("{} ({})", d.name, d.kind))- .collect::<Vec<_>>()- .join(", ")- );+/// Devices are listed by a transcription worker process (see+/// [`crate::engine_supervisor`]), so GPU driver initialization never runs in+/// the app process. Listing opens the GPU, which on macOS loads ggml's Metal+/// library and compiles it when the system shader cache does not hold it yet+/// (the first launch after an install or update), so the app calls this from+/// a background thread instead of its startup path.+pub fn report_compute_devices(tm: &TranscriptionManager) {+ if transcribe_gpu_disabled_for_host() {+ warn!(+ "Windows x64 build is running under emulation on an ARM64 host; \+ disabling transcribe.cpp GPU acceleration and using CPU"+ );+ }+ match transcribe_compute_devices(&tm.engine) {+ Ok(devices) => info!(+ "transcribe-cpp initialized with {} compute device(s): [{}]",+ devices.len(),+ devices+ .iter()+ .map(|d| format!("{} ({})", d.name, d.kind))+ .collect::<Vec<_>>()+ .join(", ")+ ),+ Err(e) => warn!("transcribe-cpp compute devices are unavailable: {}", e),+ }}-/// Human-readable list of the transcribe-cpp compute devices registered at-/// startup, for the `--list-devices` flag. The reported `index` is the-/// value to pass to `--device-index`. Backends must be initialized first-/// (see [`init_transcribe_backend`]).-pub fn describe_compute_devices() -> Vec<String> {- transcribe_compute_devices()+/// Human-readable list of the transcribe-cpp compute devices, for the+/// `--list-devices` flag. The reported `index` is the value to pass to+/// `--device-index`. Errors if the devices could not be listed.+pub fn describe_compute_devices(tm: &TranscriptionManager) -> Result<Vec<String>> {+ Ok(transcribe_compute_devices(&tm.engine)?.into_iter().map(|d| {let idx = d.index.map(|i| i.to_string()).unwrap_or_else(|| "-".to_string());- let name = if d.description.is_empty() {- d.name- } else {- d.description- };let vram_mb = d.memory_total / (1024 * 1024);format!("index={} kind={} name={} vram={}MB",- idx, d.kind, name, vram_mb+ idx,+ d.kind,+ d.label(),+ vram_mb)})- .collect()-}--/// Resolve a `--list-devices` registry index to an exact opaque device handle-/// for a transcribe-cpp model load (the `--device-index` flag). In 0.2 index 0-/// is an exact selection too; only an omitted index requests automatic device-/// selection. Errors if the index isn't a registered, loadable primary device.-fn resolve_device_index(index: usize) -> Result<(Backend, Option<transcribe_cpp::Device>)> {- let device = transcribe_compute_devices()- .into_iter()- .find(|d| d.index == Some(index))- .ok_or_else(|| {- anyhow::anyhow!("No compute device with index {index} (see --list-devices)")- })?;- if matches!(- device.device_type,- transcribe_cpp::DeviceType::Accel | transcribe_cpp::DeviceType::Unknown- ) {- return Err(anyhow::anyhow!(- "Device index {index} ({}) cannot host a model",- device.kind- ));- }-- // 0.2's opaque handle makes every index, including zero, an exact- // selection. Backend::Auto accepts any primary device and cannot conflict- // with the selected device's vendor backend.- Ok((Backend::Auto, Some(device)))+ .collect())}/// Map Handy's whisper accelerator setting to a transcribe-cpp [`Backend`].@@ -2065,45 +2014,19 @@ fn select_transcribe_backend_for_host(}}-/// Resolve the user's persisted GPU identity to a fresh opaque 0.2 device-/// handle. Registry indices and handles are process-local, so settings store a-/// key based on the backend's stable `device_id` (falling back to name for-/// backends such as Metal that do not report one).+/// Resolve the user's persisted GPU choice to the device key the worker+/// should load on. Registry indices and handles are process-local, so+/// settings store a key based on the backend's stable `device_id` (falling+/// back to name for backends such as Metal that do not report one). The+/// worker falls back to automatic selection if that device is gone.fn resolve_gpu_device(setting: TranscribeAcceleratorSetting,gpu_device: Option<&str>,-) -> Option<transcribe_cpp::Device> {+) -> Option<String> {if transcribe_gpu_disabled_for_host() || setting != TranscribeAcceleratorSetting::Gpu {return None;}- let gpu_device = gpu_device?;- let resolved = transcribe_compute_devices().into_iter().find(|device| {- is_transcribe_gpu_device(device) && transcribe_device_key(device) == gpu_device- });- if resolved.is_none() {- warn!(- "Stored transcribe GPU device '{}' is no longer available; using automatic device selection",- gpu_device- );- }- resolved-}--fn transcribe_device_key(device: &transcribe_cpp::Device) -> String {- let (identity_kind, identity) = match device.device_id.as_deref() {- Some(device_id) => ("id", device_id),- None => ("name", device.name.as_str()),- };- serde_json::to_string(&(device.kind.as_str(), identity_kind, identity))- .expect("transcribe device identity is always JSON serializable")-}--fn transcribe_device_label(device: &transcribe_cpp::Device) -> String {- if device.description.is_empty() {- device.name.clone()- } else {- device.description.clone()- }+ gpu_device.map(str::to_string)}/// Apply the user's ORT accelerator preference to the transcribe-rs global.@@ -2140,8 +2063,6 @@ pub struct GpuDeviceOption {pub total_vram_mb: usize,}-static GPU_DEVICES: OnceLock<Vec<GpuDeviceOption>> = OnceLock::new();-fn transcribe_gpu_disabled_for_host() -> bool {crate::utils::is_windows_x64_emulated_on_arm64()}@@ -2157,28 +2078,12 @@ fn effective_transcribe_accelerator(}}-fn is_transcribe_gpu_device(device: &transcribe_cpp::Device) -> bool {- matches!(- device.device_type,- transcribe_cpp::DeviceType::Gpu | transcribe_cpp::DeviceType::Igpu- )-}--fn transcribe_device_allowed(kind: &str, gpu_disabled: bool) -> bool {- !gpu_disabled || matches!(kind, "cpu" | "accel")-}--fn transcribe_compute_devices() -> Vec<transcribe_cpp::Device> {- let devices = transcribe_cpp::devices();- let gpu_disabled = transcribe_gpu_disabled_for_host();- if !gpu_disabled {- return devices;- }-- devices- .into_iter()- .filter(|device| transcribe_device_allowed(&device.kind, gpu_disabled))- .collect()+/// The compute devices the latest worker listed. On a host without GPU+/// access every worker is CPU-only, so it lists no GPU.+fn transcribe_compute_devices(engine: &EngineSupervisor) -> Result<Vec<DeviceInfo>> {+ engine+ .devices()+ .ok_or_else(|| anyhow::anyhow!("compute devices could not be listed (see the log for why)"))}fn available_transcribe_accelerators(gpu_disabled: bool) -> Vec<String> {@@ -2189,22 +2094,22 @@ fn available_transcribe_accelerators(gpu_disabled: bool) -> Vec<String> {}}-fn cached_gpu_devices() -> &'static [GpuDeviceOption] {- // GPU compute devices transcribe-cpp registered at startup. `id` is a- // persistent identity key, never the process-local registry index. It uses- // the backend's device_id where available and its name otherwise (Metal).- // `total_vram_mb` is 0 when the backend does not report capacity.- GPU_DEVICES.get_or_init(|| {- transcribe_compute_devices()- .into_iter()- .filter(is_transcribe_gpu_device)- .map(|d| GpuDeviceOption {- id: transcribe_device_key(&d),- name: transcribe_device_label(&d),- total_vram_mb: (d.memory_total / (1024 * 1024)) as usize,- })- .collect()- })+fn gpu_device_options(engine: &EngineSupervisor) -> Vec<GpuDeviceOption> {+ // GPU compute devices as the latest worker listed them, so a GPU that+ // comes back is picked up. `id` is a persistent identity key, never the+ // process-local registry index. It uses the backend's device_id where+ // available and its name otherwise (Metal). `total_vram_mb` is 0 when the+ // backend does not report capacity. Empty if listing failed.+ transcribe_compute_devices(engine)+ .unwrap_or_default()+ .into_iter()+ .filter(DeviceInfo::is_gpu)+ .map(|d| GpuDeviceOption {+ id: d.key.clone(),+ name: d.label().to_string(),+ total_vram_mb: (d.memory_total / (1024 * 1024)) as usize,+ })+ .collect()}#[derive(Serialize, Clone, Debug, Type)]@@ -2215,7 +2120,7 @@ pub struct AvailableAccelerators {}/// Return the accelerators available to this process on its current host.-pub fn get_available_accelerators() -> AvailableAccelerators {+pub fn get_available_accelerators(tm: &TranscriptionManager) -> AvailableAccelerators {use transcribe_rs::accel::OrtAccelerator;let ort_options: Vec<String> = OrtAccelerator::available()@@ -2228,7 +2133,7 @@ pub fn get_available_accelerators() -> AvailableAccelerators {AvailableAccelerators {transcribe: transcribe_options,ort: ort_options,- gpu_devices: cached_gpu_devices().to_vec(),+ gpu_devices: gpu_device_options(&tm.engine),}}@@ -2265,9 +2170,6 @@ mod tests {select_transcribe_backend_for_host(TranscribeAcceleratorSetting::Gpu, false),Backend::Auto);- for kind in ["cpu", "accel", "metal", "cuda", "vulkan", "gpu"] {- assert!(transcribe_device_allowed(kind, false));- }}#[test]@@ -2287,11 +2189,6 @@ mod tests {);}assert_eq!(available_transcribe_accelerators(true), ["cpu"]);- assert!(transcribe_device_allowed("cpu", true));- assert!(transcribe_device_allowed("accel", true));- for kind in ["metal", "cuda", "vulkan", "gpu", "unknown"] {- assert!(!transcribe_device_allowed(kind, true));- }}#[test]@@ -2605,10 +2502,10 @@ impl Drop for TranscriptionManager {// Skip shutdown unless this is the very last clone. TranscriptionManager// is cloned by initiate_model_load() and the watcher thread — those// clones dropping must not kill the watcher. The watcher thread holds- // its own clone, so engine's strong_count is always >= 2 while the+ // its own clone, so onnx's strong_count is always >= 2 while the// watcher is alive. When it reaches 1, only this instance remains// and we can safely shut down.- if Arc::strong_count(&self.engine) > 1 {+ if Arc::strong_count(&self.onnx) > 1 {return;}diff --git a/src-tauri/src/memory.rs b/src-tauri/src/memory.rsindex 2c4f940c..ce01ee31 100644--- a/src-tauri/src/memory.rs+++ b/src-tauri/src/memory.rs@@ -22,7 +22,8 @@/// Pin glibc's mmap threshold so large transient buffers keep taking the/// mmap path and are returned to the OS as soon as they are freed.///-/// Must run before the workload allocates (called at the top of `run()`);+/// Must run before the workload allocates (called at the top of `run()`,+/// and of the transcription worker's own `run()`);/// the cost is an mmap/munmap round-trip per multi-MB buffer, which is/// negligible at dictation frequency.#[cfg(all(target_os = "linux", target_env = "gnu"))]@@ -39,7 +40,8 @@ pub fn init_allocator() {}/// Return freed-but-cached malloc arena memory to the OS.///-/// Called once per finished transcription pipeline (see `FinishGuard`); it+/// Called once per finished transcription pipeline (see `FinishGuard`), and+/// by the transcription worker after each run or stream; it/// sweeps whatever smaller-than-threshold churn still accumulates in the/// arenas. Takes on the order of a millisecond, off the main thread.#[cfg(all(target_os = "linux", target_env = "gnu"))]diff --git a/src-tauri/src/shortcut/mod.rs b/src-tauri/src/shortcut/mod.rsindex 607ef521..3dbc5e3a 100644--- a/src-tauri/src/shortcut/mod.rs+++ b/src-tauri/src/shortcut/mod.rs@@ -1425,6 +1425,8 @@ pub fn change_transcribe_accelerator_setting(let mut s = settings::get_settings(&app);s.transcribe_accelerator = accelerator;save_accelerator_and_reload_next_use(&app, s);+ app.state::<std::sync::Arc<crate::managers::transcription::TranscriptionManager>>()+ .retry_transcribe_gpu("the transcribe.cpp accelerator setting changed");Ok(())}@@ -1446,6 +1448,8 @@ pub fn change_transcribe_gpu_device(app: AppHandle, device: Option<String>) -> Rlet mut s = settings::get_settings(&app);s.transcribe_gpu_device = device;save_accelerator_and_reload_next_use(&app, s);+ app.state::<std::sync::Arc<crate::managers::transcription::TranscriptionManager>>()+ .retry_transcribe_gpu("the transcribe.cpp GPU device setting changed");Ok(())}@@ -1457,10 +1461,16 @@ pub fn change_transcribe_gpu_device(app: AppHandle, device: Option<String>) -> R/// stays responsive — see also the startup pre-warm in `lib.rs`.#[tauri::command]#[specta::specta]-pub async fn get_available_accelerators() -> crate::managers::transcription::AvailableAccelerators {- tauri::async_runtime::spawn_blocking(crate::managers::transcription::get_available_accelerators)- .await- .expect("get_available_accelerators panicked")+pub async fn get_available_accelerators(+ app: AppHandle,+) -> crate::managers::transcription::AvailableAccelerators {+ tauri::async_runtime::spawn_blocking(move || {+ let tm =+ app.state::<std::sync::Arc<crate::managers::transcription::TranscriptionManager>>();+ crate::managers::transcription::get_available_accelerators(&tm)+ })+ .await+ .expect("get_available_accelerators panicked")}#[cfg(test)]diff --git a/src-tauri/src/utils.rs b/src-tauri/src/utils.rsindex d80ec2a4..0959c543 100644--- a/src-tauri/src/utils.rs+++ b/src-tauri/src/utils.rs@@ -94,15 +94,19 @@ pub fn cancel_current_operation(app: &AppHandle) {let recording_was_active = audio_manager.is_recording();audio_manager.cancel_recording();- // Abandon any live streaming transcription+ // Abandon any live streaming transcription, and stop a transcription+ // already in progresslet tm = app.state::<Arc<TranscriptionManager>>();tm.cancel_stream();+ tm.cancel_transcription();// Update tray icon and hide overlayset_tray_state(app, crate::tray::TrayIconState::Idle);hide_recording_overlay(app);- // Unload model if immediate unload is enabled+ // Unload model if immediate unload is enabled. Done before notifying the+ // coordinator, so a recording started next always loads afresh; it+ // doesn't wait for the cancelled work, so it never blocks this thread.tm.maybe_unload_immediately("cancellation");// Notify coordinator so it can keep lifecycle state coherent.