Skip to content

armgit / Handy

Public Imported from GitHub
0

A free, open source, and extensible speech-to-text application that works completely offline.

isolate transcribe.cpp backend in own worker (#2208)

CJ Paisc8262ab3ebe5
diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock
index ac879abd..77db5fcf 100644
--- a/src-tauri/Cargo.lock
+++ b/src-tauri/Cargo.lock
@@ -7285,20 +7285,21 @@ dependencies = [
[[package]]
name = "transcribe-cpp"
-version = "0.3.0"
+version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f5134e51fd0cd73fc024b8004c495d96ec1fa5d89c5e341482fd60d82483002a"
+checksum = "2f06368bb32251b5e93ceb8cae2d6ba9adff6a51693c7ce108449e65522b05c6"
dependencies = [
"log",
+ "serde",
"thiserror 2.0.18",
"transcribe-cpp-sys",
]
[[package]]
name = "transcribe-cpp-sys"
-version = "0.3.0"
+version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98c3d32764dc7ae892274022d52d369e92e1e765e403e56daaa3fe8fdbe18443"
+checksum = "fedfadb42719d92e58a2c340ccfb6ef327ac1b846fc1d4716b65384a330513a6"
dependencies = [
"cmake",
"serde_json",
diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml
index d669d29e..0a9fe242 100644
--- a/src-tauri/Cargo.toml
+++ b/src-tauri/Cargo.toml
@@ -75,11 +75,15 @@ rusqlite = { version = "0.37", features = ["bundled"] }
tar = "0.4.44"
flate2 = "1.0"
sha2 = "0.10"
+libc = "0.2"
# Whisper-family models run through transcribe-cpp (GGUF/ggml, native backends);
# transcribe-rs is now ONNX-only (Parakeet, Moonshine, SenseVoice, GigaAM,
-# Canary, Cohere). Per-platform backend features are added in the target tables.
+# Canary, Cohere). Per-platform backend features are added in the target tables
+# (Cargo unifies features, so `serde` here applies to every platform).
transcribe-rs = { version = "0.3.8", features = ["onnx"] }
-transcribe-cpp = { version = "0.3.0", default-features = false }
+transcribe-cpp = { version = "0.3.1", default-features = false, features = [
+ "serde",
+] }
handy-keys = "0.3.4"
ferrous-opencc = "0.2.3"
clap = { version = "4", features = ["derive"] }
@@ -142,17 +146,17 @@ winreg = "0.55"
# signtool verify entirely. CI pins -DGGML_NATIVE=OFF (see build.yml) so the
# static build targets a portable ARM baseline, not the build runner's CPU.
[target.'cfg(all(windows, target_arch = "x86_64"))'.dependencies]
-transcribe-cpp = { version = "0.3.0", default-features = false, features = [
+transcribe-cpp = { version = "0.3.1", default-features = false, features = [
"dynamic-backends",
"vulkan",
] }
[target.'cfg(all(windows, target_arch = "aarch64"))'.dependencies]
-transcribe-cpp = { version = "0.3.0", default-features = false }
+transcribe-cpp = { version = "0.3.1", default-features = false }
[target.'cfg(target_os = "macos")'.dependencies]
tauri-nspanel = { git = "https://github.com/ahkohd/tauri-nspanel", branch = "v2.1" }
-transcribe-cpp = { version = "0.3.0", default-features = false, features = ["metal"] }
+transcribe-cpp = { version = "0.3.1", default-features = false, features = ["metal"] }
# Used by the receipt-sequenced ("reliable") paste path for lazy NSPasteboard
# promises. Versions track what Tauri already pulls in.
objc2 = "0.6"
@@ -163,8 +167,7 @@ objc2-service-management = "0.3.2"
[target.'cfg(target_os = "linux")'.dependencies]
gtk-layer-shell = { version = "0.8", features = ["v0_6"] }
gtk = "0.18"
-libc = "0.2"
-transcribe-cpp = { version = "0.3.0", default-features = false, features = [
+transcribe-cpp = { version = "0.3.1", default-features = false, features = [
"dynamic-backends",
"vulkan",
] }
diff --git a/src-tauri/src/actions.rs b/src-tauri/src/actions.rs
index cce7cfe4..04d88644 100644
--- a/src-tauri/src/actions.rs
+++ b/src-tauri/src/actions.rs
@@ -650,11 +650,10 @@ impl ShortcutAction for TranscribeAction {
let transcription_time = Instant::now();
let transcription_result = match tm.finalize_stream() {
// A finalized stream with usable text wins. An empty result
- // (no active stream, produced nothing, or a finalize error
- // after the engine was returned) falls back to a full batch
- // transcription of the same audio. A finalize timeout is
- // surfaced instead — the worker may still hold the engine,
- // so a batch fallback would contend with it.
+ // (no active stream, produced nothing, or the stream failed
+ // or its worker crashed) falls back to a full batch
+ // transcription of the same audio. A cancelled finalize is
+ // surfaced instead, so a cancel never starts a batch run.
Ok(Some(text)) if !text.trim().is_empty() => Ok(text),
Ok(_) => tm.transcribe(samples),
Err(err) => Err(err),
diff --git a/src-tauri/src/commands/models.rs b/src-tauri/src/commands/models.rs
index b1bb8949..e49a5cf1 100644
--- a/src-tauri/src/commands/models.rs
+++ b/src-tauri/src/commands/models.rs
@@ -72,8 +72,12 @@ pub async fn delete_model(
// If deleting the active model, unload it and clear the setting
let settings = get_settings(&app_handle);
if settings.selected_model == model_id {
- transcription_manager
- .unload_model()
+ // Waits for the worker to exit (behind any transcription in progress)
+ // before the file is deleted, so keep it off the async workers.
+ let tm = Arc::clone(&transcription_manager);
+ tauri::async_runtime::spawn_blocking(move || tm.unload_model())
+ .await
+ .map_err(|e| format!("Failed to unload model: {}", e))?
.map_err(|e| format!("Failed to unload model: {}", e))?;
let mut settings = get_settings(&app_handle);
diff --git a/src-tauri/src/commands/transcription.rs b/src-tauri/src/commands/transcription.rs
index 6cfa2e6a..d51b1673 100644
--- a/src-tauri/src/commands/transcription.rs
+++ b/src-tauri/src/commands/transcription.rs
@@ -2,6 +2,7 @@ use crate::managers::transcription::TranscriptionManager;
use crate::settings::{get_settings, write_settings, ModelUnloadTimeout};
use serde::Serialize;
use specta::Type;
+use std::sync::Arc;
use tauri::{AppHandle, State};
#[derive(Serialize, Type)]
@@ -21,7 +22,7 @@ pub fn set_model_unload_timeout(app: AppHandle, timeout: ModelUnloadTimeout) {
#[tauri::command]
#[specta::specta]
pub fn get_model_load_status(
- transcription_manager: State<TranscriptionManager>,
+ transcription_manager: State<Arc<TranscriptionManager>>,
) -> Result<ModelLoadStatus, String> {
Ok(ModelLoadStatus {
is_loaded: transcription_manager.is_model_loaded(),
@@ -32,9 +33,8 @@ pub fn get_model_load_status(
#[tauri::command]
#[specta::specta]
pub fn unload_model_manually(
- transcription_manager: State<TranscriptionManager>,
+ transcription_manager: State<Arc<TranscriptionManager>>,
) -> Result<(), String> {
- transcription_manager
- .unload_model()
- .map_err(|e| format!("Failed to unload model: {}", e))
+ transcription_manager.request_unload();
+ Ok(())
}
diff --git a/src-tauri/src/engine_supervisor/mod.rs b/src-tauri/src/engine_supervisor/mod.rs
new file mode 100644
index 00000000..a88634a4
--- /dev/null
+++ b/src-tauri/src/engine_supervisor/mod.rs
@@ -0,0 +1,52 @@
+//! Process isolation for transcribe.cpp.
+//!
+//! All transcribe.cpp native code (backend init, device enumeration, model
+//! load, inference, streaming) runs in a child process: the Handy executable
+//! re-launched with [`WORKER_FLAG`]. A GPU driver fault, `GGML_ASSERT` abort,
+//! or hang then kills only the worker.
+//!
+//! [`EngineSupervisor`] is the single owner of that worker. It loads and
+//! unloads models, runs and streams transcriptions, cancels them, and
+//! recovers from crashes and hangs (retrying in a CPU-only worker when the
+//! GPU was at fault), so the rest of the app never sees processes, restarts
+//! or fallback. At most one worker holds a model at a time, and it lives
+//! exactly as long as that model, so unloading returns all of its CPU and GPU
+//! memory to the OS. Listing compute devices may briefly run a second,
+//! model-less worker.
+//!
+//! A CPU-only worker ([`CPU_ONLY_FLAG`]) restricts transcribe.cpp to its CPU
+//! backends before any are registered, so no GPU driver code runs in it at
+//! all: a driver that crashes or hangs while initializing can't take the CPU
+//! fallback down with it.
+//!
+//! - `protocol`: the framed request/response protocol between the two.
+//! - `supervisor`: the parent side ([`EngineSupervisor`]).
+//! - `worker`: the child process.
+
+mod protocol;
+mod supervisor;
+mod worker;
+
+pub use protocol::{DeviceInfo, DeviceSelector, LoadedInfo};
+pub use supervisor::{
+ EngineError, EngineSupervisor, Finalized, LoadSpec, StreamHandle, StreamProgress, Unloading,
+};
+
+/// Hidden first argument that turns the executable into a worker.
+pub const WORKER_FLAG: &str = "--transcribe-worker";
+/// Worker argument: register only the CPU backends.
+const CPU_ONLY_FLAG: &str = "--cpu-only";
+const LOG_LEVEL_ENV: &str = "HANDY_TRANSCRIBE_WORKER_LOG";
+
+/// Whether this process was launched as a transcription worker. Checked in
+/// `main` before CLI parsing, Tauri, or single-instance handling.
+pub fn is_worker_invocation() -> bool {
+ std::env::args_os()
+ .nth(1)
+ .is_some_and(|arg| arg == WORKER_FLAG)
+}
+
+/// Run the worker until the parent closes its stdin. Returns the exit code.
+pub fn run_worker() -> i32 {
+ worker::run()
+}
diff --git a/src-tauri/src/engine_supervisor/protocol.rs b/src-tauri/src/engine_supervisor/protocol.rs
new file mode 100644
index 00000000..9e353c95
--- /dev/null
+++ b/src-tauri/src/engine_supervisor/protocol.rs
@@ -0,0 +1,310 @@
+//! Wire protocol between Handy and its transcribe.cpp worker process.
+//!
+//! Strict request/response: every [`Request`] gets exactly one [`Response`].
+//! Each message is one frame: `[u32 json_len][json][u32 pcm_len][pcm bytes]`
+//! (lengths little-endian, PCM as little-endian f32). JSON (not bincode) is
+//! deliberate: transcribe-cpp's serde impls round-trip NaN confidences through
+//! `null`, which only works with a self-describing format.
+
+use serde::{Deserialize, Serialize};
+use std::io::{self, Read, Write};
+use std::path::PathBuf;
+use transcribe_cpp::{
+ Backend, Capabilities, DeviceType, RunOptions, StreamOptions, StreamText, StreamUpdate,
+ Transcript,
+};
+
+/// Refuse frames larger than this: a garbage length must not turn into a
+/// multi-gigabyte allocation. One hour of 16 kHz f32 PCM is ~230 MB.
+const MAX_SECTION_BYTES: usize = 1 << 30;
+const PCM_BYTES_PER_MINUTE: usize = 16_000 * 4 * 60;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum Request {
+ /// First request to every worker; answered once backend init is done
+ /// (with an error if it registered no compute device). With
+ /// `list_devices`, the worker also enumerates its compute devices.
+ /// CPU-only workers that hold a model don't list: they would only see
+ /// the CPU.
+ Hello {
+ list_devices: bool,
+ },
+ Load {
+ path: PathBuf,
+ backend: Backend,
+ device: DeviceSelector,
+ },
+ /// Followed by the PCM to transcribe.
+ Run {
+ options: RunOptions,
+ },
+ StreamBegin {
+ run: RunOptions,
+ stream: StreamOptions,
+ },
+ /// Followed by the PCM frame to feed.
+ Feed,
+ /// `want_language`: also report the model's detected language (costs a
+ /// transcript snapshot, so only asked for when it can change the outcome).
+ Finalize {
+ want_language: bool,
+ },
+ StreamReset,
+}
+
+/// How the worker should pick the device for a [`Request::Load`]. Device
+/// handles are process-local, so the parent names devices by stable key or by
+/// registry index and the worker resolves them.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum DeviceSelector {
+ Auto,
+ /// Persisted identity from [`device_key`]. Falls back to automatic
+ /// selection when the device is no longer present.
+ Key(String),
+ /// Registry index from `--list-devices`. Fails if not a usable device.
+ Index(usize),
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum Response {
+ /// `devices` is present only when the hello asked for them.
+ Hello {
+ devices: Option<Vec<DeviceInfo>>,
+ },
+ Loaded(LoadedInfo),
+ Transcript(Transcript),
+ Fed {
+ update: StreamUpdate,
+ /// Present only when the committed or tentative text changed.
+ text: Option<StreamText>,
+ },
+ Finalized {
+ update: StreamUpdate,
+ text: StreamText,
+ language: Option<String>,
+ },
+ Ok,
+ Error(String),
+}
+
+/// Serializable mirror of `transcribe_cpp::Device` (which holds a
+/// process-local handle and so has no serde impl).
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DeviceInfo {
+ pub name: String,
+ pub description: String,
+ pub kind: String,
+ pub device_type: DeviceType,
+ pub device_id: Option<String>,
+ pub memory_total: u64,
+ pub index: Option<usize>,
+ /// Stable identity for persisting a device choice; see [`device_key`].
+ pub key: String,
+}
+
+impl DeviceInfo {
+ pub fn from_device(device: &transcribe_cpp::Device) -> Self {
+ Self {
+ name: device.name.clone(),
+ description: device.description.clone(),
+ kind: device.kind.clone(),
+ device_type: device.device_type,
+ device_id: device.device_id.clone(),
+ memory_total: device.memory_total,
+ index: device.index,
+ key: device_key(&device.kind, device.device_id.as_deref(), &device.name),
+ }
+ }
+
+ pub fn label(&self) -> &str {
+ if self.description.is_empty() {
+ &self.name
+ } else {
+ &self.description
+ }
+ }
+
+ pub fn is_gpu(&self) -> bool {
+ matches!(self.device_type, DeviceType::Gpu | DeviceType::Igpu)
+ }
+}
+
+/// Persistent device identity. Uses the backend's stable `device_id` where
+/// available and its name otherwise (Metal reports no id). Must stay
+/// byte-identical to what older versions stored in settings.
+pub fn device_key(kind: &str, device_id: Option<&str>, name: &str) -> String {
+ let (identity_kind, identity) = match device_id {
+ Some(device_id) => ("id", device_id),
+ None => ("name", name),
+ };
+ serde_json::to_string(&(kind, identity_kind, identity))
+ .expect("transcribe device identity is always JSON serializable")
+}
+
+/// What the parent needs to know about a loaded model, read once at load.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LoadedInfo {
+ pub arch: String,
+ pub variant: String,
+ pub backend: String,
+ /// Label of the device the model actually bound to.
+ pub device: String,
+ /// Whether that device is a GPU (a crash there may be the GPU's fault).
+ pub on_gpu: bool,
+ pub capabilities: Capabilities,
+ pub supports_initial_prompt: bool,
+}
+
+/// Encode one frame. Refuses a section the reader would refuse, so an
+/// oversized request fails cleanly here instead of killing the worker (which
+/// would read as a crash and be blamed on the GPU).
+pub fn encode_message<T: Serialize>(message: &T, pcm: Option<&[f32]>) -> io::Result<Vec<u8>> {
+ let json = serde_json::to_vec(message).map_err(io::Error::other)?;
+ check_section_len(json.len())?;
+ let pcm = pcm.unwrap_or(&[]);
+ if check_section_len(pcm.len() * 4).is_err() {
+ return Err(io::Error::other(format!(
+ "audio too long for one transcription: {} minutes (limit {} minutes)",
+ pcm.len() * 4 / PCM_BYTES_PER_MINUTE,
+ MAX_SECTION_BYTES / PCM_BYTES_PER_MINUTE,
+ )));
+ }
+ let mut frame = Vec::with_capacity(8 + json.len() + pcm.len() * 4);
+ frame.extend_from_slice(&(json.len() as u32).to_le_bytes());
+ frame.extend_from_slice(&json);
+ frame.extend_from_slice(&((pcm.len() * 4) as u32).to_le_bytes());
+ for sample in pcm {
+ frame.extend_from_slice(&sample.to_le_bytes());
+ }
+ Ok(frame)
+}
+
+pub fn write_message<T: Serialize>(
+ w: &mut impl Write,
+ message: &T,
+ pcm: Option<&[f32]>,
+) -> io::Result<()> {
+ w.write_all(&encode_message(message, pcm)?)?;
+ w.flush()
+}
+
+/// Read one frame. `Ok(None)` only on a clean EOF before the first byte of a
+/// frame; a frame cut off part-way is an error.
+pub fn read_message<T: for<'de> Deserialize<'de>>(
+ r: &mut impl Read,
+) -> io::Result<Option<(T, Vec<f32>)>> {
+ let mut len = [0u8; 4];
+ let mut filled = 0;
+ while filled < len.len() {
+ match r.read(&mut len[filled..]) {
+ Ok(0) if filled == 0 => return Ok(None),
+ Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()),
+ Ok(n) => filled += n,
+ Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
+ Err(e) => return Err(e),
+ }
+ }
+ let json = read_section_body(r, len)?;
+ let message = serde_json::from_slice(&json).map_err(io::Error::other)?;
+ let pcm_bytes = read_section(r)?;
+ if pcm_bytes.len() % 4 != 0 {
+ return Err(io::Error::other(
+ "PCM section is not a whole number of f32s",
+ ));
+ }
+ let pcm = pcm_bytes
+ .as_chunks::<4>()
+ .0
+ .iter()
+ .map(|b| f32::from_le_bytes(*b))
+ .collect();
+ Ok(Some((message, pcm)))
+}
+
+fn read_section(r: &mut impl Read) -> io::Result<Vec<u8>> {
+ let mut len = [0u8; 4];
+ r.read_exact(&mut len)?;
+ read_section_body(r, len)
+}
+
+fn read_section_body(r: &mut impl Read, len: [u8; 4]) -> io::Result<Vec<u8>> {
+ let len = u32::from_le_bytes(len) as usize;
+ check_section_len(len)?;
+ let mut buf = vec![0u8; len];
+ r.read_exact(&mut buf)?;
+ Ok(buf)
+}
+
+/// Also keeps every section length within the frame's `u32` length prefix.
+fn check_section_len(len: usize) -> io::Result<()> {
+ if len > MAX_SECTION_BYTES {
+ return Err(io::Error::other(format!("frame section too large: {len}")));
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn round_trips_message_with_pcm() {
+ let mut buf = Vec::new();
+ let pcm = [0.0f32, -1.0, 0.5, f32::MIN_POSITIVE];
+ write_message(
+ &mut buf,
+ &Request::Run {
+ options: RunOptions::default(),
+ },
+ Some(&pcm),
+ )
+ .unwrap();
+ let (req, got): (Request, Vec<f32>) = read_message(&mut buf.as_slice()).unwrap().unwrap();
+ assert!(matches!(req, Request::Run { .. }));
+ assert_eq!(got, pcm);
+ }
+
+ #[test]
+ fn clean_eof_is_none() {
+ let empty: &[u8] = &[];
+ assert!(read_message::<Request>(&mut &*empty).unwrap().is_none());
+ }
+
+ #[test]
+ fn truncated_frame_is_an_error() {
+ let mut buf = Vec::new();
+ write_message(
+ &mut buf,
+ &Request::Run {
+ options: RunOptions::default(),
+ },
+ Some(&[0.5f32; 8]),
+ )
+ .unwrap();
+ // Cut inside the length prefix, the JSON, the PCM length and the PCM.
+ for cut in [1, 3, 6, buf.len() - 34, buf.len() - 1] {
+ let partial = &buf[..cut];
+ let err = read_message::<Request>(&mut &*partial).unwrap_err();
+ assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof, "cut at {cut}");
+ }
+ }
+
+ #[test]
+ fn section_limit_is_inclusive_and_fits_the_length_prefix() {
+ assert!(check_section_len(MAX_SECTION_BYTES).is_ok());
+ assert!(check_section_len(MAX_SECTION_BYTES + 1).is_err());
+ assert!(MAX_SECTION_BYTES <= u32::MAX as usize);
+ }
+
+ #[test]
+ fn device_key_matches_legacy_format() {
+ assert_eq!(
+ device_key("metal", None, "Metal"),
+ r#"["metal","name","Metal"]"#
+ );
+ assert_eq!(
+ device_key("vulkan", Some("0000:01:00.0"), "Vulkan0"),
+ r#"["vulkan","id","0000:01:00.0"]"#
+ );
+ }
+}
diff --git a/src-tauri/src/engine_supervisor/supervisor.rs b/src-tauri/src/engine_supervisor/supervisor.rs
new file mode 100644
index 00000000..8fae4f0b
--- /dev/null
+++ b/src-tauri/src/engine_supervisor/supervisor.rs
@@ -0,0 +1,1975 @@
+//! Parent side: [`EngineSupervisor`], the single owner of the transcribe.cpp
+//! worker process.
+//!
+//! One owner thread works through a command queue and holds the only worker
+//! slot, the loaded model's spec and any active stream, so "at most one
+//! model worker alive" holds by construction. Callers' commands wait their
+//! turn; waiting in the queue (e.g. behind a model load) is never a timeout.
+//! Only [`EngineSupervisor::cancel`] and device probes bypass the queue: a
+//! probe runs its own short-lived, model-less worker on the caller's thread,
+//! so a hung GPU driver can't hold up dictation behind it.
+//!
+//! A worker that will only use the CPU is started CPU-only, so it never
+//! registers a GPU backend. One rule decides: the spec it loads asks for
+//! [`Backend::Cpu`] (the user's choice, or a fallback after a GPU failure),
+//! or the host must not use the GPU at all.
+
+use super::protocol::{
+ encode_message, read_message, DeviceInfo, DeviceSelector, LoadedInfo, Request, Response,
+};
+use super::worker::LOG_LINE_PREFIX;
+use super::{CPU_ONLY_FLAG, LOG_LEVEL_ENV, WORKER_FLAG};
+use log::{debug, error, info, warn, Level};
+use std::collections::VecDeque;
+use std::fmt;
+use std::io::{self, BufRead, BufReader, Write};
+use std::panic::{catch_unwind, AssertUnwindSafe};
+use std::path::PathBuf;
+use std::process::{Child, ChildStdin, ChildStdout, Command as ProcessCommand, ExitStatus, Stdio};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{mpsc, Arc, Mutex, MutexGuard};
+use std::thread;
+use std::time::{Duration, Instant};
+use transcribe_cpp::{Backend, RunOptions, StreamOptions, StreamText, StreamUpdate, Transcript};
+
+/// Spawn, backend init and device listing. Listing first opens the GPU,
+/// which on macOS compiles ggml's Metal library when the system shader cache
+/// does not hold it yet (~15 s after an install or update).
+const HELLO_TIMEOUT: Duration = Duration::from_secs(60);
+/// Model loads stay generous: a slow load is slow, not hung (#1841).
+const LOAD_TIMEOUT: Duration = Duration::from_secs(180);
+/// Floor for stream calls that do no decoding (begin, reset).
+const CALL_FLOOR: Duration = Duration::from_secs(30);
+/// How long a worker gets to exit after its stdin closes before it's killed.
+const SHUTDOWN_GRACE: Duration = Duration::from_secs(2);
+/// Native stderr lines kept for crash reports (a `GGML_ASSERT` message lands
+/// here right before an abort).
+const STDERR_TAIL_LINES: usize = 64;
+const SAMPLE_RATE: f64 = 16_000.0;
+
+/// How long decoding work may take before its worker counts as hung, by the
+/// device the model runs on. On a GPU a hang is recovered from (retried on
+/// CPU), so it should be caught soon. On CPU a hang is never retried, so the
+/// deadline only decides when to give up: a slow machine must not be
+/// mistaken for a hung one.
+#[derive(Debug, Clone, Copy, PartialEq)]
+struct Deadlines {
+ /// Floor for a stream feed or finalize, so sub-second chunks and
+ /// first-use shader compiles are not mistaken for hangs.
+ call_floor: Duration,
+ /// Floor for a batch run. A run has a fixed cost however short the
+ /// clip: Whisper always encodes a full 30 s window, which on CPU takes
+ /// seconds even for a 1 s clip.
+ run_floor: Duration,
+ /// Work on N seconds of audio may take up to this many times N.
+ audio_factor: f64,
+}
+
+const GPU_DEADLINES: Deadlines = Deadlines {
+ call_floor: Duration::from_secs(30),
+ run_floor: Duration::from_secs(60),
+ audio_factor: 10.0,
+};
+
+/// A single CPU feed or finalize can take well over 10 s, and a large model
+/// on an older CPU runs slower than real time.
+const CPU_DEADLINES: Deadlines = Deadlines {
+ call_floor: Duration::from_secs(120),
+ run_floor: Duration::from_secs(300),
+ audio_factor: 20.0,
+};
+
+impl Deadlines {
+ fn for_device(on_gpu: bool) -> Self {
+ if on_gpu {
+ GPU_DEADLINES
+ } else {
+ CPU_DEADLINES
+ }
+ }
+
+ /// Deadline for stream work on `samples` of 16 kHz audio.
+ fn audio(&self, samples: usize) -> Duration {
+ self.call_floor.max(Duration::from_secs_f64(
+ samples as f64 / SAMPLE_RATE * self.audio_factor,
+ ))
+ }
+
+ /// Deadline for a batch run on `samples` of 16 kHz audio.
+ fn run(&self, samples: usize) -> Duration {
+ self.run_floor.max(self.audio(samples))
+ }
+}
+
+fn ms_to_samples(ms: i64) -> usize {
+ (ms.max(0) as usize) * (SAMPLE_RATE as usize / 1000)
+}
+
+/// What to load. The supervisor keeps it so a dead worker can be replaced
+/// transparently.
+#[derive(Debug, Clone)]
+pub struct LoadSpec {
+ pub path: PathBuf,
+ pub backend: Backend,
+ pub device: DeviceSelector,
+}
+
+impl LoadSpec {
+ /// An explicitly chosen device (`--device-index`) never falls back to
+ /// CPU: its failures are reported, not hidden.
+ fn pinned(&self) -> bool {
+ matches!(self.device, DeviceSelector::Index(_))
+ }
+
+ fn cpu_only(&self) -> bool {
+ self.backend == Backend::Cpu
+ }
+
+ fn cpu(&self) -> Self {
+ Self {
+ path: self.path.clone(),
+ backend: Backend::Cpu,
+ device: DeviceSelector::Auto,
+ }
+ }
+}
+
+#[derive(Debug)]
+pub enum EngineError {
+ /// [`EngineSupervisor::cancel`] stopped the work.
+ Cancelled,
+ /// No model, a transcribe.cpp error, or a worker failure that could not
+ /// be recovered.
+ Failed(String),
+}
+
+impl fmt::Display for EngineError {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ EngineError::Cancelled => f.write_str("transcription was cancelled"),
+ EngineError::Failed(message) => f.write_str(message),
+ }
+ }
+}
+
+impl std::error::Error for EngineError {}
+
+/// The result of one stream feed, handed to the stream's callback.
+pub struct StreamProgress {
+ pub update: StreamUpdate,
+ /// Present only when the committed or tentative text changed.
+ pub text: Option<StreamText>,
+ /// Time from sending the frame to the worker until its answer.
+ pub elapsed: Duration,
+}
+
+pub struct Finalized {
+ pub update: StreamUpdate,
+ pub text: StreamText,
+ /// The model's detected language, when asked for.
+ pub language: Option<String>,
+ pub elapsed: Duration,
+}
+
+type Reply<T> = mpsc::Sender<T>;
+type OnProgress = Box<dyn FnMut(StreamProgress) + Send>;
+
+enum Command {
+ Load(LoadSpec, Reply<Result<LoadedInfo, EngineError>>),
+ Unload(Reply<()>),
+ Transcribe {
+ pcm: Vec<f32>,
+ run: RunOptions,
+ epoch: u64,
+ reply: Reply<Result<Transcript, EngineError>>,
+ },
+ StreamBegin {
+ active: ActiveStream,
+ run: RunOptions,
+ stream: StreamOptions,
+ reply: Reply<Result<(), EngineError>>,
+ },
+ Feed {
+ id: u64,
+ pcm: Vec<f32>,
+ },
+ Finalize {
+ id: u64,
+ want_language: bool,
+ epoch: u64,
+ reply: Reply<Result<Option<Finalized>, EngineError>>,
+ },
+ StreamReset {
+ id: u64,
+ },
+}
+
+/// State readable without going through the queue.
+#[derive(Default)]
+struct Shared {
+ /// Whether this host may use a GPU at all. When not, every worker is
+ /// CPU-only, device probes included.
+ gpu_allowed: bool,
+ /// A worker with GPU backends crashed or hung (in backend init, device
+ /// listing, model load or inference). Models load in CPU-only workers,
+ /// and devices are not listed again, until [`EngineSupervisor::retry_gpu`].
+ gpu_unavailable: AtomicBool,
+ /// The loaded model (logically: it stays loaded while its worker is
+ /// replaced after a crash or cancel).
+ loaded: Mutex<Option<LoadedInfo>>,
+ /// The compute devices as last listed by a worker; `None` until one has.
+ devices: Mutex<Option<Vec<DeviceInfo>>>,
+ /// Held for the length of a device probe, so concurrent callers wait for
+ /// the one probe instead of starting their own.
+ probing: Mutex<()>,
+ /// Bumped by every [`EngineSupervisor::cancel`]. Work queued before the
+ /// bump counts as cancelled.
+ cancel_epoch: AtomicU64,
+ /// The call the owner is waiting on, if any. A cancel takes it and kills
+ /// its worker.
+ in_flight: Mutex<Option<InFlight>>,
+ /// Bumped by every [`EngineSupervisor::unload`] as it is called, so the
+ /// model reads as unloaded at once, even while the unload waits its turn.
+ unloads_requested: AtomicU64,
+ next_id: AtomicU64,
+}
+
+impl Shared {
+ /// Start a model-less worker just to list devices. If it may use the GPU
+ /// and crashes or hangs doing so, the GPU is marked unavailable.
+ fn probe_devices(&self) {
+ let cpu_only = !self.gpu_allowed;
+ match self.hello(cpu_only, true) {
+ Ok(worker) => drop(worker),
+ Err(failure) => {
+ error!(
+ "Could not start a transcription worker to list compute devices: {}",
+ failure
+ );
+ if failure.worker_lost() && !cpu_only {
+ self.mark_gpu_unavailable(format!(
+ "a worker with GPU backends {failure} while listing compute devices"
+ ));
+ }
+ }
+ }
+ }
+
+ /// Start a worker and say hello, publishing its device list if it lists
+ /// one.
+ fn hello(&self, cpu_only: bool, list_devices: bool) -> Result<Worker, Failure> {
+ let mut worker = Worker::spawn(cpu_only).map_err(Failure::Spawn)?;
+ match worker.call(&Request::Hello { list_devices }, None, HELLO_TIMEOUT)? {
+ Response::Hello { devices } => {
+ if let Some(devices) = devices {
+ self.set_devices(devices);
+ }
+ Ok(worker)
+ }
+ other => Err(unexpected(&other)),
+ }
+ }
+
+ fn set_devices(&self, new: Vec<DeviceInfo>) {
+ let mut current = lock(&self.devices);
+ if let Some(old) = &*current {
+ let identity = |d: &DeviceInfo| (d.key.clone(), d.index);
+ if !old.iter().map(identity).eq(new.iter().map(identity)) {
+ info!(
+ "transcribe-cpp compute devices changed: [{}]",
+ new.iter()
+ .map(|d| format!("{} ({})", d.name, d.kind))
+ .collect::<Vec<_>>()
+ .join(", ")
+ );
+ }
+ }
+ *current = Some(new);
+ }
+
+ fn gpu_unavailable(&self) -> bool {
+ self.gpu_unavailable.load(Ordering::Acquire)
+ }
+
+ /// `reason` is logged when this changes the state.
+ fn mark_gpu_unavailable(&self, reason: impl fmt::Display) {
+ if !self.gpu_unavailable.swap(true, Ordering::AcqRel) {
+ warn!("Transcription GPU marked unavailable: {}", reason);
+ }
+ }
+
+ fn epoch(&self) -> u64 {
+ self.cancel_epoch.load(Ordering::SeqCst)
+ }
+
+ fn cancelled_since(&self, epoch: u64) -> bool {
+ self.epoch() != epoch
+ }
+
+ fn next_id(&self) -> u64 {
+ self.next_id.fetch_add(1, Ordering::Relaxed) + 1
+ }
+}
+
+struct InFlight {
+ what: &'static str,
+ control: Arc<Control>,
+}
+
+fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
+ mutex.lock().unwrap_or_else(|e| e.into_inner())
+}
+
+/// Owns the one transcribe.cpp worker slot. Cheap to clone.
+#[derive(Clone)]
+pub struct EngineSupervisor {
+ commands: mpsc::Sender<Command>,
+ shared: Arc<Shared>,
+}
+
+impl EngineSupervisor {
+ /// Start the owner thread. No worker runs until something needs one.
+ /// Without `gpu_allowed`, every worker is CPU-only, so no GPU driver is
+ /// ever loaded.
+ pub fn new(gpu_allowed: bool) -> Self {
+ #[cfg(all(unix, not(test)))]
+ record_launched_exe();
+ let (commands, queue) = mpsc::channel();
+ let shared = Arc::new(Shared {
+ gpu_allowed,
+ ..Shared::default()
+ });
+ let owner = Owner {
+ shared: Arc::clone(&shared),
+ worker: None,
+ spec: None,
+ stream: None,
+ unloads_done: 0,
+ };
+ thread::Builder::new()
+ .name("transcribe-engine".into())
+ .spawn(move || owner.run(queue))
+ .expect("failed to start the transcription engine thread");
+ Self { commands, shared }
+ }
+
+ fn ask<T>(&self, command: impl FnOnce(Reply<T>) -> Command) -> Option<T> {
+ let (reply, answer) = mpsc::channel();
+ self.commands.send(command(reply)).ok()?;
+ answer.recv().ok()
+ }
+
+ /// Load a model, replacing the current one. The old worker has fully
+ /// exited before the new one starts, so two models are never held at
+ /// once.
+ pub fn load(&self, spec: LoadSpec) -> Result<LoadedInfo, EngineError> {
+ self.ask(|reply| Command::Load(spec, reply))
+ .unwrap_or_else(|| Err(engine_stopped()))
+ }
+
+ /// Unload the model, without waiting. It reads as unloaded at once and
+ /// the unload is queued ahead of anything requested later, so a new
+ /// dictation queues a fresh load behind it and is never affected by it.
+ /// [`Unloading::wait`] returns once the worker has exited.
+ pub fn unload(&self) -> Unloading {
+ let (reply, exited) = mpsc::channel();
+ {
+ let mut loaded = lock(&self.shared.loaded);
+ self.shared.unloads_requested.fetch_add(1, Ordering::SeqCst);
+ *loaded = None;
+ if self.commands.send(Command::Unload(reply)).is_err() {
+ return Unloading(None);
+ }
+ }
+ Unloading(Some(exited))
+ }
+
+ /// An explicit transcribe.cpp accelerator or GPU device change: forget
+ /// the GPU failure, if any, so the next load may use (and list) the GPU
+ /// again. A new failure sets the flag again.
+ pub fn retry_gpu(&self, reason: &str) {
+ if self.shared.gpu_unavailable.swap(false, Ordering::AcqRel) {
+ info!("Transcription GPU no longer marked unavailable: {}", reason);
+ }
+ }
+
+ /// The loaded model. A snapshot; never waits on the queue.
+ pub fn loaded(&self) -> Option<LoadedInfo> {
+ lock(&self.shared.loaded).clone()
+ }
+
+ /// Transcribe 16 kHz mono PCM with the loaded model. A worker crash or
+ /// hang is retried once in a fresh worker (on CPU if it was on the GPU).
+ pub fn transcribe(&self, pcm: Vec<f32>, run: RunOptions) -> Result<Transcript, EngineError> {
+ let epoch = self.shared.epoch();
+ self.ask(|reply| Command::Transcribe {
+ pcm,
+ run,
+ epoch,
+ reply,
+ })
+ .unwrap_or_else(|| Err(engine_stopped()))
+ }
+
+ /// Begin a live stream on the loaded model. `on_progress` runs on the
+ /// owner thread after every feed, so it must not block or call back into
+ /// the supervisor.
+ pub fn start_stream(
+ &self,
+ run: RunOptions,
+ stream: StreamOptions,
+ on_progress: impl FnMut(StreamProgress) + Send + 'static,
+ ) -> Result<StreamHandle, EngineError> {
+ let id = self.shared.next_id();
+ // The stream is one piece of cancellable work: a cancel from here on
+ // stops its feeds and its finalize.
+ let epoch = self.shared.epoch();
+ let closed = Arc::new(AtomicBool::new(false));
+ self.ask(|reply| Command::StreamBegin {
+ active: ActiveStream {
+ id,
+ epoch,
+ closed: Arc::clone(&closed),
+ on_progress: Box::new(on_progress),
+ pending_ms: 0,
+ },
+ run,
+ stream,
+ reply,
+ })
+ .unwrap_or_else(|| Err(engine_stopped()))?;
+ Ok(StreamHandle {
+ id,
+ epoch,
+ closed,
+ commands: self.commands.clone(),
+ finished: false,
+ })
+ }
+
+ /// Stop the transcription or stream in progress, and any queued behind
+ /// it. Bypasses the queue. The worker doing the work is killed, so the
+ /// next use starts a fresh one and reloads the model. Model loads are
+ /// never cancelled.
+ pub fn cancel(&self) {
+ self.shared.cancel_epoch.fetch_add(1, Ordering::SeqCst);
+ // Taking the call tells the owner its worker was killed.
+ if let Some(call) = lock(&self.shared.in_flight).take() {
+ info!(
+ "Cancelling the {} by stopping its worker (pid {})",
+ call.what, call.control.pid
+ );
+ call.control.kill();
+ }
+ }
+
+ /// The compute devices, as listed by the latest worker. If no worker has
+ /// listed them yet, starts a short-lived worker on this thread to list
+ /// them. It runs beside any model worker, which it leaves alone. Not
+ /// while the GPU is marked unavailable: that probe would crash or hang
+ /// the same way.
+ pub fn devices(&self) -> Option<Vec<DeviceInfo>> {
+ let _probing = lock(&self.shared.probing);
+ if lock(&self.shared.devices).is_none() && !self.shared.gpu_unavailable() {
+ self.shared.probe_devices();
+ }
+ lock(&self.shared.devices).clone()
+ }
+}
+
+/// An unload in progress; see [`EngineSupervisor::unload`].
+pub struct Unloading(Option<mpsc::Receiver<()>>);
+
+impl Unloading {
+ /// Wait until the worker has exited (e.g. before deleting the model file).
+ pub fn wait(self) {
+ if let Some(exited) = self.0 {
+ let _ = exited.recv();
+ }
+ }
+}
+
+/// A live stream in the worker. Feeds are queued without waiting, in order
+/// with finalize. Dropping the handle without finalizing resets the stream.
+pub struct StreamHandle {
+ id: u64,
+ /// Cancel epoch when the stream started.
+ epoch: u64,
+ /// Set when the handle is dropped; shared with the owner.
+ closed: Arc<AtomicBool>,
+ commands: mpsc::Sender<Command>,
+ finished: bool,
+}
+
+impl StreamHandle {
+ pub fn feed(&self, pcm: Vec<f32>) {
+ let _ = self.commands.send(Command::Feed { id: self.id, pcm });
+ }
+
+ /// Flush the stream and return its final text. `Ok(None)` if the stream
+ /// was lost or failed; fall back to batch transcription then.
+ /// `Err(Cancelled)` if [`EngineSupervisor::cancel`] stopped it.
+ pub fn finalize(mut self, want_language: bool) -> Result<Option<Finalized>, EngineError> {
+ self.finished = true;
+ let (reply, answer) = mpsc::channel();
+ let command = Command::Finalize {
+ id: self.id,
+ want_language,
+ epoch: self.epoch,
+ reply,
+ };
+ if self.commands.send(command).is_err() {
+ return Ok(None);
+ }
+ answer.recv().unwrap_or(Ok(None))
+ }
+}
+
+impl Drop for StreamHandle {
+ fn drop(&mut self) {
+ if !self.finished {
+ // Queued feeds for this stream are skipped from here on.
+ self.closed.store(true, Ordering::Release);
+ let _ = self.commands.send(Command::StreamReset { id: self.id });
+ }
+ }
+}
+
+fn engine_stopped() -> EngineError {
+ EngineError::Failed("the transcription engine has stopped".to_string())
+}
+
+fn not_loaded() -> EngineError {
+ EngineError::Failed("no transcription model is loaded".to_string())
+}
+
+/// How a call to the worker failed.
+#[derive(Debug)]
+enum Failure {
+ /// The worker process could not be started.
+ Spawn(io::Error),
+ /// transcribe.cpp returned an error; the worker is fine.
+ Remote(String),
+ /// The worker exited or crashed. Carries exit status and its last
+ /// native output.
+ Crashed(String),
+ /// The worker did not answer within the deadline and was killed.
+ Hung(Duration),
+ Cancelled,
+}
+
+impl Failure {
+ /// Whether the worker is gone.
+ fn worker_lost(&self) -> bool {
+ matches!(self, Failure::Crashed(_) | Failure::Hung(_))
+ }
+}
+
+impl fmt::Display for Failure {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ Failure::Spawn(e) => write!(f, "failed to start transcription worker: {e}"),
+ Failure::Remote(message) => f.write_str(message),
+ Failure::Crashed(detail) => write!(f, "transcription worker crashed ({detail})"),
+ Failure::Hung(t) => write!(f, "transcription worker did not respond within {t:?}"),
+ Failure::Cancelled => f.write_str("cancelled"),
+ }
+ }
+}
+
+impl From<Failure> for EngineError {
+ fn from(failure: Failure) -> Self {
+ match failure {
+ Failure::Cancelled => EngineError::Cancelled,
+ other => EngineError::Failed(other.to_string()),
+ }
+ }
+}
+
+fn unexpected(response: &Response) -> Failure {
+ Failure::Remote(format!(
+ "unexpected transcription worker response: {response:?}"
+ ))
+}
+
+// --- Owner thread ---------------------------------------------------------
+
+struct ActiveStream {
+ id: u64,
+ /// Cancel epoch when the stream started; a later cancel stops it.
+ epoch: u64,
+ /// Set when the caller drops its handle.
+ closed: Arc<AtomicBool>,
+ on_progress: OnProgress,
+ /// Audio received but not yet committed, which finalize must still
+ /// decode; sizes the feed and finalize deadlines.
+ pending_ms: i64,
+}
+
+struct Owner {
+ shared: Arc<Shared>,
+ /// The single worker slot.
+ worker: Option<Worker>,
+ /// The model the app asked for. Outlives its worker: after a crash,
+ /// hang or cancel the next use starts a fresh worker for it.
+ spec: Option<LoadSpec>,
+ stream: Option<ActiveStream>,
+ /// Unload commands processed, against [`Shared::unloads_requested`].
+ unloads_done: u64,
+}
+
+impl Owner {
+ fn run(mut self, queue: mpsc::Receiver<Command>) {
+ while let Ok(command) = queue.recv() {
+ match command {
+ Command::Load(spec, reply) => {
+ let _ = reply.send(self.load(spec));
+ }
+ Command::Unload(reply) => {
+ self.unload();
+ self.unloads_done += 1;
+ let _ = reply.send(());
+ }
+ Command::Transcribe {
+ pcm,
+ run,
+ epoch,
+ reply,
+ } => {
+ let _ = reply.send(self.transcribe(&pcm, &run, epoch));
+ }
+ Command::StreamBegin {
+ active,
+ run,
+ stream,
+ reply,
+ } => {
+ let _ = reply.send(self.stream_begin(active, &run, &stream));
+ }
+ Command::Feed { id, pcm } => self.feed(id, &pcm),
+ Command::Finalize {
+ id,
+ want_language,
+ epoch,
+ reply,
+ } => {
+ let _ = reply.send(self.finalize(id, want_language, epoch));
+ }
+ Command::StreamReset { id } => {
+ if self.stream.as_ref().is_some_and(|s| s.id == id) {
+ self.end_stream();
+ }
+ }
+ }
+ }
+ // Every handle is gone.
+ self.unload();
+ }
+
+ fn load(&mut self, spec: LoadSpec) -> Result<LoadedInfo, EngineError> {
+ self.unload();
+ self.spec = Some(spec);
+ let result = self.start();
+ if result.is_err() {
+ self.spec = None;
+ }
+ result
+ }
+
+ fn unload(&mut self) {
+ // The worker's exit ends any stream; no need to reset it first.
+ self.stream = None;
+ if let Some(worker) = self.worker.take() {
+ let started = Instant::now();
+ let pid = worker.control.pid;
+ drop(worker);
+ debug!(
+ "Transcription worker (pid {}) stopped in {}ms",
+ pid,
+ started.elapsed().as_millis()
+ );
+ }
+ self.spec = None;
+ *lock(&self.shared.loaded) = None;
+ }
+
+ /// The spec a fresh worker should load: CPU while the GPU is marked
+ /// unavailable, unless the device was chosen explicitly.
+ fn effective_spec(&self, spec: &LoadSpec) -> LoadSpec {
+ if spec.pinned() || spec.cpu_only() || !self.shared.gpu_unavailable() {
+ return spec.clone();
+ }
+ info!("Loading the transcription model on CPU: the GPU failed earlier");
+ spec.cpu()
+ }
+
+ /// Whether a worker for `spec` registers only the CPU backends.
+ fn cpu_only_worker(&self, spec: &LoadSpec) -> bool {
+ spec.cpu_only() || !self.shared.gpu_allowed
+ }
+
+ /// Start a worker for `spec` and load it there. A worker that may use the
+ /// GPU lists the devices in its hello, which keeps the list current (a
+ /// GPU that comes back is just noticed). A CPU-only worker would only
+ /// see the CPU, so it doesn't list.
+ fn start_worker(&self, spec: &LoadSpec) -> Result<(Worker, LoadedInfo), Failure> {
+ let cpu_only = self.cpu_only_worker(spec);
+ let mut worker = self.shared.hello(cpu_only, !cpu_only)?;
+ let info = worker.load(spec)?;
+ Ok((worker, info))
+ }
+
+ /// Start a worker and load the current spec in it. If a worker with GPU
+ /// backends is lost on the way (backend init, device listing or the
+ /// load), mark the GPU unavailable and retry once in a CPU-only worker.
+ fn start(&mut self) -> Result<LoadedInfo, EngineError> {
+ let spec = self.spec.clone().ok_or_else(not_loaded)?;
+ let started = Instant::now();
+ let wanted = self.effective_spec(&spec);
+ let (worker, info) = match self.start_worker(&wanted) {
+ Ok(started) => started,
+ Err(failure)
+ if failure.worker_lost() && !self.cpu_only_worker(&wanted) && !wanted.pinned() =>
+ {
+ self.shared.mark_gpu_unavailable(format!(
+ "a worker for '{}' with GPU backends {}",
+ wanted.path.display(),
+ failure
+ ));
+ warn!("Retrying in a CPU-only worker");
+ self.start_worker(&wanted.cpu())?
+ }
+ Err(failure) => return Err(failure.into()),
+ };
+ debug!(
+ "Worker (pid {}) spawned and loaded the model in {}ms",
+ worker.control.pid,
+ started.elapsed().as_millis()
+ );
+ self.publish_loaded(&info);
+ self.worker = Some(worker);
+ Ok(info)
+ }
+
+ /// Report the model loaded, unless an unload was requested since and is
+ /// still waiting in the queue.
+ fn publish_loaded(&self, info: &LoadedInfo) {
+ let mut loaded = lock(&self.shared.loaded);
+ if self.shared.unloads_requested.load(Ordering::SeqCst) == self.unloads_done {
+ *loaded = Some(info.clone());
+ }
+ }
+
+ /// Make sure the slot holds a worker with the model loaded, starting a
+ /// fresh one if the last one died or was cancelled.
+ fn ensure_worker(&mut self) -> Result<(), EngineError> {
+ if self.worker.is_some() {
+ return Ok(());
+ }
+ if self.spec.is_none() {
+ return Err(not_loaded());
+ }
+ info!("Starting a fresh transcription worker for the loaded model");
+ if let Err(e) = self.start() {
+ // The model can't be brought back; report it unloaded so the
+ // next use loads it from scratch (and reports why it can't).
+ self.spec = None;
+ *lock(&self.shared.loaded) = None;
+ return Err(e);
+ }
+ Ok(())
+ }
+
+ /// Drop a worker that crashed or hung (its handle already logged the
+ /// details and native output). If it was on a GPU, mark the GPU
+ /// unavailable so the next worker uses CPU. Returns whether it was.
+ fn worker_lost(&mut self, during: &str, failure: &Failure) -> bool {
+ // Dropping the handle reaps the process.
+ let info = self.worker.take().and_then(|worker| worker.info.clone());
+ let pinned = self.spec.as_ref().is_some_and(LoadSpec::pinned);
+ match info {
+ Some(info) if info.on_gpu => {
+ if !pinned {
+ self.shared.mark_gpu_unavailable(format!(
+ "{failure} during {during} on GPU '{}' ({})",
+ info.device, info.backend
+ ));
+ }
+ true
+ }
+ _ => false,
+ }
+ }
+
+ /// Deadlines for the device the current worker's model runs on.
+ fn deadlines(&self) -> Deadlines {
+ let on_gpu = self
+ .worker
+ .as_ref()
+ .and_then(|worker| worker.info.as_ref())
+ .is_some_and(|info| info.on_gpu);
+ Deadlines::for_device(on_gpu)
+ }
+
+ /// A call that [`EngineSupervisor::cancel`] can stop. Returns
+ /// `Cancelled` if a cancel came after `epoch`, whatever the worker said.
+ fn call_work(
+ &mut self,
+ request: &Request,
+ pcm: Option<&[f32]>,
+ deadline: Duration,
+ epoch: u64,
+ ) -> Result<Response, Failure> {
+ let worker = self.worker.as_mut().expect("caller ensured a worker");
+ let frame = encode_request(request, pcm)?;
+ let sent = {
+ // Check, send and publish under the lock cancel() takes after
+ // bumping the epoch: either this sees the cancel and sends
+ // nothing, or the cancel sees the call and kills its worker.
+ let mut in_flight = lock(&self.shared.in_flight);
+ if self.shared.cancelled_since(epoch) {
+ return Err(Failure::Cancelled);
+ }
+ let sent = worker.control.send(frame);
+ *in_flight = Some(InFlight {
+ what: request_name(request),
+ control: Arc::clone(&worker.control),
+ });
+ sent
+ };
+ let result = if sent {
+ worker.wait(request, deadline)
+ } else {
+ Err(worker.died())
+ };
+ // Gone if cancel() took the call, which means it killed the worker
+ // (possibly just after it answered).
+ let killed = lock(&self.shared.in_flight).take().is_none();
+ if killed || self.shared.cancelled_since(epoch) {
+ if killed || result.as_ref().is_err_and(Failure::worker_lost) {
+ // The next use starts a fresh worker.
+ self.worker = None;
+ }
+ return Err(Failure::Cancelled);
+ }
+ result
+ }
+
+ /// Retry once after a crash or hang, on CPU if it happened on the GPU.
+ /// Not after a hang on CPU, and never for an explicitly chosen device.
+ fn transcribe(
+ &mut self,
+ pcm: &[f32],
+ run: &RunOptions,
+ epoch: u64,
+ ) -> Result<Transcript, EngineError> {
+ // A stream left open would make the worker refuse the run.
+ self.end_stream();
+ let request = Request::Run {
+ options: run.clone(),
+ };
+ let mut retried = false;
+ loop {
+ if self.shared.cancelled_since(epoch) {
+ return Err(EngineError::Cancelled);
+ }
+ // A respawn here runs under the load deadline; the run's own
+ // deadline starts only once the model is loaded.
+ self.ensure_worker()?;
+ let deadline = self.deadlines().run(pcm.len());
+ match self.call_work(&request, Some(pcm), deadline, epoch) {
+ Ok(Response::Transcript(transcript)) => return Ok(transcript),
+ Ok(other) => return Err(unexpected(&other).into()),
+ Err(failure) if failure.worker_lost() => {
+ let on_gpu = self.worker_lost("transcription", &failure);
+ let pinned = self.spec.as_ref().is_some_and(LoadSpec::pinned);
+ let hung_on_cpu = matches!(failure, Failure::Hung(_)) && !on_gpu;
+ if retried || pinned || hung_on_cpu {
+ return Err(failure.into());
+ }
+ warn!("Retrying the transcription in a fresh worker");
+ retried = true;
+ }
+ Err(failure) => return Err(failure.into()),
+ }
+ }
+ }
+
+ fn stream_begin(
+ &mut self,
+ active: ActiveStream,
+ run: &RunOptions,
+ stream: &StreamOptions,
+ ) -> Result<(), EngineError> {
+ self.end_stream();
+ if self.shared.cancelled_since(active.epoch) {
+ return Err(EngineError::Cancelled);
+ }
+ self.ensure_worker()?;
+ let worker = self.worker.as_mut().expect("worker was just ensured");
+ let request = Request::StreamBegin {
+ run: run.clone(),
+ stream: stream.clone(),
+ };
+ match worker.call(&request, None, CALL_FLOOR) {
+ Ok(Response::Ok) => {
+ self.stream = Some(active);
+ Ok(())
+ }
+ Ok(other) => Err(unexpected(&other).into()),
+ Err(failure) => {
+ if failure.worker_lost() {
+ self.worker_lost("stream start", &failure);
+ }
+ Err(failure.into())
+ }
+ }
+ }
+
+ /// Feeds never respawn the worker (the stream state would be lost); a
+ /// lost stream just drops the rest of its frames and finalizes to
+ /// `None`, and the caller falls back to batch. Feeds are cancellable like
+ /// any other dictation work.
+ fn feed(&mut self, id: u64, pcm: &[f32]) {
+ let Some(stream) = self.stream.as_ref().filter(|s| s.id == id) else {
+ return;
+ };
+ // Skipped once the caller dropped its handle or cancelled.
+ if stream.closed.load(Ordering::Acquire)
+ || self.shared.cancelled_since(stream.epoch)
+ || self.worker.is_none()
+ {
+ return;
+ }
+ let epoch = stream.epoch;
+ let deadline = self
+ .deadlines()
+ .audio(pcm.len() + ms_to_samples(stream.pending_ms));
+ let started = Instant::now();
+ match self.call_work(&Request::Feed, Some(pcm), deadline, epoch) {
+ Ok(Response::Fed { update, text }) => {
+ let Some(stream) = self.stream.as_mut() else {
+ return;
+ };
+ stream.pending_ms = pending_ms(&update);
+ let progress = StreamProgress {
+ update,
+ text,
+ elapsed: started.elapsed(),
+ };
+ if catch_unwind(AssertUnwindSafe(|| (stream.on_progress)(progress))).is_err() {
+ error!("Live transcription callback panicked");
+ }
+ }
+ Ok(other) => warn!("Stream feed failed: {}", unexpected(&other)),
+ Err(Failure::Remote(message)) => warn!("Stream feed failed: {}", message),
+ // Later feeds are skipped, and finalize reports the cancel.
+ Err(Failure::Cancelled) => {}
+ Err(failure) => {
+ self.stream = None;
+ self.worker_lost("stream feed", &failure);
+ }
+ }
+ }
+
+ fn finalize(
+ &mut self,
+ id: u64,
+ want_language: bool,
+ epoch: u64,
+ ) -> Result<Option<Finalized>, EngineError> {
+ // Checked first: a stream lost to a cancel must report the cancel,
+ // not "no result", which would start a batch run.
+ if self.shared.cancelled_since(epoch) {
+ if self.stream.as_ref().is_some_and(|s| s.id == id) {
+ self.end_stream();
+ }
+ return Err(EngineError::Cancelled);
+ }
+ let Some(stream) = self.stream.take_if(|s| s.id == id) else {
+ // Lost earlier (or replaced): fall back to batch.
+ return Ok(None);
+ };
+ if self.worker.is_none() {
+ return Ok(None);
+ }
+ let deadline = self.deadlines().audio(ms_to_samples(stream.pending_ms));
+ let started = Instant::now();
+ match self.call_work(&Request::Finalize { want_language }, None, deadline, epoch) {
+ Ok(Response::Finalized {
+ update,
+ text,
+ language,
+ }) => Ok(Some(Finalized {
+ update,
+ text,
+ language,
+ elapsed: started.elapsed(),
+ })),
+ Ok(other) => {
+ error!("Stream finalize failed: {}", unexpected(&other));
+ Ok(None)
+ }
+ Err(Failure::Cancelled) => {
+ // The finalize may never have reached the worker.
+ self.reset_worker_stream();
+ Err(EngineError::Cancelled)
+ }
+ Err(Failure::Remote(message)) => {
+ error!("Stream finalize failed: {}", message);
+ Ok(None)
+ }
+ Err(failure) => {
+ self.worker_lost("stream finalize", &failure);
+ Ok(None)
+ }
+ }
+ }
+
+ fn end_stream(&mut self) {
+ if self.stream.take().is_some() {
+ self.reset_worker_stream();
+ }
+ }
+
+ fn reset_worker_stream(&mut self) {
+ let Some(worker) = self.worker.as_mut() else {
+ return;
+ };
+ match worker.call(&Request::StreamReset, None, CALL_FLOOR) {
+ Ok(_) => {}
+ // Expected when the stream already ended in the worker.
+ Err(Failure::Remote(message)) => debug!("Stream reset: {}", message),
+ Err(failure) => {
+ self.worker_lost("stream reset", &failure);
+ }
+ }
+ }
+}
+
+fn pending_ms(update: &StreamUpdate) -> i64 {
+ update
+ .buffered_ms
+ .max(update.input_received_ms - update.audio_committed_ms)
+ .max(0)
+}
+
+// --- Worker process handle ------------------------------------------------
+
+/// The parts of a worker that other threads touch: its stdin queue and the
+/// process (to kill it). Every write goes through the one writer thread, so
+/// a cancel can never interleave with a large frame.
+struct Control {
+ pid: u32,
+ child: Mutex<Child>,
+ stdin: Mutex<Option<mpsc::Sender<Vec<u8>>>>,
+}
+
+impl Control {
+ fn send(&self, frame: Vec<u8>) -> bool {
+ lock(&self.stdin)
+ .as_ref()
+ .is_some_and(|stdin| stdin.send(frame).is_ok())
+ }
+
+ /// Closing stdin tells the worker to exit (it ends on EOF).
+ fn close_stdin(&self) {
+ lock(&self.stdin).take();
+ }
+
+ fn kill(&self) {
+ let _ = lock(&self.child).kill();
+ }
+}
+
+/// One worker process, owned by the supervisor's thread. Strict
+/// request/response. Dropping it closes the worker's stdin, gives it
+/// [`SHUTDOWN_GRACE`] to exit, kills it otherwise, and reaps it: once
+/// dropped, the worker is gone.
+struct Worker {
+ control: Arc<Control>,
+ responses: mpsc::Receiver<Response>,
+ stderr_done: mpsc::Receiver<()>,
+ stderr_tail: Arc<Mutex<VecDeque<String>>>,
+ /// Set once a model is loaded.
+ info: Option<LoadedInfo>,
+}
+
+impl Worker {
+ /// Start a worker process. A `cpu_only` worker never registers a GPU
+ /// backend.
+ fn spawn(cpu_only: bool) -> io::Result<Self> {
+ // Under `cargo test` the current exe is the test harness; tests point
+ // this at the built app instead.
+ #[cfg(test)]
+ let exe = PathBuf::from(
+ std::env::var_os("HANDY_TRANSCRIBE_WORKER_EXE")
+ .expect("set HANDY_TRANSCRIBE_WORKER_EXE"),
+ );
+ #[cfg(not(test))]
+ let exe = worker_exe()?;
+ let mut command = ProcessCommand::new(exe);
+ #[cfg(target_os = "linux")]
+ if let Some(arg0) = std::env::args_os().next() {
+ // Keep `ps` showing Handy's own path rather than /proc/self/exe.
+ use std::os::unix::process::CommandExt;
+ command.arg0(arg0);
+ }
+ command.arg(WORKER_FLAG);
+ if cpu_only {
+ command.arg(CPU_ONLY_FLAG);
+ }
+ command
+ .env(LOG_LEVEL_ENV, log::max_level().as_str())
+ .stdin(Stdio::piped())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped());
+ #[cfg(windows)]
+ {
+ use std::os::windows::process::CommandExt;
+ const CREATE_NO_WINDOW: u32 = 0x0800_0000;
+ command.creation_flags(CREATE_NO_WINDOW);
+ }
+ let mut child = command.spawn()?;
+ let pid = child.id();
+ let stdin = child.stdin.take().expect("stdin is piped");
+ let stdout = child.stdout.take().expect("stdout is piped");
+ let stderr = child.stderr.take().expect("stderr is piped");
+
+ let (stdin_tx, frames) = mpsc::channel();
+ let (response_tx, responses) = mpsc::channel();
+ let (stderr_done_tx, stderr_done) = mpsc::channel();
+ #[cfg(test)]
+ LIVE_WORKERS.fetch_add(1, Ordering::SeqCst);
+ // Built before the threads so a failure below still reaps the child.
+ let worker = Worker {
+ control: Arc::new(Control {
+ pid,
+ child: Mutex::new(child),
+ stdin: Mutex::new(Some(stdin_tx)),
+ }),
+ responses,
+ stderr_done,
+ stderr_tail: Arc::new(Mutex::new(VecDeque::with_capacity(STDERR_TAIL_LINES))),
+ info: None,
+ };
+
+ thread::Builder::new()
+ .name("transcribe-worker-in".into())
+ .spawn(move || write_frames(stdin, frames))?;
+ thread::Builder::new()
+ .name("transcribe-worker-out".into())
+ .spawn(move || read_responses(stdout, response_tx))?;
+ // Always drain stderr: a full pipe would block the worker.
+ let tail = Arc::clone(&worker.stderr_tail);
+ thread::Builder::new()
+ .name("transcribe-worker-err".into())
+ .spawn(move || {
+ // Split on raw bytes: native code may write non-UTF-8 (e.g. a
+ // path in the Windows ANSI code page), and `lines()` would end
+ // the loop there, losing every later line, abort message included.
+ for line in BufReader::new(stderr).split(b'\n') {
+ let Ok(line) = line else { break };
+ let line = String::from_utf8_lossy(&line);
+ let line = line.strip_suffix('\r').unwrap_or(&line).to_string();
+ forward_worker_log(&line);
+ let mut tail = lock(&tail);
+ if tail.len() == STDERR_TAIL_LINES {
+ tail.pop_front();
+ }
+ tail.push_back(line);
+ }
+ let _ = stderr_done_tx.send(());
+ })?;
+
+ debug!(
+ "Started {}transcription worker (pid {})",
+ if cpu_only { "CPU-only " } else { "" },
+ pid
+ );
+ Ok(worker)
+ }
+
+ fn load(&mut self, spec: &LoadSpec) -> Result<LoadedInfo, Failure> {
+ let request = Request::Load {
+ path: spec.path.clone(),
+ backend: spec.backend,
+ device: spec.device.clone(),
+ };
+ match self.call(&request, None, LOAD_TIMEOUT)? {
+ Response::Loaded(info) => {
+ self.info = Some(info.clone());
+ Ok(info)
+ }
+ other => Err(unexpected(&other)),
+ }
+ }
+
+ /// Send one request and wait up to `deadline` for its response. Hitting
+ /// the deadline kills the worker; after any `Crashed`/`Hung` the handle
+ /// is dead and should be dropped.
+ fn call(
+ &mut self,
+ request: &Request,
+ pcm: Option<&[f32]>,
+ deadline: Duration,
+ ) -> Result<Response, Failure> {
+ if !self.control.send(encode_request(request, pcm)?) {
+ return Err(self.died());
+ }
+ self.wait(request, deadline)
+ }
+
+ /// Wait for the response to `request`, already sent.
+ fn wait(&mut self, request: &Request, deadline: Duration) -> Result<Response, Failure> {
+ match self.responses.recv_timeout(deadline) {
+ Ok(Response::Error(message)) => Err(Failure::Remote(message)),
+ Ok(response) => Ok(response),
+ Err(mpsc::RecvTimeoutError::Timeout) => {
+ self.control.kill();
+ let tail = self.native_tail();
+ error!(
+ "Transcription worker (pid {}) did not answer {} within {:?}; killed it{}",
+ self.control.pid,
+ request_name(request),
+ deadline,
+ if tail.is_empty() {
+ String::new()
+ } else {
+ format!(". Native output before the hang:\n{}", tail.join("\n"))
+ }
+ );
+ Err(Failure::Hung(deadline))
+ }
+ Err(mpsc::RecvTimeoutError::Disconnected) => Err(self.died()),
+ }
+ }
+
+ /// The worker's stdout closed: collect its exit status and last output.
+ fn died(&mut self) -> Failure {
+ // Let the stderr thread catch the final lines (the abort message).
+ let _ = self.stderr_done.recv_timeout(Duration::from_secs(1));
+ let status = {
+ let mut child = lock(&self.control.child);
+ match wait_exit(&mut child, Duration::from_secs(1)) {
+ Some(status) => status.to_string(),
+ None => {
+ let _ = child.kill();
+ "unresponsive after closing its output; killed".to_string()
+ }
+ }
+ };
+ let tail = self.native_tail();
+ if tail.is_empty() {
+ error!(
+ "Transcription worker (pid {}) died: {}",
+ self.control.pid, status
+ );
+ } else {
+ error!(
+ "Transcription worker (pid {}) died: {}. Native output before exit:\n{}",
+ self.control.pid,
+ status,
+ tail.join("\n")
+ );
+ }
+ Failure::Crashed(match tail.last() {
+ Some(last) => format!("{status}: {last}"),
+ None => status,
+ })
+ }
+
+ /// Recent raw native output (worker log lines are excluded; they were
+ /// already re-logged).
+ fn native_tail(&self) -> Vec<String> {
+ lock(&self.stderr_tail)
+ .iter()
+ .filter(|line| !line.starts_with(LOG_LINE_PREFIX))
+ .cloned()
+ .collect()
+ }
+}
+
+impl Drop for Worker {
+ fn drop(&mut self) {
+ self.control.close_stdin();
+ let mut child = lock(&self.control.child);
+ if wait_exit(&mut child, SHUTDOWN_GRACE).is_none() {
+ warn!(
+ "Transcription worker (pid {}) did not exit within {:?}; killing it",
+ self.control.pid, SHUTDOWN_GRACE
+ );
+ let _ = child.kill();
+ let _ = child.wait();
+ }
+ #[cfg(test)]
+ LIVE_WORKERS.fetch_sub(1, Ordering::SeqCst);
+ }
+}
+
+/// Wait up to `timeout` for `child` to exit. `None` if it is still running
+/// (or its status can't be read).
+fn wait_exit(child: &mut Child, timeout: Duration) -> Option<ExitStatus> {
+ let deadline = Instant::now() + timeout;
+ loop {
+ match child.try_wait() {
+ Ok(Some(status)) => return Some(status),
+ Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(5)),
+ _ => return None,
+ }
+ }
+}
+
+/// Worker processes started and not yet reaped. Every worker is reaped when
+/// its handle drops, so this is exactly the worker processes still alive.
+#[cfg(test)]
+static LIVE_WORKERS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
+
+/// The file Handy was started from, with its identity (device and inode),
+/// recorded at startup; see [`worker_exe`]. `None` if either couldn't be
+/// read, which skips the check.
+#[cfg(all(unix, not(test)))]
+static LAUNCHED_EXE: std::sync::OnceLock<Option<(PathBuf, (u64, u64))>> =
+ std::sync::OnceLock::new();
+
+#[cfg(all(unix, not(test)))]
+fn file_id(path: &std::path::Path) -> io::Result<(u64, u64)> {
+ use std::os::unix::fs::MetadataExt;
+ let metadata = std::fs::metadata(path)?;
+ Ok((metadata.dev(), metadata.ino()))
+}
+
+/// Must run before an upgrade can replace the file, so at startup.
+#[cfg(all(unix, not(test)))]
+fn record_launched_exe() {
+ LAUNCHED_EXE.get_or_init(|| {
+ std::env::current_exe()
+ .and_then(|path| {
+ let id = file_id(&path)?;
+ Ok((path, id))
+ })
+ .inspect_err(|e| warn!("Could not identify Handy's executable: {}", e))
+ .ok()
+ });
+}
+
+/// The executable a worker runs: always this very binary, so the two agree
+/// on the protocol and on transcribe.cpp's backend libraries. Upgrading or
+/// moving Handy while it runs replaces or removes the file it started from,
+/// and a worker started from there would be the new version (macOS), or this
+/// binary loading the new version's backend libraries (Linux). Only a
+/// restart fixes that, so refuse to start one and say so. Windows locks a
+/// running executable against replacement. On Linux the worker runs from
+/// `/proc/self/exe`, which is this binary even once its file is replaced.
+#[cfg(not(test))]
+fn worker_exe() -> io::Result<PathBuf> {
+ #[cfg(unix)]
+ if let Some((path, launched)) = LAUNCHED_EXE.get().and_then(Option::as_ref) {
+ if file_id(path).ok().as_ref() != Some(launched) {
+ return Err(io::Error::other(
+ "Handy was updated or moved while it was running; restart Handy",
+ ));
+ }
+ }
+ #[cfg(target_os = "linux")]
+ return Ok(PathBuf::from("/proc/self/exe"));
+ #[cfg(not(target_os = "linux"))]
+ std::env::current_exe()
+}
+
+fn encode_request(request: &Request, pcm: Option<&[f32]>) -> Result<Vec<u8>, Failure> {
+ encode_message(request, pcm)
+ .map_err(|e| Failure::Remote(format!("failed to encode request: {e}")))
+}
+
+fn write_frames(mut stdin: ChildStdin, frames: mpsc::Receiver<Vec<u8>>) {
+ // Ends when the handle closes stdin (dropping it here sends EOF) or the
+ // worker is gone.
+ for frame in frames {
+ if stdin
+ .write_all(&frame)
+ .and_then(|()| stdin.flush())
+ .is_err()
+ {
+ break;
+ }
+ }
+}
+
+fn read_responses(stdout: ChildStdout, responses: mpsc::Sender<Response>) {
+ let mut stdout = BufReader::new(stdout);
+ loop {
+ match read_message::<Response>(&mut stdout) {
+ Ok(Some((response, _))) => {
+ if responses.send(response).is_err() {
+ break;
+ }
+ }
+ Ok(None) => break,
+ Err(e) => {
+ error!("Transcription worker protocol error: {}", e);
+ break;
+ }
+ }
+ }
+}
+
+fn request_name(request: &Request) -> &'static str {
+ match request {
+ Request::Hello { .. } => "hello",
+ Request::Load { .. } => "model load",
+ Request::Run { .. } => "transcription",
+ Request::StreamBegin { .. } => "stream begin",
+ Request::Feed => "stream feed",
+ Request::Finalize { .. } => "stream finalize",
+ Request::StreamReset => "stream reset",
+ }
+}
+
+/// Re-log a worker stderr line in the parent. Structured lines keep their
+/// level; anything else is raw native output.
+fn forward_worker_log(line: &str) {
+ const TARGET: &str = "transcribe_worker";
+ if let Some(rest) = line.strip_prefix(LOG_LINE_PREFIX) {
+ let mut parts = rest.splitn(3, '\t');
+ if let (Some(level), Some(target), Some(message)) =
+ (parts.next(), parts.next(), parts.next())
+ {
+ let level = level.parse::<Level>().unwrap_or(Level::Info);
+ log::log!(target: TARGET, level, "[{}] {}", target, message);
+ return;
+ }
+ }
+ log::info!(target: TARGET, "{}", line);
+}
+
+#[cfg(test)]
+mod unit_tests {
+ use super::*;
+
+ #[test]
+ fn deadlines_scale_with_the_audio_above_a_floor() {
+ let gpu = Deadlines::for_device(true);
+ assert_eq!(gpu, GPU_DEADLINES);
+ assert_eq!(gpu.audio(0), gpu.call_floor);
+ assert_eq!(gpu.audio(16_000 / 2), gpu.call_floor);
+ assert_eq!(gpu.audio(16_000 * 5), Duration::from_secs(50));
+ assert_eq!(gpu.run(16_000), gpu.run_floor);
+ assert_eq!(gpu.run(16_000 * 10), Duration::from_secs(100));
+
+ let cpu = Deadlines::for_device(false);
+ assert_eq!(cpu, CPU_DEADLINES);
+ assert_eq!(cpu.audio(16_000), cpu.call_floor);
+ assert_eq!(cpu.audio(16_000 * 10), Duration::from_secs(200));
+ assert_eq!(cpu.run(16_000 * 5), cpu.run_floor);
+ assert_eq!(cpu.run(16_000 * 60), Duration::from_secs(1200));
+ // CPU is never stricter than GPU.
+ for secs in [0, 1, 5, 30, 120, 600] {
+ assert!(cpu.audio(secs * 16_000) >= gpu.audio(secs * 16_000));
+ assert!(cpu.run(secs * 16_000) >= gpu.run(secs * 16_000));
+ }
+
+ assert_eq!(ms_to_samples(1500), 24_000);
+ assert_eq!(ms_to_samples(-5), 0);
+ }
+}
+
+/// End-to-end checks against a real worker and model. Crash and hang tests
+/// need a debug worker exe (fault injection is compiled out of release), and
+/// the stream tests a model that can stream.
+/// Run with:
+/// `HANDY_TRANSCRIBE_WORKER_EXE=target/debug/handy HANDY_TEST_MODEL=<gguf>
+/// HANDY_TEST_WAV=<16 kHz mono wav> cargo test --lib engine_supervisor --
+/// --ignored --nocapture --test-threads=1`
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn env_path(name: &str) -> PathBuf {
+ PathBuf::from(std::env::var_os(name).unwrap_or_else(|| panic!("set {name}")))
+ }
+
+ fn test_pcm() -> Vec<f32> {
+ crate::audio_toolkit::read_wav_samples(env_path("HANDY_TEST_WAV")).expect("read wav")
+ }
+
+ fn spec() -> LoadSpec {
+ LoadSpec {
+ path: env_path("HANDY_TEST_MODEL"),
+ backend: Backend::Auto,
+ device: DeviceSelector::Auto,
+ }
+ }
+
+ /// Faults are read by each worker at spawn, so set them before loading.
+ fn set_fault(fault: Option<&str>, gpu_only: bool) {
+ let marker = std::env::temp_dir().join("handy-worker-fault-once");
+ let _ = std::fs::remove_file(&marker);
+ std::env::remove_var("HANDY_WORKER_FAULT_ONCE");
+ std::env::remove_var("HANDY_WORKER_FAULT_GPU_ONLY");
+ match fault {
+ Some(fault) => {
+ std::env::set_var("HANDY_WORKER_FAULT", fault);
+ if gpu_only {
+ std::env::set_var("HANDY_WORKER_FAULT_GPU_ONLY", "1");
+ } else {
+ std::env::set_var("HANDY_WORKER_FAULT_ONCE", marker);
+ }
+ }
+ None => std::env::remove_var("HANDY_WORKER_FAULT"),
+ }
+ }
+
+ /// The workers alive right now.
+ fn worker_count() -> usize {
+ LIVE_WORKERS.load(Ordering::SeqCst)
+ }
+
+ /// Feed 30 ms frames a little faster than a microphone would.
+ fn feed_paced(stream: &StreamHandle, pcm: &[f32]) {
+ for frame in pcm.chunks(480) {
+ stream.feed(frame.to_vec());
+ thread::sleep(Duration::from_millis(2));
+ }
+ }
+
+ fn stream_all(
+ engine: &EngineSupervisor,
+ pcm: &[f32],
+ ) -> Result<Option<Finalized>, EngineError> {
+ let stream =
+ engine.start_stream(RunOptions::default(), StreamOptions::default(), |_| {})?;
+ feed_paced(&stream, pcm);
+ stream.finalize(false)
+ }
+
+ /// Block until the owner is waiting on `what` (a [`request_name`]) in
+ /// the worker, so a cancel lands on that work rather than before or
+ /// after it, however fast the model is.
+ fn wait_in_flight(engine: &EngineSupervisor, what: &str) {
+ let started = Instant::now();
+ while !lock(&engine.shared.in_flight)
+ .as_ref()
+ .is_some_and(|call| call.what == what)
+ {
+ assert!(
+ started.elapsed() < Duration::from_secs(60),
+ "{what} never started"
+ );
+ thread::sleep(Duration::from_millis(1));
+ }
+ }
+
+ /// The test clip repeated to at least `secs` seconds.
+ fn long_pcm(secs: usize) -> Vec<f32> {
+ let clip = test_pcm();
+ clip.repeat((secs * 16_000).div_ceil(clip.len()))
+ }
+
+ /// Never two workers: a reload or unload has fully stopped the old
+ /// worker before anything else happens.
+ #[test]
+ #[ignore]
+ fn one_worker_at_a_time() {
+ set_fault(None, false);
+ let engine = EngineSupervisor::new(true);
+ for _ in 0..3 {
+ engine.load(spec()).unwrap();
+ assert_eq!(worker_count(), 1);
+ }
+ engine.unload().wait();
+ assert_eq!(worker_count(), 0);
+ assert!(engine.loaded().is_none());
+ // A device probe with nothing loaded leaves no worker behind.
+ assert!(engine.devices().is_some());
+ assert_eq!(worker_count(), 0);
+ }
+
+ /// A worker crash mid-stream loses the stream; the batch fallback then
+ /// runs in a fresh worker.
+ #[test]
+ #[ignore]
+ fn crash_mid_stream_falls_back_to_batch() {
+ let pcm = test_pcm();
+ set_fault(Some("abort@feed"), false);
+ let engine = EngineSupervisor::new(true);
+ engine.load(spec()).unwrap();
+ assert!(stream_all(&engine, &pcm).unwrap().is_none());
+ let t = Instant::now();
+ let transcript = engine.transcribe(pcm, RunOptions::default()).unwrap();
+ println!(
+ "batch fallback after crash {:?} on '{}': {}",
+ t.elapsed(),
+ engine.loaded().unwrap().backend,
+ transcript.text
+ );
+ assert!(engine.loaded().is_some());
+ set_fault(None, false);
+ }
+
+ /// A hung finalize is killed at its deadline (sized by the buffered
+ /// audio, at least the device's call floor) and the batch fallback runs in a fresh
+ /// worker.
+ #[test]
+ #[ignore]
+ fn hung_finalize_is_killed_and_recovers() {
+ let pcm = test_pcm();
+ set_fault(Some("hang@finalize"), false);
+ let engine = EngineSupervisor::new(true);
+ engine.load(spec()).unwrap();
+ let t = Instant::now();
+ assert!(stream_all(&engine, &pcm).unwrap().is_none());
+ println!("stream + hung finalize gave up after {:?}", t.elapsed());
+ let t = Instant::now();
+ let transcript = engine.transcribe(pcm, RunOptions::default()).unwrap();
+ println!(
+ "batch fallback after hang {:?}: {}",
+ t.elapsed(),
+ transcript.text
+ );
+ set_fault(None, false);
+ }
+
+ /// Cancelling a stream finalize reports `Cancelled`, never "no result"
+ /// (which would start a batch run), and the next use works.
+ #[test]
+ #[ignore]
+ fn cancel_during_finalize_is_cancelled() {
+ let pcm = test_pcm();
+ set_fault(Some("hang@finalize"), false);
+ let engine = EngineSupervisor::new(true);
+ engine.load(spec()).unwrap();
+ let stream = engine
+ .start_stream(RunOptions::default(), StreamOptions::default(), |_| {})
+ .unwrap();
+ feed_paced(&stream, &pcm);
+ let finalize = thread::spawn(move || stream.finalize(false));
+ wait_in_flight(&engine, "stream finalize");
+ let t = Instant::now();
+ engine.cancel();
+ let result = finalize.join().unwrap();
+ println!("finalize cancelled {:?} after cancel()", t.elapsed());
+ assert!(matches!(result, Err(EngineError::Cancelled)));
+ assert!(t.elapsed() < Duration::from_secs(2));
+ assert!(!engine.shared.gpu_unavailable());
+ engine.transcribe(pcm, RunOptions::default()).unwrap();
+ set_fault(None, false);
+ }
+
+ /// Cancelling during a hung feed stops it at once (not at the feed
+ /// deadline), skips the queued frames, and the finalize reports
+ /// `Cancelled` even though the stream's worker is gone.
+ #[test]
+ #[ignore]
+ fn cancel_during_hung_feed_is_cancelled() {
+ let pcm = test_pcm();
+ set_fault(Some("hang@feed"), false);
+ let engine = EngineSupervisor::new(true);
+ engine.load(spec()).unwrap();
+ let stream = engine
+ .start_stream(RunOptions::default(), StreamOptions::default(), |_| {})
+ .unwrap();
+ feed_paced(&stream, &pcm);
+ wait_in_flight(&engine, "stream feed");
+ let t = Instant::now();
+ engine.cancel();
+ let result = stream.finalize(false);
+ println!("hung feed cancelled {:?} after cancel()", t.elapsed());
+ assert!(matches!(result, Err(EngineError::Cancelled)));
+ assert!(t.elapsed() < Duration::from_secs(2));
+ assert!(!engine.shared.gpu_unavailable());
+ engine.transcribe(pcm, RunOptions::default()).unwrap();
+ set_fault(None, false);
+ }
+
+ /// An unload reads as unloaded at once, even while it waits behind a
+ /// run, so a new dictation queues a fresh load instead of assuming the
+ /// model is there.
+ #[test]
+ #[ignore]
+ fn unload_reads_unloaded_at_once() {
+ set_fault(None, false);
+ let engine = EngineSupervisor::new(true);
+ engine.load(spec()).unwrap();
+ let run = {
+ let engine = engine.clone();
+ thread::spawn(move || engine.transcribe(long_pcm(60), RunOptions::default()))
+ };
+ wait_in_flight(&engine, "transcription");
+ let unload = {
+ let engine = engine.clone();
+ thread::spawn(move || engine.unload().wait())
+ };
+ let t = Instant::now();
+ while engine.loaded().is_some() {
+ assert!(t.elapsed() < Duration::from_secs(1), "still reads loaded");
+ thread::sleep(Duration::from_millis(1));
+ }
+ assert!(lock(&engine.shared.in_flight).is_some(), "run still going");
+ run.join().unwrap().unwrap();
+ unload.join().unwrap();
+ assert!(engine.loaded().is_none());
+ assert_eq!(worker_count(), 0);
+ }
+
+ /// An unload that is still waiting its turn can't affect a load
+ /// requested after it (a new recording right after an "Immediately"
+ /// unload): the newer model ends up loaded and usable.
+ #[test]
+ #[ignore]
+ fn queued_unload_never_affects_a_later_load() {
+ set_fault(None, false);
+ let engine = EngineSupervisor::new(true);
+ engine.load(spec()).unwrap();
+ let run = {
+ let engine = engine.clone();
+ thread::spawn(move || engine.transcribe(long_pcm(60), RunOptions::default()))
+ };
+ wait_in_flight(&engine, "transcription");
+ drop(engine.unload());
+ assert!(
+ engine.loaded().is_none(),
+ "a new recording must see no model"
+ );
+ engine.load(spec()).unwrap();
+ run.join().unwrap().unwrap();
+ assert!(engine.loaded().is_some());
+ assert_eq!(worker_count(), 1);
+ engine
+ .transcribe(test_pcm(), RunOptions::default())
+ .unwrap();
+ }
+
+ /// A crash that only happens on the GPU: the dictation succeeds on CPU,
+ /// the GPU is marked unavailable, and later loads use CPU.
+ #[test]
+ #[ignore]
+ fn gpu_only_crash_recovers_on_cpu() {
+ let pcm = test_pcm();
+ set_fault(Some("abort@run"), true);
+ let engine = EngineSupervisor::new(true);
+ let info = engine.load(spec()).unwrap();
+ assert!(info.on_gpu, "needs a GPU to test GPU-only faults");
+ let transcript = engine
+ .transcribe(pcm.clone(), RunOptions::default())
+ .unwrap();
+ println!("recovered on CPU: {}", transcript.text);
+ assert!(engine.shared.gpu_unavailable());
+ assert!(!engine.loaded().unwrap().on_gpu);
+ let info = engine.load(spec()).unwrap();
+ assert!(
+ !info.on_gpu,
+ "loads stay on CPU while the GPU is marked unavailable"
+ );
+ engine.transcribe(pcm, RunOptions::default()).unwrap();
+ set_fault(None, false);
+ }
+
+ /// A GPU driver that crashes while its backend registers, before the
+ /// hello: only a CPU-only worker survives it, so that is where the model
+ /// loads. The GPU is marked unavailable, later loads stay CPU-only and devices
+ /// aren't probed, until "try the GPU again" (a settings change).
+ #[test]
+ #[ignore]
+ fn gpu_backend_init_crash_uses_cpu_only_worker() {
+ let pcm = test_pcm();
+ set_fault(Some("abort@init"), true);
+ let engine = EngineSupervisor::new(true);
+ let info = engine.load(spec()).unwrap();
+ assert!(!info.on_gpu, "loaded on '{}'", info.backend);
+ assert!(engine.shared.gpu_unavailable());
+ assert!(engine.devices().is_none());
+ engine.transcribe(pcm, RunOptions::default()).unwrap();
+ // The fault still fires in any worker with GPU backends.
+ assert!(!engine.load(spec()).unwrap().on_gpu);
+ set_fault(None, false);
+ engine.retry_gpu("test");
+ assert!(engine.load(spec()).unwrap().on_gpu);
+ assert!(engine.devices().is_some());
+ }
+
+ /// With the CPU accelerator, the model worker is CPU-only (a GPU-only
+ /// init fault never fires), and a device probe runs beside it rather
+ /// than stopping it.
+ #[test]
+ #[ignore]
+ fn cpu_setting_uses_cpu_only_worker_and_probe_leaves_it_running() {
+ let pcm = test_pcm();
+ set_fault(Some("abort@init"), true);
+ let engine = EngineSupervisor::new(true);
+ let info = engine
+ .load(LoadSpec {
+ backend: Backend::Cpu,
+ ..spec()
+ })
+ .unwrap();
+ assert!(!info.on_gpu);
+ assert!(!engine.shared.gpu_unavailable());
+ set_fault(None, false);
+ assert!(engine.devices().is_some());
+ assert_eq!(worker_count(), 1);
+ assert!(engine.loaded().is_some());
+ engine.transcribe(pcm, RunOptions::default()).unwrap();
+ }
+
+ /// A host that must not use the GPU: even an `Auto` load and the device
+ /// probe run CPU-only, so a GPU-only init fault never fires.
+ #[test]
+ #[ignore]
+ fn gpu_disallowed_host_never_registers_a_gpu_backend() {
+ set_fault(Some("abort@init"), true);
+ let engine = EngineSupervisor::new(false);
+ let devices = engine.devices().expect("devices not listed");
+ assert!(devices.iter().all(|d| !d.is_gpu()), "{devices:?}");
+ assert!(!engine.load(spec()).unwrap().on_gpu);
+ assert!(!engine.shared.gpu_unavailable());
+ set_fault(None, false);
+ }
+
+ /// The mask itself: a CPU-only worker registers no GPU device, including
+ /// compiled-in ones such as Metal.
+ #[test]
+ #[ignore]
+ fn cpu_only_worker_lists_no_gpu() {
+ set_fault(None, false);
+ let mut worker = Worker::spawn(true).unwrap();
+ let Ok(Response::Hello {
+ devices: Some(devices),
+ }) = worker.call(&Request::Hello { list_devices: true }, None, HELLO_TIMEOUT)
+ else {
+ panic!("no device list");
+ };
+ println!("CPU-only worker devices: {devices:?}");
+ assert!(!devices.is_empty());
+ assert!(devices.iter().all(|d| !d.is_gpu()));
+ }
+
+ /// An explicit device never falls back to CPU.
+ #[test]
+ #[ignore]
+ fn pinned_device_fails_loudly() {
+ let pcm = test_pcm();
+ set_fault(Some("abort@run"), true);
+ let engine = EngineSupervisor::new(true);
+ let devices = engine.devices().expect("devices not listed");
+ let gpu = devices.iter().find(|d| d.is_gpu()).expect("needs a GPU");
+ let info = engine
+ .load(LoadSpec {
+ device: DeviceSelector::Index(gpu.index.unwrap()),
+ ..spec()
+ })
+ .unwrap();
+ assert!(info.on_gpu);
+ assert!(engine.transcribe(pcm, RunOptions::default()).is_err());
+ assert!(!engine.shared.gpu_unavailable());
+ set_fault(None, false);
+ }
+
+ /// Cancel mid-run: the run stops promptly, without a retry, the model
+ /// still reads as loaded, and the next run works in a fresh worker.
+ #[test]
+ #[ignore]
+ fn cancel_stops_a_run() {
+ let pcm = long_pcm(60);
+ set_fault(None, false);
+ let engine = EngineSupervisor::new(true);
+ engine.load(spec()).unwrap();
+ let run = {
+ let engine = engine.clone();
+ let pcm = pcm.clone();
+ thread::spawn(move || engine.transcribe(pcm, RunOptions::default()))
+ };
+ wait_in_flight(&engine, "transcription");
+ let t = Instant::now();
+ engine.cancel();
+ let result = run.join().unwrap();
+ println!("cancelled {:?} after cancel()", t.elapsed());
+ assert!(matches!(result, Err(EngineError::Cancelled)));
+ assert!(t.elapsed() < Duration::from_secs(2));
+ assert!(engine.loaded().is_some());
+ assert!(!engine.shared.gpu_unavailable());
+ let t = Instant::now();
+ engine
+ .transcribe(test_pcm(), RunOptions::default())
+ .unwrap();
+ println!("next run took {:?}", t.elapsed());
+ }
+
+ /// Cancel a hung run: killed at once, no retry, no GPU
+ /// flag, and the next run starts a fresh worker.
+ #[test]
+ #[ignore]
+ fn cancel_kills_a_hung_run() {
+ let pcm = test_pcm();
+ set_fault(Some("hang@run"), false);
+ let engine = EngineSupervisor::new(true);
+ engine.load(spec()).unwrap();
+ let run = {
+ let engine = engine.clone();
+ let pcm = pcm.clone();
+ thread::spawn(move || engine.transcribe(pcm, RunOptions::default()))
+ };
+ wait_in_flight(&engine, "transcription");
+ // Cancel only once the worker is actually hung, not on its way there.
+ let marker = std::env::temp_dir().join("handy-worker-fault-once");
+ while !marker.exists() {
+ thread::sleep(Duration::from_millis(1));
+ }
+ let t = Instant::now();
+ engine.cancel();
+ let result = run.join().unwrap();
+ println!("hung run cancelled {:?} after cancel()", t.elapsed());
+ assert!(matches!(result, Err(EngineError::Cancelled)));
+ assert!(t.elapsed() < Duration::from_secs(2));
+ assert!(!engine.shared.gpu_unavailable());
+ engine.transcribe(pcm, RunOptions::default()).unwrap();
+ set_fault(None, false);
+ }
+
+ /// A worker hung in native code still exits when its parent goes away
+ /// (stdin closes), without being killed.
+ #[test]
+ #[ignore]
+ fn hung_worker_exits_when_stdin_closes() {
+ set_fault(Some("hang@run"), false);
+ let mut worker = Worker::spawn(false).unwrap();
+ worker
+ .call(
+ &Request::Hello {
+ list_devices: false,
+ },
+ None,
+ HELLO_TIMEOUT,
+ )
+ .unwrap();
+ let spec = spec();
+ let load = Request::Load {
+ path: spec.path,
+ backend: spec.backend,
+ device: spec.device,
+ };
+ worker.call(&load, None, LOAD_TIMEOUT).unwrap();
+ let run = Request::Run {
+ options: RunOptions::default(),
+ };
+ assert!(worker
+ .control
+ .send(encode_message(&run, Some(&test_pcm())).unwrap()));
+ thread::sleep(Duration::from_secs(1));
+ worker.control.close_stdin();
+ let t = Instant::now();
+ let status = loop {
+ if let Some(status) = lock(&worker.control.child).try_wait().unwrap() {
+ break status;
+ }
+ assert!(t.elapsed() < SHUTDOWN_GRACE, "worker outlived its stdin");
+ thread::sleep(Duration::from_millis(10));
+ };
+ println!(
+ "hung worker exited {:?} after stdin closed: {status}",
+ t.elapsed()
+ );
+ assert!(status.success());
+ set_fault(None, false);
+ }
+}
diff --git a/src-tauri/src/engine_supervisor/worker.rs b/src-tauri/src/engine_supervisor/worker.rs
new file mode 100644
index 00000000..1dd69c04
--- /dev/null
+++ b/src-tauri/src/engine_supervisor/worker.rs
@@ -0,0 +1,475 @@
+//! The worker process: the only place transcribe.cpp native code runs. Owns
+//! the model, session and stream, and serves [`Request`]s until the parent
+//! closes its stdin.
+//!
+//! A dedicated thread reads stdin so the parent's writes never block, and
+//! the worker exits the moment the parent goes away, even if hung.
+
+use super::protocol::{
+ read_message, write_message, DeviceInfo, DeviceSelector, LoadedInfo, Request, Response,
+};
+use super::{CPU_ONLY_FLAG, LOG_LEVEL_ENV};
+use log::{debug, error, warn, LevelFilter, Log, Metadata, Record};
+use std::fs::File;
+use std::io::{self, BufReader, Write};
+use std::path::Path;
+use std::sync::mpsc;
+use std::thread;
+use transcribe_cpp::{
+ Backend, BackendMask, DeviceType, Feature, Model, ModelOptions, Session, Stream,
+};
+
+/// Prefix on worker log lines so the parent can tell them apart from raw
+/// native output (e.g. a `GGML_ASSERT` message right before an abort).
+pub(super) const LOG_LINE_PREFIX: &str = "\u{1}";
+
+/// A request as handed from the stdin reader to the main loop.
+struct Incoming {
+ request: Request,
+ pcm: Vec<f32>,
+}
+
+pub fn run() -> i32 {
+ // The worker owns the per-run engine scratch, so it needs the same glibc
+ // tuning as the app (#1792). Before anything allocates.
+ crate::memory::init_allocator();
+ #[cfg(unix)]
+ ignore_app_signals();
+ #[cfg(target_os = "linux")]
+ set_process_name();
+ // Take the real stdout for the protocol before any native code runs, and
+ // point fd 1 at stderr: ggml writes to stdout in places, and any stray
+ // byte there would corrupt the protocol stream.
+ let protocol_out = match take_stdout_for_protocol() {
+ Ok(file) => file,
+ Err(e) => {
+ eprintln!("transcribe worker: failed to claim stdout: {e}");
+ return 2;
+ }
+ };
+ init_logger();
+
+ let (requests_tx, requests) = mpsc::channel();
+ if let Err(e) = thread::Builder::new()
+ .name("transcribe-worker-in".into())
+ .spawn(move || read_requests(requests_tx))
+ {
+ error!("Failed to start the request reader: {}", e);
+ return 2;
+ }
+
+ transcribe_cpp::init_logging();
+ // Registering a GPU backend runs its driver code, so this is where a
+ // broken driver crashes or hangs, before the hello is answered.
+ let cpu_only = std::env::args_os().any(|arg| arg == CPU_ONLY_FLAG);
+ inject_fault("init", !cpu_only);
+ let init = if cpu_only {
+ transcribe_cpp::init_backends_with(None::<&Path>, BackendMask::CPU)
+ } else {
+ transcribe_cpp::init_backends_default()
+ };
+ // Init only fails when no compute device registered at all, which no
+ // other worker could fix either: the hello reports it.
+ let init_error = init
+ .err()
+ .map(|e| format!("Failed to initialize transcribe.cpp backends: {e}"));
+ debug!(
+ "transcribe.cpp allowed backends: {:#x}",
+ transcribe_cpp::allowed_backends().bits()
+ );
+
+ let mut output = protocol_out;
+ let mut session: Option<(Session, LoadedInfo)> = None;
+
+ while let Ok(Incoming { request, pcm }) = requests.recv() {
+ let finished_run = matches!(request, Request::Run { .. });
+ let response = match request {
+ Request::Hello { list_devices } => match &init_error {
+ Some(message) => Response::Error(message.clone()),
+ None => Response::Hello {
+ devices: list_devices.then(|| {
+ inject_fault("list", !cpu_only);
+ transcribe_cpp::devices()
+ .iter()
+ .map(DeviceInfo::from_device)
+ .collect()
+ }),
+ },
+ },
+ Request::Load {
+ path,
+ backend,
+ device,
+ } => {
+ // Free any previous model first to avoid holding two at once.
+ session = None;
+ match load(&path, backend, device) {
+ Ok(loaded) => {
+ let info = loaded.1.clone();
+ session = Some(loaded);
+ Response::Loaded(info)
+ }
+ Err(e) => Response::Error(e),
+ }
+ }
+ Request::Run { options } => match session.as_mut() {
+ Some((session, info)) => {
+ inject_fault("run", info.on_gpu);
+ match session.run(&pcm, &options) {
+ Ok(transcript) => Response::Transcript(transcript),
+ Err(e) => Response::Error(e.to_string()),
+ }
+ }
+ None => not_loaded(),
+ },
+ Request::StreamBegin { run, stream } => match session.as_mut() {
+ Some((session, info)) => match session.stream(&run, &stream) {
+ Ok(stream) => {
+ if write_message(&mut output, &Response::Ok, None).is_err() {
+ return 1;
+ }
+ let on_gpu = info.on_gpu;
+ if let Err(e) = serve_stream(stream, &requests, &mut output, on_gpu) {
+ error!("Protocol write failed during stream: {}", e);
+ return 1;
+ }
+ crate::memory::trim_freed_memory();
+ continue;
+ }
+ Err(e) => Response::Error(e.to_string()),
+ },
+ None => not_loaded(),
+ },
+ Request::Feed | Request::Finalize { .. } | Request::StreamReset => {
+ Response::Error("no active stream".to_string())
+ }
+ };
+ if let Err(e) = write_message(&mut output, &response, None) {
+ error!("Protocol write failed: {}", e);
+ return 1;
+ }
+ // After the reply, so it never delays the transcript.
+ if finished_run {
+ drop(pcm);
+ crate::memory::trim_freed_memory();
+ }
+ }
+ // Unreachable in practice: the reader exits the process on EOF.
+ 0
+}
+
+/// The stdin reader. Forwards requests to the main loop, and ends the
+/// process when the parent closes stdin (unload, quit, or the parent died).
+/// Cancels never reach the worker: the parent kills it instead. `_exit`
+/// skips C++ static destructors, so a model still alive at that point can't
+/// trip ggml-metal's teardown asserts, and the OS reclaims its CPU and GPU
+/// memory.
+fn read_requests(requests: mpsc::Sender<Incoming>) -> ! {
+ let mut input = BufReader::new(io::stdin().lock());
+ loop {
+ match read_message::<Request>(&mut input) {
+ Ok(Some((request, pcm))) => {
+ if requests.send(Incoming { request, pcm }).is_err() {
+ exit_now(1);
+ }
+ }
+ Ok(None) => exit_now(0),
+ Err(e) => {
+ error!("Protocol read failed: {}", e);
+ exit_now(1);
+ }
+ }
+ }
+}
+
+fn exit_now(code: i32) -> ! {
+ let _ = io::stderr().flush();
+ // SAFETY: `_exit` only ends the process; nothing runs afterwards.
+ unsafe { libc::_exit(code) }
+}
+
+/// Serve requests against an active stream until it is finalized or reset.
+fn serve_stream(
+ mut stream: Stream<'_>,
+ requests: &mpsc::Receiver<Incoming>,
+ output: &mut impl Write,
+ on_gpu: bool,
+) -> io::Result<()> {
+ while let Ok(Incoming { request, pcm }) = requests.recv() {
+ let (response, done) = match request {
+ Request::Feed => {
+ inject_fault("feed", on_gpu);
+ match stream.feed(&pcm) {
+ Ok(update) => {
+ let text = (update.committed_changed || update.tentative_changed)
+ .then(|| stream.text());
+ (Response::Fed { update, text }, false)
+ }
+ Err(e) => (Response::Error(e.to_string()), false),
+ }
+ }
+ Request::Finalize { want_language } => {
+ inject_fault("finalize", on_gpu);
+ match stream.finalize() {
+ Ok(update) => {
+ let language = if want_language {
+ stream.snapshot().language
+ } else {
+ None
+ };
+ let text = stream.text();
+ (
+ Response::Finalized {
+ update,
+ text,
+ language,
+ },
+ true,
+ )
+ }
+ // Finalize ends the stream even when it fails.
+ Err(e) => (Response::Error(e.to_string()), true),
+ }
+ }
+ Request::StreamReset => {
+ stream.reset();
+ (Response::Ok, true)
+ }
+ _ => (
+ Response::Error("a stream is active; finalize or reset it first".to_string()),
+ false,
+ ),
+ };
+ write_message(output, &response, None)?;
+ if done {
+ break;
+ }
+ }
+ Ok(())
+}
+
+fn not_loaded() -> Response {
+ Response::Error("no model loaded".to_string())
+}
+
+fn load(
+ path: &Path,
+ backend: Backend,
+ selector: DeviceSelector,
+) -> Result<(Session, LoadedInfo), String> {
+ let device = match selector {
+ DeviceSelector::Auto => None,
+ DeviceSelector::Key(key) => {
+ let found = transcribe_cpp::devices().into_iter().find(|d| {
+ let info = DeviceInfo::from_device(d);
+ info.is_gpu() && info.key == key
+ });
+ if found.is_none() {
+ warn!(
+ "Stored transcribe GPU device '{}' is no longer available; using automatic device selection",
+ key
+ );
+ }
+ found
+ }
+ DeviceSelector::Index(index) => {
+ let device = transcribe_cpp::devices()
+ .into_iter()
+ .find(|d| d.index == Some(index))
+ .ok_or_else(|| {
+ format!("No compute device with index {index} (see --list-devices)")
+ })?;
+ if matches!(device.device_type, DeviceType::Accel | DeviceType::Unknown) {
+ return Err(format!(
+ "Device index {index} ({}) cannot host a model",
+ device.kind
+ ));
+ }
+ Some(device)
+ }
+ };
+
+ inject_fault("load", backend != Backend::Cpu);
+ let model = Model::load_with(path, &ModelOptions { backend, device })
+ .map_err(|e| format!("Failed to load model: {e}"))?;
+ let session = model
+ .session()
+ .map_err(|e| format!("Failed to create session: {e}"))?;
+ let bound = model.device().ok().map(|d| DeviceInfo::from_device(&d));
+ let backend_name = model.backend();
+ let info = LoadedInfo {
+ arch: model.arch(),
+ variant: model.variant(),
+ on_gpu: match &bound {
+ Some(device) => device.is_gpu(),
+ None => !backend_name.is_empty() && !backend_name.eq_ignore_ascii_case("cpu"),
+ },
+ backend: backend_name,
+ device: bound
+ .as_ref()
+ .map(|d| d.label().to_string())
+ .unwrap_or_else(|| "unknown".to_string()),
+ capabilities: model.capabilities(),
+ supports_initial_prompt: model.supports(Feature::InitialPrompt),
+ };
+ Ok((session, info))
+}
+
+/// Debug-build fault injection for exercising crash/hang recovery:
+/// `HANDY_WORKER_FAULT=<abort|segv|hang>@<init|list|load|run|feed|finalize>`
+/// (`abort` is what `GGML_ASSERT` ends in, `segv` a real invalid memory write)
+/// (`init` is backend registration, before the hello). With `HANDY_WORKER_FAULT_ONCE=<marker
+/// path>` it fires only in the first worker to reach that stage (the marker
+/// file records that it fired). With `HANDY_WORKER_FAULT_GPU_ONLY=1` it fires
+/// only on a GPU (backend init and device listing count as GPU work unless
+/// the worker is CPU-only), so recovery on CPU can succeed.
+#[cfg(debug_assertions)]
+fn inject_fault(stage: &str, on_gpu: bool) {
+ let Ok(spec) = std::env::var("HANDY_WORKER_FAULT") else {
+ return;
+ };
+ let Some((kind, at)) = spec.split_once('@') else {
+ return;
+ };
+ if at != stage {
+ return;
+ }
+ if !on_gpu && std::env::var_os("HANDY_WORKER_FAULT_GPU_ONLY").is_some() {
+ return;
+ }
+ if let Some(marker) = std::env::var_os("HANDY_WORKER_FAULT_ONCE") {
+ if std::fs::OpenOptions::new()
+ .write(true)
+ .create_new(true)
+ .open(&marker)
+ .is_err()
+ {
+ return;
+ }
+ }
+ eprintln!("injected fault: {kind} at {stage}");
+ match kind {
+ "abort" => std::process::abort(),
+ // SAFETY: deliberately crash the worker with a real invalid memory
+ // write, as a faulting driver does: SIGSEGV on Unix, an access
+ // violation on Windows (where raising SIGSEGV only exits with code
+ // 3). On Unix, restore the default disposition first so Rust's
+ // stack-overflow handler doesn't get involved.
+ "segv" => unsafe {
+ #[cfg(unix)]
+ libc::signal(libc::SIGSEGV, libc::SIG_DFL);
+ std::ptr::write_volatile(std::ptr::null_mut::<u32>(), 1);
+ },
+ "hang" => loop {
+ std::thread::sleep(std::time::Duration::from_secs(3600));
+ },
+ _ => {}
+ }
+}
+
+#[cfg(not(debug_assertions))]
+fn inject_fault(_stage: &str, _on_gpu: bool) {}
+
+/// Logs go to stderr as `\x01LEVEL\ttarget\tmessage` lines; the parent
+/// re-logs them at the same level.
+struct StderrLogger;
+
+impl Log for StderrLogger {
+ fn enabled(&self, metadata: &Metadata) -> bool {
+ metadata.level() <= log::max_level()
+ }
+
+ fn log(&self, record: &Record) {
+ if !self.enabled(record.metadata()) {
+ return;
+ }
+ let message = record.args().to_string();
+ let mut stderr = io::stderr().lock();
+ for line in message.lines() {
+ let _ = writeln!(
+ stderr,
+ "{LOG_LINE_PREFIX}{}\t{}\t{}",
+ record.level(),
+ record.target(),
+ line
+ );
+ }
+ }
+
+ fn flush(&self) {
+ let _ = io::stderr().flush();
+ }
+}
+
+fn init_logger() {
+ static LOGGER: StderrLogger = StderrLogger;
+ let level = std::env::var(LOG_LEVEL_ENV)
+ .ok()
+ .and_then(|v| v.parse::<LevelFilter>().ok())
+ .unwrap_or(LevelFilter::Info);
+ if log::set_logger(&LOGGER).is_ok() {
+ log::set_max_level(level);
+ }
+}
+
+/// SIGUSR2 toggles transcription in the app (and WebKitGTK uses SIGUSR1).
+/// Aimed at Handy by name, e.g. the README's `pkill -USR2 -n handy`, one
+/// can reach this worker instead, and its default action would kill it.
+#[cfg(unix)]
+fn ignore_app_signals() {
+ // SAFETY: setting a signal's disposition to SIG_IGN has no preconditions.
+ unsafe {
+ libc::signal(libc::SIGUSR1, libc::SIG_IGN);
+ libc::signal(libc::SIGUSR2, libc::SIG_IGN);
+ }
+}
+
+/// Started from /proc/self/exe, the kernel names this process "exe". Name it
+/// as Handy's, but not `handy`: `pkill`/`killall` match names case-sensitively,
+/// so `pkill -USR2 -n handy` keeps reaching the app, never this newer process.
+#[cfg(target_os = "linux")]
+fn set_process_name() {
+ // SAFETY: PR_SET_NAME copies a NUL-terminated name of up to 16 bytes.
+ unsafe {
+ libc::prctl(libc::PR_SET_NAME, c"Handy-worker".as_ptr());
+ }
+}
+
+#[cfg(unix)]
+fn take_stdout_for_protocol() -> io::Result<File> {
+ use std::os::fd::FromRawFd;
+ // SAFETY: plain fd duplication; the new fd is owned by the returned File.
+ unsafe {
+ let fd = libc::dup(libc::STDOUT_FILENO);
+ if fd < 0 {
+ return Err(io::Error::last_os_error());
+ }
+ if libc::dup2(libc::STDERR_FILENO, libc::STDOUT_FILENO) < 0 {
+ return Err(io::Error::last_os_error());
+ }
+ Ok(File::from_raw_fd(fd))
+ }
+}
+
+#[cfg(windows)]
+fn take_stdout_for_protocol() -> io::Result<File> {
+ use std::os::windows::io::FromRawHandle;
+ // Native code writes through the CRT's fd 1, so redirect that one. The
+ // protocol uses the OS handle behind a CRT duplicate of the original.
+ // SAFETY: CRT fd duplication; the duplicate's handle is owned by the File
+ // for the rest of the process.
+ unsafe {
+ let fd = libc::dup(1);
+ if fd < 0 {
+ return Err(io::Error::last_os_error());
+ }
+ if libc::dup2(2, 1) < 0 {
+ return Err(io::Error::last_os_error());
+ }
+ let handle = libc::get_osfhandle(fd);
+ if handle == -1 {
+ return Err(io::Error::last_os_error());
+ }
+ Ok(File::from_raw_handle(handle as _))
+ }
+}
diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs
index 898770bc..a41d62c5 100644
--- a/src-tauri/src/lib.rs
+++ b/src-tauri/src/lib.rs
@@ -9,6 +9,7 @@ mod chinese_script;
pub mod cli;
mod clipboard;
mod commands;
+pub mod engine_supervisor;
mod helpers;
mod input;
mod llm_client;
@@ -206,10 +207,6 @@ fn initialize_core_logic(app_handle: &AppHandle) {
let history_manager =
Arc::new(HistoryManager::new(app_handle).expect("Failed to initialize history manager"));
- // Initialize the transcribe-cpp native backend (logging + backend module
- // registration) once, before any whisper model is loaded.
- managers::transcription::init_transcribe_backend();
-
// Apply accelerator preferences before any model loads
managers::transcription::apply_accelerator_settings(app_handle);
@@ -309,10 +306,8 @@ fn initialize_core_logic(app_handle: &AppHandle) {
log::warn!("No model is currently loaded.");
return;
}
- match transcription_manager.unload_model() {
- Ok(()) => log::info!("Model unloaded via tray."),
- Err(e) => log::error!("Failed to unload model via tray: {}", e),
- }
+ transcription_manager.request_unload();
+ log::info!("Model unloaded via tray.");
}
"cancel" => {
use crate::utils::cancel_current_operation;
@@ -438,7 +433,14 @@ fn run_headless_transcription(app: &AppHandle, args: &CliArgs) -> i32 {
// --list-devices: print registered compute devices (with indices) and exit.
// Useful on multi-GPU machines to discover the index for --device-index.
if args.list_devices {
- let devices = crate::managers::transcription::describe_compute_devices();
+ let tm = app.state::<Arc<TranscriptionManager>>();
+ let devices = match crate::managers::transcription::describe_compute_devices(&tm) {
+ Ok(devices) => devices,
+ Err(e) => {
+ eprintln!("error: {}", e);
+ return 1;
+ }
+ };
if devices.is_empty() {
println!("No transcribe-cpp compute devices registered.");
} else {
@@ -914,19 +916,17 @@ pub fn run(cli_args: CliArgs) {
TranscriptionManager::new(&app_handle, model_manager.clone())
.expect("Failed to initialize transcription manager"),
);
+ managers::transcription::report_compute_devices(&transcription_manager);
app_handle.manage(model_manager);
app_handle.manage(transcription_manager);
- managers::transcription::init_transcribe_backend();
- managers::transcription::report_compute_devices();
managers::transcription::apply_accelerator_settings(&app_handle);
let handle = app_handle.clone();
let args = cli_args.clone();
std::thread::spawn(move || {
let code = run_headless_guarded(|| run_headless_transcription(&handle, &args));
- // Drop the loaded engine before teardown: ggml-metal's global
- // device free asserts (SIGABRT) if a model's Metal resources
- // are still alive at C++ static-destructor time.
+ // Drop the loaded engine before exit so its transcription
+ // worker is told to shut down cleanly.
if let Some(tm) = handle.try_state::<Arc<TranscriptionManager>>() {
let _ = tm.unload_model();
}
@@ -1031,10 +1031,14 @@ pub fn run(cli_args: CliArgs) {
// stays off the startup path. get_available_accelerators then enumerates
// ORT execution providers and transcribe-cpp compute devices; without this
// the cost is paid synchronously when the user first opens Advanced
- // settings, freezing the UI. Result is cached in a OnceLock.
- std::thread::spawn(|| {
- crate::managers::transcription::report_compute_devices();
- let _ = crate::managers::transcription::get_available_accelerators();
+ // settings, freezing the UI. The device list is kept by the
+ // transcription engine and refreshed by every worker it starts that may
+ // use the GPU (CPU-only workers can't see it).
+ let devices_app_handle = app_handle.clone();
+ std::thread::spawn(move || {
+ let tm = devices_app_handle.state::<Arc<TranscriptionManager>>();
+ crate::managers::transcription::report_compute_devices(&tm);
+ let _ = crate::managers::transcription::get_available_accelerators(&tm);
});
// Hide tray icon if --no-tray was passed
@@ -1095,6 +1099,8 @@ pub fn run(cli_args: CliArgs) {
#[cfg(target_os = "macos")]
apply_startup_activation_policy(&mut app, headless_mode);
+ // `app` is only used by the macOS arm.
+ #[cfg_attr(not(target_os = "macos"), allow(unused_variables))]
app.run(|app, event| match &event {
#[cfg(target_os = "macos")]
tauri::RunEvent::Reopen { .. } => {
@@ -1113,12 +1119,10 @@ pub fn run(cli_args: CliArgs) {
}
show_main_window(app);
}
- // Teardown transcribe.cpp before exit
- tauri::RunEvent::Exit => {
- if let Some(tm) = app.try_state::<Arc<TranscriptionManager>>() {
- let _ = tm.unload_model();
- }
- }
+ // No transcription teardown on exit: transcribe.cpp runs only in the
+ // worker process, which exits by itself as soon as this process's end
+ // of its stdin closes, even if it is hung. Waiting on an unload here
+ // could block quitting behind a hung model load.
_ => {}
});
}
diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs
index 5322eb5b..db502060 100644
--- a/src-tauri/src/main.rs
+++ b/src-tauri/src/main.rs
@@ -5,8 +5,6 @@ use clap::Parser;
use handy_app_lib::CliArgs;
fn main() {
- let cli_args = CliArgs::parse();
-
#[cfg(target_os = "linux")]
{
// DMABUF renderer causes crashes on various GPU/display server configurations
@@ -25,5 +23,13 @@ fn main() {
}
}
+ // Transcription worker: runs transcribe.cpp in isolation and never touches
+ // Tauri, the CLI, or single-instance handling. Checked after the env setup
+ // above so the worker inherits it (e.g. the Vulkan layer opt-out).
+ if handy_app_lib::engine_supervisor::is_worker_invocation() {
+ std::process::exit(handy_app_lib::engine_supervisor::run_worker());
+ }
+
+ let cli_args = CliArgs::parse();
handy_app_lib::run(cli_args)
}
diff --git a/src-tauri/src/managers/transcription.rs b/src-tauri/src/managers/transcription.rs
index 32de0c59..e3907236 100644
--- a/src-tauri/src/managers/transcription.rs
+++ b/src-tauri/src/managers/transcription.rs
@@ -3,6 +3,10 @@ use crate::audio_toolkit::{
remove_filler_words, OutputLanguageEvidence,
};
use crate::chinese_script::{convert_chinese_script, ChineseVariety};
+use crate::engine_supervisor::{
+ DeviceInfo, DeviceSelector, EngineError, EngineSupervisor, LoadSpec, LoadedInfo,
+ StreamProgress, Unloading,
+};
use crate::managers::audio::AudioRecordingManager;
use crate::managers::model::{EngineType, ModelManager};
use crate::settings::{
@@ -15,15 +19,12 @@ use serde::{Deserialize, Serialize};
use specta::Type;
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
-use std::sync::{mpsc, Arc, Condvar, Mutex, MutexGuard, OnceLock};
+use std::sync::{mpsc, Arc, Condvar, Mutex, MutexGuard};
use std::thread;
use std::time::{Duration, Instant, SystemTime};
use tauri::{AppHandle, Emitter, Manager};
use tauri_specta::Event;
-use transcribe_cpp::{
- Backend, Feature, Model, ModelOptions, RunExtension, RunOptions, Session, StreamOptions, Task,
- WhisperRunOptions,
-};
+use transcribe_cpp::{Backend, RunExtension, RunOptions, StreamOptions, Task, WhisperRunOptions};
use transcribe_rs::{
onnx::{
canary::CanaryModel,
@@ -38,7 +39,6 @@ use transcribe_rs::{
};
const STREAM_PERF_LOG_INTERVAL: Duration = Duration::from_secs(5);
-const STREAM_FINALIZE_REPLY_TIMEOUT: Duration = Duration::from_secs(30);
fn panic_payload_message(payload: &(dyn std::any::Any + Send)) -> String {
if let Some(message) = payload.downcast_ref::<&str>() {
@@ -101,12 +101,15 @@ pub struct StreamPhaseEvent {
/// is processed before finalize runs.
enum StreamCmd {
Feed(Vec<f32>),
- /// Flush the stream and reply with the final text, or `None` if no stream
- /// was ever active (caller should fall back to batch transcription).
- Finalize(mpsc::Sender<Option<FinalizedStreamText>>),
+ /// Flush the stream and reply with the final text, or `None` if no
+ /// usable stream exists (caller should fall back to batch transcription).
+ /// `Err(Cancelled)` if the finalize itself was cancelled.
+ Finalize(mpsc::Sender<StreamFinalizeReply>),
Cancel,
}
+type StreamFinalizeReply = Result<Option<FinalizedStreamText>, EngineError>;
+
struct FinalizedStreamText {
text: String,
output_language: OutputLanguageEvidence,
@@ -177,11 +180,10 @@ impl StreamRouter {
}
}
-enum LoadedEngine {
- /// Whisper-family models (whisper, breeze-asr, custom .bin/.gguf) via
- /// transcribe-cpp. Holds the live `Session`, which keeps its `Model` alive
- /// internally, so repeated dictation reuses the session without reloading.
- TranscribeCpp(Session),
+/// In-process ONNX engines. transcribe-cpp models (whisper family, parakeet,
+/// custom .bin/.gguf) live in the [`EngineSupervisor`]'s worker process
+/// instead, so a native crash or hang there cannot take the app down.
+enum OnnxEngine {
Parakeet(ParakeetModel),
Moonshine(MoonshineModel),
MoonshineStreaming(StreamingModel),
@@ -214,14 +216,13 @@ impl Drop for LoadingGuard {
}
}
-/// RAII guard that clears the streaming worker/lease flags on any worker exit -
-/// normal return, early return, or a panic in an engine call that unwinds the
-/// detached worker thread. Tokens prevent an older worker from clearing a newer
-/// worker's state if a start/finalize race ever slips through.
+/// RAII guard that clears the streaming worker flags on any worker exit -
+/// normal return, early return, or a panic that unwinds the detached worker
+/// thread. Tokens prevent an older worker from clearing a newer worker's
+/// state if a start/finalize race ever slips through.
struct StreamWorkerGuard {
worker_id: u64,
active_stream_worker: Arc<AtomicU64>,
- active_engine_lease: Arc<AtomicU64>,
stream_active: Arc<AtomicBool>,
}
@@ -230,12 +231,6 @@ impl Drop for StreamWorkerGuard {
if self.active_stream_worker.load(Ordering::Acquire) == self.worker_id {
self.stream_active.store(false, Ordering::Release);
}
- let _ = self.active_engine_lease.compare_exchange(
- self.worker_id,
- 0,
- Ordering::AcqRel,
- Ordering::Acquire,
- );
let _ = self.active_stream_worker.compare_exchange(
self.worker_id,
0,
@@ -247,7 +242,11 @@ impl Drop for StreamWorkerGuard {
#[derive(Clone)]
pub struct TranscriptionManager {
- engine: Arc<Mutex<Option<LoadedEngine>>>,
+ /// The transcribe-cpp engine: owns the worker process, its model, and
+ /// recovery from crashes and hangs.
+ engine: EngineSupervisor,
+ /// The loaded ONNX engine, if the current model is an ONNX one.
+ onnx: Arc<Mutex<Option<OnnxEngine>>>,
model_manager: Arc<ModelManager>,
app_handle: AppHandle,
current_model_id: Arc<Mutex<Option<String>>>,
@@ -261,12 +260,12 @@ pub struct TranscriptionManager {
/// [`StreamRouter`]. Shared with the audio recorder so per-frame feeds skip
/// Tauri state and the manager lock.
router: Arc<StreamRouter>,
- /// True only while a transcribe-cpp `Stream` is actually in flight (set by
- /// the worker once `stream()` succeeds). Used for overlay/UI decisions.
+ /// True only while a transcribe-cpp stream is actually in flight (set by
+ /// the worker once the stream begins). Used for overlay/UI decisions.
stream_active: Arc<AtomicBool>,
- /// Streaming uses four independent flags: router open = frames should route,
- /// worker active = no second worker may start, engine lease = engine is out
- /// of the mutex, stream active = UI should show a live session.
+ /// Streaming uses three independent flags: router open = frames should
+ /// route, worker active = no second worker may start, stream active = UI
+ /// should show a live session.
///
/// Monotonic id source for stream workers; zero means "no worker".
next_stream_worker_id: Arc<AtomicU64>,
@@ -274,16 +273,13 @@ pub struct TranscriptionManager {
/// yet. This prevents a second worker from starting after finalize/cancel
/// closes the router but before the first worker has fully exited.
active_stream_worker: Arc<AtomicU64>,
- /// Nonzero while the streaming worker has taken the engine out of `engine`.
- /// `is_model_loaded()` consults this so the model still reports "loaded"
- /// while the worker holds it.
- active_engine_lease: Arc<AtomicU64>,
}
impl TranscriptionManager {
pub fn new(app_handle: &AppHandle, model_manager: Arc<ModelManager>) -> Result<Self> {
let manager = Self {
- engine: Arc::new(Mutex::new(None)),
+ engine: EngineSupervisor::new(!transcribe_gpu_disabled_for_host()),
+ onnx: Arc::new(Mutex::new(None)),
model_manager,
app_handle: app_handle.clone(),
current_model_id: Arc::new(Mutex::new(None)),
@@ -297,7 +293,6 @@ impl TranscriptionManager {
stream_active: Arc::new(AtomicBool::new(false)),
next_stream_worker_id: Arc::new(AtomicU64::new(1)),
active_stream_worker: Arc::new(AtomicU64::new(0)),
- active_engine_lease: Arc::new(AtomicU64::new(0)),
};
// Start the idle watcher
@@ -374,18 +369,18 @@ impl TranscriptionManager {
Ok(manager)
}
- /// Lock the engine mutex, recovering from poison if a previous transcription panicked.
- fn lock_engine(&self) -> MutexGuard<'_, Option<LoadedEngine>> {
- self.engine.lock().unwrap_or_else(|poisoned| {
+ /// Lock the ONNX engine mutex, recovering from poison if a previous transcription panicked.
+ fn lock_onnx(&self) -> MutexGuard<'_, Option<OnnxEngine>> {
+ self.onnx.lock().unwrap_or_else(|poisoned| {
warn!("Engine mutex was poisoned by a previous panic, recovering");
poisoned.into_inner()
})
}
pub fn is_model_loaded(&self) -> bool {
- // The engine may be leased out to the streaming worker (taken out of
- // the mutex). It's still loaded, just in use, so report true.
- self.lock_engine().is_some() || self.active_engine_lease.load(Ordering::Acquire) != 0
+ // A transcribe-cpp model stays loaded while it is busy, so a batch
+ // run or stream in progress never reads as "unloaded".
+ self.engine.loaded().is_some() || self.lock_onnx().is_some()
}
/// Accelerator changes should not disturb the current transcription. Mark
@@ -395,6 +390,44 @@ impl TranscriptionManager {
self.reload_model_on_next_use.store(true, Ordering::Release);
}
+ /// An explicit transcribe.cpp accelerator or GPU device change is the
+ /// user's way of trying the GPU again: forget every GPU failure so far,
+ /// so the next load may use (and list) the GPU. The ONNX accelerator does
+ /// not affect this.
+ pub fn retry_transcribe_gpu(&self, reason: &str) {
+ self.engine.retry_gpu(reason);
+ }
+
+ /// The engine gives up on a model by itself when its worker can't be
+ /// restarted (e.g. the file is gone, or it crashes on CPU too). Keep the
+ /// loaded model id and the UI in step, so the next use loads the model
+ /// again and reports why it can't.
+ fn forget_model_if_engine_dropped(&self, model_id: &str, error: &str) {
+ if self.engine.loaded().is_some() {
+ return;
+ }
+ {
+ let mut current_model = self.current_model_id.lock().unwrap();
+ if current_model.as_deref() != Some(model_id) {
+ return;
+ }
+ *current_model = None;
+ }
+ warn!(
+ "Transcription model '{}' is no longer loaded: {}",
+ model_id, error
+ );
+ let _ = self.app_handle.emit(
+ "model-state-changed",
+ ModelStateEvent {
+ event_type: "unloaded".to_string(),
+ model_id: None,
+ model_name: None,
+ error: Some(error.to_string()),
+ },
+ );
+ }
+
/// Atomically check whether a model load is in progress and, if not, mark
/// one as starting. Returns a [`LoadingGuard`] whose [`Drop`] impl will
/// clear the flag and wake waiters. Returns `None` if a load is already in
@@ -411,15 +444,34 @@ impl TranscriptionManager {
})
}
+ /// Unload the model. Returns once the transcribe-cpp worker has exited,
+ /// which frees all of its memory (e.g. before its file is deleted).
pub fn unload_model(&self) -> Result<()> {
let unload_start = std::time::Instant::now();
debug!("Starting to unload model");
+ self.begin_unload().wait();
+ debug!(
+ "Model unloaded manually (took {}ms)",
+ unload_start.elapsed().as_millis()
+ );
+ Ok(())
+ }
- {
- let mut engine = self.lock_engine();
- // Dropping the engine frees all resources
- *engine = None;
- }
+ /// Unload the model without waiting for the transcribe-cpp worker to
+ /// exit, so it never blocks the caller (e.g. the event loop) behind a
+ /// transcription in progress. The model reads as unloaded at once.
+ pub fn request_unload(&self) {
+ drop(self.begin_unload());
+ }
+
+ /// Every state change of an unload, done now on the caller's thread: a
+ /// dictation started right after sees no model and loads it again, and
+ /// its load is queued behind this unload, so the unload can never affect
+ /// it. Only the worker's exit is left to wait for.
+ fn begin_unload(&self) -> Unloading {
+ let unloading = self.engine.unload();
+ // Dropping an ONNX engine frees its resources.
+ *self.lock_onnx() = None;
{
let mut current_model = self.current_model_id.lock().unwrap();
*current_model = None;
@@ -435,13 +487,7 @@ impl TranscriptionManager {
error: None,
},
);
-
- let unload_duration = unload_start.elapsed();
- debug!(
- "Model unloaded manually (took {}ms)",
- unload_duration.as_millis()
- );
- Ok(())
+ unloading
}
fn now_ms() -> u64 {
@@ -456,16 +502,16 @@ impl TranscriptionManager {
self.last_activity.store(Self::now_ms(), Ordering::Relaxed);
}
- /// Unloads the model immediately if the setting is enabled and the model is loaded
+ /// Unloads the model immediately if the setting is enabled and the model
+ /// is loaded. Never waits for the worker to exit, so it is safe on any
+ /// thread, including the event loop.
pub fn maybe_unload_immediately(&self, context: &str) {
let settings = get_settings(&self.app_handle);
if settings.model_unload_timeout == ModelUnloadTimeout::Immediately
&& self.is_model_loaded()
{
info!("Immediately unloading model after {}", context);
- if let Err(e) = self.unload_model() {
- warn!("Failed to immediately unload model: {}", e);
- }
+ self.request_unload();
}
}
@@ -474,7 +520,7 @@ impl TranscriptionManager {
}
/// Like [`load_model`](Self::load_model), but lets a caller hard-select the
- /// compute device for this one load by its `transcribe_cpp::devices()`
+ /// compute device for this one load by its transcribe-cpp device
/// registry index (the index shown by `--list-devices`). `None` keeps the
/// persisted accelerator setting (which may be Auto). Only affects
/// transcribe-cpp (whisper-family) models; the selection is not persisted.
@@ -541,13 +587,14 @@ impl TranscriptionManager {
.get_model_path(model_id)
.inspect_err(|error| emit_loading_failed(&error.to_string()))?;
- // Drop the current engine BEFORE building the new one so transcribe-cpp
- // frees the previous native context first — avoids holding two models at
- // once (peak memory on large GGUFs). Clear the id too: if the new load
- // fails, status should read "no loaded model", not the dropped engine.
- {
- let mut engine = self.lock_engine();
- *engine = None;
+ // Drop the current engine BEFORE building the new one so the previous
+ // model is freed first — avoids holding two models at once (peak memory
+ // on large GGUFs). A transcribe-cpp load replaces its worker the same
+ // way. Clear the id too: if the new load fails, status should read "no
+ // loaded model", not the dropped engine.
+ *self.lock_onnx() = None;
+ if !matches!(model_info.engine_type, EngineType::TranscribeCpp) {
+ self.engine.unload().wait();
}
{
let mut current_model = self.current_model_id.lock().unwrap();
@@ -556,61 +603,51 @@ impl TranscriptionManager {
// Create appropriate engine based on model type
- let loaded_engine = match model_info.engine_type {
+ let loaded_onnx = match model_info.engine_type {
EngineType::TranscribeCpp => {
- // The whisper backend is chosen at load time (transcribe-cpp has
- // no runtime global). With an explicit `device_index` (the
- // --device-index flag) hard-select that registered device;
- // otherwise re-read the persisted accelerator preference (so an
- // accelerator change marked for reload takes effect here).
+ // The backend is chosen at load time. With an explicit
+ // `device_index` (the --device-index flag) hard-select that
+ // registered device; otherwise re-read the persisted
+ // accelerator preference (so an accelerator change marked for
+ // reload takes effect here). The worker resolves the selector
+ // to a device handle, and rejects an index that isn't a
+ // loadable device (a host without GPU access lists none).
let (backend, device) = match device_index {
- Some(index) => resolve_device_index(index).inspect_err(|e| {
- emit_loading_failed(&e.to_string());
- })?,
+ Some(index) => (Backend::Auto, DeviceSelector::Index(index)),
None => {
let settings = get_settings(&self.app_handle);
let accelerator = settings.transcribe_accelerator;
- let device = resolve_gpu_device(
+ match resolve_gpu_device(
accelerator,
settings.transcribe_gpu_device.as_deref(),
- );
- // Backend::Auto accepts an exact GPU device. Without a
- // valid exact device, backend selection handles the
- // retired generic GPU state and host CPU guard.
- let backend = if device.is_some() {
- Backend::Auto
- } else {
- select_transcribe_backend(accelerator)
- };
- (backend, device)
+ ) {
+ // Backend::Auto accepts an exact GPU device.
+ Some(key) => (Backend::Auto, DeviceSelector::Key(key)),
+ // Without a valid exact device, backend selection
+ // handles the retired generic GPU state and host
+ // CPU guard.
+ None => (select_transcribe_backend(accelerator), DeviceSelector::Auto),
+ }
}
};
- let requested_device = device
- .as_ref()
- .map(transcribe_device_label)
- .unwrap_or_else(|| "automatic".to_string());
- let model_options = ModelOptions { backend, device };
- let model = Model::load_with(&model_path, &model_options).map_err(|e| {
- let error_msg = format!("Failed to load whisper model {}: {}", model_id, e);
- emit_loading_failed(&error_msg);
- anyhow::anyhow!(error_msg)
- })?;
- // The bound backend may differ from the request (e.g. CPU
- // fallback under Auto); log what actually loaded.
- let bound_backend = model.backend();
- let session = model.session().map_err(|e| {
- let error_msg = format!(
- "Failed to create session for whisper model {}: {}",
- model_id, e
- );
- emit_loading_failed(&error_msg);
- anyhow::anyhow!(error_msg)
- })?;
+ let requested_device = format!("{:?}", device);
+ let info = self
+ .engine
+ .load(LoadSpec {
+ path: model_path.clone(),
+ backend,
+ device,
+ })
+ .map_err(|e| {
+ let error_msg = format!("Failed to load model {}: {}", model_id, e);
+ emit_loading_failed(&error_msg);
+ anyhow::anyhow!(error_msg)
+ })?;
// Reconcile the registry's advertised capabilities with the
// loaded model's real ones (GGUF metadata) so badges/gating
// reflect runtime truth, not the pre-download probe. The
// load-completed event below triggers the frontend refresh.
- let caps = session.model().capabilities();
+ let caps = &info.capabilities;
self.model_manager.set_runtime_capabilities(
model_id,
caps.supports_streaming,
@@ -618,24 +655,22 @@ impl TranscriptionManager {
caps.supports_language_detect,
caps.languages.clone(),
);
- let bound_device = model
- .device()
- .map(|device| transcribe_device_label(&device))
- .unwrap_or_else(|_| "unknown".to_string());
+ // The bound backend may differ from the request (e.g. CPU
+ // fallback under Auto or after a GPU crash); log what loaded.
info!(
- "Loaded whisper model '{}' (requested {:?}, requested device '{}', \
+ "Loaded transcribe-cpp model '{}' (requested {:?}, requested device {}, \
bound backend '{}', bound device '{}', supports_streaming={}, \
supports_translate={}, supports_language_detect={})",
model_id,
backend,
requested_device,
- bound_backend,
- bound_device,
+ info.backend,
+ info.device,
caps.supports_streaming,
caps.supports_translate,
caps.supports_language_detect
);
- LoadedEngine::TranscribeCpp(session)
+ None
}
EngineType::Parakeet => {
let engine =
@@ -645,7 +680,7 @@ impl TranscriptionManager {
emit_loading_failed(&error_msg);
anyhow::anyhow!(error_msg)
})?;
- LoadedEngine::Parakeet(engine)
+ Some(OnnxEngine::Parakeet(engine))
}
EngineType::Moonshine => {
let engine = MoonshineModel::load(
@@ -658,7 +693,7 @@ impl TranscriptionManager {
emit_loading_failed(&error_msg);
anyhow::anyhow!(error_msg)
})?;
- LoadedEngine::Moonshine(engine)
+ Some(OnnxEngine::Moonshine(engine))
}
EngineType::MoonshineStreaming => {
let engine = StreamingModel::load(&model_path, 0, &Quantization::default())
@@ -670,7 +705,7 @@ impl TranscriptionManager {
emit_loading_failed(&error_msg);
anyhow::anyhow!(error_msg)
})?;
- LoadedEngine::MoonshineStreaming(engine)
+ Some(OnnxEngine::MoonshineStreaming(engine))
}
EngineType::SenseVoice => {
let engine =
@@ -680,7 +715,7 @@ impl TranscriptionManager {
emit_loading_failed(&error_msg);
anyhow::anyhow!(error_msg)
})?;
- LoadedEngine::SenseVoice(engine)
+ Some(OnnxEngine::SenseVoice(engine))
}
EngineType::GigaAM => {
let engine = GigaAMModel::load(&model_path, &Quantization::Int8).map_err(|e| {
@@ -688,7 +723,7 @@ impl TranscriptionManager {
emit_loading_failed(&error_msg);
anyhow::anyhow!(error_msg)
})?;
- LoadedEngine::GigaAM(engine)
+ Some(OnnxEngine::GigaAM(engine))
}
EngineType::Canary => {
let engine = CanaryModel::load(&model_path, &Quantization::Int8).map_err(|e| {
@@ -696,7 +731,7 @@ impl TranscriptionManager {
emit_loading_failed(&error_msg);
anyhow::anyhow!(error_msg)
})?;
- LoadedEngine::Canary(engine)
+ Some(OnnxEngine::Canary(engine))
}
EngineType::Cohere => {
let engine = CohereModel::load(&model_path, &Quantization::Int8).map_err(|e| {
@@ -704,15 +739,12 @@ impl TranscriptionManager {
emit_loading_failed(&error_msg);
anyhow::anyhow!(error_msg)
})?;
- LoadedEngine::Cohere(engine)
+ Some(OnnxEngine::Cohere(engine))
}
};
// Update the current engine and model ID
- {
- let mut engine = self.lock_engine();
- *engine = Some(loaded_engine);
- }
+ *self.lock_onnx() = loaded_onnx;
{
let mut current_model = self.current_model_id.lock().unwrap();
*current_model = Some(model_id.to_string());
@@ -782,13 +814,10 @@ impl TranscriptionManager {
/// its real backend string; ONNX engines report "onnx"; `None` when no
/// model is loaded.
pub fn current_backend(&self) -> Option<String> {
- match self.lock_engine().as_ref() {
- Some(LoadedEngine::TranscribeCpp(session)) => {
- Some(session.model().backend().to_string())
- }
- Some(_) => Some("onnx".to_string()),
- None => None,
+ if let Some(info) = self.engine.loaded() {
+ return Some(info.backend);
}
+ self.lock_onnx().as_ref().map(|_| "onnx".to_string())
}
/// Whether a live streaming run is currently in flight.
@@ -837,7 +866,6 @@ impl TranscriptionManager {
let _worker = StreamWorkerGuard {
worker_id,
active_stream_worker: Arc::clone(&self.active_stream_worker),
- active_engine_lease: Arc::clone(&self.active_engine_lease),
stream_active: Arc::clone(&self.stream_active),
};
@@ -852,79 +880,36 @@ impl TranscriptionManager {
let model_id = self.get_current_model().unwrap_or_default();
- // Take the engine out of the mutex so we own it during streaming,
- // structurally excluding any concurrent batch transcription (which
- // transcribe-cpp's compute_lock would refuse anyway). Returned when the
- // worker exits, or dropped if the model was switched/unloaded mid-stream.
- if self
- .active_engine_lease
- .compare_exchange(0, worker_id, Ordering::AcqRel, Ordering::Acquire)
- .is_err()
- {
- warn!("Live preview: another worker already holds the transcription engine");
+ // Only transcribe-cpp models expose streaming; ONNX engines fall back to
+ // batch. The loaded model (not the ModelManager copy) is the source of
+ // truth for run-path capabilities.
+ let Some(info) = self.engine.loaded() else {
+ info!(
+ "Live preview: model '{}' is not a loaded transcribe-cpp model; \
+ streaming is unavailable, using batch transcription",
+ model_id
+ );
self.router.clear();
drain_until_finalize(rx);
return;
- }
- let mut engine = match self.lock_engine().take() {
- Some(e) => e,
- None => {
- info!(
- "Live preview: model '{}' was unloaded before streaming could begin; \
- falling back to batch transcription",
- model_id
- );
- let _ = self.active_engine_lease.compare_exchange(
- worker_id,
- 0,
- Ordering::AcqRel,
- Ordering::Acquire,
- );
- self.router.clear();
- drain_until_finalize(rx);
- return;
- }
- };
-
- // Only transcribe-cpp models expose streaming; ONNX engines fall back to
- // batch. The loaded session (not the ModelManager copy) is the source of
- // truth for run-path capabilities.
- let (supports_streaming, supports_translate, languages) = match &engine {
- LoadedEngine::TranscribeCpp(session) => {
- let model = session.model();
- let caps = model.capabilities();
- info!(
- "Live preview: model '{}' arch='{}' variant='{}' supports_streaming={} \
- supports_translate={} languages={:?}",
- model_id,
- model.arch(),
- model.variant(),
- caps.supports_streaming,
- caps.supports_translate,
- caps.languages,
- );
- (
- caps.supports_streaming,
- caps.supports_translate,
- caps.languages,
- )
- }
- _ => {
- info!(
- "Live preview: model '{}' is not a transcribe-cpp model; \
- streaming is unavailable, using batch transcription",
- model_id
- );
- (false, false, Vec::new())
- }
};
-
- if !supports_streaming {
- self.return_engine(engine, &model_id);
+ let caps = &info.capabilities;
+ info!(
+ "Live preview: model '{}' arch='{}' variant='{}' supports_streaming={} \
+ supports_translate={} languages={:?}",
+ model_id,
+ info.arch,
+ info.variant,
+ caps.supports_streaming,
+ caps.supports_translate,
+ caps.languages,
+ );
+ if !caps.supports_streaming {
self.router.clear();
drain_until_finalize(rx);
return;
}
+ let languages = caps.languages.clone();
// Build run options mirroring the offline transcribe-cpp path: task +
// language gated against what the model actually advertises.
@@ -935,7 +920,7 @@ impl TranscriptionManager {
settings.translate_to_english,
&effective_language,
&languages,
- supports_translate,
+ caps.supports_translate,
);
let output_language = resolve_output_language_evidence(
&settings,
@@ -950,160 +935,126 @@ impl TranscriptionManager {
..Default::default()
};
- // Run the stream on the held session. The Stream borrows the session
- // (and thus the engine) for its lifetime, so the feed/finalize loop
- // lives in a labeled block — when it exits, the borrow is released and
- // the engine can be moved into return_engine().
- let mut preview_script = PreviewScript::new(settings.chinese_script, &output_language);
- let mut finalize_reply: Option<mpsc::Sender<Option<FinalizedStreamText>>> = None;
- let mut finalize_result: Option<Option<FinalizedStreamText>> = None;
- let stream_started = 'stream: {
- let session = match &mut engine {
- LoadedEngine::TranscribeCpp(s) => s,
- _ => break 'stream false,
- };
-
- // Read the backend string before beginning the stream — the
- // `Stream` borrows `session` mutably for its lifetime, so we can't
- // call `session.model()` once it exists.
- let backend = session.model().backend();
+ // Feed results arrive on the engine's thread; this callback only
+ // converts and emits preview text.
+ let perf = Arc::new(Mutex::new(StreamPerf::new()));
+ let on_progress = {
+ let perf = Arc::clone(&perf);
+ let app_handle = self.app_handle.clone();
+ let languages = languages.clone();
+ let mut preview_script = PreviewScript::new(settings.chinese_script, &output_language);
+ move |progress: StreamProgress| {
+ let mut perf = lock_perf(&perf);
+ perf.record_compute(progress.elapsed);
+ perf.record_update(&progress.update);
+ if let Some(text) = progress.text {
+ perf.record_emit();
+ let (committed, tentative) =
+ preview_script.convert(&text.committed, &text.tentative, &languages);
+ emit_stream_text(&app_handle, &committed, &tentative);
+ }
+ perf.maybe_log();
+ }
+ };
- // StreamOptions::default() uses CommitPolicy::Auto and lets the
- // family pick its own streaming strategy (no family-specific ext).
- let mut stream = match session.stream(&run_options, &StreamOptions::default()) {
- Ok(s) => s,
+ // Run the stream in the engine's worker process. Feeds are queued
+ // without waiting; a crashed or hung worker makes finalize report no
+ // result, and the caller falls back to batch transcription. StreamOptions::default()
+ // uses CommitPolicy::Auto and lets the family pick its own streaming
+ // strategy (no family-specific ext).
+ let stream =
+ match self
+ .engine
+ .start_stream(run_options, StreamOptions::default(), on_progress)
+ {
+ Ok(stream) => stream,
Err(e) => {
error!("Failed to begin stream: {}", e);
- break 'stream false;
+ self.forget_model_if_engine_dropped(&model_id, &e.to_string());
+ drain_until_finalize(rx);
+ return;
}
};
- self.stream_active.store(true, Ordering::Release);
- self.touch_activity();
- info!(
- "Live streaming transcription started (model '{}', backend '{}')",
- model_id, backend
- );
+ self.stream_active.store(true, Ordering::Release);
+ self.touch_activity();
+ info!(
+ "Live streaming transcription started (model '{}', backend '{}')",
+ model_id, info.backend
+ );
- let mut perf = StreamPerf::new();
- while let Ok(cmd) = rx.recv() {
- match cmd {
- StreamCmd::Feed(pcm) => {
- self.touch_activity();
- perf.record_feed(pcm.len());
- let feed_start = Instant::now();
- match stream.feed(&pcm) {
- Ok(update) => {
- perf.record_compute(feed_start.elapsed());
- perf.record_update(
- update.revision,
- update.input_received_ms,
- update.audio_committed_ms,
- update.buffered_ms,
- );
- if update.committed_changed || update.tentative_changed {
- let text = stream.text();
- perf.record_emit();
- let (committed, tentative) = preview_script.convert(
- &text.committed,
- &text.tentative,
- &languages,
- );
- self.emit_stream_text(&committed, &tentative);
- }
- perf.maybe_log();
- }
- Err(e) => {
- perf.record_compute(feed_start.elapsed());
- warn!("stream feed failed: {}", e);
- }
+ while let Ok(cmd) = rx.recv() {
+ match cmd {
+ StreamCmd::Feed(pcm) => {
+ self.touch_activity();
+ lock_perf(&perf).record_feed(pcm.len());
+ stream.feed(pcm);
+ }
+ StreamCmd::Finalize(reply) => {
+ // In auto mode the model's own LID is the best remaining
+ // evidence; the snapshot is only materialized when it can
+ // change the outcome.
+ let want_language = matches!(output_language, OutputLanguageEvidence::Unknown);
+ let result = match stream.finalize(want_language) {
+ // After finalize the committed prefix holds the full
+ // text; display() = committed + tentative is the safe read.
+ Ok(Some(finalized)) => {
+ let mut perf = lock_perf(&perf);
+ perf.record_compute(finalized.elapsed);
+ perf.record_update(&finalized.update);
+ let output_language = match &output_language {
+ OutputLanguageEvidence::Unknown => with_model_detected_language(
+ OutputLanguageEvidence::Unknown,
+ finalized.language,
+ ),
+ resolved => resolved.clone(),
+ };
+ Ok(Some(FinalizedStreamText {
+ text: finalized.text.full,
+ output_language,
+ supported_languages: languages.clone(),
+ }))
}
- }
- StreamCmd::Finalize(reply) => {
- let finalize_start = Instant::now();
- let result = match stream.finalize() {
- // After finalize the committed prefix holds the full
- // text; display() = committed + tentative is the safe read.
- Ok(update) => {
- perf.record_compute(finalize_start.elapsed());
- perf.record_update(
- update.revision,
- update.input_received_ms,
- update.audio_committed_ms,
- update.buffered_ms,
- );
- // In auto mode the model's own LID is the best
- // remaining evidence; the snapshot is only
- // materialized when it can change the outcome.
- let output_language = match &output_language {
- OutputLanguageEvidence::Unknown => {
- with_model_detected_language(
- OutputLanguageEvidence::Unknown,
- stream.snapshot().language,
- )
- }
- resolved => resolved.clone(),
- };
- Some(FinalizedStreamText {
- text: stream.text().full,
- output_language,
- supported_languages: languages.clone(),
- })
- }
- Err(e) => {
- perf.record_compute(finalize_start.elapsed());
- error!(
- "stream finalize failed: {}; falling back to batch transcription",
- e
- );
- None
- }
- };
- let chars = match &result {
- Some(finalized) => finalized.text.len(),
- _ => 0,
- };
- perf.log_finalized(chars);
- finalize_reply = Some(reply);
- finalize_result = Some(result);
- break;
- }
- StreamCmd::Cancel => {
- stream.reset();
- break;
- }
+ Ok(None) => {
+ error!(
+ "Live transcription could not be finalized; \
+ falling back to batch transcription"
+ );
+ self.forget_model_if_engine_dropped(
+ &model_id,
+ "live transcription failed and the model could not be restarted",
+ );
+ Ok(None)
+ }
+ Err(e) => {
+ info!("Live transcription finalize stopped: {}", e);
+ Err(e)
+ }
+ };
+ let chars = match &result {
+ Ok(Some(finalized)) => finalized.text.len(),
+ _ => 0,
+ };
+ lock_perf(&perf).log_finalized(chars);
+ let _ = reply.send(result);
+ return;
}
+ // Dropping the stream handle resets the worker's stream.
+ StreamCmd::Cancel => return,
}
-
- true
- };
- // `stream` + the `&mut engine` borrow are released here.
-
- if !stream_started {
- // Stream never began (model doesn't support streaming or begin
- // failed); drain so the finalize handshake still completes and the
- // caller falls back to batch transcription. Return the engine first
- // so the fallback can immediately use it.
- self.return_engine(engine, &model_id);
- drain_until_finalize(rx);
- return;
}
-
- self.return_engine(engine, &model_id);
- if let (Some(reply), Some(result)) = (finalize_reply, finalize_result) {
- let _ = reply.send(result);
- }
- // `_worker` drops here, clearing this worker's active/lease flags after
- // the engine has been returned to the pool.
+ // The channel closed without finalize/cancel; dropping the stream
+ // handle resets the worker's stream. `_worker` drops after, clearing
+ // this worker's flags.
}
- /// Return the leased engine to the mutex, unless the model was switched or
- /// unloaded during transcription (in which case the stale engine is dropped).
- fn return_engine(&self, engine: LoadedEngine, expected_model_id: &str) {
+ /// Return the taken ONNX engine to the mutex, unless the model was switched
+ /// or unloaded during transcription (in which case the stale engine is dropped).
+ fn return_engine(&self, engine: OnnxEngine, expected_model_id: &str) {
let still_current =
self.current_model_id.lock().unwrap().as_deref() == Some(expected_model_id);
if still_current {
- *self.lock_engine() = Some(engine);
+ *self.lock_onnx() = Some(engine);
} else {
info!(
"Model changed/unloaded during transcription; dropping stale engine (was '{}')",
@@ -1115,10 +1066,11 @@ impl TranscriptionManager {
/// Flush the active stream and return its final, post-filtered text.
///
- /// `Ok(None)` means no usable stream was active and the caller may fall back
- /// to batch transcription. `Err` means finalize itself failed or timed out.
- /// A timeout may still leave the worker holding the engine, so callers
- /// should surface it instead of immediately starting a batch fallback.
+ /// `Ok(None)` means no usable stream result exists and the caller should
+ /// fall back to batch transcription. `Err` means the transcription was
+ /// cancelled. There is no timeout here: waiting behind a model load is a
+ /// queue position, not a failure (#1841), and the engine bounds the
+ /// finalize itself by the audio it still has to decode.
pub fn finalize_stream(&self) -> Result<Option<String>> {
let Some(tx) = self.router.take() else {
return Ok(None);
@@ -1127,17 +1079,10 @@ impl TranscriptionManager {
if tx.send(StreamCmd::Finalize(reply_tx)).is_err() {
return Ok(None);
}
- let finalized = match reply_rx.recv_timeout(STREAM_FINALIZE_REPLY_TIMEOUT) {
- Ok(Some(finalized)) => finalized,
- Ok(None) => return Ok(None),
- Err(mpsc::RecvTimeoutError::Disconnected) => return Ok(None),
- Err(mpsc::RecvTimeoutError::Timeout) => {
- self.stream_active.store(false, Ordering::Release);
- return Err(anyhow::anyhow!(
- "Timed out waiting {:?} for live transcription to finalize",
- STREAM_FINALIZE_REPLY_TIMEOUT
- ));
- }
+ let finalized = match reply_rx.recv() {
+ Ok(Ok(Some(finalized))) => finalized,
+ Ok(Ok(None)) | Err(_) => return Ok(None),
+ Ok(Err(e)) => return Err(e.into()),
};
let settings = get_settings(&self.app_handle);
@@ -1163,6 +1108,14 @@ impl TranscriptionManager {
self.stream_active.store(false, Ordering::Release);
}
+ /// Stop the transcription in progress (a batch run or stream finalize),
+ /// and any queued behind it, for an explicit user cancel. The worker doing
+ /// it is stopped; the model stays loaded and the next use starts a fresh
+ /// worker for it.
+ pub fn cancel_transcription(&self) {
+ self.engine.cancel();
+ }
+
/// Emit a working-phase event to the streaming overlay (spinner + label).
pub fn emit_stream_working(&self, kind: StreamWorkKind) {
let _ = StreamPhaseEvent {
@@ -1172,14 +1125,6 @@ impl TranscriptionManager {
.emit(&self.app_handle);
}
- fn emit_stream_text(&self, committed: &str, tentative: &str) {
- let _ = StreamTextEvent {
- committed: committed.to_string(),
- tentative: tentative.to_string(),
- }
- .emit(&self.app_handle);
- }
-
pub fn transcribe(&self, audio: Vec<f32>) -> Result<String> {
#[cfg(debug_assertions)]
if std::env::var("HANDY_FORCE_TRANSCRIPTION_FAILURE").is_ok() {
@@ -1210,8 +1155,7 @@ impl TranscriptionManager {
is_loading = self.loading_condvar.wait(is_loading).unwrap();
}
- let engine_guard = self.lock_engine();
- if engine_guard.is_none() {
+ if !self.is_model_loaded() {
return Err(anyhow::anyhow!("Model is not loaded for transcription."));
}
}
@@ -1241,271 +1185,36 @@ impl TranscriptionManager {
);
}
- // Whether the loaded transcribe-cpp model advertises
- // Feature::InitialPrompt. Informational (logged below); the whisper
- // run extension and the fuzzy-correction skip are gated on
- // `model_is_whisper` instead, since non-whisper archs can advertise
- // the feature while rejecting the whisper-kind extension.
- let mut model_takes_initial_prompt = false;
- // Whether the loaded model is actually whisper-family (arch string).
- // Non-whisper archs (e.g. Voxtral Small) can advertise
- // Feature::InitialPrompt yet reject the whisper-kind run extension
- // with INVALID_ARG, so the whisper extension must be gated on the
- // arch, not on the feature (see #1601).
- let mut model_is_whisper = false;
-
// Perform transcription with the appropriate engine.
- // We use catch_unwind to prevent engine panics from poisoning the mutex,
- // which would make the app hang indefinitely on subsequent operations.
- let (result, output_language, model_languages) = {
- let mut engine_guard = self.lock_engine();
-
- // Take the engine out so we own it during transcription.
- // If the engine panics, we simply don't put it back (effectively unloading it)
- // instead of poisoning the mutex.
- let mut engine = match engine_guard.take() {
- Some(e) => e,
- None => {
- return Err(anyhow::anyhow!(
- "Model failed to load after auto-load attempt. Please check your model settings."
- ));
- }
- };
-
- // Release the lock before transcribing — no mutex held during the engine call
- drop(engine_guard);
-
- // Probe live transcribe-cpp capabilities once (cheap GGUF-metadata
- // reads); the loaded session is the source of truth, not the
- // ModelManager copy. The whisper run extension is kind-tagged, so
- // non-whisper archs (parakeet, voxtral, …) reject it with
- // INVALID_ARG; attach it — and translate — only where supported.
- let mut model_supports_translate = false;
- let mut model_languages = self
- .model_manager
- .get_model_info(&active_model)
- .map(|info| info.supported_languages)
- .unwrap_or_default();
- let mut output_was_translated = false;
- let mut applied_language_hint: Option<String> = None;
- let mut model_detected_language: Option<String> = None;
- if let LoadedEngine::TranscribeCpp(session) = &engine {
- let model = session.model();
- let caps = model.capabilities();
- model_takes_initial_prompt = model.supports(Feature::InitialPrompt);
- model_is_whisper = model.arch() == "whisper";
- model_supports_translate = caps.supports_translate;
- model_languages = caps.languages;
- debug!(
- "transcribe-cpp model '{}' on '{}': initial_prompt={}, translate={}, languages={:?}",
- settings.selected_model,
- model.backend(),
- model_takes_initial_prompt,
- model_supports_translate,
- model_languages
- );
+ let run = match self.engine.loaded() {
+ Some(info) => {
+ self.transcribe_cpp(info, audio, &settings, &validated_language, &active_model)?
}
-
- let transcribe_result = catch_unwind(AssertUnwindSafe(|| -> Result<String> {
- match &mut engine {
- LoadedEngine::TranscribeCpp(session) => {
- // Custom words become the initial prompt ONLY for models
- // that accept one (whisper family). Attaching the
- // whisper run extension to a non-whisper arch is rejected
- // with INVALID_ARG, so skip it there and let the fuzzy
- // post-correction handle custom words instead.
- let family = if settings.custom_words.is_empty() || !model_is_whisper {
- None
- } else {
- Some(RunExtension::Whisper(WhisperRunOptions {
- initial_prompt: Some(settings.custom_words.join(", ")),
- ..Default::default()
- }))
- };
-
- let run_plan = transcribe_cpp_run_plan(
- settings.translate_to_english,
- &validated_language,
- &model_languages,
- model_supports_translate,
- );
- output_was_translated = run_plan.target_language.as_deref() == Some("en");
- applied_language_hint = run_plan.language.clone();
-
- let run_options = RunOptions {
- task: run_plan.task,
- language: run_plan.language,
- target_language: run_plan.target_language,
- family,
- ..Default::default()
- };
-
- debug!(
- "transcribe-cpp run: task={:?}, language={:?}, initial_prompt={}",
- run_options.task,
- run_options.language,
- run_options.family.is_some()
- );
-
- session
- .run(&audio, &run_options)
- .map(|t| {
- // Whisper's audio-based LID (auto mode only;
- // `None` when a language hint was passed).
- model_detected_language = t.language;
- t.text
- })
- .map_err(|e| {
- anyhow::anyhow!("transcribe-cpp transcription failed: {}", e)
- })
- }
- LoadedEngine::Parakeet(parakeet_engine) => {
- let params = ParakeetParams {
- timestamp_granularity: Some(TimestampGranularity::Segment),
- ..Default::default()
- };
- parakeet_engine
- .transcribe_with(&audio, &params)
- .map(|r| r.text)
- .map_err(|e| anyhow::anyhow!("Parakeet transcription failed: {}", e))
- }
- LoadedEngine::Moonshine(moonshine_engine) => moonshine_engine
- .transcribe(&audio, &TranscribeOptions::default())
- .map(|r| r.text)
- .map_err(|e| anyhow::anyhow!("Moonshine transcription failed: {}", e)),
- LoadedEngine::MoonshineStreaming(streaming_engine) => streaming_engine
- .transcribe(&audio, &TranscribeOptions::default())
- .map(|r| r.text)
- .map_err(|e| {
- anyhow::anyhow!("Moonshine streaming transcription failed: {}", e)
- }),
- LoadedEngine::SenseVoice(sense_voice_engine) => {
- let language = match validated_language.as_str() {
- "zh" => Some("zh".to_string()),
- "en" => Some("en".to_string()),
- "ja" => Some("ja".to_string()),
- "ko" => Some("ko".to_string()),
- "yue" => Some("yue".to_string()),
- _ => None,
- };
- applied_language_hint = language.clone();
- let params = SenseVoiceParams {
- language,
- use_itn: Some(true),
- };
- sense_voice_engine
- .transcribe_with(&audio, &params)
- .map(|r| r.text)
- .map_err(|e| anyhow::anyhow!("SenseVoice transcription failed: {}", e))
- }
- LoadedEngine::GigaAM(gigaam_engine) => gigaam_engine
- .transcribe(&audio, &TranscribeOptions::default())
- .map(|r| r.text)
- .map_err(|e| anyhow::anyhow!("GigaAM transcription failed: {}", e)),
- LoadedEngine::Canary(canary_engine) => {
- output_was_translated = settings.translate_to_english;
- let lang = if validated_language == "auto" {
- None
- } else {
- Some(validated_language.clone())
- };
- applied_language_hint = lang.clone();
- let options = TranscribeOptions {
- language: lang,
- translate: settings.translate_to_english,
- ..Default::default()
- };
- canary_engine
- .transcribe(&audio, &options)
- .map(|r| r.text)
- .map_err(|e| anyhow::anyhow!("Canary transcription failed: {}", e))
- }
- LoadedEngine::Cohere(cohere_engine) => {
- let lang = if validated_language == "auto" {
- None
- } else {
- Some(validated_language.clone())
- };
- applied_language_hint = lang.clone();
- let options = TranscribeOptions {
- language: lang,
- ..Default::default()
- };
- cohere_engine
- .transcribe(&audio, &options)
- .map(|r| r.text)
- .map_err(|e| anyhow::anyhow!("Cohere transcription failed: {}", e))
- }
- }
- }));
-
- let text = match transcribe_result {
- Ok(inner_result) => {
- // Success or normal error: return the engine unless a model
- // switch/unload invalidated it while it was in use.
- self.return_engine(engine, &active_model);
- inner_result?
- }
- Err(panic_payload) => {
- // Engine panicked — do NOT put it back (it's in an unknown state).
- // The engine is dropped here, effectively unloading it.
- let panic_msg = panic_payload_message(panic_payload.as_ref());
- error!(
- "Transcription engine panicked: {}. Model has been unloaded.",
- panic_msg
- );
-
- // Clear the model ID so it will be reloaded on next attempt
- {
- let mut current_model = self
- .current_model_id
- .lock()
- .unwrap_or_else(|e| e.into_inner());
- *current_model = None;
- }
-
- let _ = self.app_handle.emit(
- "model-state-changed",
- ModelStateEvent {
- event_type: "unloaded".to_string(),
- model_id: None,
- model_name: None,
- error: Some(format!("Engine panicked: {}", panic_msg)),
- },
- );
-
- return Err(anyhow::anyhow!(
- "Transcription engine panicked: {}. The model has been unloaded and will reload on next attempt.",
- panic_msg
- ));
- }
- };
-
- let output_language = with_model_detected_language(
- resolve_output_language_evidence(
- &settings,
- applied_language_hint.as_deref(),
- &model_languages,
- output_was_translated,
- ),
- model_detected_language,
- );
- debug!("Output language evidence: {:?}", output_language);
-
- (text, output_language, model_languages)
+ None => self.transcribe_onnx(&audio, &settings, &validated_language, &active_model)?,
};
+ let output_language = with_model_detected_language(
+ resolve_output_language_evidence(
+ &settings,
+ run.applied_language_hint.as_deref(),
+ &run.languages,
+ run.output_was_translated,
+ ),
+ run.model_detected_language,
+ );
+ debug!("Output language evidence: {:?}", output_language);
+
// Apply fuzzy word correction if custom words are configured — UNLESS the
// words were already handed to the model as an initial prompt (whisper
// family). We don't pass a prompt to non-whisper models (it requires the
// whisper-kind run extension), so they still get fuzzy correction here,
// same as the ONNX engines.
let filtered_result = post_process_transcription_text(
- result,
+ run.text,
&settings,
- model_is_whisper,
+ run.model_is_whisper,
&output_language,
- &model_languages,
+ &run.languages,
);
let et = std::time::Instant::now();
@@ -1540,6 +1249,293 @@ impl TranscriptionManager {
Ok(final_result)
}
+
+ /// transcribe-cpp: the model runs in the engine's worker process, which
+ /// recovers from crashes and hangs itself. The loaded model's live
+ /// capabilities (cheap GGUF-metadata reads) are the source of truth, not
+ /// the ModelManager copy. The whisper run extension is kind-tagged, so
+ /// non-whisper archs (parakeet, voxtral, …) reject it with INVALID_ARG;
+ /// attach it — and translate — only where supported.
+ fn transcribe_cpp(
+ &self,
+ info: LoadedInfo,
+ audio: Vec<f32>,
+ settings: &AppSettings,
+ validated_language: &str,
+ active_model: &str,
+ ) -> Result<RunOutcome> {
+ // Whether the model advertises Feature::InitialPrompt. Informational
+ // (logged below); the whisper run extension and the fuzzy-correction
+ // skip are gated on `model_is_whisper` instead, since non-whisper
+ // archs can advertise the feature while rejecting the whisper-kind
+ // extension.
+ let model_takes_initial_prompt = info.supports_initial_prompt;
+ let model_is_whisper = info.arch == "whisper";
+ let model_supports_translate = info.capabilities.supports_translate;
+ let languages = info.capabilities.languages;
+ debug!(
+ "transcribe-cpp model '{}' on '{}': initial_prompt={}, translate={}, languages={:?}",
+ settings.selected_model,
+ info.backend,
+ model_takes_initial_prompt,
+ model_supports_translate,
+ languages
+ );
+
+ // Custom words become the initial prompt ONLY for models that accept
+ // one (whisper family). Attaching the whisper run extension to a
+ // non-whisper arch is rejected with INVALID_ARG, so skip it there and
+ // let the fuzzy post-correction handle custom words instead.
+ let family = if settings.custom_words.is_empty() || !model_is_whisper {
+ None
+ } else {
+ Some(RunExtension::Whisper(WhisperRunOptions {
+ initial_prompt: Some(settings.custom_words.join(", ")),
+ ..Default::default()
+ }))
+ };
+
+ let run_plan = transcribe_cpp_run_plan(
+ settings.translate_to_english,
+ validated_language,
+ &languages,
+ model_supports_translate,
+ );
+ let output_was_translated = run_plan.target_language.as_deref() == Some("en");
+ let applied_language_hint = run_plan.language.clone();
+
+ let run_options = RunOptions {
+ task: run_plan.task,
+ language: run_plan.language,
+ target_language: run_plan.target_language,
+ family,
+ ..Default::default()
+ };
+
+ debug!(
+ "transcribe-cpp run: task={:?}, language={:?}, initial_prompt={}",
+ run_options.task,
+ run_options.language,
+ run_options.family.is_some()
+ );
+
+ let transcript = match self.engine.transcribe(audio, run_options) {
+ Ok(transcript) => transcript,
+ Err(EngineError::Cancelled) => return Err(EngineError::Cancelled.into()),
+ Err(e) => {
+ self.forget_model_if_engine_dropped(active_model, &e.to_string());
+ return Err(anyhow::anyhow!(
+ "transcribe-cpp transcription failed: {}",
+ e
+ ));
+ }
+ };
+ Ok(RunOutcome {
+ text: transcript.text,
+ languages,
+ applied_language_hint,
+ output_was_translated,
+ // Whisper's audio-based LID (auto mode only; `None` when a
+ // language hint was passed).
+ model_detected_language: transcript.language,
+ model_is_whisper,
+ })
+ }
+
+ /// In-process ONNX engines.
+ fn transcribe_onnx(
+ &self,
+ audio: &[f32],
+ settings: &AppSettings,
+ validated_language: &str,
+ active_model: &str,
+ ) -> Result<RunOutcome> {
+ let languages = self
+ .model_manager
+ .get_model_info(active_model)
+ .map(|info| info.supported_languages)
+ .unwrap_or_default();
+ let mut output_was_translated = false;
+ let mut applied_language_hint: Option<String> = None;
+
+ // Take the engine out so we own it during transcription. If the
+ // engine panics, we simply don't put it back (effectively unloading
+ // it) instead of poisoning the mutex. No lock is held during the
+ // engine call.
+ let mut engine = match self.lock_onnx().take() {
+ Some(e) => e,
+ None => {
+ return Err(anyhow::anyhow!(
+ "Model failed to load after auto-load attempt. Please check your model settings."
+ ));
+ }
+ };
+
+ // We use catch_unwind to prevent engine panics from poisoning the
+ // mutex, which would make the app hang indefinitely on subsequent
+ // operations.
+ let transcribe_result = catch_unwind(AssertUnwindSafe(|| -> Result<String> {
+ match &mut engine {
+ OnnxEngine::Parakeet(parakeet_engine) => {
+ let params = ParakeetParams {
+ timestamp_granularity: Some(TimestampGranularity::Segment),
+ ..Default::default()
+ };
+ parakeet_engine
+ .transcribe_with(audio, &params)
+ .map(|r| r.text)
+ .map_err(|e| anyhow::anyhow!("Parakeet transcription failed: {}", e))
+ }
+ OnnxEngine::Moonshine(moonshine_engine) => moonshine_engine
+ .transcribe(audio, &TranscribeOptions::default())
+ .map(|r| r.text)
+ .map_err(|e| anyhow::anyhow!("Moonshine transcription failed: {}", e)),
+ OnnxEngine::MoonshineStreaming(streaming_engine) => streaming_engine
+ .transcribe(audio, &TranscribeOptions::default())
+ .map(|r| r.text)
+ .map_err(|e| {
+ anyhow::anyhow!("Moonshine streaming transcription failed: {}", e)
+ }),
+ OnnxEngine::SenseVoice(sense_voice_engine) => {
+ let language = match validated_language {
+ "zh" => Some("zh".to_string()),
+ "en" => Some("en".to_string()),
+ "ja" => Some("ja".to_string()),
+ "ko" => Some("ko".to_string()),
+ "yue" => Some("yue".to_string()),
+ _ => None,
+ };
+ applied_language_hint = language.clone();
+ let params = SenseVoiceParams {
+ language,
+ use_itn: Some(true),
+ };
+ sense_voice_engine
+ .transcribe_with(audio, &params)
+ .map(|r| r.text)
+ .map_err(|e| anyhow::anyhow!("SenseVoice transcription failed: {}", e))
+ }
+ OnnxEngine::GigaAM(gigaam_engine) => gigaam_engine
+ .transcribe(audio, &TranscribeOptions::default())
+ .map(|r| r.text)
+ .map_err(|e| anyhow::anyhow!("GigaAM transcription failed: {}", e)),
+ OnnxEngine::Canary(canary_engine) => {
+ output_was_translated = settings.translate_to_english;
+ let lang = if validated_language == "auto" {
+ None
+ } else {
+ Some(validated_language.to_string())
+ };
+ applied_language_hint = lang.clone();
+ let options = TranscribeOptions {
+ language: lang,
+ translate: settings.translate_to_english,
+ ..Default::default()
+ };
+ canary_engine
+ .transcribe(audio, &options)
+ .map(|r| r.text)
+ .map_err(|e| anyhow::anyhow!("Canary transcription failed: {}", e))
+ }
+ OnnxEngine::Cohere(cohere_engine) => {
+ let lang = if validated_language == "auto" {
+ None
+ } else {
+ Some(validated_language.to_string())
+ };
+ applied_language_hint = lang.clone();
+ let options = TranscribeOptions {
+ language: lang,
+ ..Default::default()
+ };
+ cohere_engine
+ .transcribe(audio, &options)
+ .map(|r| r.text)
+ .map_err(|e| anyhow::anyhow!("Cohere transcription failed: {}", e))
+ }
+ }
+ }));
+
+ let text = match transcribe_result {
+ Ok(inner_result) => {
+ // Success or normal error: return the engine unless a model
+ // switch/unload invalidated it while it was in use.
+ self.return_engine(engine, active_model);
+ inner_result?
+ }
+ Err(panic_payload) => {
+ // Engine panicked — do NOT put it back (it's in an unknown state).
+ // The engine is dropped here, effectively unloading it.
+ let panic_msg = panic_payload_message(panic_payload.as_ref());
+ error!(
+ "Transcription engine panicked: {}. Model has been unloaded.",
+ panic_msg
+ );
+
+ // Clear the model ID so it will be reloaded on next attempt
+ {
+ let mut current_model = self
+ .current_model_id
+ .lock()
+ .unwrap_or_else(|e| e.into_inner());
+ *current_model = None;
+ }
+
+ let _ = self.app_handle.emit(
+ "model-state-changed",
+ ModelStateEvent {
+ event_type: "unloaded".to_string(),
+ model_id: None,
+ model_name: None,
+ error: Some(format!("Engine panicked: {}", panic_msg)),
+ },
+ );
+
+ return Err(anyhow::anyhow!(
+ "Transcription engine panicked: {}. The model has been unloaded and will reload on next attempt.",
+ panic_msg
+ ));
+ }
+ };
+ Ok(RunOutcome {
+ text,
+ languages,
+ applied_language_hint,
+ output_was_translated,
+ model_detected_language: None,
+ model_is_whisper: false,
+ })
+ }
+}
+
+/// What a batch transcription produced, plus what post-processing needs to
+/// know about how it was produced.
+struct RunOutcome {
+ text: String,
+ /// The model's supported languages.
+ languages: Vec<String>,
+ applied_language_hint: Option<String>,
+ output_was_translated: bool,
+ /// The language the model itself detected, if it reported one.
+ model_detected_language: Option<String>,
+ /// Whether the loaded model is actually whisper-family (arch string).
+ /// Non-whisper archs (e.g. Voxtral Small) can advertise
+ /// Feature::InitialPrompt yet reject the whisper-kind run extension with
+ /// INVALID_ARG, so the whisper extension must be gated on the arch, not
+ /// on the feature (see #1601).
+ model_is_whisper: bool,
+}
+
+fn emit_stream_text(app_handle: &AppHandle, committed: &str, tentative: &str) {
+ let _ = StreamTextEvent {
+ committed: committed.to_string(),
+ tentative: tentative.to_string(),
+ }
+ .emit(app_handle);
+}
+
+fn lock_perf(perf: &Mutex<StreamPerf>) -> MutexGuard<'_, StreamPerf> {
+ perf.lock().unwrap_or_else(|e| e.into_inner())
}
struct StreamPerf {
@@ -1578,17 +1574,11 @@ impl StreamPerf {
self.stream_compute_elapsed += elapsed;
}
- fn record_update(
- &mut self,
- revision: i32,
- input_received_ms: i64,
- audio_committed_ms: i64,
- buffered_ms: i64,
- ) {
- self.latest_revision = revision;
- self.latest_input_received_ms = input_received_ms;
- self.latest_audio_committed_ms = audio_committed_ms;
- self.latest_buffered_ms = buffered_ms;
+ fn record_update(&mut self, update: &transcribe_cpp::StreamUpdate) {
+ self.latest_revision = update.revision;
+ self.latest_input_received_ms = update.input_received_ms;
+ self.latest_audio_committed_ms = update.audio_committed_ms;
+ self.latest_buffered_ms = update.buffered_ms;
}
fn record_emit(&mut self) {
@@ -1943,7 +1933,7 @@ fn drain_until_finalize(rx: mpsc::Receiver<StreamCmd>) {
match cmd {
StreamCmd::Feed(_) => {}
StreamCmd::Finalize(reply) => {
- let _ = reply.send(None);
+ let _ = reply.send(Ok(None));
break;
}
StreamCmd::Cancel => break,
@@ -1951,97 +1941,56 @@ fn drain_until_finalize(rx: mpsc::Receiver<StreamCmd>) {
}
}
-/// Initialize the transcribe-cpp native backend once at startup: route native +
-/// ggml diagnostics into the `log` facade and register compute backend modules.
-/// In a static build (macOS Metal) `init_backends_default` is a harmless no-op;
-/// in a `dynamic-backends` build it loads the per-ISA CPU / GPU modules. Must run
-/// before the first model load.
-pub fn init_transcribe_backend() {
- transcribe_cpp::init_logging();
- match transcribe_cpp::init_backends_default() {
- Ok(()) => {
- if transcribe_gpu_disabled_for_host() {
- warn!(
- "Windows x64 build is running under emulation on an ARM64 host; \
- disabling transcribe.cpp GPU acceleration and using CPU"
- );
- }
- }
- Err(e) => warn!("Failed to initialize transcribe-cpp backends: {}", e),
- }
-}
-
-/// Log the compute devices [`init_transcribe_backend`] registered.
+/// Log the compute devices transcribe-cpp registers.
///
-/// Listing devices is what first opens the GPU. On macOS that loads ggml's
-/// Metal library, which is compiled from source whenever the system's shader
-/// cache does not hold it yet (the first launch after an install or update),
-/// so the app calls this from a background thread instead of its startup
-/// path. A model load that comes first waits on the same one-time compile.
-pub fn report_compute_devices() {
- let devices = transcribe_compute_devices();
- info!(
- "transcribe-cpp initialized with {} compute device(s): [{}]",
- devices.len(),
- devices
- .iter()
- .map(|d| format!("{} ({})", d.name, d.kind))
- .collect::<Vec<_>>()
- .join(", ")
- );
+/// Devices are listed by a transcription worker process (see
+/// [`crate::engine_supervisor`]), so GPU driver initialization never runs in
+/// the app process. Listing opens the GPU, which on macOS loads ggml's Metal
+/// library and compiles it when the system shader cache does not hold it yet
+/// (the first launch after an install or update), so the app calls this from
+/// a background thread instead of its startup path.
+pub fn report_compute_devices(tm: &TranscriptionManager) {
+ if transcribe_gpu_disabled_for_host() {
+ warn!(
+ "Windows x64 build is running under emulation on an ARM64 host; \
+ disabling transcribe.cpp GPU acceleration and using CPU"
+ );
+ }
+ match transcribe_compute_devices(&tm.engine) {
+ Ok(devices) => info!(
+ "transcribe-cpp initialized with {} compute device(s): [{}]",
+ devices.len(),
+ devices
+ .iter()
+ .map(|d| format!("{} ({})", d.name, d.kind))
+ .collect::<Vec<_>>()
+ .join(", ")
+ ),
+ Err(e) => warn!("transcribe-cpp compute devices are unavailable: {}", e),
+ }
}
-/// Human-readable list of the transcribe-cpp compute devices registered at
-/// startup, for the `--list-devices` flag. The reported `index` is the
-/// value to pass to `--device-index`. Backends must be initialized first
-/// (see [`init_transcribe_backend`]).
-pub fn describe_compute_devices() -> Vec<String> {
- transcribe_compute_devices()
+/// Human-readable list of the transcribe-cpp compute devices, for the
+/// `--list-devices` flag. The reported `index` is the value to pass to
+/// `--device-index`. Errors if the devices could not be listed.
+pub fn describe_compute_devices(tm: &TranscriptionManager) -> Result<Vec<String>> {
+ Ok(transcribe_compute_devices(&tm.engine)?
.into_iter()
.map(|d| {
let idx = d
.index
.map(|i| i.to_string())
.unwrap_or_else(|| "-".to_string());
- let name = if d.description.is_empty() {
- d.name
- } else {
- d.description
- };
let vram_mb = d.memory_total / (1024 * 1024);
format!(
"index={} kind={} name={} vram={}MB",
- idx, d.kind, name, vram_mb
+ idx,
+ d.kind,
+ d.label(),
+ vram_mb
)
})
- .collect()
-}
-
-/// Resolve a `--list-devices` registry index to an exact opaque device handle
-/// for a transcribe-cpp model load (the `--device-index` flag). In 0.2 index 0
-/// is an exact selection too; only an omitted index requests automatic device
-/// selection. Errors if the index isn't a registered, loadable primary device.
-fn resolve_device_index(index: usize) -> Result<(Backend, Option<transcribe_cpp::Device>)> {
- let device = transcribe_compute_devices()
- .into_iter()
- .find(|d| d.index == Some(index))
- .ok_or_else(|| {
- anyhow::anyhow!("No compute device with index {index} (see --list-devices)")
- })?;
- if matches!(
- device.device_type,
- transcribe_cpp::DeviceType::Accel | transcribe_cpp::DeviceType::Unknown
- ) {
- return Err(anyhow::anyhow!(
- "Device index {index} ({}) cannot host a model",
- device.kind
- ));
- }
-
- // 0.2's opaque handle makes every index, including zero, an exact
- // selection. Backend::Auto accepts any primary device and cannot conflict
- // with the selected device's vendor backend.
- Ok((Backend::Auto, Some(device)))
+ .collect())
}
/// Map Handy's whisper accelerator setting to a transcribe-cpp [`Backend`].
@@ -2065,45 +2014,19 @@ fn select_transcribe_backend_for_host(
}
}
-/// Resolve the user's persisted GPU identity to a fresh opaque 0.2 device
-/// handle. Registry indices and handles are process-local, so settings store a
-/// key based on the backend's stable `device_id` (falling back to name for
-/// backends such as Metal that do not report one).
+/// Resolve the user's persisted GPU choice to the device key the worker
+/// should load on. Registry indices and handles are process-local, so
+/// settings store a key based on the backend's stable `device_id` (falling
+/// back to name for backends such as Metal that do not report one). The
+/// worker falls back to automatic selection if that device is gone.
fn resolve_gpu_device(
setting: TranscribeAcceleratorSetting,
gpu_device: Option<&str>,
-) -> Option<transcribe_cpp::Device> {
+) -> Option<String> {
if transcribe_gpu_disabled_for_host() || setting != TranscribeAcceleratorSetting::Gpu {
return None;
}
- let gpu_device = gpu_device?;
- let resolved = transcribe_compute_devices().into_iter().find(|device| {
- is_transcribe_gpu_device(device) && transcribe_device_key(device) == gpu_device
- });
- if resolved.is_none() {
- warn!(
- "Stored transcribe GPU device '{}' is no longer available; using automatic device selection",
- gpu_device
- );
- }
- resolved
-}
-
-fn transcribe_device_key(device: &transcribe_cpp::Device) -> String {
- let (identity_kind, identity) = match device.device_id.as_deref() {
- Some(device_id) => ("id", device_id),
- None => ("name", device.name.as_str()),
- };
- serde_json::to_string(&(device.kind.as_str(), identity_kind, identity))
- .expect("transcribe device identity is always JSON serializable")
-}
-
-fn transcribe_device_label(device: &transcribe_cpp::Device) -> String {
- if device.description.is_empty() {
- device.name.clone()
- } else {
- device.description.clone()
- }
+ gpu_device.map(str::to_string)
}
/// Apply the user's ORT accelerator preference to the transcribe-rs global.
@@ -2140,8 +2063,6 @@ pub struct GpuDeviceOption {
pub total_vram_mb: usize,
}
-static GPU_DEVICES: OnceLock<Vec<GpuDeviceOption>> = OnceLock::new();
-
fn transcribe_gpu_disabled_for_host() -> bool {
crate::utils::is_windows_x64_emulated_on_arm64()
}
@@ -2157,28 +2078,12 @@ fn effective_transcribe_accelerator(
}
}
-fn is_transcribe_gpu_device(device: &transcribe_cpp::Device) -> bool {
- matches!(
- device.device_type,
- transcribe_cpp::DeviceType::Gpu | transcribe_cpp::DeviceType::Igpu
- )
-}
-
-fn transcribe_device_allowed(kind: &str, gpu_disabled: bool) -> bool {
- !gpu_disabled || matches!(kind, "cpu" | "accel")
-}
-
-fn transcribe_compute_devices() -> Vec<transcribe_cpp::Device> {
- let devices = transcribe_cpp::devices();
- let gpu_disabled = transcribe_gpu_disabled_for_host();
- if !gpu_disabled {
- return devices;
- }
-
- devices
- .into_iter()
- .filter(|device| transcribe_device_allowed(&device.kind, gpu_disabled))
- .collect()
+/// The compute devices the latest worker listed. On a host without GPU
+/// access every worker is CPU-only, so it lists no GPU.
+fn transcribe_compute_devices(engine: &EngineSupervisor) -> Result<Vec<DeviceInfo>> {
+ engine
+ .devices()
+ .ok_or_else(|| anyhow::anyhow!("compute devices could not be listed (see the log for why)"))
}
fn available_transcribe_accelerators(gpu_disabled: bool) -> Vec<String> {
@@ -2189,22 +2094,22 @@ fn available_transcribe_accelerators(gpu_disabled: bool) -> Vec<String> {
}
}
-fn cached_gpu_devices() -> &'static [GpuDeviceOption] {
- // GPU compute devices transcribe-cpp registered at startup. `id` is a
- // persistent identity key, never the process-local registry index. It uses
- // the backend's device_id where available and its name otherwise (Metal).
- // `total_vram_mb` is 0 when the backend does not report capacity.
- GPU_DEVICES.get_or_init(|| {
- transcribe_compute_devices()
- .into_iter()
- .filter(is_transcribe_gpu_device)
- .map(|d| GpuDeviceOption {
- id: transcribe_device_key(&d),
- name: transcribe_device_label(&d),
- total_vram_mb: (d.memory_total / (1024 * 1024)) as usize,
- })
- .collect()
- })
+fn gpu_device_options(engine: &EngineSupervisor) -> Vec<GpuDeviceOption> {
+ // GPU compute devices as the latest worker listed them, so a GPU that
+ // comes back is picked up. `id` is a persistent identity key, never the
+ // process-local registry index. It uses the backend's device_id where
+ // available and its name otherwise (Metal). `total_vram_mb` is 0 when the
+ // backend does not report capacity. Empty if listing failed.
+ transcribe_compute_devices(engine)
+ .unwrap_or_default()
+ .into_iter()
+ .filter(DeviceInfo::is_gpu)
+ .map(|d| GpuDeviceOption {
+ id: d.key.clone(),
+ name: d.label().to_string(),
+ total_vram_mb: (d.memory_total / (1024 * 1024)) as usize,
+ })
+ .collect()
}
#[derive(Serialize, Clone, Debug, Type)]
@@ -2215,7 +2120,7 @@ pub struct AvailableAccelerators {
}
/// Return the accelerators available to this process on its current host.
-pub fn get_available_accelerators() -> AvailableAccelerators {
+pub fn get_available_accelerators(tm: &TranscriptionManager) -> AvailableAccelerators {
use transcribe_rs::accel::OrtAccelerator;
let ort_options: Vec<String> = OrtAccelerator::available()
@@ -2228,7 +2133,7 @@ pub fn get_available_accelerators() -> AvailableAccelerators {
AvailableAccelerators {
transcribe: transcribe_options,
ort: ort_options,
- gpu_devices: cached_gpu_devices().to_vec(),
+ gpu_devices: gpu_device_options(&tm.engine),
}
}
@@ -2265,9 +2170,6 @@ mod tests {
select_transcribe_backend_for_host(TranscribeAcceleratorSetting::Gpu, false),
Backend::Auto
);
- for kind in ["cpu", "accel", "metal", "cuda", "vulkan", "gpu"] {
- assert!(transcribe_device_allowed(kind, false));
- }
}
#[test]
@@ -2287,11 +2189,6 @@ mod tests {
);
}
assert_eq!(available_transcribe_accelerators(true), ["cpu"]);
- assert!(transcribe_device_allowed("cpu", true));
- assert!(transcribe_device_allowed("accel", true));
- for kind in ["metal", "cuda", "vulkan", "gpu", "unknown"] {
- assert!(!transcribe_device_allowed(kind, true));
- }
}
#[test]
@@ -2605,10 +2502,10 @@ impl Drop for TranscriptionManager {
// Skip shutdown unless this is the very last clone. TranscriptionManager
// is cloned by initiate_model_load() and the watcher thread — those
// clones dropping must not kill the watcher. The watcher thread holds
- // its own clone, so engine's strong_count is always >= 2 while the
+ // its own clone, so onnx's strong_count is always >= 2 while the
// watcher is alive. When it reaches 1, only this instance remains
// and we can safely shut down.
- if Arc::strong_count(&self.engine) > 1 {
+ if Arc::strong_count(&self.onnx) > 1 {
return;
}
diff --git a/src-tauri/src/memory.rs b/src-tauri/src/memory.rs
index 2c4f940c..ce01ee31 100644
--- a/src-tauri/src/memory.rs
+++ b/src-tauri/src/memory.rs
@@ -22,7 +22,8 @@
/// Pin glibc's mmap threshold so large transient buffers keep taking the
/// mmap path and are returned to the OS as soon as they are freed.
///
-/// Must run before the workload allocates (called at the top of `run()`);
+/// Must run before the workload allocates (called at the top of `run()`,
+/// and of the transcription worker's own `run()`);
/// the cost is an mmap/munmap round-trip per multi-MB buffer, which is
/// negligible at dictation frequency.
#[cfg(all(target_os = "linux", target_env = "gnu"))]
@@ -39,7 +40,8 @@ pub fn init_allocator() {}
/// Return freed-but-cached malloc arena memory to the OS.
///
-/// Called once per finished transcription pipeline (see `FinishGuard`); it
+/// Called once per finished transcription pipeline (see `FinishGuard`), and
+/// by the transcription worker after each run or stream; it
/// sweeps whatever smaller-than-threshold churn still accumulates in the
/// arenas. Takes on the order of a millisecond, off the main thread.
#[cfg(all(target_os = "linux", target_env = "gnu"))]
diff --git a/src-tauri/src/shortcut/mod.rs b/src-tauri/src/shortcut/mod.rs
index 607ef521..3dbc5e3a 100644
--- a/src-tauri/src/shortcut/mod.rs
+++ b/src-tauri/src/shortcut/mod.rs
@@ -1425,6 +1425,8 @@ pub fn change_transcribe_accelerator_setting(
let mut s = settings::get_settings(&app);
s.transcribe_accelerator = accelerator;
save_accelerator_and_reload_next_use(&app, s);
+ app.state::<std::sync::Arc<crate::managers::transcription::TranscriptionManager>>()
+ .retry_transcribe_gpu("the transcribe.cpp accelerator setting changed");
Ok(())
}
@@ -1446,6 +1448,8 @@ pub fn change_transcribe_gpu_device(app: AppHandle, device: Option<String>) -> R
let mut s = settings::get_settings(&app);
s.transcribe_gpu_device = device;
save_accelerator_and_reload_next_use(&app, s);
+ app.state::<std::sync::Arc<crate::managers::transcription::TranscriptionManager>>()
+ .retry_transcribe_gpu("the transcribe.cpp GPU device setting changed");
Ok(())
}
@@ -1457,10 +1461,16 @@ pub fn change_transcribe_gpu_device(app: AppHandle, device: Option<String>) -> R
/// stays responsive — see also the startup pre-warm in `lib.rs`.
#[tauri::command]
#[specta::specta]
-pub async fn get_available_accelerators() -> crate::managers::transcription::AvailableAccelerators {
- tauri::async_runtime::spawn_blocking(crate::managers::transcription::get_available_accelerators)
- .await
- .expect("get_available_accelerators panicked")
+pub async fn get_available_accelerators(
+ app: AppHandle,
+) -> crate::managers::transcription::AvailableAccelerators {
+ tauri::async_runtime::spawn_blocking(move || {
+ let tm =
+ app.state::<std::sync::Arc<crate::managers::transcription::TranscriptionManager>>();
+ crate::managers::transcription::get_available_accelerators(&tm)
+ })
+ .await
+ .expect("get_available_accelerators panicked")
}
#[cfg(test)]
diff --git a/src-tauri/src/utils.rs b/src-tauri/src/utils.rs
index d80ec2a4..0959c543 100644
--- a/src-tauri/src/utils.rs
+++ b/src-tauri/src/utils.rs
@@ -94,15 +94,19 @@ pub fn cancel_current_operation(app: &AppHandle) {
let recording_was_active = audio_manager.is_recording();
audio_manager.cancel_recording();
- // Abandon any live streaming transcription
+ // Abandon any live streaming transcription, and stop a transcription
+ // already in progress
let tm = app.state::<Arc<TranscriptionManager>>();
tm.cancel_stream();
+ tm.cancel_transcription();
// Update tray icon and hide overlay
set_tray_state(app, crate::tray::TrayIconState::Idle);
hide_recording_overlay(app);
- // Unload model if immediate unload is enabled
+ // Unload model if immediate unload is enabled. Done before notifying the
+ // coordinator, so a recording started next always loads afresh; it
+ // doesn't wait for the cancelled work, so it never blocks this thread.
tm.maybe_unload_immediately("cancellation");
// Notify coordinator so it can keep lifecycle state coherent.