Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(voice): the local Whisper engine loads off the event loop, and a cold load is waited for instead of restarted

rubenluetke10-beep5cffd682392a
diff --git a/jarvis/plugins/stt/fwhisper.py b/jarvis/plugins/stt/fwhisper.py
index 13507db13a..95e6adc4e9 100644
--- a/jarvis/plugins/stt/fwhisper.py
+++ b/jarvis/plugins/stt/fwhisper.py
@@ -649,6 +649,20 @@ class FasterWhisperProvider:
# 2026-06-29) or return garbage. This lock makes the model call mutually
# exclusive per instance so the two callers serialize instead of racing.
self._infer_lock = threading.Lock()
+ # Serialize the model CONSTRUCTION separately from inference. Building a
+ # ``WhisperModel`` reads gigabytes of weights and, on CUDA, resolves DLLs
+ # and allocates VRAM — a cold large-v3 measured 90+ s (live forensic
+ # 2026-08-24). Two callers arriving during that window used to each start
+ # their OWN build (nothing guarded ``_ensure_model``), doubling both the
+ # wait and the VRAM. A DEDICATED lock (never the inference lock) means the
+ # second caller waits for the first engine instead of building a rival
+ # one, and it can never deadlock: ``_ensure_model`` only ever takes this
+ # lock, so the order is always infer -> load, never the reverse.
+ self._load_lock = threading.Lock()
+ # True while a build is actually in flight. Callers that supervise the
+ # warm-up read it to tell "still loading a cold engine" (wait) apart from
+ # "wedged" (replace the instance, AP-24) — see ``is_loading``.
+ self._loading = False
# One-shot latch for the "auto-detect on an English-only model" warning
# below, so a long dictation cannot fill the log with the same line.
self._warned_english_only_autodetect = False
@@ -671,6 +685,19 @@ class FasterWhisperProvider:
"""True when the model is constructed AND primed (safe to poll)."""
return self._warm
+ @property
+ def is_loading(self) -> bool:
+ """True while the native engine is being CONSTRUCTED right now.
+
+ A cold local checkpoint takes as long as it takes — 90+ s for large-v3 on
+ CUDA with a cold file cache. A supervisor that gives up on a slow warm-up
+ and builds a REPLACEMENT instance does not shorten that wait, it restarts
+ it from zero (the load cascade that turned a 4 s load into 114.7 s, TTU
+ forensic 2026-07-02; and again 2026-08-24 on the dictation path). This
+ flag lets the supervisor keep waiting while the engine is demonstrably
+ making progress and reserve instance replacement for a real wedge."""
+ return self._loading
+
@property
def last_used_model(self) -> str:
"""Effective local checkpoint used for transcription telemetry."""
@@ -698,8 +725,24 @@ class FasterWhisperProvider:
self._initial_prompt = normalized
def _ensure_model(self) -> None:
+ """Build the native engine if it is missing. BLOCKING — never call this
+ from the event loop; every async entry point ships it to a thread."""
if self._model is not None:
return
+ # Capture the lock locally so a concurrent ``recover()`` swapping in a
+ # fresh one still releases THIS one (same reasoning as the infer lock).
+ lock = self._load_lock
+ with lock:
+ if self._model is not None:
+ # Another caller finished the build while we waited on the lock.
+ return
+ self._loading = True
+ try:
+ self._build_model()
+ finally:
+ self._loading = False
+
+ def _build_model(self) -> None:
model_name = _normalize_model_name(self._model_name)
device, compute_type = self._device, self._compute_type
# Adopt the boot-prefetched engine when its exact key matches (see the
@@ -762,6 +805,12 @@ class FasterWhisperProvider:
)
self._model = None
self._infer_lock = threading.Lock()
+ # The load lock is swapped for the same reason as the inference one: a
+ # thread wedged INSIDE the native build still holds the old one, and the
+ # fresh path must not queue up behind it. The orphaned lock dies with the
+ # orphaned build if it ever returns.
+ self._load_lock = threading.Lock()
+ self._loading = False
self._warm = False
self._adopted_primed = False
# (recover() drops a wedged engine; the next _ensure_model must load
@@ -817,7 +866,7 @@ class FasterWhisperProvider:
This is enough for Phase 1 — the VAD layer in front already delivers
clean utterances, so "in one go" is the natural granularity.
"""
- self._ensure_model()
+ await self._ensure_model_async()
# Concatenate all chunks into one float32 array
pieces: list[np.ndarray] = []
@@ -848,13 +897,34 @@ class FasterWhisperProvider:
the same window separates a genuine wake (the unprimed ear still
hears speech) from a prompt echo on noise/breath (it hears nothing).
"""
- self._ensure_model()
+ await self._ensure_model_async()
audio_np = pcm_bytes_to_np(pcm_bytes)
return await self._transcribe_np(
audio_np, sample_rate, language=language,
ignore_initial_prompt=ignore_initial_prompt,
)
+ async def _ensure_model_async(self) -> None:
+ """Build the engine off the event loop.
+
+ Constructing a ``WhisperModel`` is a long BLOCKING call: weights off
+ disk, DLL resolution and a VRAM allocation on CUDA. Awaiting it inline
+ froze the whole backend — every WebSocket frame, HTTP route and brain
+ turn queues behind the loop thread, and the caller's own
+ ``asyncio.wait_for`` cannot even fire, because a timeout needs a running
+ loop to fire ON (live forensic 2026-08-24: the loop watchdog logged a
+ 75 s stall with ``_ensure_model`` -> ``WhisperModel`` on the stack, and
+ the abandoned build kept the engine busy so the retries all came back
+ ``TranscribeBusy`` and the dictation ended with zero characters).
+
+ The fast path — model already built — costs one attribute read and no
+ thread hop at all, so the steady state is unchanged."""
+ if self._model is not None:
+ return
+ import asyncio
+
+ await asyncio.to_thread(self._ensure_model)
+
async def _transcribe_np(
self, audio_np: np.ndarray, sample_rate: int,
language: str | None = None,
diff --git a/jarvis/speech/pipeline.py b/jarvis/speech/pipeline.py
index a7af9af989..a0b3094152 100644
--- a/jarvis/speech/pipeline.py
+++ b/jarvis/speech/pipeline.py
@@ -947,6 +947,16 @@ _DICTATION_TRANSCRIBE_TIMEOUT_PER_AUDIO_S = 2.0
# abandon its whole provider instance rather than racing its native session.
_DICTATION_WARMUP_JOIN_TIMEOUT_S = 20.0
+# ...unless the provider says it is still BUILDING its native engine. That is
+# not an unhealthy warmer, it is an honest cold start: a large local checkpoint
+# on a cold file cache measured 90+ s (live forensic 2026-08-24, whisper-large-v3
+# on CUDA). Replacing the instance there does not shorten the wait — it throws
+# the half-built engine away and starts the same load from zero, so the user
+# waits twice and the first dictation comes back empty. While the engine reports
+# progress we keep waiting, up to this hard ceiling; only silence past it is a
+# genuine wedge worth replacing (AP-24).
+_DICTATION_WARMUP_COLD_LOAD_TIMEOUT_S = 180.0
+
def _dictation_retry_worthwhile(exc: BaseException | None) -> bool:
"""Whether re-sending the SAME audio to this provider could work.
@@ -10554,6 +10564,58 @@ class SpeechPipeline:
name="dictation-stt-warmup",
)
+ async def _await_warmup_or_cold_load(
+ self, task: asyncio.Task, provider: Any
+ ) -> None:
+ """Join a warm-up, staying patient while its engine is still building.
+
+ Raises ``TimeoutError`` exactly like a bare ``wait_for`` would, so the
+ caller's wedge handling is unchanged — but only after the provider has
+ stopped making progress. A provider that does not expose ``is_loading``
+ (every cloud one) keeps the plain short join it has always had.
+ """
+ try:
+ await asyncio.wait_for(
+ asyncio.shield(task), timeout=_DICTATION_WARMUP_JOIN_TIMEOUT_S
+ )
+ return
+ except TimeoutError:
+ if not getattr(provider, "is_loading", False):
+ raise
+ # Still constructing the native engine. Keep joining in short hops so a
+ # build that finishes — or stalls — is noticed promptly either way. The
+ # load gets the long ceiling; once it ends, whatever the warm-up still
+ # owes (its priming inference) gets a fresh short one, because that part
+ # IS bounded work and a hang there is the wedge the caller handles.
+ log.info(
+ "Dictation STT is still building its local engine after %.0fs; "
+ "waiting for it instead of restarting the load from zero.",
+ _DICTATION_WARMUP_JOIN_TIMEOUT_S,
+ )
+ load_deadline = (
+ time.monotonic()
+ + _DICTATION_WARMUP_COLD_LOAD_TIMEOUT_S
+ - _DICTATION_WARMUP_JOIN_TIMEOUT_S
+ )
+ prime_deadline: float | None = None
+ while True:
+ deadline = load_deadline if prime_deadline is None else prime_deadline
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise TimeoutError
+ try:
+ await asyncio.wait_for(
+ asyncio.shield(task), timeout=min(2.0, remaining)
+ )
+ return
+ except TimeoutError:
+ if prime_deadline is None and not getattr(
+ provider, "is_loading", False
+ ):
+ prime_deadline = (
+ time.monotonic() + _DICTATION_WARMUP_JOIN_TIMEOUT_S
+ )
+
async def _join_dictation_warmup(self, provider: Any) -> Any:
"""Join matching warm-up before authoritative STT, returning its provider.
@@ -10593,9 +10655,7 @@ class SpeechPipeline:
):
return provider
try:
- await asyncio.wait_for(
- asyncio.shield(task), timeout=_DICTATION_WARMUP_JOIN_TIMEOUT_S
- )
+ await self._await_warmup_or_cold_load(task, provider)
return provider
except TimeoutError:
log.warning(
diff --git a/tests/unit/dictation/test_warmup_cold_load_patience.py b/tests/unit/dictation/test_warmup_cold_load_patience.py
new file mode 100644
index 0000000000..8e36b6b53b
--- /dev/null
+++ b/tests/unit/dictation/test_warmup_cold_load_patience.py
@@ -0,0 +1,111 @@
+"""A dictation warm-up that is still BUILDING its local engine must be waited
+for, not replaced.
+
+Live forensic 2026-08-24 (local models, first dictation after a restart): the
+join gave up after 20 s with "warm-up did not finish; replacing the provider
+instance", built a second instance, and that one restarted the same cold
+whisper-large-v3 load from zero. Replacing an instance never shortens a load —
+it repeats it. The user waited through both and the dictation still came back
+with zero characters.
+
+The rule pinned here: patience is granted only on evidence. A provider that
+reports ``is_loading`` gets the long ceiling while it says so; a provider that
+does not report it (every cloud one) keeps the short join it always had; and a
+build that ENDS without the warm-up finishing falls back to the short join,
+because a hang in the priming inference is a real wedge (AP-24).
+"""
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+
+from jarvis.speech import pipeline as pipe
+from jarvis.speech.pipeline import SpeechPipeline
+
+
+class _LocalProvider:
+ """A local engine that reports its build honestly."""
+
+ def __init__(self) -> None:
+ self.is_loading = True
+
+
+class _CloudProvider:
+ """No ``is_loading`` at all — the contract every cloud provider meets."""
+
+
+async def _join(task: asyncio.Task, provider: object) -> None:
+ # The method only reads module constants, so an unbound call is enough and
+ # keeps the test off the SpeechPipeline constructor's whole dependency tree.
+ await SpeechPipeline._await_warmup_or_cold_load(object(), task, provider) # noqa: SLF001
+
+
[email protected](autouse=True)
+def _fast_timeouts(monkeypatch: pytest.MonkeyPatch) -> None:
+ # Same logic, test-sized clocks: a 20 s / 180 s pair would make this a
+ # three-minute test.
+ monkeypatch.setattr(pipe, "_DICTATION_WARMUP_JOIN_TIMEOUT_S", 0.1)
+ monkeypatch.setattr(pipe, "_DICTATION_WARMUP_COLD_LOAD_TIMEOUT_S", 5.0)
+
+
+async def test_a_still_loading_engine_is_waited_for() -> None:
+ provider = _LocalProvider()
+
+ async def _warm() -> None:
+ await asyncio.sleep(0.5) # 5x the short join — a cold load, in miniature
+ provider.is_loading = False
+
+ task = asyncio.create_task(_warm())
+ await _join(task, provider) # must NOT raise: the engine was making progress
+
+ assert task.done() and not task.cancelled(), (
+ "the warm-up was abandoned while its engine was still building — the "
+ "replacement would restart the same cold load from zero"
+ )
+
+
+async def test_a_cloud_provider_keeps_the_short_join() -> None:
+ async def _warm() -> None:
+ await asyncio.sleep(5.0)
+
+ task = asyncio.create_task(_warm())
+ try:
+ with pytest.raises(TimeoutError):
+ await _join(task, _CloudProvider())
+ finally:
+ task.cancel()
+
+
+async def test_a_hang_after_the_build_is_still_a_wedge() -> None:
+ # The engine finished building, but the warm-up never returns: that is the
+ # priming inference hanging — the wedge the caller must replace.
+ provider = _LocalProvider()
+
+ async def _warm() -> None:
+ await asyncio.sleep(0.2)
+ provider.is_loading = False # build done...
+ await asyncio.sleep(30.0) # ...priming hangs forever
+
+ task = asyncio.create_task(_warm())
+ try:
+ with pytest.raises(TimeoutError):
+ await _join(task, provider)
+ finally:
+ task.cancel()
+
+
+async def test_an_endless_build_eventually_gives_up() -> None:
+ # is_loading stuck True forever is still a wedge, just a patient one: the
+ # cold-load ceiling has to end the wait rather than hanging the dictation.
+ provider = _LocalProvider()
+
+ async def _warm() -> None:
+ await asyncio.sleep(30.0)
+
+ task = asyncio.create_task(_warm())
+ try:
+ with pytest.raises(TimeoutError):
+ await _join(task, provider)
+ finally:
+ task.cancel()
diff --git a/tests/unit/plugins/stt/test_fwhisper_model_load_offloop.py b/tests/unit/plugins/stt/test_fwhisper_model_load_offloop.py
new file mode 100644
index 0000000000..917ccaa126
--- /dev/null
+++ b/tests/unit/plugins/stt/test_fwhisper_model_load_offloop.py
@@ -0,0 +1,143 @@
+"""Building the native Whisper engine must never block the event loop, and two
+callers arriving during a cold load must share ONE build.
+
+Live forensic 2026-08-24 (dictation on local models, right after a restart): the
+loop watchdog logged a 75 s stall whose stack was
+``transcribe_pcm`` -> ``_ensure_model`` -> ``WhisperModel(...)``. ``transcribe_pcm``
+is a coroutine, so awaiting the blocking construction inline froze the whole
+backend: every WebSocket frame, HTTP route and brain turn queued behind the loop
+thread, and the caller's own ``asyncio.wait_for`` could not fire either — a
+timeout needs a running loop to fire ON. The abandoned build kept the engine
+busy, so the retries came back ``TranscribeBusy`` and the dictation ended with
+zero characters.
+
+Three contracts pinned here:
+ 1. The construction runs OFF the loop, so the loop keeps ticking during it.
+ 2. A concurrent second caller waits for the first engine instead of building a
+ rival one (doubling both the wait and the VRAM).
+ 3. ``is_loading`` reports the build honestly, so a warm-up supervisor can tell
+ "cold start" apart from "wedged" instead of restarting the load from zero.
+"""
+from __future__ import annotations
+
+import asyncio
+import threading
+import time
+from types import SimpleNamespace
+
+from jarvis.plugins.stt import fwhisper
+from jarvis.plugins.stt.fwhisper import FasterWhisperProvider
+
+PCM_1S = b"\x00\x00" * 16_000
+
+# Long enough that an on-loop build is unmistakable, short enough to stay a unit
+# test: the loop ticks every 10 ms below, so a free loop racks up dozens of them.
+_BUILD_SECONDS = 0.5
+
+
+class _StubModel:
+ """Stands in for a built WhisperModel — returns an empty transcript."""
+
+ def transcribe(self, audio, **kwargs): # noqa: ANN001, ANN003
+ return iter(()), SimpleNamespace(language="de")
+
+
+async def test_model_build_does_not_block_the_event_loop(monkeypatch) -> None:
+ def _slow_build(*_a, **_kw):
+ time.sleep(_BUILD_SECONDS) # a cold ctranslate2 load, in miniature
+ return _StubModel()
+
+ monkeypatch.setattr(fwhisper, "_new_whisper_model", _slow_build)
+ prov = FasterWhisperProvider(device="cpu", compute_type="int8")
+
+ ticks = 0
+
+ async def _heartbeat() -> None:
+ # Stands in for every WebSocket frame, HTTP route and brain turn: work
+ # the loop owes other callers while the engine loads.
+ nonlocal ticks
+ while True:
+ await asyncio.sleep(0.01)
+ ticks += 1
+
+ beat = asyncio.create_task(_heartbeat())
+ try:
+ await prov.transcribe_pcm(PCM_1S)
+ finally:
+ beat.cancel()
+
+ assert ticks >= 10, (
+ f"the event loop only ticked {ticks} times during a "
+ f"{_BUILD_SECONDS}s model build — the native load is blocking the loop "
+ "again (the 75 s backend freeze of 2026-08-24)"
+ )
+
+
+async def test_a_concurrent_caller_shares_one_build(monkeypatch) -> None:
+ builds = 0
+ build_lock = threading.Lock()
+
+ def _counting_build(*_a, **_kw):
+ nonlocal builds
+ with build_lock:
+ builds += 1
+ time.sleep(_BUILD_SECONDS)
+ return _StubModel()
+
+ monkeypatch.setattr(fwhisper, "_new_whisper_model", _counting_build)
+ prov = FasterWhisperProvider(device="cpu", compute_type="int8")
+
+ # Both arrive while the engine is still cold — the wake poll loop and the
+ # dictation final pass in production.
+ await asyncio.gather(
+ prov.transcribe_pcm(PCM_1S),
+ prov.transcribe_pcm(PCM_1S),
+ return_exceptions=True, # one may lose the inference lock; that is fine
+ )
+
+ assert builds == 1, (
+ f"{builds} engines were built for one provider — a concurrent caller "
+ "started a rival load instead of waiting for the first"
+ )
+
+
+async def test_is_loading_is_true_only_while_building(monkeypatch) -> None:
+ building = threading.Event()
+ release = threading.Event()
+
+ def _held_build(*_a, **_kw):
+ building.set()
+ release.wait(timeout=5.0)
+ return _StubModel()
+
+ monkeypatch.setattr(fwhisper, "_new_whisper_model", _held_build)
+ prov = FasterWhisperProvider(device="cpu", compute_type="int8")
+
+ assert prov.is_loading is False, "a provider that never loaded is not loading"
+ call = asyncio.create_task(prov.transcribe_pcm(PCM_1S))
+ try:
+ await asyncio.to_thread(building.wait, 5.0)
+ assert prov.is_loading is True, (
+ "is_loading stayed False during the build — a warm-up supervisor "
+ "cannot tell a cold start from a wedge and will restart the load"
+ )
+ finally:
+ release.set()
+ await call
+
+ assert prov.is_loading is False, "is_loading must clear once the build ends"
+
+
+def test_recover_frees_the_load_lock_too() -> None:
+ # A thread wedged INSIDE the native build still holds the old load lock;
+ # the fresh path must not queue up behind it (AP-24).
+ prov = FasterWhisperProvider(device="cpu", compute_type="int8")
+ old_lock = prov._load_lock # noqa: SLF001
+ old_lock.acquire() # a wedged build "holds" it
+
+ prov.recover()
+
+ assert prov._load_lock is not old_lock, "recover() must swap in a fresh load lock" # noqa: SLF001
+ assert prov._load_lock.acquire(blocking=False) is True # noqa: SLF001
+ prov._load_lock.release() # noqa: SLF001
+ old_lock.release()