Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
fix(voice): the local Whisper engine loads off the event loop, and a cold load is waited for instead of restarted
rubenluetke10-beep5cffd682392a
diff --git a/jarvis/plugins/stt/fwhisper.py b/jarvis/plugins/stt/fwhisper.pyindex 13507db13a..95e6adc4e9 100644--- a/jarvis/plugins/stt/fwhisper.py+++ b/jarvis/plugins/stt/fwhisper.py@@ -649,6 +649,20 @@ class FasterWhisperProvider:# 2026-06-29) or return garbage. This lock makes the model call mutually# exclusive per instance so the two callers serialize instead of racing.self._infer_lock = threading.Lock()+ # Serialize the model CONSTRUCTION separately from inference. Building a+ # ``WhisperModel`` reads gigabytes of weights and, on CUDA, resolves DLLs+ # and allocates VRAM — a cold large-v3 measured 90+ s (live forensic+ # 2026-08-24). Two callers arriving during that window used to each start+ # their OWN build (nothing guarded ``_ensure_model``), doubling both the+ # wait and the VRAM. A DEDICATED lock (never the inference lock) means the+ # second caller waits for the first engine instead of building a rival+ # one, and it can never deadlock: ``_ensure_model`` only ever takes this+ # lock, so the order is always infer -> load, never the reverse.+ self._load_lock = threading.Lock()+ # True while a build is actually in flight. Callers that supervise the+ # warm-up read it to tell "still loading a cold engine" (wait) apart from+ # "wedged" (replace the instance, AP-24) — see ``is_loading``.+ self._loading = False# One-shot latch for the "auto-detect on an English-only model" warning# below, so a long dictation cannot fill the log with the same line.self._warned_english_only_autodetect = False@@ -671,6 +685,19 @@ class FasterWhisperProvider:"""True when the model is constructed AND primed (safe to poll)."""return self._warm+ @property+ def is_loading(self) -> bool:+ """True while the native engine is being CONSTRUCTED right now.++ A cold local checkpoint takes as long as it takes — 90+ s for large-v3 on+ CUDA with a cold file cache. A supervisor that gives up on a slow warm-up+ and builds a REPLACEMENT instance does not shorten that wait, it restarts+ it from zero (the load cascade that turned a 4 s load into 114.7 s, TTU+ forensic 2026-07-02; and again 2026-08-24 on the dictation path). This+ flag lets the supervisor keep waiting while the engine is demonstrably+ making progress and reserve instance replacement for a real wedge."""+ return self._loading+@propertydef last_used_model(self) -> str:"""Effective local checkpoint used for transcription telemetry."""@@ -698,8 +725,24 @@ class FasterWhisperProvider:self._initial_prompt = normalizeddef _ensure_model(self) -> None:+ """Build the native engine if it is missing. BLOCKING — never call this+ from the event loop; every async entry point ships it to a thread."""if self._model is not None:return+ # Capture the lock locally so a concurrent ``recover()`` swapping in a+ # fresh one still releases THIS one (same reasoning as the infer lock).+ lock = self._load_lock+ with lock:+ if self._model is not None:+ # Another caller finished the build while we waited on the lock.+ return+ self._loading = True+ try:+ self._build_model()+ finally:+ self._loading = False++ def _build_model(self) -> None:model_name = _normalize_model_name(self._model_name)device, compute_type = self._device, self._compute_type# Adopt the boot-prefetched engine when its exact key matches (see the@@ -762,6 +805,12 @@ class FasterWhisperProvider:)self._model = Noneself._infer_lock = threading.Lock()+ # The load lock is swapped for the same reason as the inference one: a+ # thread wedged INSIDE the native build still holds the old one, and the+ # fresh path must not queue up behind it. The orphaned lock dies with the+ # orphaned build if it ever returns.+ self._load_lock = threading.Lock()+ self._loading = Falseself._warm = Falseself._adopted_primed = False# (recover() drops a wedged engine; the next _ensure_model must load@@ -817,7 +866,7 @@ class FasterWhisperProvider:This is enough for Phase 1 — the VAD layer in front already deliversclean utterances, so "in one go" is the natural granularity."""- self._ensure_model()+ await self._ensure_model_async()# Concatenate all chunks into one float32 arraypieces: list[np.ndarray] = []@@ -848,13 +897,34 @@ class FasterWhisperProvider:the same window separates a genuine wake (the unprimed ear stillhears speech) from a prompt echo on noise/breath (it hears nothing)."""- self._ensure_model()+ await self._ensure_model_async()audio_np = pcm_bytes_to_np(pcm_bytes)return await self._transcribe_np(audio_np, sample_rate, language=language,ignore_initial_prompt=ignore_initial_prompt,)+ async def _ensure_model_async(self) -> None:+ """Build the engine off the event loop.++ Constructing a ``WhisperModel`` is a long BLOCKING call: weights off+ disk, DLL resolution and a VRAM allocation on CUDA. Awaiting it inline+ froze the whole backend — every WebSocket frame, HTTP route and brain+ turn queues behind the loop thread, and the caller's own+ ``asyncio.wait_for`` cannot even fire, because a timeout needs a running+ loop to fire ON (live forensic 2026-08-24: the loop watchdog logged a+ 75 s stall with ``_ensure_model`` -> ``WhisperModel`` on the stack, and+ the abandoned build kept the engine busy so the retries all came back+ ``TranscribeBusy`` and the dictation ended with zero characters).++ The fast path — model already built — costs one attribute read and no+ thread hop at all, so the steady state is unchanged."""+ if self._model is not None:+ return+ import asyncio++ await asyncio.to_thread(self._ensure_model)+async def _transcribe_np(self, audio_np: np.ndarray, sample_rate: int,language: str | None = None,diff --git a/jarvis/speech/pipeline.py b/jarvis/speech/pipeline.pyindex a7af9af989..a0b3094152 100644--- a/jarvis/speech/pipeline.py+++ b/jarvis/speech/pipeline.py@@ -947,6 +947,16 @@ _DICTATION_TRANSCRIBE_TIMEOUT_PER_AUDIO_S = 2.0# abandon its whole provider instance rather than racing its native session._DICTATION_WARMUP_JOIN_TIMEOUT_S = 20.0+# ...unless the provider says it is still BUILDING its native engine. That is+# not an unhealthy warmer, it is an honest cold start: a large local checkpoint+# on a cold file cache measured 90+ s (live forensic 2026-08-24, whisper-large-v3+# on CUDA). Replacing the instance there does not shorten the wait — it throws+# the half-built engine away and starts the same load from zero, so the user+# waits twice and the first dictation comes back empty. While the engine reports+# progress we keep waiting, up to this hard ceiling; only silence past it is a+# genuine wedge worth replacing (AP-24).+_DICTATION_WARMUP_COLD_LOAD_TIMEOUT_S = 180.0+def _dictation_retry_worthwhile(exc: BaseException | None) -> bool:"""Whether re-sending the SAME audio to this provider could work.@@ -10554,6 +10564,58 @@ class SpeechPipeline:name="dictation-stt-warmup",)+ async def _await_warmup_or_cold_load(+ self, task: asyncio.Task, provider: Any+ ) -> None:+ """Join a warm-up, staying patient while its engine is still building.++ Raises ``TimeoutError`` exactly like a bare ``wait_for`` would, so the+ caller's wedge handling is unchanged — but only after the provider has+ stopped making progress. A provider that does not expose ``is_loading``+ (every cloud one) keeps the plain short join it has always had.+ """+ try:+ await asyncio.wait_for(+ asyncio.shield(task), timeout=_DICTATION_WARMUP_JOIN_TIMEOUT_S+ )+ return+ except TimeoutError:+ if not getattr(provider, "is_loading", False):+ raise+ # Still constructing the native engine. Keep joining in short hops so a+ # build that finishes — or stalls — is noticed promptly either way. The+ # load gets the long ceiling; once it ends, whatever the warm-up still+ # owes (its priming inference) gets a fresh short one, because that part+ # IS bounded work and a hang there is the wedge the caller handles.+ log.info(+ "Dictation STT is still building its local engine after %.0fs; "+ "waiting for it instead of restarting the load from zero.",+ _DICTATION_WARMUP_JOIN_TIMEOUT_S,+ )+ load_deadline = (+ time.monotonic()+ + _DICTATION_WARMUP_COLD_LOAD_TIMEOUT_S+ - _DICTATION_WARMUP_JOIN_TIMEOUT_S+ )+ prime_deadline: float | None = None+ while True:+ deadline = load_deadline if prime_deadline is None else prime_deadline+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ raise TimeoutError+ try:+ await asyncio.wait_for(+ asyncio.shield(task), timeout=min(2.0, remaining)+ )+ return+ except TimeoutError:+ if prime_deadline is None and not getattr(+ provider, "is_loading", False+ ):+ prime_deadline = (+ time.monotonic() + _DICTATION_WARMUP_JOIN_TIMEOUT_S+ )+async def _join_dictation_warmup(self, provider: Any) -> Any:"""Join matching warm-up before authoritative STT, returning its provider.@@ -10593,9 +10655,7 @@ class SpeechPipeline:):return providertry:- await asyncio.wait_for(- asyncio.shield(task), timeout=_DICTATION_WARMUP_JOIN_TIMEOUT_S- )+ await self._await_warmup_or_cold_load(task, provider)return providerexcept TimeoutError:log.warning(diff --git a/tests/unit/dictation/test_warmup_cold_load_patience.py b/tests/unit/dictation/test_warmup_cold_load_patience.pynew file mode 100644index 0000000000..8e36b6b53b--- /dev/null+++ b/tests/unit/dictation/test_warmup_cold_load_patience.py@@ -0,0 +1,111 @@+"""A dictation warm-up that is still BUILDING its local engine must be waited+for, not replaced.++Live forensic 2026-08-24 (local models, first dictation after a restart): the+join gave up after 20 s with "warm-up did not finish; replacing the provider+instance", built a second instance, and that one restarted the same cold+whisper-large-v3 load from zero. Replacing an instance never shortens a load —+it repeats it. The user waited through both and the dictation still came back+with zero characters.++The rule pinned here: patience is granted only on evidence. A provider that+reports ``is_loading`` gets the long ceiling while it says so; a provider that+does not report it (every cloud one) keeps the short join it always had; and a+build that ENDS without the warm-up finishing falls back to the short join,+because a hang in the priming inference is a real wedge (AP-24).+"""+from __future__ import annotations++import asyncio++import pytest++from jarvis.speech import pipeline as pipe+from jarvis.speech.pipeline import SpeechPipeline+++class _LocalProvider:+ """A local engine that reports its build honestly."""++ def __init__(self) -> None:+ self.is_loading = True+++class _CloudProvider:+ """No ``is_loading`` at all — the contract every cloud provider meets."""+++async def _join(task: asyncio.Task, provider: object) -> None:+ # The method only reads module constants, so an unbound call is enough and+ # keeps the test off the SpeechPipeline constructor's whole dependency tree.+ await SpeechPipeline._await_warmup_or_cold_load(object(), task, provider) # noqa: SLF001++[email protected](autouse=True)+def _fast_timeouts(monkeypatch: pytest.MonkeyPatch) -> None:+ # Same logic, test-sized clocks: a 20 s / 180 s pair would make this a+ # three-minute test.+ monkeypatch.setattr(pipe, "_DICTATION_WARMUP_JOIN_TIMEOUT_S", 0.1)+ monkeypatch.setattr(pipe, "_DICTATION_WARMUP_COLD_LOAD_TIMEOUT_S", 5.0)+++async def test_a_still_loading_engine_is_waited_for() -> None:+ provider = _LocalProvider()++ async def _warm() -> None:+ await asyncio.sleep(0.5) # 5x the short join — a cold load, in miniature+ provider.is_loading = False++ task = asyncio.create_task(_warm())+ await _join(task, provider) # must NOT raise: the engine was making progress++ assert task.done() and not task.cancelled(), (+ "the warm-up was abandoned while its engine was still building — the "+ "replacement would restart the same cold load from zero"+ )+++async def test_a_cloud_provider_keeps_the_short_join() -> None:+ async def _warm() -> None:+ await asyncio.sleep(5.0)++ task = asyncio.create_task(_warm())+ try:+ with pytest.raises(TimeoutError):+ await _join(task, _CloudProvider())+ finally:+ task.cancel()+++async def test_a_hang_after_the_build_is_still_a_wedge() -> None:+ # The engine finished building, but the warm-up never returns: that is the+ # priming inference hanging — the wedge the caller must replace.+ provider = _LocalProvider()++ async def _warm() -> None:+ await asyncio.sleep(0.2)+ provider.is_loading = False # build done...+ await asyncio.sleep(30.0) # ...priming hangs forever++ task = asyncio.create_task(_warm())+ try:+ with pytest.raises(TimeoutError):+ await _join(task, provider)+ finally:+ task.cancel()+++async def test_an_endless_build_eventually_gives_up() -> None:+ # is_loading stuck True forever is still a wedge, just a patient one: the+ # cold-load ceiling has to end the wait rather than hanging the dictation.+ provider = _LocalProvider()++ async def _warm() -> None:+ await asyncio.sleep(30.0)++ task = asyncio.create_task(_warm())+ try:+ with pytest.raises(TimeoutError):+ await _join(task, provider)+ finally:+ task.cancel()diff --git a/tests/unit/plugins/stt/test_fwhisper_model_load_offloop.py b/tests/unit/plugins/stt/test_fwhisper_model_load_offloop.pynew file mode 100644index 0000000000..917ccaa126--- /dev/null+++ b/tests/unit/plugins/stt/test_fwhisper_model_load_offloop.py@@ -0,0 +1,143 @@+"""Building the native Whisper engine must never block the event loop, and two+callers arriving during a cold load must share ONE build.++Live forensic 2026-08-24 (dictation on local models, right after a restart): the+loop watchdog logged a 75 s stall whose stack was+``transcribe_pcm`` -> ``_ensure_model`` -> ``WhisperModel(...)``. ``transcribe_pcm``+is a coroutine, so awaiting the blocking construction inline froze the whole+backend: every WebSocket frame, HTTP route and brain turn queued behind the loop+thread, and the caller's own ``asyncio.wait_for`` could not fire either — a+timeout needs a running loop to fire ON. The abandoned build kept the engine+busy, so the retries came back ``TranscribeBusy`` and the dictation ended with+zero characters.++Three contracts pinned here:+ 1. The construction runs OFF the loop, so the loop keeps ticking during it.+ 2. A concurrent second caller waits for the first engine instead of building a+ rival one (doubling both the wait and the VRAM).+ 3. ``is_loading`` reports the build honestly, so a warm-up supervisor can tell+ "cold start" apart from "wedged" instead of restarting the load from zero.+"""+from __future__ import annotations++import asyncio+import threading+import time+from types import SimpleNamespace++from jarvis.plugins.stt import fwhisper+from jarvis.plugins.stt.fwhisper import FasterWhisperProvider++PCM_1S = b"\x00\x00" * 16_000++# Long enough that an on-loop build is unmistakable, short enough to stay a unit+# test: the loop ticks every 10 ms below, so a free loop racks up dozens of them.+_BUILD_SECONDS = 0.5+++class _StubModel:+ """Stands in for a built WhisperModel — returns an empty transcript."""++ def transcribe(self, audio, **kwargs): # noqa: ANN001, ANN003+ return iter(()), SimpleNamespace(language="de")+++async def test_model_build_does_not_block_the_event_loop(monkeypatch) -> None:+ def _slow_build(*_a, **_kw):+ time.sleep(_BUILD_SECONDS) # a cold ctranslate2 load, in miniature+ return _StubModel()++ monkeypatch.setattr(fwhisper, "_new_whisper_model", _slow_build)+ prov = FasterWhisperProvider(device="cpu", compute_type="int8")++ ticks = 0++ async def _heartbeat() -> None:+ # Stands in for every WebSocket frame, HTTP route and brain turn: work+ # the loop owes other callers while the engine loads.+ nonlocal ticks+ while True:+ await asyncio.sleep(0.01)+ ticks += 1++ beat = asyncio.create_task(_heartbeat())+ try:+ await prov.transcribe_pcm(PCM_1S)+ finally:+ beat.cancel()++ assert ticks >= 10, (+ f"the event loop only ticked {ticks} times during a "+ f"{_BUILD_SECONDS}s model build — the native load is blocking the loop "+ "again (the 75 s backend freeze of 2026-08-24)"+ )+++async def test_a_concurrent_caller_shares_one_build(monkeypatch) -> None:+ builds = 0+ build_lock = threading.Lock()++ def _counting_build(*_a, **_kw):+ nonlocal builds+ with build_lock:+ builds += 1+ time.sleep(_BUILD_SECONDS)+ return _StubModel()++ monkeypatch.setattr(fwhisper, "_new_whisper_model", _counting_build)+ prov = FasterWhisperProvider(device="cpu", compute_type="int8")++ # Both arrive while the engine is still cold — the wake poll loop and the+ # dictation final pass in production.+ await asyncio.gather(+ prov.transcribe_pcm(PCM_1S),+ prov.transcribe_pcm(PCM_1S),+ return_exceptions=True, # one may lose the inference lock; that is fine+ )++ assert builds == 1, (+ f"{builds} engines were built for one provider — a concurrent caller "+ "started a rival load instead of waiting for the first"+ )+++async def test_is_loading_is_true_only_while_building(monkeypatch) -> None:+ building = threading.Event()+ release = threading.Event()++ def _held_build(*_a, **_kw):+ building.set()+ release.wait(timeout=5.0)+ return _StubModel()++ monkeypatch.setattr(fwhisper, "_new_whisper_model", _held_build)+ prov = FasterWhisperProvider(device="cpu", compute_type="int8")++ assert prov.is_loading is False, "a provider that never loaded is not loading"+ call = asyncio.create_task(prov.transcribe_pcm(PCM_1S))+ try:+ await asyncio.to_thread(building.wait, 5.0)+ assert prov.is_loading is True, (+ "is_loading stayed False during the build — a warm-up supervisor "+ "cannot tell a cold start from a wedge and will restart the load"+ )+ finally:+ release.set()+ await call++ assert prov.is_loading is False, "is_loading must clear once the build ends"+++def test_recover_frees_the_load_lock_too() -> None:+ # A thread wedged INSIDE the native build still holds the old load lock;+ # the fresh path must not queue up behind it (AP-24).+ prov = FasterWhisperProvider(device="cpu", compute_type="int8")+ old_lock = prov._load_lock # noqa: SLF001+ old_lock.acquire() # a wedged build "holds" it++ prov.recover()++ assert prov._load_lock is not old_lock, "recover() must swap in a fresh load lock" # noqa: SLF001+ assert prov._load_lock.acquire(blocking=False) is True # noqa: SLF001+ prov._load_lock.release() # noqa: SLF001+ old_lock.release()