Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
fix(dictation): the GPU preview stops rebuilding itself every seven seconds
rubenluetke10-beep3d2a3761e93d
diff --git a/jarvis/dictation/local_preview.py b/jarvis/dictation/local_preview.pyindex 27c42e0209..cbd6cbcbb1 100644--- a/jarvis/dictation/local_preview.py+++ b/jarvis/dictation/local_preview.py@@ -36,6 +36,7 @@ from __future__ import annotationsimport asyncioimport loggingimport threading+import timefrom typing import Anylog = logging.getLogger(__name__)@@ -55,6 +56,21 @@ PREVIEW_TIMEOUT_S = 2.0#: use. A native engine that has wedged never recovers by being asked again._MAX_FAILURES = 3+#: The clip the freshly built engine decodes before it is advertised — four+#: seconds of near-silence, the shape of an ordinary preview tail. Its timing+#: is logged, never judged: the first CUDA decode in a process pays context+#: init and, on a cold driver cache, a one-off kernel compile measured at 19 s,+#: so a clock here would reject a perfectly good GPU on its first day.+#:+#: What IS judged is the live tick. Measured 2026-08-22 on a Blackwell card+#: (ctranslate2 4.8): ``int8_float16`` decoded 8 s of real speech in 10-11 s,+#: ``float16`` in 0.7 s — so the engine timed out on every tick, was dropped,+#: rebuilt with the SAME settings and dropped again, every 7 s for the whole+#: dictation. A drop therefore now advances to the next engine in the attempt+#: list (``_load_model``) instead of rebuilding the one that just failed, and+#: stays on the CPU floor once it gets there.+_WARM_CLIP_S = 4.0+def faster_whisper_available() -> bool:"""Is a local engine importable here? Cheap spec probe, no heavy import."""@@ -78,6 +94,11 @@ class LocalPreviewTranscriber:self._model: Any = Noneself._lock = threading.Lock()self._busy = threading.Lock()+ # Which entry of the attempt list the next build starts from, and what+ # the current engine runs on (for the log line and the status card).+ self._attempt_index = 0+ self._engine_device = ""+ self._engine_compute = ""self._failures = 0self._unavailable = Falseself._loading = False@@ -110,8 +131,17 @@ class LocalPreviewTranscriber:preferred = self._pick_device()attempts = [preferred]+ if preferred == ("cuda", "float16"):+ # The quantized pair is the old GPU default and still the+ # right answer on a GPU where it is fast; tried second so a+ # host that cannot do float16 keeps its GPU preview.+ attempts.append(("cuda", "int8_float16"))if preferred[0] != "cpu":attempts.append(("cpu", "int8"))+ # A rebuild after repeated live failures starts one entry further+ # down: the engine that just kept timing out is not asked again.+ skip = min(self._attempt_index, len(attempts) - 1)+ attempts = attempts[skip:]last_error: Exception | None = Nonefor device, compute in attempts:try:@@ -129,13 +159,20 @@ class LocalPreviewTranscriber:# Constructing the model does not pay CUDA's first-decode# setup cost. Prime one throwaway greedy decode here, while# the preview is still advertised as unavailable, so the- # first visible preview keeps the steady-state latency.+ # first visible preview keeps the steady-state latency —+ # and TIME it: a GPU engine that cannot decode a preview-+ # sized clip well inside the tick ceiling is not a preview+ # engine, whatever the constructor said.import numpy as nprng = np.random.default_rng(0)warm_audio = (- rng.standard_normal(16_000).astype(np.float32) * 0.001+ rng.standard_normal(int(16_000 * _WARM_CLIP_S)).astype(+ np.float32+ )+ * 0.001)+ started = time.perf_counter()segments, _info = model.transcribe(warm_audio,beam_size=1,@@ -143,6 +180,7 @@ class LocalPreviewTranscriber:condition_on_previous_text=False,)list(segments)+ warm_s = time.perf_counter() - startedexcept Exception as exc: # noqa: BLE001 — try the portable floorlast_error = exclog.info(@@ -155,11 +193,16 @@ class LocalPreviewTranscriber:continuewith self._lock:self._model = model+ self._engine_device = device+ self._engine_compute = computelog.info(- "Dictation preview engine ready: %s on %s (%s).",+ "Dictation preview engine ready: %s on %s (%s), %.0fs clip "+ "in %.2fs.",self._model_name,device,compute,+ _WARM_CLIP_S,+ warm_s,)breakelse:@@ -195,7 +238,15 @@ class LocalPreviewTranscriber:with inference_only_import_shield():import ctranslate2- supported = ctranslate2.get_supported_compute_types("cuda")+ supported = set(ctranslate2.get_supported_compute_types("cuda"))+ # Plain half precision first: on the one GPU measured so far the+ # quantized int8 pairs were an order of magnitude SLOWER than+ # float16 (10-11 s against 0.7 s for 8 s of speech, ctranslate2+ # 4.8 on a Blackwell card) — the preview model is tiny, so the+ # memory int8 saves buys nothing here. The warm-decode gate in+ # ``_load_model`` is the backstop for the next surprise.+ if "float16" in supported:+ return "cuda", "float16"if "int8_float16" in supported:return "cuda", "int8_float16"except Exception as exc: # noqa: BLE001 — a probe must never decide by raising@@ -311,6 +362,11 @@ class LocalPreviewTranscriber:passexcept Exception as exc: # noqa: BLE001 — detached worker is containedlog.debug("Detached dictation preview failed: %s", exc)+ else:+ # It came back late, but it came back: a slow engine is+ # not a wedged one, and the streak that would have+ # dropped it is over.+ self._failures = 0work.add_done_callback(_release_finished_worker)if release_here:@@ -350,8 +406,19 @@ class LocalPreviewTranscriber:# fresh session rather than re-polling a wedged engine (AP-24).self._model = Noneself._busy = threading.Lock()+ # Not the same engine again: the next build takes the next entry+ # of the attempt list (float16 -> int8_float16 -> cpu) and stays+ # on the last one. An engine that keeps failing live is slow or+ # wedged on THIS device/compute pair; the floor below it is not.+ self._attempt_index += 1self._failures = 0- log.info("Dictation preview engine dropped (%s); rebuilding on the next tick.", why)+ log.info(+ "Dictation preview engine dropped (%s on %s/%s); rebuilding on the "+ "next tick with the next engine.",+ why,+ self._engine_device or "?",+ self._engine_compute or "?",+ )_INSTANCE: LocalPreviewTranscriber | None = Nonediff --git a/tests/unit/dictation/test_local_preview.py b/tests/unit/dictation/test_local_preview.pyindex 062ae75952..4906082b79 100644--- a/tests/unit/dictation/test_local_preview.py+++ b/tests/unit/dictation/test_local_preview.py@@ -352,3 +352,88 @@ def test_an_unpinned_decode_reports_the_language_it_found():assert (detected, probability) == ("de", 0.98)assert text == "Hallo Welt" # i18n-allow: German test fixtureassert engine._model.options[-1]["temperature"] == 0.0+++# --------------------------------------------------------------------------+# The GPU engine must be FAST, not merely present (live 2026-08-22)+# --------------------------------------------------------------------------+++def test_the_device_probe_prefers_plain_half_precision_on_cuda(monkeypatch):+ """int8 on CUDA measured 10x slower than float16 on a Blackwell card."""+ import sys+ from types import SimpleNamespace++ monkeypatch.setitem(+ sys.modules,+ "ctranslate2",+ SimpleNamespace(+ get_supported_compute_types=lambda device: (+ {"int8", "int8_float16", "float16"} if device == "cuda" else {"int8"}+ )+ ),+ )++ assert LocalPreviewTranscriber._pick_device() == ("cuda", "float16")+++def test_a_dropped_engine_is_rebuilt_on_the_next_device_not_the_same_one(monkeypatch):+ """The rebuild spiral: an engine that keeps timing out was rebuilt with the+ SAME settings and timed out again, every 7 s. A drop now advances."""+ import jarvis.plugins.stt.fwhisper as fwhisper++ calls: list[tuple[str, str]] = []++ class _Model:+ def __init__(self, device: str, compute: str) -> None:+ self.device, self.compute = device, compute++ def transcribe(self, _audio, **_kwargs):+ return [], object()++ def _build(_name: str, device: str, compute: str):+ calls.append((device, compute))+ return _Model(device, compute)++ monkeypatch.setattr(fwhisper, "_new_whisper_model", _build)+ engine = LocalPreviewTranscriber()+ engine._pick_device = lambda: ("cuda", "float16") # type: ignore[method-assign]++ engine._load_model()+ assert (engine._model.device, engine._model.compute) == ("cuda", "float16")++ for _ in range(3):+ engine._note_failure("timed out")+ assert engine.ready is False+ engine._load_model()+ assert (engine._model.device, engine._model.compute) == ("cuda", "int8_float16")++ for _ in range(3):+ engine._note_failure("timed out")+ engine._load_model()+ assert (engine._model.device, engine._model.compute) == ("cpu", "int8")++ # The floor is the floor: more failures rebuild the CPU engine, not beyond.+ for _ in range(3):+ engine._note_failure("timed out")+ engine._load_model()+ assert (engine._model.device, engine._model.compute) == ("cpu", "int8")+ assert calls[0] == ("cuda", "float16")+ assert calls[-1] == ("cpu", "int8")+++async def test_a_late_but_successful_preview_ends_the_failure_streak():+ """Slow is not wedged: a worker that comes back resets the drop counter."""+ import jarvis.dictation.local_preview as mod++ engine = _Engine(delay=0.05)+ original = mod.PREVIEW_TIMEOUT_S+ mod.PREVIEW_TIMEOUT_S = 0.01+ try:+ assert await engine.transcribe(b"\x00" * 32000) is None # times out, worker runs on+ assert engine._failures == 1+ await asyncio.sleep(0.15) # the worker finishes late+ assert engine._failures == 0+ assert engine.ready is True+ finally:+ mod.PREVIEW_TIMEOUT_S = original