Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(dictation): the GPU preview stops rebuilding itself every seven seconds

rubenluetke10-beep3d2a3761e93d
diff --git a/jarvis/dictation/local_preview.py b/jarvis/dictation/local_preview.py
index 27c42e0209..cbd6cbcbb1 100644
--- a/jarvis/dictation/local_preview.py
+++ b/jarvis/dictation/local_preview.py
@@ -36,6 +36,7 @@ from __future__ import annotations
import asyncio
import logging
import threading
+import time
from typing import Any
log = logging.getLogger(__name__)
@@ -55,6 +56,21 @@ PREVIEW_TIMEOUT_S = 2.0
#: use. A native engine that has wedged never recovers by being asked again.
_MAX_FAILURES = 3
+#: The clip the freshly built engine decodes before it is advertised — four
+#: seconds of near-silence, the shape of an ordinary preview tail. Its timing
+#: is logged, never judged: the first CUDA decode in a process pays context
+#: init and, on a cold driver cache, a one-off kernel compile measured at 19 s,
+#: so a clock here would reject a perfectly good GPU on its first day.
+#:
+#: What IS judged is the live tick. Measured 2026-08-22 on a Blackwell card
+#: (ctranslate2 4.8): ``int8_float16`` decoded 8 s of real speech in 10-11 s,
+#: ``float16`` in 0.7 s — so the engine timed out on every tick, was dropped,
+#: rebuilt with the SAME settings and dropped again, every 7 s for the whole
+#: dictation. A drop therefore now advances to the next engine in the attempt
+#: list (``_load_model``) instead of rebuilding the one that just failed, and
+#: stays on the CPU floor once it gets there.
+_WARM_CLIP_S = 4.0
+
def faster_whisper_available() -> bool:
"""Is a local engine importable here? Cheap spec probe, no heavy import."""
@@ -78,6 +94,11 @@ class LocalPreviewTranscriber:
self._model: Any = None
self._lock = threading.Lock()
self._busy = threading.Lock()
+ # Which entry of the attempt list the next build starts from, and what
+ # the current engine runs on (for the log line and the status card).
+ self._attempt_index = 0
+ self._engine_device = ""
+ self._engine_compute = ""
self._failures = 0
self._unavailable = False
self._loading = False
@@ -110,8 +131,17 @@ class LocalPreviewTranscriber:
preferred = self._pick_device()
attempts = [preferred]
+ if preferred == ("cuda", "float16"):
+ # The quantized pair is the old GPU default and still the
+ # right answer on a GPU where it is fast; tried second so a
+ # host that cannot do float16 keeps its GPU preview.
+ attempts.append(("cuda", "int8_float16"))
if preferred[0] != "cpu":
attempts.append(("cpu", "int8"))
+ # A rebuild after repeated live failures starts one entry further
+ # down: the engine that just kept timing out is not asked again.
+ skip = min(self._attempt_index, len(attempts) - 1)
+ attempts = attempts[skip:]
last_error: Exception | None = None
for device, compute in attempts:
try:
@@ -129,13 +159,20 @@ class LocalPreviewTranscriber:
# Constructing the model does not pay CUDA's first-decode
# setup cost. Prime one throwaway greedy decode here, while
# the preview is still advertised as unavailable, so the
- # first visible preview keeps the steady-state latency.
+ # first visible preview keeps the steady-state latency —
+ # and TIME it: a GPU engine that cannot decode a preview-
+ # sized clip well inside the tick ceiling is not a preview
+ # engine, whatever the constructor said.
import numpy as np
rng = np.random.default_rng(0)
warm_audio = (
- rng.standard_normal(16_000).astype(np.float32) * 0.001
+ rng.standard_normal(int(16_000 * _WARM_CLIP_S)).astype(
+ np.float32
+ )
+ * 0.001
)
+ started = time.perf_counter()
segments, _info = model.transcribe(
warm_audio,
beam_size=1,
@@ -143,6 +180,7 @@ class LocalPreviewTranscriber:
condition_on_previous_text=False,
)
list(segments)
+ warm_s = time.perf_counter() - started
except Exception as exc: # noqa: BLE001 — try the portable floor
last_error = exc
log.info(
@@ -155,11 +193,16 @@ class LocalPreviewTranscriber:
continue
with self._lock:
self._model = model
+ self._engine_device = device
+ self._engine_compute = compute
log.info(
- "Dictation preview engine ready: %s on %s (%s).",
+ "Dictation preview engine ready: %s on %s (%s), %.0fs clip "
+ "in %.2fs.",
self._model_name,
device,
compute,
+ _WARM_CLIP_S,
+ warm_s,
)
break
else:
@@ -195,7 +238,15 @@ class LocalPreviewTranscriber:
with inference_only_import_shield():
import ctranslate2
- supported = ctranslate2.get_supported_compute_types("cuda")
+ supported = set(ctranslate2.get_supported_compute_types("cuda"))
+ # Plain half precision first: on the one GPU measured so far the
+ # quantized int8 pairs were an order of magnitude SLOWER than
+ # float16 (10-11 s against 0.7 s for 8 s of speech, ctranslate2
+ # 4.8 on a Blackwell card) — the preview model is tiny, so the
+ # memory int8 saves buys nothing here. The warm-decode gate in
+ # ``_load_model`` is the backstop for the next surprise.
+ if "float16" in supported:
+ return "cuda", "float16"
if "int8_float16" in supported:
return "cuda", "int8_float16"
except Exception as exc: # noqa: BLE001 — a probe must never decide by raising
@@ -311,6 +362,11 @@ class LocalPreviewTranscriber:
pass
except Exception as exc: # noqa: BLE001 — detached worker is contained
log.debug("Detached dictation preview failed: %s", exc)
+ else:
+ # It came back late, but it came back: a slow engine is
+ # not a wedged one, and the streak that would have
+ # dropped it is over.
+ self._failures = 0
work.add_done_callback(_release_finished_worker)
if release_here:
@@ -350,8 +406,19 @@ class LocalPreviewTranscriber:
# fresh session rather than re-polling a wedged engine (AP-24).
self._model = None
self._busy = threading.Lock()
+ # Not the same engine again: the next build takes the next entry
+ # of the attempt list (float16 -> int8_float16 -> cpu) and stays
+ # on the last one. An engine that keeps failing live is slow or
+ # wedged on THIS device/compute pair; the floor below it is not.
+ self._attempt_index += 1
self._failures = 0
- log.info("Dictation preview engine dropped (%s); rebuilding on the next tick.", why)
+ log.info(
+ "Dictation preview engine dropped (%s on %s/%s); rebuilding on the "
+ "next tick with the next engine.",
+ why,
+ self._engine_device or "?",
+ self._engine_compute or "?",
+ )
_INSTANCE: LocalPreviewTranscriber | None = None
diff --git a/tests/unit/dictation/test_local_preview.py b/tests/unit/dictation/test_local_preview.py
index 062ae75952..4906082b79 100644
--- a/tests/unit/dictation/test_local_preview.py
+++ b/tests/unit/dictation/test_local_preview.py
@@ -352,3 +352,88 @@ def test_an_unpinned_decode_reports_the_language_it_found():
assert (detected, probability) == ("de", 0.98)
assert text == "Hallo Welt" # i18n-allow: German test fixture
assert engine._model.options[-1]["temperature"] == 0.0
+
+
+# --------------------------------------------------------------------------
+# The GPU engine must be FAST, not merely present (live 2026-08-22)
+# --------------------------------------------------------------------------
+
+
+def test_the_device_probe_prefers_plain_half_precision_on_cuda(monkeypatch):
+ """int8 on CUDA measured 10x slower than float16 on a Blackwell card."""
+ import sys
+ from types import SimpleNamespace
+
+ monkeypatch.setitem(
+ sys.modules,
+ "ctranslate2",
+ SimpleNamespace(
+ get_supported_compute_types=lambda device: (
+ {"int8", "int8_float16", "float16"} if device == "cuda" else {"int8"}
+ )
+ ),
+ )
+
+ assert LocalPreviewTranscriber._pick_device() == ("cuda", "float16")
+
+
+def test_a_dropped_engine_is_rebuilt_on_the_next_device_not_the_same_one(monkeypatch):
+ """The rebuild spiral: an engine that keeps timing out was rebuilt with the
+ SAME settings and timed out again, every 7 s. A drop now advances."""
+ import jarvis.plugins.stt.fwhisper as fwhisper
+
+ calls: list[tuple[str, str]] = []
+
+ class _Model:
+ def __init__(self, device: str, compute: str) -> None:
+ self.device, self.compute = device, compute
+
+ def transcribe(self, _audio, **_kwargs):
+ return [], object()
+
+ def _build(_name: str, device: str, compute: str):
+ calls.append((device, compute))
+ return _Model(device, compute)
+
+ monkeypatch.setattr(fwhisper, "_new_whisper_model", _build)
+ engine = LocalPreviewTranscriber()
+ engine._pick_device = lambda: ("cuda", "float16") # type: ignore[method-assign]
+
+ engine._load_model()
+ assert (engine._model.device, engine._model.compute) == ("cuda", "float16")
+
+ for _ in range(3):
+ engine._note_failure("timed out")
+ assert engine.ready is False
+ engine._load_model()
+ assert (engine._model.device, engine._model.compute) == ("cuda", "int8_float16")
+
+ for _ in range(3):
+ engine._note_failure("timed out")
+ engine._load_model()
+ assert (engine._model.device, engine._model.compute) == ("cpu", "int8")
+
+ # The floor is the floor: more failures rebuild the CPU engine, not beyond.
+ for _ in range(3):
+ engine._note_failure("timed out")
+ engine._load_model()
+ assert (engine._model.device, engine._model.compute) == ("cpu", "int8")
+ assert calls[0] == ("cuda", "float16")
+ assert calls[-1] == ("cpu", "int8")
+
+
+async def test_a_late_but_successful_preview_ends_the_failure_streak():
+ """Slow is not wedged: a worker that comes back resets the drop counter."""
+ import jarvis.dictation.local_preview as mod
+
+ engine = _Engine(delay=0.05)
+ original = mod.PREVIEW_TIMEOUT_S
+ mod.PREVIEW_TIMEOUT_S = 0.01
+ try:
+ assert await engine.transcribe(b"\x00" * 32000) is None # times out, worker runs on
+ assert engine._failures == 1
+ await asyncio.sleep(0.15) # the worker finishes late
+ assert engine._failures == 0
+ assert engine.ready is True
+ finally:
+ mod.PREVIEW_TIMEOUT_S = original