Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(dictation): re-measure a busy GPU in the background, not on a dictation (#1009)

Ruben Lütkefbf9481f2a11
diff --git a/dodaios/dictation/local_final.py b/dodaios/dictation/local_final.py
index 7fdd675c5b..b5e78f70a5 100644
--- a/dodaios/dictation/local_final.py
+++ b/dodaios/dictation/local_final.py
@@ -75,15 +75,17 @@ CONTENDED_BEAM_RTF = 0.2
#: so one quick call on a still-busy card does not flip the width back.
UNCONTENDED_GREEDY_RTF = 0.06
-#: How long a busy card hands dictations to the cloud provider behind it before
-#: the local engine is tried again. The retry is what notices the card is free
-#: once the other program finishes; it decodes greedily, so it costs at most
-#: the greedy wait on a card that is still busy.
-_BUSY_RECHECK_S = 90.0
+#: How often a busy card is measured again while the cloud provider behind it
+#: takes the dictations. The measurement is a greedy decode of the dictation
+#: that was just handed over, run in the BACKGROUND after the cloud answered,
+#: so nobody waits on it. (The first version retried a real dictation locally
+#: every 90 s; with a dictation every couple of minutes that made every
+#: dictation a slow one — live logs 2026-10-27.)
+_BUSY_RECHECK_S = 60.0
#: Audio shorter than this never moves the decode width: a one-word clip is
#: dominated by fixed per-call overhead and says nothing about the card.
-_MIN_RTF_SAMPLE_S = 2.0
+_MIN_RTF_SAMPLE_S = 1.0
#: Free accelerator memory the spawn asks for by default — the turbo weights
#: plus a CUDA context, with a little headroom.
@@ -164,6 +166,8 @@ class LocalFinalSTT:
#: Set by the dictation lane: True when a cloud provider stands behind
#: this one in the chain. Only then may a busy card hand calls over.
self.cloud_behind = False
+ #: True while a background measurement holds the call lock.
+ self._probe_running = False
self._worker: Any = None
self._device = ""
self._engine_compute = ""
@@ -265,6 +269,11 @@ class LocalFinalSTT:
f"the dictation worker takes 16 kHz audio, not {sample_rate} Hz"
)
if not self._call_lock.acquire(blocking=False):
+ if self._probe_running:
+ # Only ever set while the card is busy and a cloud is behind.
+ raise LocalEngineUnavailable(
+ "the card is busy with another program; the next provider takes this dictation"
+ )
if self.is_loading:
raise LocalEngineUnavailable("the local dictation engine is still starting")
await asyncio.to_thread(self.recover)
@@ -304,13 +313,11 @@ class LocalFinalSTT:
raise
def _transcribe_blocking(self, pcm_bytes: bytes, language: str | None) -> Transcript:
- if (
- self._contended
- and self.cloud_behind
- and time.monotonic() - self._contended_at < _BUSY_RECHECK_S
- ):
+ if self._contended and self.cloud_behind:
# A saturated card decodes at speech speed: the cloud behind us
# answers in about a second. Crossable, so the chain moves on.
+ if time.monotonic() - self._contended_at >= _BUSY_RECHECK_S:
+ self._start_background_probe(pcm_bytes, language)
raise LocalEngineUnavailable(
"the card is busy with another program; the next provider takes this dictation"
)
@@ -382,6 +389,37 @@ class LocalFinalSTT:
# Worker ownership
# ------------------------------------------------------------------
+ def _start_background_probe(self, pcm_bytes: bytes, language: str | None) -> None:
+ """Measure the card again on this audio, after the caller has moved on.
+
+ Runs on its own thread once the current call has released the lock, so
+ the dictation itself goes to the cloud without waiting. A greedy decode
+ that is fast again brings the local engine back for the next press.
+ """
+ self._contended_at = time.monotonic() # one probe per interval
+
+ def probe() -> None:
+ if not self._call_lock.acquire(timeout=5.0):
+ return
+ self._probe_running = True
+ try:
+ worker = self._worker
+ if worker is None:
+ return
+ import numpy as np
+
+ samples = np.frombuffer(pcm_bytes, dtype=np.int16).astype(np.float32) / 32768.0
+ started = time.perf_counter()
+ worker.transcribe(samples, language=_concrete_language(language), beam_size=1)
+ self._note_decode_speed(len(pcm_bytes) / 32_000.0, time.perf_counter() - started, 1)
+ except Exception as exc: # noqa: BLE001 — a failed probe just waits for the next
+ log.debug("local dictation: busy-card probe failed (%s)", exc)
+ finally:
+ self._probe_running = False
+ self._call_lock.release()
+
+ threading.Thread(target=probe, name="dictation-busy-probe", daemon=True).start()
+
@property
def current_beam_size(self) -> int:
"""Beam width the next decode uses: greedy while the card is contended."""
diff --git a/tests/unit/dictation/test_local_final.py b/tests/unit/dictation/test_local_final.py
index 433f019aa3..f32979ef94 100644
--- a/tests/unit/dictation/test_local_final.py
+++ b/tests/unit/dictation/test_local_final.py
@@ -598,8 +598,8 @@ def test_a_short_clip_never_moves_the_decode_width(
_clock, worker = _clocked
stt = LocalFinalSTT()
- asyncio.run(stt.transcribe_pcm(_pcm(1.0), language="de"))
- asyncio.run(stt.transcribe_pcm(_pcm(1.0), language="de"))
+ asyncio.run(stt.transcribe_pcm(_pcm(0.5), language="de"))
+ asyncio.run(stt.transcribe_pcm(_pcm(0.5), language="de"))
assert [c["beam"] for c in worker.calls] == [FINAL_BEAM_SIZE, FINAL_BEAM_SIZE]
@@ -617,14 +617,57 @@ def test_a_busy_card_hands_dictations_to_the_cloud_behind_it(
assert is_crossable_failure(classify_stt_failure(caught.value))
assert len(worker.calls) == 1
- # After the recheck interval the local engine is tried again, greedily.
- clock.now += local_final_mod._BUSY_RECHECK_S + 1
- worker.rtf = {FINAL_BEAM_SIZE: 0.04, 1: 0.02}
+
+def test_a_busy_card_is_measured_again_in_the_background(
+ _clocked: tuple[_Clock, _TimedWorker], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Nobody waits on the re-measurement: the dictation still goes to the
+ cloud, and a greedy decode of the same audio runs after it."""
+ clock, worker = _clocked
+ started: list[threading.Thread] = []
+ real_thread = threading.Thread
+
+ def _thread(*args: Any, **kwargs: Any) -> threading.Thread:
+ t = real_thread(*args, **kwargs)
+ if kwargs.get("name") == "dictation-busy-probe":
+ started.append(t)
+ return t
+
+ monkeypatch.setattr(local_final_mod.threading, "Thread", _thread)
+ stt = LocalFinalSTT()
+ stt.cloud_behind = True
asyncio.run(stt.transcribe_pcm(_pcm(10.0), language="de"))
+ clock.now += local_final_mod._BUSY_RECHECK_S + 1
+ worker.rtf = {FINAL_BEAM_SIZE: 0.04, 1: 0.02} # the render finished
+
+ with pytest.raises(LocalEngineUnavailable):
+ asyncio.run(stt.transcribe_pcm(_pcm(10.0), language="de"))
+ assert len(started) == 1
+ started[0].join(timeout=5)
+
+ assert [c["beam"] for c in worker.calls] == [FINAL_BEAM_SIZE, 1]
asyncio.run(stt.transcribe_pcm(_pcm(10.0), language="de"))
assert [c["beam"] for c in worker.calls] == [FINAL_BEAM_SIZE, 1, FINAL_BEAM_SIZE]
+def test_a_dictation_during_the_probe_goes_to_the_cloud_without_killing_the_worker(
+ _clocked: tuple[_Clock, _TimedWorker],
+) -> None:
+ _clock, worker = _clocked
+ stt = LocalFinalSTT()
+ stt.cloud_behind = True
+ asyncio.run(stt.transcribe_pcm(_pcm(10.0), language="de"))
+ assert stt._call_lock.acquire(blocking=False)
+ stt._probe_running = True
+ try:
+ with pytest.raises(LocalEngineUnavailable):
+ asyncio.run(stt.transcribe_pcm(_pcm(10.0), language="de"))
+ finally:
+ stt._probe_running = False
+ stt._call_lock.release()
+ assert stt._worker is worker and not worker.closed
+
+
def test_without_a_cloud_behind_a_busy_card_still_decodes_locally(
_clocked: tuple[_Clock, _TimedWorker],
) -> None: