Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(dictation): host the preview engine out of process

rubenluetke10-beepfb5a1bd5d176
diff --git a/jarvis/dictation/local_preview.py b/jarvis/dictation/local_preview.py
index cbd6cbcbb1..379fb2a834 100644
--- a/jarvis/dictation/local_preview.py
+++ b/jarvis/dictation/local_preview.py
@@ -29,18 +29,38 @@ Degrades honestly (CLAUDE.md §3): on a host without ``faster_whisper`` — a ba
or headless install — this reports unavailable and the caller falls back to the
budgeted cloud preview. No GPU is required either; the engine picks CPU and the
caller simply sees a slower preview.
+
+In the desktop app the engine is hosted OUT OF PROCESS
+(``jarvis/dictation/preview_worker.py``): building it in-process loaded the
+CUDA DLLs (~600 MB of cublas) while holding the Windows loader lock, freezing
+the whole window for 15-30 s on the first dictation after a cold boot. The
+worker runs this same class; the parent talks to it through a faster-whisper-
+shaped proxy (``_WorkerModel``), so every timeout/failure/recovery path below
+applies to both hostings. ``JARVIS_DICTATION_PREVIEW_IN_PROCESS=1`` is the
+debug escape back to the in-process build.
"""
from __future__ import annotations
import asyncio
import logging
+import subprocess
+import sys
import threading
import time
+from pathlib import Path
+from types import SimpleNamespace
from typing import Any
log = logging.getLogger(__name__)
+#: How long the out-of-process engine may take to build + warm before the
+#: spawn attempt is abandoned. Generous on purpose: the first CUDA decode on a
+#: cold driver cache pays a one-off kernel compile measured at 19 s, and a
+#: cold-boot disk multiplies everything — the parent loses nothing by waiting,
+#: the preview simply stays on its cloud fallback meanwhile.
+_WORKER_BOOT_TIMEOUT_S = 180.0
+
#: Model for the preview. Small enough to answer in tens of milliseconds, good
#: enough that the line on screen reads as what was said. Never used for the
#: text that is actually delivered.
@@ -79,6 +99,132 @@ def faster_whisper_available() -> bool:
return importlib.util.find_spec("faster_whisper") is not None
+def _child_python() -> str:
+ """The interpreter for the preview worker, beside the running executable.
+
+ ``sys.executable`` in the desktop app is the venv's GUI entry point
+ (``PersonalJarvis.exe``), which cannot take ``-m`` — the real
+ ``python(.exe)`` lives in the same Scripts/bin directory.
+ """
+ exe = Path(sys.executable)
+ if exe.stem.lower().startswith("python"):
+ return str(exe)
+ for name in ("python.exe", "python"):
+ candidate = exe.parent / name
+ if candidate.exists():
+ return str(candidate)
+ raise RuntimeError(f"no python interpreter beside {exe}")
+
+
+class _WorkerModel:
+ """faster-whisper-shaped proxy whose decodes run in the preview worker.
+
+ Presents the same ``transcribe(...)`` surface ``_transcribe_sync`` uses,
+ so every failure/timeout/recovery path of the parent class applies
+ unchanged — with one upgrade: dropping this "model" (``close``) KILLS the
+ worker process, which is the recovery an in-process wedged native engine
+ never had (AP-24).
+ """
+
+ def __init__(self, proc: subprocess.Popen[bytes], device: str, compute: str) -> None:
+ self._proc = proc
+ self.device = device
+ self.compute = compute
+
+ def transcribe(
+ self, samples: Any, language: str | None = None, **_ignored: Any
+ ) -> tuple[list[Any], Any]:
+ import numpy as np
+
+ from jarvis.dictation.preview_worker import read_message, write_message
+
+ pcm = (np.clip(samples, -1.0, 1.0) * 32767.0).astype("<i2").tobytes()
+ stdin, stdout = self._proc.stdin, self._proc.stdout
+ if stdin is None or stdout is None: # pragma: no cover — Popen(PIPE) guarantees both
+ raise RuntimeError("preview worker pipes are gone")
+ write_message(stdin, {"n": len(pcm), "language": language})
+ stdin.write(pcm)
+ stdin.flush()
+ response = read_message(stdout)
+ if response is None:
+ raise RuntimeError("preview worker exited")
+ if "error" in response:
+ raise RuntimeError(str(response["error"]))
+ segments = [SimpleNamespace(text=str(response.get("text", "")))]
+ info = SimpleNamespace(
+ language=str(response.get("language", "") or ""),
+ language_probability=float(response.get("probability", 0.0) or 0.0),
+ )
+ return segments, info
+
+ def close(self) -> None:
+ """Kill the worker. Never raises — this runs on drop/replace paths."""
+ try:
+ self._proc.kill()
+ except Exception as exc: # noqa: BLE001 — an already-dead worker is the goal state
+ log.debug("Preview worker kill skipped: %s", exc)
+
+
+def _spawn_worker_model(model_name: str) -> _WorkerModel | None:
+ """Start the out-of-process engine; ``None`` when this host cannot.
+
+ A refused spawn is not an error — the caller falls back to the in-process
+ CPU floor, which never touches the CUDA DLLs and therefore never holds
+ the loader lock.
+ """
+ from jarvis.core.process_utils import NO_WINDOW_CREATIONFLAGS
+ from jarvis.dictation.preview_worker import read_message
+
+ try:
+ proc = subprocess.Popen( # noqa: S603 — our own interpreter + module, no user input
+ [_child_python(), "-X", "utf8", "-m", "jarvis.dictation.preview_worker", model_name],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL,
+ creationflags=NO_WINDOW_CREATIONFLAGS,
+ )
+ except Exception as exc: # noqa: BLE001 — no interpreter / spawn refused: use the floor
+ log.info(
+ "Dictation preview worker could not start (%s: %s) — staying in-process.",
+ type(exc).__name__,
+ exc,
+ )
+ return None
+ result: dict[str, Any] = {}
+ done = threading.Event()
+
+ def _handshake() -> None:
+ try:
+ result["msg"] = read_message(proc.stdout) # type: ignore[arg-type]
+ except Exception as exc: # noqa: BLE001 — reported via the empty result
+ result["exc"] = exc
+ done.set()
+
+ threading.Thread(target=_handshake, name="dictation-preview-handshake", daemon=True).start()
+ if not done.wait(_WORKER_BOOT_TIMEOUT_S):
+ log.info(
+ "Dictation preview worker did not become ready within %.0f s — killed.",
+ _WORKER_BOOT_TIMEOUT_S,
+ )
+ proc.kill()
+ return None
+ message = result.get("msg")
+ if not isinstance(message, dict) or not message.get("ready"):
+ detail = (
+ message.get("error")
+ if isinstance(message, dict)
+ else result.get("exc") or "worker exited during startup"
+ )
+ log.info("Dictation preview worker reported no engine (%s).", detail)
+ proc.kill()
+ return None
+ return _WorkerModel(
+ proc,
+ str(message.get("device", "?") or "?"),
+ str(message.get("compute", "?") or "?"),
+ )
+
+
class LocalPreviewTranscriber:
"""Lazily-built local engine for preview text. Never raises to the caller.
@@ -89,8 +235,13 @@ class LocalPreviewTranscriber:
preview away is free; the next tick asks again.
"""
- def __init__(self, model_name: str = PREVIEW_MODEL) -> None:
+ def __init__(self, model_name: str = PREVIEW_MODEL, *, prefer_worker: bool = False) -> None:
self._model_name = model_name
+ #: True in the DESKTOP process (set by the ``local_preview`` factory):
+ #: the engine is hosted out of process so the CUDA DLL load never
+ #: holds this process's loader lock. False inside the worker itself
+ #: and in tests, where the in-process ladder below runs unchanged.
+ self._prefer_worker = prefer_worker
self._model: Any = None
self._lock = threading.Lock()
self._busy = threading.Lock()
@@ -129,7 +280,27 @@ class LocalPreviewTranscriber:
try:
from jarvis.plugins.stt.fwhisper import _new_whisper_model
- preferred = self._pick_device()
+ if self._prefer_worker:
+ worker = _spawn_worker_model(self._model_name)
+ if worker is not None:
+ with self._lock:
+ self._model = worker
+ self._engine_device = worker.device
+ self._engine_compute = worker.compute
+ log.info(
+ "Dictation preview engine ready: %s on %s (%s), out of process.",
+ self._model_name,
+ worker.device,
+ worker.compute,
+ )
+ return
+ # No worker on this host: stay on the CPU floor IN process.
+ # Deliberately never CUDA here — the in-process cublas load
+ # holds the Windows loader lock for 15-30 s on a cold boot,
+ # which is the freeze this worker exists to remove.
+ preferred = ("cpu", "int8")
+ else:
+ preferred = self._pick_device()
attempts = [preferred]
if preferred == ("cuda", "float16"):
# The quantized pair is the old GPU default and still the
@@ -404,6 +575,7 @@ class LocalPreviewTranscriber:
# The old native worker keeps its captured model and guard. Rotate
# both references atomically so the next tick builds a genuinely
# fresh session rather than re-polling a wedged engine (AP-24).
+ dropped = self._model
self._model = None
self._busy = threading.Lock()
# Not the same engine again: the next build takes the next entry
@@ -412,6 +584,13 @@ class LocalPreviewTranscriber:
# wedged on THIS device/compute pair; the floor below it is not.
self._attempt_index += 1
self._failures = 0
+ # An out-of-process engine can be recovered for REAL: killing the
+ # worker unwedges whatever its native session was stuck in, and the
+ # detached transcribe thread's blocking read ends with the pipe. An
+ # in-process model has no close() and is left to the GC as before.
+ close = getattr(dropped, "close", None)
+ if callable(close):
+ close()
log.info(
"Dictation preview engine dropped (%s on %s/%s); rebuilding on the "
"next tick with the next engine.",
@@ -437,7 +616,13 @@ def local_preview() -> LocalPreviewTranscriber | None:
return None
with _INSTANCE_LOCK:
if _INSTANCE is None:
- _INSTANCE = LocalPreviewTranscriber()
+ import os
+
+ # Out of process by default (the CUDA DLL load must never hold
+ # this process's loader lock); the env switch is the debug escape
+ # back to the old in-process build.
+ in_process = os.environ.get("JARVIS_DICTATION_PREVIEW_IN_PROCESS") == "1"
+ _INSTANCE = LocalPreviewTranscriber(prefer_worker=not in_process)
return _INSTANCE if _INSTANCE.available else None
diff --git a/jarvis/dictation/preview_worker.py b/jarvis/dictation/preview_worker.py
new file mode 100644
index 0000000000..04b8f34545
--- /dev/null
+++ b/jarvis/dictation/preview_worker.py
@@ -0,0 +1,114 @@
+"""Out-of-process host for the dictation preview engine.
+
+Why a subprocess: building the CUDA engine in the DESKTOP process loads
+cublas (~600 MB of DLLs) while holding the Windows loader lock — no other
+thread can start during that, the audio queue overflows and the window stops
+answering for 15-30 s on a cold boot (2026-08-28 forensics, BUG-189 class).
+In its own process the load costs the app nothing, and a wedged native
+engine (AP-24) is finally recoverable for real: the parent kills this
+process and starts a fresh one — an in-process timeout could only ever stop
+waiting.
+
+The engine itself is the unchanged :class:`LocalPreviewTranscriber` device
+ladder (cuda/float16 → cuda/int8_float16 → cpu/int8, warm-decode gated): this
+module only moves WHERE it runs.
+
+Protocol (stdin/stdout, binary, length-prefixed):
+
+* every message is ``4-byte big-endian length`` + that many bytes of UTF-8
+ JSON; a transcribe REQUEST is followed by exactly ``n`` raw PCM bytes
+ (16 kHz s16le), where ``n`` comes from the JSON header
+* worker → parent, once, after the engine built and warmed:
+ ``{"ready": true, "device": ..., "compute": ...}`` or
+ ``{"ready": false, "error": ...}`` (the worker then exits)
+* parent → worker: ``{"n": <pcm byte count>, "language": <code or null>}``
+* worker → parent: ``{"text": ..., "language": ..., "probability": ...}``
+ or ``{"error": ...}``
+
+stdout carries ONLY the protocol (stderr is discarded by the parent). The
+worker exits when its stdin closes — parent exit or kill — so no orphan can
+outlive the app.
+"""
+
+from __future__ import annotations
+
+import json
+import struct
+import sys
+from typing import Any, BinaryIO
+
+
+def _read_exact(stream: BinaryIO, n: int) -> bytes | None:
+ """``n`` bytes, or ``None`` on EOF — a torn read means the peer is gone."""
+ chunks: list[bytes] = []
+ remaining = n
+ while remaining > 0:
+ chunk = stream.read(remaining)
+ if not chunk:
+ return None
+ chunks.append(chunk)
+ remaining -= len(chunk)
+ return b"".join(chunks)
+
+
+def read_message(stream: BinaryIO) -> dict[str, Any] | None:
+ """One length-prefixed JSON message, or ``None`` on EOF."""
+ header = _read_exact(stream, 4)
+ if header is None:
+ return None
+ (length,) = struct.unpack(">I", header)
+ payload = _read_exact(stream, length)
+ if payload is None:
+ return None
+ return json.loads(payload.decode("utf-8"))
+
+
+def write_message(stream: BinaryIO, message: dict[str, Any]) -> None:
+ payload = json.dumps(message, ensure_ascii=False).encode("utf-8")
+ stream.write(struct.pack(">I", len(payload)) + payload)
+ stream.flush()
+
+
+def serve(stdin: BinaryIO, stdout: BinaryIO, model_name: str) -> int:
+ """Build the engine, report readiness, then answer requests until EOF."""
+ from jarvis.dictation.local_preview import LocalPreviewTranscriber
+
+ engine = LocalPreviewTranscriber(model_name)
+ engine._load_model() # noqa: SLF001 — the worker IS the engine's host
+ if not engine.ready:
+ write_message(stdout, {"ready": False, "error": "no usable local engine"})
+ return 1
+ write_message(
+ stdout,
+ {
+ "ready": True,
+ "device": engine._engine_device, # noqa: SLF001
+ "compute": engine._engine_compute, # noqa: SLF001
+ },
+ )
+ while True:
+ request = read_message(stdin)
+ if request is None:
+ return 0 # parent closed stdin: normal shutdown
+ pcm = _read_exact(stdin, int(request.get("n", 0)))
+ if pcm is None:
+ return 0
+ try:
+ text, language, probability = engine._transcribe_sync( # noqa: SLF001
+ pcm, request.get("language") or None
+ )
+ write_message(
+ stdout,
+ {"text": text, "language": language, "probability": probability},
+ )
+ except Exception as exc: # noqa: BLE001 — one bad clip must not kill the worker
+ write_message(stdout, {"error": f"{type(exc).__name__}: {exc}"})
+
+
+def main() -> int:
+ model_name = sys.argv[1] if len(sys.argv) > 1 else "base"
+ return serve(sys.stdin.buffer, sys.stdout.buffer, model_name)
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/unit/dictation/test_preview_worker.py b/tests/unit/dictation/test_preview_worker.py
new file mode 100644
index 0000000000..c74e284d5f
--- /dev/null
+++ b/tests/unit/dictation/test_preview_worker.py
@@ -0,0 +1,189 @@
+"""The out-of-process dictation preview: protocol, proxy, and fallbacks.
+
+The worker exists so the CUDA DLL load (cublas, ~600 MB, held Windows loader
+lock, 15-30 s whole-app freezes on a cold boot) never happens in the desktop
+process. These tests run the protocol in-memory — no subprocess, no model.
+"""
+
+from __future__ import annotations
+
+import io
+from typing import Any
+
+import numpy as np
+import pytest
+
+import jarvis.dictation.local_preview as local_preview_mod
+from jarvis.dictation.local_preview import LocalPreviewTranscriber, _WorkerModel
+from jarvis.dictation.preview_worker import read_message, serve, write_message
+
+
+def _messages_from(buffer: bytes) -> list[dict[str, Any]]:
+ stream = io.BytesIO(buffer)
+ out: list[dict[str, Any]] = []
+ while True:
+ message = read_message(stream)
+ if message is None:
+ return out
+ out.append(message)
+
+
+class _FakeEngine:
+ """Stands in for LocalPreviewTranscriber inside the worker."""
+
+ def __init__(self, model_name: str, *, ready: bool = True) -> None:
+ self._model_name = model_name
+ self.ready = ready
+ self._engine_device = "cpu"
+ self._engine_compute = "int8"
+ self.requests: list[tuple[bytes, str | None]] = []
+
+ def _load_model(self) -> None:
+ return None
+
+ def _transcribe_sync(self, pcm: bytes, language: str | None) -> tuple[str, str, float]:
+ self.requests.append((pcm, language))
+ if language == "boom":
+ raise RuntimeError("bad clip")
+ return f"heard {len(pcm)} bytes", "de", 0.9
+
+
+def _request_bytes(pcm: bytes, language: str | None) -> bytes:
+ header = io.BytesIO()
+ write_message(header, {"n": len(pcm), "language": language})
+ return header.getvalue() + pcm
+
+
+def test_serve_answers_ready_then_transcribes(monkeypatch: pytest.MonkeyPatch) -> None:
+ engine = _FakeEngine("base")
+ monkeypatch.setattr(local_preview_mod, "LocalPreviewTranscriber", lambda name: engine)
+ stdin = io.BytesIO(_request_bytes(b"\x00\x01" * 100, "de") + _request_bytes(b"", None))
+ stdout = io.BytesIO()
+
+ assert serve(stdin, stdout, "base") == 0
+
+ ready, first, second = _messages_from(stdout.getvalue())
+ assert ready == {"ready": True, "device": "cpu", "compute": "int8"}
+ assert first == {"text": "heard 200 bytes", "language": "de", "probability": 0.9}
+ assert second["text"] == "heard 0 bytes"
+ assert engine.requests[0][1] == "de"
+
+
+def test_serve_reports_a_failed_build_and_exits(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(
+ local_preview_mod,
+ "LocalPreviewTranscriber",
+ lambda name: _FakeEngine(name, ready=False),
+ )
+ stdout = io.BytesIO()
+
+ assert serve(io.BytesIO(b""), stdout, "base") == 1
+
+ (message,) = _messages_from(stdout.getvalue())
+ assert message["ready"] is False
+
+
+def test_serve_turns_a_bad_clip_into_an_error_message_not_a_crash(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ engine = _FakeEngine("base")
+ monkeypatch.setattr(local_preview_mod, "LocalPreviewTranscriber", lambda name: engine)
+ stdin = io.BytesIO(_request_bytes(b"\x00\x00", "boom") + _request_bytes(b"\x00\x00", "de"))
+ stdout = io.BytesIO()
+
+ assert serve(stdin, stdout, "base") == 0
+
+ _ready, error, ok = _messages_from(stdout.getvalue())
+ assert "bad clip" in error["error"]
+ assert ok["language"] == "de"
+
+
+class _FakePipeProc:
+ """A Popen double whose pipes loop back through the worker protocol."""
+
+ def __init__(self, responses: list[dict[str, Any]]) -> None:
+ self.stdin = io.BytesIO()
+ out = io.BytesIO()
+ for response in responses:
+ write_message(out, response)
+ out.seek(0)
+ self.stdout = out
+ self.killed = False
+
+ def kill(self) -> None:
+ self.killed = True
+
+
+def test_worker_model_speaks_the_protocol_and_reports_the_language() -> None:
+ proc = _FakePipeProc([{"text": "hallo", "language": "de", "probability": 0.8}])
+ model = _WorkerModel(proc, "cuda", "float16") # type: ignore[arg-type]
+
+ samples = np.zeros(160, dtype=np.float32)
+ segments, info = model.transcribe(samples, language=None)
+
+ assert [seg.text for seg in segments] == ["hallo"]
+ assert (info.language, info.language_probability) == ("de", 0.8)
+ header = read_message(io.BytesIO(proc.stdin.getvalue()))
+ assert header == {"n": 320, "language": None}
+
+
+def test_worker_model_raises_on_error_and_on_a_dead_worker() -> None:
+ erring = _WorkerModel(_FakePipeProc([{"error": "boom"}]), "cpu", "int8") # type: ignore[arg-type]
+ with pytest.raises(RuntimeError, match="boom"):
+ erring.transcribe(np.zeros(10, dtype=np.float32))
+
+ dead = _WorkerModel(_FakePipeProc([]), "cpu", "int8") # type: ignore[arg-type]
+ with pytest.raises(RuntimeError, match="exited"):
+ dead.transcribe(np.zeros(10, dtype=np.float32))
+
+
+def test_dropping_a_worker_engine_kills_the_process() -> None:
+ """The upgrade over in-process: recovery is a kill, not a shrug (AP-24)."""
+ proc = _FakePipeProc([])
+ engine = LocalPreviewTranscriber()
+ engine._model = _WorkerModel(proc, "cuda", "float16") # type: ignore[arg-type] # noqa: SLF001
+
+ for _ in range(3):
+ engine._note_failure("timed out") # noqa: SLF001
+
+ assert proc.killed is True
+ assert engine._model is None # noqa: SLF001
+
+
+def test_worker_preferring_load_falls_back_to_the_cpu_floor(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """No worker on this host: the in-process build must pin CPU — never the
+ CUDA ladder whose DLL load holds the loader lock."""
+ from jarvis.plugins.stt import fwhisper
+
+ built: list[tuple[str, str]] = []
+
+ class _Model:
+ def transcribe(self, *_a: Any, **_k: Any) -> tuple[list[Any], Any]:
+ return [], None
+
+ def _build(name: str, device: str, compute: str, *a: Any, **k: Any) -> Any:
+ built.append((device, compute))
+ return _Model()
+
+ monkeypatch.setattr(fwhisper, "_new_whisper_model", _build)
+ monkeypatch.setattr(local_preview_mod, "_spawn_worker_model", lambda name: None)
+ engine = LocalPreviewTranscriber(prefer_worker=True)
+
+ engine._load_model() # noqa: SLF001
+
+ assert built == [("cpu", "int8")]
+ assert engine.ready
+
+
+def test_the_factory_defaults_to_the_worker_hosting(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(local_preview_mod, "faster_whisper_available", lambda: True)
+ monkeypatch.delenv("JARVIS_DICTATION_PREVIEW_IN_PROCESS", raising=False)
+ local_preview_mod.reset_local_preview_for_tests()
+ try:
+ engine = local_preview_mod.local_preview()
+ assert engine is not None
+ assert engine._prefer_worker is True # noqa: SLF001
+ finally:
+ local_preview_mod.reset_local_preview_for_tests()