Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(voice): isolate local STT and defer unsafe audio refreshes

rubenluetke10-beep8370fd47432a
diff --git a/docs/os-parity.md b/docs/os-parity.md
index a31c1f56bc..bc8a1b8a92 100644
--- a/docs/os-parity.md
+++ b/docs/os-parity.md
@@ -1,5 +1,20 @@
# OS Feature Parity — macOS / Linux Gap Register
+## Local voice and audio topology refresh (issue #464, T2)
+
+Local final transcription uses its configured model independently of the wake
+model. Preview is disabled when it would share a non-concurrent native engine
+with final transcription; cloud preview remains available. CUDA construction
+preserves the configured compute type, and STT failure speech uses the shared
+reply/conversation/UI language resolution.
+
+On Windows, macOS and Linux, topology refresh waits for playback and native
+stream operations to finish, refuses to terminate PortAudio after a failed
+stream close, and retries a deferred refresh. Transient device enumeration
+changes that disappear during settling do not trigger re-initialization.
+Regression tests use fake devices and models on Windows. Ubuntu/PipeWire,
+macOS audio devices and NVIDIA T600 inference have not been verified live.
+
## Screenshots paste anywhere (2026-10-05, T2)
Appshots taken by shortcut or button and Jarvis X screenshots go on the
diff --git a/jarvis/audio/capture.py b/jarvis/audio/capture.py
index 0b3abace03..064f13f02b 100644
--- a/jarvis/audio/capture.py
+++ b/jarvis/audio/capture.py
@@ -1124,6 +1124,7 @@ class MicrophoneCapture:
dtype=DTYPE,
callback=self._callback,
)
+ topology.register_native_stream(opened)
try:
opened.start()
except BaseException:
@@ -1131,15 +1132,19 @@ class MicrophoneCapture:
# outer candidate loop only sees the exception.
try:
opened.close()
+ topology.unregister_native_stream(opened)
except Exception: # noqa: BLE001, S110 - preserve the original open failure
pass
raise
+ # Publish before releasing the refresh guard. Otherwise
+ # the watcher can miss this live stream while its async
+ # caller is still waiting for the worker result.
+ self._stream = opened
return opened
stream = await asyncio.to_thread(
_guarded_open, attempt, capture_rate, capture_blocksize
)
- self._stream = stream
_remember_input_latency(stream)
self._device = attempt
self._using_physical_fallback = physical_fallback
@@ -1409,7 +1414,8 @@ class MicrophoneCapture:
except Exception: # noqa: BLE001 - a notice must never break the capture
_log.debug("Mic silence notice could not be published.", exc_info=True)
- def discard_native_stream(self) -> None:
+ @topology.native_stream_operation()
+ def discard_native_stream(self) -> bool:
"""Drop the native stream so the stall watchdog reopens it (BUG-102).
Called by the topology watcher (worker thread) right before a
@@ -1420,7 +1426,9 @@ class MicrophoneCapture:
"""
stream, self._stream = self._stream, None
if stream is not None:
- self._discard_stream(stream)
+ if self._discard_stream(stream) is False:
+ self._stream = stream
+ return False
if not isinstance(self._device_spec, int):
# Indices renumber across a PortAudio re-init; retrying the stale
# resolved index first could silently open a DIFFERENT physical
@@ -1430,6 +1438,7 @@ class MicrophoneCapture:
self._last_chunk_monotonic,
time.monotonic() - self._STALL_THRESHOLD_S - 1.0,
)
+ return True
#: How long a coroutine waits for a native stream to shut down before it
#: leaves the rest to a worker thread. Long enough that the ordinary close
@@ -1468,7 +1477,8 @@ class MicrophoneCapture:
)
@staticmethod
- def _discard_stream(stream: Any) -> None:
+ @topology.native_stream_operation()
+ def _discard_stream(stream: Any) -> bool:
"""Abort-then-close a (possibly wedged) native stream, never raising.
Abort discards the dead native stream immediately; CoreAudio can
@@ -1492,6 +1502,9 @@ class MicrophoneCapture:
stream.close()
except Exception as exc: # noqa: BLE001
_log.debug("Mic discard: close() ignored ({}).", exc)
+ return False
+ topology.unregister_native_stream(stream)
+ return True
async def __aenter__(self) -> MicrophoneCapture:
self._loop = asyncio.get_running_loop()
diff --git a/jarvis/audio/player.py b/jarvis/audio/player.py
index 378ff6df7c..a7b41958b5 100644
--- a/jarvis/audio/player.py
+++ b/jarvis/audio/player.py
@@ -718,7 +718,14 @@ class AudioPlayer:
return 0.0
return min(latency, _MAX_REPORTED_OUTPUT_LATENCY_S)
- def invalidate_device_cache(self) -> None:
+ @property
+ def audio_backend_busy(self) -> bool:
+ """Whether a topology refresh must wait for this player's output."""
+ lock = getattr(self, "_play_lock", None)
+ return bool(lock is not None and lock.locked()) or self._native_playback_poisoned()
+
+ @topology.native_stream_operation()
+ def invalidate_device_cache(self) -> bool:
"""Forget every cached device_rate and tear down the active stream.
Call this on USB hot-swap / device-disconnect / device-reset events
@@ -739,12 +746,17 @@ class AudioPlayer:
self._active_source_rate = None
self._active_device_rate = None
if stream is not None:
- self._close_output_stream(stream)
+ if self._close_output_stream(stream) is False:
+ with self._get_stream_state_lock():
+ self._active_stream = stream
+ self._unclean_stream = stream
+ return False
self._device_rate_cache.clear()
self._device_rate_failed.clear()
# A swapped device also invalidates the played-envelope record: echo
# correlation against a vanished device's output is meaningless.
echo_reference.reset()
+ return True
def set_device(self, device: int | str | None) -> None:
"""Re-resolve the output device and drop any cached state tied to
@@ -829,6 +841,7 @@ class AudioPlayer:
"""Invalidate queued PCM on both mute edges, independently of stop()."""
return getattr(self, "_output_generation", 0)
+ @topology.native_stream_operation()
def set_muted(self, muted: bool) -> None:
"""Discard native buffered speech without cancelling its producer or mic."""
with self._get_stream_state_lock():
@@ -946,6 +959,7 @@ class AudioPlayer:
except _PlaybackSuperseded: # stop() won; its generation owns cleanup
return
+ @topology.native_stream_operation()
def _play_blob(
self, pcm: bytes, source_rate: int, playback_generation: int,
output_generation: int | None = None,
@@ -1022,6 +1036,7 @@ class AudioPlayer:
elif closed is False:
self._unclean_stream = stream
+ @topology.native_stream_operation()
def _open_output_stream(self, source_rate: int) -> tuple[sd.OutputStream, int]:
"""Open a persistent ``sd.OutputStream`` (float32 stereo or mono).
@@ -1126,7 +1141,12 @@ class AudioPlayer:
blocksize=0,
latency=self._output_buffer_seconds(),
)
- stream.start()
+ topology.register_native_stream(stream)
+ try:
+ stream.start()
+ except BaseException:
+ self._close_output_stream(stream)
+ raise
self._stream_channels = stream_channels
self._device_rate_cache[cache_key] = target_rate
log.info(
@@ -1160,6 +1180,7 @@ class AudioPlayer:
raise last_exc
raise RuntimeError("No supported sample rate found")
+ @topology.native_stream_operation()
def _write_samples(
self,
stream: sd.OutputStream,
@@ -1316,6 +1337,7 @@ class AudioPlayer:
out.shape[0] / device_rate,
)
+ @topology.native_stream_operation()
def _close_output_stream(self, stream: sd.OutputStream) -> bool:
"""Flush and stop: ``stream.stop()`` blocks until the buffer is empty."""
try:
@@ -1327,6 +1349,7 @@ class AudioPlayer:
except Exception: # noqa: BLE001
log.debug("Output stream close failed", exc_info=True)
return False
+ topology.unregister_native_stream(stream)
return True
async def play_chunks(
@@ -1437,6 +1460,7 @@ class AudioPlayer:
# the main answer is dropped here rather than queued behind it.
if should_play is not None and not should_play():
return False
+ @topology.native_stream_operation()
def _ensure_stream(
needed_rate: int, output_generation: int,
) -> tuple[sd.OutputStream, int]:
@@ -1673,6 +1697,7 @@ class AudioPlayer:
if next_pull is not None:
next_pull.cancel()
+ @topology.native_stream_operation()
def abort_active(self) -> None:
"""Force-abort the live OutputStream to unblock a wedged ``stream.write``.
@@ -1706,6 +1731,7 @@ class AudioPlayer:
log.debug("abort_active: stream.abort() failed: %s", exc)
try:
stream.close()
+ topology.unregister_native_stream(stream)
except Exception as exc: # noqa: BLE001
log.warning("abort_active: stream.close() failed: %s", exc)
# Preserve ownership so the worker's ``finally`` retries the
diff --git a/jarvis/audio/topology.py b/jarvis/audio/topology.py
index c33c393f5b..de2415c473 100644
--- a/jarvis/audio/topology.py
+++ b/jarvis/audio/topology.py
@@ -14,7 +14,8 @@ This module owns the ONE cross-platform cure, in three parts:
(`jarvis.audio.devices._query_tables_fresh`), which is safe while this
process holds live streams, and compare a name-based topology signature.
No OS-specific listener; Windows, macOS, and Linux share the poller.
-2. **Coordinated refresh** — when the topology changed: discard every
+2. **Coordinated refresh** — when a topology change persists and native
+ playback is idle: discard every
registered capture stream and the player's output stream, re-initialize
PortAudio under the established re-init lock (safe ONLY with no live
streams — the BUG-058 native-fault hazard), and invalidate every resolve
@@ -24,8 +25,8 @@ This module owns the ONE cross-platform cure, in three parts:
3. **The open guard** — every native stream open holds ``stream_open_guard``
so no stream can come to life BETWEEN terminate and initialize.
-Everything fails open: a failed probe means "no judgment", a failed refresh
-leaves the process exactly where it was, and a headless install without
+Everything degrades safely: a failed probe means "no judgment", a failed
+stream close prevents re-initialization, and a headless install without
sounddevice never gets past the first probe. Nothing here runs on the boot
critical path (AP-26) — the watcher starts after honest voice readiness.
"""
@@ -37,7 +38,7 @@ import contextlib
import logging
import threading
import weakref
-from collections.abc import Callable
+from collections.abc import Callable, Iterator
from typing import Any
log = logging.getLogger("jarvis.audio.topology")
@@ -45,6 +46,12 @@ log = logging.getLogger("jarvis.audio.topology")
# Serializes native stream opens against the PortAudio re-init window. An
# RLock so a guarded open path may nest (candidate loops re-enter freely).
_OPEN_LOCK = threading.RLock()
+# A cancelled asyncio waiter can leave a native output worker running. Count
+# the synchronous operations themselves, including open-to-publication gaps.
+_native_operations = 0
+# Keep handles until close succeeds, including streams awaiting publication or
+# asynchronous disposal. Losing the Python owner does not close a native handle.
+_native_streams: dict[int, Any] = {}
# Live MicrophoneCapture instances that a refresh must quiesce. WeakSet: a
# capture that forgot to unregister can never be kept alive by the watcher.
@@ -93,11 +100,41 @@ def stream_open_guard() -> threading.RLock:
return _OPEN_LOCK
+def native_stream_operation() -> Iterator[None]:
+ """Keep PortAudio alive until a synchronous native operation has exited.
+
+ Only admission holds the open lock. Playback and abort must remain able
+ to run concurrently, and a topology refresh defers instead of waiting on
+ a potentially wedged native write.
+ """
+ global _native_operations
+ with _OPEN_LOCK:
+ _native_operations += 1
+ try:
+ yield
+ finally:
+ with _OPEN_LOCK:
+ _native_operations -= 1
+
+
def register_capture(capture: Any) -> None:
with _captures_lock:
_captures.add(capture)
+def register_native_stream(stream: Any) -> None:
+ """Called under the open guard as soon as PortAudio returns a handle."""
+ with _OPEN_LOCK:
+ _native_streams[id(stream)] = stream
+
+
+def unregister_native_stream(stream: Any) -> None:
+ """Remove a handle only after its native close completed successfully."""
+ with _OPEN_LOCK:
+ _native_streams.pop(id(stream), None)
+
+
def unregister_capture(capture: Any) -> None:
with _captures_lock:
_captures.discard(capture)
@@ -161,22 +198,32 @@ def refresh_audio_backend(player: Any, output_device: Any = None) -> bool:
from jarvis.audio.device_init import _REINIT_LOCK
with _OPEN_LOCK:
+ if _native_operations or getattr(player, "audio_backend_busy", False):
+ log.debug("Audio backend refresh deferred until playback is idle")
+ return False
with _captures_lock:
captures = list(_captures)
for capture in captures:
try:
- capture.discard_native_stream()
- except Exception: # noqa: BLE001 — one dead capture never blocks
- log.debug(
+ if capture.discard_native_stream() is False:
+ return False
+ except Exception: # noqa: BLE001 — a live stream forbids re-init
+ log.warning(
"Capture discard during refresh failed", exc_info=True
)
+ return False
if player is not None:
try:
- player.invalidate_device_cache()
+ if player.invalidate_device_cache() is False:
+ return False
except Exception: # noqa: BLE001
log.debug(
"Player invalidate during refresh failed", exc_info=True
)
+ return False
+ if _native_streams:
+ log.debug("Audio backend refresh deferred: native streams remain open")
+ return False
with _REINIT_LOCK:
with contextlib.suppress(Exception):
sd._terminate()
@@ -232,19 +279,25 @@ async def watch_topology(
if signature == last:
continue
await asyncio.sleep(_SETTLE_S)
- settled = await asyncio.to_thread(probe) or signature
+ settled = await asyncio.to_thread(probe)
+ if settled is None or settled == last:
+ # A transient enumeration change (e.g. PipeWire closing a mic)
+ # does not justify terminating the process-wide audio backend.
+ continue
log.info(
"Audio device topology changed — refreshing the audio backend "
"(streams reopen automatically)."
)
ok = await asyncio.to_thread(refresh)
- log.info("Audio backend refresh %s.", "completed" if ok else "failed")
- last = settled
+ log.info("Audio backend refresh %s.", "completed" if ok else "deferred or failed")
+ if ok:
+ last = settled
__all__ = [
"DEFAULT_POLL_S",
"refresh_audio_backend",
+ "native_stream_operation",
"register_capture",
"stream_open_guard",
"topology_signature",
diff --git a/jarvis/plugins/stt/fwhisper.py b/jarvis/plugins/stt/fwhisper.py
index 09101cac95..8f53fab07d 100644
--- a/jarvis/plugins/stt/fwhisper.py
+++ b/jarvis/plugins/stt/fwhisper.py
@@ -404,46 +404,6 @@ def ensure_cuda_libraries_findable() -> None:
return
-_GPU_COMPUTE_REMAP_LOGGED: set[str] = set()
-
-
-def _gpu_compute_type(device: str, compute_type: str) -> str:
- """The compute type a GPU engine is REALLY built with.
-
- Measured 2026-08-22 on a Blackwell card with ctranslate2 4.8: the int8
- pairs (``int8_float16``, ``int8``) decode 8 s of real speech in 10-11 s,
- ``float16`` in 0.7 s, the CPU in 1.2 s — so the shipped GPU default was
- slower than no GPU at all. With large-v3 that became "one to two minutes
- for three words" and every dictation ran into the 15 s provider ceiling.
- The int8 path on a GPU saves memory and nothing else a recognizer needs,
- so on CUDA an int8 request becomes ``float16`` whenever the runtime says
- it can do it; CPU requests are untouched (``_cpu_safe_compute_type`` owns
- those). Gated on the runtime's own capability list, never on a GPU name
- (AP-21). Never raises: a runtime that cannot answer keeps the request.
- """
- if device.lower() == "cpu" or not compute_type.lower().startswith("int8"):
- return compute_type
- try:
- import ctranslate2 # noqa: PLC0415 — lazy, the GPU path only
-
- supported = set(ctranslate2.get_supported_compute_types(device))
- except Exception as exc: # noqa: BLE001 — no answer means no remap
- log.debug("GPU compute-type probe failed (%s); keeping %s.", exc, compute_type)
- return compute_type
- if "float16" not in supported:
- return compute_type
- if compute_type not in _GPU_COMPUTE_REMAP_LOGGED:
- _GPU_COMPUTE_REMAP_LOGGED.add(compute_type)
- log.info(
- "Whisper on %s: building with float16 instead of the configured %s "
- "— the int8 kernels run an order of magnitude slower on this GPU "
- "(measured 10-11 s vs 0.7 s for 8 s of speech).",
- device,
- compute_type,
- )
- return "float16"
-
-
def _new_whisper_model(
model_name: str, device: str, compute_type: str, cpu_threads: int = 0
) -> Any:
@@ -472,7 +432,8 @@ def _new_whisper_model(
with inference_only_import_shield():
from faster_whisper import WhisperModel
- compute_type = _gpu_compute_type(device, compute_type)
+ # Preserve the requested precision. Relative int8/float16 performance and
+ # memory use vary across GPUs; one host's benchmark is not a safe default.
kwargs: dict[str, Any] = {"device": device, "compute_type": compute_type}
if cpu_threads and cpu_threads > 0:
kwargs["cpu_threads"] = int(cpu_threads)
@@ -574,6 +535,7 @@ class FasterWhisperProvider:
name = "faster-whisper"
supports_streaming = False # we can add stream_transcribe later
+ supports_concurrent_transcription = False
def __init__(
self,
diff --git a/jarvis/speech/pipeline.py b/jarvis/speech/pipeline.py
index 62e608e6ac..4ccd283fa4 100644
--- a/jarvis/speech/pipeline.py
+++ b/jarvis/speech/pipeline.py
@@ -2524,11 +2524,9 @@ class SpeechPipeline:
from jarvis.plugins.stt import build_stt_from_config
resolved = build_stt_from_config(config.stt)
- # Swap in the resolved provider when it differs from the local
- # instance — or whenever there is no local instance at all.
- if resolved is not self._stt and (
- self._stt is None or type(resolved) is not type(self._stt)
- ):
+ # The configured final recognizer owns its model and settings,
+ # even when the wake recognizer uses the same provider class.
+ if resolved is not None and resolved is not self._stt:
self._utterance_stt = resolved
log.info(
"Utterance-STT provider resolved: %s (wake stays local)",
@@ -2575,7 +2573,7 @@ class SpeechPipeline:
# In lightweight mode there is no local Whisper, but the post-wake
# utterance STT may still exist (cloud provider). Keep that path alive
# so the Listening bubble does not stay stuck on "...".
- self._probe_stt: Any = self._stt or self._utterance_stt
+ self._probe_stt: Any = self._select_probe_stt(self._utterance_stt)
# User STT dictionary (dictation-tool-style custom vocabulary): wrap the
# utterance + preview handles so EVERY provider's transcript gets the
# user's corrections — brain turns, chat dictation, and the live
@@ -3374,10 +3372,7 @@ class SpeechPipeline:
# A lightweight/cloud-first pipeline has no local preview recognizer,
# so its preview must follow the selected provider. A heavy wake path
# keeps its dedicated local probe.
- if getattr(self, "_stt", None) is None or getattr(
- self, "_probe_stt", None
- ) is previous:
- self._probe_stt = rebuilt
+ self._probe_stt = self._select_probe_stt(rebuilt)
try:
cfg_stt.provider = normalized
if model is not None:
@@ -3439,16 +3434,12 @@ class SpeechPipeline:
except Exception as exc: # noqa: BLE001 — corrections are not load-bearing here
log.debug("STT dictionary wrapper unavailable on live switch: %s", exc)
- previous = self._utterance_stt
# Same rule as ``set_stt_provider``: the meter wraps the new instance.
self._utterance_stt = meter_stt(
rebuilt, getattr(self, "_speech_spend", None), trace_id=self._speech_trace
)
- # The preview probe follows only when it was the SAME object — in the
- # local-Whisper path it is the wake model, which must keep its own
- # language (AP-27: the wake never rides on the utterance setting).
- if previous is not None and self._probe_stt is previous:
- self._probe_stt = rebuilt
+ # Re-select after wrapping: wrapper identity is not provider identity.
+ self._probe_stt = self._select_probe_stt(rebuilt)
# The dictation lane holds its OWN instance (no voice bias prompt), so a
# switch that only rebuilt the voice one would leave dictation
# transcribing in the previous recognition language for the rest of the
@@ -4189,6 +4180,20 @@ class SpeechPipeline:
self._probe_task = None
self._probe_in_flight = False
+ def _select_probe_stt(self, utterance: Any) -> Any:
+ """Use a separate wake engine or a concurrency-safe final provider."""
+ wake = getattr(self, "_stt", None)
+ candidate = wake if wake is not None else utterance
+ if candidate is utterance and not getattr(
+ candidate, "supports_concurrent_transcription", True
+ ):
+ # Cancelling an async preview does not stop its native worker.
+ # A local final decode must never compete with that same engine.
+ return None
+ from jarvis.speech.stt_dictionary import wrap_stt_with_dictionary
+
+ return wrap_stt_with_dictionary(candidate)
+
def _on_vad_probe(self, pcm: bytes, tail_loud: bool = True) -> None:
"""Sync callback from SileroEndpointer; spawns the async STT probe task.
@@ -17607,16 +17612,15 @@ class SpeechPipeline:
except Exception as exc: # noqa: BLE001
log.warning("Brain-unavailable fallback speak failed: %s", exc)
- async def _speak_stt_unavailable(self, lang: str = "de") -> None:
+ async def _speak_stt_unavailable(self, lang: str | None = None) -> None:
"""Zero-silent-drop (AD-OE6) for STT: say we couldn't transcribe the
utterance instead of dropping back to LISTENING mute when
``_transcribe_final`` exhausted its retries (sustained cloud rate-limit
/ outage). Mirrors ``_speak_brain_unavailable``. No transcript exists
- yet, so there is no detected language — default to German (the user's
- primary; runtime TTS auto-detects anyway). Failures here are swallowed:
+ yet, so use the shared output-language resolver. Failures are logged:
the fallback must never itself crash the turn.
"""
- picker_lang = _phrase_lang(lang)
+ picker_lang = _phrase_lang(lang or self._output_language(None, ""))
phrase = _STT_UNAVAILABLE_PHRASE[picker_lang]
try:
await self._set_turn_state(TurnTakingState.JARVIS_SPEAKING)
@@ -18374,8 +18378,10 @@ class SpeechPipeline:
# running conversation language instead of flipping ack/phrases/TTS
# (forensic 2026-06-18). The brain owns the sticky conversation language.
conv = getattr(brain, "conversation_language", "")
+ ui = getattr(getattr(self, "_config", None), "ui", None)
return resolve_output_language(
- pin, stt_language, text, conversation_language=conv
+ pin, stt_language, text, conversation_language=conv,
+ default=_phrase_lang(getattr(ui, "language", None)),
)
async def _speak(
diff --git a/tests/unit/audio/test_topology.py b/tests/unit/audio/test_topology.py
index 36ec3c32f1..69acc7d741 100644
--- a/tests/unit/audio/test_topology.py
+++ b/tests/unit/audio/test_topology.py
@@ -9,12 +9,21 @@ from __future__ import annotations
import asyncio
import sys
+import threading
import time
+import numpy as np
import pytest
import jarvis.audio.topology as topology
from jarvis.audio.capture import MicrophoneCapture
+from jarvis.audio.player import AudioPlayer
+
+
[email protected](autouse=True)
+def _isolated_native_handles(monkeypatch):
+ # Other audio tests deliberately return fake streams without closing them.
+ monkeypatch.setattr(topology, "_native_streams", {})
def _tables(names, default_in=0, default_out=0):
@@ -221,3 +230,158 @@ def test_capture_discard_backdates_watchdog_heartbeat() -> None:
time.monotonic() - capture._last_chunk_monotonic
> MicrophoneCapture._STALL_THRESHOLD_S
)
+
+
+async def test_watcher_retries_deferred_refresh_without_another_change(monkeypatch):
+ monkeypatch.setattr(topology, "_SETTLE_S", 0.0)
+ signatures = iter(["a", "b"])
+ attempts = []
+ completed = asyncio.Event()
+ loop = asyncio.get_running_loop()
+
+ def refresh():
+ attempts.append(True)
+ if len(attempts) == 1:
+ return False
+ loop.call_soon_threadsafe(completed.set)
+ return True
+
+ task = asyncio.create_task(topology.watch_topology(
+ None, poll_s=0.005, probe=lambda: next(signatures, "b"), refresh=refresh,
+ ))
+ try:
+ await asyncio.wait_for(completed.wait(), 2)
+ await asyncio.sleep(0.03)
+ assert len(attempts) == 2
+ finally:
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+
+async def test_watcher_ignores_change_that_disappears_during_settle(monkeypatch):
+ monkeypatch.setattr(topology, "_SETTLE_S", 0.0)
+ signatures = iter(["a", "b", "a"])
+ refreshes = []
+ task = asyncio.create_task(topology.watch_topology(
+ None, poll_s=0.005, probe=lambda: next(signatures, "a"),
+ refresh=lambda: refreshes.append(True) or True,
+ ))
+ try:
+ await asyncio.sleep(0.1)
+ assert refreshes == []
+ finally:
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+
+def _idle_player():
+ player = AudioPlayer.__new__(AudioPlayer)
+ player._play_lock = None
+ player._active_stream = None
+ player._device_rate_cache = {}
+ player._device_rate_failed = set()
+ player._device = None
+ return player
+
+
+def test_refresh_waits_for_native_write_even_without_async_playback_owner(monkeypatch):
+ entered = threading.Event()
+ release = threading.Event()
+ fake_sd = _FakeSd()
+ monkeypatch.setitem(sys.modules, "sounddevice", fake_sd)
+ player = _idle_player()
+
+ class Stream:
+ latency = 0.0
+
+ def write(self, samples):
+ entered.set()
+ assert release.wait(2)
+ return False
+
+ def stop(self):
+ pass
+
+ def close(self):
+ pass
+
+ player._active_stream = Stream()
+ failures = []
+
+ def write():
+ try:
+ player._write_samples(player._active_stream, np.zeros(16, dtype=np.int16), 16000, 16000)
+ except BaseException as exc:
+ failures.append(exc)
+
+ worker = threading.Thread(target=write)
+ worker.start()
+ try:
+ assert entered.wait(2)
+ assert topology.refresh_audio_backend(player) is False
+ assert fake_sd.calls == []
+ finally:
+ release.set()
+ worker.join(2)
+ assert not worker.is_alive()
+ assert failures == []
+ assert topology.refresh_audio_backend(player) is True
+ assert fake_sd.calls == ["terminate", "initialize"]
+
+
+async def test_refresh_defers_between_tts_chunks(monkeypatch):
+ fake_sd = _FakeSd()
+ monkeypatch.setitem(sys.modules, "sounddevice", fake_sd)
+ player = _idle_player()
+ async with player._get_play_lock():
+ assert topology.refresh_audio_backend(player) is False
+ assert fake_sd.calls == []
+ assert topology.refresh_audio_backend(player) is True
+
+
[email protected]("owner", ["capture", "player"])
+def test_failed_stream_close_prevents_termination(monkeypatch, owner):
+ fake_sd = _FakeSd()
+ monkeypatch.setitem(sys.modules, "sounddevice", fake_sd)
+
+ class UnclosableStream:
+ def abort(self):
+ pass
+
+ def stop(self):
+ pass
+
+ def close(self):
+ raise RuntimeError("device is still busy")
+
+ stream = UnclosableStream()
+ player = _idle_player()
+ capture = MicrophoneCapture.__new__(MicrophoneCapture)
+ capture._stream = stream
+ if owner == "capture":
+ topology.register_capture(capture)
+ else:
+ player._active_stream = stream
+ try:
+ assert topology.refresh_audio_backend(player) is False
+ assert fake_sd.calls == []
+ assert (capture._stream if owner == "capture" else player._active_stream) is stream
+ finally:
+ topology.unregister_capture(capture)
+
+
+def test_unpublished_or_detached_native_stream_prevents_termination(monkeypatch):
+ fake_sd = _FakeSd()
+ monkeypatch.setitem(sys.modules, "sounddevice", fake_sd)
+ stream = object()
+ with topology.stream_open_guard():
+ topology.register_native_stream(stream)
+ assert topology.refresh_audio_backend(None) is False
+ assert fake_sd.calls == []
+ topology.unregister_native_stream(stream)
+ assert topology.refresh_audio_backend(None) is True
diff --git a/tests/unit/plugins/stt/test_fwhisper_gpu_compute_type.py b/tests/unit/plugins/stt/test_fwhisper_gpu_compute_type.py
index 8862f5e119..c72e71c84f 100644
--- a/tests/unit/plugins/stt/test_fwhisper_gpu_compute_type.py
+++ b/tests/unit/plugins/stt/test_fwhisper_gpu_compute_type.py
@@ -1,7 +1,4 @@
-"""On CUDA an int8 request is built as float16 — the int8 kernels measured 10x
-slower on a Blackwell card (ctranslate2 4.8): large-v3 took 1-2 minutes for
-three words and ran into the 15 s provider ceiling. CPU requests are untouched.
-"""
+"""GPU construction preserves precision chosen for the user's hardware."""
from __future__ import annotations
@@ -13,56 +10,19 @@ import pytest
import jarvis.plugins.stt.fwhisper as fwhisper
-def _runtime(monkeypatch: pytest.MonkeyPatch, supported: set[str]) -> None:
- monkeypatch.setitem(
- sys.modules,
- "ctranslate2",
- SimpleNamespace(get_supported_compute_types=lambda _device: supported),
- )
-
-
-def test_int8_on_cuda_becomes_float16_when_the_runtime_can(monkeypatch: pytest.MonkeyPatch) -> None:
- _runtime(monkeypatch, {"int8", "int8_float16", "float16"})
- assert fwhisper._gpu_compute_type("cuda", "int8_float16") == "float16"
- assert fwhisper._gpu_compute_type("cuda", "int8") == "float16"
-
-
-def test_float16_requests_and_cpu_requests_are_untouched(monkeypatch: pytest.MonkeyPatch) -> None:
- _runtime(monkeypatch, {"int8", "int8_float16", "float16"})
- assert fwhisper._gpu_compute_type("cuda", "float16") == "float16"
- assert fwhisper._gpu_compute_type("cpu", "int8_float16") == "int8_float16"
- assert fwhisper._gpu_compute_type("cpu", "int8") == "int8"
-
-
-def test_a_runtime_without_float16_keeps_the_request(monkeypatch: pytest.MonkeyPatch) -> None:
- _runtime(monkeypatch, {"int8", "int8_float16"})
- assert fwhisper._gpu_compute_type("cuda", "int8_float16") == "int8_float16"
-
-
-def test_a_runtime_that_cannot_answer_keeps_the_request(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setitem(
- sys.modules,
- "ctranslate2",
- SimpleNamespace(
- get_supported_compute_types=lambda _d: (_ for _ in ()).throw(RuntimeError("no CUDA"))
- ),
- )
- assert fwhisper._gpu_compute_type("cuda", "int8_float16") == "int8_float16"
-
-
-def test_the_constructor_hands_the_remapped_type_to_the_engine(monkeypatch: pytest.MonkeyPatch) -> None:
- _runtime(monkeypatch, {"int8", "int8_float16", "float16"})
[email protected]("device", ["cuda", "auto", "cpu"])
[email protected]("compute_type", ["int8", "int8_float16", "float16", "float32"])
+def test_constructor_preserves_requested_precision(monkeypatch, device, compute_type) -> None:
built: list[dict] = []
- class _Model:
+ class Model:
def __init__(self, name: str, **kwargs: object) -> None:
built.append({"name": name, **kwargs})
- monkeypatch.setitem(sys.modules, "faster_whisper", SimpleNamespace(WhisperModel=_Model))
+ monkeypatch.setitem(sys.modules, "faster_whisper", SimpleNamespace(WhisperModel=Model))
monkeypatch.setattr(fwhisper, "ensure_cuda_libraries_findable", lambda: None)
- fwhisper._new_whisper_model("large-v3", "cuda", "int8_float16")
- fwhisper._new_whisper_model("large-v3", "cpu", "int8")
+ fwhisper._new_whisper_model("large-v3-turbo", device, compute_type)
- assert built[0]["compute_type"] == "float16"
- assert built[1]["compute_type"] == "int8"
+ assert built[0]["device"] == device
+ assert built[0]["compute_type"] == compute_type
diff --git a/tests/unit/speech/test_local_stt_isolation.py b/tests/unit/speech/test_local_stt_isolation.py
new file mode 100644
index 0000000000..af88a8b494
--- /dev/null
+++ b/tests/unit/speech/test_local_stt_isolation.py
@@ -0,0 +1,119 @@
+"""Local preview must not occupy the final decoder (issue #464)."""
+
+from __future__ import annotations
+
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+
+from jarvis.core.config import STTConfig
+from jarvis.core.protocols import Transcript
+from jarvis.plugins.stt.fwhisper import FasterWhisperProvider
+from jarvis.speech.pipeline import (
+ _STT_UNAVAILABLE_PHRASE,
+ SpeechPipeline,
+ TurnTakingState,
+)
+
+
+def _pipeline(monkeypatch, *, wake=None):
+ final = FasterWhisperProvider(model="small", device="cpu", language="en")
+ monkeypatch.setattr("jarvis.plugins.stt.build_stt_from_config", lambda cfg: final)
+ monkeypatch.setattr(
+ "jarvis.speech.pipeline._stt_crossover_would_leave_the_machine", lambda: True
+ )
+ pipe = SpeechPipeline(
+ stt=wake,
+ tts=SimpleNamespace(name="test-tts"),
+ config=SimpleNamespace(stt=STTConfig(provider="faster-whisper")),
+ enable_local_whisper=False,
+ enable_openwakeword=False,
+ enable_whisper_wake=False,
+ )
+ return pipe, final
+
+
+async def test_local_final_without_wake_model_never_starts_preview(monkeypatch) -> None:
+ pipe, final = _pipeline(monkeypatch)
+ calls = []
+
+ async def transcribe(pcm):
+ calls.append(pcm)
+ return Transcript(text="Open the calendar", language="en", confidence=0.99)
+
+ monkeypatch.setattr(final, "transcribe_pcm", transcribe)
+ pcm = b"\x00\x01" * 1600
+ pipe._on_vad_probe(pcm)
+ await asyncio.sleep(0)
+ assert pipe._probe_stt is None
+ assert not pipe._probe_in_flight
+ result = await pipe._transcribe_final(pcm)
+ assert result is not None and result.text == "Open the calendar"
+ assert calls == [pcm]
+
+
+def test_same_provider_class_keeps_configured_final_model_separate(monkeypatch) -> None:
+ wake = FasterWhisperProvider(model="tiny", device="cpu")
+ pipe, final = _pipeline(monkeypatch, wake=wake)
+ assert pipe._utterance_stt._model_name == "small"
+ assert pipe._probe_stt._model_name == "tiny"
+ assert pipe._probe_stt._inner is wake
+ assert pipe._utterance_stt._inner._inner is final
+
+
+def test_explicit_shared_native_provider_disables_preview() -> None:
+ wake = FasterWhisperProvider(model="tiny", device="cpu")
+ pipe = SpeechPipeline(
+ stt=wake, tts=SimpleNamespace(name="test-tts"),
+ enable_openwakeword=False, enable_whisper_wake=False,
+ )
+ assert pipe._probe_stt is None
+
+
[email protected]("setting", ["provider", "language"])
+def test_live_switch_does_not_share_local_final_with_preview(monkeypatch, setting) -> None:
+ pipe, final = _pipeline(monkeypatch)
+ pipe._probe_stt = SimpleNamespace(name="old-cloud-preview")
+ if setting == "provider":
+ assert pipe.set_stt_provider("faster-whisper")
+ else:
+ assert pipe.set_stt_language("en")
+ assert pipe._probe_stt is None
+ assert pipe._utterance_stt._model_name == final._model_name
+
+
+ "pin,conversation,ui,explicit,expected",
+ [
+ ("en", "de", "de", None, "en"),
+ ("de", "en", "en", None, "de"),
+ ("es", "en", "en", None, "es"),
+ ("auto", "es", "de", None, "es"),
+ ("auto", "", "de", None, "de"),
+ ("auto", "", "en", None, "en"),
+ ("en", "", "en", "es", "es"),
+ ],
+)
+async def test_stt_apology_uses_output_language(pin, conversation, ui, explicit, expected):
+ pipe = SpeechPipeline.__new__(SpeechPipeline)
+ pipe._config = SimpleNamespace(
+ brain=SimpleNamespace(reply_language=pin), ui=SimpleNamespace(language=ui)
+ )
+ pipe._brain = SimpleNamespace(conversation_language=conversation)
+ spoken = []
+ states = []
+
+ async def speak(text, language=None, **kwargs):
+ spoken.append((text, language))
+
+ async def set_state(state):
+ states.append(state)
+
+ pipe._speak = speak
+ pipe._set_turn_state = set_state
+ await pipe._speak_stt_unavailable(explicit)
+ assert spoken == [(_STT_UNAVAILABLE_PHRASE[expected], expected)]
+ assert states == [TurnTakingState.JARVIS_SPEAKING]