Skip to content

Personal AI, On Personal Devices

Python

feat: add voice interaction to jarvis chat (--voice flag) (#539)

jeremyboles76996bf97d5d90
diff --git a/pyproject.toml b/pyproject.toml
index ae4f00fc..382282ed 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -93,6 +93,9 @@ desktop = [
"pydantic>=2.0",
"python-multipart>=0.0.9",
"faster-whisper>=1.0",
+ "numpy>=1.24",
+ "sounddevice>=0.5",
+ "soundfile>=0.12",
]
openhands = ["openhands-sdk>=1.0; python_version >= '3.12'"]
gpu-metrics = ["pynvml>=12.0"]
@@ -131,7 +134,12 @@ security-signing = ["cryptography>=43"]
sandbox-wasm = ["wasmtime>=25"]
sandbox-docker = ["docker>=7.0"]
dashboard = ["textual>=0.80"]
-speech = ["faster-whisper>=1.0"]
+speech = [
+ "faster-whisper>=1.0",
+ "numpy>=1.24",
+ "sounddevice>=0.5",
+ "soundfile>=0.12",
+]
speech-deepgram = ["deepgram-sdk>=3.0"]
eval-wandb = ["wandb>=0.17"]
eval-sheets = ["gspread>=6.0", "google-auth>=2.0"]
diff --git a/src/openjarvis/cli/_voice_chat.py b/src/openjarvis/cli/_voice_chat.py
new file mode 100644
index 00000000..76324f68
--- /dev/null
+++ b/src/openjarvis/cli/_voice_chat.py
@@ -0,0 +1,147 @@
+"""Voice input/output helpers for the interactive chat session."""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from rich.markup import escape
+
+VOICE_EXIT = object()
+_TTS_BACKEND_ORDER = ("kokoro", "openai_tts", "cartesia")
+
+
+def _terminal_safe_text(value: object) -> str:
+ """Remove terminal controls and escape Rich markup from dynamic text."""
+ printable = "".join(
+ char for char in str(value) if char in ("\n", "\t") or char.isprintable()
+ )
+ return escape(printable)
+
+
+class VoiceSession:
+ """Cache healthy speech backends for one interactive chat session."""
+
+ def __init__(self, config: object | None = None) -> None:
+ self._config = config
+ self._stt_resolved = False
+ self._stt_backend: Any = None
+ self._tts_backend: Any = None
+ self._tts_attempted: set[str] = set()
+
+ def get_stt_backend(self) -> Any:
+ """Resolve and health-check STT once, then reuse the loaded backend."""
+ if not self._stt_resolved:
+ from openjarvis.core.config import load_config
+ from openjarvis.speech._discovery import get_speech_backend
+
+ config = self._config if self._config is not None else load_config()
+ self._stt_backend = get_speech_backend(config)
+ self._stt_resolved = True
+ return self._stt_backend
+
+ def get_tts_backend(self) -> Any:
+ """Return a cached healthy TTS backend, falling through once per key."""
+ if self._tts_backend is not None:
+ return self._tts_backend
+
+ # Import triggers built-in backend registration only when voice output
+ # is actually requested.
+ import openjarvis.speech # noqa: F401
+ from openjarvis.core.registry import TTSRegistry
+
+ for key in _TTS_BACKEND_ORDER:
+ if key in self._tts_attempted:
+ continue
+ self._tts_attempted.add(key)
+ if not TTSRegistry.contains(key):
+ continue
+ try:
+ candidate = TTSRegistry.get(key)()
+ if candidate.health():
+ self._tts_backend = candidate
+ return candidate
+ except Exception:
+ continue
+ return None
+
+ def discard_tts_backend(self) -> None:
+ """Forget a backend that failed synthesis and allow the next fallback."""
+ self._tts_backend = None
+
+
+def read_voice_input(console: Any, session: VoiceSession) -> Optional[str] | object:
+ """Accept a typed command/message, or record after an empty submission."""
+ try:
+ typed = input("You> [type, or press Enter to speak] ")
+ except (EOFError, KeyboardInterrupt):
+ return VOICE_EXIT
+ typed = typed.strip()
+ return typed if typed else record_voice(console, session)
+
+
+def record_voice(
+ console: Any,
+ session: VoiceSession | None = None,
+) -> Optional[str] | object:
+ """Record from mic, transcribe, and return text or a loop sentinel."""
+ from openjarvis.speech.voice_io import record_until_silence
+
+ active_session = session or VoiceSession()
+ backend = active_session.get_stt_backend()
+ if backend is None:
+ console.print(
+ "[red]No speech-to-text backend available. "
+ "Install the voice dependencies with: "
+ "pip install 'OpenJarvis[speech]', or configure a healthy "
+ "OpenAI/Deepgram backend.[/red]"
+ )
+ return VOICE_EXIT
+
+ console.print("[dim cyan]Listening… (speak now, stops on silence)[/dim cyan]")
+ try:
+ audio_bytes = record_until_silence()
+ except KeyboardInterrupt:
+ return VOICE_EXIT
+ except Exception as exc:
+ # PortAudio failures are regular Exceptions (and can surface as
+ # OSError), so report them without letting terminal control sequences
+ # through. SystemExit deliberately continues to propagate, while
+ # Ctrl-C maps to the chat loop's graceful-exit sentinel above.
+ console.print(f"[red]Mic error: {_terminal_safe_text(exc)}[/red]")
+ return VOICE_EXIT
+
+ console.print("[dim]Transcribing…[/dim]")
+ try:
+ result = backend.transcribe(audio_bytes, format="wav")
+ text = result.text.strip()
+ if text:
+ console.print(f"[bold]You (voice):[/bold] {_terminal_safe_text(text)}")
+ return text
+ console.print("[dim]Nothing heard — try again.[/dim]")
+ return None
+ except Exception as exc:
+ console.print(f"[red]Transcription error: {_terminal_safe_text(exc)}[/red]")
+ return None
+
+
+def speak(text: str, console: Any, session: VoiceSession | None = None) -> None:
+ """Synthesize and play text, reusing a healthy backend for the session."""
+ from openjarvis.speech.voice_io import play_wav
+
+ active_session = session or VoiceSession()
+ while (backend := active_session.get_tts_backend()) is not None:
+ try:
+ result = backend.synthesize(text, output_format="wav")
+ if result.audio:
+ play_wav(result.audio, sample_rate=result.sample_rate)
+ return
+ except Exception:
+ active_session.discard_tts_backend()
+
+ console.print(
+ "[dim yellow]No TTS backend available — install kokoro: "
+ "pip install kokoro[/dim yellow]"
+ )
+
+
+__all__ = ["VOICE_EXIT", "VoiceSession", "read_voice_input", "record_voice", "speak"]
diff --git a/src/openjarvis/cli/chat_cmd.py b/src/openjarvis/cli/chat_cmd.py
index c78f2fb7..7c767f89 100644
--- a/src/openjarvis/cli/chat_cmd.py
+++ b/src/openjarvis/cli/chat_cmd.py
@@ -13,6 +13,7 @@ from rich.markup import escape
from openjarvis.cli._runtime_panel import runtime_cli_options
from openjarvis.cli._tool_names import resolve_tool_names
+from openjarvis.cli._voice_chat import VOICE_EXIT, VoiceSession, read_voice_input, speak
from openjarvis.core.config import load_config
from openjarvis.core.events import EventBus
from openjarvis.core.types import Message, Role
@@ -70,6 +71,13 @@ def _read_input(prompt: str = "You> ") -> Optional[str]:
"(overrides config). Pass 'none' to disable all persona files."
),
)
+ "--voice",
+ "voice_mode",
+ is_flag=True,
+ default=False,
+ help="Enable voice I/O: mic input with silence detection + TTS response playback.",
+)
@runtime_cli_options
def chat(
engine_key: str | None,
@@ -79,6 +87,7 @@ def chat(
tools: str | None,
system_prompt: str | None,
persona_name: str | None,
+ voice_mode: bool,
num_ctx: int | None,
num_gpu: int | None,
skip_runtime_panel: bool,
@@ -97,6 +106,9 @@ def chat(
/runtime — Ollama context + GPU offload for this session
/help — show available commands
/history — show conversation history
+
+ Pass --voice to use microphone input (silence-detection) and hear responses
+ read back via text-to-speech (kokoro local or OpenAI TTS).
"""
console = Console(stderr=True)
@@ -247,13 +259,24 @@ def chat(
f"{escape(str(exc))}[/yellow]"
)
+ # Keep voice state outside the core chat path so picker/runtime changes can
+ # be layered independently. Loaded speech models live for this session.
+ voice_session = VoiceSession(config) if voice_mode else None
+
# Print banner
+ voice_hint = (
+ " [magenta]Voice mode ON[/magenta] — type normally, or press Enter "
+ "to speak; silence stops recording.\n"
+ if voice_mode
+ else ""
+ )
console.print(
f"[green bold]OpenJarvis Chat[/green bold]\n"
f" Engine: [cyan]{_safe_rich_label(engine_name)}[/cyan] "
f"Model: [cyan]{_safe_rich_label(model)}[/cyan]"
f" Agent: [cyan]{_safe_rich_label(agent_key or 'direct')}[/cyan]\n"
f" Runtime: [cyan]{runtime_opts.summary(engine_name=engine_name)}[/cyan]\n"
+ f"{voice_hint}"
f" Type /help for commands, /quit to exit.\n",
)
@@ -312,14 +335,23 @@ def chat(
for note in _notifications.diff(get_status()):
console.print(f"[dim cyan]{note}[/dim cyan]")
- user_input = _read_input()
- if user_input is None:
- console.print("\n[dim]Goodbye![/dim]")
- break
-
- user_input = user_input.strip()
- if not user_input:
- continue
+ if voice_mode:
+ assert voice_session is not None
+ result = read_voice_input(console, voice_session)
+ if result is VOICE_EXIT:
+ console.print("\n[dim]Goodbye![/dim]")
+ break
+ if result is None:
+ continue # nothing heard, loop again
+ user_input = result
+ else:
+ user_input = _read_input()
+ if user_input is None:
+ console.print("\n[dim]Goodbye![/dim]")
+ break
+ user_input = user_input.strip()
+ if not user_input:
+ continue
# Handle slash commands
cmd = user_input.lower()
@@ -434,6 +466,9 @@ def chat(
console.print()
console.print(Markdown(content))
console.print()
+ if voice_mode:
+ assert voice_session is not None
+ speak(content, console, voice_session)
publish_completed_exchange(
bus,
diff --git a/src/openjarvis/speech/_discovery.py b/src/openjarvis/speech/_discovery.py
index 0319193b..b7e4a6f0 100644
--- a/src/openjarvis/speech/_discovery.py
+++ b/src/openjarvis/speech/_discovery.py
@@ -64,12 +64,23 @@ def get_speech_backend(config: "JarvisConfig") -> Optional["SpeechBackend"]:
backend_key = config.speech.backend
if backend_key != "auto":
- return _create_backend(backend_key, config)
+ backend = _create_backend(backend_key, config)
+ if backend is None:
+ return None
+ try:
+ return backend if backend.health() else None
+ except Exception:
+ return None
# Auto-discovery: try each in priority order
for key in DISCOVERY_ORDER:
backend = _create_backend(key, config)
- if backend is not None:
- return backend
+ if backend is None:
+ continue
+ try:
+ if backend.health():
+ return backend
+ except Exception:
+ continue
return None
diff --git a/src/openjarvis/speech/voice_io.py b/src/openjarvis/speech/voice_io.py
new file mode 100644
index 00000000..0c0e6de5
--- /dev/null
+++ b/src/openjarvis/speech/voice_io.py
@@ -0,0 +1,126 @@
+"""Microphone recording with silence detection and audio playback helpers."""
+
+from __future__ import annotations
+
+import io
+import wave
+
+_SAMPLE_RATE = 16000
+_CHANNELS = 1
+_CHUNK = 1024
+_SILENCE_THRESHOLD = 500 # RMS below this → silence
+_SILENCE_SECONDS = 1.5 # seconds of silence before auto-stop
+_STARTUP_SILENCE_SECONDS = 5.0 # give up early if speech never begins
+_MAX_RECORD_SECONDS = 30 # safety ceiling
+
+
+def _rms(data: bytes) -> float:
+ """Compute RMS amplitude of 16-bit PCM bytes."""
+ import struct
+
+ n = len(data) // 2
+ if n == 0:
+ return 0.0
+ shorts = struct.unpack(f"{n}h", data[: n * 2])
+ return (sum(s * s for s in shorts) / n) ** 0.5
+
+
+def record_until_silence(
+ *,
+ sample_rate: int = _SAMPLE_RATE,
+ silence_threshold: int = _SILENCE_THRESHOLD,
+ silence_seconds: float = _SILENCE_SECONDS,
+ startup_silence_seconds: float = _STARTUP_SILENCE_SECONDS,
+ max_seconds: float = _MAX_RECORD_SECONDS,
+) -> bytes:
+ """Record from the default microphone until silence is detected.
+
+ Returns raw WAV bytes (16-bit mono).
+ Raises RuntimeError if sounddevice is not installed.
+ """
+ try:
+ import sounddevice as sd
+ except ImportError:
+ raise RuntimeError(
+ "sounddevice is required for voice input. "
+ "Install with: pip install sounddevice"
+ )
+
+ chunks_per_second = sample_rate / _CHUNK
+ silence_chunks = int(silence_seconds * chunks_per_second)
+ startup_silence_chunks = max(1, int(startup_silence_seconds * chunks_per_second))
+ max_chunks = int(max_seconds * chunks_per_second)
+
+ frames: list[bytes] = []
+ silence_count = 0
+ has_speech = False
+
+ with sd.RawInputStream(
+ samplerate=sample_rate,
+ channels=_CHANNELS,
+ dtype="int16",
+ blocksize=_CHUNK,
+ ) as stream:
+ for _ in range(max_chunks):
+ raw, _ = stream.read(_CHUNK)
+ data = bytes(raw)
+ frames.append(data)
+
+ amplitude = _rms(data)
+ if amplitude > silence_threshold:
+ has_speech = True
+ silence_count = 0
+ elif not has_speech and len(frames) >= startup_silence_chunks:
+ break
+ elif has_speech:
+ silence_count += 1
+ if silence_count >= silence_chunks:
+ break
+
+ return _frames_to_wav(frames, sample_rate)
+
+
+def _frames_to_wav(frames: list[bytes], sample_rate: int) -> bytes:
+ buf = io.BytesIO()
+ with wave.open(buf, "wb") as wf:
+ wf.setnchannels(_CHANNELS)
+ wf.setsampwidth(2) # 16-bit
+ wf.setframerate(sample_rate)
+ wf.writeframes(b"".join(frames))
+ return buf.getvalue()
+
+
+def play_wav(audio: bytes, sample_rate: int = 24000) -> None:
+ """Play raw WAV bytes through the default output device.
+
+ If the bytes are a valid WAV file, sample rate is read from the header;
+ otherwise ``sample_rate`` is used as a fallback.
+ """
+ try:
+ import numpy as np
+ import sounddevice as sd
+ import soundfile as sf
+ except ImportError:
+ raise RuntimeError(
+ "sounddevice, numpy, and soundfile are required for voice output. "
+ "Install with: pip install sounddevice numpy soundfile"
+ )
+
+ buf = io.BytesIO(audio)
+ try:
+ data, sr = sf.read(buf, dtype="float32")
+ except Exception:
+ # Fall back: treat as raw PCM
+ import struct
+
+ n = len(audio) // 2
+ data = (
+ np.array(struct.unpack(f"{n}h", audio[: n * 2]), dtype="float32") / 32768.0
+ )
+ sr = sample_rate
+
+ sd.play(data, sr)
+ sd.wait()
+
+
+__all__ = ["play_wav", "record_until_silence"]
diff --git a/tests/cli/test_chat_cmd.py b/tests/cli/test_chat_cmd.py
index 7509f73b..5631ca38 100644
--- a/tests/cli/test_chat_cmd.py
+++ b/tests/cli/test_chat_cmd.py
@@ -2,10 +2,14 @@
from __future__ import annotations
+from io import StringIO
+from types import SimpleNamespace
from unittest import mock
from unittest.mock import MagicMock, patch
+import pytest
from click.testing import CliRunner
+from rich.console import Console
from openjarvis.agents._stubs import (
AgentContext,
@@ -13,6 +17,7 @@ from openjarvis.agents._stubs import (
BaseAgent,
ToolUsingAgent,
)
+from openjarvis.cli._voice_chat import VOICE_EXIT, VoiceSession, record_voice, speak
from openjarvis.cli.chat_cmd import _read_input, chat
from openjarvis.core.config import JarvisConfig
from openjarvis.core.events import Event, EventBus, EventType
@@ -83,6 +88,58 @@ class TestChatCommand:
assert result.exit_code == 0
assert "/quit" in result.output
+ def test_voice_mode_preserves_typed_slash_commands(self) -> None:
+ engine = MagicMock()
+ engine.engine_id = "mock"
+ config = JarvisConfig()
+ config.intelligence.default_model = "test-model"
+
+ with (
+ patch("openjarvis.cli.chat_cmd.load_config", return_value=config),
+ patch("openjarvis.engine.get_engine", return_value=("mock", engine)),
+ patch("openjarvis.intelligence.register_builtin_models"),
+ patch("openjarvis.cli._voice_chat.record_voice") as record_voice,
+ ):
+ result = CliRunner().invoke(
+ chat,
+ ["--voice", "--model", "test-model"],
+ input="/model\n/quit\n",
+ )
+
+ assert result.exit_code == 0
+ assert "Model:" in result.output
+ record_voice.assert_not_called()
+
+ def test_voice_mode_ctrl_c_during_recording_exits_cleanly(self) -> None:
+ engine = MagicMock()
+ engine.engine_id = "mock"
+ backend = MagicMock()
+ config = JarvisConfig()
+ config.intelligence.default_model = "test-model"
+
+ with (
+ patch("openjarvis.cli.chat_cmd.load_config", return_value=config),
+ patch("openjarvis.engine.get_engine", return_value=("mock", engine)),
+ patch("openjarvis.intelligence.register_builtin_models"),
+ patch(
+ "openjarvis.speech._discovery.get_speech_backend",
+ return_value=backend,
+ ),
+ patch(
+ "openjarvis.speech.voice_io.record_until_silence",
+ side_effect=KeyboardInterrupt,
+ ),
+ ):
+ result = CliRunner().invoke(
+ chat,
+ ["--voice", "--model", "test-model"],
+ input="\n",
+ )
+
+ assert result.exit_code == 0
+ assert result.exception is None
+ assert "Goodbye!" in result.output
+
class TestReadInput:
"""Test the _read_input helper function."""
@@ -100,6 +157,157 @@ class TestReadInput:
assert _read_input() == "hello"
+class TestVoiceInput:
+ def test_unavailable_stt_is_detected_before_recording(self) -> None:
+ console = MagicMock()
+ with (
+ patch(
+ "openjarvis.speech._discovery.get_speech_backend",
+ return_value=None,
+ ),
+ patch("openjarvis.speech.voice_io.record_until_silence") as record,
+ ):
+ assert record_voice(console) is VOICE_EXIT
+
+ record.assert_not_called()
+ assert "OpenJarvis[speech]" in str(console.print.call_args)
+
+ def test_stt_backend_is_cached_for_the_chat_session(self) -> None:
+ backend = MagicMock()
+ backend.transcribe.side_effect = [
+ SimpleNamespace(text="first message"),
+ SimpleNamespace(text="second message"),
+ ]
+ session = VoiceSession(JarvisConfig())
+ console = MagicMock()
+
+ with (
+ patch(
+ "openjarvis.speech._discovery.get_speech_backend",
+ return_value=backend,
+ ) as discover,
+ patch(
+ "openjarvis.speech.voice_io.record_until_silence",
+ return_value=b"wav",
+ ),
+ ):
+ assert record_voice(console, session) == "first message"
+ assert record_voice(console, session) == "second message"
+
+ discover.assert_called_once()
+ assert backend.transcribe.call_count == 2
+
+ def test_voice_transcript_cannot_inject_rich_hyperlink(self) -> None:
+ backend = MagicMock()
+ backend.transcribe.return_value = SimpleNamespace(
+ text="[link=https://attacker.invalid]trusted[/link]\x1b]8;;evil\x07"
+ )
+ session = VoiceSession(JarvisConfig())
+ output = StringIO()
+ console = Console(file=output, force_terminal=True)
+
+ with (
+ patch(
+ "openjarvis.speech._discovery.get_speech_backend",
+ return_value=backend,
+ ),
+ patch(
+ "openjarvis.speech.voice_io.record_until_silence",
+ return_value=b"wav",
+ ),
+ ):
+ result = record_voice(console, session)
+
+ assert result.startswith("[link=https://attacker.invalid]")
+ assert "\x1b]8;" not in output.getvalue()
+ assert "[link=https://attacker.invalid]trusted[/link]" in output.getvalue()
+
+ def test_microphone_oserror_is_sanitized_and_ends_voice_session(self) -> None:
+ backend = MagicMock()
+ session = VoiceSession(JarvisConfig())
+ output = StringIO()
+ console = Console(file=output, force_terminal=True)
+ error = OSError(
+ "[link=https://attacker.invalid]permission denied[/link]\x1b]8;;evil\x07"
+ )
+
+ with (
+ patch(
+ "openjarvis.speech._discovery.get_speech_backend",
+ return_value=backend,
+ ),
+ patch(
+ "openjarvis.speech.voice_io.record_until_silence",
+ side_effect=error,
+ ),
+ ):
+ result = record_voice(console, session)
+
+ rendered = output.getvalue()
+ assert result is VOICE_EXIT
+ assert "Mic error:" in rendered
+ assert "\x1b]8;" not in rendered
+ assert "[link=https://attacker.invalid]permission denied[/link]" in rendered
+
+ def test_microphone_keyboard_interrupt_returns_voice_exit(self) -> None:
+ backend = MagicMock()
+ session = VoiceSession(JarvisConfig())
+
+ with (
+ patch(
+ "openjarvis.speech._discovery.get_speech_backend",
+ return_value=backend,
+ ),
+ patch(
+ "openjarvis.speech.voice_io.record_until_silence",
+ side_effect=KeyboardInterrupt,
+ ),
+ ):
+ assert record_voice(MagicMock(), session) is VOICE_EXIT
+
+ def test_microphone_system_exit_is_not_swallowed(self) -> None:
+ backend = MagicMock()
+ session = VoiceSession(JarvisConfig())
+
+ with (
+ patch(
+ "openjarvis.speech._discovery.get_speech_backend",
+ return_value=backend,
+ ),
+ patch(
+ "openjarvis.speech.voice_io.record_until_silence",
+ side_effect=SystemExit(),
+ ),
+ pytest.raises(SystemExit),
+ ):
+ record_voice(MagicMock(), session)
+
+ def test_tts_backend_is_cached_for_the_chat_session(self) -> None:
+ from openjarvis.core.registry import TTSRegistry
+
+ backend = MagicMock()
+ backend.health.return_value = True
+ backend.synthesize.return_value = SimpleNamespace(
+ audio=b"wav",
+ sample_rate=24000,
+ )
+ factory = MagicMock(return_value=backend)
+ session = VoiceSession(JarvisConfig())
+
+ with (
+ patch.object(TTSRegistry, "contains", return_value=True),
+ patch.object(TTSRegistry, "get", return_value=factory),
+ patch("openjarvis.speech.voice_io.play_wav") as play,
+ ):
+ speak("first", MagicMock(), session)
+ speak("second", MagicMock(), session)
+
+ factory.assert_called_once()
+ backend.health.assert_called_once()
+ assert backend.synthesize.call_count == 2
+ assert play.call_count == 2
+
+
class TestChatAgents:
def test_direct_chat_injects_auto_memory_facts(self, tmp_path) -> None:
facts_path = tmp_path / "facts.jsonl"
diff --git a/tests/speech/test_discovery.py b/tests/speech/test_discovery.py
index 50f44220..f271589b 100644
--- a/tests/speech/test_discovery.py
+++ b/tests/speech/test_discovery.py
@@ -1,6 +1,6 @@
"""Tests for speech backend auto-discovery."""
-from unittest.mock import patch
+from unittest.mock import MagicMock, call, patch
from openjarvis.core.config import JarvisConfig
@@ -46,3 +46,59 @@ def test_auto_discovery_priority():
assert DISCOVERY_ORDER[0] == "faster-whisper"
assert "openai" in DISCOVERY_ORDER
assert "deepgram" in DISCOVERY_ORDER
+
+
+def test_auto_discovery_skips_unhealthy_backend() -> None:
+ """Do not open the microphone for a backend that cannot transcribe."""
+ from openjarvis.speech._discovery import get_speech_backend
+
+ config = JarvisConfig()
+ config.speech.backend = "auto"
+ unhealthy = MagicMock(backend_id="faster-whisper")
+ unhealthy.health.return_value = False
+ healthy = MagicMock(backend_id="openai")
+ healthy.health.return_value = True
+
+ with patch(
+ "openjarvis.speech._discovery._create_backend",
+ side_effect=[unhealthy, healthy],
+ ) as create:
+ result = get_speech_backend(config)
+
+ assert result is healthy
+ assert create.call_args_list == [
+ call("faster-whisper", config),
+ call("openai", config),
+ ]
+
+
+def test_auto_discovery_continues_when_health_check_raises() -> None:
+ from openjarvis.speech._discovery import get_speech_backend
+
+ config = JarvisConfig()
+ config.speech.backend = "auto"
+ broken = MagicMock(backend_id="faster-whisper")
+ broken.health.side_effect = RuntimeError("model cannot load")
+ healthy = MagicMock(backend_id="openai")
+ healthy.health.return_value = True
+
+ with patch(
+ "openjarvis.speech._discovery._create_backend",
+ side_effect=[broken, healthy],
+ ):
+ assert get_speech_backend(config) is healthy
+
+
+def test_explicit_unhealthy_backend_is_unavailable() -> None:
+ from openjarvis.speech._discovery import get_speech_backend
+
+ config = JarvisConfig()
+ config.speech.backend = "faster-whisper"
+ backend = MagicMock()
+ backend.health.return_value = False
+
+ with patch(
+ "openjarvis.speech._discovery._create_backend",
+ return_value=backend,
+ ):
+ assert get_speech_backend(config) is None
diff --git a/tests/speech/test_voice_io.py b/tests/speech/test_voice_io.py
new file mode 100644
index 00000000..2d172e47
--- /dev/null
+++ b/tests/speech/test_voice_io.py
@@ -0,0 +1,61 @@
+"""Timing behavior for microphone silence detection."""
+
+from __future__ import annotations
+
+import struct
+import sys
+from types import SimpleNamespace
+
+from openjarvis.speech.voice_io import record_until_silence
+
+
+class _FakeStream:
+ def __init__(self, frames: list[bytes]) -> None:
+ self.frames = iter(frames)
+ self.reads = 0
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc_info):
+ return None
+
+ def read(self, chunk: int):
+ self.reads += 1
+ return next(self.frames), False
+
+
+def _install_audio(monkeypatch, stream: _FakeStream) -> None:
+ fake_sd = SimpleNamespace(RawInputStream=lambda **kwargs: stream)
+ monkeypatch.setitem(sys.modules, "numpy", SimpleNamespace())
+ monkeypatch.setitem(sys.modules, "sounddevice", fake_sd)
+
+
+def test_initial_silence_stops_at_startup_timeout(monkeypatch) -> None:
+ silence = bytes(1024 * 2)
+ stream = _FakeStream([silence] * 30)
+ _install_audio(monkeypatch, stream)
+
+ record_until_silence(
+ sample_rate=1024,
+ startup_silence_seconds=2,
+ max_seconds=30,
+ )
+
+ assert stream.reads == 2
+
+
+def test_post_speech_uses_normal_silence_window(monkeypatch) -> None:
+ speech = struct.pack("1024h", *([1000] * 1024))
+ silence = bytes(1024 * 2)
+ stream = _FakeStream([speech, silence, silence, silence])
+ _install_audio(monkeypatch, stream)
+
+ record_until_silence(
+ sample_rate=1024,
+ silence_seconds=2,
+ startup_silence_seconds=1,
+ max_seconds=30,
+ )
+
+ assert stream.reads == 3
diff --git a/uv.lock b/uv.lock
index 301495c6..1acee5e5 100644
--- a/uv.lock
+++ b/uv.lock
@@ -4206,8 +4206,11 @@ dashboard = [
desktop = [
{ name = "fastapi" },
{ name = "faster-whisper" },
+ { name = "numpy" },
{ name = "pydantic" },
{ name = "python-multipart" },
+ { name = "sounddevice" },
+ { name = "soundfile" },
{ name = "uvicorn" },
]
dev = [
@@ -4328,6 +4331,9 @@ server = [
]
speech = [
{ name = "faster-whisper" },
+ { name = "numpy" },
+ { name = "sounddevice" },
+ { name = "soundfile" },
]
speech-deepgram = [
{ name = "deepgram-sdk" },
@@ -4387,7 +4393,9 @@ requires-dist = [
{ name = "mkdocs-material", marker = "extra == 'docs'", specifier = ">=9.5" },
{ name = "mkdocstrings", extras = ["python"], marker = "extra == 'docs'", specifier = ">=0.25" },
{ name = "mlx-lm", marker = "sys_platform == 'darwin' and extra == 'inference-mlx'", specifier = ">=0.31.1" },
+ { name = "numpy", marker = "extra == 'desktop'", specifier = ">=1.24" },
{ name = "numpy", marker = "extra == 'memory-faiss'", specifier = ">=1.24" },
+ { name = "numpy", marker = "extra == 'speech'", specifier = ">=1.24" },
{ name = "nvidia-ml-py", specifier = ">=12.560.30" },
{ name = "openai", specifier = ">=1.30" },
{ name = "openai", marker = "extra == 'inference-cloud'", specifier = ">=1.30" },
@@ -4423,6 +4431,10 @@ requires-dist = [
{ name = "sentence-transformers", marker = "extra == 'memory-faiss'", specifier = ">=2.2" },
{ name = "slack-sdk", marker = "extra == 'channel-slack'", specifier = ">=3.27" },
{ name = "slixmpp", marker = "extra == 'channel-xmpp'", specifier = ">=1.8" },
+ { name = "sounddevice", marker = "extra == 'desktop'", specifier = ">=0.5" },
+ { name = "sounddevice", marker = "extra == 'speech'", specifier = ">=0.5" },
+ { name = "soundfile", marker = "extra == 'desktop'", specifier = ">=0.12" },
+ { name = "soundfile", marker = "extra == 'speech'", specifier = ">=0.12" },
{ name = "tavily-python", marker = "extra == 'tools-search'", specifier = ">=0.3" },
{ name = "textual", marker = "extra == 'dashboard'", specifier = ">=0.80" },
{ name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0" },
@@ -7202,6 +7214,43 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" },
]
+[[package]]
+name = "sounddevice"
+version = "0.5.6"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "cffi" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ec/db/0c890e2d9aab9ba284021efc02e1d3aebfecab1b611762d7434602209bcf/sounddevice-0.5.6.tar.gz", hash = "sha256:8ec9fbfde2e32f020b167e348f3ab3bac6625a5f15af524d790108ac7147a410", size = 1120094, upload-time = "2026-08-17T07:55:05.048Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/72/1f/62eef605172bddc1017508469a12f75bc7c4194ece35c734f822795f53b1/sounddevice-0.5.6-py3-none-any.whl", hash = "sha256:de099612311ad81e55d31ccbd83f43ea6bf4d87b48f9b6ea55a1fbcde0eee4e0", size = 32793, upload-time = "2026-08-17T07:54:57.507Z" },
+ { url = "https://files.pythonhosted.org/packages/b6/84/85e719d49cf98b2f406d9ac9c338892286c4448eb42ef0b2625ccf159616/sounddevice-0.5.6-py3-none-macosx_10_6_x86_64.macosx_10_6_universal2.whl", hash = "sha256:e3aef00ad8b1d1740eb66d9a7671eab88a4d2b8fa4ab33498d742e63b65c309c", size = 1009647, upload-time = "2026-08-17T07:54:58.814Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/6f/6292145099f72a153a710245f46ae43e5fb6c77bec1b6086cb76c12dc280/sounddevice-0.5.6-py3-none-win32.whl", hash = "sha256:b36b807eb02abd257198bf84b2af05e4fea199a9d2f0019014169c7136d45e9c", size = 1009627, upload-time = "2026-08-17T07:55:00.401Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/3e/cbc593c31a5f0d817b3fe97e64aa8461bd0f55cb07b67ce1b776296ae336/sounddevice-0.5.6-py3-none-win_amd64.whl", hash = "sha256:7f4162f514f007b0bf25a3ccfed3f1705bc2ec311888a90232729eec4f57a4f4", size = 1009630, upload-time = "2026-08-17T07:55:02.088Z" },
+ { url = "https://files.pythonhosted.org/packages/60/a4/b0c21c9f215a6fd9606b8f8748c21212dc098e5d5a2d93068c50edcf19b4/sounddevice-0.5.6-py3-none-win_arm64.whl", hash = "sha256:c8ae19173e5f27f8c12d4b5eee2dbfe542cee125d591e663e0fb4dfb75246d45", size = 1009630, upload-time = "2026-08-17T07:55:03.689Z" },
+]
+
+[[package]]
+name = "soundfile"
+version = "0.14.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "cffi" },
+ { name = "numpy" },
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d2/db/949331952a6fb1c5b12e9de80fd08747966c2039d1a61db4764fbd3981c2/soundfile-0.14.0.tar.gz", hash = "sha256:ba1c1a2d618bca5c406647c83b89f07cc8810fa506a50622a6993ba130c1de11", size = 47842, upload-time = "2026-06-06T08:58:47.869Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b1/d1/5e338af9ca6ed0786cd5bb03f6d60de1c325728c1189014f3b59aae7403c/soundfile-0.14.0-py2.py3-none-any.whl", hash = "sha256:8ba81ae3a89fd5ab3bef8a8eb481fbbe794e806309675a89b4df48b8d31908a8", size = 26799, upload-time = "2026-06-06T08:58:33.269Z" },
+ { url = "https://files.pythonhosted.org/packages/7e/72/c6b21e58d3113596e7e8de0a08d6f1d95173492cfbca0a4db14148cbba2a/soundfile-0.14.0-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:19be05428da76ed61a4cad29b8e4bcf43a3e5c100089d2ec81dc961eed1b0dd4", size = 1144568, upload-time = "2026-06-06T08:58:35.231Z" },
+ { url = "https://files.pythonhosted.org/packages/63/7a/dfdd6f8c748988427119f75eb860a3cedd858d1aea1fe28f39ad8559ef22/soundfile-0.14.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:d828d35a059626da52f1415b5faee610aeab393319cb3fc4a9aef47b619fc14c", size = 1103726, upload-time = "2026-06-06T08:58:37.948Z" },
+ { url = "https://files.pythonhosted.org/packages/4a/f8/fc39fad6f879633461d27394cd1ddaf1f769ffa0597dca35872f51b16461/soundfile-0.14.0-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:e85724a90bc99a6e8062c0b4ddf725f53b2a3b70afd4da875e9d2cfc4e92f377", size = 1238050, upload-time = "2026-06-06T08:58:39.932Z" },
+ { url = "https://files.pythonhosted.org/packages/7b/a2/70fd4432b924684c372df8b0a45708c36c057ef3596c9eb53e0a806b980b/soundfile-0.14.0-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:1e38bac1853412871318e82a1ba69a8be677619b56025bbfcccdb41b6cafe82d", size = 1315963, upload-time = "2026-06-06T08:58:41.716Z" },
+ { url = "https://files.pythonhosted.org/packages/d9/34/c9e80783d83eab739a9531fdee03675d53e0bf1b2ccb4bb3af5844675046/soundfile-0.14.0-py2.py3-none-win32.whl", hash = "sha256:0a6ae43c50c71b4e020cc55382925cb89451c1ed1a0c3d0f5d802da269226849", size = 902199, upload-time = "2026-06-06T08:58:43.289Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/97/b39c18ac1df45e755ca22b8b00e872929da5d107998a207a5e4ac831bfda/soundfile-0.14.0-py2.py3-none-win_amd64.whl", hash = "sha256:299491d3499460fb1b74bb4bd78b57ffc2d243a5fafa7b6ec1b264875c78453e", size = 1021480, upload-time = "2026-06-06T08:58:45.016Z" },
+ { url = "https://files.pythonhosted.org/packages/f4/83/55c65e61cf457805ce2ec157c1c6ae17715d0851aa2374422de0538838ca/soundfile-0.14.0-py2.py3-none-win_arm64.whl", hash = "sha256:e090704718e124e7c844695236f1fce8d18a5e761eaf7c82dfcd124620805f98", size = 888858, upload-time = "2026-06-06T08:58:46.593Z" },
+]
+
[[package]]
name = "sqlalchemy"
version = "2.0.48"