Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
feat(voice-engine): local-voice realtime provider adapter (not yet registered)
rubenluetke10-beepe4a09ce6e68d
diff --git a/jarvis/plugins/realtime/local_voice.py b/jarvis/plugins/realtime/local_voice.pynew file mode 100644index 0000000000..76c58c602e--- /dev/null+++ b/jarvis/plugins/realtime/local_voice.py@@ -0,0 +1,452 @@+"""The Jarvis-owned local voice engine as a realtime provider (ADR-0037).++A thin adapter: the engine runs as one child process per app (started through+``jarvis.voice_engine.client``), and every live call is a session inside it.+``NativeLiveVoiceSession`` drives the session through the realtime contract+(``jarvis/realtime/protocol.py``); this module only translates between that+contract and the engine's wire protocol (``jarvis/voice_engine/protocol.py``).++Plugin rule: nothing from ``jarvis.*`` is imported at module import.+"""++from __future__ import annotations++import asyncio+import contextlib+import logging+import os+import time+from collections.abc import AsyncIterator+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any, ClassVar++log = logging.getLogger(__name__)++_INPUT_RATE = 16_000+_OUTPUT_RATE = 24_000+_READY_TIMEOUT_S = 180.0+_RESTART_BACKOFF_S = (1.0, 5.0, 30.0)+++@dataclass(frozen=True, slots=True)+class _PcmChunk:+ pcm: bytes+ sample_rate: int+ timestamp_ns: int = 0+ channels: int = 1+++@dataclass(frozen=True, slots=True)+class _Event:+ """Duck-typed ``RealtimeEvent`` (the plugin cannot import it at module import)."""++ type: str+ audio: _PcmChunk | None = None+ text: str | None = None+ is_final: bool = False+ shadow: bool = False+ voiced_ms: int = 0+ ms_played: int | None = None+ error: str | None = None+ recoverable: bool = False+ reconnect_advised: bool = False+ item_id: str | None = None+ call_id: str | None = None+ tool_name: str | None = None+ tool_args: dict[str, Any] | None = None+ handoff_id: str | None = None+ provider_turn_id: str | None = None+ usage: dict[str, int] | None = None+ self_initiated: bool = False+ superseded: bool = False+++@dataclass+class EngineSettings:+ python: str+ package_root: str | None+ home: str | None = None+ languages: list[str] = field(default_factory=lambda: ["de", "en"])+ tts: str = "pocket"+ tts_options: dict[str, Any] = field(default_factory=dict)+ llm_model: str = "qwen3.5:4b-voice-8k"+ llm_base_url: str = "http://127.0.0.1:11434"++ @classmethod+ def from_config(cls, cfg: Any) -> EngineSettings:+ section = getattr(cfg, "voice_engine", None)++ def pick(name: str, default: Any) -> Any:+ value = getattr(section, name, None) if section is not None else None+ return default if value in (None, "") else value++ home = pick("home", os.environ.get("JARVIS_VOICE_ENGINE_HOME") or None)+ return cls(+ python=str(pick("python", os.environ.get("JARVIS_VOICE_ENGINE_PYTHON")+ or _default_python(home))),+ package_root=pick("package_root", _package_root()),+ home=home,+ languages=list(pick("languages", ["de", "en"])),+ tts=str(pick("tts", "pocket")),+ tts_options=dict(pick("tts_options", {})),+ llm_model=str(pick("llm_model", "qwen3.5:4b-voice-8k")),+ llm_base_url=str(pick("llm_base_url", "http://127.0.0.1:11434")),+ )++ def configure_message(self) -> dict[str, Any]:+ return {"type": "configure", "languages": self.languages, "tts": self.tts,+ "tts_options": self.tts_options,+ "llm": {"model": self.llm_model, "base_url": self.llm_base_url}}+++def _package_root() -> str | None:+ """Directory that holds the ``jarvis`` package, for a source checkout."""+ try:+ import jarvis # noqa: PLC0415 - lazily, plugin rule++ return str(Path(jarvis.__file__).resolve().parent.parent)+ except (ImportError, AttributeError, TypeError):+ # Frozen builds install the engine package into the engine's own+ # environment instead (plan section 4.10).+ return None+++def _default_python(home: str | None) -> str:+ from jarvis.voice_engine.paths import engine_home # noqa: PLC0415++ root = Path(home) if home else engine_home()+ venv = root / "venv"+ return str(venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python"))+++class _Engine:+ """The one worker process of this app, its state and the routing of its output."""++ def __init__(self, settings: EngineSettings) -> None:+ self.settings = settings+ self.phase = "stopped"+ self.stage = ""+ self.progress = 0.0+ self.reason = ""+ self.detail: dict[str, Any] = {}+ self._client: Any = None+ self._router: asyncio.Task[None] | None = None+ self._audio_router: asyncio.Task[None] | None = None+ self._ready = asyncio.Event()+ self._start_lock = asyncio.Lock()+ self._sessions: dict[str, LocalVoiceSession] = {}+ self._slots: dict[int, LocalVoiceSession] = {}+ self._next_slot = 1+ self._failures = 0++ async def ensure_started(self) -> None:+ async with self._start_lock:+ if self._client is not None:+ return+ from jarvis.voice_engine.client import EngineClient, worker_env # noqa: PLC0415++ settings = self.settings+ env = worker_env(+ package_root=Path(settings.package_root) if settings.package_root else None,+ home=Path(settings.home) if settings.home else None,+ )+ stderr = Path(settings.home) / "worker.log" if settings.home else None+ client = EngineClient(settings.python, env=env, stderr_path=stderr)+ self.phase, self.stage, self.reason = "starting", "process", ""+ self._ready.clear()+ await client.start()+ self._client = client+ self._router = asyncio.get_running_loop().create_task(self._route_messages(client))+ self._audio_router = asyncio.get_running_loop().create_task(self._route_audio(client))+ await client.send(settings.configure_message())++ async def wait_ready(self, timeout_s: float) -> bool:+ try:+ await asyncio.wait_for(self._ready.wait(), timeout_s)+ except TimeoutError:+ return False+ return self.phase == "ready"++ async def _route_messages(self, client: Any) -> None:+ while True:+ message = await client.messages.get()+ kind = message.get("type")+ if kind == "_exited":+ await self._on_exit()+ return+ if kind == "state":+ self.phase = str(message.get("phase", self.phase))+ self.stage = str(message.get("stage", ""))+ self.progress = float(message.get("progress") or 0.0)+ self.reason = str(message.get("reason", ""))+ if message.get("detail"):+ self.detail = dict(message["detail"])+ if self.phase in ("ready", "failed"):+ self._failures = 0 if self.phase == "ready" else self._failures+ self._ready.set()+ continue+ session = self._sessions.get(str(message.get("session", "")))+ if session is not None:+ session._deliver(message)+ elif kind == "error":+ log.warning("local voice engine: %s", message.get("message"))++ async def _route_audio(self, client: Any) -> None:+ while True:+ frame = await client.audio_frames.get()+ session = self._slots.get(frame.slot)+ if session is not None:+ session._deliver_audio(frame.pcm)++ async def _on_exit(self) -> None:+ log.warning("local voice engine exited (phase %s)", self.phase)+ for session in list(self._sessions.values()):+ session._deliver({"type": "_engine_exited"})+ self._sessions.clear()+ self._slots.clear()+ self._client = None+ self._failures += 1+ self.phase = "failed" if self._failures >= len(_RESTART_BACKOFF_S) else "stopped"+ self.reason = self.reason or "The local voice engine stopped."+ self._ready.set()++ async def open(self, provider: LocalVoiceProvider, cfg: Any) -> LocalVoiceSession:+ await self.ensure_started()+ if not await self.wait_ready(_READY_TIMEOUT_S) or self._client is None:+ raise RuntimeError(self.reason or "The local voice is not ready.")+ slot = self._next_slot+ self._next_slot = self._next_slot % 60000 + 1+ session = LocalVoiceSession(self, slot=slot, language=getattr(cfg, "language", "en"))+ self._sessions[session.session_id] = session+ self._slots[slot] = session+ await self._client.send({+ "type": "session.open", "session": session.session_id, "slot": slot,+ "language": session.language, "instructions": getattr(cfg, "instructions", ""),+ "tools": list(getattr(cfg, "tools", ()) or ()),+ "history": list(getattr(cfg, "history", ()) or ()),+ "turn_pause_ms": getattr(cfg, "turn_pause_ms", None),+ })+ return session++ async def send(self, message: dict[str, Any]) -> None:+ if self._client is None:+ raise ConnectionError("the local voice engine is not running")+ await self._client.send(message)++ async def send_audio(self, slot: int, seq: int, pcm: bytes) -> None:+ if self._client is None:+ raise ConnectionError("the local voice engine is not running")+ await self._client.send_audio(slot, seq, pcm)++ def forget(self, session: LocalVoiceSession) -> None:+ self._sessions.pop(session.session_id, None)+ self._slots.pop(session.slot, None)++ async def stop(self) -> None:+ client, self._client = self._client, None+ for task in (self._router, self._audio_router):+ if task is not None:+ task.cancel()+ if client is not None:+ await client.close()+ self.phase = "stopped"+++class LocalVoiceSession:+ """One live call inside the engine, seen through the realtime contract."""++ creates_responses_automatically = False+ isolates_response_generations = True+ supports_tool_results = True+ supports_direct_tools = True++ def __init__(self, engine: _Engine, *, slot: int, language: str) -> None:+ import uuid # noqa: PLC0415++ self._engine = engine+ self.slot = slot+ self.language = language+ self.session_id = f"lv-{uuid.uuid4().hex[:12]}"+ self.model = engine.settings.llm_model+ self._events: asyncio.Queue[_Event | None] = asyncio.Queue()+ self._seq = 0+ self._closed = False++ # engine -> app+ def _deliver(self, message: dict[str, Any]) -> None:+ kind = message.get("type")+ if kind == "speech_started":+ self._events.put_nowait(_Event(type="speech_started"))+ elif kind == "transcript.input":+ self._events.put_nowait(_Event(+ type="input_transcript", text=str(message.get("text", "")),+ is_final=bool(message.get("final")), voiced_ms=int(message.get("voiced_ms") or 0),+ ))+ elif kind == "transcript.output":+ self._events.put_nowait(_Event(type="output_transcript_delta",+ text=str(message.get("delta", ""))))+ elif kind == "tool.call":+ args = message.get("arguments")+ self._events.put_nowait(_Event(type="tool_call", call_id=str(message.get("call_id")),+ tool_name=str(message.get("name")),+ tool_args=args if isinstance(args, dict) else {}))+ elif kind == "interrupted":+ self._events.put_nowait(_Event(type="interrupted",+ self_initiated=bool(message.get("self_initiated"))))+ elif kind == "response.done":+ self._events.put_nowait(_Event(type="turn_complete"))+ elif kind == "error":+ self._events.put_nowait(_Event(type="error", error=str(message.get("message", "")),+ recoverable=bool(message.get("recoverable"))))+ elif kind == "_engine_exited":+ self._events.put_nowait(_Event(type="error", error="The local voice engine stopped.",+ recoverable=False))+ self._events.put_nowait(None)++ def _deliver_audio(self, pcm: bytes) -> None:+ self._events.put_nowait(_Event(type="audio_delta", audio=_PcmChunk(+ pcm=pcm, sample_rate=_OUTPUT_RATE, timestamp_ns=time.monotonic_ns())))++ async def receive(self) -> AsyncIterator[_Event]:+ while True:+ event = await self._events.get()+ if event is None:+ return+ yield event++ # app -> engine+ async def send_audio(self, chunk: Any) -> None:+ if self._closed:+ return+ if int(getattr(chunk, "sample_rate", _INPUT_RATE)) != _INPUT_RATE:+ raise ValueError("the local voice engine takes 16 kHz audio")+ self._seq += 1+ await self._engine.send_audio(self.slot, self._seq, bytes(chunk.pcm))++ async def request_response(self, *, required_tool: str | None = None) -> None:+ del required_tool # the engine has no forced-tool mode+ await self._engine.send({"type": "response.request", "session": self.session_id,+ "language": self.language})++ async def update_session(self, *, instructions: str | None = None,+ language: str | None = None,+ tools: tuple[dict[str, Any], ...] | None = None,+ turn_directive: str | None = None,+ standing_directive: str | None = None) -> None:+ if language:+ self.language = language+ text = None+ if instructions is not None or turn_directive or standing_directive:+ text = "\n\n".join(p for p in (instructions, standing_directive, turn_directive) if p)+ await self._engine.send({"type": "session.update", "session": self.session_id,+ "instructions": text, "language": language,+ "tools": list(tools) if tools is not None else None})++ async def send_text(self, text: str) -> None:+ await self._engine.send({"type": "text", "session": self.session_id, "content": text})++ async def truncate(self, audio_end_ms: int) -> None:+ await self._engine.send({"type": "truncate", "session": self.session_id,+ "audio_end_ms": int(audio_end_ms)})++ async def interrupt(self, *, retire_input_entitlement: bool = False) -> None:+ del retire_input_entitlement # one response per request; nothing to retire+ await self._engine.send({"type": "interrupt", "session": self.session_id})++ async def send_tool_result(self, call_id: str, name: str, result: dict[str, Any]) -> None:+ del name+ await self._engine.send({"type": "tool.result", "session": self.session_id,+ "call_id": call_id, "result": result})++ async def close(self) -> None:+ if self._closed:+ return+ self._closed = True+ with contextlib.suppress(ConnectionError, OSError, RuntimeError):+ await self._engine.send({"type": "session.close", "session": self.session_id})+ self._engine.forget(self)+ self._events.put_nowait(None)+++class LocalVoiceProvider:+ """Entry point class: keyless, local, native tool orchestration, browser audio."""++ name = "local-voice"+ supports_realtime = True+ native_tool_orchestration = True+ browser_audio = True+ implicit_usage_fallback_allowed = False+ eager_warm_as_fallback = False+ credential_candidates: tuple[tuple[str, str | None], ...] = ()+ input_sample_rate = _INPUT_RATE+ output_sample_rate = _OUTPUT_RATE+ handshake_budget_s = 5.0++ _engine: ClassVar[_Engine | None] = None++ def __init__(self, settings: EngineSettings | None = None) -> None:+ self._settings = settings+ self.duplex_unavailable_reason = ""++ @classmethod+ def from_runtime_config(cls, cfg: Any) -> LocalVoiceProvider:+ return cls(EngineSettings.from_config(cfg))++ @classmethod+ def external_login_ready(cls, cfg: Any = None) -> bool:+ """Installed means: the engine's Python exists and the core models are on disk."""+ settings = EngineSettings.from_config(cfg)+ if not Path(settings.python).is_file():+ return False+ try:+ from jarvis.voice_engine import models # noqa: PLC0415++ return all(models.is_present(n) for n in+ ("silero-vad-v6", "smart-turn-v3.2", "parakeet-tdt-0.6b-v3-int8"))+ except (ImportError, OSError):+ # An unreadable model store is "not installed", never "ready".+ return False++ @classmethod+ def _shared(cls, settings: EngineSettings) -> _Engine:+ if cls._engine is None or cls._engine.settings != settings:+ cls._engine = _Engine(settings)+ return cls._engine++ @classmethod+ async def prespawn_transport(cls, cfg: Any) -> bool:+ if not cls.external_login_ready(cfg):+ return False+ await cls._shared(EngineSettings.from_config(cfg)).ensure_started()+ return True++ @classmethod+ async def warm_transport(cls, cfg: Any) -> bool:+ if not cls.external_login_ready(cfg):+ return False+ engine = cls._shared(EngineSettings.from_config(cfg))+ await engine.ensure_started()+ return await engine.wait_ready(_READY_TIMEOUT_S)++ async def can_open_duplex_session(self) -> bool:+ settings = self._settings or EngineSettings.from_config(None)+ engine = self._shared(settings)+ if engine.phase == "ready":+ self.duplex_unavailable_reason = ""+ return True+ if engine.phase in ("stopped", "failed") and engine._client is None:+ with contextlib.suppress(OSError, RuntimeError):+ await engine.ensure_started()+ if engine.phase == "failed":+ self.duplex_unavailable_reason = engine.reason or "The local voice could not start."+ else:+ percent = int(engine.progress * 100)+ self.duplex_unavailable_reason = (+ f"The local voice is still loading ({engine.stage or 'starting'}, {percent} %)."+ )+ return False++ async def open_session(self, cfg: Any) -> LocalVoiceSession:+ settings = self._settings or EngineSettings.from_config(None)+ return await self._shared(settings).open(self, cfg)diff --git a/tests/unit/voice_engine/test_local_voice_adapter.py b/tests/unit/voice_engine/test_local_voice_adapter.pynew file mode 100644index 0000000000..28d7a16a69--- /dev/null+++ b/tests/unit/voice_engine/test_local_voice_adapter.py@@ -0,0 +1,150 @@+"""The local-voice provider adapter translates between the realtime contract and the engine."""++from __future__ import annotations++import asyncio+from dataclasses import dataclass, field+from types import SimpleNamespace+from typing import Any++import pytest++from jarvis.plugins.realtime.local_voice import (+ EngineSettings,+ LocalVoiceProvider,+ _Engine,+)+from jarvis.realtime.protocol import RealtimeProvider+from jarvis.voice_engine.protocol import AudioFrame+++@dataclass+class FakeClient:+ """Stands in for EngineClient: records what the adapter sends."""++ sent: list[dict[str, Any]] = field(default_factory=list)+ audio: list[tuple[int, int, bytes]] = field(default_factory=list)+ messages: asyncio.Queue[dict[str, Any]] = field(default_factory=asyncio.Queue)+ audio_frames: asyncio.Queue[AudioFrame] = field(default_factory=asyncio.Queue)++ async def send(self, message: dict[str, Any]) -> None:+ self.sent.append(message)++ async def send_audio(self, slot: int, seq: int, pcm: bytes) -> None:+ self.audio.append((slot, seq, pcm))++ async def close(self) -> int:+ return 0+++def _settings() -> EngineSettings:+ return EngineSettings(python="python", package_root=None, languages=["de", "en"])+++async def _running_engine() -> tuple[_Engine, FakeClient]:+ engine = _Engine(_settings())+ client = FakeClient()+ engine._client = client+ loop = asyncio.get_running_loop()+ engine._router = loop.create_task(engine._route_messages(client))+ engine._audio_router = loop.create_task(engine._route_audio(client))+ client.messages.put_nowait({"type": "state", "phase": "ready", "detail": {"voices": {}}})+ assert await engine.wait_ready(1.0)+ return engine, client+++def test_provider_satisfies_the_realtime_contract() -> None:+ provider = LocalVoiceProvider(_settings())+ assert isinstance(provider, RealtimeProvider)+ assert LocalVoiceProvider.native_tool_orchestration and LocalVoiceProvider.browser_audio+ assert LocalVoiceProvider.implicit_usage_fallback_allowed is False+ assert LocalVoiceProvider.credential_candidates == ()+++async def test_a_session_round_trip_is_translated_both_ways() -> None:+ engine, client = await _running_engine()+ cfg = SimpleNamespace(language="de", instructions="Du bist Jarvis.", # i18n-allow+ tools=({"name": "set_timer", "description": "", "parameters": {}},),+ history=(), turn_pause_ms=None)+ session = await engine.open(LocalVoiceProvider(_settings()), cfg)+ opened = client.sent[-1]+ assert opened["type"] == "session.open" and opened["language"] == "de"+ assert opened["tools"][0]["name"] == "set_timer"++ await session.send_audio(SimpleNamespace(pcm=b"\x00\x00" * 320, sample_rate=16_000))+ assert client.audio[-1][0] == session.slot++ for message in (+ {"type": "speech_started"},+ {"type": "transcript.input", "text": "Stell einen Timer", "final": True, # i18n-allow+ "voiced_ms": 900},+ {"type": "tool.call", "call_id": "c1", "name": "set_timer", "arguments": {"minutes": 5}},+ {"type": "transcript.output", "delta": "Erledigt. "}, # i18n-allow+ {"type": "interrupted", "self_initiated": False},+ {"type": "response.done", "status": "completed"},+ ):+ client.messages.put_nowait({**message, "session": session.session_id})+ client.audio_frames.put_nowait(AudioFrame(slot=session.slot, seq=1, pcm=b"\x01\x00" * 480))++ seen = []+ async with asyncio.timeout(2):+ async for event in session.receive():+ seen.append(event)+ if len(seen) == 7:+ break+ kinds = [e.type for e in seen]+ assert kinds.count("audio_delta") == 1+ assert [k for k in kinds if k != "audio_delta"] == [+ "speech_started", "input_transcript", "tool_call", "output_transcript_delta",+ "interrupted", "turn_complete",+ ]+ final = next(e for e in seen if e.type == "input_transcript")+ assert final.is_final and final.voiced_ms == 900+ call = next(e for e in seen if e.type == "tool_call")+ assert (call.call_id, call.tool_name, call.tool_args) == ("c1", "set_timer", {"minutes": 5})+ audio = next(e for e in seen if e.type == "audio_delta")+ assert audio.audio.sample_rate == 24_000++ await session.request_response()+ await session.send_tool_result("c1", "set_timer", {"success": True})+ await session.interrupt()+ await session.truncate(1200)+ await session.update_session(language="en", turn_directive="Be brief.")+ await session.close()+ kinds_sent = [m["type"] for m in client.sent[1:]]+ assert kinds_sent == ["response.request", "tool.result", "interrupt", "truncate",+ "session.update", "session.close"]+ assert client.sent[1]["language"] == "de"+ assert client.sent[5]["language"] == "en" and "Be brief." in client.sent[5]["instructions"]+ await engine.stop()+++async def test_an_engine_exit_ends_the_call_with_an_error() -> None:+ engine, client = await _running_engine()+ session = await engine.open(LocalVoiceProvider(_settings()),+ SimpleNamespace(language="en", instructions="", tools=(),+ history=(), turn_pause_ms=None))+ client.messages.put_nowait({"type": "_exited"})+ events = []+ async with asyncio.timeout(2):+ async for event in session.receive():+ events.append(event)+ assert [e.type for e in events] == ["error"]+ assert events[0].recoverable is False+ assert engine._client is None+++async def test_a_loading_engine_refuses_fast_with_its_stage() -> None:+ provider = LocalVoiceProvider(_settings())+ engine = LocalVoiceProvider._shared(_settings())+ engine._client = FakeClient()+ engine.phase, engine.stage, engine.progress = "loading", "stt", 0.2+ try:+ assert await provider.can_open_duplex_session() is False+ assert "stt" in provider.duplex_unavailable_reason+ assert "20 %" in provider.duplex_unavailable_reason+ finally:+ LocalVoiceProvider._engine = None