Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

feat(voice-engine): local-voice realtime provider adapter (not yet registered)

rubenluetke10-beepe4a09ce6e68d
diff --git a/jarvis/plugins/realtime/local_voice.py b/jarvis/plugins/realtime/local_voice.py
new file mode 100644
index 0000000000..76c58c602e
--- /dev/null
+++ b/jarvis/plugins/realtime/local_voice.py
@@ -0,0 +1,452 @@
+"""The Jarvis-owned local voice engine as a realtime provider (ADR-0037).
+
+A thin adapter: the engine runs as one child process per app (started through
+``jarvis.voice_engine.client``), and every live call is a session inside it.
+``NativeLiveVoiceSession`` drives the session through the realtime contract
+(``jarvis/realtime/protocol.py``); this module only translates between that
+contract and the engine's wire protocol (``jarvis/voice_engine/protocol.py``).
+
+Plugin rule: nothing from ``jarvis.*`` is imported at module import.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+import os
+import time
+from collections.abc import AsyncIterator
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, ClassVar
+
+log = logging.getLogger(__name__)
+
+_INPUT_RATE = 16_000
+_OUTPUT_RATE = 24_000
+_READY_TIMEOUT_S = 180.0
+_RESTART_BACKOFF_S = (1.0, 5.0, 30.0)
+
+
+@dataclass(frozen=True, slots=True)
+class _PcmChunk:
+ pcm: bytes
+ sample_rate: int
+ timestamp_ns: int = 0
+ channels: int = 1
+
+
+@dataclass(frozen=True, slots=True)
+class _Event:
+ """Duck-typed ``RealtimeEvent`` (the plugin cannot import it at module import)."""
+
+ type: str
+ audio: _PcmChunk | None = None
+ text: str | None = None
+ is_final: bool = False
+ shadow: bool = False
+ voiced_ms: int = 0
+ ms_played: int | None = None
+ error: str | None = None
+ recoverable: bool = False
+ reconnect_advised: bool = False
+ item_id: str | None = None
+ call_id: str | None = None
+ tool_name: str | None = None
+ tool_args: dict[str, Any] | None = None
+ handoff_id: str | None = None
+ provider_turn_id: str | None = None
+ usage: dict[str, int] | None = None
+ self_initiated: bool = False
+ superseded: bool = False
+
+
+@dataclass
+class EngineSettings:
+ python: str
+ package_root: str | None
+ home: str | None = None
+ languages: list[str] = field(default_factory=lambda: ["de", "en"])
+ tts: str = "pocket"
+ tts_options: dict[str, Any] = field(default_factory=dict)
+ llm_model: str = "qwen3.5:4b-voice-8k"
+ llm_base_url: str = "http://127.0.0.1:11434"
+
+ @classmethod
+ def from_config(cls, cfg: Any) -> EngineSettings:
+ section = getattr(cfg, "voice_engine", None)
+
+ def pick(name: str, default: Any) -> Any:
+ value = getattr(section, name, None) if section is not None else None
+ return default if value in (None, "") else value
+
+ home = pick("home", os.environ.get("JARVIS_VOICE_ENGINE_HOME") or None)
+ return cls(
+ python=str(pick("python", os.environ.get("JARVIS_VOICE_ENGINE_PYTHON")
+ or _default_python(home))),
+ package_root=pick("package_root", _package_root()),
+ home=home,
+ languages=list(pick("languages", ["de", "en"])),
+ tts=str(pick("tts", "pocket")),
+ tts_options=dict(pick("tts_options", {})),
+ llm_model=str(pick("llm_model", "qwen3.5:4b-voice-8k")),
+ llm_base_url=str(pick("llm_base_url", "http://127.0.0.1:11434")),
+ )
+
+ def configure_message(self) -> dict[str, Any]:
+ return {"type": "configure", "languages": self.languages, "tts": self.tts,
+ "tts_options": self.tts_options,
+ "llm": {"model": self.llm_model, "base_url": self.llm_base_url}}
+
+
+def _package_root() -> str | None:
+ """Directory that holds the ``jarvis`` package, for a source checkout."""
+ try:
+ import jarvis # noqa: PLC0415 - lazily, plugin rule
+
+ return str(Path(jarvis.__file__).resolve().parent.parent)
+ except (ImportError, AttributeError, TypeError):
+ # Frozen builds install the engine package into the engine's own
+ # environment instead (plan section 4.10).
+ return None
+
+
+def _default_python(home: str | None) -> str:
+ from jarvis.voice_engine.paths import engine_home # noqa: PLC0415
+
+ root = Path(home) if home else engine_home()
+ venv = root / "venv"
+ return str(venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python"))
+
+
+class _Engine:
+ """The one worker process of this app, its state and the routing of its output."""
+
+ def __init__(self, settings: EngineSettings) -> None:
+ self.settings = settings
+ self.phase = "stopped"
+ self.stage = ""
+ self.progress = 0.0
+ self.reason = ""
+ self.detail: dict[str, Any] = {}
+ self._client: Any = None
+ self._router: asyncio.Task[None] | None = None
+ self._audio_router: asyncio.Task[None] | None = None
+ self._ready = asyncio.Event()
+ self._start_lock = asyncio.Lock()
+ self._sessions: dict[str, LocalVoiceSession] = {}
+ self._slots: dict[int, LocalVoiceSession] = {}
+ self._next_slot = 1
+ self._failures = 0
+
+ async def ensure_started(self) -> None:
+ async with self._start_lock:
+ if self._client is not None:
+ return
+ from jarvis.voice_engine.client import EngineClient, worker_env # noqa: PLC0415
+
+ settings = self.settings
+ env = worker_env(
+ package_root=Path(settings.package_root) if settings.package_root else None,
+ home=Path(settings.home) if settings.home else None,
+ )
+ stderr = Path(settings.home) / "worker.log" if settings.home else None
+ client = EngineClient(settings.python, env=env, stderr_path=stderr)
+ self.phase, self.stage, self.reason = "starting", "process", ""
+ self._ready.clear()
+ await client.start()
+ self._client = client
+ self._router = asyncio.get_running_loop().create_task(self._route_messages(client))
+ self._audio_router = asyncio.get_running_loop().create_task(self._route_audio(client))
+ await client.send(settings.configure_message())
+
+ async def wait_ready(self, timeout_s: float) -> bool:
+ try:
+ await asyncio.wait_for(self._ready.wait(), timeout_s)
+ except TimeoutError:
+ return False
+ return self.phase == "ready"
+
+ async def _route_messages(self, client: Any) -> None:
+ while True:
+ message = await client.messages.get()
+ kind = message.get("type")
+ if kind == "_exited":
+ await self._on_exit()
+ return
+ if kind == "state":
+ self.phase = str(message.get("phase", self.phase))
+ self.stage = str(message.get("stage", ""))
+ self.progress = float(message.get("progress") or 0.0)
+ self.reason = str(message.get("reason", ""))
+ if message.get("detail"):
+ self.detail = dict(message["detail"])
+ if self.phase in ("ready", "failed"):
+ self._failures = 0 if self.phase == "ready" else self._failures
+ self._ready.set()
+ continue
+ session = self._sessions.get(str(message.get("session", "")))
+ if session is not None:
+ session._deliver(message)
+ elif kind == "error":
+ log.warning("local voice engine: %s", message.get("message"))
+
+ async def _route_audio(self, client: Any) -> None:
+ while True:
+ frame = await client.audio_frames.get()
+ session = self._slots.get(frame.slot)
+ if session is not None:
+ session._deliver_audio(frame.pcm)
+
+ async def _on_exit(self) -> None:
+ log.warning("local voice engine exited (phase %s)", self.phase)
+ for session in list(self._sessions.values()):
+ session._deliver({"type": "_engine_exited"})
+ self._sessions.clear()
+ self._slots.clear()
+ self._client = None
+ self._failures += 1
+ self.phase = "failed" if self._failures >= len(_RESTART_BACKOFF_S) else "stopped"
+ self.reason = self.reason or "The local voice engine stopped."
+ self._ready.set()
+
+ async def open(self, provider: LocalVoiceProvider, cfg: Any) -> LocalVoiceSession:
+ await self.ensure_started()
+ if not await self.wait_ready(_READY_TIMEOUT_S) or self._client is None:
+ raise RuntimeError(self.reason or "The local voice is not ready.")
+ slot = self._next_slot
+ self._next_slot = self._next_slot % 60000 + 1
+ session = LocalVoiceSession(self, slot=slot, language=getattr(cfg, "language", "en"))
+ self._sessions[session.session_id] = session
+ self._slots[slot] = session
+ await self._client.send({
+ "type": "session.open", "session": session.session_id, "slot": slot,
+ "language": session.language, "instructions": getattr(cfg, "instructions", ""),
+ "tools": list(getattr(cfg, "tools", ()) or ()),
+ "history": list(getattr(cfg, "history", ()) or ()),
+ "turn_pause_ms": getattr(cfg, "turn_pause_ms", None),
+ })
+ return session
+
+ async def send(self, message: dict[str, Any]) -> None:
+ if self._client is None:
+ raise ConnectionError("the local voice engine is not running")
+ await self._client.send(message)
+
+ async def send_audio(self, slot: int, seq: int, pcm: bytes) -> None:
+ if self._client is None:
+ raise ConnectionError("the local voice engine is not running")
+ await self._client.send_audio(slot, seq, pcm)
+
+ def forget(self, session: LocalVoiceSession) -> None:
+ self._sessions.pop(session.session_id, None)
+ self._slots.pop(session.slot, None)
+
+ async def stop(self) -> None:
+ client, self._client = self._client, None
+ for task in (self._router, self._audio_router):
+ if task is not None:
+ task.cancel()
+ if client is not None:
+ await client.close()
+ self.phase = "stopped"
+
+
+class LocalVoiceSession:
+ """One live call inside the engine, seen through the realtime contract."""
+
+ creates_responses_automatically = False
+ isolates_response_generations = True
+ supports_tool_results = True
+ supports_direct_tools = True
+
+ def __init__(self, engine: _Engine, *, slot: int, language: str) -> None:
+ import uuid # noqa: PLC0415
+
+ self._engine = engine
+ self.slot = slot
+ self.language = language
+ self.session_id = f"lv-{uuid.uuid4().hex[:12]}"
+ self.model = engine.settings.llm_model
+ self._events: asyncio.Queue[_Event | None] = asyncio.Queue()
+ self._seq = 0
+ self._closed = False
+
+ # engine -> app
+ def _deliver(self, message: dict[str, Any]) -> None:
+ kind = message.get("type")
+ if kind == "speech_started":
+ self._events.put_nowait(_Event(type="speech_started"))
+ elif kind == "transcript.input":
+ self._events.put_nowait(_Event(
+ type="input_transcript", text=str(message.get("text", "")),
+ is_final=bool(message.get("final")), voiced_ms=int(message.get("voiced_ms") or 0),
+ ))
+ elif kind == "transcript.output":
+ self._events.put_nowait(_Event(type="output_transcript_delta",
+ text=str(message.get("delta", ""))))
+ elif kind == "tool.call":
+ args = message.get("arguments")
+ self._events.put_nowait(_Event(type="tool_call", call_id=str(message.get("call_id")),
+ tool_name=str(message.get("name")),
+ tool_args=args if isinstance(args, dict) else {}))
+ elif kind == "interrupted":
+ self._events.put_nowait(_Event(type="interrupted",
+ self_initiated=bool(message.get("self_initiated"))))
+ elif kind == "response.done":
+ self._events.put_nowait(_Event(type="turn_complete"))
+ elif kind == "error":
+ self._events.put_nowait(_Event(type="error", error=str(message.get("message", "")),
+ recoverable=bool(message.get("recoverable"))))
+ elif kind == "_engine_exited":
+ self._events.put_nowait(_Event(type="error", error="The local voice engine stopped.",
+ recoverable=False))
+ self._events.put_nowait(None)
+
+ def _deliver_audio(self, pcm: bytes) -> None:
+ self._events.put_nowait(_Event(type="audio_delta", audio=_PcmChunk(
+ pcm=pcm, sample_rate=_OUTPUT_RATE, timestamp_ns=time.monotonic_ns())))
+
+ async def receive(self) -> AsyncIterator[_Event]:
+ while True:
+ event = await self._events.get()
+ if event is None:
+ return
+ yield event
+
+ # app -> engine
+ async def send_audio(self, chunk: Any) -> None:
+ if self._closed:
+ return
+ if int(getattr(chunk, "sample_rate", _INPUT_RATE)) != _INPUT_RATE:
+ raise ValueError("the local voice engine takes 16 kHz audio")
+ self._seq += 1
+ await self._engine.send_audio(self.slot, self._seq, bytes(chunk.pcm))
+
+ async def request_response(self, *, required_tool: str | None = None) -> None:
+ del required_tool # the engine has no forced-tool mode
+ await self._engine.send({"type": "response.request", "session": self.session_id,
+ "language": self.language})
+
+ async def update_session(self, *, instructions: str | None = None,
+ language: str | None = None,
+ tools: tuple[dict[str, Any], ...] | None = None,
+ turn_directive: str | None = None,
+ standing_directive: str | None = None) -> None:
+ if language:
+ self.language = language
+ text = None
+ if instructions is not None or turn_directive or standing_directive:
+ text = "\n\n".join(p for p in (instructions, standing_directive, turn_directive) if p)
+ await self._engine.send({"type": "session.update", "session": self.session_id,
+ "instructions": text, "language": language,
+ "tools": list(tools) if tools is not None else None})
+
+ async def send_text(self, text: str) -> None:
+ await self._engine.send({"type": "text", "session": self.session_id, "content": text})
+
+ async def truncate(self, audio_end_ms: int) -> None:
+ await self._engine.send({"type": "truncate", "session": self.session_id,
+ "audio_end_ms": int(audio_end_ms)})
+
+ async def interrupt(self, *, retire_input_entitlement: bool = False) -> None:
+ del retire_input_entitlement # one response per request; nothing to retire
+ await self._engine.send({"type": "interrupt", "session": self.session_id})
+
+ async def send_tool_result(self, call_id: str, name: str, result: dict[str, Any]) -> None:
+ del name
+ await self._engine.send({"type": "tool.result", "session": self.session_id,
+ "call_id": call_id, "result": result})
+
+ async def close(self) -> None:
+ if self._closed:
+ return
+ self._closed = True
+ with contextlib.suppress(ConnectionError, OSError, RuntimeError):
+ await self._engine.send({"type": "session.close", "session": self.session_id})
+ self._engine.forget(self)
+ self._events.put_nowait(None)
+
+
+class LocalVoiceProvider:
+ """Entry point class: keyless, local, native tool orchestration, browser audio."""
+
+ name = "local-voice"
+ supports_realtime = True
+ native_tool_orchestration = True
+ browser_audio = True
+ implicit_usage_fallback_allowed = False
+ eager_warm_as_fallback = False
+ credential_candidates: tuple[tuple[str, str | None], ...] = ()
+ input_sample_rate = _INPUT_RATE
+ output_sample_rate = _OUTPUT_RATE
+ handshake_budget_s = 5.0
+
+ _engine: ClassVar[_Engine | None] = None
+
+ def __init__(self, settings: EngineSettings | None = None) -> None:
+ self._settings = settings
+ self.duplex_unavailable_reason = ""
+
+ @classmethod
+ def from_runtime_config(cls, cfg: Any) -> LocalVoiceProvider:
+ return cls(EngineSettings.from_config(cfg))
+
+ @classmethod
+ def external_login_ready(cls, cfg: Any = None) -> bool:
+ """Installed means: the engine's Python exists and the core models are on disk."""
+ settings = EngineSettings.from_config(cfg)
+ if not Path(settings.python).is_file():
+ return False
+ try:
+ from jarvis.voice_engine import models # noqa: PLC0415
+
+ return all(models.is_present(n) for n in
+ ("silero-vad-v6", "smart-turn-v3.2", "parakeet-tdt-0.6b-v3-int8"))
+ except (ImportError, OSError):
+ # An unreadable model store is "not installed", never "ready".
+ return False
+
+ @classmethod
+ def _shared(cls, settings: EngineSettings) -> _Engine:
+ if cls._engine is None or cls._engine.settings != settings:
+ cls._engine = _Engine(settings)
+ return cls._engine
+
+ @classmethod
+ async def prespawn_transport(cls, cfg: Any) -> bool:
+ if not cls.external_login_ready(cfg):
+ return False
+ await cls._shared(EngineSettings.from_config(cfg)).ensure_started()
+ return True
+
+ @classmethod
+ async def warm_transport(cls, cfg: Any) -> bool:
+ if not cls.external_login_ready(cfg):
+ return False
+ engine = cls._shared(EngineSettings.from_config(cfg))
+ await engine.ensure_started()
+ return await engine.wait_ready(_READY_TIMEOUT_S)
+
+ async def can_open_duplex_session(self) -> bool:
+ settings = self._settings or EngineSettings.from_config(None)
+ engine = self._shared(settings)
+ if engine.phase == "ready":
+ self.duplex_unavailable_reason = ""
+ return True
+ if engine.phase in ("stopped", "failed") and engine._client is None:
+ with contextlib.suppress(OSError, RuntimeError):
+ await engine.ensure_started()
+ if engine.phase == "failed":
+ self.duplex_unavailable_reason = engine.reason or "The local voice could not start."
+ else:
+ percent = int(engine.progress * 100)
+ self.duplex_unavailable_reason = (
+ f"The local voice is still loading ({engine.stage or 'starting'}, {percent} %)."
+ )
+ return False
+
+ async def open_session(self, cfg: Any) -> LocalVoiceSession:
+ settings = self._settings or EngineSettings.from_config(None)
+ return await self._shared(settings).open(self, cfg)
diff --git a/tests/unit/voice_engine/test_local_voice_adapter.py b/tests/unit/voice_engine/test_local_voice_adapter.py
new file mode 100644
index 0000000000..28d7a16a69
--- /dev/null
+++ b/tests/unit/voice_engine/test_local_voice_adapter.py
@@ -0,0 +1,150 @@
+"""The local-voice provider adapter translates between the realtime contract and the engine."""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, field
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+from jarvis.plugins.realtime.local_voice import (
+ EngineSettings,
+ LocalVoiceProvider,
+ _Engine,
+)
+from jarvis.realtime.protocol import RealtimeProvider
+from jarvis.voice_engine.protocol import AudioFrame
+
+
+@dataclass
+class FakeClient:
+ """Stands in for EngineClient: records what the adapter sends."""
+
+ sent: list[dict[str, Any]] = field(default_factory=list)
+ audio: list[tuple[int, int, bytes]] = field(default_factory=list)
+ messages: asyncio.Queue[dict[str, Any]] = field(default_factory=asyncio.Queue)
+ audio_frames: asyncio.Queue[AudioFrame] = field(default_factory=asyncio.Queue)
+
+ async def send(self, message: dict[str, Any]) -> None:
+ self.sent.append(message)
+
+ async def send_audio(self, slot: int, seq: int, pcm: bytes) -> None:
+ self.audio.append((slot, seq, pcm))
+
+ async def close(self) -> int:
+ return 0
+
+
+def _settings() -> EngineSettings:
+ return EngineSettings(python="python", package_root=None, languages=["de", "en"])
+
+
+async def _running_engine() -> tuple[_Engine, FakeClient]:
+ engine = _Engine(_settings())
+ client = FakeClient()
+ engine._client = client
+ loop = asyncio.get_running_loop()
+ engine._router = loop.create_task(engine._route_messages(client))
+ engine._audio_router = loop.create_task(engine._route_audio(client))
+ client.messages.put_nowait({"type": "state", "phase": "ready", "detail": {"voices": {}}})
+ assert await engine.wait_ready(1.0)
+ return engine, client
+
+
+def test_provider_satisfies_the_realtime_contract() -> None:
+ provider = LocalVoiceProvider(_settings())
+ assert isinstance(provider, RealtimeProvider)
+ assert LocalVoiceProvider.native_tool_orchestration and LocalVoiceProvider.browser_audio
+ assert LocalVoiceProvider.implicit_usage_fallback_allowed is False
+ assert LocalVoiceProvider.credential_candidates == ()
+
+
+async def test_a_session_round_trip_is_translated_both_ways() -> None:
+ engine, client = await _running_engine()
+ cfg = SimpleNamespace(language="de", instructions="Du bist Jarvis.", # i18n-allow
+ tools=({"name": "set_timer", "description": "", "parameters": {}},),
+ history=(), turn_pause_ms=None)
+ session = await engine.open(LocalVoiceProvider(_settings()), cfg)
+ opened = client.sent[-1]
+ assert opened["type"] == "session.open" and opened["language"] == "de"
+ assert opened["tools"][0]["name"] == "set_timer"
+
+ await session.send_audio(SimpleNamespace(pcm=b"\x00\x00" * 320, sample_rate=16_000))
+ assert client.audio[-1][0] == session.slot
+
+ for message in (
+ {"type": "speech_started"},
+ {"type": "transcript.input", "text": "Stell einen Timer", "final": True, # i18n-allow
+ "voiced_ms": 900},
+ {"type": "tool.call", "call_id": "c1", "name": "set_timer", "arguments": {"minutes": 5}},
+ {"type": "transcript.output", "delta": "Erledigt. "}, # i18n-allow
+ {"type": "interrupted", "self_initiated": False},
+ {"type": "response.done", "status": "completed"},
+ ):
+ client.messages.put_nowait({**message, "session": session.session_id})
+ client.audio_frames.put_nowait(AudioFrame(slot=session.slot, seq=1, pcm=b"\x01\x00" * 480))
+
+ seen = []
+ async with asyncio.timeout(2):
+ async for event in session.receive():
+ seen.append(event)
+ if len(seen) == 7:
+ break
+ kinds = [e.type for e in seen]
+ assert kinds.count("audio_delta") == 1
+ assert [k for k in kinds if k != "audio_delta"] == [
+ "speech_started", "input_transcript", "tool_call", "output_transcript_delta",
+ "interrupted", "turn_complete",
+ ]
+ final = next(e for e in seen if e.type == "input_transcript")
+ assert final.is_final and final.voiced_ms == 900
+ call = next(e for e in seen if e.type == "tool_call")
+ assert (call.call_id, call.tool_name, call.tool_args) == ("c1", "set_timer", {"minutes": 5})
+ audio = next(e for e in seen if e.type == "audio_delta")
+ assert audio.audio.sample_rate == 24_000
+
+ await session.request_response()
+ await session.send_tool_result("c1", "set_timer", {"success": True})
+ await session.interrupt()
+ await session.truncate(1200)
+ await session.update_session(language="en", turn_directive="Be brief.")
+ await session.close()
+ kinds_sent = [m["type"] for m in client.sent[1:]]
+ assert kinds_sent == ["response.request", "tool.result", "interrupt", "truncate",
+ "session.update", "session.close"]
+ assert client.sent[1]["language"] == "de"
+ assert client.sent[5]["language"] == "en" and "Be brief." in client.sent[5]["instructions"]
+ await engine.stop()
+
+
+async def test_an_engine_exit_ends_the_call_with_an_error() -> None:
+ engine, client = await _running_engine()
+ session = await engine.open(LocalVoiceProvider(_settings()),
+ SimpleNamespace(language="en", instructions="", tools=(),
+ history=(), turn_pause_ms=None))
+ client.messages.put_nowait({"type": "_exited"})
+ events = []
+ async with asyncio.timeout(2):
+ async for event in session.receive():
+ events.append(event)
+ assert [e.type for e in events] == ["error"]
+ assert events[0].recoverable is False
+ assert engine._client is None
+
+
+async def test_a_loading_engine_refuses_fast_with_its_stage() -> None:
+ provider = LocalVoiceProvider(_settings())
+ engine = LocalVoiceProvider._shared(_settings())
+ engine._client = FakeClient()
+ engine.phase, engine.stage, engine.progress = "loading", "stt", 0.2
+ try:
+ assert await provider.can_open_duplex_session() is False
+ assert "stt" in provider.duplex_unavailable_reason
+ assert "20 %" in provider.duplex_unavailable_reason
+ finally:
+ LocalVoiceProvider._engine = None