Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
fix(realtime): let an unpinned engine decide the browser-audio transport (#403)
Ruben Lütkefd29b230d4e5
diff --git a/docs/BUGS.md b/docs/BUGS.mdindex 3b579c1782..635a3af118 100644--- a/docs/BUGS.md+++ b/docs/BUGS.md@@ -16036,8 +16036,8 @@ never has; the only browser-side start was the desktop's realtime hand-over(`BrowserVoiceRequested`). (3) The visible browser-microphone card that couldstart a call itself left the sidebar on 2026-09-12 (`51b4d4015`). The issue'snote that Gemini Live lacks browser audio is not the cause: the provider-declares it; `browser_audio` in `/api/settings/voice-mode` reads only an-explicitly pinned realtime provider.+declares it; `browser_audio` in `/api/settings/voice-mode` read only an+explicitly pinned realtime provider (fixed separately as BUG-232).**Fix.** `BrowserVoiceConfig(enabled=True)` is now `JarvisConfig.browser_voice`;`browser_voice_enabled(cfg)` serves the classic bridge in pipeline mode by@@ -16050,3 +16050,27 @@ Guards: `tests/unit/web/test_voice_mode_route.py`,`tests/unit/browser_voice/test_route.py`,`tests/unit/ui/test_voice_call_routes.py`, `useVoiceCall.test.tsx`,`BrowserRealtimeControl.test.tsx`.++## BUG-232: an unpinned browser-audio engine ran on the desktop's half-duplex path and was reported as "no browser audio" (MEDIUM, FIXED 2026-10-05)++**Symptom.** With no `[brain.realtime].provider` pinned, a Gemini-only install+showed `active_provider: gemini-live` next to `browser_audio: false` in+`GET /api/settings/voice-mode` (seen in GitHub issue #399). On the desktop the+same call ran on the native half-duplex path instead of the browser hand-over+that a pinned Gemini Live call takes.++**Cause.** `realtime_browser_audio(cfg)` read only the explicitly pinned+primary. Without a pin, the session builder opens on the first+credential-ready provider in effective order, so the transport decision and+the provider the call actually used disagreed for every unpinned install.++**Fix.** Without a pin, `realtime_browser_audio` reads the capability of the+first credential-ready provider (`_identified_provider_candidates(...,+limit=1)`: the same order and the same refusal rules as+`build_realtime_session`); a pinned primary still answers for itself without+reading credentials. The settings route and both pipeline call sites now read+it off their loops. Speech-suite tests pick the transport as a keyless host+does (`tests/unit/speech/conftest.py`), so a developer's real key no longer+flips desktop-path tests. Guards: `tests/unit/realtime/test_factory.py`,+`tests/unit/web/test_voice_mode_route.py`,+`tests/unit/speech/test_realtime_mode.py`.diff --git a/jarvis/realtime/factory.py b/jarvis/realtime/factory.pyindex 1a25c8efed..8881f419ee 100644--- a/jarvis/realtime/factory.py+++ b/jarvis/realtime/factory.py@@ -88,7 +88,10 @@ def realtime_implicit_usage_fallback_allowed(cfg: Any) -> bool:def _identified_provider_candidates(- cfg: Any, *, defer_external_login_probe: bool = False+ cfg: Any,+ *,+ defer_external_login_probe: bool = False,+ limit: int | None = None,) -> list[tuple[str, Any]]:"""Instantiate every credential-ready plugin in effective fallback order.@@ -100,6 +103,9 @@ def _identified_provider_candidates(Each candidate is paired with the entry-point id it was loaded from, whicha caller cannot recover from the instance: ``name`` is a provider-chosenlabel and may differ from the id the registry knows.++ ``limit`` stops after that many candidates, so a caller that only needs+ the provider a call would open first reads no further credentials."""candidates: list[tuple[str, Any]] = []explicit_ids = set(_explicit_provider_ids(cfg))@@ -168,6 +174,8 @@ def _identified_provider_candidates()continuecandidates.append((provider_id, provider))+ if limit is not None and len(candidates) >= limit:+ breakexcept Exception as exc: # noqa: BLE001 — one plugin must not brick otherslog.warning("Realtime plugin %s is unavailable: %s", provider_id, exc)return candidates@@ -523,14 +531,37 @@ def realtime_webrtc_start_event_required(cfg: Any) -> bool:def realtime_browser_audio(cfg: Any) -> bool:- """Whether the selected engine needs browser echo-cancelled audio."""+ """Whether the engine a call would open needs browser echo-cancelled audio.++ A pinned primary answers for itself. Without a pin the call opens on the+ first credential-ready provider in effective order — the one+ ``build_realtime_session`` builds and ``/api/settings/voice-mode``+ reports as ``active_provider`` — so that provider's capability decides.+ Reading the pin alone answered ``False`` for an automatically chosen+ browser-audio provider such as Gemini Live: the desktop then ran that call+ on the half-duplex native path and every surface reported the wrong+ transport (follow-up to issue #399).++ The unpinned answer reads credentials, so callers keep it off the voice+ and request loops.+ """+ if cfg is None:+ return Falsefor provider_id in _explicit_provider_ids(cfg)[:1]:try:provider = load(_GROUP, provider_id, protocol=RealtimeProvider)return bool(getattr(provider, "browser_audio", False))except Exception:log.warning("Browser audio capability unavailable", exc_info=True)- return False+ return False+ # The same candidate order the session builder uses, including its refusal+ # to fall back ambiently behind a pin that is missing or does not load.+ candidates = _identified_provider_candidates(+ cfg, defer_external_login_probe=True, limit=1+ )+ if not candidates:+ return False+ return bool(getattr(candidates[0][1], "browser_audio", False))def build_realtime_session(diff --git a/jarvis/speech/pipeline.py b/jarvis/speech/pipeline.pyindex d62a6a0245..9cdda91461 100644--- a/jarvis/speech/pipeline.py+++ b/jarvis/speech/pipeline.py@@ -8227,10 +8227,15 @@ class SpeechPipeline:"""Borrow the wake mic or open exactly one fallback session mic."""from jarvis.realtime.factory import realtime_browser_audio+ config = getattr(self, "_config", None)+ # An unpinned engine is resolved from the credentials, so the answer+ # is read off the voice loop, like the session build.browser_media = (not getattr(self, "_ptt_mode", False)and getattr(self, "_active_voice_mode", None) == "realtime"- and realtime_browser_audio(getattr(self, "_config", None))+ and bool(+ await _run_voice_critical_thread(lambda: realtime_browser_audio(config))+ ))buffer = await self._claim_wake_capture_for_session()if browser_media and buffer is None:@@ -9812,12 +9817,25 @@ class SpeechPipeline:"""from jarvis.realtime.factory import realtime_browser_audio, realtime_handshake_budget_s- if realtime_browser_audio(self._config):+ config = self._config++ def _browser_handover_budget_s() -> float | None:+ """The hand-over budget when the browser owns this call, else ``None``.++ Both answers can read credentials (an unpinned engine is resolved+ from the keys), so they run off the voice loop together.+ """+ if not realtime_browser_audio(config):+ return None+ return realtime_handshake_budget_s(config)++ handover_budget_s = await _run_voice_critical_thread(_browser_handover_budget_s)+ if handover_budget_s is not None:from jarvis.live.runtime import run_browser_callreturn await run_browser_call(self._bus, self._hangup_event,- timeout_s=max(45.0, realtime_handshake_budget_s(self._config) + 5.0),+ timeout_s=max(45.0, handover_budget_s + 5.0),input_buffer=input_buffer,session_id=self._current_voice_session_id or "",)diff --git a/jarvis/ui/web/settings_routes.py b/jarvis/ui/web/settings_routes.pyindex 4d57a693d6..35adfbca4b 100644--- a/jarvis/ui/web/settings_routes.py+++ b/jarvis/ui/web/settings_routes.py@@ -325,7 +325,9 @@ async def get_voice_mode(request: Request) -> dict[str, object]:# Capability, not a provider id (AP-21): the surface must not call a start# attempt dead while the backend is still inside a budget it declared.handshake_budget_s = await asyncio.to_thread(_realtime_handshake_budget_s, cfg)- browser_audio = realtime_browser_audio(cfg)+ # Without a pin the answer follows the first credential-ready provider,+ # which reads credentials — off the loop like the lookups above.+ browser_audio = await asyncio.to_thread(realtime_browser_audio, cfg)transport_offer_ready = (None if browser_audio else await _realtime_transport_offer_ready(requires_webrtc_offer))diff --git a/tests/unit/realtime/test_factory.py b/tests/unit/realtime/test_factory.pyindex 87ceaecdac..a61769a204 100644--- a/tests/unit/realtime/test_factory.py+++ b/tests/unit/realtime/test_factory.py@@ -13,6 +13,7 @@ from jarvis.realtime.factory import (_resolve_realtime_provider,build_realtime_session,realtime_available_provider,+ realtime_browser_audio,realtime_requires_webrtc_offer,)@@ -41,6 +42,8 @@ class _OpenAIProvider(_BaseProvider):class _GeminiProvider(_BaseProvider):name = "gemini-live"credential_candidates = (("gemini_api_key", "GEMINI_API_KEY"),)+ # Mirrors the real adapter: its calls need browser echo-cancelled audio.+ browser_audio = Trueclass _AcmeProvider(_BaseProvider):@@ -200,6 +203,52 @@ def test_logged_in_subscription_is_never_an_ambient_candidate(monkeypatch):assert realtime_available_provider(config) is None+def test_unpinned_browser_audio_follows_the_automatically_chosen_provider(monkeypatch):+ # Issue #399: a Gemini-only install without a pin opens its call on+ # Gemini Live, yet browser_audio read only the (absent) pin and said no.+ _fake_registry(monkeypatch, {"gemini"})+ config = _cfg(provider="")++ assert realtime_available_provider(config) == "gemini-live"+ assert realtime_browser_audio(config) is True+++def test_unpinned_browser_audio_is_false_without_a_ready_provider(monkeypatch):+ _fake_registry(monkeypatch, set())++ assert realtime_browser_audio(_cfg(provider="")) is False+ assert realtime_browser_audio(None) is False+++def test_unpinned_browser_audio_reads_only_up_to_the_first_ready_provider(monkeypatch):+ # The first installed provider with a key decides; credentials of the+ # providers behind it are never read on this path.+ _fake_registry(monkeypatch, {"openai", "gemini"})+ read: list[str] = []+ resolve = factory.get_secret_any++ def _spy(candidates):+ read.append(candidates[0][0])+ return resolve(candidates)++ monkeypatch.setattr(factory, "get_secret_any", _spy)++ assert realtime_browser_audio(_cfg(provider="")) is False+ assert read == ["openai_api_key"]+++def test_pinned_browser_audio_answers_for_the_pin_without_reading_credentials(monkeypatch):+ _fake_registry(monkeypatch, {"openai", "gemini"})++ def _no_credential_reads(_candidates):+ raise AssertionError("a pinned provider must answer without reading credentials")++ monkeypatch.setattr(factory, "get_secret_any", _no_credential_reads)++ assert realtime_browser_audio(_cfg(provider="gemini-live")) is True+ assert realtime_browser_audio(_cfg(provider="openai-realtime")) is False++def test_pipeline_mode_never_builds_realtime_session(monkeypatch):_fake_registry(monkeypatch, {"openai"})assert (diff --git a/tests/unit/realtime/test_usage_billed_fallback_guard.py b/tests/unit/realtime/test_usage_billed_fallback_guard.pyindex 70be669303..d7068f81e1 100644--- a/tests/unit/realtime/test_usage_billed_fallback_guard.py+++ b/tests/unit/realtime/test_usage_billed_fallback_guard.py@@ -51,6 +51,19 @@ def _registered_browser_token():revoke_token(_VALID_TOKEN)[email protected](autouse=True)+def _realtime_transport_sees_no_host_credentials(monkeypatch):+ """The desktop-path guards here must not follow the host's real keys.++ Without a pin, ``realtime_browser_audio`` follows the first+ credential-ready provider, so a real Gemini key on a developer machine+ would hand these desktop calls to the browser while CI never does.+ """+ monkeypatch.setattr(+ "jarvis.realtime.factory.get_secret_any", lambda _candidates: None+ )++class _FailingDesktopSession:"""A realtime session that dies in its handshake, like a refused plan."""diff --git a/tests/unit/speech/conftest.py b/tests/unit/speech/conftest.pyindex a2c483f2d2..f7a3f5b9a1 100644--- a/tests/unit/speech/conftest.py+++ b/tests/unit/speech/conftest.py@@ -1,10 +1,29 @@-"""Explicit OS permission fixtures for synthetic speech consumers."""+"""Shared fixtures for the speech suite.++Explicit OS permission fixtures for synthetic speech consumers, and the one+autouse guarantee that a realtime transport is picked as on a keyless host.+"""import pytestfrom tests.fakes.fake_permission_service import FakePermissionService[email protected](autouse=True)+def _realtime_transport_sees_no_host_credentials(monkeypatch):+ """Pick the realtime transport as a keyless host (and CI) does.++ Without a pinned provider, ``realtime_browser_audio`` follows the first+ credential-ready provider. On a developer machine with a real Gemini key+ every desktop-path test would then hand its call to the browser, while CI+ never does. A test that needs a ready provider patches ``get_secret_any``+ in its own body, which runs after this fixture and wins.+ """+ monkeypatch.setattr(+ "jarvis.realtime.factory.get_secret_any", lambda _candidates: None+ )++@pytest.fixturedef granted_microphone(monkeypatch):"""A fake capture device needs a fake grant, independent of host TCC.diff --git a/tests/unit/speech/test_realtime_mode.py b/tests/unit/speech/test_realtime_mode.pyindex 49c91090c3..48381093f9 100644--- a/tests/unit/speech/test_realtime_mode.py+++ b/tests/unit/speech/test_realtime_mode.py@@ -215,6 +215,38 @@ def _pipe(mode: str = "realtime") -> SpeechPipeline:return pipe+async def test_unpinned_browser_audio_engine_hands_the_call_to_the_browser(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Issue #399 follow-up: a Gemini-only install without a pinned provider+ # ran Gemini Live on the half-duplex native path, because the transport+ # decision read only the (absent) pin. The real registry decides here.+ handed_over: list[dict] = []++ async def _browser_call(_bus, _hangup, **kwargs):+ handed_over.append(kwargs)+ return "client_stop"++ def _native_build(**_kwargs):+ raise AssertionError("the native desktop session must not be built")++ monkeypatch.setattr(+ "jarvis.realtime.factory.get_secret_any",+ lambda candidates: (+ "gemini-key" if any(slot == "gemini_api_key" for slot, _ in candidates) else None+ ),+ )+ monkeypatch.setattr("jarvis.live.runtime.run_browser_call", _browser_call)+ monkeypatch.setattr("jarvis.realtime.factory.build_realtime_session", _native_build)++ reason = await asyncio.wait_for(_pipe()._active_realtime_session(), timeout=2.0)++ assert reason == "client_stop"+ assert len(handed_over) == 1+ assert handed_over[0]["session_id"] == "desktop-session"++@pytest.mark.asyncioasync def test_desktop_realtime_handshake_streams_audio_and_ends_single_turn(monkeypatch: pytest.MonkeyPatch,diff --git a/tests/unit/web/test_voice_mode_route.py b/tests/unit/web/test_voice_mode_route.pyindex 96ba4f6db3..340c2021dc 100644--- a/tests/unit/web/test_voice_mode_route.py+++ b/tests/unit/web/test_voice_mode_route.py@@ -104,8 +104,12 @@ def test_get_voice_mode(monkeypatch):def test_get_voice_mode_reports_browser_offer_capability(monkeypatch):+ import jarvis.realtime.factory as rffrom jarvis.ui.web import settings_routes+ # An offer-requiring transport without browser audio. Fixed here, because+ # without a pin the answer follows the host's real credentials.+ monkeypatch.setattr(rf, "realtime_browser_audio", lambda _cfg: False)monkeypatch.setattr(settings_routes,"_realtime_available_provider",@@ -191,6 +195,22 @@ def test_get_voice_mode_cross_family_gemini_only(monkeypatch):assert body["active_model"] == "gemini-3.1-flash-live-preview"+def test_get_voice_mode_reports_browser_audio_of_the_automatic_provider(monkeypatch):+ """Issue #399: a Gemini-only install with no pinned realtime provider+ reported ``active_provider: gemini-live`` next to ``browser_audio: false``+ although Gemini Live declares browser audio."""+ import jarvis.realtime.factory as rf++ def only_gemini(candidates):+ keys = {c[0] for c in candidates}+ return "sk-x" if "gemini_api_key" in keys else None++ monkeypatch.setattr(rf, "get_secret_any", only_gemini)+ body = TestClient(_app(mode="realtime")).get("/api/settings/voice-mode").json()+ assert body["active_provider"] == "gemini-live"+ assert body["browser_audio"] is True++def test_get_voice_mode_reports_pinned_realtime_model(monkeypatch):"""A model pinned in [brain.providers.<id>].model outranks the catalogdefault in the sidebar display fields."""