Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(realtime): let an unpinned engine decide the browser-audio transport (#403)

Ruben Lütkefd29b230d4e5
diff --git a/docs/BUGS.md b/docs/BUGS.md
index 3b579c1782..635a3af118 100644
--- a/docs/BUGS.md
+++ b/docs/BUGS.md
@@ -16036,8 +16036,8 @@ never has; the only browser-side start was the desktop's realtime hand-over
(`BrowserVoiceRequested`). (3) The visible browser-microphone card that could
start a call itself left the sidebar on 2026-09-12 (`51b4d4015`). The issue's
note that Gemini Live lacks browser audio is not the cause: the provider
-declares it; `browser_audio` in `/api/settings/voice-mode` reads only an
-explicitly pinned realtime provider.
+declares it; `browser_audio` in `/api/settings/voice-mode` read only an
+explicitly pinned realtime provider (fixed separately as BUG-232).
**Fix.** `BrowserVoiceConfig(enabled=True)` is now `JarvisConfig.browser_voice`;
`browser_voice_enabled(cfg)` serves the classic bridge in pipeline mode by
@@ -16050,3 +16050,27 @@ Guards: `tests/unit/web/test_voice_mode_route.py`,
`tests/unit/browser_voice/test_route.py`,
`tests/unit/ui/test_voice_call_routes.py`, `useVoiceCall.test.tsx`,
`BrowserRealtimeControl.test.tsx`.
+
+## BUG-232: an unpinned browser-audio engine ran on the desktop's half-duplex path and was reported as "no browser audio" (MEDIUM, FIXED 2026-10-05)
+
+**Symptom.** With no `[brain.realtime].provider` pinned, a Gemini-only install
+showed `active_provider: gemini-live` next to `browser_audio: false` in
+`GET /api/settings/voice-mode` (seen in GitHub issue #399). On the desktop the
+same call ran on the native half-duplex path instead of the browser hand-over
+that a pinned Gemini Live call takes.
+
+**Cause.** `realtime_browser_audio(cfg)` read only the explicitly pinned
+primary. Without a pin, the session builder opens on the first
+credential-ready provider in effective order, so the transport decision and
+the provider the call actually used disagreed for every unpinned install.
+
+**Fix.** Without a pin, `realtime_browser_audio` reads the capability of the
+first credential-ready provider (`_identified_provider_candidates(...,
+limit=1)`: the same order and the same refusal rules as
+`build_realtime_session`); a pinned primary still answers for itself without
+reading credentials. The settings route and both pipeline call sites now read
+it off their loops. Speech-suite tests pick the transport as a keyless host
+does (`tests/unit/speech/conftest.py`), so a developer's real key no longer
+flips desktop-path tests. Guards: `tests/unit/realtime/test_factory.py`,
+`tests/unit/web/test_voice_mode_route.py`,
+`tests/unit/speech/test_realtime_mode.py`.
diff --git a/jarvis/realtime/factory.py b/jarvis/realtime/factory.py
index 1a25c8efed..8881f419ee 100644
--- a/jarvis/realtime/factory.py
+++ b/jarvis/realtime/factory.py
@@ -88,7 +88,10 @@ def realtime_implicit_usage_fallback_allowed(cfg: Any) -> bool:
def _identified_provider_candidates(
- cfg: Any, *, defer_external_login_probe: bool = False
+ cfg: Any,
+ *,
+ defer_external_login_probe: bool = False,
+ limit: int | None = None,
) -> list[tuple[str, Any]]:
"""Instantiate every credential-ready plugin in effective fallback order.
@@ -100,6 +103,9 @@ def _identified_provider_candidates(
Each candidate is paired with the entry-point id it was loaded from, which
a caller cannot recover from the instance: ``name`` is a provider-chosen
label and may differ from the id the registry knows.
+
+ ``limit`` stops after that many candidates, so a caller that only needs
+ the provider a call would open first reads no further credentials.
"""
candidates: list[tuple[str, Any]] = []
explicit_ids = set(_explicit_provider_ids(cfg))
@@ -168,6 +174,8 @@ def _identified_provider_candidates(
)
continue
candidates.append((provider_id, provider))
+ if limit is not None and len(candidates) >= limit:
+ break
except Exception as exc: # noqa: BLE001 — one plugin must not brick others
log.warning("Realtime plugin %s is unavailable: %s", provider_id, exc)
return candidates
@@ -523,14 +531,37 @@ def realtime_webrtc_start_event_required(cfg: Any) -> bool:
def realtime_browser_audio(cfg: Any) -> bool:
- """Whether the selected engine needs browser echo-cancelled audio."""
+ """Whether the engine a call would open needs browser echo-cancelled audio.
+
+ A pinned primary answers for itself. Without a pin the call opens on the
+ first credential-ready provider in effective order — the one
+ ``build_realtime_session`` builds and ``/api/settings/voice-mode``
+ reports as ``active_provider`` — so that provider's capability decides.
+ Reading the pin alone answered ``False`` for an automatically chosen
+ browser-audio provider such as Gemini Live: the desktop then ran that call
+ on the half-duplex native path and every surface reported the wrong
+ transport (follow-up to issue #399).
+
+ The unpinned answer reads credentials, so callers keep it off the voice
+ and request loops.
+ """
+ if cfg is None:
+ return False
for provider_id in _explicit_provider_ids(cfg)[:1]:
try:
provider = load(_GROUP, provider_id, protocol=RealtimeProvider)
return bool(getattr(provider, "browser_audio", False))
except Exception:
log.warning("Browser audio capability unavailable", exc_info=True)
- return False
+ return False
+ # The same candidate order the session builder uses, including its refusal
+ # to fall back ambiently behind a pin that is missing or does not load.
+ candidates = _identified_provider_candidates(
+ cfg, defer_external_login_probe=True, limit=1
+ )
+ if not candidates:
+ return False
+ return bool(getattr(candidates[0][1], "browser_audio", False))
def build_realtime_session(
diff --git a/jarvis/speech/pipeline.py b/jarvis/speech/pipeline.py
index d62a6a0245..9cdda91461 100644
--- a/jarvis/speech/pipeline.py
+++ b/jarvis/speech/pipeline.py
@@ -8227,10 +8227,15 @@ class SpeechPipeline:
"""Borrow the wake mic or open exactly one fallback session mic."""
from jarvis.realtime.factory import realtime_browser_audio
+ config = getattr(self, "_config", None)
+ # An unpinned engine is resolved from the credentials, so the answer
+ # is read off the voice loop, like the session build.
browser_media = (
not getattr(self, "_ptt_mode", False)
and getattr(self, "_active_voice_mode", None) == "realtime"
- and realtime_browser_audio(getattr(self, "_config", None))
+ and bool(
+ await _run_voice_critical_thread(lambda: realtime_browser_audio(config))
+ )
)
buffer = await self._claim_wake_capture_for_session()
if browser_media and buffer is None:
@@ -9812,12 +9817,25 @@ class SpeechPipeline:
"""
from jarvis.realtime.factory import realtime_browser_audio, realtime_handshake_budget_s
- if realtime_browser_audio(self._config):
+ config = self._config
+
+ def _browser_handover_budget_s() -> float | None:
+ """The hand-over budget when the browser owns this call, else ``None``.
+
+ Both answers can read credentials (an unpinned engine is resolved
+ from the keys), so they run off the voice loop together.
+ """
+ if not realtime_browser_audio(config):
+ return None
+ return realtime_handshake_budget_s(config)
+
+ handover_budget_s = await _run_voice_critical_thread(_browser_handover_budget_s)
+ if handover_budget_s is not None:
from jarvis.live.runtime import run_browser_call
return await run_browser_call(
self._bus, self._hangup_event,
- timeout_s=max(45.0, realtime_handshake_budget_s(self._config) + 5.0),
+ timeout_s=max(45.0, handover_budget_s + 5.0),
input_buffer=input_buffer,
session_id=self._current_voice_session_id or "",
)
diff --git a/jarvis/ui/web/settings_routes.py b/jarvis/ui/web/settings_routes.py
index 4d57a693d6..35adfbca4b 100644
--- a/jarvis/ui/web/settings_routes.py
+++ b/jarvis/ui/web/settings_routes.py
@@ -325,7 +325,9 @@ async def get_voice_mode(request: Request) -> dict[str, object]:
# Capability, not a provider id (AP-21): the surface must not call a start
# attempt dead while the backend is still inside a budget it declared.
handshake_budget_s = await asyncio.to_thread(_realtime_handshake_budget_s, cfg)
- browser_audio = realtime_browser_audio(cfg)
+ # Without a pin the answer follows the first credential-ready provider,
+ # which reads credentials — off the loop like the lookups above.
+ browser_audio = await asyncio.to_thread(realtime_browser_audio, cfg)
transport_offer_ready = (
None if browser_audio else await _realtime_transport_offer_ready(requires_webrtc_offer)
)
diff --git a/tests/unit/realtime/test_factory.py b/tests/unit/realtime/test_factory.py
index 87ceaecdac..a61769a204 100644
--- a/tests/unit/realtime/test_factory.py
+++ b/tests/unit/realtime/test_factory.py
@@ -13,6 +13,7 @@ from jarvis.realtime.factory import (
_resolve_realtime_provider,
build_realtime_session,
realtime_available_provider,
+ realtime_browser_audio,
realtime_requires_webrtc_offer,
)
@@ -41,6 +42,8 @@ class _OpenAIProvider(_BaseProvider):
class _GeminiProvider(_BaseProvider):
name = "gemini-live"
credential_candidates = (("gemini_api_key", "GEMINI_API_KEY"),)
+ # Mirrors the real adapter: its calls need browser echo-cancelled audio.
+ browser_audio = True
class _AcmeProvider(_BaseProvider):
@@ -200,6 +203,52 @@ def test_logged_in_subscription_is_never_an_ambient_candidate(monkeypatch):
assert realtime_available_provider(config) is None
+def test_unpinned_browser_audio_follows_the_automatically_chosen_provider(monkeypatch):
+ # Issue #399: a Gemini-only install without a pin opens its call on
+ # Gemini Live, yet browser_audio read only the (absent) pin and said no.
+ _fake_registry(monkeypatch, {"gemini"})
+ config = _cfg(provider="")
+
+ assert realtime_available_provider(config) == "gemini-live"
+ assert realtime_browser_audio(config) is True
+
+
+def test_unpinned_browser_audio_is_false_without_a_ready_provider(monkeypatch):
+ _fake_registry(monkeypatch, set())
+
+ assert realtime_browser_audio(_cfg(provider="")) is False
+ assert realtime_browser_audio(None) is False
+
+
+def test_unpinned_browser_audio_reads_only_up_to_the_first_ready_provider(monkeypatch):
+ # The first installed provider with a key decides; credentials of the
+ # providers behind it are never read on this path.
+ _fake_registry(monkeypatch, {"openai", "gemini"})
+ read: list[str] = []
+ resolve = factory.get_secret_any
+
+ def _spy(candidates):
+ read.append(candidates[0][0])
+ return resolve(candidates)
+
+ monkeypatch.setattr(factory, "get_secret_any", _spy)
+
+ assert realtime_browser_audio(_cfg(provider="")) is False
+ assert read == ["openai_api_key"]
+
+
+def test_pinned_browser_audio_answers_for_the_pin_without_reading_credentials(monkeypatch):
+ _fake_registry(monkeypatch, {"openai", "gemini"})
+
+ def _no_credential_reads(_candidates):
+ raise AssertionError("a pinned provider must answer without reading credentials")
+
+ monkeypatch.setattr(factory, "get_secret_any", _no_credential_reads)
+
+ assert realtime_browser_audio(_cfg(provider="gemini-live")) is True
+ assert realtime_browser_audio(_cfg(provider="openai-realtime")) is False
+
+
def test_pipeline_mode_never_builds_realtime_session(monkeypatch):
_fake_registry(monkeypatch, {"openai"})
assert (
diff --git a/tests/unit/realtime/test_usage_billed_fallback_guard.py b/tests/unit/realtime/test_usage_billed_fallback_guard.py
index 70be669303..d7068f81e1 100644
--- a/tests/unit/realtime/test_usage_billed_fallback_guard.py
+++ b/tests/unit/realtime/test_usage_billed_fallback_guard.py
@@ -51,6 +51,19 @@ def _registered_browser_token():
revoke_token(_VALID_TOKEN)
[email protected](autouse=True)
+def _realtime_transport_sees_no_host_credentials(monkeypatch):
+ """The desktop-path guards here must not follow the host's real keys.
+
+ Without a pin, ``realtime_browser_audio`` follows the first
+ credential-ready provider, so a real Gemini key on a developer machine
+ would hand these desktop calls to the browser while CI never does.
+ """
+ monkeypatch.setattr(
+ "jarvis.realtime.factory.get_secret_any", lambda _candidates: None
+ )
+
+
class _FailingDesktopSession:
"""A realtime session that dies in its handshake, like a refused plan."""
diff --git a/tests/unit/speech/conftest.py b/tests/unit/speech/conftest.py
index a2c483f2d2..f7a3f5b9a1 100644
--- a/tests/unit/speech/conftest.py
+++ b/tests/unit/speech/conftest.py
@@ -1,10 +1,29 @@
-"""Explicit OS permission fixtures for synthetic speech consumers."""
+"""Shared fixtures for the speech suite.
+
+Explicit OS permission fixtures for synthetic speech consumers, and the one
+autouse guarantee that a realtime transport is picked as on a keyless host.
+"""
import pytest
from tests.fakes.fake_permission_service import FakePermissionService
[email protected](autouse=True)
+def _realtime_transport_sees_no_host_credentials(monkeypatch):
+ """Pick the realtime transport as a keyless host (and CI) does.
+
+ Without a pinned provider, ``realtime_browser_audio`` follows the first
+ credential-ready provider. On a developer machine with a real Gemini key
+ every desktop-path test would then hand its call to the browser, while CI
+ never does. A test that needs a ready provider patches ``get_secret_any``
+ in its own body, which runs after this fixture and wins.
+ """
+ monkeypatch.setattr(
+ "jarvis.realtime.factory.get_secret_any", lambda _candidates: None
+ )
+
+
@pytest.fixture
def granted_microphone(monkeypatch):
"""A fake capture device needs a fake grant, independent of host TCC.
diff --git a/tests/unit/speech/test_realtime_mode.py b/tests/unit/speech/test_realtime_mode.py
index 49c91090c3..48381093f9 100644
--- a/tests/unit/speech/test_realtime_mode.py
+++ b/tests/unit/speech/test_realtime_mode.py
@@ -215,6 +215,38 @@ def _pipe(mode: str = "realtime") -> SpeechPipeline:
return pipe
+async def test_unpinned_browser_audio_engine_hands_the_call_to_the_browser(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Issue #399 follow-up: a Gemini-only install without a pinned provider
+ # ran Gemini Live on the half-duplex native path, because the transport
+ # decision read only the (absent) pin. The real registry decides here.
+ handed_over: list[dict] = []
+
+ async def _browser_call(_bus, _hangup, **kwargs):
+ handed_over.append(kwargs)
+ return "client_stop"
+
+ def _native_build(**_kwargs):
+ raise AssertionError("the native desktop session must not be built")
+
+ monkeypatch.setattr(
+ "jarvis.realtime.factory.get_secret_any",
+ lambda candidates: (
+ "gemini-key" if any(slot == "gemini_api_key" for slot, _ in candidates) else None
+ ),
+ )
+ monkeypatch.setattr("jarvis.live.runtime.run_browser_call", _browser_call)
+ monkeypatch.setattr("jarvis.realtime.factory.build_realtime_session", _native_build)
+
+ reason = await asyncio.wait_for(_pipe()._active_realtime_session(), timeout=2.0)
+
+ assert reason == "client_stop"
+ assert len(handed_over) == 1
+ assert handed_over[0]["session_id"] == "desktop-session"
+
+
@pytest.mark.asyncio
async def test_desktop_realtime_handshake_streams_audio_and_ends_single_turn(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/web/test_voice_mode_route.py b/tests/unit/web/test_voice_mode_route.py
index 96ba4f6db3..340c2021dc 100644
--- a/tests/unit/web/test_voice_mode_route.py
+++ b/tests/unit/web/test_voice_mode_route.py
@@ -104,8 +104,12 @@ def test_get_voice_mode(monkeypatch):
def test_get_voice_mode_reports_browser_offer_capability(monkeypatch):
+ import jarvis.realtime.factory as rf
from jarvis.ui.web import settings_routes
+ # An offer-requiring transport without browser audio. Fixed here, because
+ # without a pin the answer follows the host's real credentials.
+ monkeypatch.setattr(rf, "realtime_browser_audio", lambda _cfg: False)
monkeypatch.setattr(
settings_routes,
"_realtime_available_provider",
@@ -191,6 +195,22 @@ def test_get_voice_mode_cross_family_gemini_only(monkeypatch):
assert body["active_model"] == "gemini-3.1-flash-live-preview"
+def test_get_voice_mode_reports_browser_audio_of_the_automatic_provider(monkeypatch):
+ """Issue #399: a Gemini-only install with no pinned realtime provider
+ reported ``active_provider: gemini-live`` next to ``browser_audio: false``
+ although Gemini Live declares browser audio."""
+ import jarvis.realtime.factory as rf
+
+ def only_gemini(candidates):
+ keys = {c[0] for c in candidates}
+ return "sk-x" if "gemini_api_key" in keys else None
+
+ monkeypatch.setattr(rf, "get_secret_any", only_gemini)
+ body = TestClient(_app(mode="realtime")).get("/api/settings/voice-mode").json()
+ assert body["active_provider"] == "gemini-live"
+ assert body["browser_audio"] is True
+
+
def test_get_voice_mode_reports_pinned_realtime_model(monkeypatch):
"""A model pinned in [brain.providers.<id>].model outranks the catalog
default in the sidebar display fields."""