Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
feat(voice): speaking while Jarvis thinks now interrupts him
rubenluetke10-beepe5c8714f4ce0
diff --git a/jarvis/realtime/session.py b/jarvis/realtime/session.pyindex d746e6f4cd..2564a14b0a 100644--- a/jarvis/realtime/session.py+++ b/jarvis/realtime/session.py@@ -2868,6 +2868,37 @@ class RealtimeVoiceSession:return Falsereturn (time.monotonic() - stamp) < _USER_SPEAKING_HOLD_S+ def owes_the_user_a_reply(self) -> bool:+ """True while a turn is being worked on and nothing is audible yet.++ The THINKING phase, named for the one surface that needs it: the+ desktop microphone pump. Barge-in during playback is detected locally+ and fires in milliseconds; during this phase there was no local+ detector at all, because the one that exists is armed only while audio+ plays (``jarvis/speech/pipeline.py``, the ``echo_guard_active``+ branch). The only remaining signal was the provider's own VAD, which+ on a Live-API transport reports room noise and a real interruption as+ the SAME event and is therefore parked while an action runs+ (``_pending_delegate_needs_endpoint_protection``). Net effect, live+ 2026-08-13 12:11:12: the user spoke into a 11.7 s silent wait, the+ edge was deferred, and Jarvis answered the original question anyway.++ Deliberately a capability question ("may the user take the floor?"),+ not a state enum: the pump must not learn the turn machinery, and+ every caller wants the same thing — is a reply owed, and is the room+ still silent enough that speech can only be the user's.+ """+ return bool(+ not self._ended+ and self._session is not None+ and not self._output_active+ and (+ self._response_requested_for_turn+ or self._turn_has_pending_delegate(self._turn_id)+ or self._has_pending_delegate_from_earlier_turn()+ )+ )+async def handle_audio_frame(self, pcm_native: bytes) -> None:if self._ended or self._session is None or not pcm_native:returndiff --git a/jarvis/speech/pipeline.py b/jarvis/speech/pipeline.pyindex 1b5ef3b4ae..22ea75dff9 100644--- a/jarvis/speech/pipeline.py+++ b/jarvis/speech/pipeline.py@@ -794,6 +794,15 @@ _REALTIME_HW_ECHO_TAIL_MARGIN_S = 0.25# _REALTIME_POST_OUTPUT_ECHO_GUARD_S + _REALTIME_OUTPUT_LATENCY_CAP_S (~5.5 s# of 16 kHz mono PCM16 ≈ 176 kB), so this never truncates in practice._REALTIME_TAIL_PENDING_MAX_BYTES = 256_000+# How long the microphone must stay quiet after a turn is committed before+# speaking into the silent THINKING wait counts as an interruption rather than+# the tail of the utterance that started it. The provider commits on ITS OWN+# VAD, mid-sentence when a filler pause fools it, so the first moments after a+# commit are exactly when the user is most likely to still be talking (the turn+# fragmentation the session's own _user_is_speaking hold answers). Long enough+# to clear that overlap, short enough that a deliberate interruption a beat+# after Jarvis goes quiet still lands.+_THINKING_BARGE_QUIET_S = 0.35def _feed_live_mic_level(chunk: AudioChunk) -> None:@@ -7930,6 +7939,47 @@ class SpeechPipeline:barge_feed_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix=f"rt-barge-feed-{session_id}")+ # Second detector for the SILENT wait (see the thinking-phase branch in+ # _send_microphone). A separate instance rather than a mode flag on the+ # one above: that one's state machine is keyed to a playback response+ # (start_output/stop_output per answer, per-response echo calibration),+ # and interleaving a second, differently-armed window through it would+ # corrupt the calibration the playback barge depends on.+ #+ # ``output_active=None`` is what makes it work at all: with the+ # playback probe attached, ``feed`` returns None forever while nothing+ # plays (jarvis/realtime/desktop.py — the ``_playback_started`` gate),+ # which is precisely why no detector existed for this phase. With no+ # probe the window opens on start_output(). For the same reason the+ # echo machinery is inert here and wants to be: nothing is playing, so+ # there is no echo to discriminate, and the only question left is the+ # one Silero answers — is this sustained, deliberate speech.+ #+ # The grace period drops to a token value: on the playback detector it+ # exists to collect echo for the adaptive floor, and here there is+ # none to collect. Detection strictness is unchanged (0.97 / 12 frames+ # ~= 0.4 s of speech), so a cough or a door does not take the floor.+ thinking_detector = DesktopRealtimeBargeInDetector(+ grace_s=0.05, output_active=None+ )+ # The microphone must fall quiet ONCE before the thinking barge arms,+ # or the user's own trailing words would arm it against themselves.+ thinking_barge_quiet_since = 0.0++ def thinking_barge_armed(active_session: Any, now: float) -> bool:+ nonlocal thinking_barge_quiet_since+ owed = getattr(active_session, "owes_the_user_a_reply", None)+ if not callable(owed) or not owed():+ thinking_barge_quiet_since = 0.0+ return False+ speaking_now = getattr(active_session, "_user_is_speaking", None)+ if callable(speaking_now) and speaking_now():+ # Still the same utterance the provider just committed.+ thinking_barge_quiet_since = 0.0+ return False+ if not thinking_barge_quiet_since:+ thinking_barge_quiet_since = now+ return (now - thinking_barge_quiet_since) >= _THINKING_BARGE_QUIET_S# Provider PCM is owned by ``playback``; the independent surface-TTS# fallback used to bypass that adapter and was awaited inline. A barge# then stopped PortAudio but left the TTS producer alive, so its next@@ -8043,6 +8093,12 @@ class SpeechPipeline:async def _warm_barge_detector() -> None:try:await _run_voice_critical_thread(barge_detector.warmup)+ # BOTH detectors, or the thinking-phase one is silently dead:+ # ``feed`` returns None while ``_ready`` is false, so an+ # unwarmed detector looks exactly like a room that never+ # speaks. They share the bundled ONNX model, so the second+ # warmup is a cache hit, not a second load.+ await _run_voice_critical_thread(thinking_detector.warmup)except Exception as exc: # noqa: BLE001 -- voice still works without local VADlog.warning("Realtime desktop barge-in detector unavailable; "@@ -8522,7 +8578,7 @@ class SpeechPipeline:async def _send_microphone() -> None:nonlocal post_output_echo_guard_until, preroll_bytes- nonlocal tail_pending_bytes+ nonlocal tail_pending_bytes, thinking_barge_quiet_sinceasync for chunk in self._session_input_stream(input_chunks):if not provider_ready.is_set():if (@@ -8586,6 +8642,52 @@ class SpeechPipeline:if bool(getattr(barge_detector, "active", False)):barge_detector.stop_output()await _flush_tail_pending()+ # THINKING-PHASE BARGE-IN. Everything above this line+ # only runs while audio plays, which is why speaking+ # over Jarvis worked and speaking over his THINKING+ # did nothing (live 2026-08-13 12:11:12: the provider+ # edge was deferred and he answered the original+ # question 11.7 s later regardless). The same Silero+ # detector, armed for the silent wait, closes that+ # hole with the mechanism that already works.+ #+ # Arming is deliberately two-condition:+ # - the session owes a reply and nothing is audible;+ # - the microphone has fallen quiet since the turn+ # was committed, so the user's OWN trailing words+ # cannot arm it against themselves (that shape is+ # turn fragmentation, handled by the session's+ # _user_is_speaking hold, not by a barge).+ # The frame is still uploaded either way — unlike the+ # playback branch there is no echo to withhold, and+ # the words have to reach the provider or the+ # interruption would take the floor and say nothing.+ if thinking_barge_armed(session, now):+ if not thinking_detector.active:+ thinking_detector.start_output()+ confirmed_pcm = await asyncio.get_running_loop(+ ).run_in_executor(+ barge_feed_executor,+ thinking_detector.feed,+ chunk.pcm,+ )+ if confirmed_pcm is not None:+ log.info(+ "Realtime desktop barge-in confirmed by "+ "local CPU VAD while Jarvis was thinking"+ )+ thinking_detector.stop_output()+ thinking_barge_quiet_since = 0.0+ await session.handle_control(+ {"type": "barge_in"}+ )+ # The detector's own capture carries the+ # opening syllables its confirmation consumed;+ # uploading the raw frame too would double them.+ await session.handle_audio_frame(confirmed_pcm)+ continue+ elif thinking_detector.active:+ thinking_detector.stop_output()await session.handle_audio_frame(chunk.pcm)# A shared capture buffer already owns and meters productiondiff --git a/tests/unit/speech/test_realtime_mode.py b/tests/unit/speech/test_realtime_mode.pyindex c6cf0b9099..f5cef95a95 100644--- a/tests/unit/speech/test_realtime_mode.py+++ b/tests/unit/speech/test_realtime_mode.py@@ -867,11 +867,14 @@ async def test_desktop_cpu_barge_in_cancels_and_forwards_user_preroll(pipe = _pipe()output_started = asyncio.Event()forwarded = b"\x09\x00" * 32- detector_kwargs: dict[str, object] = {}+ # Per INSTANCE: the session builds two detectors — one for playback and one+ # for the silent thinking wait — and they are configured differently on+ # purpose (see the output_active assertions below).+ detector_kwargs: list[dict[str, object]] = []class _Detector:def __init__(self, **kwargs: object) -> None:- detector_kwargs.update(kwargs)+ detector_kwargs.append(dict(kwargs))def warmup(self) -> None:return None@@ -954,7 +957,19 @@ async def test_desktop_cpu_barge_in_cancels_and_forwards_user_preroll(assert {"type": "barge_in"} in session.controlsassert session.audio_frames[-1] == forwardedassert built["half_duplex"] is True- assert detector_kwargs["output_active"] is pipeline_mod.level_tap.playback_active+ # The PLAYBACK detector keeps the probe: it must judge frames against what+ # the speakers are emitting, or its echo gates cannot tell our own voice+ # from the user's.+ assert (+ detector_kwargs[0]["output_active"]+ is pipeline_mod.level_tap.playback_active+ )+ # The THINKING detector must NOT have one. With the probe attached, feed()+ # returns None for as long as nothing is playing, which is exactly the+ # silence it exists to listen through.+ assert any(kwargs.get("output_active") is None for kwargs in detector_kwargs[1:]), (+ "no detector is armed for the silent thinking wait"+ )@pytest.mark.asyncio@@ -1857,3 +1872,203 @@ async def test_capture_latency_extends_the_audible_phase(# silent reached the provider. Without the capture latency, tail_pcm was# uploaded and the model answered its own echo.assert session.audio_frames == [live_pcm]+++async def test_thinking_phase_barge_in_takes_the_floor_without_a_stop_word(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Speaking into the SILENT wait interrupts, exactly like speaking over him.++ The reported failure (live 2026-08-13 12:11:12): barge-in during playback+ works perfectly, barge-in while Jarvis thinks does nothing. The playback+ detector is armed only inside the ``echo_guard_active`` branch, so during+ the silent wait there was no local detector at all and the only remaining+ signal — the provider's own VAD — is deliberately parked while an action+ runs. No stop word is spoken here on purpose: ordinary speech has to be+ enough.+ """+ pipe = _pipe()+ ready = asyncio.Event()+ captured = b"\x0b\x00" * 32++ class _Detector:+ """Confirms on any frame — the arming logic is what is under test."""++ def __init__(self, **kwargs: object) -> None:+ self._probe = kwargs.get("output_active")+ self.active = False++ def warmup(self) -> None:+ return None++ def start_output(self) -> None:+ self.active = True++ def stop_output(self) -> None:+ self.active = False++ def feed(self, _pcm: bytes) -> bytes | None:+ # Only the probe-less (thinking) detector ever confirms, mirroring+ # the real gate: with a playback probe attached feed() returns None+ # while nothing is playing.+ return None if self._probe is not None else captured++ class _Mic:+ async def __aenter__(self):+ return self++ async def __aexit__(self, *_exc: object) -> bool:+ return False++ async def stream(self):+ await ready.wait()+ # Enough frames to clear _THINKING_BARGE_QUIET_S at 32 ms/frame.+ for _ in range(40):+ yield AudioChunk(+ pcm=b"\x01\x00" * 256, sample_rate=16_000, timestamp_ns=0+ )+ await asyncio.sleep(0.02)+ await asyncio.Event().wait()++ class _ThinkingSession(_HandshakeOnlyRealtimeSession):+ """Owes a reply, nothing audible — the phase under test."""++ def __init__(self, send_binary, send_json) -> None:+ super().__init__(send_binary, send_json)+ self.audio_frames: list[bytes] = []+ self.barged = asyncio.Event()++ def owes_the_user_a_reply(self) -> bool:+ return True++ def _user_is_speaking(self) -> bool:+ return False++ async def handle_control(self, message) -> None:+ await super().handle_control(message)+ if message.get("type") == "barge_in":+ self.barged.set()++ async def handle_audio_frame(self, pcm: bytes) -> None:+ self.audio_frames.append(pcm)++ async def wait_finished(self) -> None:+ await self.barged.wait()++ built: dict[str, object] = {}++ def _build(**kwargs):+ session = _ThinkingSession(kwargs["send_binary"], kwargs["send_json"])+ built["session"] = session+ ready.set()+ return session++ monkeypatch.setattr("jarvis.realtime.factory.build_realtime_session", _build)+ monkeypatch.setattr(+ "jarvis.realtime.desktop.DesktopRealtimeBargeInDetector", _Detector+ )+ monkeypatch.setattr(pipeline_mod, "MicrophoneCapture", lambda **_kwargs: _Mic())++ await asyncio.wait_for(pipe._active_realtime_session(), timeout=5.0)++ session = built["session"]+ assert {"type": "barge_in"} in session.controls, (+ "speaking during the thinking phase did not interrupt"+ )+ # ...and what the user said reaches the provider, or the interruption+ # would take the floor and deliver no context.+ assert captured in session.audio_frames+++async def test_thinking_barge_stays_disarmed_while_the_user_is_still_talking(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The user's own trailing words must not arm the barge against themselves.++ A provider that commits on its own VAD cuts mid-sentence when a filler+ pause fools it. That shape is turn fragmentation (872b05051), not an+ interruption, and firing a barge there would abandon the order the user is+ still in the middle of giving.+ """+ pipe = _pipe()+ ready = asyncio.Event()++ class _Detector:+ def __init__(self, **kwargs: object) -> None:+ self._probe = kwargs.get("output_active")+ self.active = False+ self.fed = 0++ def warmup(self) -> None:+ return None++ def start_output(self) -> None:+ self.active = True++ def stop_output(self) -> None:+ self.active = False++ def feed(self, _pcm: bytes) -> bytes | None:+ self.fed += 1+ return None if self._probe is not None else b"\x0c\x00" * 32++ class _Mic:+ async def __aenter__(self):+ return self++ async def __aexit__(self, *_exc: object) -> bool:+ return False++ async def stream(self):+ await ready.wait()+ for _ in range(30):+ yield AudioChunk(+ pcm=b"\x01\x00" * 256, sample_rate=16_000, timestamp_ns=0+ )+ await asyncio.sleep(0.02)+ await asyncio.Event().wait()++ class _StillTalkingSession(_HandshakeOnlyRealtimeSession):+ def __init__(self, send_binary, send_json) -> None:+ super().__init__(send_binary, send_json)+ self.audio_frames: list[bytes] = []+ self.settled = asyncio.Event()++ def owes_the_user_a_reply(self) -> bool:+ return True++ def _user_is_speaking(self) -> bool:+ return True++ async def handle_audio_frame(self, pcm: bytes) -> None:+ self.audio_frames.append(pcm)+ if len(self.audio_frames) >= 20:+ self.settled.set()++ async def wait_finished(self) -> None:+ await self.settled.wait()++ built: dict[str, object] = {}++ def _build(**kwargs):+ session = _StillTalkingSession(kwargs["send_binary"], kwargs["send_json"])+ built["session"] = session+ ready.set()+ return session++ monkeypatch.setattr("jarvis.realtime.factory.build_realtime_session", _build)+ monkeypatch.setattr(+ "jarvis.realtime.desktop.DesktopRealtimeBargeInDetector", _Detector+ )+ monkeypatch.setattr(pipeline_mod, "MicrophoneCapture", lambda **_kwargs: _Mic())++ await asyncio.wait_for(pipe._active_realtime_session(), timeout=5.0)++ session = built["session"]+ assert {"type": "barge_in"} not in session.controls, (+ "a still-running utterance was mistaken for an interruption"+ )+ # The audio still flows: withholding it is what would lose the tail.+ assert session.audio_frames