Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

feat(voice): speaking while Jarvis thinks now interrupts him

rubenluetke10-beepe5c8714f4ce0
diff --git a/jarvis/realtime/session.py b/jarvis/realtime/session.py
index d746e6f4cd..2564a14b0a 100644
--- a/jarvis/realtime/session.py
+++ b/jarvis/realtime/session.py
@@ -2868,6 +2868,37 @@ class RealtimeVoiceSession:
return False
return (time.monotonic() - stamp) < _USER_SPEAKING_HOLD_S
+ def owes_the_user_a_reply(self) -> bool:
+ """True while a turn is being worked on and nothing is audible yet.
+
+ The THINKING phase, named for the one surface that needs it: the
+ desktop microphone pump. Barge-in during playback is detected locally
+ and fires in milliseconds; during this phase there was no local
+ detector at all, because the one that exists is armed only while audio
+ plays (``jarvis/speech/pipeline.py``, the ``echo_guard_active``
+ branch). The only remaining signal was the provider's own VAD, which
+ on a Live-API transport reports room noise and a real interruption as
+ the SAME event and is therefore parked while an action runs
+ (``_pending_delegate_needs_endpoint_protection``). Net effect, live
+ 2026-08-13 12:11:12: the user spoke into a 11.7 s silent wait, the
+ edge was deferred, and Jarvis answered the original question anyway.
+
+ Deliberately a capability question ("may the user take the floor?"),
+ not a state enum: the pump must not learn the turn machinery, and
+ every caller wants the same thing — is a reply owed, and is the room
+ still silent enough that speech can only be the user's.
+ """
+ return bool(
+ not self._ended
+ and self._session is not None
+ and not self._output_active
+ and (
+ self._response_requested_for_turn
+ or self._turn_has_pending_delegate(self._turn_id)
+ or self._has_pending_delegate_from_earlier_turn()
+ )
+ )
+
async def handle_audio_frame(self, pcm_native: bytes) -> None:
if self._ended or self._session is None or not pcm_native:
return
diff --git a/jarvis/speech/pipeline.py b/jarvis/speech/pipeline.py
index 1b5ef3b4ae..22ea75dff9 100644
--- a/jarvis/speech/pipeline.py
+++ b/jarvis/speech/pipeline.py
@@ -794,6 +794,15 @@ _REALTIME_HW_ECHO_TAIL_MARGIN_S = 0.25
# _REALTIME_POST_OUTPUT_ECHO_GUARD_S + _REALTIME_OUTPUT_LATENCY_CAP_S (~5.5 s
# of 16 kHz mono PCM16 ≈ 176 kB), so this never truncates in practice.
_REALTIME_TAIL_PENDING_MAX_BYTES = 256_000
+# How long the microphone must stay quiet after a turn is committed before
+# speaking into the silent THINKING wait counts as an interruption rather than
+# the tail of the utterance that started it. The provider commits on ITS OWN
+# VAD, mid-sentence when a filler pause fools it, so the first moments after a
+# commit are exactly when the user is most likely to still be talking (the turn
+# fragmentation the session's own _user_is_speaking hold answers). Long enough
+# to clear that overlap, short enough that a deliberate interruption a beat
+# after Jarvis goes quiet still lands.
+_THINKING_BARGE_QUIET_S = 0.35
def _feed_live_mic_level(chunk: AudioChunk) -> None:
@@ -7930,6 +7939,47 @@ class SpeechPipeline:
barge_feed_executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix=f"rt-barge-feed-{session_id}"
)
+ # Second detector for the SILENT wait (see the thinking-phase branch in
+ # _send_microphone). A separate instance rather than a mode flag on the
+ # one above: that one's state machine is keyed to a playback response
+ # (start_output/stop_output per answer, per-response echo calibration),
+ # and interleaving a second, differently-armed window through it would
+ # corrupt the calibration the playback barge depends on.
+ #
+ # ``output_active=None`` is what makes it work at all: with the
+ # playback probe attached, ``feed`` returns None forever while nothing
+ # plays (jarvis/realtime/desktop.py — the ``_playback_started`` gate),
+ # which is precisely why no detector existed for this phase. With no
+ # probe the window opens on start_output(). For the same reason the
+ # echo machinery is inert here and wants to be: nothing is playing, so
+ # there is no echo to discriminate, and the only question left is the
+ # one Silero answers — is this sustained, deliberate speech.
+ #
+ # The grace period drops to a token value: on the playback detector it
+ # exists to collect echo for the adaptive floor, and here there is
+ # none to collect. Detection strictness is unchanged (0.97 / 12 frames
+ # ~= 0.4 s of speech), so a cough or a door does not take the floor.
+ thinking_detector = DesktopRealtimeBargeInDetector(
+ grace_s=0.05, output_active=None
+ )
+ # The microphone must fall quiet ONCE before the thinking barge arms,
+ # or the user's own trailing words would arm it against themselves.
+ thinking_barge_quiet_since = 0.0
+
+ def thinking_barge_armed(active_session: Any, now: float) -> bool:
+ nonlocal thinking_barge_quiet_since
+ owed = getattr(active_session, "owes_the_user_a_reply", None)
+ if not callable(owed) or not owed():
+ thinking_barge_quiet_since = 0.0
+ return False
+ speaking_now = getattr(active_session, "_user_is_speaking", None)
+ if callable(speaking_now) and speaking_now():
+ # Still the same utterance the provider just committed.
+ thinking_barge_quiet_since = 0.0
+ return False
+ if not thinking_barge_quiet_since:
+ thinking_barge_quiet_since = now
+ return (now - thinking_barge_quiet_since) >= _THINKING_BARGE_QUIET_S
# Provider PCM is owned by ``playback``; the independent surface-TTS
# fallback used to bypass that adapter and was awaited inline. A barge
# then stopped PortAudio but left the TTS producer alive, so its next
@@ -8043,6 +8093,12 @@ class SpeechPipeline:
async def _warm_barge_detector() -> None:
try:
await _run_voice_critical_thread(barge_detector.warmup)
+ # BOTH detectors, or the thinking-phase one is silently dead:
+ # ``feed`` returns None while ``_ready`` is false, so an
+ # unwarmed detector looks exactly like a room that never
+ # speaks. They share the bundled ONNX model, so the second
+ # warmup is a cache hit, not a second load.
+ await _run_voice_critical_thread(thinking_detector.warmup)
except Exception as exc: # noqa: BLE001 -- voice still works without local VAD
log.warning(
"Realtime desktop barge-in detector unavailable; "
@@ -8522,7 +8578,7 @@ class SpeechPipeline:
async def _send_microphone() -> None:
nonlocal post_output_echo_guard_until, preroll_bytes
- nonlocal tail_pending_bytes
+ nonlocal tail_pending_bytes, thinking_barge_quiet_since
async for chunk in self._session_input_stream(input_chunks):
if not provider_ready.is_set():
if (
@@ -8586,6 +8642,52 @@ class SpeechPipeline:
if bool(getattr(barge_detector, "active", False)):
barge_detector.stop_output()
await _flush_tail_pending()
+ # THINKING-PHASE BARGE-IN. Everything above this line
+ # only runs while audio plays, which is why speaking
+ # over Jarvis worked and speaking over his THINKING
+ # did nothing (live 2026-08-13 12:11:12: the provider
+ # edge was deferred and he answered the original
+ # question 11.7 s later regardless). The same Silero
+ # detector, armed for the silent wait, closes that
+ # hole with the mechanism that already works.
+ #
+ # Arming is deliberately two-condition:
+ # - the session owes a reply and nothing is audible;
+ # - the microphone has fallen quiet since the turn
+ # was committed, so the user's OWN trailing words
+ # cannot arm it against themselves (that shape is
+ # turn fragmentation, handled by the session's
+ # _user_is_speaking hold, not by a barge).
+ # The frame is still uploaded either way — unlike the
+ # playback branch there is no echo to withhold, and
+ # the words have to reach the provider or the
+ # interruption would take the floor and say nothing.
+ if thinking_barge_armed(session, now):
+ if not thinking_detector.active:
+ thinking_detector.start_output()
+ confirmed_pcm = await asyncio.get_running_loop(
+ ).run_in_executor(
+ barge_feed_executor,
+ thinking_detector.feed,
+ chunk.pcm,
+ )
+ if confirmed_pcm is not None:
+ log.info(
+ "Realtime desktop barge-in confirmed by "
+ "local CPU VAD while Jarvis was thinking"
+ )
+ thinking_detector.stop_output()
+ thinking_barge_quiet_since = 0.0
+ await session.handle_control(
+ {"type": "barge_in"}
+ )
+ # The detector's own capture carries the
+ # opening syllables its confirmation consumed;
+ # uploading the raw frame too would double them.
+ await session.handle_audio_frame(confirmed_pcm)
+ continue
+ elif thinking_detector.active:
+ thinking_detector.stop_output()
await session.handle_audio_frame(chunk.pcm)
# A shared capture buffer already owns and meters production
diff --git a/tests/unit/speech/test_realtime_mode.py b/tests/unit/speech/test_realtime_mode.py
index c6cf0b9099..f5cef95a95 100644
--- a/tests/unit/speech/test_realtime_mode.py
+++ b/tests/unit/speech/test_realtime_mode.py
@@ -867,11 +867,14 @@ async def test_desktop_cpu_barge_in_cancels_and_forwards_user_preroll(
pipe = _pipe()
output_started = asyncio.Event()
forwarded = b"\x09\x00" * 32
- detector_kwargs: dict[str, object] = {}
+ # Per INSTANCE: the session builds two detectors — one for playback and one
+ # for the silent thinking wait — and they are configured differently on
+ # purpose (see the output_active assertions below).
+ detector_kwargs: list[dict[str, object]] = []
class _Detector:
def __init__(self, **kwargs: object) -> None:
- detector_kwargs.update(kwargs)
+ detector_kwargs.append(dict(kwargs))
def warmup(self) -> None:
return None
@@ -954,7 +957,19 @@ async def test_desktop_cpu_barge_in_cancels_and_forwards_user_preroll(
assert {"type": "barge_in"} in session.controls
assert session.audio_frames[-1] == forwarded
assert built["half_duplex"] is True
- assert detector_kwargs["output_active"] is pipeline_mod.level_tap.playback_active
+ # The PLAYBACK detector keeps the probe: it must judge frames against what
+ # the speakers are emitting, or its echo gates cannot tell our own voice
+ # from the user's.
+ assert (
+ detector_kwargs[0]["output_active"]
+ is pipeline_mod.level_tap.playback_active
+ )
+ # The THINKING detector must NOT have one. With the probe attached, feed()
+ # returns None for as long as nothing is playing, which is exactly the
+ # silence it exists to listen through.
+ assert any(kwargs.get("output_active") is None for kwargs in detector_kwargs[1:]), (
+ "no detector is armed for the silent thinking wait"
+ )
@pytest.mark.asyncio
@@ -1857,3 +1872,203 @@ async def test_capture_latency_extends_the_audible_phase(
# silent reached the provider. Without the capture latency, tail_pcm was
# uploaded and the model answered its own echo.
assert session.audio_frames == [live_pcm]
+
+
+async def test_thinking_phase_barge_in_takes_the_floor_without_a_stop_word(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Speaking into the SILENT wait interrupts, exactly like speaking over him.
+
+ The reported failure (live 2026-08-13 12:11:12): barge-in during playback
+ works perfectly, barge-in while Jarvis thinks does nothing. The playback
+ detector is armed only inside the ``echo_guard_active`` branch, so during
+ the silent wait there was no local detector at all and the only remaining
+ signal — the provider's own VAD — is deliberately parked while an action
+ runs. No stop word is spoken here on purpose: ordinary speech has to be
+ enough.
+ """
+ pipe = _pipe()
+ ready = asyncio.Event()
+ captured = b"\x0b\x00" * 32
+
+ class _Detector:
+ """Confirms on any frame — the arming logic is what is under test."""
+
+ def __init__(self, **kwargs: object) -> None:
+ self._probe = kwargs.get("output_active")
+ self.active = False
+
+ def warmup(self) -> None:
+ return None
+
+ def start_output(self) -> None:
+ self.active = True
+
+ def stop_output(self) -> None:
+ self.active = False
+
+ def feed(self, _pcm: bytes) -> bytes | None:
+ # Only the probe-less (thinking) detector ever confirms, mirroring
+ # the real gate: with a playback probe attached feed() returns None
+ # while nothing is playing.
+ return None if self._probe is not None else captured
+
+ class _Mic:
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *_exc: object) -> bool:
+ return False
+
+ async def stream(self):
+ await ready.wait()
+ # Enough frames to clear _THINKING_BARGE_QUIET_S at 32 ms/frame.
+ for _ in range(40):
+ yield AudioChunk(
+ pcm=b"\x01\x00" * 256, sample_rate=16_000, timestamp_ns=0
+ )
+ await asyncio.sleep(0.02)
+ await asyncio.Event().wait()
+
+ class _ThinkingSession(_HandshakeOnlyRealtimeSession):
+ """Owes a reply, nothing audible — the phase under test."""
+
+ def __init__(self, send_binary, send_json) -> None:
+ super().__init__(send_binary, send_json)
+ self.audio_frames: list[bytes] = []
+ self.barged = asyncio.Event()
+
+ def owes_the_user_a_reply(self) -> bool:
+ return True
+
+ def _user_is_speaking(self) -> bool:
+ return False
+
+ async def handle_control(self, message) -> None:
+ await super().handle_control(message)
+ if message.get("type") == "barge_in":
+ self.barged.set()
+
+ async def handle_audio_frame(self, pcm: bytes) -> None:
+ self.audio_frames.append(pcm)
+
+ async def wait_finished(self) -> None:
+ await self.barged.wait()
+
+ built: dict[str, object] = {}
+
+ def _build(**kwargs):
+ session = _ThinkingSession(kwargs["send_binary"], kwargs["send_json"])
+ built["session"] = session
+ ready.set()
+ return session
+
+ monkeypatch.setattr("jarvis.realtime.factory.build_realtime_session", _build)
+ monkeypatch.setattr(
+ "jarvis.realtime.desktop.DesktopRealtimeBargeInDetector", _Detector
+ )
+ monkeypatch.setattr(pipeline_mod, "MicrophoneCapture", lambda **_kwargs: _Mic())
+
+ await asyncio.wait_for(pipe._active_realtime_session(), timeout=5.0)
+
+ session = built["session"]
+ assert {"type": "barge_in"} in session.controls, (
+ "speaking during the thinking phase did not interrupt"
+ )
+ # ...and what the user said reaches the provider, or the interruption
+ # would take the floor and deliver no context.
+ assert captured in session.audio_frames
+
+
+async def test_thinking_barge_stays_disarmed_while_the_user_is_still_talking(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The user's own trailing words must not arm the barge against themselves.
+
+ A provider that commits on its own VAD cuts mid-sentence when a filler
+ pause fools it. That shape is turn fragmentation (872b05051), not an
+ interruption, and firing a barge there would abandon the order the user is
+ still in the middle of giving.
+ """
+ pipe = _pipe()
+ ready = asyncio.Event()
+
+ class _Detector:
+ def __init__(self, **kwargs: object) -> None:
+ self._probe = kwargs.get("output_active")
+ self.active = False
+ self.fed = 0
+
+ def warmup(self) -> None:
+ return None
+
+ def start_output(self) -> None:
+ self.active = True
+
+ def stop_output(self) -> None:
+ self.active = False
+
+ def feed(self, _pcm: bytes) -> bytes | None:
+ self.fed += 1
+ return None if self._probe is not None else b"\x0c\x00" * 32
+
+ class _Mic:
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *_exc: object) -> bool:
+ return False
+
+ async def stream(self):
+ await ready.wait()
+ for _ in range(30):
+ yield AudioChunk(
+ pcm=b"\x01\x00" * 256, sample_rate=16_000, timestamp_ns=0
+ )
+ await asyncio.sleep(0.02)
+ await asyncio.Event().wait()
+
+ class _StillTalkingSession(_HandshakeOnlyRealtimeSession):
+ def __init__(self, send_binary, send_json) -> None:
+ super().__init__(send_binary, send_json)
+ self.audio_frames: list[bytes] = []
+ self.settled = asyncio.Event()
+
+ def owes_the_user_a_reply(self) -> bool:
+ return True
+
+ def _user_is_speaking(self) -> bool:
+ return True
+
+ async def handle_audio_frame(self, pcm: bytes) -> None:
+ self.audio_frames.append(pcm)
+ if len(self.audio_frames) >= 20:
+ self.settled.set()
+
+ async def wait_finished(self) -> None:
+ await self.settled.wait()
+
+ built: dict[str, object] = {}
+
+ def _build(**kwargs):
+ session = _StillTalkingSession(kwargs["send_binary"], kwargs["send_json"])
+ built["session"] = session
+ ready.set()
+ return session
+
+ monkeypatch.setattr("jarvis.realtime.factory.build_realtime_session", _build)
+ monkeypatch.setattr(
+ "jarvis.realtime.desktop.DesktopRealtimeBargeInDetector", _Detector
+ )
+ monkeypatch.setattr(pipeline_mod, "MicrophoneCapture", lambda **_kwargs: _Mic())
+
+ await asyncio.wait_for(pipe._active_realtime_session(), timeout=5.0)
+
+ session = built["session"]
+ assert {"type": "barge_in"} not in session.controls, (
+ "a still-running utterance was mistaken for an interruption"
+ )
+ # The audio still flows: withholding it is what would lose the tail.
+ assert session.audio_frames