Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
fix(voice): retain agent replies until playback completes
rubenluetke10-beep8dfc2ff478ff
diff --git a/jarvis/realtime/desktop.py b/jarvis/realtime/desktop.pyindex d3580cf0c1..0f6ba435fb 100644--- a/jarvis/realtime/desktop.py+++ b/jarvis/realtime/desktop.py@@ -494,7 +494,7 @@ class DesktopRealtimePlayback:0.0, float(resume_without_prebuffer_s))self._queue: asyncio.Queue[AudioChunk | None] | None = None- self._task: asyncio.Task[None] | None = None+ self._task: asyncio.Task[bool | None] | None = Noneself._closed = Falseself._last_finish_at = 0.0@@ -552,13 +552,15 @@ class DesktopRealtimePlayback:raise RuntimeError("Cannot change realtime sample rate during playback")self._sample_rate = rate- async def finish_turn(self) -> None:+ async def finish_turn(self) -> bool:+ """Confirm only a turn that drained without cancellation or timeout."""queue, task = self._queue, self._taskif queue is None or task is None:- return+ return Falseawait queue.put(None)try:- await asyncio.wait_for(task, timeout=self._finish_timeout_s)+ result = await asyncio.wait_for(task, timeout=self._finish_timeout_s)+ return result is not False and self._task is taskexcept TimeoutError:self._player.stop()task.cancel()@@ -591,6 +593,8 @@ class DesktopRealtimePlayback:# its jitter reserve.self._last_finish_at = time.monotonic()+ return False+async def cancel(self) -> None:queue, task = self._detach()task_was_done = task is not None and task.done()@@ -629,7 +633,7 @@ class DesktopRealtimePlayback:await self.cancel()@staticmethod- def _observe_playback_result(task: asyncio.Task[None]) -> None:+ def _observe_playback_result(task: asyncio.Task[bool | None]) -> None:if task.cancelled():returntry:@@ -641,7 +645,7 @@ class DesktopRealtimePlayback:def _detach(self,- ) -> tuple[asyncio.Queue[AudioChunk | None] | None, asyncio.Task[None] | None]:+ ) -> tuple[asyncio.Queue[AudioChunk | None] | None, asyncio.Task[bool | None] | None]:queue, task = self._queue, self._taskself._queue = Noneself._task = Nonediff --git a/jarvis/speech/pipeline.py b/jarvis/speech/pipeline.pyindex c92c6b3fc4..ce248bd7cb 100644--- a/jarvis/speech/pipeline.py+++ b/jarvis/speech/pipeline.py@@ -3004,6 +3004,10 @@ class SpeechPipeline:# mid-utterance. Preambles are dropped, not parked — they are stale by# the time the user finishes. See ``_on_announcement`` + ``_set_turn_state``.self._deferred_announcements: list[AnnouncementRequested] = []+ self._agent_reply_inflight: AnnouncementRequested | None = None+ self._agent_reply_inflight_text = ""+ self._agent_reply_retries: list[AnnouncementRequested] = []+ self._agent_reply_retry_task: asyncio.Task[None] | None = None# Optional pre-rendered acknowledgement PCM, populated during warmup.self._ack_pcm: bytes = b""# Pre-rendered Task-Ack-Phrasen ("Sofort.", "Right away." …) als PCM-Cache.@@ -3805,6 +3809,11 @@ class SpeechPipeline:):self._last_answer_floor_monotonic = time.monotonic()self._turn_state = new_state+ if new_state is TurnTakingState.IDLE:+ retry = getattr(self, "_agent_reply_retry_task", None)+ if retry is not None and retry is not asyncio.current_task():+ retry.cancel()+ await asyncio.gather(retry, return_exceptions=True)await self._transition(self._supervisor_state_for_turn(new_state))# Turn-boundary: the floor has cleared → flush any announcements that# were deferred while the user was speaking (AD-OE6 zero-silent-drop).@@ -3818,6 +3827,9 @@ class SpeechPipeline:pending = self._deferred_announcementsself._deferred_announcements = []for event in pending:+ if new_state is TurnTakingState.IDLE and self._is_agent_reply(event):+ self._defer_agent_reply(event)+ continueasyncio.create_task(self._on_announcement(event), name="deferred-announcement")@@ -4805,20 +4817,96 @@ class SpeechPipeline:log.warning("cron skill announcement failed: %s", exc)log.info("Cron skill '%s' executed via brain turn", skill.name)- async def _on_announcement(self, event: AnnouncementRequested) -> None:- """TTS-Bypass-Handler (CL-13): spricht sofort, ohne Brain-Pfad.+ @staticmethod+ def _is_agent_reply(event: AnnouncementRequested) -> bool:+ return event.source_layer == "society.lead" and event.kind in _READBACK_KINDS++ def _agent_reply_needs_session(self) -> bool:+ hangup = getattr(self, "_hangup_event", None)+ return bool(+ getattr(self, "_muted", False)+ or (hangup is not None and hangup.is_set())+ or getattr(self, "_turn_state", TurnTakingState.IDLE) is TurnTakingState.IDLE+ or getattr(self, "_voice_engine_transitioning", False)+ )++ def _defer_agent_reply(self, event: AnnouncementRequested) -> None:+ pending = getattr(self, "_deferred_announcements", None)+ if pending is None:+ pending = self._deferred_announcements = []+ if event not in pending:+ pending.append(event)++ def _settle_agent_reply(self, *, completed: bool) -> None:+ """Only a drained speaker queue confirms a realtime agent readback."""+ event = getattr(self, "_agent_reply_inflight", None)+ self._agent_reply_inflight = None+ self._agent_reply_inflight_text = ""+ if event is not None and not completed:+ retries = getattr(self, "_agent_reply_retries", None)+ if retries is None:+ retries = self._agent_reply_retries = []+ if event not in retries:+ retries.append(event)+ log.info("Agent reply interrupted; retained for the next voice session")++ def _restore_agent_replies(self) -> None:+ """Rejoin the existing floor-aware queue when a new call starts."""+ for event in getattr(self, "_agent_reply_retries", []):+ self._defer_agent_reply(event)+ self._agent_reply_retries = []+ pending = getattr(self, "_deferred_announcements", None)+ if pending:+ pending.sort(key=lambda event: event.timestamp_ns)- Bei ``priority="interrupt"`` wird laufendes Audio-Playback via- ``AudioPlayer.stop()`` abgebrochen (Barge-in-aequivalent).+ def _retry_agent_reply_after_boundary(self) -> None:+ """Let the live wrapper finish resetting after the speaker drains."""+ previous = getattr(self, "_agent_reply_retry_task", None)+ if previous is not None and not previous.done():+ return+ if self._agent_reply_needs_session():+ return+ self._agent_reply_retry_task = asyncio.create_task(+ self._retry_pending_agent_replies(), name="agent-reply-boundary"+ )- Nutzt ``synthesize()`` + ``player.play_chunks()`` — denselben Pfad wie- die normale Antwort-Ausgabe. Frueher stand hier ``self._tts.speak(...)``;- das war ein Phantom-Call, da ``GeminiFlashTTS`` nur ``synthesize()``- exponiert. Der stumme ``AttributeError`` machte Sub-Agent-Announcements- unhoerbar (Silent-Failure, entdeckt 2026-04-23).+ async def _retry_pending_agent_replies(self) -> None:+ while True:+ # No provider request is made while its wrapper reports busy.+ # This closes the gap between the desktop's LISTENING callback+ # and the provider wrapper clearing its completed turn state.+ await asyncio.sleep(0.1)+ if (+ self._agent_reply_needs_session()+ or self._turn_state is not TurnTakingState.LISTENING+ or getattr(self, "_agent_reply_inflight", None) is not None+ ):+ return+ event = next((event for event in self._deferred_announcements+ if self._is_agent_reply(event)), None)+ if event is None:+ return+ self._deferred_announcements.remove(event)+ try:+ await self._on_announcement(event)+ except asyncio.CancelledError:+ self._defer_agent_reply(event)+ raise+ except Exception:+ self._defer_agent_reply(event)+ log.warning("Agent reply retry failed; retained for a later turn", exc_info=True)+ return++ async def _on_announcement(self, event: AnnouncementRequested) -> None:+ """Deliver a readback through the live voice or the classic TTS path.++ Agent replies wait for an open call. Interrupt-priority announcements+ retain the existing player-stop behavior. Classic playback uses+ synthesize() and play_chunks(), the same path as ordinary answers."""event_kind = getattr(event, "kind", None)is_readback = event_kind in _READBACK_KINDS+ is_agent_reply = self._is_agent_reply(event)if is_readback:# Muting controls audio, not conversational memory. A mission that# finishes while muted must still be available to the next follow-up.@@ -4836,6 +4924,13 @@ class SpeechPipeline:"Realtime announcement context mirror failed",exc_info=True,)+ if is_agent_reply:+ if event == getattr(self, "_agent_reply_inflight", None):+ return+ if self._agent_reply_needs_session():+ self._defer_agent_reply(event)+ log.info("Agent reply retained until an unmuted voice session is available")+ returnif getattr(self, "_muted", False):log.debug("Announcement suppressed — voice muted: %r", event.text)return@@ -4949,7 +5044,10 @@ class SpeechPipeline:return# Completion/readback owes the user information → park it and flush# at the next turn-boundary (AD-OE6 zero-silent-drop).- self._deferred_announcements.append(event)+ if is_agent_reply:+ self._defer_agent_reply(event)+ else:+ self._deferred_announcements.append(event)log.info("Announcement deferred — user holds the floor: %r",event.text[:80],@@ -5158,13 +5256,20 @@ class SpeechPipeline:event.text[:80],)return- self._deferred_announcements.append(event)+ if is_agent_reply:+ self._defer_agent_reply(event)+ self._retry_agent_reply_after_boundary()+ else:+ self._deferred_announcements.append(event)log.info("Announcement deferred — a live realtime call owns the ""voice: %r",event.text[:80],)return+ if is_agent_reply and self._agent_reply_needs_session():+ self._defer_agent_reply(event)+ return# Re-check the hangup gate at the moment of SPEAKING, not only at# arrival: the user can hang up during the seconds between the two# (scrub, dedup, the realtime-delivery probe), and the entry check@@ -5206,6 +5311,7 @@ class SpeechPipeline:if animate:await self._transition("SPEAKING")self._assistant_work_count = getattr(self, "_assistant_work_count", 0) + 1+ agent_reply_completed = Falsetry:# Drive the TTS pin from the SAME resolved language as the scrub,# not from event.language again — a None/auto tag here used to send@@ -5218,7 +5324,7 @@ class SpeechPipeline:chunks = self._tts.synthesize(scrubbed.cleaned, language_code=lang_code)except TypeError:chunks = self._tts.synthesize(scrubbed.cleaned)- if is_preamble:+ if is_preamble or is_agent_reply:# Staleness gate evaluated by the player right before it writes# audio: if the answer has started speaking by the time the# preamble's synthesis + play-lock wait completes, drop it so it@@ -5231,7 +5337,9 @@ class SpeechPipeline:playback_result = await self._player.play_chunks(chunks,should_play=lambda: (- getattr(self, "_turn_state", TurnTakingState.IDLE)+ not self._agent_reply_needs_session()+ if is_agent_reply+ else getattr(self, "_turn_state", TurnTakingState.IDLE)is not TurnTakingState.JARVIS_SPEAKING),)@@ -5239,7 +5347,10 @@ class SpeechPipeline:playback_result = await self._player.play_chunks(chunks)else:playback_result = await self._player.play_chunks(chunks)- if self._playback_confirmed(playback_result):+ agent_reply_completed = self._playback_confirmed(playback_result) and (+ not is_agent_reply or not self._agent_reply_needs_session()+ )+ if agent_reply_completed:self._emit_spoken(scrubbed.cleaned,ann_lang,@@ -5247,8 +5358,10 @@ class SpeechPipeline:getattr(event, "detail", None),)except Exception as exc: # noqa: BLE001- log.warning("Announcement-Speak fehlgeschlagen: %s", exc)+ log.warning("Announcement playback failed: %s", exc)finally:+ if is_agent_reply and not agent_reply_completed:+ self._defer_agent_reply(event)self._assistant_work_count -= 1self._last_announcement_spoken_monotonic = time.monotonic()if animate:@@ -5286,6 +5399,14 @@ class SpeechPipeline:deliver = getattr(session, "deliver_announcement", None)if not callable(deliver):return False+ agent_reply = self._is_agent_reply(event)+ if agent_reply:+ if getattr(self, "_agent_reply_inflight", None) is not None:+ return False+ # Register before transport I/O so fast playback cannot race acceptance.+ self._agent_reply_inflight = event+ self._agent_reply_inflight_text = text+ accepted = Falsetry:accepted = bool(await deliver(@@ -5300,6 +5421,10 @@ class SpeechPipeline:except Exception as exc: # noqa: BLE001 -- classic path is load-bearinglog.warning("Realtime announcement handoff failed: %s", exc)return False+ finally:+ if agent_reply and not accepted and self._agent_reply_inflight is event:+ self._agent_reply_inflight = None+ self._agent_reply_inflight_text = ""if accepted:log.info("Announcement handed to active realtime provider %s: %r",@@ -8514,6 +8639,7 @@ class SpeechPipeline:self._last_announcement_spoken_monotonic = Noneself._previous_turn_state = Noneself._current_voice_session_id = session_id+ self._restore_agent_replies()self._active_voice_mode = self._configured_voice_mode()self._active_realtime_provider = ""self._active_realtime_model = ""@@ -9376,10 +9502,12 @@ class SpeechPipeline:"at": time.time(),}elif kind == "tts_cancel":+ self._settle_agent_reply(completed=False)_close_output_segment(preserve_echo_tail=False)await _cancel_output_playback()await self._set_turn_state(TurnTakingState.LISTENING)elif kind == "hangup":+ self._settle_agent_reply(completed=False)semantic_turn_committed = True_close_output_segment(preserve_echo_tail=False)await _cancel_output_playback()@@ -9389,8 +9517,12 @@ class SpeechPipeline:recover()elif kind == "turn_complete":semantic_turn_committed = True- await playback.finish_turn()+ playback_completed = await playback.finish_turn()interrupted_during_drain = not speaking+ self._settle_agent_reply(completed=bool(+ playback_completed and not interrupted_during_drain+ and not self._agent_reply_needs_session()+ ))_close_output_segment(preserve_echo_tail=not interrupted_during_drain,)@@ -9554,6 +9686,11 @@ class SpeechPipeline:if surface_playback_task is active_surface_task:surface_playback_task = Noneif self._playback_confirmed(playback_result):+ if (+ cleaned == getattr(self, "_agent_reply_inflight_text", "")+ and not self._agent_reply_needs_session()+ ):+ self._settle_agent_reply(completed=True)self._emit_spoken(cleaned,language,@@ -9971,6 +10108,7 @@ class SpeechPipeline:return HANGUP_ERRORfinally:if getattr(self, "_active_realtime_handle", None) is session:+ self._settle_agent_reply(completed=False)self._active_realtime_handle = None# Invalidate even when synthesis is paused before child-task# creation; terminal teardown must own that pre-playback window too.diff --git a/tests/unit/realtime/test_desktop.py b/tests/unit/realtime/test_desktop.pyindex 092e9a19ee..e4545ecbb5 100644--- a/tests/unit/realtime/test_desktop.py+++ b/tests/unit/realtime/test_desktop.py@@ -84,7 +84,7 @@ async def test_turn_uses_one_stream_and_drains_in_order():await playback.send_binary(b"\x01\x00" * 8)await playback.send_binary(b"\x02\x00" * 8)- await playback.finish_turn()+ assert await playback.finish_turn() is Trueassert [chunk.pcm for chunk in player.chunks] == [b"\x01\x00" * 8,@@ -151,7 +151,7 @@ async def test_cancel_can_interrupt_a_turn_already_draining() -> None:await asyncio.sleep(0)await playback.cancel()- await drain+ assert await drain is Falseassert player.stopped == 1assert player.chunks == []@@ -178,7 +178,7 @@ async def test_cancel_during_drain_treats_stopped_stream_as_expected() -> None:await asyncio.sleep(0)await playback.cancel()- await drain+ assert await drain is Falseassert player.stopped == 1@@ -197,6 +197,21 @@ async def test_finish_turn_still_surfaces_an_unrelated_playback_failure() -> Nonawait playback.finish_turn()+async def test_finish_turn_does_not_confirm_missing_or_timed_out_audio() -> None:+ class StalledPlayer(FakePlayer):+ async def play_chunks(self, chunks) -> None:+ await asyncio.Event().wait()++ player = StalledPlayer()+ playback = DesktopRealtimePlayback(player)+ assert await playback.finish_turn() is False+ await playback.send_binary(b"\x01\x00" * 8)+ playback._finish_timeout_s = 0.01+ assert await playback.finish_turn() is False+ assert player.stopped == 1++@pytest.mark.asyncioasync def test_handshake_sample_rate_is_used_for_the_next_turn():player = FakePlayer()diff --git a/tests/unit/speech/test_agent_reply_readbacks.py b/tests/unit/speech/test_agent_reply_readbacks.pynew file mode 100644index 0000000000..7fec34ac87--- /dev/null+++ b/tests/unit/speech/test_agent_reply_readbacks.py@@ -0,0 +1,207 @@+"""Agent replies remain owed until audio finishes, across voice-call boundaries."""++import asyncio++import pytest++from jarvis.core.events import AnnouncementRequested+from jarvis.speech.pipeline import TurnTakingState+from tests.unit.speech.test_realtime_announcement_bridge import _pipeline+from tests.unit.speech.test_realtime_mode import _FakeRealtimeSession, _FakeTTS, _pipe, _SilentMic+++def reply(name="Nala"):+ return AnnouncementRequested(+ source_layer="society.lead",+ kind="completion",+ language="en",+ text=f"{name} reports: The draft is ready.",+ detail=f"agent={name} trace=conversation",+ )++[email protected]("blocked", ["idle", "hangup", "muted", "connecting"])+async def test_agent_reply_waits_for_an_available_call(blocked):+ pipe, tts, player, realtime = _pipeline(accepted=True)+ event = reply()+ if blocked == "idle":+ pipe._turn_state = TurnTakingState.IDLE+ elif blocked == "hangup":+ pipe._hangup_event.set()+ elif blocked == "muted":+ pipe._muted = True+ else:+ pipe._voice_engine_transitioning = True+ await pipe._on_announcement(event)+ await pipe._on_announcement(event)+ assert pipe._deferred_announcements == [event]+ assert not realtime.calls and not tts.calls and not player.plays+ pipe._hangup_event.clear()+ pipe._muted = False+ pipe._voice_engine_transitioning = False+ await pipe._set_turn_state(TurnTakingState.LISTENING)+ await asyncio.sleep(0.02)+ assert len(realtime.calls) == 1+ assert pipe._agent_reply_inflight == event++[email protected]("name", ["Nala", "Scout", "Mail agent"])+async def test_interrupted_reply_retries_next_session_and_completed_reply_does_not(name):+ pipe, tts, _, realtime = _pipeline(accepted=True)+ event = reply(name)+ await pipe._on_announcement(event)+ await pipe._on_announcement(event)+ assert len(realtime.calls) == 1+ pipe._settle_agent_reply(completed=False)+ await pipe._set_turn_state(TurnTakingState.LISTENING)+ await asyncio.sleep(0.02)+ assert len(realtime.calls) == 1 # No immediate repetition after a barge-in.+ pipe._restore_agent_replies()+ await pipe._set_turn_state(TurnTakingState.LISTENING)+ await asyncio.sleep(0.02)+ assert len(realtime.calls) == 2+ assert realtime.calls[0] == realtime.calls[1]+ assert not tts.calls+ pipe._settle_agent_reply(completed=True)+ pipe._restore_agent_replies()+ await pipe._set_turn_state(TurnTakingState.LISTENING)+ await asyncio.sleep(0.02)+ assert len(realtime.calls) == 2+ assert pipe._agent_reply_inflight is None+++async def test_multiple_agent_replies_are_spoken_one_at_a_time():+ pipe, _, _, realtime = _pipeline(accepted=True)+ first, second = reply(), reply("Scout")+ await pipe._on_announcement(first)+ await pipe._on_announcement(second)+ assert pipe._agent_reply_inflight == first+ assert pipe._deferred_announcements == [second]+ pipe._settle_agent_reply(completed=True)+ await pipe._set_turn_state(TurnTakingState.LISTENING)+ await asyncio.sleep(0.02)+ assert len(realtime.calls) == 2+ assert pipe._agent_reply_inflight == second+++async def test_hangup_during_handoff_does_not_start_classic_tts():+ pipe, tts, player, _ = _pipeline(accepted=False)++ class Handle:+ async def deliver_announcement(self, **kwargs):+ pipe._hangup_event.set()+ pipe._active_realtime_handle = None+ return False++ pipe._active_realtime_handle = Handle()+ event = reply()+ await pipe._on_announcement(event)+ assert pipe._agent_reply_inflight is None+ assert pipe._deferred_announcements == [event]+ assert not tts.calls and not player.plays+++async def test_idle_boundary_retry_does_not_need_another_user_utterance():+ pipe, _, _, realtime = _pipeline(accepted=False)+ event = reply()+ await pipe._on_announcement(event)+ # The speaker is idle, but the provider wrapper is still resetting.+ realtime.accepted = True+ await asyncio.sleep(0.15)+ assert len(realtime.calls) == 2+ assert pipe._agent_reply_inflight == event+ await pipe._set_turn_state(TurnTakingState.IDLE)+++async def test_hangup_cancels_a_boundary_retry_without_losing_the_message():+ pipe, _, _, realtime = _pipeline(accepted=False)+ event = reply()+ await pipe._on_announcement(event)+ pipe._hangup_event.set()+ await pipe._set_turn_state(TurnTakingState.IDLE)+ assert pipe._agent_reply_retry_task.done()+ assert len(realtime.calls) == 1+ assert pipe._deferred_announcements == [event]+++async def test_classic_playback_failure_keeps_the_agent_reply():+ pipe, _, _, _ = _pipeline(accepted=False)+ pipe._active_voice_mode = "pipeline"+ pipe._active_realtime_handle = None++ class Player:+ async def play_chunks(self, chunks, **kwargs):+ async for _ in chunks:+ pass+ return False++ pipe._player = Player()+ event = reply()+ await pipe._on_announcement(event)+ assert pipe._deferred_announcements == [event]++[email protected]("ending", ["complete", "cancel", "hangup", "surface"])+async def test_real_desktop_callbacks_confirm_only_a_completed_readback(monkeypatch, ending):+ pipe = _pipe()+ pipe._turn_state = TurnTakingState.LISTENING+ event = reply()+ monkeypatch.setattr(+ "jarvis.plugins.tts.build_realtime_surface_tts",+ lambda *args: _FakeTTS(b"\x02\x00" * 16),+ )++ class Session(_FakeRealtimeSession):+ async def handle_control(self, message):+ await self._send_json(+ {+ "type": "audio_ready",+ "provider": "fake-live",+ "input_sample_rate": 16000,+ "output_sample_rate": 24000,+ }+ )++ async def wait_finished(self):+ pipe._agent_reply_inflight = event+ pipe._agent_reply_inflight_text = event.text+ if ending == "surface":+ await self._send_json(+ {+ "type": "error_spoken",+ "text": event.text,+ "language": "en",+ "spoken_kind": "completion",+ "detail": event.detail,+ }+ )+ pipe._hangup_event.set()+ await self._forever.wait()+ return+ await self._send_binary(b"\x01\x00" * 16)+ if ending == "cancel":+ await self._send_json({"type": "tts_cancel"})+ if ending != "hangup":+ await self._send_json({"type": "turn_complete"})+ if ending != "complete":+ pipe._hangup_event.set()+ await self._forever.wait()++ monkeypatch.setattr(+ "jarvis.realtime.factory.build_realtime_session",+ lambda **kw: Session(kw["send_binary"], kw["send_json"]),+ )+ monkeypatch.setattr("jarvis.speech.pipeline.MicrophoneCapture", lambda **kw: _SilentMic())+ await asyncio.wait_for(pipe._active_realtime_session(), timeout=3)+ assert pipe._agent_reply_inflight is None+ assert getattr(pipe, "_agent_reply_retries", []) == (+ [] if ending in {"complete", "surface"} else [event]+ )