Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

feat(realtime): a readback that renames the pane it reports on is now recorded

rubenluetke10-beepcb64947df21f
diff --git a/jarvis/realtime/readback_check.py b/jarvis/realtime/readback_check.py
new file mode 100644
index 0000000000..d514893ebe
--- /dev/null
+++ b/jarvis/realtime/readback_check.py
@@ -0,0 +1,84 @@
+"""Did the spoken readback name a different pane than the result it renders?
+
+A trusted action result is handed to the live model as a RENDERING ORDER: say
+this, in your own voice, keeping every material fact. The order has always
+carried an identifier-fidelity clause — "never swap in a name from the user's
+request that the result itself does not contain" — and the model has always
+been free to ignore it. Twice now it did:
+
+* 2026-08-12 — result opened T5 and T6, the voice said "ich habe T2
+ angewiesen"; T2 was the pane the user had asked about.
+* 2026-08-13 — result opened T5 (a fresh pane, because the call-sign in the
+ user's sentence was garbled), the voice said "ich habe T1 den Auftrag
+ erteilt". The user only found out by looking at the screen.
+
+Both are the same shape, and it is the worst one a voice assistant has: the
+action is reported as the user WANTED it, so a wrong action sounds exactly
+like a right one and there is nothing to notice. ``_delegate_result_prompt``
+says in its own docstring that if the class resurfaces, the deterministic fix
+belongs at the readback boundary rather than in more prompt wording. This is
+that check.
+
+**Deliberately narrow.** It fires only when the result names a live pane, the
+rendering names a live pane, and the rendering names one the result does not.
+That is the failure verbatim. A rendering that names no pane at all is not a
+swap — paraphrasing "T5 is open" as "the new terminal is running" is a
+perfectly good reading, and treating it as a lie would make the check fire
+constantly on correct behaviour. A rendering that names the SAME pane plus
+prose is likewise clean. False positives here cost the user an unnecessary spoken correction, so
+the bar is set where only a contradiction can clear it.
+
+Spoken forms are folded into call-signs first (``names.canonical_positions``),
+so "Terminal eins" is compared as "T1" rather than missed. Everything fails
+OPEN: any fault answers "no swap", because a broken check must never invent a
+correction on a live call.
+"""
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+
+__all__ = ["swapped_call_signs"]
+
+
+def _canonical(text: str, roster: list[str]) -> str:
+ """``text`` with every spoken pane position rewritten as its call-sign."""
+ from jarvis.agentic_ide.names import canonical_positions
+
+ return canonical_positions(text, roster)
+
+
+def _named(text: str, roster: Sequence[str]) -> set[str]:
+ """Which of ``roster`` the text names, matched whole-word."""
+ found: set[str] = set()
+ for name in roster:
+ if re.search(rf"(?<!\w){re.escape(name)}(?!\w)", text, re.IGNORECASE):
+ found.add(name)
+ return found
+
+
+def swapped_call_signs(
+ result: str, rendering: str, *, roster: Sequence[str]
+) -> tuple[str, ...]:
+ """Panes the SPOKEN rendering claims that the trusted result never names.
+
+ ``result`` is the deterministic sentence the action layer produced,
+ ``rendering`` what the model actually said, and ``roster`` the workspace's
+ live call-signs. Returns the offending names sorted, or ``()`` when the
+ rendering is consistent with the result — which includes every case where
+ one of the three inputs is missing, empty, or unparseable.
+ """
+ names = [str(n).strip() for n in roster if str(n or "").strip()]
+ if not names or not str(result or "").strip() or not str(rendering or "").strip():
+ return ()
+ try:
+ stated = _named(_canonical(result, names), names)
+ if not stated:
+ # The result names no pane, so the rendering cannot contradict it
+ # about one. Guarding here and not only below keeps a result about
+ # something else entirely (a file, a setting) out of this check.
+ return ()
+ spoken = _named(_canonical(rendering, names), names)
+ except Exception: # noqa: BLE001 - a faulty check must never fire
+ return ()
+ return tuple(sorted(spoken - stated))
diff --git a/jarvis/realtime/session.py b/jarvis/realtime/session.py
index ee74a19cdd..d746e6f4cd 100644
--- a/jarvis/realtime/session.py
+++ b/jarvis/realtime/session.py
@@ -6450,6 +6450,7 @@ class RealtimeVoiceSession:
self.session_id,
)
external_update = None
+ await self._check_readback_fidelity(answer, delegate_state, external_update)
response_text = answer or (
delegate_state.last_reply if delegate_state is not None else ""
)
@@ -7111,6 +7112,70 @@ class RealtimeVoiceSession:
for turn_id, tasks in self._delegate_tasks_by_turn.items()
)
+ async def _check_readback_fidelity(
+ self,
+ rendering: str,
+ delegate_state: Any,
+ external_update: _ExternalUpdateState | None,
+ ) -> None:
+ """Record it when the spoken readback renamed the pane it reported on.
+
+ The rendering order forbids swapping in a name the result does not
+ contain, and the model did it anyway twice — 2026-08-12 and 2026-08-13
+ — each time substituting the pane the USER had named for the one the
+ action actually touched. It is the one wrong readback nobody catches by
+ ear: it reports the action the user wanted, so a wrong action and a
+ right one sound identical, and the user finds out by looking at the
+ screen or not at all.
+
+ This is the boundary ``_delegate_result_prompt`` points at when it says
+ the deterministic fix does not belong in more prompt wording. It only
+ OBSERVES: a spoken correction has to be a same-voice provider
+ re-render, because the 2026-07-21 maintainer verdict rules out claiming
+ the turn for the surface TTS (it flipped the voice on every delegated
+ turn), and how often a correction would fire is not yet measured. What
+ this buys today is that the failure stops being invisible — it lands in
+ the log and on the bus with both texts side by side, so a recurrence is
+ a search rather than a reconstruction from provider rollout files.
+
+ Never raises. An observation must not be able to end a live call.
+ """
+ try:
+ trusted = ""
+ if delegate_state is not None:
+ trusted = str(getattr(delegate_state, "last_reply", "") or "")
+ elif external_update is not None:
+ trusted = str(external_update.source_text or "")
+ if not trusted.strip() or not str(rendering or "").strip():
+ return
+ from jarvis.realtime.readback_check import swapped_call_signs
+
+ swapped = swapped_call_signs(
+ trusted, rendering, roster=self._workspace_call_signs()
+ )
+ if not swapped:
+ return
+ log.warning(
+ "realtime[%s] readback named %s, which the trusted result does "
+ "not mention — spoken: %s | result: %s",
+ self.session_id,
+ ", ".join(swapped),
+ safe_preview(rendering, max_chars=200),
+ safe_preview(trusted, max_chars=200),
+ )
+ await self._publish_error(
+ "readback_identifier_swap",
+ f"The spoken readback named {', '.join(swapped)}, which the "
+ f"action result does not mention.",
+ recoverable=True,
+ )
+ except Exception: # noqa: BLE001 - an observation never breaks a call
+ log.debug(
+ "realtime[%s] readback fidelity check failed",
+ self.session_id,
+ exc_info=True,
+ )
+
def _workspace_owns_turn(self, text: str) -> bool:
"""True when THIS utterance addresses an open Agentic-IDE pane itself.
diff --git a/tests/unit/realtime/test_readback_check.py b/tests/unit/realtime/test_readback_check.py
new file mode 100644
index 0000000000..b4e89227a3
--- /dev/null
+++ b/tests/unit/realtime/test_readback_check.py
@@ -0,0 +1,168 @@
+"""The readback may paraphrase freely, but it may not rename the pane.
+
+Both live failures this pins have the same shape: the deterministic result
+named one pane, the voice named the pane the USER had asked about, and nothing
+downstream compared the two. What matters as much as catching that is NOT
+catching ordinary paraphrase — a check that cried wolf would be turned off
+within a day, so the negative cases outnumber the positive ones here.
+"""
+from __future__ import annotations
+
+import pytest
+
+from jarvis.realtime.readback_check import swapped_call_signs
+
+ROSTER = ["T1", "T2", "T3", "T4", "T6", "T8"]
+
+
+class TestTheLiveFailures:
+ def test_the_2026_08_13_readback_is_caught(self) -> None:
+ """Result opened T5; the voice claimed the pane the user had named."""
+ assert swapped_call_signs(
+ "T5 ist offen. Ich briefe ihn gleich.", # i18n-allow: quoted live result
+ "Alles klar, ich habe T1 den Auftrag erteilt.", # i18n-allow: live transcript
+ roster=[*ROSTER, "T5"],
+ ) == ("T1",)
+
+ def test_the_2026_08_12_readback_is_caught(self) -> None:
+ """Result opened T5 and T6; the voice named T2."""
+ assert swapped_call_signs(
+ "2 neue Terminals: T5, T6.", # i18n-allow: quoted live result
+ "Ich habe T2 angewiesen, das zu übernehmen.", # i18n-allow: quoted live transcript
+ roster=[*ROSTER, "T5"],
+ ) == ("T2",)
+
+ def test_a_spoken_position_is_folded_before_comparing(self) -> None:
+ """"Terminal eins" is T1 — a swap said in words is still a swap."""
+ assert swapped_call_signs(
+ "T3 ist offen.", # i18n-allow: quoted result
+ "Ich habe Terminal eins gebrieft.", # i18n-allow: quoted transcript
+ roster=ROSTER,
+ ) == ("T1",)
+
+
+class TestOrdinaryRenderingIsLeftAlone:
+ @pytest.mark.parametrize(
+ ("result", "rendering"),
+ [
+ # The same pane, reworded — the whole point of the rendering order.
+ ("T3 ist offen.", "Alles klar, T3 läuft jetzt."), # i18n-allow: quoted
+ # Named plus prose around it.
+ ("T3 ist offen.", "Ich habe T3 gebrieft."), # i18n-allow: quoted
+ # No pane named at all: a legitimate paraphrase, not a claim.
+ ("T3 ist offen.", "Das neue Terminal läuft."), # i18n-allow: quoted
+ # Every pane of a multi-pane result, in a different order.
+ ("2 neue Terminals: T5, T6.", "T6 und T5 sind offen."), # i18n-allow: quoted
+ # A subset — saying less is not saying something false.
+ ("2 neue Terminals: T5, T6.", "T5 ist offen."), # i18n-allow: quoted
+ # The result is about no pane at all, so nothing can contradict it.
+ ("Alles gespeichert.", "Ich habe T1 gespeichert."), # i18n-allow: quoted
+ # A pane name inside a longer word must not match.
+ ("T3 ist offen.", "Das T3000-Modul."), # i18n-allow: quoted
+ ],
+ )
+ def test_no_correction_is_invented(self, result: str, rendering: str) -> None:
+ assert swapped_call_signs(result, rendering, roster=ROSTER) == ()
+
+
+class TestItFailsOpen:
+ @pytest.mark.parametrize(
+ ("result", "rendering", "roster"),
+ [
+ ("", "Ich habe T1 gebrieft.", ROSTER),
+ ("T3 ist offen.", "", ROSTER),
+ ("T3 ist offen.", "Ich habe T1 gebrieft.", []),
+ ("T3 ist offen.", "Ich habe T1 gebrieft.", ["", " "]),
+ ],
+ )
+ def test_missing_inputs_report_nothing(
+ self, result: str, rendering: str, roster: list[str]
+ ) -> None:
+ assert swapped_call_signs(result, rendering, roster=roster) == ()
+
+ def test_a_broken_canonicaliser_reports_nothing(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ """A fault in the check must never invent a spoken correction."""
+ from jarvis.agentic_ide import names
+
+ def _boom(*_args: object, **_kwargs: object) -> str:
+ raise RuntimeError("canonicaliser is down")
+
+ monkeypatch.setattr(names, "canonical_positions", _boom)
+ assert swapped_call_signs(
+ "T3 ist offen.", "Ich habe T1 gebrieft.", roster=ROSTER
+ ) == ()
+
+
+def test_a_custom_pane_name_is_compared_too() -> None:
+ """Panes may carry any label; the check must not be positions-only."""
+ assert swapped_call_signs(
+ "Nova ist offen.", "Ich habe Mika gebrieft.", roster=["Mika", "Nova"]
+ ) == ("Mika",)
+
+
+class TestTheSessionWiring:
+ """The check is only worth anything if the turn boundary actually calls it.
+
+ Driven through the real method with a stand-in ``self``: building a whole
+ live session to observe one observation would pin the scaffolding rather
+ than the behaviour, and the method touches exactly these four attributes.
+ """
+
+ @staticmethod
+ def _session(roster: tuple[str, ...], published: list[tuple[str, str]]):
+ from types import SimpleNamespace
+
+ async def _publish_error(error_type: str, message: str, *, recoverable: bool):
+ assert recoverable is True
+ published.append((error_type, message))
+
+ return SimpleNamespace(
+ session_id="test-session",
+ _workspace_call_signs=lambda: roster,
+ _publish_error=_publish_error,
+ )
+
+ async def test_a_swapped_readback_is_published(self) -> None:
+ from jarvis.realtime.session import RealtimeVoiceSession
+
+ published: list[tuple[str, str]] = []
+ session = self._session(("T1", "T5"), published)
+ delegate = type("_State", (), {"last_reply": "T5 ist offen."})()
+ await RealtimeVoiceSession._check_readback_fidelity(
+ session,
+ "Alles klar, ich habe T1 den Auftrag erteilt.", # i18n-allow: quoted
+ delegate,
+ None,
+ )
+ assert len(published) == 1
+ error_type, message = published[0]
+ assert error_type == "readback_identifier_swap"
+ assert "T1" in message
+
+ async def test_a_faithful_readback_publishes_nothing(self) -> None:
+ from jarvis.realtime.session import RealtimeVoiceSession
+
+ published: list[tuple[str, str]] = []
+ session = self._session(("T1", "T5"), published)
+ delegate = type("_State", (), {"last_reply": "T5 ist offen."})()
+ await RealtimeVoiceSession._check_readback_fidelity(
+ session, "Alles klar, T5 läuft jetzt.", delegate, None # i18n-allow: quoted
+ )
+ assert published == []
+
+ async def test_a_broken_session_never_raises(self) -> None:
+ """The observation must not be able to end a live call."""
+ from jarvis.realtime.session import RealtimeVoiceSession
+
+ class _Exploding:
+ session_id = "test-session"
+
+ @staticmethod
+ def _workspace_call_signs():
+ raise RuntimeError("registry is down")
+
+ await RealtimeVoiceSession._check_readback_fidelity(
+ _Exploding(), "Ich habe T1 gebrieft.", None, None
+ )