Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(gemini-live): deliver per-turn directives instead of discarding them

Harald79c5790643ec
diff --git a/jarvis/plugins/realtime/gemini_live.py b/jarvis/plugins/realtime/gemini_live.py
index eb67a05128..d527625a0f 100644
--- a/jarvis/plugins/realtime/gemini_live.py
+++ b/jarvis/plugins/realtime/gemini_live.py
@@ -62,6 +62,22 @@ _END_SENSITIVITY_MEMBERS = {
"high": "END_SENSITIVITY_HIGH",
}
+# Per-turn steering keys, in the order they are written into one delta.
+# ``language`` first: it changes how everything after it is spoken.
+_STEERING_ORDER = ("language", "turn", "standing")
+
+# Framing for the steering delta. The session's own one-speaker directive —
+# already in this connection's fixed system instruction — declares developer
+# messages silent configuration that the model must never acknowledge, with
+# exactly one opener as the exception. This header deliberately does NOT use
+# that opener, and restates the silence so a model whose connect-time rule has
+# faded still stays quiet.
+_STEERING_HEADER = (
+ "Developer message — silent configuration update. It is NOT something the "
+ "user said and NOT a request to speak. Apply it from now on: do not "
+ "acknowledge it, do not answer it, and do not mention it."
+)
+
def _end_of_speech_sensitivity(types: Any, preference: str | None) -> Any | None:
"""Resolve the requested end-of-speech patience, or None on an old SDK.
@@ -175,6 +191,13 @@ class _GeminiLiveSession:
# reason=error (BUG-071). This adapter has no in-protocol resume, so the
# orchestrator may reopen a fresh session in place and continue the call.
rebuild_on_transport_death = True
+ # An answer blocked at the speech boundary cannot be re-requested here:
+ # this transport only ever generates on its own VAD boundary, so the
+ # orchestrator's plain ``request_response()`` fallback is a no-op and the
+ # turn stays silent until the 20 s stall watchdog fires. A developer text
+ # turn IS a working trigger on this channel (the same one that carries
+ # every delegate readback), so the retry travels that way instead.
+ supports_prompted_response_retry = True
# Gemini's native audio is a GENERATIVE renderer, not a fixed-voice
# synthesizer: the pinned prebuilt voice is a starting point the model
# can audibly drift from when the spoken content reads as a performance
@@ -195,6 +218,8 @@ class _GeminiLiveSession:
connection_cm: Any,
client: Any,
session_id: str,
+ instructions: str = "",
+ language: str = "",
) -> None:
self._session = session
self._connection_cm = connection_cm
@@ -206,6 +231,23 @@ class _GeminiLiveSession:
# orchestrator can sum generations without double counting the
# progressive snapshots some SDK versions repeat mid-generation.
self._pending_usage: dict[str, int] | None = None
+ # The system instruction THIS connection was opened with. It can never
+ # be replaced (see update_session), so it is also the baseline: a
+ # directive whose exact text already stands in it needs no delivery.
+ self._connect_instructions = str(instructions or "")
+ # Steering the model has already been told, and steering that still
+ # has to travel: key -> (compared value, text to send).
+ self._delivered_steering: dict[str, str] = {
+ "language": str(language or "")
+ }
+ self._pending_steering: dict[str, tuple[str, str]] = {}
+ # Where the conversation stands, read off the stream this adapter
+ # already parses. A steering text is a USER-side realtime input, so it
+ # may only travel while the user's turn is open and the model is not
+ # generating — anywhere else it would interrupt a reply in flight or
+ # open a turn of its own.
+ self._user_turn_open = False
+ self._model_generating = False
async def send_audio(self, chunk: Any) -> None:
from google.genai import types # lazy (AP-26)
@@ -238,6 +280,7 @@ class _GeminiLiveSession:
# including Gemini 3.1 events that carry multiple parts at once.
data = getattr(message, "data", None)
if data:
+ self._note_model_output()
yield _ProviderEvent(
type="audio_delta",
audio=_PcmChunk(pcm=bytes(data), sample_rate=_OUTPUT_RATE),
@@ -258,6 +301,8 @@ class _GeminiLiveSession:
# the follow-up generation's snapshot cannot overwrite it.
yield _ProviderEvent(type="usage", usage=self._pending_usage)
self._pending_usage = None
+ if function_calls:
+ self._note_model_output()
for function_call in function_calls:
raw_args = getattr(function_call, "args", None) or {}
if hasattr(raw_args, "model_dump"):
@@ -282,6 +327,7 @@ class _GeminiLiveSession:
getattr(output_transcription, "text", "") or ""
)
if output_text:
+ self._note_model_output()
yield _ProviderEvent(
type="output_transcript_delta", text=output_text
)
@@ -298,6 +344,9 @@ class _GeminiLiveSession:
type="usage", usage=self._pending_usage
)
self._pending_usage = None
+ # A barge-in ends the generation, so the channel is
+ # free again for the user turn that just started.
+ self._model_generating = False
yield _ProviderEvent(type="interrupted")
input_transcription = getattr(
@@ -307,12 +356,21 @@ class _GeminiLiveSession:
getattr(input_transcription, "text", "") or ""
)
if input_text:
+ # The user is audibly mid-turn: this is the one window
+ # in which steering may travel. Flush whatever an
+ # earlier update_session could not deliver BEFORE the
+ # orchestrator reacts to the transcript, so the delta
+ # is in context for the reply this turn produces.
+ self._user_turn_open = True
+ await self._flush_steering()
yield _ProviderEvent(
type="input_transcript", text=input_text, is_final=True
)
if bool(getattr(content, "turn_complete", False)):
turn_boundary_seen = True
+ self._model_generating = False
+ self._user_turn_open = False
# Every named TurnCompleteReason except UNSPECIFIED is
# an ABNORMAL stop (safety filter, response rejection,
# regeneration limit, ...). A natural end leaves the
@@ -369,6 +427,83 @@ class _GeminiLiveSession:
if not turn_boundary_seen:
return
+ def _note_model_output(self) -> None:
+ """The model is producing this turn — the user's turn is over."""
+ self._model_generating = True
+ self._user_turn_open = False
+
+ def _note_steering(self, key: str, value: str, rendered: str) -> None:
+ """Queue one steering key when it differs from what the model knows."""
+ value = str(value or "")
+ if not value:
+ return
+ queued = self._pending_steering.get(key)
+ known = queued[0] if queued is not None else self._delivered_steering.get(key)
+ if known == value:
+ # Unchanged since the model was last told: nothing travels. The
+ # orchestrator rebuilds these strings every turn even when nothing
+ # in them moved, and re-asserting an identical directive is pure
+ # cost on a channel that re-bills its whole context per turn.
+ return
+ if (
+ key not in self._delivered_steering
+ and queued is None
+ and value in self._connect_instructions
+ ):
+ # First sighting, and this exact text already stands in the fixed
+ # system instruction of this connection — the model has it.
+ self._delivered_steering[key] = value
+ return
+ self._pending_steering[key] = (value, rendered)
+
+ async def _flush_steering(self) -> None:
+ """Send the queued steering delta, if the channel is safe right now."""
+ if not self._pending_steering:
+ return
+ if self._model_generating or not self._user_turn_open:
+ # Outside the user's open turn a text input is a turn of its own:
+ # it would interrupt a reply in flight or provoke an unprompted
+ # one. The delta waits for the next user utterance instead.
+ return
+ # ONE ordered pass decides both what travels and what counts as told.
+ # Building the text from one sequence and marking delivered from
+ # another is how a key silently becomes "the model knows this" without
+ # ever leaving the process — the same swallow this method exists to
+ # remove, one level deeper. A key outside the known order is appended
+ # last instead of vanishing.
+ ordered = [key for key in _STEERING_ORDER if key in self._pending_steering]
+ ordered += [
+ key for key in self._pending_steering if key not in _STEERING_ORDER
+ ]
+ ordered = [key for key in ordered if self._pending_steering[key][1]]
+ if not ordered:
+ # Every queued key renders to nothing, so there is nothing to say.
+ self._pending_steering.clear()
+ return
+ body = "\n\n".join(self._pending_steering[key][1] for key in ordered)
+ try:
+ await self._session.send_realtime_input(
+ text=f"{_STEERING_HEADER}\n\n{body}"
+ )
+ except Exception: # noqa: BLE001 — steering must not kill the call
+ log.warning(
+ "gemini-live: delivering the per-turn directive update failed; "
+ "the model keeps the previous one and the next user turn "
+ "retries",
+ exc_info=True,
+ )
+ return
+ for key in ordered:
+ self._delivered_steering[key] = self._pending_steering[key][0]
+ self._pending_steering.clear()
+ # Keys only, never the directive text: this line exists so a call where
+ # the model "did not answer the way it normally would" can be checked
+ # against what actually reached it.
+ log.debug(
+ "gemini-live: delivered a per-turn steering delta (%s)",
+ ", ".join(ordered),
+ )
+
async def update_session(
self,
*,
@@ -378,12 +513,41 @@ class _GeminiLiveSession:
turn_directive: str | None = None,
standing_directive: str | None = None,
) -> None:
- # Gemini fixes system instructions at connect time. The orchestrator
- # reconnects on a substantive language change in a later session.
- # Accepting the directive keywords keeps the per-turn update off the
- # session's TypeError-retry path; both are already embedded in the
- # (fixed) instructions.
- del instructions, language, tools, turn_directive, standing_directive
+ """Deliver what CHANGED; everything fixed stays at connect time.
+
+ The Live protocol has no mid-session setup message — a
+ ``LiveClientMessage`` carries setup, client_content, realtime_input or
+ tool_response, and setup is sent once at connect — so this
+ connection's system instruction and tool declarations genuinely cannot
+ be replaced. That used to mean the orchestrator's per-turn rebuild was
+ dropped whole: a delegate directive or a language pin that changed
+ mid-call never reached the model, which then answered from a frozen
+ connect-time prompt while the session filtered as if the new rules
+ were in force.
+
+ The changed part now travels as a short developer text turn on the
+ same realtime-input channel that carries every delegate readback.
+ Only the DELTA travels: the ~21k instruction block is never re-sent,
+ an unchanged directive sends nothing at all, and a newer value
+ supersedes an older undelivered one whole.
+ """
+ # Both are connect-time on this transport. ``tools`` is not silently
+ # lost either: the session reads ``supports_tool_updates = False``
+ # above and warns that changed declarations stand until the next
+ # session. ``instructions`` is the block whose changing parts arrive
+ # below as the delta.
+ del instructions, tools
+ self._note_steering(
+ "language",
+ str(language or ""),
+ "Output language pin: speak every following reply in this "
+ f"language only: {language}.",
+ )
+ turn_text = str(turn_directive or "")
+ self._note_steering("turn", turn_text, turn_text)
+ standing_text = str(standing_directive or "")
+ self._note_steering("standing", standing_text, standing_text)
+ await self._flush_steering()
async def request_response(self, *, required_tool: str | None = None) -> None:
# Gemini Live creates a response automatically at the VAD turn boundary.
@@ -731,6 +895,11 @@ class GeminiLiveProvider:
connection_cm=connection_cm,
client=client,
session_id=str(uuid4()),
+ # The baseline for the per-turn steering delta: whatever already
+ # stands in this connection's fixed instruction never has to be
+ # re-delivered (see update_session).
+ instructions=str(getattr(cfg, "instructions", "") or ""),
+ language=str(getattr(cfg, "language", "") or ""),
)
diff --git a/tests/unit/realtime/test_gemini_live.py b/tests/unit/realtime/test_gemini_live.py
index 4a631b7a9f..2f3205bba8 100644
--- a/tests/unit/realtime/test_gemini_live.py
+++ b/tests/unit/realtime/test_gemini_live.py
@@ -246,6 +246,288 @@ async def test_text_update_uses_realtime_input_for_gemini_31() -> None:
assert calls == [{"text": "Deliver the completed mission update."}]
+_TURN_DIRECTIVE = "Delegate this turn: call jarvis_action before you answer."
+_STANDING_DIRECTIVE = "Live-call discipline: you are ONE voice."
+
+
+def _user_speaking_message(text="was ist die uhrzeit", turn_complete=False):
+ """One server message that puts the user mid-turn (no model output yet)."""
+ return _fake_message(
+ server_content=SimpleNamespace(
+ output_transcription=None,
+ input_transcription=SimpleNamespace(text=text),
+ interrupted=False,
+ turn_complete=turn_complete,
+ )
+ )
+
+
+def _model_speaking_message(text="hier ist die antwort"):
+ return _fake_message(
+ server_content=SimpleNamespace(
+ output_transcription=SimpleNamespace(text=text),
+ input_transcription=None,
+ interrupted=False,
+ turn_complete=False,
+ )
+ )
+
+
+def _turn_complete_message():
+ return _fake_message(
+ server_content=SimpleNamespace(
+ output_transcription=None,
+ input_transcription=None,
+ interrupted=False,
+ turn_complete=True,
+ )
+ )
+
+
+def _steering_session(sent, *, instructions="", language="", session_id="s-steer"):
+ """A session whose realtime-input sends are captured, driven by messages."""
+ queue: list[list[object]] = []
+
+ async def fake_receive():
+ if queue:
+ for message in queue.pop(0):
+ yield message
+
+ async def send_realtime_input(**kwargs):
+ sent.append(kwargs)
+
+ session = _GeminiLiveSession(
+ session=SimpleNamespace(
+ receive=fake_receive, send_realtime_input=send_realtime_input
+ ),
+ connection_cm=SimpleNamespace(),
+ client=SimpleNamespace(),
+ session_id=session_id,
+ instructions=instructions,
+ language=language,
+ )
+ return session, queue
+
+
+async def _drive(session, queue, messages):
+ queue.append(list(messages))
+ return [event async for event in session.receive()]
+
+
+async def test_changed_turn_directive_reaches_the_model_as_a_text_turn() -> None:
+ sent: list[dict[str, str]] = []
+ session, queue = _steering_session(sent, instructions="fixed prompt")
+
+ await _drive(session, queue, [_user_speaking_message()])
+ await session.update_session(
+ instructions="x" * 21_000, turn_directive=_TURN_DIRECTIVE
+ )
+
+ assert len(sent) == 1
+ text = sent[0]["text"]
+ assert _TURN_DIRECTIVE in text
+ # A developer message, explicitly not a request to speak.
+ assert "silent configuration" in text
+ # The rebuilt instruction block never travels; only the delta does.
+ assert "x" * 100 not in text
+ assert len(text) < 1_000
+
+
+async def test_unchanged_directive_sends_nothing_on_the_next_turn() -> None:
+ sent: list[dict[str, str]] = []
+ session, queue = _steering_session(sent, instructions="fixed prompt")
+
+ await _drive(session, queue, [_user_speaking_message()])
+ await session.update_session(turn_directive=_TURN_DIRECTIVE)
+ assert len(sent) == 1
+
+ # Second turn, same directive: the model already has it.
+ await _drive(session, queue, [_user_speaking_message("und morgen")])
+ await session.update_session(turn_directive=_TURN_DIRECTIVE)
+ assert len(sent) == 1
+
+ # Third turn with a DIFFERENT directive travels again.
+ await _drive(session, queue, [_user_speaking_message("mach es")])
+ await session.update_session(turn_directive="Answer directly this turn.")
+ assert len(sent) == 2
+ assert "Answer directly this turn." in sent[1]["text"]
+
+
+async def test_directive_already_in_the_fixed_instructions_sends_nothing() -> None:
+ sent: list[dict[str, str]] = []
+ session, queue = _steering_session(
+ sent,
+ instructions=f"You are Jarvis.\n\n{_STANDING_DIRECTIVE}\n\n{_TURN_DIRECTIVE}",
+ )
+
+ await _drive(session, queue, [_user_speaking_message()])
+ await session.update_session(
+ turn_directive=_TURN_DIRECTIVE, standing_directive=_STANDING_DIRECTIVE
+ )
+
+ assert sent == []
+
+
+async def test_language_pin_travels_only_when_it_changes() -> None:
+ sent: list[dict[str, str]] = []
+ session, queue = _steering_session(sent, instructions="fixed", language="de")
+
+ await _drive(session, queue, [_user_speaking_message()])
+ await session.update_session(language="de")
+ assert sent == []
+
+ await _drive(session, queue, [_user_speaking_message("switch please")])
+ await session.update_session(language="en")
+ assert len(sent) == 1
+ assert "en" in sent[0]["text"]
+
+
+async def test_steering_waits_while_the_model_is_generating() -> None:
+ sent: list[dict[str, str]] = []
+ session, queue = _steering_session(sent, instructions="fixed")
+
+ # The model is mid-reply: a text input here would interrupt it.
+ await _drive(session, queue, [_model_speaking_message()])
+ await session.update_session(turn_directive=_TURN_DIRECTIVE)
+ assert sent == []
+
+ # The finished model turn alone is not the window either — between two
+ # turns a text input would open one of its own.
+ await _drive(session, queue, [_turn_complete_message()])
+ assert sent == []
+
+ # The next user utterance IS the window, and the delta is delivered
+ # BEFORE the transcript reaches the orchestrator.
+ events = await _drive(session, queue, [_user_speaking_message()])
+ assert [event.type for event in events] == ["input_transcript"]
+ assert len(sent) == 1
+ assert _TURN_DIRECTIVE in sent[0]["text"]
+
+
+async def test_newer_directive_supersedes_an_undelivered_one() -> None:
+ sent: list[dict[str, str]] = []
+ session, queue = _steering_session(sent, instructions="fixed")
+
+ await _drive(
+ session, queue, [_model_speaking_message(), _turn_complete_message()]
+ )
+ await session.update_session(turn_directive=_TURN_DIRECTIVE)
+ await session.update_session(turn_directive="Answer directly this turn.")
+ assert sent == []
+
+ await _drive(session, queue, [_user_speaking_message()])
+ assert len(sent) == 1
+ assert "Answer directly this turn." in sent[0]["text"]
+ assert _TURN_DIRECTIVE not in sent[0]["text"]
+
+
+async def test_a_failed_steering_send_keeps_the_delta_for_the_next_turn() -> None:
+ sent: list[dict[str, str]] = []
+ failures = {"count": 1}
+
+ async def send_realtime_input(**kwargs):
+ if failures["count"] > 0:
+ failures["count"] -= 1
+ raise RuntimeError("socket hiccup")
+ sent.append(kwargs)
+
+ queue: list[list[object]] = []
+
+ async def fake_receive():
+ if queue:
+ for message in queue.pop(0):
+ yield message
+
+ session = _GeminiLiveSession(
+ session=SimpleNamespace(
+ receive=fake_receive, send_realtime_input=send_realtime_input
+ ),
+ connection_cm=SimpleNamespace(),
+ client=SimpleNamespace(),
+ session_id="s-retry",
+ instructions="fixed",
+ )
+
+ await _drive(session, queue, [_user_speaking_message()])
+ await session.update_session(turn_directive=_TURN_DIRECTIVE)
+ assert sent == []
+
+ await _drive(session, queue, [_user_speaking_message("noch mal")])
+ assert len(sent) == 1
+ assert _TURN_DIRECTIVE in sent[0]["text"]
+
+
+async def test_barge_in_reopens_the_steering_window() -> None:
+ sent: list[dict[str, str]] = []
+ session, queue = _steering_session(sent, instructions="fixed")
+
+ await session.update_session(turn_directive=_TURN_DIRECTIVE)
+ assert sent == []
+
+ # Model output, then the user cutting in inside ONE server message.
+ await _drive(
+ session,
+ queue,
+ [
+ _fake_message(
+ server_content=SimpleNamespace(
+ output_transcription=SimpleNamespace(text="ich erklaere"),
+ input_transcription=SimpleNamespace(text="stop"),
+ interrupted=True,
+ turn_complete=False,
+ )
+ )
+ ],
+ )
+
+ assert len(sent) == 1
+ assert _TURN_DIRECTIVE in sent[0]["text"]
+
+
+async def test_each_steering_key_is_tracked_on_its_own() -> None:
+ sent: list[dict[str, str]] = []
+ session, queue = _steering_session(sent, instructions="fixed", language="de")
+
+ await _drive(session, queue, [_user_speaking_message()])
+ await session.update_session(
+ turn_directive=_TURN_DIRECTIVE, standing_directive=_STANDING_DIRECTIVE
+ )
+ assert len(sent) == 1
+ assert _TURN_DIRECTIVE in sent[0]["text"]
+ assert _STANDING_DIRECTIVE in sent[0]["text"]
+
+ # Only the turn directive moves: the standing one is already told, so it
+ # must NOT ride along again on a channel that re-bills its whole context.
+ await _drive(session, queue, [_user_speaking_message("und jetzt")])
+ await session.update_session(
+ turn_directive="Answer directly this turn.",
+ standing_directive=_STANDING_DIRECTIVE,
+ )
+ assert len(sent) == 2
+ assert "Answer directly this turn." in sent[1]["text"]
+ assert _STANDING_DIRECTIVE not in sent[1]["text"]
+
+
+def test_prompted_response_retry_is_advertised() -> None:
+ # request_response() is a no-op on this transport, so without the
+ # capability an answer blocked at the speech boundary is never replaced.
+ assert _GeminiLiveSession.supports_prompted_response_retry is True
+ # The capability is a PAIR: the orchestrator's retry path reads the flag
+ # and then calls send_text, and raises "advertises prompted retries
+ # without send_text" if the adapter has none. Advertising one without the
+ # other turns a recoverable blocked answer into a failed retry.
+ assert callable(_GeminiLiveSession.send_text)
+
+
class _FakeConnectCM:
def __init__(self) -> None:
self.exited = False