Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(voice): keep running actions when the user only says "mhm"

rubenluetke10-beepdc1657c6530b
diff --git a/jarvis/live/session.py b/jarvis/live/session.py
index 32c7d7fe9d..24471f72bd 100644
--- a/jarvis/live/session.py
+++ b/jarvis/live/session.py
@@ -7,6 +7,7 @@ import base64
import json
import logging
import random
+import re
import time
from typing import Any, Literal
from uuid import uuid4
@@ -29,6 +30,19 @@ REASONING_SNAPSHOT_INTERVAL_S = 0.3
#: Reasoning items whose summary may be streaming at once (a bound, not a goal).
_REASONING_ITEMS_MAX = 16
+#: Backchannel sounds that acknowledge without asking for anything new. "Ja"
+#: and "ok" are NOT here: they answer a pending approval.
+_BACKCHANNEL = frozenset(
+ {"m", "mm", "mhm", "hm", "hmm", "ah", "aha", "oh", "eh", "ehm", "uh", "um", "uhm"}
+ | {"äh", "ähm"} # i18n-allow: spoken backchannel vocabulary
+) # fmt: skip
+
+
+def _says_something(fragment: str) -> bool:
+ """True when a user transcript fragment holds a word, not just a backchannel."""
+ words = re.findall(r"\w+", fragment.casefold())
+ return any(word not in _BACKCHANNEL for word in words)
+
def _pipeline_input_muted() -> bool:
"""Jarvis's microphone mute. The speech pipeline is its only writer."""
@@ -718,7 +732,10 @@ class LiveVoiceSession:
self._last_end[role] = max(self._last_end[role], fragment.end_ms)
if role == "user" and current:
self._tools.user_text = self._captions[role]
- self._tools.revision += 1
+ # "Mhm" while Jarvis works is not a new request; counting it
+ # discarded running actions as superseded (live 2026-10-01).
+ if _says_something(delta):
+ self._tools.revision += 1
if not self._closing and not self._recovering:
self._resume_needs_input = False
self._tools.accept_new_input()
diff --git a/jarvis/live/state.py b/jarvis/live/state.py
index 5c22ba932e..78136f280f 100644
--- a/jarvis/live/state.py
+++ b/jarvis/live/state.py
@@ -144,6 +144,10 @@ class LiveLedger:
result = json.loads(raw)
if result.get("confirmation_required") or result.get("blocked"):
continue
+ # A refusal that ran nothing (schema error, unapproved yes, refused
+ # hang-up) leaves nothing half-done; it must not end the call.
+ if result.get("executed") is False:
+ continue
if result.get("status") in {
"superseded",
"voice_closed_before_execution",
diff --git a/jarvis/live/tools.py b/jarvis/live/tools.py
index 770bb37396..d1ad8d745c 100644
--- a/jarvis/live/tools.py
+++ b/jarvis/live/tools.py
@@ -233,9 +233,13 @@ class LiveTools:
operation_token = self.cancel_token
if operation_token.is_cancelled():
return {"success": False, "status": "cancelled"}
- if revision != self.revision:
+ if revision != self.revision and not self._reads_only(name, args):
+ # Only an action can be overtaken by what the user said meanwhile;
+ # a search or a read answers the same either way (11 reads were
+ # discarded live and re-bought as paid rounds).
return {
"success": False,
+ "executed": False,
"status": "superseded",
"error": "The request changed before execution. Re-evaluate the latest user input.",
}
@@ -261,6 +265,7 @@ class LiveTools:
# called end_call instead of confirm_action and the call dropped.
refusal: dict = {
"success": False,
+ "executed": False,
"error": "The user did not ask to hang up. Stay on the call.",
}
if self._pending:
@@ -294,6 +299,7 @@ class LiveTools:
# Say why, or the model asks the same question again and again.
return {
"success": False,
+ "executed": False,
"error": "This action has not been explicitly approved.",
"reason": "no pending approval"
if pending is None
@@ -328,6 +334,7 @@ class LiveTools:
if descriptor is None:
return {
"success": False,
+ "executed": False,
"error": "Tool is no longer available. Discover the current catalog.",
}
import jsonschema # type: ignore[import-untyped]
@@ -383,6 +390,15 @@ class LiveTools:
}
return self._result(result)
+ def _reads_only(self, name: str, args: dict) -> bool:
+ """True for a call that only reads: tool search or a safe-tier tool."""
+ if name == "discover_tools":
+ return True
+ if name == "call_tool":
+ name = str(args.get("name", ""))
+ canonical = self._names.get(name, name)
+ return any(d.name == canonical and d.risk_tier == "safe" for d in self.catalog())
+
def _user_approved(self) -> bool:
"""True when the latest user text is a short, unvetoed yes in any locale.
diff --git a/tests/contract/test_gpt_live.py b/tests/contract/test_gpt_live.py
index 0a4ff3f75f..4f6737bc4d 100644
--- a/tests/contract/test_gpt_live.py
+++ b/tests/contract/test_gpt_live.py
@@ -1125,3 +1125,38 @@ def test_steering_tools_are_declared_before_the_size_budget_runs_out(ledger):
runtime = LiveTools(Crowded(), ledger, "s", language="en", backend_model="")
declared = runtime.declarations()
assert "workspace-orchestrate" in {runtime._names.get(d["name"]) for d in declared}
+
+
+async def test_a_read_is_not_discarded_when_the_user_kept_talking(ledger):
+ class Reads(Gateway):
+ def catalog(self):
+ return (
+ SupervisorToolDescriptor("list-agents", "List agents", {"type": "object"}, "safe"),
+ *super().catalog(),
+ )
+
+ gateway = Reads()
+ runtime = LiveTools(gateway, ledger, "s", language="en", backend_model="")
+ runtime.revision = 2
+ read = {"name": "list-agents", "arguments_json": "{}"}
+ assert (await runtime.execute("r", "call_tool", read, 1))["success"]
+ action = {"name": "write-file", "arguments_json": '{"text":"x"}'}
+ assert (await runtime.execute("w", "call_tool", action, 1))["status"] == "superseded"
+
+
+ ("fragment", "counts"),
+ [("Mhm.", False), (" hm", False), ("Ähm", False), (" Ja", True), ("stop", True)], # i18n-allow
+)
+def test_only_words_beyond_a_backchannel_count_as_new_input(fragment, counts):
+ from jarvis.live.session import _says_something
+
+ assert _says_something(fragment) is counts
+
+
+def test_a_refusal_that_ran_nothing_does_not_block_reconnecting(ledger):
+ ledger.claim("s", "bye", "end_call", {}, 0)
+ ledger.finish("s", "bye", {"success": False, "executed": False, "error": "Stay on the call."})
+ resumable, _ = ledger.recovery_state("s")
+ assert resumable