Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(brain): run a tool call the model wrote inside prose

Haralddf2fd3da108b
diff --git a/jarvis/brain/tool_call_recovery.py b/jarvis/brain/tool_call_recovery.py
index 5743d8e277..5d626dd784 100644
--- a/jarvis/brain/tool_call_recovery.py
+++ b/jarvis/brain/tool_call_recovery.py
@@ -4,29 +4,135 @@ Some model transports occasionally serialize a function call into the text
stream instead of returning a structured tool-call delta. This module parses
only explicit tool-call envelopes; arbitrary JSON is never treated as an
action. Recovered calls still pass through ToolUseLoop and ToolExecutor.
+
+The envelope does not have to be the whole response. A model that writes
+"I'm opening Spotify. {"type":"tool_use",...}" used to execute NOTHING: the
+whole-text ``json.loads`` failed, the call was dropped, and the turn ended as
+plain text — the maintainer's "nothing happens" (audit GT-13). Envelopes are
+now located by balanced-brace scanning, so prose before, after, or on both
+sides of the call no longer defeats the recovery.
+
+Widening WHERE we look must not widen WHAT counts as a call. An answer that
+merely QUOTES a tool call ("For example, a provider might emit {...}") must
+never fire it: a missed recovery costs one answer, a wrongly executed call is
+a real side effect. Four gates keep the two apart — see
+:func:`extract_leaked_tool_calls`.
"""
from __future__ import annotations
import json
+import re
from hashlib import sha256
from typing import Any
_CALL_MARKERS = ("tool_use", "function_call", "tool_calls", '"function"')
_MISSING = object()
+# A fence marker (``` optionally followed by a language tag). Masked out before
+# the prose is judged, so a fenced envelope is not mistaken for a quoted one.
+_FENCE_RE = re.compile(r"```[A-Za-z0-9_+-]*")
+# Only a plain or explicitly JSON fence may carry an executable envelope. A
+# ```python / ```tool_code block is documentation about a call, not the call.
+_ALLOWED_FENCE_TAGS = frozenset({"", "json"})
+# Quote characters around an envelope mark it as CITED text, not an action.
+_QUOTE_CHARS = ("`", '"', "'", "„", "“", "”", "‚", "‘", "’", "«", "»")
+
+# Prose that DESCRIBES a call instead of performing one. Deliberately broad:
+# every marker here costs at most one missed recovery, while a miss in the
+# other direction executes something the user never asked for. Covers the
+# three product locales plus the vocabulary of an answer *about* tool calls.
+_EXAMPLE_MARKER_RE = re.compile(
+ r"(?:"
+ r"\be\.\s?g\.|\bi\.\s?e\.|\bz\.\s?b\.|\bp\.\s?ej\.|"
+ r"\b(?:"
+ # EN
+ r"example|examples|sample|samples|for instance|such as|pseudo|"
+ r"looks?\s+like|would\s+look|might|would|template|schema|format|syntax|"
+ r"illustrat\w*|json|tool[\s_-]?use|tool\s+calls?|function\s+calls?|"
+ # DE
+ r"beispiel\w*|etwa\s+so|so\s+aus|vorlage|schema|format|syntax|"
+ r"werkzeugaufruf\w*|"
+ # ES
+ r"ejemplo\w*|formato|esquema|plantilla|sintaxis"
+ r")\b"
+ r")",
+ re.IGNORECASE,
+)
+# A leak announces an action in a sentence or two ("Ich öffne Spotify.").
+# Paragraphs of prose around an envelope are an explanation that happens to
+# contain JSON, so the recovery stands down.
+_MAX_SURROUNDING_PROSE_CHARS = 240
+
+
+def _mask_fences(text: str) -> str:
+ """Blank out fence markers, preserving every character offset.
+
+ The masked copy is what the quote and prose gates read: a fenced envelope
+ must not look "quoted" just because a fence ends in a backtick.
+ """
+ return _FENCE_RE.sub(lambda m: " " * len(m.group(0)), text)
+
+
+def _fence_tags(text: str) -> list[str]:
+ return [match.group(0).lstrip("`").lower() for match in _FENCE_RE.finditer(text)]
-def _strip_json_fence(text: str) -> str | None:
- """Return an entire JSON response, optionally inside one strict fence."""
- stripped = text.strip()
- if not stripped.startswith("```"):
- return stripped
- lines = stripped.splitlines()
- if len(lines) < 3 or lines[-1].strip() != "```":
- return None
- if lines[0].strip().lower() not in {"```", "```json"}:
- return None
- return "\n".join(lines[1:-1]).strip()
+
+def _json_spans(text: str) -> list[tuple[int, int]]:
+ """Return ``(start, end)`` of every balanced top-level JSON object/array.
+
+ Brace counting, not a parser: string literals and backslash escapes are
+ tracked so a ``}`` inside a JSON string never closes a span. An opener
+ that is never closed (a truncated payload) yields no span at all.
+ """
+ spans: list[tuple[int, int]] = []
+ depth = 0
+ start = -1
+ in_string = False
+ escaped = False
+ for index, char in enumerate(text):
+ if depth == 0:
+ if char in "{[":
+ depth = 1
+ start = index
+ in_string = False
+ escaped = False
+ continue
+ if in_string:
+ if escaped:
+ escaped = False
+ elif char == "\\":
+ escaped = True
+ elif char == '"':
+ in_string = False
+ continue
+ if char == '"':
+ in_string = True
+ elif char in "{[":
+ depth += 1
+ elif char in "}]":
+ depth -= 1
+ if depth == 0:
+ spans.append((start, index + 1))
+ return spans
+
+
+def _is_quoted(masked: str, start: int, end: int) -> bool:
+ """True when the span sits inside quotation marks or an inline code span."""
+ before = masked[:start].rstrip()
+ after = masked[end:].lstrip()
+ return before.endswith(_QUOTE_CHARS) or after.startswith(_QUOTE_CHARS)
+
+
+def _surrounding_prose(masked: str, spans: list[tuple[int, int]]) -> str:
+ """Everything outside the executed envelopes, whitespace-collapsed."""
+ parts: list[str] = []
+ cursor = 0
+ for start, end in spans:
+ parts.append(masked[cursor:start])
+ cursor = end
+ parts.append(masked[cursor:])
+ return " ".join("".join(parts).split())
def _decode_arguments(value: Any) -> dict[str, Any] | None:
@@ -36,6 +142,9 @@ def _decode_arguments(value: Any) -> dict[str, Any] | None:
try:
decoded = json.loads(value)
except (json.JSONDecodeError, ValueError):
+ # Silence is the decision: arguments we cannot decode mean we do
+ # not know what the call would do, so the envelope is refused
+ # rather than guessed at. The caller sees "no call found".
return None
return dict(decoded) if isinstance(decoded, dict) else None
return None
@@ -105,20 +214,81 @@ def _calls_from_candidate(candidate: Any) -> list[dict[str, Any]]:
return calls
+def _call_identity(call: dict[str, Any]) -> str:
+ """Name + arguments — the same action, however the provider labelled it."""
+ return json.dumps(
+ [call["name"], call["input"]],
+ sort_keys=True,
+ separators=(",", ":"),
+ default=str,
+ )
+
+
def extract_leaked_tool_calls(text: str) -> list[dict[str, Any]]:
- """Parse explicit text-serialized tool calls into normal loop call records."""
+ """Parse explicit text-serialized tool calls into normal loop call records.
+
+ An envelope may sit anywhere in the response — alone, after prose, before
+ prose, between two sentences, or inside a ```json fence. Every envelope
+ found in the text is returned, in document order, with identical calls
+ collapsed to one.
+
+ Four gates separate an intended call from a quoted sample. All of them
+ fail closed, i.e. they return ``[]`` for the WHOLE text rather than
+ executing a subset:
+
+ 1. Shape — unchanged: only an explicit ``tool_use`` / ``function_call`` /
+ ``function`` envelope with a non-empty name and object arguments.
+ 2. Fences — an unbalanced ``` count means a truncated payload, and a
+ language tag other than ``json`` means documentation about a call.
+ 3. Quoting — an envelope wrapped in quotation marks or inline backticks is
+ being cited, not invoked.
+ 4. Prose — the text outside the envelopes must be short and free of
+ example vocabulary ("for example", "z. B.", "schema", "json", …).
+ """
if not text or not any(marker in text.lower() for marker in _CALL_MARKERS):
return []
- stripped = _strip_json_fence(text)
- if stripped is None:
+ masked = _mask_fences(text)
+ tags = _fence_tags(text)
+ if len(tags) % 2 or any(tag not in _ALLOWED_FENCE_TAGS for tag in tags):
return []
- try:
- candidate = json.loads(stripped)
- except (json.JSONDecodeError, ValueError):
- # Never execute quoted examples or JSON embedded in ordinary prose.
+
+ calls: list[dict[str, Any]] = []
+ consumed: list[tuple[int, int]] = []
+ for start, end in _json_spans(masked):
+ try:
+ candidate = json.loads(text[start:end])
+ except (json.JSONDecodeError, ValueError):
+ # Expected on ordinary prose: balanced braces are not proof of
+ # JSON (a smiley, a Python repr with single quotes). Nothing to
+ # report — this span is simply not an envelope.
+ continue
+ found = _calls_from_candidate(candidate)
+ if not found:
+ continue
+ if _is_quoted(masked, start, end):
+ return []
+ calls.extend(found)
+ consumed.append((start, end))
+
+ if not calls:
return []
- return _calls_from_candidate(candidate)
+
+ prose = _surrounding_prose(masked, consumed)
+ if len(prose) > _MAX_SURROUNDING_PROSE_CHARS:
+ return []
+ if _EXAMPLE_MARKER_RE.search(prose):
+ return []
+
+ unique: list[dict[str, Any]] = []
+ seen: set[str] = set()
+ for call in calls:
+ identity = _call_identity(call)
+ if identity in seen:
+ continue
+ seen.add(identity)
+ unique.append(call)
+ return unique
__all__ = ["extract_leaked_tool_calls"]
diff --git a/tests/unit/brain/test_leaked_spawn_recovery.py b/tests/unit/brain/test_leaked_spawn_recovery.py
index 0860fe0f55..5b3a001df8 100644
--- a/tests/unit/brain/test_leaked_spawn_recovery.py
+++ b/tests/unit/brain/test_leaked_spawn_recovery.py
@@ -46,10 +46,16 @@ def test_extract_detects_leaked_spawn_variants() -> None:
'```json\n[{"type":"tool_use","name":"spawn_worker",'
'"input":{"action":"y"}}]\n```'
) == {"action": "y"}
- # Quoted/example JSON inside prose is not an executable provider envelope.
+ # Embedded in prose (audit GT-13). The model announced the action AND
+ # emitted the call; dropping it is the maintainer's "nothing happens".
assert _extract_leaked_spawn_call(
'Klar! {"type":"tool_use","name":"spawn_worker",'
'"input":{"target":"z"}} erledige ich.'
+ ) == {"target": "z"}
+ # A QUOTED sample in an answer about tool calls is still never executed.
+ assert _extract_leaked_spawn_call(
+ 'Ein Block sieht zum Beispiel so aus: {"type":"tool_use",'
+ '"name":"spawn_worker","input":{"target":"z"}}'
) is None
diff --git a/tests/unit/brain/test_leaked_tool_call_extraction.py b/tests/unit/brain/test_leaked_tool_call_extraction.py
new file mode 100644
index 0000000000..e95d92c7ed
--- /dev/null
+++ b/tests/unit/brain/test_leaked_tool_call_extraction.py
@@ -0,0 +1,141 @@
+"""Envelope extraction for tool calls a provider emitted as response TEXT.
+
+Audit GT-13: the recovery only fired when the ENTIRE response was valid JSON.
+A model that wrote "Ich öffne Spotify. {tool_use…}" therefore executed
+nothing — the whole-text parse failed, the call was dropped, the turn ended as
+plain text, and the maintainer's complaint was the literal "nothing happens".
+Half a recovery is worse than none: it produces exactly the silent failure it
+was built to prevent.
+
+These tests pin both directions at once: an embedded envelope IS recovered,
+and a QUOTED one is still never executed. The second half matters more — a
+missed recovery costs one answer, a wrongly executed call is a real side
+effect.
+"""
+from __future__ import annotations
+
+import pytest
+
+from jarvis.brain.tool_call_recovery import extract_leaked_tool_calls
+
+_ENVELOPE = '{"type":"tool_use","name":"open_app","input":{"app":"spotify"}}'
+_SECOND_ENVELOPE = '{"type":"tool_use","name":"open_app","input":{"app":"notepad"}}'
+
+
+def _names_and_inputs(text: str) -> list[tuple[str, dict[str, object]]]:
+ return [(call["name"], call["input"]) for call in extract_leaked_tool_calls(text)]
+
+
+# ---------------------------------------------------------------------------
+# Recovered: the envelope is the action, whatever prose surrounds it.
+# ---------------------------------------------------------------------------
+
+
+ ("label", "text"),
+ [
+ ("prose before", f"Ich öffne Spotify. {_ENVELOPE}"),
+ ("prose after", f"{_ENVELOPE} Ich öffne Spotify."),
+ ("prose on both sides", f"Klar! {_ENVELOPE} — läuft."),
+ ("fenced after prose", f"Ich öffne Spotify.\n```json\n{_ENVELOPE}\n```"),
+ ("bare fence", f"```\n{_ENVELOPE}\n```"),
+ ("whole text", _ENVELOPE),
+ ("whole-text list", f"[{_ENVELOPE}]"),
+ ("newline separated", f"Ich öffne Spotify.\n\n{_ENVELOPE}\n"),
+ ],
+)
+def test_embedded_envelope_is_recovered(label: str, text: str) -> None:
+ assert _names_and_inputs(text) == [("open_app", {"app": "spotify"})], label
+
+
+def test_two_distinct_embedded_envelopes_both_run_in_order() -> None:
+ """Each envelope passed the same gates on its own, so each is honoured."""
+ text = f"Ich öffne Spotify. {_ENVELOPE} Und den Editor. {_SECOND_ENVELOPE}"
+
+ assert _names_and_inputs(text) == [
+ ("open_app", {"app": "spotify"}),
+ ("open_app", {"app": "notepad"}),
+ ]
+
+
+def test_repeated_identical_envelope_executes_once() -> None:
+ """Same name, same arguments — one action, however often it was written."""
+ assert _names_and_inputs(f"{_ENVELOPE} {_ENVELOPE}") == [
+ ("open_app", {"app": "spotify"})
+ ]
+
+
+def test_brace_inside_a_json_string_does_not_end_the_envelope() -> None:
+ text = (
+ 'Ich tippe das ein. {"type":"tool_use","name":"type_text",'
+ '"input":{"text":"} nicht das Ende \\" auch nicht"}}'
+ )
+
+ assert _names_and_inputs(text) == [
+ ("type_text", {"text": '} nicht das Ende " auch nicht'})
+ ]
+
+
+# ---------------------------------------------------------------------------
+# Never recovered: the envelope is being TALKED ABOUT, not invoked.
+# ---------------------------------------------------------------------------
+
+
+ ("label", "text"),
+ [
+ (
+ "english example marker",
+ "For example, a provider might emit "
+ '{"type":"tool_use","name":"gmail","input":{}} in prose.',
+ ),
+ ("german example marker", f"Ein Block sieht zum Beispiel so aus: {_ENVELOPE}"),
+ ("spanish example marker", f"Por ejemplo, el modelo emite {_ENVELOPE}"),
+ ("names the format", f"Das JSON dafür lautet: {_ENVELOPE}"),
+ ("names the mechanism", f"Ein tool_use-Block trägt zwei Felder: {_ENVELOPE}"),
+ ("inline code span", f"Der Block `{_ENVELOPE}` gehört dazu."),
+ ("quoted string", f'Er antwortete "{_ENVELOPE}" und ging.'),
+ ("non-json fence", f"```python\n{_ENVELOPE}\n```"),
+ ("truncated fence", f"```json\n{_ENVELOPE}"),
+ (
+ "paragraph of explanation",
+ "Der Aufbau ist einfach: das Feld type sagt, worum es geht, name "
+ "trägt den Werkzeugnamen und input die Argumente. Ein "
+ f"vollständiger Block sieht dann so aus: {_ENVELOPE} Mehr steckt "
+ "nicht dahinter, und mehr braucht es auch gar nicht.",
+ ),
+ ],
+)
+def test_quoted_sample_is_never_executed(label: str, text: str) -> None:
+ assert extract_leaked_tool_calls(text) == [], label
+
+
+def test_one_quoted_envelope_stands_down_the_whole_text() -> None:
+ """Mixed intent is not a licence to execute the half that looks clean."""
+ text = f"Ich öffne Spotify. {_ENVELOPE} Der Block `{_SECOND_ENVELOPE}` erklärt es."
+
+ assert extract_leaked_tool_calls(text) == []
+
+
+ "text",
+ [
+ "",
+ "Ich habe Spotify geöffnet.",
+ '{"app":"spotify","name":"open_app"}', # plain JSON, no envelope
+ '{"type":"tool_use","name":"","input":{}}', # no tool name
+ '{"type":"tool_use","name":"gmail","input":"{bad json"}',
+ '{"type":"tool_use","name":"gmail","input":["not","an","object"]}',
+ ],
+)
+def test_non_envelopes_stay_unexecuted(text: str) -> None:
+ assert extract_leaked_tool_calls(text) == []
+
+
+def test_recovered_call_carries_a_stable_content_derived_id() -> None:
+ """The loop de-duplicates on the id, so the same call must hash the same."""
+ embedded = extract_leaked_tool_calls(f"Ich öffne Spotify. {_ENVELOPE}")
+ whole_text = extract_leaked_tool_calls(_ENVELOPE)
+
+ assert embedded[0]["id"] == whole_text[0]["id"]
+ assert embedded[0]["id"].startswith("leaked_")
diff --git a/tests/unit/brain/test_tool_use_loop.py b/tests/unit/brain/test_tool_use_loop.py
index 90d5cc303b..bf15f4afed 100644
--- a/tests/unit/brain/test_tool_use_loop.py
+++ b/tests/unit/brain/test_tool_use_loop.py
@@ -240,6 +240,49 @@ async def test_text_leaked_read_call_rejoins_normal_tool_loop(
assert any(message.role == "tool" for message in brain.requests[1].messages)
+class _AnnouncedThenLeakedBrain:
+ """The model says what it is about to do, then leaks the call as text."""
+
+ def __init__(self) -> None:
+ self.requests: list[BrainRequest] = []
+
+ async def complete(self, req: BrainRequest) -> AsyncIterator[BrainDelta]:
+ self.requests.append(req)
+ if len(self.requests) == 1:
+ yield BrainDelta(
+ content=(
+ "Ich sehe nach. " # i18n-allow: simulated German model reply
+ '{"type":"tool_use","name":"gmail","input":{"item_id":"m1"}}'
+ )
+ )
+ yield BrainDelta(finish_reason="stop")
+ return
+ yield BrainDelta(content="The connected service returned a grounded result.")
+ yield BrainDelta(finish_reason="stop")
+
+
+async def test_announced_then_leaked_call_runs_and_never_speaks_the_json() -> None:
+ """Audit GT-13: prose in front of the envelope no longer drops the action."""
+ brain = _AnnouncedThenLeakedBrain()
+ executor = _ExecStructuredRead()
+ loop = ToolUseLoop(
+ brain,
+ {"gmail": _NamedReadTool("gmail")},
+ executor, # type: ignore[arg-type]
+ )
+
+ result = await loop.run([], user_utterance="Read the latest connected item.")
+
+ assert len(executor.calls) == 1
+ assert executor.calls[0][1] == {"item_id": "m1"}
+ assert result.text == "The connected service returned a grounded result."
+ # The announcement round is dropped whole — the raw envelope must never
+ # reach the answer, and neither half of that round is spoken twice.
+ assert "tool_use" not in result.text
+ assert "Ich sehe nach" not in result.text
+
+
class _StructuredThenLeakedGmailBrain:
"""Exact incident shape: list call, leaked detail call, final answer."""