Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
"""Safety and provider-neutrality tests for realtime tool execution."""
from __future__ import annotations
from types import SimpleNamespace
import pytest
from dodaios.brain.tool_gateway import BrainSupervisorToolGateway
from dodaios.core import runtime_refs
from dodaios.realtime.tools import RealtimeToolBridge
from dodaios.safety.tool_executor import VOICE_CONFIRM_SENTINEL
class FakeTool:
def __init__(self, name: str = "open_app") -> None:
self.name = name
self.description = "Open a local application."
self.schema = {
"type": "object",
"properties": {"app_name": {"type": "string"}},
"required": ["app_name"],
}
self.risk_tier = "ask"
async def execute(self, *_args, **_kwargs):
raise AssertionError("Realtime must never execute a tool directly")
class FakeExecutor:
def __init__(self, *, confirmation_required: bool = False, output="opened") -> None:
self.confirmation_required = confirmation_required
self.output = output
self.execute_calls = []
self.confirmed_calls = []
self.cancelled = []
self.denied = []
async def execute(self, tool, arguments, **kwargs):
self.execute_calls.append((tool, arguments, kwargs))
if self.confirmation_required:
return SimpleNamespace(
success=False,
output={"tool_name": tool.name},
error=VOICE_CONFIRM_SENTINEL,
)
return SimpleNamespace(success=True, output=self.output, error=None)
async def execute_confirmed(self, trace_id, **kwargs):
self.confirmed_calls.append((trace_id, kwargs))
return SimpleNamespace(success=True, output="confirmed", error=None)
async def cancel_pending(self, trace_id):
self.cancelled.append(trace_id)
return True
async def publish_guard_denied(self, name, reason, *, trace_id):
self.denied.append((name, reason, trace_id))
@pytest.fixture
def wire_gateway():
previous = runtime_refs.get_supervisor_tool_gateway()
def _wire(brain):
gateway = BrainSupervisorToolGateway(brain)
runtime_refs.set_supervisor_tool_gateway(gateway)
return gateway
yield _wire
runtime_refs.set_supervisor_tool_gateway(previous)
def _bridge(*, name: str = "open_app", confirmation_required: bool = False):
tool = FakeTool(name)
executor = FakeExecutor(confirmation_required=confirmation_required)
bridge = RealtimeToolBridge(
tools={name: tool}, executor=executor, language="en"
)
return bridge, tool, executor
def test_declaration_uses_a_cross_provider_safe_wire_name():
bridge, _tool, _executor = _bridge(name="call-contact")
declaration = bridge.declarations[0]
assert declaration["name"].replace("_", "").isalnum()
assert len(declaration["name"]) <= 64
assert declaration["parameters"]["required"] == ["app_name"]
def test_supervisor_gateway_bridge_needs_no_classic_brain_argument(
wire_gateway,
) -> None:
tool = FakeTool("open_app")
manager = SimpleNamespace(
_tools={"open_app": tool},
_tool_executor=FakeExecutor(),
)
wire_gateway(manager)
bridge = RealtimeToolBridge.from_supervisor_gateway(language="en")
assert bridge is not None
assert [item["name"] for item in bridge.declarations] == ["open_app"]
@pytest.mark.asyncio
async def test_available_tool_runs_only_through_tool_executor():
bridge, tool, executor = _bridge()
await bridge.handle_user_transcript("Open Calculator")
name, result = await bridge.execute(
wire_name="open_app", arguments={"app_name": "Calculator"}
)
assert name == "open_app"
assert result == {"success": True, "output": "opened", "error": None}
assert executor.execute_calls[0][0] is tool
assert executor.execute_calls[0][1] == {"app_name": "Calculator"}
assert executor.execute_calls[0][2]["config_snapshot"]["voice_confirm"] is True
@pytest.mark.asyncio
async def test_unverified_outcome_carries_the_attempt_not_outcome_instruction():
"""A tool that acted blind (``verified: false``) must reach the model as
an attempt: live 2026-08-26 19:50 a media-key "play" came back ok and
the voice announced music that never started."""
tool = FakeTool("youtube_music")
executor = FakeExecutor(output={"ok": True, "action": "play", "verified": False})
bridge = RealtimeToolBridge(tools={"youtube_music": tool}, executor=executor, language="en")
_name, result = await bridge.execute(
wire_name="youtube_music", arguments={"app_name": "x"}
)
assert result["success"] is True and result["unverified"] is True
assert "not a confirmed outcome" in result["instruction"]
assert "never say it is done" in result["instruction"]
@pytest.mark.asyncio
async def test_verified_outcome_carries_no_instruction():
tool = FakeTool("youtube_music")
executor = FakeExecutor(output={"ok": True, "action": "play", "verified": True})
bridge = RealtimeToolBridge(tools={"youtube_music": tool}, executor=executor, language="en")
_name, result = await bridge.execute(
wire_name="youtube_music", arguments={"app_name": "x"}
)
assert result == {
"success": True,
"output": {"ok": True, "action": "play", "verified": True},
"error": None,
}
@pytest.mark.asyncio
async def test_vertex_toolset_prefix_resolves_to_the_declared_tool():
"""Vertex Live prefixes user-declared functions with the tool-set name.
Live 2026-08-19 16:07: hybrid vertex-live called ``default:run_shell``.
The bridge looked that name up verbatim, returned "Tool is not available
in this session.", and the model spoke that error. The prefix is a
transport artifact; the declared tool must still run.
"""
from dodaios.realtime.tools import canonical_tool_wire_name
assert canonical_tool_wire_name("default:run_shell") == "run_shell"
assert canonical_tool_wire_name("open_app") == "open_app"
assert canonical_tool_wire_name("default:jarvis_action") == "jarvis_action"
bridge, tool, executor = _bridge()
await bridge.handle_user_transcript("Open Calculator")
name, result = await bridge.execute(
wire_name="default:open_app", arguments={"app_name": "Calculator"}
)
assert name == "open_app"
assert result == {"success": True, "output": "opened", "error": None}
assert executor.execute_calls[0][0] is tool
@pytest.mark.asyncio
async def test_vertex_prefix_on_a_hashed_wire_name_still_runs():
bridge, tool, executor = _bridge(name="call-contact")
wire = bridge.declarations[0]["name"]
await bridge.handle_user_transcript("Open Calculator")
name, result = await bridge.execute(
wire_name=f"default:{wire}", arguments={"app_name": "Calculator"}
)
assert name == "call-contact"
assert result["success"] is True
assert executor.execute_calls[0][0] is tool
@pytest.mark.asyncio
async def test_underscore_alias_runs_a_hyphenated_tool():
"""Live models drop the hash and call ``run_skill`` for ``run-skill``.
Live 2026-08-20: that miss was "unknown realtime tool" and the user
heard that the skill could not be loaded.
"""
bridge, tool, executor = _bridge(name="run-skill")
await bridge.handle_user_transcript("Open Calculator")
name, result = await bridge.execute(
wire_name="run_skill", arguments={"app_name": "Calculator"}
)
assert name == "run-skill"
assert result["success"] is True
assert executor.execute_calls[0][0] is tool
name, result = await bridge.execute(
wire_name="default:run_skill",
arguments={"app_name": "Calculator"},
)
assert name == "run-skill"
assert result["success"] is True
@pytest.mark.asyncio
async def test_missing_required_argument_is_denied_before_executor():
bridge, _tool, executor = _bridge()
await bridge.handle_user_transcript("Open it")
_name, result = await bridge.execute(wire_name="open_app", arguments={})
assert result["success"] is False
assert "Missing required" in result["error"]
assert executor.execute_calls == []
assert executor.denied
@pytest.mark.asyncio
async def test_clear_yes_resumes_the_original_pending_action_once():
bridge, _tool, executor = _bridge(confirmation_required=True)
await bridge.handle_user_transcript("Open Calculator")
_name, first = await bridge.execute(
wire_name="open_app", arguments={"app_name": "Calculator"}
)
await bridge.handle_user_transcript("Yes")
_name, second = await bridge.execute(
wire_name="open_app", arguments={"app_name": "Something else"}
)
assert first["confirmation_required"] is True
assert second == {"success": True, "output": "confirmed", "error": None}
assert len(executor.execute_calls) == 1
assert len(executor.confirmed_calls) == 1
@pytest.mark.asyncio
async def test_clear_no_cancels_and_blocks_same_turn_retry():
bridge, _tool, executor = _bridge(confirmation_required=True)
await bridge.handle_user_transcript("Open Calculator")
await bridge.execute(
wire_name="open_app", arguments={"app_name": "Calculator"}
)
await bridge.handle_user_transcript("No")
_name, result = await bridge.execute(
wire_name="open_app", arguments={"app_name": "Calculator"}
)
assert len(executor.cancelled) == 1
assert result["success"] is False
assert "declined" in result["error"]
assert len(executor.execute_calls) == 1
def test_bridge_declarations_keep_the_full_json_schema() -> None:
"""Sanitizing is Gemini's wire-format concern: the provider-neutral
bridge declarations (and thus the OpenAI path) keep the full schema,
including additionalProperties."""
tool = FakeTool("strict_tool")
tool.schema = {
"type": "object",
"additionalProperties": False,
"properties": {"value": {"type": "string"}},
}
executor = FakeExecutor()
bridge = RealtimeToolBridge(
tools={"strict_tool": tool}, executor=executor, language="en"
)
declaration = bridge.declarations[0]
assert declaration["parameters"]["additionalProperties"] is False
@pytest.mark.asyncio
async def test_bridge_refreshes_replaced_brain_tools_and_denies_removed_tool(
wire_gateway,
):
executor = FakeExecutor()
old_tool = FakeTool("old_tool")
new_tool = FakeTool("new_tool")
brain = SimpleNamespace(
_tools={"old_tool": old_tool},
_tool_executor=executor,
)
wire_gateway(brain)
bridge = RealtimeToolBridge.from_brain(brain, language="en")
assert bridge is not None
brain._tools = {"new_tool": new_tool}
assert bridge.refresh_from_source() is True
assert [item["name"] for item in bridge.declarations] == ["new_tool"]
_name, removed = await bridge.execute(
wire_name="old_tool", arguments={"app_name": "Calculator"}
)
_name, added = await bridge.execute(
wire_name="new_tool", arguments={"app_name": "Calculator"}
)
assert removed["success"] is False
assert added["success"] is True
@pytest.mark.asyncio
async def test_bridge_refresh_retains_a_tool_awaiting_voice_confirmation(
wire_gateway,
):
executor = FakeExecutor(confirmation_required=True)
pending_tool = FakeTool("pending_tool")
brain = SimpleNamespace(
_tools={"pending_tool": pending_tool},
_tool_executor=executor,
)
wire_gateway(brain)
bridge = RealtimeToolBridge.from_brain(brain, language="en")
assert bridge is not None
await bridge.handle_user_transcript("Open Calculator")
await bridge.execute(
wire_name="pending_tool", arguments={"app_name": "Calculator"}
)
brain._tools = {"new_tool": FakeTool("new_tool")}
assert bridge.refresh_from_source() is True
assert {item["name"] for item in bridge.declarations} == {
"new_tool",
"pending_tool",
}
await bridge.handle_user_transcript("Yes")
_name, result = await bridge.execute(
wire_name="pending_tool", arguments={"app_name": "Calculator"}
)
assert result["success"] is True
assert bridge.refresh_from_source() is True
assert [item["name"] for item in bridge.declarations] == ["new_tool"]
class FakeSpawnTool:
"""Minimal spawn_worker stand-in — no required args, monitor tier."""
def __init__(self) -> None:
self.name = "spawn_worker"
self.description = "Delegates a heavy task to a background worker."
self.schema = {"type": "object", "properties": {}, "required": []}
self.risk_tier = "monitor"
async def execute(self, *_args, **_kwargs):
raise AssertionError("Realtime must never execute a tool directly")
def _spawn_bridge():
tool = FakeSpawnTool()
executor = FakeExecutor()
bridge = RealtimeToolBridge(
tools={"spawn_worker": tool}, executor=executor, language="en"
)
return bridge, executor
@pytest.fixture(autouse=True)
def _fresh_spawn_offer_window():
from dodaios.brain.spawn_gate import OFFER_WINDOW
OFFER_WINDOW.disarm()
yield
OFFER_WINDOW.disarm()
@pytest.mark.asyncio
async def test_conversational_turn_blocks_realtime_spawn():
"""Explicit-delegation gate (mandate 2026-07-18): the realtime model chose
spawn_worker on a plain conversational remark — blocked before the
executor, with the redirect message fed back to the model."""
bridge, executor = _spawn_bridge()
await bridge.handle_user_transcript(
"Ah, ich will gucken, wo ich als nächstes hinziehe." # i18n-allow: live utterance
)
name, result = await bridge.execute(wire_name="spawn_worker", arguments={})
assert name == "spawn_worker"
assert result["success"] is False
assert "did not explicitly ask" in result["error"]
assert executor.execute_calls == []
# A guard decision is NOT a broken tool. Without this flag the voice layer
# reads the block as the turn's failure and speaks a stock line, throwing
# away the real reason an earlier tool reported (live 2026-08-20 13:41:24 —
# see tests/unit/realtime/test_multi_step_turn_is_finished.py).
assert result["blocked"] is True
@pytest.mark.asyncio
async def test_explicit_agent_request_executes_realtime_spawn():
bridge, executor = _spawn_bridge()
await bridge.handle_user_transcript("Spawn an agent to research this.")
_name, result = await bridge.execute(wire_name="spawn_worker", arguments={})
assert result["success"] is True
assert len(executor.execute_calls) == 1
@pytest.mark.asyncio
async def test_confirmed_delegation_offer_unlocks_realtime_spawn():
"""Blocked turn → the model offers delegation → a short yes unlocks it."""
bridge, executor = _spawn_bridge()
await bridge.handle_user_transcript("Figure out where I should move next.")
_name, blocked = await bridge.execute(wire_name="spawn_worker", arguments={})
assert blocked["success"] is False
assert executor.execute_calls == []
await bridge.handle_user_transcript("Yes, go ahead.")
_name, result = await bridge.execute(wire_name="spawn_worker", arguments={})
assert result["success"] is True
assert len(executor.execute_calls) == 1
# ── the 2026-08-24 10:24 session: three asks, zero agents on the board ─────
@pytest.mark.asyncio
async def test_correcting_the_assistant_still_starts_the_worker():
"""Turn 4 of the live session, verbatim.
The model had just offered to do the job itself. "No, no, which a worker
should do it." is the user OVERRULING that offer and naming the vehicle,
and it must reach the executor. It used to score as a spawn DECLINE: the
leading "no" was read as a determiner negating the vehicle rather than as
a contradiction of the assistant's offer, so the board stayed empty at
"0 running · 0 in total" while the user asked three times.
"""
bridge, executor = _spawn_bridge()
await bridge.handle_user_transcript("No, no, which a worker should do it.")
_name, result = await bridge.execute(wire_name="spawn_worker", arguments={})
assert result["success"] is True
assert len(executor.execute_calls) == 1
@pytest.mark.asyncio
async def test_spoken_confirmation_naming_the_vehicle_starts_the_worker():
"""Turn 5 of the live session: a confirmation refused for being 8 words."""
bridge, executor = _spawn_bridge()
await bridge.handle_user_transcript("Give me an overview of my calendar.")
_name, blocked = await bridge.execute(wire_name="spawn_worker", arguments={})
assert blocked["success"] is False
await bridge.handle_user_transcript("Just follow the sub and then do it.")
_name, result = await bridge.execute(wire_name="spawn_worker", arguments={})
assert result["success"] is True
assert len(executor.execute_calls) == 1
@pytest.mark.asyncio
async def test_a_real_decline_still_starts_nothing():
"""The guard this all protects must survive: an explicit no spawns nothing."""
bridge, executor = _spawn_bridge()
await bridge.handle_user_transcript(
"No, do not spawn a subagent, talk to me directly."
)
_name, result = await bridge.execute(wire_name="spawn_worker", arguments={})
assert result["success"] is False
assert executor.execute_calls == []