Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""The tool loop over a faked transport, with scripted llama.cpp, vLLM and Ollama streams."""
from __future__ import annotations
import asyncio
import json
import threading
import pytest
from core.inference import passthrough_healing
from core.inference import studio_tool_loop as loop_mod
from core.inference.studio_tool_loop import (
ToolLoopPolicy,
ToolLoopRun,
stream_with_studio_tools,
)
from core.inference.tool_loop_controller import _reject_json_constant
def _shared_setup_1():
transport = FakeTransport(
[
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "c1",
"function": {"name": "python", "arguments": "{}"},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "ok"}), _sse(finish = "stop"), _DONE],
]
)
return transport
def _sse(
delta = None,
finish = None,
**extra,
) -> str:
choice: dict = {"index": 0, "delta": delta or {}}
if finish is not None:
choice["finish_reason"] = finish
payload: dict = {"choices": [choice]}
payload.update(extra)
return "data: " + json.dumps(payload)
_DONE = "data: [DONE]"
def _tool(name: str, description: str = "") -> dict:
return {
"type": "function",
"function": {
"name": name,
"description": description,
"parameters": {
"type": "object",
"properties": {"query": {"type": "string"}},
"required": ["query"],
},
},
}
WEB = _tool("web_search")
PY = _tool("python")
class FakeTransport:
"""Replays scripted turns and records what the loop asked for each time."""
def __init__(
self,
turns,
*,
heals = True,
):
self.turns = [list(turn) for turn in turns]
self.heals_text_tool_calls = heals
self.requests: list[dict] = []
def stream(self, *, messages, tools, tool_choice, cancel_event):
self.requests.append(
{
"messages": [dict(message) for message in messages],
"tools": tools,
"tool_choice": tool_choice,
}
)
lines = self.turns.pop(0) if self.turns else [_DONE]
async def _gen():
for line in lines:
yield line
return _gen()
@pytest.fixture
def executed(monkeypatch):
"""Record every execute_tool call and return a canned result."""
calls: list[dict] = []
def _execute(name, arguments, **kwargs):
calls.append({"name": name, "arguments": arguments, **kwargs})
return f"RESULT<{name}>"
monkeypatch.setattr(loop_mod, "execute_tool", _execute)
monkeypatch.setattr(loop_mod, "build_rag_autoinject", lambda *a, **k: None)
monkeypatch.setattr(loop_mod, "is_high_risk_tool_call", lambda name, args: name == "python")
return calls
def _run(
transport,
*,
tools = None,
tool_choice = None,
messages = None,
supports_vision = False,
mcp_image = None,
continue_final_message = False,
**policy_kwargs,
):
policy_fields = {
"tools": tools if tools is not None else [WEB],
"max_calls": 25,
"timeout": 300,
"permission_mode": "off",
"confirm_calls": False,
"bypass_permissions": False,
"rag_scope": None,
}
policy_fields.update(policy_kwargs)
cancel_event = threading.Event()
async def _collect():
out = []
agen = stream_with_studio_tools(
transport,
run = ToolLoopRun(
messages = messages or [{"role": "user", "content": "hi"}],
session_id = "s1",
thread_id = "t1",
tool_choice = tool_choice,
supports_vision = supports_vision,
continue_final_message = continue_final_message,
),
policy = ToolLoopPolicy(**policy_fields),
cancel_event = cancel_event,
mcp_image = mcp_image,
)
async for line in agen:
out.append(line)
return out
return asyncio.run(_collect())
def _events(lines, kind):
out = []
for line in lines:
if not line.startswith("data: "):
continue
raw = line[6:]
if raw == "[DONE]":
continue
payload = json.loads(raw)
if payload.get("type") == kind:
out.append(payload)
return out
def _visible_text(lines) -> str:
text = []
for line in lines:
if not line.startswith("data: "):
continue
raw = line[6:]
if raw == "[DONE]":
continue
payload = json.loads(raw)
if payload.get("type") in ("tool_start", "tool_end"):
continue
for choice in payload.get("choices") or []:
content = (choice.get("delta") or {}).get("content")
if isinstance(content, str):
text.append(content)
return "".join(text)
def test_structured_call_executes_and_continues(executed):
transport = FakeTransport(
[
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "call_a",
"function": {
"name": "web_search",
"arguments": '{"query":"unsloth"}',
},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "Here is what I found."}), _sse(finish = "stop"), _DONE],
]
)
lines = _run(transport)
assert [call["name"] for call in executed] == ["web_search"]
assert executed[0]["arguments"] == {"query": "unsloth"}
assert len(_events(lines, "tool_start")) == 1
assert _events(lines, "tool_end")[0]["result"] == "RESULT<web_search>"
assert "Here is what I found." in _visible_text(lines)
follow_up = transport.requests[1]["messages"]
assert [message["role"] for message in follow_up[-2:]] == ["assistant", "tool"]
assert follow_up[-1]["content"] == "RESULT<web_search>"
@pytest.mark.parametrize(
"policy", [None, {"allowedDomains": ["docs.python.org"], "blockedDomains": []}]
)
def test_web_search_domain_limits_reach_execute_tool(executed, policy):
call = {
"index": 0,
"id": "call_a",
"function": {"name": "web_search", "arguments": '{"query":"asyncio"}'},
}
transport = FakeTransport(
[
[_sse({"tool_calls": [call]}), _sse(finish = "tool_calls"), _DONE],
[_sse({"content": "Done."}), _sse(finish = "stop"), _DONE],
]
)
_run(transport, website_policy = policy)
[call] = executed
assert call.get("website_policy") == policy and ("website_policy" in call) == bool(policy)
def test_provider_compaction_replaces_history_before_tool_follow_up(executed):
transport = FakeTransport(
[
[
_sse(
{},
_toolEvent = {
"type": "compaction_block",
"encrypted_content": "opaque-current",
},
),
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "call_a",
"function": {
"name": "web_search",
"arguments": '{"query":"current"}',
},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "Done."}), _sse(finish = "stop"), _DONE],
]
)
# ExternalProviderClient synthesizes this control frame after sanitizing the provider stream.
transport.sanitizes_provider_frames = True
_run(
transport,
messages = [
{"role": "system", "content": "Keep this instruction."},
{"role": "user", "content": "old question"},
{"role": "assistant", "content": "old answer"},
{"role": "user", "content": "current question"},
],
)
follow_up = transport.requests[1]["messages"]
assert follow_up[0] == {"role": "system", "content": "Keep this instruction."}
assert follow_up[1] == {
"role": "assistant",
"content": "",
"extra_content": {"openai_responses_compaction": "opaque-current"},
}
serialized = json.dumps(follow_up)
assert "old question" not in serialized
assert "current question" not in serialized
assert [message["role"] for message in follow_up[-2:]] == ["assistant", "tool"]
def test_a_resumed_reply_does_not_merge_over_the_compaction_it_follows(executed):
summary = {"type": "compaction", "content": "Earlier conversation summary"}
native = {"anthropic": {"content": [summary, {"type": "text", "text": " rest"}]}}
call = {"index": 0, "id": "call_a", "function": {"name": "web_search", "arguments": "{}"}}
transport = FakeTransport(
[
[
_sse({}, _toolEvent = {"type": "compaction_block", "content": summary["content"]}),
_sse({"content": " rest", "extra_content": native, "tool_calls": [call]}),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "Done."}), _sse(finish = "stop"), _DONE],
]
)
transport.sanitizes_provider_frames = True
_run(
transport,
messages = [
{"role": "user", "content": "question"},
{"role": "assistant", "content": "A partial"},
],
continue_final_message = True,
)
follow_up = transport.requests[1]["messages"]
assert [message["role"] for message in follow_up] == ["assistant", "assistant", "tool"]
assert follow_up[0]["extra_content"]["anthropic"]["content"] == [summary]
def test_the_searched_branch_carries_replayed_results_as_the_model_saw_them(executed):
from core.inference.tool_loop_controller import TOOL_ERROR_NUDGE
branch = [
{"role": "user", "content": "run it"},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call_a",
"type": "function",
"function": {"name": "python", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_a", "name": "python", "content": "Error: boom"},
{"role": "user", "content": "what failed earlier"},
]
transport = FakeTransport(
[
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "call_c",
"function": {
"name": "search_conversation",
"arguments": '{"query":"failed"}',
},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "boom"}), _sse(finish = "stop"), _DONE],
]
)
_run(transport, tools = [_tool("search_conversation")], messages = branch)
searched = executed[0]["conversation_branch"]
assert searched[2]["content"] == "Error: boom" + TOOL_ERROR_NUDGE
assert searched[2]["content"] == transport.requests[0]["messages"][2]["content"]
def test_a_conversation_search_here_gets_the_active_branch(executed):
"""search_conversation needs the active branch, since stored rows are the whole DAG, Retry included."""
branch = [
{"role": "user", "content": "what was the code"},
{"role": "assistant", "content": "let me look"},
{"role": "user", "content": "please"},
]
transport = FakeTransport(
[
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "call_c",
"function": {
"name": "search_conversation",
"arguments": '{"query":"the code"}',
},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "It was 5150."}), _sse(finish = "stop"), _DONE],
]
)
_run(transport, tools = [_tool("search_conversation")], messages = branch)
assert [call["name"] for call in executed] == ["search_conversation"]
assert executed[0]["conversation_branch"] == branch
# Pass a budget, or a model-chosen top_k skips the tool's clamp and bloats the replayed prompt.
from core.rag import config as rag_config
assert (
executed[0]["conversation_budget_tokens"]
== rag_config.CHUNK_TOKENS * rag_config.CONVERSATION_ARCHIVE_TOP_K
)
def test_streamed_tool_name_fragments_are_not_concatenated(executed):
"""llama-server re-sends the whole name as it grows: web -> web_search."""
transport = FakeTransport(
[
[
_sse({"tool_calls": [{"index": 0, "id": "c1", "function": {"name": "web"}}]}),
_sse({"tool_calls": [{"index": 0, "function": {"name": "web_search"}}]}),
_sse({"tool_calls": [{"index": 0, "function": {"arguments": '{"query":'}}]}),
_sse({"tool_calls": [{"index": 0, "function": {"arguments": '"x"}'}}]}),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "done"}), _sse(finish = "stop"), _DONE],
]
)
_run(transport)
assert [call["name"] for call in executed] == ["web_search"]
assert executed[0]["arguments"] == {"query": "x"}
def test_text_form_tool_call_is_healed_and_executed(executed):
transport = FakeTransport(
[
[
_sse({"content": "Let me look. "}),
_sse(
{
"content": '<tool_call>{"name": "web_search", "arguments": {"query": "unsloth"}}</tool_call>'
}
),
_sse(finish = "stop"),
_DONE,
],
[_sse({"content": "Found it."}), _sse(finish = "stop"), _DONE],
]
)
lines = _run(transport)
assert [call["name"] for call in executed] == ["web_search"]
assert executed[0]["arguments"] == {"query": "unsloth"}
visible = _visible_text(lines)
assert "Let me look." in visible
assert "<tool_call>" not in visible
def test_partial_marker_split_across_deltas_is_not_broken(executed):
"""The signal itself straddles a chunk boundary."""
transport = FakeTransport(
[
[
_sse({"content": "<tool"}),
_sse({"content": '_call>{"name": "web_search", "arg'}),
_sse({"content": 'uments": {"query": "split"}}</tool_call>'}),
_sse(finish = "stop"),
_DONE,
],
[_sse({"content": "ok"}), _sse(finish = "stop"), _DONE],
]
)
lines = _run(transport)
assert [call["name"] for call in executed] == ["web_search"]
assert executed[0]["arguments"] == {"query": "split"}
assert "<tool" not in _visible_text(lines)
def test_unterminated_envelope_is_released_as_prose_and_terminates(executed):
"""An unclosed tool envelope must surface as prose, never be swallowed or run as a call."""
transport = FakeTransport(
[
[
_sse({"content": "thinking... "}),
_sse({"content": '<tool_call>{"name": "web_sea'}),
_sse(finish = "stop"),
_DONE,
]
]
)
lines = _run(transport)
assert executed == []
visible = _visible_text(lines)
assert "thinking..." in visible
assert "web_sea" in visible
def test_undeclared_text_call_is_not_promoted(executed):
"""A name outside the selected catalog is data, not a call."""
transport = FakeTransport(
[
[
_sse(
{
"content": '<tool_call>{"name": "terminal", "arguments": {"command": "id"}}</tool_call>'
}
),
_sse(finish = "stop"),
_DONE,
]
]
)
lines = _run(transport, tools = [WEB])
assert executed == []
assert "terminal" in _visible_text(lines)
def test_fenced_rehearsal_is_documentation_not_a_call(executed):
"""Markerless syntax quoted in markdown code must never execute (#6967, #8312)."""
transport = FakeTransport(
[
[
_sse({"content": 'Docs:\n```\npython[ARGS]{"code": "1"}\n```\n'}),
_sse(finish = "stop"),
_DONE,
]
]
)
lines = _run(transport, tools = [WEB, PY])
assert executed == []
assert "python[ARGS]" in _visible_text(lines)
def test_healing_is_off_for_a_transport_that_does_not_need_it(executed):
"""Codex emits structured calls; its text stream is relayed untouched."""
transport = FakeTransport(
[
[
_sse(
{
"content": '<tool_call>{"name": "web_search", "arguments": {"query": "x"}}</tool_call>'
}
),
_sse(finish = "stop"),
_DONE,
]
],
heals = False,
)
lines = _run(transport)
assert executed == []
assert "<tool_call>" in _visible_text(lines)
def test_structured_call_makes_the_healer_dormant(executed):
"""A provider that emits both must not have its text double-counted."""
transport = FakeTransport(
[
[
_sse({"content": "prefix <tool_c"}),
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "c1",
"function": {"name": "web_search", "arguments": "{}"},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "ok"}), _sse(finish = "stop"), _DONE],
]
)
lines = _run(transport)
assert [call["name"] for call in executed] == ["web_search"]
assert "prefix <tool_c" in _visible_text(lines)
def test_zero_budget_withdraws_the_catalog(executed):
transport = FakeTransport([[_sse({"content": "no tools for me"}), _sse(finish = "stop"), _DONE]])
_run(transport, max_calls = 0)
assert executed == []
assert transport.requests[0]["tools"] is None
assert transport.requests[0]["tool_choice"] == "none"
def test_denied_call_does_not_spend_an_iteration(executed, monkeypatch):
decisions = ["deny", "allow"]
monkeypatch.setattr(loop_mod, "begin_tool_decision", lambda session, approval: object())
monkeypatch.setattr(
loop_mod,
"wait_tool_decision",
lambda slot, approval, cancel_event = None: decisions.pop(0),
)
monkeypatch.setattr(loop_mod, "abort_tool_decision", lambda slot, approval: None)
monkeypatch.setattr(loop_mod, "new_approval_id", lambda: "ap1")
transport = FakeTransport(
[
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "c1",
"function": {"name": "python", "arguments": "{}"},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "c2",
"function": {"name": "python", "arguments": '{"query":"2"}'},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "ok"}), _sse(finish = "stop"), _DONE],
]
)
_run(
transport,
tools = [PY],
max_calls = 1,
permission_mode = "auto",
confirm_calls = True,
)
assert [call["name"] for call in executed] == ["python"]
assert executed[0]["arguments"] == {"query": "2"}
@pytest.mark.parametrize("verdict", ["allow", "deny"])
def test_an_allowed_call_tells_every_follower_before_tool_end(executed, monkeypatch, verdict):
"""Only the client that answered clears its own prompt; other tabs learn it from tool_approved."""
monkeypatch.setattr(loop_mod, "begin_tool_decision", lambda session, approval: object())
monkeypatch.setattr(
loop_mod, "wait_tool_decision", lambda slot, approval, cancel_event = None: verdict
)
monkeypatch.setattr(loop_mod, "abort_tool_decision", lambda slot, approval: None)
monkeypatch.setattr(loop_mod, "new_approval_id", lambda: "ap1")
call = {"index": 0, "id": "c1", "function": {"name": "python", "arguments": "{}"}}
transport = FakeTransport(
[
[_sse({"tool_calls": [call]}), _sse(finish = "tool_calls"), _DONE],
[_sse({"content": "ok"}), _sse(finish = "stop"), _DONE],
]
)
lines = _run(transport, tools = [PY], permission_mode = "ask", confirm_calls = True)
gated = [e for e in _events(lines, "tool_start") if e.get("awaiting_confirmation")]
approved = _events(lines, "tool_approved")
assert len(gated) == 1 and len(_events(lines, "tool_end")) == 1, lines
if verdict == "deny":
assert approved == []
return
assert approved == [
{
"type": "tool_approved",
"tool_name": "python",
"tool_call_id": gated[0]["tool_call_id"],
"approval_id": "ap1",
}
]
kinds = [
json.loads(line[6:]).get("type")
for line in lines
if line.startswith("data: ") and line[6:] != "[DONE]"
]
assert kinds.index("tool_start") < kinds.index("tool_approved") < kinds.index("tool_end")
# Same order as the GGUF loop: the approval before the running status.
assert "tool_status" not in kinds[kinds.index("tool_start") + 1 : kinds.index("tool_approved")]
def test_auto_mode_prompts_only_for_high_risk_calls(executed, monkeypatch):
slots: list = []
monkeypatch.setattr(
loop_mod,
"begin_tool_decision",
lambda session, approval: slots.append(approval) or object(),
)
monkeypatch.setattr(
loop_mod, "wait_tool_decision", lambda slot, approval, cancel_event = None: "allow"
)
monkeypatch.setattr(loop_mod, "abort_tool_decision", lambda slot, approval: None)
monkeypatch.setattr(loop_mod, "new_approval_id", lambda: "ap1")
transport = FakeTransport(
[
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "c1",
"function": {"name": "web_search", "arguments": "{}"},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "ok"}), _sse(finish = "stop"), _DONE],
]
)
lines = _run(
transport,
tools = [WEB, PY],
permission_mode = "auto",
confirm_calls = True,
)
assert slots == []
assert _events(lines, "tool_start")[0]["awaiting_confirmation"] is False
assert [call["name"] for call in executed] == ["web_search"]
def test_sharing_the_attached_image_asks_even_with_bypass(executed, monkeypatch):
from core.inference.mcp_image import McpImage
image = McpImage(mime = "image/png", data = b"IMG")
disclosure = {"server": "Trace", "tool": "lookup", "size_bytes": 3}
shared = {"disclosure": disclosure, "image": image.approved_for("r1")}
asked: list = []
monkeypatch.setattr(
loop_mod, "mcp_image_share", lambda name, args, image: shared if image else None
)
monkeypatch.setattr(
loop_mod, "begin_tool_decision", lambda session, approval: asked.append(approval) or 1
)
monkeypatch.setattr(
loop_mod, "wait_tool_decision", lambda slot, approval, cancel_event = None: "allow"
)
monkeypatch.setattr(loop_mod, "abort_tool_decision", lambda slot, approval: None)
call = {"index": 0, "id": "c1", "function": {"name": "web_search", "arguments": "{}"}}
turns = [
[_sse({"tool_calls": [call]}), _sse(finish = "tool_calls"), _DONE],
[_sse({"content": "ok"}), _sse(finish = "stop"), _DONE],
]
lines = _run(FakeTransport(turns), mcp_image = image, bypass_permissions = True)
start = _events(lines, "tool_start")[0]
assert start["awaiting_confirmation"] is True and start["image_disclosure"] == disclosure
assert len(asked) == 1 and executed[0]["mcp_image"] == image.approved_for("r1")
executed.clear()
lines = _run(FakeTransport(turns), bypass_permissions = True)
assert _events(lines, "tool_start")[0]["awaiting_confirmation"] is False
assert "mcp_image" not in executed[0]
def test_settled_image_call_is_replayed_and_deduped_with_sent_arguments(executed, monkeypatch):
from core.inference.mcp_image import ATTACHED_IMAGE, McpImage
image = McpImage(mime = "image/png", data = b"IMG")
shared = {
"disclosure": {"server": "Trace", "tool": "lookup", "size_bytes": 3},
"image": image.approved_for("r1"),
}
def settle(_name, arguments, _image):
arguments["image"] = ATTACHED_IMAGE
return shared
monkeypatch.setattr(loop_mod, "mcp_image_share", settle)
monkeypatch.setattr(loop_mod, "begin_tool_decision", lambda *_args: object())
monkeypatch.setattr(loop_mod, "wait_tool_decision", lambda *_args, **_kwargs: "allow")
monkeypatch.setattr(loop_mod, "abort_tool_decision", lambda *_args: None)
call = {"index": 0, "function": {"name": "python", "arguments": "{}"}}
transport = FakeTransport(
[
[_sse({"tool_calls": [{**call, "id": "c1"}]}), _sse(finish = "tool_calls"), _DONE],
[_sse({"tool_calls": [{**call, "id": "c2"}]}), _sse(finish = "tool_calls"), _DONE],
[_sse({"content": "done"}), _sse(finish = "stop"), _DONE],
],
heals = False,
)
_run(transport, tools = [PY], mcp_image = image, bypass_permissions = True)
assert len(executed) == 1
replayed = next(
message
for message in reversed(transport.requests[1]["messages"])
if message.get("role") == "assistant" and message.get("tool_calls")
)
assert json.loads(replayed["tool_calls"][0]["function"]["arguments"]) == {
"image": ATTACHED_IMAGE
}
def test_settled_image_call_keeps_signed_provider_replay_exact(executed, monkeypatch):
from core.inference.mcp_image import ATTACHED_IMAGE, McpImage
image = McpImage(mime = "image/png", data = b"IMG")
def settle(_name, arguments, _image):
arguments["image"] = ATTACHED_IMAGE
return {
"disclosure": {"server": "Trace", "tool": "lookup", "size_bytes": 3},
"image": image.approved_for("r1"),
}
monkeypatch.setattr(loop_mod, "mcp_image_share", settle)
monkeypatch.setattr(loop_mod, "begin_tool_decision", lambda *_args: object())
monkeypatch.setattr(loop_mod, "wait_tool_decision", lambda *_args, **_kwargs: "allow")
monkeypatch.setattr(loop_mod, "abort_tool_decision", lambda *_args: None)
call = {
"index": 0,
"id": "c1",
"function": {"name": "python", "arguments": "{ }"},
"extra_content": {"google": {"thought_signature": "SIG-A"}},
}
transport = FakeTransport(
[
[_sse({"tool_calls": [call]}), _sse(finish = "tool_calls"), _DONE],
[_sse({"content": "done"}), _sse(finish = "stop"), _DONE],
],
heals = False,
)
_run(transport, tools = [PY], mcp_image = image, bypass_permissions = True)
assert executed[0]["arguments"] == {"image": ATTACHED_IMAGE}
replayed = next(
message
for message in reversed(transport.requests[1]["messages"])
if message.get("role") == "assistant" and message.get("tool_calls")
)["tool_calls"][0]
assert replayed["function"]["arguments"] == "{ }"
assert replayed["extra_content"] == {"google": {"thought_signature": "SIG-A"}}
def test_the_model_is_told_about_the_attached_image(executed, monkeypatch):
from core.inference.mcp_image import McpImage
monkeypatch.setattr(
loop_mod, "mcp_image_targets", lambda names: [("mcp__srv1__lookup", "image")]
)
turns = [[_sse({"content": "ok"}), _sse(finish = "stop"), _DONE]]
messages = [{"role": "user", "content": "what anime is this?"}]
transport = FakeTransport(turns)
_run(transport, messages = messages, mcp_image = McpImage(mime = "image/png", data = b"IMG"))
sent = transport.requests[0]["messages"][-1]["content"]
assert sent.startswith("what anime is this?\n\n[The user attached an image")
assert 'mcp__srv1__lookup with {"image": "attached_image"}' in sent
assert messages == [{"role": "user", "content": "what anime is this?"}]
transport = FakeTransport(turns)
_run(transport, messages = messages)
assert transport.requests[0]["messages"][-1]["content"] == "what anime is this?"
def test_full_access_disables_the_sandbox_at_execution(executed):
transport = _shared_setup_1()
_run(transport, tools = [PY], bypass_permissions = True)
assert executed[0]["disable_sandbox"] is True
def test_sandbox_stays_on_by_default(executed):
transport = _shared_setup_1()
_run(transport, tools = [PY])
assert executed[0]["disable_sandbox"] is False
# without approval, the executor keeps the jail even for a host path.
assert "host_access_approved" not in executed[0]
@pytest.mark.parametrize("verdict", ["allow", "deny"])
def test_only_an_approved_call_is_marked_approved(executed, monkeypatch, verdict):
monkeypatch.setattr(loop_mod, "begin_tool_decision", lambda session, approval: object())
monkeypatch.setattr(
loop_mod, "wait_tool_decision", lambda slot, approval, cancel_event = None: verdict
)
monkeypatch.setattr(loop_mod, "abort_tool_decision", lambda slot, approval: None)
monkeypatch.setattr(loop_mod, "new_approval_id", lambda: "ap1")
_run(_shared_setup_1(), tools = [PY], permission_mode = "auto", confirm_calls = True)
if verdict == "allow":
assert executed[0]["host_access_approved"] is True
else:
assert executed == []
def test_forced_choice_is_cleared_after_the_first_execution(executed):
transport = FakeTransport(
[
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "c1",
"function": {"name": "web_search", "arguments": "{}"},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "answer"}), _sse(finish = "stop"), _DONE],
]
)
_run(transport, tool_choice = "required")
assert transport.requests[0]["tool_choice"] == "required"
assert transport.requests[1]["tool_choice"] == "auto"
def test_usage_is_summed_into_one_chunk(executed):
"""A multi-turn answer reports the shape a single-turn one does."""
transport = FakeTransport(
[
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "c1",
"function": {"name": "web_search", "arguments": "{}"},
}
]
}
),
_sse(finish = "tool_calls"),
"data: "
+ json.dumps(
{
"choices": [],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 5,
"total_tokens": 15,
"prompt_tokens_details": {"cached_tokens": 4},
},
}
),
_DONE,
],
[
_sse({"content": "done"}),
_sse(finish = "stop"),
"data: "
+ json.dumps(
{
"choices": [],
"usage": {
"prompt_tokens": 20,
"completion_tokens": 7,
"total_tokens": 27,
"prompt_tokens_details": {"cached_tokens": 6},
},
}
),
_DONE,
],
]
)
lines = _run(transport)
usages = []
for line in lines:
raw = line[6:] if line.startswith("data: ") else ""
if not raw or raw == "[DONE]":
continue
payload = json.loads(raw)
if "usage" in payload:
usages.append(payload["usage"])
assert len(usages) == 1
assert usages[0]["prompt_tokens"] == 30
assert usages[0]["completion_tokens"] == 12
assert usages[0]["total_tokens"] == 42
assert usages[0]["prompt_tokens_details"]["cached_tokens"] == 10
assert usages[0]["context_tokens"] == 27
def _usage_line(prompt, completion, **extra):
usage = {
"prompt_tokens": prompt,
"completion_tokens": completion,
"total_tokens": prompt + completion,
}
usage.update(extra)
return "data: " + json.dumps({"choices": [], "usage": usage})
def _usage_payloads(lines):
found = []
for line in lines:
raw = line[6:] if line.startswith("data: ") else ""
if not raw or raw == "[DONE]":
continue
payload = json.loads(raw)
if "usage" in payload:
found.append(payload["usage"])
return found
_WEB_SEARCH_CALL = _sse(
{
"tool_calls": [
{"index": 0, "id": "c1", "function": {"name": "web_search", "arguments": "{}"}}
]
}
)
def test_usage_snapshots_sum_once_per_turn_while_context_uses_the_last_turn(executed):
transport = FakeTransport(
[
[
_WEB_SEARCH_CALL,
_usage_line(
10,
2,
prompt_tokens_details = {"cached_tokens": 4},
completion_tokens_details = {"reasoning_tokens": 1},
cache_creation_input_tokens = 3,
cache_read_input_tokens = 4,
),
_sse(finish = "tool_calls"),
_usage_line(
10,
5,
prompt_tokens_details = {"cached_tokens": 6},
completion_tokens_details = {"reasoning_tokens": 2},
cache_creation_input_tokens = 5,
cache_read_input_tokens = 6,
),
_DONE,
],
[
_sse({"content": "done"}),
_sse(finish = "stop"),
_usage_line(
20,
7,
prompt_tokens_details = {"cached_tokens": 8},
completion_tokens_details = {"reasoning_tokens": 3},
cache_creation_input_tokens = 7,
cache_read_input_tokens = 8,
),
_DONE,
],
]
)
usages = _usage_payloads(_run(transport))
assert len(usages) == 1
assert usages[0]["prompt_tokens"] == 30
assert usages[0]["completion_tokens"] == 12
assert usages[0]["total_tokens"] == 42
assert usages[0]["prompt_tokens_details"] == {"cached_tokens": 14}
assert usages[0]["completion_tokens_details"] == {"reasoning_tokens": 5}
assert usages[0]["cache_creation_input_tokens"] == 12
assert usages[0]["cache_read_input_tokens"] == 14
assert usages[0]["context_tokens"] == 27
def test_a_turn_reporting_its_own_context_keeps_it_while_billing_sums(executed):
transport = FakeTransport(
[
[_WEB_SEARCH_CALL, _sse(finish = "tool_calls"), _usage_line(50, 3), _DONE],
[
_sse({"content": "done"}),
_sse(finish = "stop"),
_usage_line(203_000, 4_500, context_tokens = 24_000),
_DONE,
],
]
)
usages = _usage_payloads(_run(transport))
assert len(usages) == 1
assert usages[0]["prompt_tokens"] == 203_050
assert usages[0]["completion_tokens"] == 4_503
assert usages[0]["total_tokens"] == 207_553
assert usages[0]["context_tokens"] == 24_000
@pytest.mark.parametrize("last_usage", [None, {}, {"total_tokens": 0}])
def test_a_final_turn_without_context_usage_keeps_the_latest_reported_context(executed, last_usage):
final_turn = [_sse({"content": "done"}), _sse(finish = "stop")]
if last_usage is not None:
final_turn.append("data: " + json.dumps({"choices": [], "usage": last_usage}))
final_turn.append(_DONE)
transport = FakeTransport(
[
[_WEB_SEARCH_CALL, _sse(finish = "tool_calls"), _usage_line(100, 5), _DONE],
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "c2",
"function": {
"name": "web_search",
"arguments": '{"query":"follow-up"}',
},
}
]
}
),
_sse(finish = "tool_calls"),
_usage_line(200, 10),
_DONE,
],
final_turn,
]
)
usages = _usage_payloads(_run(transport))
assert len(transport.requests) == 3
assert len(usages) == 1
assert usages[0]["prompt_tokens"] == 300
assert usages[0]["completion_tokens"] == 15
assert usages[0]["total_tokens"] == 315
assert usages[0]["context_tokens"] == 210
def test_a_partial_report_after_a_full_one_keeps_the_fields_it_omits(executed):
transport = FakeTransport(
[
[
_sse({"content": "done"}),
"data: "
+ json.dumps(
{
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
"usage": {
"prompt_tokens": 100,
"completion_tokens": 20,
"total_tokens": 120,
},
}
),
"data: "
+ json.dumps(
{"choices": [], "usage": {"completion_tokens": 20, "total_tokens": None}}
),
_DONE,
],
]
)
usages = _usage_payloads(_run(transport))
assert len(usages) == 1
assert usages[0]["prompt_tokens"] == 100
assert usages[0]["completion_tokens"] == 20
assert usages[0]["total_tokens"] == 120
assert usages[0]["context_tokens"] == 120
@pytest.mark.parametrize("finish", ["stop", "length"])
def test_one_response_cumulative_usage_is_not_billed_per_chunk(executed, finish):
chunks = [
_sse(
{"content": "answer "},
usage = {
"prompt_tokens": 2_000,
"completion_tokens": output,
"total_tokens": 2_000 + output,
"completion_tokens_details": {"reasoning_tokens": min(output, 12_000)},
},
)
for output in range(1_000, 20_001, 1_000)
]
chunks.extend(
[
_sse(finish = finish),
_usage_line(2_000, 20_000, completion_tokens_details = {"reasoning_tokens": 12_000}),
_DONE,
]
)
transport = FakeTransport([chunks])
lines = _run(transport)
assert len(transport.requests) == 1
assert executed == []
assert _visible_text(lines) == "answer " * 20
assert _usage_payloads(lines) == [
{
"prompt_tokens": 2_000,
"completion_tokens": 20_000,
"total_tokens": 22_000,
"completion_tokens_details": {"reasoning_tokens": 12_000},
"context_tokens": 22_000,
}
]
def test_partial_usage_preserves_nested_counts_and_reconciles_the_total(executed):
reports = [
{
"prompt_tokens": 100,
"completion_tokens": 20,
"total_tokens": 120,
"prompt_tokens_details": {"cached_tokens": 60, "audio_tokens": 10},
"completion_tokens_details": {"reasoning_tokens": 12, "audio_tokens": 3},
"cache_creation": {"ephemeral_5m_input_tokens": 50, "ephemeral_1h_input_tokens": 20},
},
{
"prompt_tokens": None,
"completion_tokens": 25,
"total_tokens": None,
"prompt_tokens_details": {"cached_tokens": 70, "audio_tokens": None},
"completion_tokens_details": {"reasoning_tokens": None, "audio_tokens": 4},
"cache_creation": {"ephemeral_5m_input_tokens": None, "ephemeral_1h_input_tokens": 30},
},
None,
{},
{"prompt_tokens": None, "completion_tokens_details": None, "cache_creation": {}},
]
transport = FakeTransport(
[
[
_sse({"content": "done"}),
*("data: " + json.dumps({"choices": [], "usage": report}) for report in reports),
_sse(finish = "stop"),
_DONE,
]
]
)
assert _usage_payloads(_run(transport)) == [
{
"prompt_tokens": 100,
"completion_tokens": 25,
"total_tokens": 125,
"prompt_tokens_details": {"cached_tokens": 70, "audio_tokens": 10},
"completion_tokens_details": {"reasoning_tokens": 12, "audio_tokens": 4},
"cache_creation": {"ephemeral_5m_input_tokens": 50, "ephemeral_1h_input_tokens": 30},
"context_tokens": 125,
}
]
def test_final_usage_corrections_can_lower_or_zero_counts(executed):
transport = FakeTransport(
[
[
_sse({"content": "done"}),
_usage_line(
100,
20,
prompt_tokens_details = {"cached_tokens": 60},
completion_tokens_details = {"reasoning_tokens": 12},
),
_usage_line(
80,
0,
prompt_tokens_details = {"cached_tokens": 0},
completion_tokens_details = {"reasoning_tokens": 0},
),
_sse(finish = "stop"),
_DONE,
]
]
)
assert _usage_payloads(_run(transport)) == [
{
"prompt_tokens": 80,
"completion_tokens": 0,
"total_tokens": 80,
"prompt_tokens_details": {"cached_tokens": 0},
"completion_tokens_details": {"reasoning_tokens": 0},
"context_tokens": 80,
}
]
@pytest.mark.parametrize("ending", ["cancel", "error_event", "exception"])
def test_interrupted_usage_is_not_counted_repeatedly(executed, ending):
class InterruptedTransport(FakeTransport):
def stream(self, *, cancel_event, **kwargs):
async def generate():
yield _sse({"content": "partial"})
yield _usage_line(100, 2)
yield _usage_line(100, 5)
if ending == "cancel":
cancel_event.set()
elif ending == "error_event":
yield 'data: {"error":{"message":"provider stopped"}}'
else:
raise RuntimeError("provider stopped")
return generate()
transport = InterruptedTransport([])
if ending == "exception":
with pytest.raises(RuntimeError, match = "provider stopped"):
_run(transport)
else:
lines = _run(transport)
assert _usage_payloads(lines) == [
{
"prompt_tokens": 100,
"completion_tokens": 5,
"total_tokens": 105,
"context_tokens": 105,
}
]
if ending == "error_event":
assert any('"error"' in line for line in lines)
assert executed == []
@pytest.mark.parametrize(
"last_usage",
[{"total_tokens": 300}, {"completion_tokens": 50, "total_tokens": 300}],
ids = ["total_only", "total_above_its_parts"],
)
def test_a_turn_whose_total_exceeds_its_parts_prices_the_context_from_it(executed, last_usage):
transport = FakeTransport(
[
[_WEB_SEARCH_CALL, _sse(finish = "tool_calls"), _usage_line(100, 5), _DONE],
[
_sse({"content": "done"}),
_sse(finish = "stop"),
"data: " + json.dumps({"choices": [], "usage": last_usage}),
_DONE,
],
]
)
usages = _usage_payloads(_run(transport))
assert len(usages) == 1
assert usages[0]["total_tokens"] == 405
assert usages[0]["context_tokens"] == 300
def test_a_repeated_identical_call_does_not_re_execute(executed):
"""The controller's dedup stops a model spending the budget on one call."""
call = {
"index": 0,
"id": "c1",
"function": {"name": "web_search", "arguments": '{"query":"x"}'},
}
transport = FakeTransport(
[
[_sse({"tool_calls": [call]}), _sse(finish = "tool_calls"), _DONE],
[_sse({"tool_calls": [dict(call, id = "c2")]}), _sse(finish = "tool_calls"), _DONE],
[_sse({"content": "answer"}), _sse(finish = "stop"), _DONE],
]
)
_run(transport)
assert [c["name"] for c in executed] == ["web_search"]
def test_a_stalled_model_is_nudged_to_act(executed):
"""Small models often say what they will do instead of doing it."""
transport = FakeTransport(
[
[_sse({"content": "I'll search for that now."}), _sse(finish = "stop"), _DONE],
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "c1",
"function": {"name": "web_search", "arguments": "{}"},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "answer"}), _sse(finish = "stop"), _DONE],
]
)
_run(transport, nudge_tool_calls = True)
assert [c["name"] for c in executed] == ["web_search"]
second = transport.requests[1]["messages"]
assert second[-1]["role"] == "user"
def test_a_stalled_model_is_not_nudged_by_default(executed, monkeypatch):
"""The external loop must not invent a retry for an omitted opt-in flag."""
# Pin it: _NUDGE_DEFAULT is import-time, so otherwise this passes only where
# UNSLOTH_TOOL_CALL_NUDGE happens to be unset.
monkeypatch.setattr(passthrough_healing, "_NUDGE_DEFAULT", False)
transport = FakeTransport(
[
[_sse({"content": "I'll search for that now."}), _sse(finish = "stop"), _DONE],
[_sse({"content": "SHOULD NOT APPEAR"}), _sse(finish = "stop"), _DONE],
]
)
lines = _run(transport)
assert executed == []
assert len(transport.requests) == 1
assert "SHOULD NOT APPEAR" not in _visible_text(lines)
def test_an_explicit_false_beats_a_process_default_of_on(executed, monkeypatch):
"""What chat-adapter.ts sends externally, and it must beat a default of on."""
monkeypatch.setattr(passthrough_healing, "_NUDGE_DEFAULT", True)
transport = FakeTransport(
[
[_sse({"content": "I'll search for that now."}), _sse(finish = "stop"), _DONE],
[_sse({"content": "SHOULD NOT APPEAR"}), _sse(finish = "stop"), _DONE],
]
)
lines = _run(transport, nudge_tool_calls = False)
assert executed == []
assert len(transport.requests) == 1
assert "SHOULD NOT APPEAR" not in _visible_text(lines)
def test_an_omitted_flag_still_follows_a_process_default_of_on(executed, monkeypatch):
"""Omission still follows the process default; if not, reconsider the explicit false."""
monkeypatch.setattr(passthrough_healing, "_NUDGE_DEFAULT", True)
transport = FakeTransport(
[
[_sse({"content": "I'll search for that now."}), _sse(finish = "stop"), _DONE],
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "c1",
"function": {"name": "web_search", "arguments": "{}"},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "answer"}), _sse(finish = "stop"), _DONE],
]
)
_run(transport)
assert [c["name"] for c in executed] == ["web_search"]
assert len(transport.requests) == 3
def test_a_finished_answer_is_not_nudged(executed):
"""A real answer must never be re-prompted into calling a tool."""
answer = (
"The capital of France is Paris, which has been the seat of government "
"since the tenth century and remains the largest city in the country."
)
transport = FakeTransport([[_sse({"content": answer}), _sse(finish = "stop"), _DONE]])
_run(transport)
assert executed == []
assert len(transport.requests) == 1
def test_the_loop_terminates_against_an_endlessly_calling_model(executed):
"""A provider that always asks for a tool must still stop."""
class Endless:
heals_text_tool_calls = True
def __init__(self):
self.turns = 0
def stream(self, *, messages, tools, tool_choice, cancel_event):
self.turns += 1
n = self.turns
async def _gen():
yield _sse(
{
"tool_calls": [
{
"index": 0,
"id": f"c{n}",
"function": {
"name": "web_search",
"arguments": json.dumps({"query": f"q{n}"}),
},
}
]
}
)
yield _sse(finish = "tool_calls")
yield _DONE
return _gen()
transport = Endless()
_run(transport, max_calls = 3)
assert len(executed) == 3
assert transport.turns <= 5
def test_tool_stdout_streams_while_the_call_runs(executed, monkeypatch):
"""A long call must emit progress rather than going silent."""
def _execute(
name,
arguments,
output_callback = None,
**kwargs,
):
if output_callback:
output_callback("partial line 1\n")
output_callback("partial line 2\n")
return "final"
monkeypatch.setattr(loop_mod, "execute_tool", _execute)
transport = _shared_setup_1()
lines = _run(transport, tools = [PY])
progress = [line for line in lines if line.startswith("data: ") and "partial line" in line]
assert progress, "no live tool output reached the client"
def test_replayed_assistant_content_carries_no_markup(executed):
"""Markup must not go back to the provider: the call replays structurally."""
transport = FakeTransport(
[
[
_sse(
{
"content": 'Sure. <tool_call>{"name": "web_search", "arguments": {"query": "x"}}</tool_call>'
}
),
_sse(finish = "stop"),
_DONE,
],
[_sse({"content": "ok"}), _sse(finish = "stop"), _DONE],
]
)
_run(transport)
replayed = transport.requests[1]["messages"]
assistant = [m for m in replayed if m.get("role") == "assistant"][-1]
assert "<tool_call>" not in (assistant.get("content") or "")
assert assistant.get("tool_calls")
def test_conversation_roles_stay_alternating_for_a_strict_server(executed):
"""A no-op only turn must not leave two user turns in a row."""
transport = FakeTransport(
[
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "c1",
"function": {"name": "not_a_tool", "arguments": "{}"},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "ok"}), _sse(finish = "stop"), _DONE],
]
)
_run(transport)
roles = [m["role"] for m in transport.requests[1]["messages"]]
assert all(not (a == "user" and b == "user") for a, b in zip(roles, roles[1:])), roles
def test_gemini_thought_signature_is_replayed_on_the_assistant_turn(executed):
"""Gemini 3 rejects a replayed functionCall lacking its thoughtSignature; keep it in the accumulator."""
transport = FakeTransport(
[
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "call_a",
"function": {"name": "web_search", "arguments": ""},
"extra_content": {"google": {"thought_signature": "SIG-A"}},
}
]
}
),
_sse({"tool_calls": [{"index": 0, "function": {"arguments": '{"query":"u"}'}}]}),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "ok"}), _sse(finish = "stop"), _DONE],
],
heals = False,
)
_run(transport)
assistant = [m for m in transport.requests[1]["messages"] if m.get("role") == "assistant"][-1]
call = assistant["tool_calls"][0]
assert call["extra_content"] == {"google": {"thought_signature": "SIG-A"}}
assert call["function"]["name"] == "web_search"
assert json.loads(call["function"]["arguments"]) == {"query": "u"}
def test_gemini_part_replay_ledgers_accumulate_across_a_tool_turn(executed):
transport = FakeTransport(
[
[
_sse(
{
"reasoning_content": "signed thought",
"extra_content": {
"google": {
"thought": True,
"thought_signature": "SIG-T",
"thought_part": {
"text": "signed thought",
"thought_signature": "SIG-T",
},
}
},
}
),
_sse(
{
"reasoning_content": " unsigned thought",
"extra_content": {
"google": {"thought_part": {"text": " unsigned thought"}}
},
}
),
_sse(
{
"content": "answer ",
"extra_content": {"google": {"answer_part": {"text": "answer "}}},
}
),
_sse(
{
"content": "tail",
"extra_content": {"google": {"answer_part": {"text": "tail"}}},
}
),
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "call_a",
"function": {
"name": "web_search",
"arguments": '{"query":"u"}',
},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "ok"}), _sse(finish = "stop"), _DONE],
],
heals = False,
)
transport.preserves_reasoning = True
_run(transport)
assistant = [
message
for message in transport.requests[1]["messages"]
if message.get("role") == "assistant"
][-1]
assert assistant["extra_content"]["google"]["thought_parts"] == [
{"text": "signed thought", "thought_signature": "SIG-T"},
{"text": " unsigned thought"},
]
assert assistant["extra_content"]["google"]["answer_parts"] == [{"text": "answer tail"}]
assert "thought_signature" not in assistant["extra_content"]["google"]
def _call_delta(index, call_id, name, arguments):
return {
"index": index,
"id": call_id,
"function": {"name": name, "arguments": arguments},
}
def test_a_decoded_object_arguments_delta_reaches_the_tool(executed):
"""Object-valued argument deltas must reach the tool; a string-only accumulator ran it with {}."""
transport = FakeTransport(
[
[
_sse(
{"tool_calls": [_call_delta(0, "call_obj", "web_search", {"query": "value"})]}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "done"}), _sse(finish = "stop"), _DONE],
],
heals = False,
)
lines = _run(transport)
assert [call["arguments"] for call in executed] == [{"query": "value"}]
assert _events(lines, "tool_start")[0]["arguments"] == {"query": "value"}
def test_a_decoded_object_lands_where_its_string_spelling_would():
"""A decoded object must land where its string spelling would, or the dialects disagree."""
streams = [
(
"one payload",
[[_call_delta(0, "c1", "s", {"query": "v"})]],
[[_call_delta(0, "c1", "s", '{"query":"v"}')]],
),
(
"an empty opening, then the rest as fragments",
[
[_call_delta(0, "c1", "s", {})],
[{"index": 0, "function": {"arguments": '{"query":'}}],
[{"index": 0, "function": {"arguments": '"v"}'}}],
],
[
[_call_delta(0, "c1", "s", "{}")],
[{"index": 0, "function": {"arguments": '{"query":'}}],
[{"index": 0, "function": {"arguments": '"v"}'}}],
],
),
(
"two snapshots under one id",
[
[_call_delta(0, "c1", "s", {"query": "a"})],
[_call_delta(0, "c1", "s", {"query": "ab"})],
],
[
[_call_delta(0, "c1", "s", '{"query":"a"}')],
[_call_delta(0, "c1", "s", '{"query":"ab"}')],
],
),
]
def _shape(batches):
turn = loop_mod._Turn(round = 1)
for batch in batches:
turn.merge_structured(batch)
return [(c["function"]["name"], c["function"]["arguments"]) for c in turn.calls()]
for label, as_object, as_string in streams:
assert _shape(as_object) == _shape(as_string), label
def test_budget_exhausted_parallel_call_is_replayed_with_its_call(executed):
"""A refused parallel call's role=tool result needs its tool_calls entry, or providers reject it."""
transport = FakeTransport(
[
[
_sse(
{
"tool_calls": [
_call_delta(0, "call_a", "web_search", '{"query":"a"}'),
_call_delta(1, "call_b", "web_search", '{"query":"b"}'),
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "done"}), _sse(finish = "stop"), _DONE],
],
heals = False,
)
lines = _run(transport, max_calls = 1)
assert [call["name"] for call in executed] == ["web_search"]
replayed = transport.requests[1]["messages"]
called_ids = {
call["id"]
for message in replayed
if message.get("role") == "assistant"
for call in message.get("tool_calls") or []
}
result_ids = {message["tool_call_id"] for message in replayed if message.get("role") == "tool"}
assert result_ids == {"call_a", "call_b"}
assert not result_ids - called_ids
# Refused calls replay in OpenAI shape only; the parsed arguments dict stays in the loop.
exhausted = [
call
for message in replayed
if message.get("role") == "assistant"
for call in message.get("tool_calls") or []
if call["id"] == "call_b"
][0]
assert set(exhausted) == {"id", "type", "function"}
assert exhausted["function"]["name"] == "web_search"
assert len(_events(lines, "tool_end")) == 2
@pytest.mark.parametrize("heals", [False, True])
@pytest.mark.parametrize(
"fragment",
[
'{"query": "b',
'{"query":' + "1" * 4301 + "}",
"[" * 100_000,
'{"query":NaN}',
],
ids = ["cut-off", "digit-limit", "nesting-limit", "nan-constant"],
)
def test_budget_exhausted_call_replays_arguments_a_provider_will_parse(executed, heals, fragment):
"""llama-server parses every replayed tool_call's arguments, so an unparseable one 500s the next turn."""
transport = FakeTransport(
[
[
_sse(
{
"tool_calls": [
_call_delta(0, "call_a", "web_search", '{"query":"a"}'),
_call_delta(1, "call_b", "web_search", fragment),
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "done"}), _sse(finish = "stop"), _DONE],
],
heals = heals,
)
_run(transport, max_calls = 1)
assert [call["name"] for call in executed] == ["web_search"]
assert len(transport.requests) == 2
replayed = transport.requests[1]["messages"]
exhausted = [
call
for message in replayed
if message.get("role") == "assistant"
for call in message.get("tool_calls") or []
if call["id"] == "call_b"
][0]
json.loads(exhausted["function"]["arguments"], parse_constant = _reject_json_constant)
def test_unlimited_budget_runs_past_the_old_fixed_turn_cap(executed):
"""Unlimited means unlimited: the fruitless-turn guard already ends runs that execute nothing."""
turns = [
[
_sse(
{"tool_calls": [_call_delta(0, f"call_{n}", "web_search", f'{{"query":"q{n}"}}')]}
),
_sse(finish = "tool_calls"),
_DONE,
]
for n in range(40)
]
turns.append([_sse({"content": "done"}), _sse(finish = "stop"), _DONE])
transport = FakeTransport(turns, heals = False)
lines = _run(transport, max_calls = 9999)
assert len(executed) == 40
assert _visible_text(lines) == "done"
def test_a_skipped_duplicate_closes_the_card_the_provider_already_painted(executed):
"""A skipped duplicate must still emit its provider id, so the client can close its card."""
repeat = [
_sse({"tool_calls": [_call_delta(0, "call_a", "web_search", '{"query":"a"}')]}),
_sse(finish = "tool_calls"),
_DONE,
]
transport = FakeTransport(
[
list(repeat),
list(repeat),
[_sse({"content": "done"}), _sse(finish = "stop"), _DONE],
],
heals = False,
)
lines = _run(transport)
assert len(executed) == 1
ends = _events(lines, "tool_end")
assert len(ends) == 2
assert [end["tool_call_id"] for end in ends] == ["call_a", "call_a"]
assert ends[1]["result"].startswith("Unsloth did not run this call")
# Announce the card first: a second tool_end on a retired id is dropped by the client.
starts = _events(lines, "tool_start")
assert [start["tool_call_id"] for start in starts] == ["call_a", "call_a"]
def test_a_second_call_at_one_index_keeps_its_own_argument_fragments(executed):
"""Providers restart tool_calls index each round; route fragments by call id, not index."""
transport = FakeTransport(
[
[
_sse({"tool_calls": [_call_delta(0, "call_a", "web_search", '{"query":')]}),
_sse({"tool_calls": [{"index": 0, "function": {"arguments": '"first"}'}}]}),
_sse({"tool_calls": [_call_delta(0, "call_b", "web_search", '{"query":')]}),
_sse({"tool_calls": [{"index": 0, "function": {"arguments": '"second"}'}}]}),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "done"}), _sse(finish = "stop"), _DONE],
],
heals = False,
)
_run(transport)
assert [call["arguments"] for call in executed] == [{"query": "first"}, {"query": "second"}]
def test_a_fragment_naming_its_call_goes_back_to_that_call(executed):
"""A fragment naming its call must return to that call, not fork a third slot with truncated JSON."""
transport = FakeTransport(
[
[
_sse({"tool_calls": [_call_delta(0, "call_a", "web_search", '{"query":')]}),
_sse({"tool_calls": [_call_delta(0, "call_b", "web_search", '{"query":')]}),
_sse(
{
"tool_calls": [
{"index": 0, "id": "call_a", "function": {"arguments": '"first"}'}}
]
}
),
_sse(
{
"tool_calls": [
{"index": 0, "id": "call_b", "function": {"arguments": '"second"}'}}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "done"}), _sse(finish = "stop"), _DONE],
],
heals = False,
)
_run(transport)
assert [call["arguments"] for call in executed] == [{"query": "first"}, {"query": "second"}]
def _mcp_image_transport() -> FakeTransport:
return FakeTransport(
[
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "call_img",
"function": {
"name": "mcp__fs__read_media_file",
"arguments": '{"path":"cat.png"}',
},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "A tabby cat."}), _sse(finish = "stop"), _DONE],
]
)
@pytest.fixture
def mcp_image_result(monkeypatch):
import base64
import io
from PIL import Image
from core.inference import mcp_images
buffer = io.BytesIO()
Image.new("RGB", (6, 6), (10, 120, 200)).save(buffer, format = "PNG")
envelope = json.dumps(
[{"data": base64.b64encode(buffer.getvalue()).decode(), "mimeType": "image/png"}]
)
monkeypatch.setattr(
loop_mod,
"execute_tool",
lambda name, arguments, **kwargs: "[1 image returned]\n" + mcp_images.SENTINEL + envelope,
)
monkeypatch.setattr(loop_mod, "build_rag_autoinject", lambda *a, **k: None)
monkeypatch.setattr(loop_mod, "is_high_risk_tool_call", lambda name, args: False)
def test_mcp_images_reach_a_vision_provider_as_their_own_user_turn(mcp_image_result):
transport = _mcp_image_transport()
_run(
transport,
tools = [_tool("mcp__fs__read_media_file")],
supports_vision = True,
)
follow_up = transport.requests[1]["messages"]
assert [message["role"] for message in follow_up[-3:]] == ["assistant", "tool", "user"]
assert "__MCP_IMAGES__" not in follow_up[-2]["content"]
assert follow_up[-1]["content"][1]["image_url"]["url"].startswith("data:image/png;base64,")
def test_mcp_images_are_not_sent_to_a_text_only_provider(mcp_image_result):
transport = _mcp_image_transport()
_run(transport, tools = [_tool("mcp__fs__read_media_file")])
follow_up = transport.requests[1]["messages"]
assert follow_up[-1]["role"] == "tool"
assert "__MCP_IMAGES__" not in follow_up[-1]["content"]
@pytest.mark.parametrize("preserve", [True, False])
def test_reasoning_replay_during_tool_rounds_is_opt_in(executed, preserve):
transport = FakeTransport(
[
[
_sse({"reasoning_content": "First "}),
_sse({"reasoning_content": "thought."}),
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "call_1",
"type": "function",
"function": {
"name": "web_search",
"arguments": '{"query":"unsloth"}',
},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "Done."}), _sse(finish = "stop"), _DONE],
]
)
transport.preserves_reasoning = preserve
_run(transport)
assistant = transport.requests[1]["messages"][-2]
assert assistant.get("reasoning_content") == ("First thought." if preserve else None)
assert assistant["tool_calls"][0]["function"]["name"] == "web_search"
assert transport.requests[1]["messages"][-1]["role"] == "tool"
@pytest.mark.parametrize("field", ["reasoning", "reasoning_content"])
def test_preserved_thinking_before_a_tool_call_reaches_the_next_turn(executed, field):
# vLLM 0.11+ streams thinking as `reasoning`, llama-server as `reasoning_content`.
transport = FakeTransport(
[
[
_sse({field: "look it up"}),
_sse({"tool_calls": [_call_delta(0, "call_r", "web_search", '{"query":"u"}')]}),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "ok"}), _sse(finish = "stop"), _DONE],
],
heals = False,
)
transport.preserves_reasoning = True
_run(transport)
assistant = [
message
for message in transport.requests[1]["messages"]
if message.get("role") == "assistant"
][-1]
assert assistant["reasoning_content"] == "look it up"
def test_a_trailing_image_turn_takes_the_notice_as_a_text_part():
from core.inference.studio_tool_loop import _append_user_turn
image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}
conversation = [{"role": "user", "content": [{"type": "text", "text": "Screenshot:"}, image]}]
_append_user_turn(conversation, "Stop now.")
assert conversation == [
{
"role": "user",
"content": [
{"type": "text", "text": "Screenshot:"},
image,
{"type": "text", "text": "\n\nStop now."},
],
}
]
def test_deferred_mcp_schema_reaches_external_transports(monkeypatch, executed):
from core.inference import tool_loop_controller as controller_mod
search = _tool("mcp_tool_search")
ping = _tool("mcp__srv1__ping")
monkeypatch.setattr(controller_mod, "_mcp_tool_search_specs", lambda args: [ping])
monkeypatch.setattr(controller_mod, "_mcp_tool_spec", lambda name: ping)
transport = FakeTransport(
[
[
_sse(
{
"tool_calls": [
{
"index": 0,
"id": "search",
"function": {
"name": "mcp_tool_search",
"arguments": '{"query":"ping"}',
},
}
]
}
),
_sse(finish = "tool_calls"),
_DONE,
],
[_sse({"content": "ready"}), _sse(finish = "stop"), _DONE],
]
)
_run(transport, tools = [search])
assert transport.requests[1]["tools"] == [search, ping]