Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
from __future__ import annotations
import json
import sys
import time
from pathlib import Path
import pytest
def _shared_setup_1(controller):
assert not controller.force_final_answer
assert [tool["function"]["name"] for tool in controller.active_tools()] == [
"web_search",
"python",
]
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
from core.inference.tool_loop_controller import (
UNPARSED_ARGUMENTS_KEY,
ToolLoopController,
append_deferred_nudges,
canonical_tool_call_key,
coerce_arguments_by_schema,
coerce_tool_arguments,
is_tool_error,
status_for_tool,
strip_result_for_model,
tool_call_limit_nudge,
tool_event_provenance,
)
from core.inference import tool_facts
from core.inference.tool_call_parser import TOOL_ERROR_NUDGE, parse_tool_calls_from_text
from core.inference.tools import (
ALL_TOOLS,
MAX_TOOL_TEXT_CHARS,
_mcp_specs_for_server,
_tool_text_notice_head,
cap_tool_text,
execute_tool,
)
_NOTICE = _tool_text_notice_head() + " the full output is not retained in model context.)"
def test_append_deferred_nudges_merges_deduped_into_one_message():
conversation = [{"role": "assistant", "tool_calls": [1]}, {"role": "tool", "content": "r"}]
nudges = [
{"role": "user", "content": "duplicate"},
{"role": "user", "content": "duplicate"},
{"role": "user", "content": "disabled foo"},
]
append_deferred_nudges(conversation, nudges)
assert conversation[2:] == [{"role": "user", "content": "duplicate\n\ndisabled foo"}]
before = list(conversation)
append_deferred_nudges(conversation, [])
assert conversation == before
def _tool(name: str) -> dict:
return {"type": "function", "function": {"name": name}}
def _call(
name: str,
args,
call_id: str = "call_0",
) -> dict:
return {
"id": call_id,
"type": "function",
"function": {
"name": name,
"arguments": json.dumps(args) if isinstance(args, dict) else args,
},
}
def test_canonical_tool_call_key_sorts_arguments():
a = canonical_tool_call_key("web_search", {"query": "gpu", "limit": 5})
b = canonical_tool_call_key("web_search", {"limit": 5, "query": "gpu"})
c = canonical_tool_call_key("python", {"limit": 5, "query": "gpu"})
assert a == b
assert a != c
assert a == 'web_search:{"limit":5,"query":"gpu"}'
def test_coerce_tool_arguments_parses_json_and_heals_raw_strings():
parsed = coerce_tool_arguments('{"query":"gpu prices"}', heal = True)
healed = coerce_tool_arguments("print(1)", heal = True, tool_name = "python")
raw = coerce_tool_arguments("not-json", heal = False, tool_name = "python")
assert parsed.arguments == {"query": "gpu prices"}
assert not parsed.healed
assert healed.arguments == {"code": "print(1)"}
assert healed.healed
assert raw.arguments == {"raw": "not-json"}
assert not raw.healed
def test_status_and_provenance_match_local_event_conventions():
assert status_for_tool("web_search", {"query": "gpus"}) == "Searching: gpus"
assert (
status_for_tool("web_search", {"url": "https://www.example.com/a"})
== "Reading: example.com"
)
assert status_for_tool("python", {"code": "print(1)\nprint(2)"}) == "Running Python: print(1)"
assert tool_event_provenance(healed = True, forced = False, provisional = None) == {
"source": "local",
"healed": True,
}
@pytest.mark.parametrize(
"url, expected",
[
# bare hosts are fetched, so the badge must name them
("google.com", "Reading: google.com"),
("www.google.com/x", "Reading: google.com"),
("//google.com", "Reading: google.com"),
("example.com:8443/path", "Reading: example.com"),
("github.com/unslothai/unsloth", "Reading: github.com"),
("/login", "Reading page..."),
("javascript:alert(1)", "Reading page..."),
# urlparse raises on these, outside the fetch's handler: degrade, not raise
("https://[::1", "Reading page..."),
("https://::1]", "Reading page..."),
("//exam/ple.com", "Reading page..."),
("//example.com@", "Reading page..."),
],
)
def test_status_names_the_host_for_schemeless_urls(url, expected):
assert status_for_tool("web_search", {"url": url}) == expected
def test_prepare_execute_builds_visible_events_and_model_tool_message():
controller = ToolLoopController(tools = [_tool("web_search")])
decision = controller.prepare_call(_call("web_search", {"query": "gpu prices"}))
assert decision.should_execute
assert decision.emit_visible_events
assert decision.status_text == "Searching: gpu prices"
assert decision.tool_start_payload()["arguments"] == {"query": "gpu prices"}
assert decision.tool_start_event()["type"] == "tool_start"
assert decision.as_assistant_tool_call()["function"]["arguments"] == '{"query":"gpu prices"}'
envelope = 'Search result\n__WEB_IMAGES__:[{"id": "a1b2c3d4e5f6", "title": "A chart", "domain": "example.com", "source": "https://example.com/a.png"}]'
completion = controller.record_result(decision, envelope)
assert completion.tool_end_payload()["result"] == envelope
assert completion.tool_end_event()["type"] == "tool_end"
assert completion.tool_message() == {
"role": "tool",
"name": "web_search",
"content": "Search result",
"tool_call_id": "call_0",
}
def test_successful_duplicate_is_internal_noop_and_keeps_remaining_tools():
controller = ToolLoopController(tools = [_tool("web_search"), _tool("python")])
first = controller.prepare_call(_call("web_search", {"query": "gpu prices"}, "call_a"))
controller.record_result(first, "ok")
duplicate = controller.prepare_call(_call("web_search", {"query": "gpu prices"}, "call_b"))
completion = controller.record_noop(duplicate)
assert duplicate.action == "duplicate"
assert not duplicate.should_execute
assert not duplicate.emit_visible_events
duplicate_nudge = completion.model_message()["content"]
assert duplicate_nudge.startswith(
"One earlier request to call tool 'web_search' in this batch was not executed"
)
assert "previous tool request" not in duplicate_nudge.lower()
assert "already completed successfully" in duplicate_nudge
assert "call a different tool only if" in duplicate_nudge
assert completion.model_message()["role"] == "user"
_shared_setup_1(controller)
def test_reprepare_uses_normalized_arguments_for_replay_and_dedup():
tool_name = "mcp__srv1__lookup"
controller = ToolLoopController(tools = [_tool(tool_name)])
call = _call(tool_name, {}, "call_a")
call["card_id"] = "painted_a"
first = controller.prepare_call(call)
first.arguments["image"] = "attached_image"
first = controller.reprepare_call(first)
assert first.card_id == "painted_a"
assert first.as_assistant_tool_call()["function"]["arguments"] == ('{"image":"attached_image"}')
assert first.key == canonical_tool_call_key(tool_name, {"image": "attached_image"})
controller.record_result(first, "ok")
repeated = controller.prepare_call(_call(tool_name, {}, "call_b"))
repeated.arguments["image"] = "attached_image"
repeated = controller.reprepare_call(repeated)
assert repeated.action == "duplicate"
assert repeated.key == first.key
def test_web_search_alias_args_share_the_duplicate_key():
controller = ToolLoopController(tools = [_tool("web_search")])
first = controller.prepare_call(_call("web_search", {"q": "unsloth"}, "call_a"))
assert first.should_execute
assert first.arguments == {"query": "unsloth"}
assert first.status_text == "Searching: unsloth"
controller.record_result(first, "ok")
duplicate = controller.prepare_call(_call("web_search", {"query": "unsloth"}, "call_b"))
assert duplicate.action == "duplicate"
assert duplicate.key == first.key
assert not duplicate.should_execute
def test_truncated_web_search_args_keep_the_unparsed_sentinel():
controller = ToolLoopController(tools = [_tool("web_search")])
decision = controller.prepare_call(_call("web_search", '{"query":"weather in S'))
assert decision.arguments == {UNPARSED_ARGUMENTS_KEY: '{"query":"weather in S'}
assert "cut off" in execute_tool("web_search", decision.arguments)
def test_web_search_url_mode_ignores_unused_args_for_duplicate_key():
controller = ToolLoopController(tools = [_tool("web_search")])
first = controller.prepare_call(
_call(
"web_search",
{
"url": "https://example.com/page",
"query": "unused",
"image_queries": ["unused"],
},
"call_a",
)
)
assert first.arguments == {"url": "https://example.com/page"}
controller.record_result(first, "page")
duplicate = controller.prepare_call(
_call(
"web_search",
{
"href": "https://example.com/page",
"query": "different",
"image_queries": ["different"],
},
"call_b",
)
)
assert duplicate.action == "duplicate"
assert duplicate.arguments == first.arguments
assert duplicate.key == first.key
def test_repeated_successful_duplicate_becomes_terminal_after_one_recovery_nudge():
controller = ToolLoopController(tools = [_tool("web_search"), _tool("python")])
first = controller.prepare_call(_call("web_search", {"query": "gpu prices"}, "call_a"))
controller.record_result(first, "ok")
duplicate_one = controller.prepare_call(_call("web_search", {"query": "gpu prices"}, "call_b"))
completion_one = controller.record_noop(duplicate_one)
assert duplicate_one.action == "duplicate"
assert "already completed successfully" in completion_one.model_message()["content"]
_shared_setup_1(controller)
duplicate_two = controller.prepare_call(_call("web_search", {"query": "gpu prices"}, "call_c"))
completion_two = controller.record_noop(duplicate_two)
assert duplicate_two.action == "duplicate"
assert "already completed successfully" in completion_two.model_message()["content"]
assert controller.force_final_answer
assert controller.active_tools() == []
def test_deduplication_can_be_disabled():
controller = ToolLoopController(
tools = [_tool("web_search"), _tool("python")],
deduplicate_tool_calls = False,
)
first = controller.prepare_call(_call("web_search", {"query": "gpu prices"}, "call_a"))
controller.record_result(first, "ok")
repeat = controller.prepare_call(_call("web_search", {"query": "gpu prices"}, "call_b"))
assert repeat.action == "execute"
assert repeat.should_execute
controller.record_result(repeat, "ok")
assert not controller.force_final_answer
_shared_setup_1(controller)
def test_command_can_run_again_after_a_file_edit():
controller = ToolLoopController(
tools = [_tool("terminal"), _tool("edit_file"), _tool("web_search")]
)
run = _call("terminal", {"command": "python calc.py"})
edit = _call(
"edit_file", {"path": "calc.py", "edits": [{"old_string": "a", "new_string": "b"}]}
)
search = _call("web_search", {"query": "gpu prices"})
controller.record_result(controller.prepare_call(search), "ok")
controller.record_result(controller.prepare_call(run), "3")
assert controller.prepare_call(run).action == "duplicate"
controller.record_noop(controller.prepare_call(run))
controller.record_result(controller.prepare_call(edit), "Edited calc.py")
rerun = controller.prepare_call(run)
assert rerun.action == "execute"
controller.record_result(rerun, "-1")
controller.record_noop(controller.prepare_call(run))
assert not controller.force_final_answer
# Identical replays are duplicates: a non-idempotent edit would otherwise land twice.
assert controller.prepare_call(edit).action == "duplicate"
assert controller.prepare_call(search).action == "duplicate"
def test_each_independent_edit_buys_back_its_own_verification():
"""A rerun is worth one piece of new work, not one per call, so B's check is not lost."""
controller = ToolLoopController(tools = [_tool("terminal"), _tool("edit_file")])
test = _call("terminal", {"command": "pytest -q"})
edit_a = _call("edit_file", {"path": "a.py", "edits": [{"old_string": "a", "new_string": "b"}]})
edit_b = _call("edit_file", {"path": "b.py", "edits": [{"old_string": "c", "new_string": "d"}]})
controller.record_result(controller.prepare_call(test), "1 failed")
controller.record_result(controller.prepare_call(edit_a), "Edited a.py")
after_a = controller.prepare_call(test)
assert after_a.action == "execute"
controller.record_result(after_a, "1 failed")
controller.record_result(controller.prepare_call(edit_b), "Edited b.py")
after_b = controller.prepare_call(test)
assert after_b.action == "execute"
def test_a_repeating_workspace_block_stops_replaying_itself():
"""read, edit, read, edit: the second edit is the block repeating, not new work."""
controller = ToolLoopController(tools = [_tool("terminal"), _tool("edit_file")])
read = _call("terminal", {"command": "cat notes.txt"})
edit = _call(
"edit_file", {"path": "notes.txt", "edits": [{"old_string": "a", "new_string": "b"}]}
)
controller.record_result(controller.prepare_call(read), "version one")
controller.record_result(controller.prepare_call(edit), "Edited notes.txt")
reread = controller.prepare_call(read)
assert reread.action == "execute"
controller.record_result(reread, "version two")
assert controller.prepare_call(edit).action == "duplicate"
def test_mcp_call_can_run_again_after_another_mcp_call_changed_state():
"""#10348: `list, create, list` on an MCP server must re-list, not answer from a stale duplicate."""
name = "mcp__docs__unified_document_manager"
controller = ToolLoopController(tools = [_tool(name)])
listing = _call(name, {"action": "list"})
create = _call(name, {"action": "create", "name": "a.md"})
controller.record_result(controller.prepare_call(listing), "[]")
assert controller.prepare_call(listing).action == "duplicate"
controller.record_result(controller.prepare_call(create), "created a.md")
relist = controller.prepare_call(listing)
assert relist.action == "execute"
controller.record_result(relist, '["a.md"]')
assert controller.prepare_call(listing).action == "duplicate"
assert controller.prepare_call(create).action == "duplicate"
def test_failed_call_does_not_block_retry():
controller = ToolLoopController(tools = [_tool("web_search")])
first = controller.prepare_call(_call("web_search", {"query": "gpu prices"}))
controller.record_result(first, "Error: temporary failure")
retry = controller.prepare_call(_call("web_search", {"query": "gpu prices"}))
assert retry.should_execute
assert retry.action == "execute"
def test_empty_enabled_tool_list_blocks_all_tool_calls():
controller = ToolLoopController(tools = [])
decision = controller.prepare_call(_call("web_search", {"query": "gpu prices"}))
completion = controller.record_noop(decision)
assert decision.action == "disabled"
assert not decision.emit_visible_events
assert completion.model_message()["role"] == "user"
disabled_nudge = completion.model_message()["content"]
assert disabled_nudge.startswith(
"One earlier request to call tool 'web_search' in this batch was not executed"
)
assert "previous tool request" not in disabled_nudge.lower()
assert "not enabled" in disabled_nudge
assert controller.force_final_answer
assert controller.active_tools() == []
def test_disabled_tool_is_internal_noop_not_visible_tool_error():
controller = ToolLoopController(tools = [_tool("web_search")])
decision = controller.prepare_call(_call("python", {"code": "print(1)"}))
completion = controller.record_noop(decision)
assert decision.action == "disabled"
assert not decision.emit_visible_events
assert completion.model_message()["role"] == "user"
assert "not enabled" in completion.model_message()["content"]
assert controller.force_final_answer
assert controller.active_tools() == []
def test_forced_mismatch_keeps_the_required_tool_active():
controller = ToolLoopController(tools = [_tool("web_search"), _tool("python")])
decision = controller.prepare_call(
_call("python", {"code": "print(1)"}),
allowed_tool_names = {"web_search"},
)
completion = controller.record_noop(decision)
assert decision.action == "forced_mismatch"
assert "required tool choice" in completion.model_message()["content"]
_shared_setup_1(controller)
def test_render_html_success_filters_active_tools_and_repeat_is_internal():
controller = ToolLoopController(tools = [_tool("render_html"), _tool("web_search")])
assert [t["function"]["name"] for t in controller.active_tools()] == [
"render_html",
"web_search",
]
first = controller.prepare_call(_call("render_html", {"code": "<html></html>"}, "call_html_1"))
controller.record_result(first, "Rendered HTML canvas: Demo")
assert [t["function"]["name"] for t in controller.active_tools()] == ["web_search"]
repeat = controller.prepare_call(_call("render_html", {"code": "<html></html>"}, "call_html_2"))
completion = controller.record_noop(repeat)
assert repeat.action == "render_html_repeat"
assert not repeat.emit_visible_events
assert completion.model_message()["role"] == "user"
assert "Do not call render_html again" in completion.model_message()["content"]
assert controller.force_final_answer
assert controller.active_tools() == []
def test_strip_result_for_model_removes_frontend_image_sentinel():
assert strip_result_for_model('text\n__IMAGES__:["a.png"]') == "text"
assert strip_result_for_model("plain text") == "plain text"
def test_a_result_that_only_quotes_the_image_or_source_marker_is_kept_whole():
"""Only a structurally valid trailing envelope is stripped, the way
`_strip_files_sentinel` and `_strip_mcp_image_suffix` beside it already are."""
quoted = 'tools.py:16134: out += f"\\n__IMAGES__:{_json.dumps(images)}"\nsecond hit\nthird hit'
for name in (None, "terminal", "mcp__fs__read"):
assert strip_result_for_model(quoted, name) == quoted
# What `_defuse_sentinels` leaves behind when a program prints the marker itself.
defused = "line one\n __IMAGES__:printed by the program\nline three"
assert strip_result_for_model(defused, "python") == defused
sources = 'line one\nRAG_SOURCES_SENTINEL = "\\n__RAG_SOURCES__:"\nline three'
assert strip_result_for_model(sources, "search_knowledge_base") == sources
assert (
strip_result_for_model('text\n__IMAGES__:{"paths":[]}') == 'text\n__IMAGES__:{"paths":[]}'
)
assert strip_result_for_model('output\n__IMAGES__:["a.png"]', "python") == "output"
assert (
strip_result_for_model(
'answer\n__RAG_SOURCES__:[{"filename": "a.pdf"}]', "search_knowledge_base"
)
== "answer"
)
def test_two_plots_in_one_code_execution_turn_replay_no_base64():
"""A Gemini `code_execution` turn that draws two figures stacks one envelope per
`inlineData` part (external_provider re-appends to the result it just emitted), so
peeling once would hand the model the earlier plot's whole data URI."""
first = "data:image/png;base64," + "A" * 64
second = "data:image/png;base64," + "B" * 64
stacked = (
"Figures saved."
+ f"\n__IMAGES__:{json.dumps([first])}"
+ f"\n__IMAGES__:{json.dumps([second])}"
)
assert strip_result_for_model(stacked, "code_execution") == "Figures saved."
def test_a_flood_of_stacked_image_markers_stays_linear():
"""Stacked image markers must cost linear time; repartitioning the string per marker is quadratic."""
def cost(markers: int) -> float:
flood = '\n__IMAGES__:["x"]' * markers
best = float("inf")
for _ in range(5):
started = time.process_time()
assert strip_result_for_model(flood, "code_execution") == ""
best = min(best, time.process_time() - started)
return max(best, 1e-4)
small = cost(20_000)
large = cost(80_000)
assert large / small < 10.0, f"4x the markers cost {large / small:.1f}x the work"
def test_a_large_source_map_is_still_taken_off_the_result():
"""A big source map must still be stripped, or _fit_result_to_room cuts it into malformed JSON."""
import json as _json
chunk = "retrieved text. " * 400
sources = [
{
"citationId": i,
"chunkId": f"c{i}",
"filename": "handbook.pdf",
"page": i,
"text": chunk,
"score": 0.5,
}
for i in range(200)
]
result = "answer\n__RAG_SOURCES__:" + _json.dumps(sources, ensure_ascii = False)
assert len(result) > 1 << 20
assert strip_result_for_model(result, "search_knowledge_base") == "answer"
def test_a_large_plot_is_still_taken_off_the_result():
"""No length bound on images on purpose: keeping a large data URI would leak base64 to the model."""
big = 'output\n__IMAGES__:["data:image/png;base64,' + "A" * (12 << 20) + '"]'
assert len(big) > 8 << 20
assert strip_result_for_model(big, "code_execution") == "output"
def test_only_the_tools_that_emit_an_envelope_have_one_taken_off():
"""Only envelope-emitting tools lose it; a document an MCP tool read or a fetched page is content."""
manifest = 'icons/\n__IMAGES__:["icon.png"]'
citation = 'notes\n__RAG_SOURCES__:[{"filename": "a.pdf"}]'
for reader in ("mcp__fs__read_file", "web_search", "fetch_url"):
assert strip_result_for_model(manifest, reader) == manifest
assert strip_result_for_model(citation, reader) == citation
for emitter in ("python", "terminal", "code_execution"):
assert strip_result_for_model(manifest, emitter) == "icons/"
for emitter in ("search_knowledge_base", "search_conversation"):
assert strip_result_for_model(citation, emitter) == "notes"
assert strip_result_for_model(manifest) == "icons/"
assert strip_result_for_model(citation) == "notes"
def test_the_card_text_keeps_digits_the_browser_would_round():
"""`JSON.parse` reads 9007199254740993 back as ...992, so a card that re-encodes the
parsed arguments in the browser would show a record the tool is not being run with."""
exact = 9007199254740993
controller = ToolLoopController(tools = [_tool("del_rec")])
decision = controller.prepare_call(_call("del_rec", {"id": exact}))
payload = decision.tool_start_payload()
assert payload["arguments"]["id"] == exact
assert payload["arguments_text"] == '{"id":9007199254740993}'
assert payload["arguments_text"] == decision.as_assistant_tool_call()["function"]["arguments"]
def test_an_unreadable_fragment_is_carried_as_the_text_the_card_shows():
"""The replay substitutes a summary for the fragment, so the two texts are meant to differ."""
truncated = '{"path": "a.py", "edits"'
controller = ToolLoopController(tools = [_tool("edit_file")])
decision = controller.prepare_call(_call("edit_file", truncated))
payload = decision.tool_start_payload()
assert payload["arguments"] == {"raw": truncated}
assert json.loads(payload["arguments_text"]) == {"raw": truncated}
assert payload["arguments_text"] != decision.as_assistant_tool_call()["function"]["arguments"]
_MCP_SERVER = {"id": "notes", "display_name": "Notes"}
_MCP_TOOL = {
"name": "search",
"inputSchema": {
"type": "object",
"properties": {
"query": {"type": "string"},
"limit": {"type": "integer"},
"fuzzy": {"type": "boolean"},
"tags": {"type": "array", "items": {"type": "string"}},
"depth": {"type": ["integer", "null"]},
},
},
}
_MCP_PROPS = _MCP_TOOL["inputSchema"]["properties"]
_UNCHANGED = object()
def _mcp_tool_schemas():
return _mcp_specs_for_server(_MCP_SERVER, [_MCP_TOOL])
def _coerce_one(key, value, props):
return coerce_arguments_by_schema({key: value}, props)[key]
@pytest.mark.parametrize(
"key, text, expected",
[
("fuzzy", " False ", False),
("limit", "25", 25),
("limit", "9007199254740993", 9007199254740993), # float() would round this
("tags", "('a', 'b')", ["a", "b"]),
("depth", "null", None),
("fuzzy", "null", _UNCHANGED), # null is not a boolean; None would mean False
],
)
def test_a_value_reads_as_the_type_its_schema_declares(key, text, expected):
got = _coerce_one(key, text, _MCP_PROPS)
want = text if expected is _UNCHANGED else expected
# Types too: `False == 0` and `25 == 25.0`, so equality alone would miss a swap.
assert got == want and type(got) is type(want)
seam = coerce_tool_arguments(
json.dumps({key: text}),
heal = False,
tool_name = "mcp__notes__search",
tool_schemas = _mcp_tool_schemas(),
)
assert seam.arguments[key] == want and seam.healed is False
@pytest.mark.parametrize(
"spec",
[
{"type": "boolean", "$ref": "#/$defs/Flag"},
# Collapsing a union discards the rest, so it is read only where it is all there is.
{"anyOf": [{"type": "boolean"}], "properties": {"x": {"type": "boolean"}}},
],
)
def test_a_schema_this_walk_cannot_read_is_left_alone(spec):
assert _coerce_one("f", "false", {"f": spec}) == "false"
assert _coerce_one("f", "null", {"f": {"anyOf": [spec, {"type": "null"}]}}) == "null"
# Falsified so the refusal cannot swallow real schemas: `nullable` spells a type union.
annotated = {"type": "boolean", "title": "F", "enum": [False]}
assert _coerce_one("f", "false", {"f": annotated}) is False
assert _coerce_one("f", "null", {"f": {"type": "boolean", "nullable": True}}) is None
@pytest.mark.parametrize(
"text, repaired",
[
('["a","b"}', ["a", "b"]), # a closer not matching the innermost opener
('["a","b\\"', ["a", 'b"']), # an open string ending on an ESCAPED quote
],
)
def test_a_malformed_container_is_repaired_only_when_healing(text, repaired):
"""Rewriting brackets invents structure, which is what the auto-heal opt-out is for."""
def at_seam(heal):
return coerce_tool_arguments(
json.dumps({"tags": text}),
heal = heal,
tool_name = "mcp__notes__search",
tool_schemas = _mcp_tool_schemas(),
).arguments["tags"]
assert at_seam(True) == repaired
assert at_seam(False) == text
def test_an_mcp_tool_call_parsed_from_xml_arrives_typed():
content = (
"<function=mcp__notes__search>"
"<parameter=query>ship dates</parameter>"
"<parameter=limit>25</parameter>"
"<parameter=fuzzy>false</parameter>"
'<parameter=tags>["a","b"]</parameter>'
"<parameter=depth>null</parameter>"
"</function>"
)
calls = parse_tool_calls_from_text(content)
decision = ToolLoopController(tools = _mcp_tool_schemas()).prepare_call(calls[0])
assert decision.arguments == {
"query": "ship dates",
"limit": 25,
"fuzzy": False,
"tags": ["a", "b"],
"depth": None,
}
assert decision.as_assistant_tool_call()["function"]["arguments"] == (
'{"query":"ship dates","limit":25,"fuzzy":false,"tags":["a","b"],"depth":null}'
)
def test_replayed_arguments_keep_the_order_the_model_generated():
"""The replay is the text the model is told it wrote, so its key order must survive (#10791)."""
edit_file = next(t for t in ALL_TOOLS if t["function"]["name"] == "edit_file")
arguments = {
"path": "calc.py",
"edits": [{"new_string": "def subtract", "old_string": "def add"}],
}
controller = ToolLoopController(tools = [edit_file])
decision = controller.prepare_call(
{"function": {"name": "edit_file", "arguments": json.dumps(arguments)}}
)
replayed = decision.as_assistant_tool_call()["function"]["arguments"]
assert (
replayed
== '{"path":"calc.py","edits":[{"new_string":"def subtract","old_string":"def add"}]}'
)
assert decision.tool_start_payload()["arguments_text"] == replayed
flipped = {
"path": "calc.py",
"edits": [{"old_string": "def add", "new_string": "def subtract"}],
}
assert canonical_tool_call_key("edit_file", decision.arguments) == (
canonical_tool_call_key("edit_file", flipped)
)
def test_two_xml_calls_written_in_different_orders_replay_in_their_own():
"""Each XML call replays in its own parameter order; a sorted encoder would replay both the same."""
def replay(*parameters):
content = (
"<function=mcp__notes__search>"
+ "".join(f"<parameter={key}>{value}</parameter>" for key, value in parameters)
+ "</function>"
)
calls = parse_tool_calls_from_text(content)
decision = ToolLoopController(tools = _mcp_tool_schemas()).prepare_call(calls[0])
return decision.as_assistant_tool_call()["function"]["arguments"]
assert replay(("query", "ship dates"), ("limit", 25), ("fuzzy", "false")) == (
'{"query":"ship dates","limit":25,"fuzzy":false}'
)
assert replay(("fuzzy", "false"), ("query", "ship dates"), ("limit", 25)) == (
'{"fuzzy":false,"query":"ship dates","limit":25}'
)
def test_a_declared_type_nested_in_a_container_is_read_too():
"""Nested replace_all inside edits.items is read; boolean-spelled text stays text."""
edits = (
'[{"old_string":"a","new_string":"b","replace_all":"false"},'
'{"old_string":"false","new_string":"null","replace_all":"true"}]'
)
edit_file = next(t for t in ALL_TOOLS if t["function"]["name"] == "edit_file")
props = edit_file["function"]["parameters"]["properties"]
typed = [
{"old_string": "a", "new_string": "b", "replace_all": False},
{"old_string": "false", "new_string": "null", "replace_all": True},
]
call = {"path": "app.py", "edits": edits}
assert coerce_arguments_by_schema(call, props) == {"path": "app.py", "edits": typed}
call = {"path": "app.py", "edits": json.loads(edits)}
assert coerce_arguments_by_schema(call, props) == {"path": "app.py", "edits": typed}
@pytest.mark.parametrize(
"result, failed",
[
("Error: boom", True),
(" Error: boom", True),
("Error executing tool remote_thing: disk full", True),
("Error running command `git push`: permission denied", True),
("Errors: 0", False),
("Errors: none found", False),
("Errored, then recovered", False),
("Error-free run", False),
],
)
def test_only_a_delimited_error_marks_a_result_failed(result, failed):
assert is_tool_error(result) is failed
def test_a_success_that_opens_with_error_is_not_nudged_as_a_failure():
controller = ToolLoopController(tools = [_tool("web_search")])
decision = controller.prepare_call(_call("web_search", {"url": "https://example.com/log"}))
completion = controller.record_result(decision, "Errors: 0 across 128 files")
assert not completion.is_error
assert TOOL_ERROR_NUDGE not in completion.model_message()["content"]
@pytest.mark.parametrize("tool_name", ["python", "terminal", "edit_file"])
def test_failed_workspace_execution_invalidates_previous_reads(tool_name):
controller = ToolLoopController(tools = [_tool("terminal"), _tool(tool_name)])
read = _call("terminal", {"command": "cat notes.txt"})
controller.record_result(controller.prepare_call(read), "before")
write = _call(tool_name, {"code": "write_then_fail", "command": "write_then_fail"})
controller.record_result(controller.prepare_call(write), "Error: failed after writing")
assert controller.prepare_call(read).action == "execute"
assert controller.prepare_call(write).action == "execute"
def test_tool_call_limit_nudge_keeps_long_arguments_whole():
code = "print(1)\n" * 60
notice = tool_call_limit_nudge(
[{"function": {"name": "python", "arguments": json.dumps({"code": code})}}], 8
)
assert json.dumps({"code": code}) in notice["content"]
def test_cap_tool_text_passes_results_at_or_under_the_floor_through_unchanged():
assert cap_tool_text("short result") == "short result"
exact = "x" * MAX_TOOL_TEXT_CHARS
assert cap_tool_text(exact) is exact
def test_cap_tool_text_cuts_oversized_text_at_a_nearby_line_break_and_appends_notice():
line = "x" * 100 + "\n"
big = line * (MAX_TOOL_TEXT_CHARS // len(line) + 100)
out = cap_tool_text(big)
assert out.endswith(_NOTICE)
body = out[: -len(_NOTICE)]
assert big.startswith(body)
assert len(body) < MAX_TOOL_TEXT_CHARS
assert big[len(body)] == "\n"
assert len(out) <= MAX_TOOL_TEXT_CHARS + len(_NOTICE)
def test_cap_tool_text_cuts_a_single_line_mid_line_when_no_break_is_near():
big = "y" * (MAX_TOOL_TEXT_CHARS + 500)
out = cap_tool_text(big)
assert out.endswith(_NOTICE)
body = out[: -len(_NOTICE)]
assert body == big[:MAX_TOOL_TEXT_CHARS]
def test_text_that_merely_quotes_the_notice_is_still_capped():
big = "x" * (MAX_TOOL_TEXT_CHARS + 50_000) + _NOTICE
assert cap_tool_text(big) == "x" * MAX_TOOL_TEXT_CHARS + _NOTICE
def test_an_oversized_result_is_capped_for_the_card_and_the_model_alike():
controller = ToolLoopController(tools = [_tool("terminal")])
decision = controller.prepare_call(_call("terminal", {"command": "cat big.log"}))
huge = "line of output\n" * 400_000
completion = controller.record_result(decision, huge)
content = completion.tool_message()["content"]
assert completion.tool_end_payload()["result"] == content
assert content.endswith(_NOTICE)
body = content[: -len(_NOTICE)]
assert len(body) <= MAX_TOOL_TEXT_CHARS
assert huge.startswith(body)
assert huge[len(body)] == "\n"
def test_the_cap_keeps_the_card_envelope_whole_and_off_the_model():
envelope = '\n__WEB_IMAGES__:[{"id": "a1b2c3d4e5f6", "title": "A chart", "domain": "example.com", "source": "https://example.com/a.png"}]'
huge = ("z" * (MAX_TOOL_TEXT_CHARS + 2000)) + envelope
controller = ToolLoopController(tools = [_tool("web_search")])
completion = controller.record_result(
controller.prepare_call(_call("web_search", {"url": "https://example.com/a"})), huge
)
capped = "z" * MAX_TOOL_TEXT_CHARS + _NOTICE
assert completion.tool_end_payload()["result"] == capped + envelope
assert completion.tool_message()["content"] == capped
@pytest.fixture
def _sandbox(tmp_path, monkeypatch):
import core.inference.tools as tools
records = tmp_path / "records"
records.mkdir()
workdir = tmp_path / "sandbox"
workdir.mkdir()
monkeypatch.setattr(tools, "_spill_records_dir", lambda: str(records))
monkeypatch.setattr(tools, "_get_workdir", lambda session_id = None: str(workdir))
return workdir
def _spilled(
names,
result,
session_id = "chat-1",
):
controller = ToolLoopController(
tools = [_tool(name) for name in names], session_id = session_id, thread_id = "t1"
)
return controller.record_result(
controller.prepare_call(_call(names[0], {"server": "x", "q": "y"})), result
)
def test_an_oversized_result_is_spilled_and_the_model_is_told_how_to_search_it(_sandbox):
huge = "".join(f"row {i}\n" for i in range(60_000))
content = _spilled(["mcp__docs__dump", "terminal"], huge).tool_message()["content"]
notice = content[content.index(_tool_text_notice_head()) :]
path = notice.split("saved to ")[1].split(" ")[0]
assert (_sandbox / path).read_text() == huge
assert f"grep -n 'pattern' {path}" in notice
assert f"sed -n '1,200p' {path}" in notice
assert "open(" not in notice
def test_the_hint_names_only_the_reader_the_model_has(_sandbox):
huge = "q" * (MAX_TOOL_TEXT_CHARS + 1)
content = _spilled(["mcp__docs__dump", "python"], huge).tool_message()["content"]
assert "open('.unsloth_tool_output/" in content
assert "grep" not in content
def test_a_cmd_only_windows_host_gets_findstr_with_a_backslash_path(_sandbox, monkeypatch):
import core.inference.tools as tools
monkeypatch.setattr(tools, "_posix_tools_available", lambda: False)
huge = "q" * (MAX_TOOL_TEXT_CHARS + 1)
content = _spilled(["mcp__docs__dump", "terminal"], huge).tool_message()["content"]
assert 'findstr /n "pattern" .unsloth_tool_output\\' in content
assert "grep" not in content
def test_no_reader_tool_or_a_shared_sandbox_gets_the_plain_notice(_sandbox):
huge = "q" * (MAX_TOOL_TEXT_CHARS + 1)
plain = "q" * MAX_TOOL_TEXT_CHARS + _NOTICE
assert _spilled(["mcp__docs__dump"], huge).tool_message()["content"] == plain
assert (
_spilled(["mcp__docs__dump", "terminal"], huge, session_id = None).tool_message()["content"]
== plain
)
assert not (_sandbox / ".unsloth_tool_output").exists()
def test_model_message_leaves_an_already_capped_result_alone():
big = "a" * (MAX_TOOL_TEXT_CHARS + 10_000)
already = cap_tool_text(big)
controller = ToolLoopController(tools = [_tool("terminal")])
completion = controller.record_result(
controller.prepare_call(_call("terminal", {"command": "ls"})), already
)
assert completion.tool_message()["content"] == already
def test_model_message_keeps_the_error_nudge_on_an_oversized_error_result():
huge = "Error: " + "e" * (MAX_TOOL_TEXT_CHARS + 10_000)
controller = ToolLoopController(tools = [_tool("terminal")])
completion = controller.record_result(
controller.prepare_call(_call("terminal", {"command": "ls"})), huge
)
assert completion.is_error
assert completion.tool_message()["content"] == cap_tool_text(huge) + TOOL_ERROR_NUDGE
def test_the_hard_cap_is_tunable(monkeypatch):
import core.inference.tools as tools
monkeypatch.setenv("UNSLOTH_TOOL_RESULT_HARD_CAP_CHARS", "1000")
assert tools._env_int("UNSLOTH_TOOL_RESULT_HARD_CAP_CHARS", 256_000) == 1000
monkeypatch.setattr(tools, "MAX_TOOL_TEXT_CHARS", 50_000)
completion = _spilled(["mcp__docs__dump"], "w" * 60_000)
content = completion.tool_message()["content"]
assert content == "w" * 50_000 + _tool_text_notice_head() + (
" the full output is not retained in model context.)"
)
assert "truncated to 50,000 chars" in content
def test_the_spill_masks_studio_credentials_like_the_model_copy(_sandbox):
key = "sk-unsloth-" + "ab12" * 8
huge = f"token {key}\n" + "r\n" * (MAX_TOOL_TEXT_CHARS // 2 + 10)
content = _spilled(["mcp__docs__dump", "terminal"], huge).tool_message()["content"]
path = content.split("saved to ")[1].split(" ")[0]
spilled = (_sandbox / path).read_text()
assert key not in spilled and key not in content
assert spilled.startswith("token [redacted]")
def test_a_hard_cap_below_the_window_cap_leaves_truncated_terminal_output_alone(monkeypatch):
import core.inference.tools as tools
monkeypatch.setattr(tools, "MAX_TOOL_TEXT_CHARS", 1000)
already = tools._truncate("t\n" * 50_000, workdir = None)
assert len(already) > 1000
controller = ToolLoopController(tools = [_tool("terminal")])
completion = controller.record_result(
controller.prepare_call(_call("terminal", {"command": "yes t"})), already
)
assert completion.tool_message()["content"] == already
def test_a_read_only_mcp_result_is_not_new_work_until_its_hint_is_dropped():
status, models, generate = (f"mcp__unsloth__{n}" for n in ("status", "models", "generate"))
for name in (status, models, generate):
tool_facts.note_mcp_tool(name, {"annotations": {"readOnlyHint": name != generate}})
controller = ToolLoopController(tools = [_tool(status), _tool(models), _tool(generate)])
controller.record_result(controller.prepare_call(_call(status, {})), '{"chat":{}}')
controller.record_result(controller.prepare_call(_call(models, {})), '{"models":[]}')
duplicate = controller.prepare_call(_call(status, {}))
assert duplicate.action == "duplicate" and "Answer from the results" in duplicate.noop_result
controller.record_result(controller.prepare_call(_call(generate, {"text": "x"})), "Error: x")
assert controller.prepare_call(_call(status, {})).action == "execute"
tool_facts.note_mcp_tool(models, {"name": "models"})
assert not tool_facts.reads_only(models)
def test_only_a_result_the_user_must_unblock_marks_the_turn_as_awaiting_them():
name = "mcp__unsloth__generate_audio"
controller = ToolLoopController(tools = [_tool(name)])
for model, result, awaiting in (
("m", "Error: [needs_user_load kind=audio page=Audio] The music model is off.", True),
("n", "Error: [not_loaded kind=audio] Load one with load_model(kind='tts') first.", False),
("o", "Error: [not_loaded kind=audio] Ask the user to load one on the Audio page.", True),
(
"p",
"Error: [multi_model_always kind=chat] (HTTP 409) Call unload_model(kind='llm').",
False,
),
("q", "Error: [multi_model_always kind=chat] (HTTP 409) Ask the user to unload it.", True),
):
controller.begin_batch()
controller.record_result(controller.prepare_call(_call(name, {"model": model})), result)
assert controller.awaiting_user is awaiting, result
def test_a_refusal_the_user_must_unblock_survives_a_later_success_in_the_same_batch():
load, status = "mcp__unsloth__generate_audio", "mcp__unsloth__status"
controller = ToolLoopController(tools = [_tool(load), _tool(status)])
refusal = "Error: [needs_user_load kind=audio page=Audio] The music model is off."
controller.record_result(controller.prepare_call(_call(load, {"model": "m"})), refusal)
controller.record_result(controller.prepare_call(_call(status, {})), '{"chat":{}}')
assert controller.awaiting_user
controller.begin_batch()
controller.record_result(controller.prepare_call(_call(load, {"model": "n"})), "ok")
assert not controller.awaiting_user
def test_active_tools_are_private_copies_of_every_nested_schema_node():
"""Each call hands out its own copy: a caller rewriting one cannot reach the ledger or a later call."""
from collections import OrderedDict
source = {
"type": "function",
"function": {
"name": "lookup",
"description": "d",
"parameters": OrderedDict(
type = "object",
properties = {
"mode": {"type": "string", "enum": ["a", "b"]},
"limit": {"type": "integer", "minimum": 1.0, "default": True},
"pairs": {"type": "array", "items": {"enum": (1, [2])}},
},
),
},
}
controller = ToolLoopController(tools = [source])
first = controller.active_tools()
assert repr(first) == repr([source])
first[0]["function"]["parameters"]["properties"]["mode"]["enum"].append("c")
first[0]["function"]["parameters"]["properties"]["pairs"]["items"]["enum"][1].append(3)
first[0]["function"]["name"] = "renamed"
assert repr(controller.active_tools()) == repr([source])
assert source["function"]["parameters"]["properties"]["mode"]["enum"] == ["a", "b"]
assert source["function"]["parameters"]["properties"]["pairs"]["items"]["enum"] == (1, [2])
def test_has_active_tools_matches_active_tools_without_copying():
controller = ToolLoopController(tools = [_tool("web_search"), _tool("python")])
assert controller.has_active_tools() is bool(controller.active_tools()) is True
assert ToolLoopController(tools = []).has_active_tools() is False
controller._completed_one_shot_tools.update({"web_search", "python"})
assert controller.has_active_tools() is bool(controller.active_tools()) is False
controller._completed_one_shot_tools.clear()
controller._force_final_answer = True
assert controller.has_active_tools() is bool(controller.active_tools()) is False
def test_polling_unsloths_own_get_job_is_never_a_duplicate():
video, job, status = (
f"mcp__unsloth__{n}" for n in ("generate_video", "get_job", "studio_status")
)
for name in (video, job, status):
tool_facts.note_mcp_tool(
name,
{"name": name.split("__")[2], "annotations": {"readOnlyHint": name != video}},
studio = True,
)
controller = ToolLoopController(tools = [_tool(video), _tool(job), _tool(status)])
poll = {"kind": "video", "id": "j1"}
controller.record_result(
controller.prepare_call(_call(video, {"prompt": "a whale"})), '{"id":"j1"}'
)
for name, args in ((job, poll), (status, {}), (job, poll), (job, poll)):
decision = controller.prepare_call(_call(name, args))
assert decision.action == "execute", name
controller.record_result(decision, '{"state":"running"}')
assert not controller.force_final_answer
# Another server's get_job is still deduplicated.
tool_facts.note_mcp_tool("mcp__other__get_job", {"name": "get_job"})
other = ToolLoopController(tools = [_tool("mcp__other__get_job")])
other.record_result(other.prepare_call(_call("mcp__other__get_job", poll)), "{}")
assert other.prepare_call(_call("mcp__other__get_job", poll)).action == "duplicate"
def test_a_second_identical_generation_lets_the_gallery_be_read_again():
image, media = "mcp__unsloth__generate_image", "mcp__unsloth__list_media"
tool_facts.note_mcp_tool(image, {"name": "generate_image"}, studio = True)
tool_facts.note_mcp_tool(
media, {"name": "list_media", "annotations": {"readOnlyHint": True}}, studio = True
)
controller = ToolLoopController(tools = [_tool(image), _tool(media)])
for name, args in (
(image, {"prompt": "a cat"}),
(media, {}),
(image, {"prompt": "a cat"}),
(media, {}),
):
decision = controller.prepare_call(_call(name, args))
assert decision.action == "execute", name
controller.record_result(decision, '{"ok":true}')
# Nothing new since that read: a third identical list is still a duplicate.
assert controller.prepare_call(_call(media, {})).action == "duplicate"
def test_unsloths_own_generators_run_again_on_identical_arguments():
# No seed: each call draws one, so "two images of a cat" is two calls with the same arguments.
names = [f"mcp__unsloth__{n}" for n in ("generate_image", "generate_audio", "generate_video")]
for name in names:
tool_facts.note_mcp_tool(name, {"name": name.split("__")[2]}, studio = True)
controller = ToolLoopController(tools = [_tool(name) for name in names])
for name in names:
for _ in range(2):
decision = controller.prepare_call(_call(name, {"prompt": "a cat"}))
assert decision.action == "execute", name
controller.record_result(decision, '{"ok":true}')
# The same tool name on another server is still deduplicated.
tool_facts.note_mcp_tool("mcp__other__generate_image", {"name": "generate_image"})
other = ToolLoopController(tools = [_tool("mcp__other__generate_image")])
other.record_result(
other.prepare_call(_call("mcp__other__generate_image", {"prompt": "a cat"})), "{}"
)
assert (
other.prepare_call(_call("mcp__other__generate_image", {"prompt": "a cat"})).action
== "duplicate"
)