Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: fit long tool results to the window on connections (#17553)
Nilayfccc8c4df4c8
diff --git a/studio/backend/core/inference/external_tool_transport.py b/studio/backend/core/inference/external_tool_transport.pyindex b75ed344b1d..42f8fe4e03b 100644--- a/studio/backend/core/inference/external_tool_transport.py+++ b/studio/backend/core/inference/external_tool_transport.py@@ -59,6 +59,7 @@ class OAICompatTransport:# Anthropic can leave a hosted call pending beside a client call; its continuation accepts only tool results.self.tool_result_only_continuation = client.provider_type == "anthropic"self._initial_message_count: int | None = None+ self.sent_messages: list[dict[str, Any]] | None = Noneself._last_tools: list[dict[str, Any]] | None = None# Always, as the local GGUF loop does: preserve_thinking hides earlier turns, never this turn's tool rounds.self.preserves_reasoning = client.provider_type in (@@ -116,6 +117,7 @@ class OAICompatTransport:request_kwargs["max_tokens"] = max_tokensif truncation_line:yield truncation_line+ self.sent_messages = messages# Re-derived each turn: a tool or user message at the end must not be resumed as an# assistant turn.diff --git a/studio/backend/core/inference/studio_tool_loop.py b/studio/backend/core/inference/studio_tool_loop.pyindex da2dee6ec17..e181c3f7fb6 100644--- a/studio/backend/core/inference/studio_tool_loop.py+++ b/studio/backend/core/inference/studio_tool_loop.py@@ -45,11 +45,17 @@ import urllib.parsefrom dataclasses import dataclass, fieldfrom functools import partial-from collections.abc import AsyncIterator, Callable+from collections.abc import AsyncIterator, Awaitable, Callablefrom typing import Any, Protocolfrom core.inference import tools as tools_modulefrom core.inference.chat_template_helpers import append_assistant_turn+from core.inference.context_window import (+ _reply_floor,+ estimate_messages_tokens_conservative,+ messages_without_unpriced_media,+ tool_result_budget,+)from core.inference.passthrough_healing import StreamToolCallHealer, heal_gate, nudge_enabledfrom core.inference.sse_control_frames import sanitize_provider_sse_linefrom core.inference.tool_call_parser import (@@ -391,6 +397,8 @@ class ToolLoopRun:supports_vision: bool = False# Counted toward the loop's image cap so resumed chats cannot exceed it.promoted_image_parts: tuple = ()+ served_window: Callable[[], Awaitable[int | None]] | None = None+ max_tokens: int | None = None@dataclass(frozen = True)@@ -416,6 +424,46 @@ class ToolLoopPolicy:website_policy: dict[str, Any] | None = None+def _result_budget_tokens(+ window: int,+ max_tokens: int | None,+ conversation: list[dict[str, Any]],+ tools: list[dict[str, Any]],+ turn: Any,+ calls: list[dict[str, Any]],+ tool_messages: list[dict[str, Any]],+ call_index: int,+) -> int:+ thread = messages_without_unpriced_media(conversation)+ results = [m for m in (*thread, *tool_messages) if m.get("role") == "tool"]+ rest = [m for m in thread if m.get("role") != "tool"]+ rest.append(+ {+ "role": "assistant",+ "content": "".join(turn.text),+ "reasoning_content": "".join(turn.reasoning),+ }+ )+ replayed = [+ {"name": (call.get("function") or {}).get("name"), "arguments": call.get("arguments")}+ for call in calls+ ]+ batch = [{"role": "assistant", "tool_calls": replayed}]+ held = estimate_messages_tokens_conservative(results, dense_ascii = True)+ spent = (+ estimate_messages_tokens_conservative(rest)+ + estimate_messages_tokens_conservative(tools or [])+ + held+ + estimate_messages_tokens_conservative(batch, dense_ascii = True)+ )+ # Uncounted results get two characters a budget token: fit the window even at one token each.+ room = min(+ tool_result_budget(window, max_tokens, spent),+ tool_result_budget(window, _reply_floor(window), spent + held) // 2,+ )+ return room // (len(calls) - call_index)++def _split_top_level_json_objects(text: str) -> tuple[list[str], str]:"""Top-level JSON objects plus any unfinished one; non-object text comes back whole. Must agree with splitTopLevelJsonObjects in chat/tool-call-arguments.ts."""unsplit: tuple[list[str], str] = ([], text)@@ -1879,6 +1927,8 @@ async def _stream_with_studio_tools(turn_executed_real_tool = Falsestarted_fetches.clear()+ # Read per batch: Ollama reports a window only once the first turn has loaded the model.+ window = await run.served_window() if run.served_window is not None else Nonefor call_index, call in enumerate(calls):if cancel_event.is_set():break@@ -2088,6 +2138,7 @@ async def _stream_with_studio_tools(approved = host_access_approved,strict = strict_isolation,share = image_share,+ index = call_index,) -> str:kwargs: dict[str, Any] = {"cancel_event": cancel_event,@@ -2113,9 +2164,20 @@ async def _stream_with_studio_tools(kwargs["conversation_branch"] = extend_live_branch()if approved and accepts_kwarg(execute_tool, "host_access_approved"):kwargs["host_access_approved"] = True- # External window is unknowable: 0 keeps the default page cap; unmeasurable budget means one recall, since 8 chunks is ~4K tokens replayed each later call.+ # 0, not the served window: tools would count with a local GGUF loaded at that window.if accepts_kwarg(execute_tool, "context_tokens"):kwargs["context_tokens"] = 0+ if window and accepts_kwarg(execute_tool, "result_budget_tokens"):+ kwargs["result_budget_tokens"] = _result_budget_tokens(+ window,+ run.max_tokens,+ getattr(transport, "sent_messages", None) or conversation,+ controller.active_tools(include_deferred = True),+ turn,+ calls,+ tool_messages,+ index,+ )if accepts_kwarg(execute_tool, "conversation_budget_tokens"):try:from core.rag import config as rag_configdiff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 499659f7b3e..20fc9bdb0b7 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -29833,6 +29833,11 @@ async def _proxy_to_external_provider(),})++ async def _tool_result_window() -> Optional[int]:+ served = await client.served_context_window(model)+ return min(served, payload.context_window or served) if served else None+_external_truncation = None_external_max_tokens = _effective_max_tokens(payload)_external_context_fitter = None@@ -29959,6 +29964,8 @@ async def _proxy_to_external_provider(continue_final_message = _continue_final_message(payload),supports_vision = _takes_tool_images,promoted_image_parts = tuple(_external_promoted_parts),+ served_window = _tool_result_window if managed is None else None,+ max_tokens = _external_max_tokens,),policy = ToolLoopPolicy(tools = external_studio_tools,diff --git a/studio/backend/tests/test_external_tool_result_fits_window.py b/studio/backend/tests/test_external_tool_result_fits_window.pynew file mode 100644index 00000000000..2cb70c67c6c--- /dev/null+++ b/studio/backend/tests/test_external_tool_result_fits_window.py@@ -0,0 +1,284 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A tool result on a connection is sized to the window its server reports."""++from __future__ import annotations++import asyncio+import json+import threading++import pytest++from core.inference import studio_tool_loop as loop_mod+from core.inference import tools+from core.inference.context_window import estimate_messages_tokens_conservative+from core.inference.studio_tool_loop import (+ ToolLoopPolicy,+ ToolLoopRun,+ stream_with_studio_tools,+)+++_DONE = "data: [DONE]"+_LONG = "\n".join(f"row {i}: the order shipped on time" for i in range(500))++WEB = {+ "type": "function",+ "function": {+ "name": "web_search",+ "description": "",+ "parameters": {+ "type": "object",+ "properties": {"query": {"type": "string"}},+ "required": ["query"],+ },+ },+}+++def _sse(delta = None, finish = None) -> str:+ choice: dict = {"index": 0, "delta": delta if delta is not None else {}}+ if finish is not None:+ choice["finish_reason"] = finish+ return "data: " + json.dumps({"choices": [choice]})+++def _call_turn(+ *call_ids,+ escaped = False,+ name = "web_search",+):+ return [+ _sse(+ {+ "tool_calls": [+ {+ "index": index,+ "id": call_id,+ "function": {+ "name": name,+ "arguments": json.dumps({"query": call_id}, ensure_ascii = escaped),+ },+ }+ for index, call_id in enumerate(call_ids)+ ]+ }+ ),+ _sse(finish = "tool_calls"),+ _DONE,+ ]+++_ANSWER_TURN = [_sse({"content": "done"}), _sse(finish = "stop"), _DONE]+++class FakeTransport:+ heals_text_tool_calls = True++ def __init__(self, turns):+ self.turns = [list(turn) for turn in turns]+ self.requests: list[list[dict]] = []++ def stream(self, *, messages, tools, tool_choice, cancel_event):+ self.requests.append([dict(message) for message in messages])+ self.tools = tools+ assert len(self.requests) <= 5, "loop never terminated"+ lines = self.turns.pop(0) if self.turns else [_DONE]++ async def _gen():+ for line in lines:+ yield line++ return _gen()++[email protected](autouse = True)+def _long_search_results(monkeypatch):+ monkeypatch.setattr(tools, "_web_search", lambda *a, **k: _LONG)+ monkeypatch.setattr(tools, "_loaded_context_tokens", lambda: None)+ monkeypatch.setattr(loop_mod, "build_rag_autoinject", lambda *a, **k: None)+ monkeypatch.setattr(loop_mod, "is_high_risk_tool_call", lambda name, args: False)+++def _serves(window):+ async def _read():+ return window++ return _read+++def _forwarded(+ turns,+ history = (),+ catalog = (WEB,),+ **run_kwargs,+):+ transport = turns if isinstance(turns, FakeTransport) else FakeTransport(turns)++ async def _collect():+ agen = stream_with_studio_tools(+ transport,+ run = ToolLoopRun(+ messages = [*history, {"role": "user", "content": "which orders shipped?"}],+ session_id = "s1",+ thread_id = "t1",+ **run_kwargs,+ ),+ policy = ToolLoopPolicy(+ tools = list(catalog),+ max_calls = 25,+ timeout = 300,+ permission_mode = "off",+ confirm_calls = False,+ bypass_permissions = False,+ rag_scope = None,+ ),+ cancel_event = threading.Event(),+ )+ async for _ in agen:+ pass++ asyncio.run(asyncio.wait_for(_collect(), timeout = 30.0))+ return transport.requests[-1]+++def _results(messages):+ return [message["content"] for message in messages if message.get("role") == "tool"]+++def test_a_long_result_is_trimmed_to_the_served_window():+ messages = _forwarded([_call_turn("c1"), _ANSWER_TURN], served_window = _serves(4096))++ (result,) = _results(messages)+ assert len(result) < len(_LONG)+ assert result.startswith(_LONG[:200])+ assert f"{len(_LONG)} chars total" in result+ assert estimate_messages_tokens_conservative(messages) < 4096+++def test_a_digit_heavy_result_fits_at_one_token_a_character(monkeypatch):+ digits = str(list(range(4000)))+ monkeypatch.setattr(tools, "_web_search", lambda *a, **k: digits)++ messages = _forwarded([_call_turn("c1"), _ANSWER_TURN], served_window = _serves(8192))++ (result,) = _results(messages)+ rest = [message for message in messages if message.get("role") != "tool"]+ assert len(result) + estimate_messages_tokens_conservative(rest) < 8192+++def test_parallel_results_share_the_served_window():+ messages = _forwarded([_call_turn("c1", "c2", "c3"), _ANSWER_TURN], served_window = _serves(4096))++ results = _results(messages)+ assert len(results) == 3+ assert all(len(result) < len(_LONG) for result in results)+ assert estimate_messages_tokens_conservative(messages) < 4096+++def test_a_window_reported_after_the_first_turn_still_sizes_the_result():+ transport = FakeTransport([_call_turn("c1"), _ANSWER_TURN])++ async def _loaded_by_the_first_turn():+ return 4096 if transport.requests else None++ messages = _forwarded(transport, served_window = _loaded_by_the_first_turn)++ (result,) = _results(messages)+ assert len(result) < len(_LONG)++[email protected]("run_kwargs", [{}, {"served_window": _serves(None)}])+def test_a_result_is_forwarded_whole_when_the_window_is_unknown(run_kwargs):+ messages = _forwarded([_call_turn("c1"), _ANSWER_TURN], **run_kwargs)++ assert _results(messages) == [_LONG]+++class CompactingTransport(FakeTransport):+ def stream(self, *, messages, **kwargs):+ self.sent_messages = messages[-1:] if messages[-1]["role"] == "user" else messages[-3:]+ return super().stream(messages = self.sent_messages, **kwargs)+++def test_a_result_is_sized_to_the_history_left_after_compaction():+ history = [+ {"role": role, "content": f"turn {i} " + "x" * 2000}+ for i in range(20)+ for role in ("user", "assistant")+ ]+ transport = CompactingTransport([_call_turn("c1"), _ANSWER_TURN])++ messages = _forwarded(transport, history = history, served_window = _serves(4096))++ (result,) = _results(messages)+ assert result.startswith(_LONG[:200])+ assert estimate_messages_tokens_conservative(messages) < 4096++[email protected]("escaped", [False, True])+def test_non_ascii_call_arguments_do_not_use_up_the_result_room(monkeypatch, escaped):+ reply = "字" * 600+ monkeypatch.setattr(tools, "_web_search", lambda *a, **k: reply)++ messages = _forwarded(+ [_call_turn("中" * 1000, escaped = escaped), _ANSWER_TURN], served_window = _serves(4096)+ )++ (result,) = _results(messages)+ assert result.startswith(reply[:200])+ assert estimate_messages_tokens_conservative(messages) < 4096+++def test_tool_schemas_found_by_search_count_against_the_result_room(monkeypatch):+ from core.inference import tool_loop_controller as controller_mod++ def _spec(name, description = ""):+ return {+ "type": "function",+ "function": {**WEB["function"], "name": name, "description": description},+ }++ search = _spec("mcp_tool_search")+ found = _spec("mcp__srv1__ping", "pings the server " * 300)+ monkeypatch.setattr(controller_mod, "_mcp_tool_search_specs", lambda result: [found])+ monkeypatch.setattr(controller_mod, "_mcp_tool_spec", lambda name: found)+ monkeypatch.setattr(tools, "_web_search", lambda *a, **k: str(list(range(4000))))+ transport = FakeTransport(+ [_call_turn("s1", name = "mcp_tool_search"), _call_turn("c1"), _ANSWER_TURN]+ )++ messages = _forwarded(transport, catalog = (WEB, search), served_window = _serves(8192))++ assert found in transport.tools+ result = _results(messages)[-1]+ rest = [message for message in messages if message.get("content") != result]+ prompt = estimate_messages_tokens_conservative(rest + transport.tools)+ assert len(result) + prompt < 8192+++def test_a_local_model_at_the_same_window_does_not_price_the_connection(monkeypatch):+ import sys+ import types++ class _LocalGguf:+ is_loaded = True+ context_length = 8192++ def count_chat_tokens(self, messages, *args, **kwargs):+ # A tokenizer that packs digits several to a token, unlike the served model's.+ return max(1, len(json.dumps(messages)) // 8)++ stub = sys.modules.get("routes.inference") or types.ModuleType("routes.inference")+ monkeypatch.setitem(sys.modules, "routes.inference", stub)+ monkeypatch.setattr(stub, "get_llama_cpp_backend", lambda: _LocalGguf(), raising = False)+ digits = str(list(range(4000)))+ monkeypatch.setattr(tools, "_web_search", lambda *a, **k: digits)++ messages = _forwarded([_call_turn("c1"), _ANSWER_TURN], served_window = _serves(8192))++ (result,) = _results(messages)+ rest = [message for message in messages if message.get("role") != "tool"]+ assert len(result) + estimate_messages_tokens_conservative(rest) < 8192