Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fit long tool results to the window on connections (#17553)

Nilayfccc8c4df4c8
diff --git a/studio/backend/core/inference/external_tool_transport.py b/studio/backend/core/inference/external_tool_transport.py
index b75ed344b1d..42f8fe4e03b 100644
--- a/studio/backend/core/inference/external_tool_transport.py
+++ b/studio/backend/core/inference/external_tool_transport.py
@@ -59,6 +59,7 @@ class OAICompatTransport:
# Anthropic can leave a hosted call pending beside a client call; its continuation accepts only tool results.
self.tool_result_only_continuation = client.provider_type == "anthropic"
self._initial_message_count: int | None = None
+ self.sent_messages: list[dict[str, Any]] | None = None
self._last_tools: list[dict[str, Any]] | None = None
# Always, as the local GGUF loop does: preserve_thinking hides earlier turns, never this turn's tool rounds.
self.preserves_reasoning = client.provider_type in (
@@ -116,6 +117,7 @@ class OAICompatTransport:
request_kwargs["max_tokens"] = max_tokens
if truncation_line:
yield truncation_line
+ self.sent_messages = messages
# Re-derived each turn: a tool or user message at the end must not be resumed as an
# assistant turn.
diff --git a/studio/backend/core/inference/studio_tool_loop.py b/studio/backend/core/inference/studio_tool_loop.py
index da2dee6ec17..e181c3f7fb6 100644
--- a/studio/backend/core/inference/studio_tool_loop.py
+++ b/studio/backend/core/inference/studio_tool_loop.py
@@ -45,11 +45,17 @@ import urllib.parse
from dataclasses import dataclass, field
from functools import partial
-from collections.abc import AsyncIterator, Callable
+from collections.abc import AsyncIterator, Awaitable, Callable
from typing import Any, Protocol
from core.inference import tools as tools_module
from core.inference.chat_template_helpers import append_assistant_turn
+from core.inference.context_window import (
+ _reply_floor,
+ estimate_messages_tokens_conservative,
+ messages_without_unpriced_media,
+ tool_result_budget,
+)
from core.inference.passthrough_healing import StreamToolCallHealer, heal_gate, nudge_enabled
from core.inference.sse_control_frames import sanitize_provider_sse_line
from core.inference.tool_call_parser import (
@@ -391,6 +397,8 @@ class ToolLoopRun:
supports_vision: bool = False
# Counted toward the loop's image cap so resumed chats cannot exceed it.
promoted_image_parts: tuple = ()
+ served_window: Callable[[], Awaitable[int | None]] | None = None
+ max_tokens: int | None = None
@dataclass(frozen = True)
@@ -416,6 +424,46 @@ class ToolLoopPolicy:
website_policy: dict[str, Any] | None = None
+def _result_budget_tokens(
+ window: int,
+ max_tokens: int | None,
+ conversation: list[dict[str, Any]],
+ tools: list[dict[str, Any]],
+ turn: Any,
+ calls: list[dict[str, Any]],
+ tool_messages: list[dict[str, Any]],
+ call_index: int,
+) -> int:
+ thread = messages_without_unpriced_media(conversation)
+ results = [m for m in (*thread, *tool_messages) if m.get("role") == "tool"]
+ rest = [m for m in thread if m.get("role") != "tool"]
+ rest.append(
+ {
+ "role": "assistant",
+ "content": "".join(turn.text),
+ "reasoning_content": "".join(turn.reasoning),
+ }
+ )
+ replayed = [
+ {"name": (call.get("function") or {}).get("name"), "arguments": call.get("arguments")}
+ for call in calls
+ ]
+ batch = [{"role": "assistant", "tool_calls": replayed}]
+ held = estimate_messages_tokens_conservative(results, dense_ascii = True)
+ spent = (
+ estimate_messages_tokens_conservative(rest)
+ + estimate_messages_tokens_conservative(tools or [])
+ + held
+ + estimate_messages_tokens_conservative(batch, dense_ascii = True)
+ )
+ # Uncounted results get two characters a budget token: fit the window even at one token each.
+ room = min(
+ tool_result_budget(window, max_tokens, spent),
+ tool_result_budget(window, _reply_floor(window), spent + held) // 2,
+ )
+ return room // (len(calls) - call_index)
+
+
def _split_top_level_json_objects(text: str) -> tuple[list[str], str]:
"""Top-level JSON objects plus any unfinished one; non-object text comes back whole. Must agree with splitTopLevelJsonObjects in chat/tool-call-arguments.ts."""
unsplit: tuple[list[str], str] = ([], text)
@@ -1879,6 +1927,8 @@ async def _stream_with_studio_tools(
turn_executed_real_tool = False
started_fetches.clear()
+ # Read per batch: Ollama reports a window only once the first turn has loaded the model.
+ window = await run.served_window() if run.served_window is not None else None
for call_index, call in enumerate(calls):
if cancel_event.is_set():
break
@@ -2088,6 +2138,7 @@ async def _stream_with_studio_tools(
approved = host_access_approved,
strict = strict_isolation,
share = image_share,
+ index = call_index,
) -> str:
kwargs: dict[str, Any] = {
"cancel_event": cancel_event,
@@ -2113,9 +2164,20 @@ async def _stream_with_studio_tools(
kwargs["conversation_branch"] = extend_live_branch()
if approved and accepts_kwarg(execute_tool, "host_access_approved"):
kwargs["host_access_approved"] = True
- # External window is unknowable: 0 keeps the default page cap; unmeasurable budget means one recall, since 8 chunks is ~4K tokens replayed each later call.
+ # 0, not the served window: tools would count with a local GGUF loaded at that window.
if accepts_kwarg(execute_tool, "context_tokens"):
kwargs["context_tokens"] = 0
+ if window and accepts_kwarg(execute_tool, "result_budget_tokens"):
+ kwargs["result_budget_tokens"] = _result_budget_tokens(
+ window,
+ run.max_tokens,
+ getattr(transport, "sent_messages", None) or conversation,
+ controller.active_tools(include_deferred = True),
+ turn,
+ calls,
+ tool_messages,
+ index,
+ )
if accepts_kwarg(execute_tool, "conversation_budget_tokens"):
try:
from core.rag import config as rag_config
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 499659f7b3e..20fc9bdb0b7 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -29833,6 +29833,11 @@ async def _proxy_to_external_provider(
),
}
)
+
+ async def _tool_result_window() -> Optional[int]:
+ served = await client.served_context_window(model)
+ return min(served, payload.context_window or served) if served else None
+
_external_truncation = None
_external_max_tokens = _effective_max_tokens(payload)
_external_context_fitter = None
@@ -29959,6 +29964,8 @@ async def _proxy_to_external_provider(
continue_final_message = _continue_final_message(payload),
supports_vision = _takes_tool_images,
promoted_image_parts = tuple(_external_promoted_parts),
+ served_window = _tool_result_window if managed is None else None,
+ max_tokens = _external_max_tokens,
),
policy = ToolLoopPolicy(
tools = external_studio_tools,
diff --git a/studio/backend/tests/test_external_tool_result_fits_window.py b/studio/backend/tests/test_external_tool_result_fits_window.py
new file mode 100644
index 00000000000..2cb70c67c6c
--- /dev/null
+++ b/studio/backend/tests/test_external_tool_result_fits_window.py
@@ -0,0 +1,284 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A tool result on a connection is sized to the window its server reports."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import threading
+
+import pytest
+
+from core.inference import studio_tool_loop as loop_mod
+from core.inference import tools
+from core.inference.context_window import estimate_messages_tokens_conservative
+from core.inference.studio_tool_loop import (
+ ToolLoopPolicy,
+ ToolLoopRun,
+ stream_with_studio_tools,
+)
+
+
+_DONE = "data: [DONE]"
+_LONG = "\n".join(f"row {i}: the order shipped on time" for i in range(500))
+
+WEB = {
+ "type": "function",
+ "function": {
+ "name": "web_search",
+ "description": "",
+ "parameters": {
+ "type": "object",
+ "properties": {"query": {"type": "string"}},
+ "required": ["query"],
+ },
+ },
+}
+
+
+def _sse(delta = None, finish = None) -> str:
+ choice: dict = {"index": 0, "delta": delta if delta is not None else {}}
+ if finish is not None:
+ choice["finish_reason"] = finish
+ return "data: " + json.dumps({"choices": [choice]})
+
+
+def _call_turn(
+ *call_ids,
+ escaped = False,
+ name = "web_search",
+):
+ return [
+ _sse(
+ {
+ "tool_calls": [
+ {
+ "index": index,
+ "id": call_id,
+ "function": {
+ "name": name,
+ "arguments": json.dumps({"query": call_id}, ensure_ascii = escaped),
+ },
+ }
+ for index, call_id in enumerate(call_ids)
+ ]
+ }
+ ),
+ _sse(finish = "tool_calls"),
+ _DONE,
+ ]
+
+
+_ANSWER_TURN = [_sse({"content": "done"}), _sse(finish = "stop"), _DONE]
+
+
+class FakeTransport:
+ heals_text_tool_calls = True
+
+ def __init__(self, turns):
+ self.turns = [list(turn) for turn in turns]
+ self.requests: list[list[dict]] = []
+
+ def stream(self, *, messages, tools, tool_choice, cancel_event):
+ self.requests.append([dict(message) for message in messages])
+ self.tools = tools
+ assert len(self.requests) <= 5, "loop never terminated"
+ lines = self.turns.pop(0) if self.turns else [_DONE]
+
+ async def _gen():
+ for line in lines:
+ yield line
+
+ return _gen()
+
+
[email protected](autouse = True)
+def _long_search_results(monkeypatch):
+ monkeypatch.setattr(tools, "_web_search", lambda *a, **k: _LONG)
+ monkeypatch.setattr(tools, "_loaded_context_tokens", lambda: None)
+ monkeypatch.setattr(loop_mod, "build_rag_autoinject", lambda *a, **k: None)
+ monkeypatch.setattr(loop_mod, "is_high_risk_tool_call", lambda name, args: False)
+
+
+def _serves(window):
+ async def _read():
+ return window
+
+ return _read
+
+
+def _forwarded(
+ turns,
+ history = (),
+ catalog = (WEB,),
+ **run_kwargs,
+):
+ transport = turns if isinstance(turns, FakeTransport) else FakeTransport(turns)
+
+ async def _collect():
+ agen = stream_with_studio_tools(
+ transport,
+ run = ToolLoopRun(
+ messages = [*history, {"role": "user", "content": "which orders shipped?"}],
+ session_id = "s1",
+ thread_id = "t1",
+ **run_kwargs,
+ ),
+ policy = ToolLoopPolicy(
+ tools = list(catalog),
+ max_calls = 25,
+ timeout = 300,
+ permission_mode = "off",
+ confirm_calls = False,
+ bypass_permissions = False,
+ rag_scope = None,
+ ),
+ cancel_event = threading.Event(),
+ )
+ async for _ in agen:
+ pass
+
+ asyncio.run(asyncio.wait_for(_collect(), timeout = 30.0))
+ return transport.requests[-1]
+
+
+def _results(messages):
+ return [message["content"] for message in messages if message.get("role") == "tool"]
+
+
+def test_a_long_result_is_trimmed_to_the_served_window():
+ messages = _forwarded([_call_turn("c1"), _ANSWER_TURN], served_window = _serves(4096))
+
+ (result,) = _results(messages)
+ assert len(result) < len(_LONG)
+ assert result.startswith(_LONG[:200])
+ assert f"{len(_LONG)} chars total" in result
+ assert estimate_messages_tokens_conservative(messages) < 4096
+
+
+def test_a_digit_heavy_result_fits_at_one_token_a_character(monkeypatch):
+ digits = str(list(range(4000)))
+ monkeypatch.setattr(tools, "_web_search", lambda *a, **k: digits)
+
+ messages = _forwarded([_call_turn("c1"), _ANSWER_TURN], served_window = _serves(8192))
+
+ (result,) = _results(messages)
+ rest = [message for message in messages if message.get("role") != "tool"]
+ assert len(result) + estimate_messages_tokens_conservative(rest) < 8192
+
+
+def test_parallel_results_share_the_served_window():
+ messages = _forwarded([_call_turn("c1", "c2", "c3"), _ANSWER_TURN], served_window = _serves(4096))
+
+ results = _results(messages)
+ assert len(results) == 3
+ assert all(len(result) < len(_LONG) for result in results)
+ assert estimate_messages_tokens_conservative(messages) < 4096
+
+
+def test_a_window_reported_after_the_first_turn_still_sizes_the_result():
+ transport = FakeTransport([_call_turn("c1"), _ANSWER_TURN])
+
+ async def _loaded_by_the_first_turn():
+ return 4096 if transport.requests else None
+
+ messages = _forwarded(transport, served_window = _loaded_by_the_first_turn)
+
+ (result,) = _results(messages)
+ assert len(result) < len(_LONG)
+
+
[email protected]("run_kwargs", [{}, {"served_window": _serves(None)}])
+def test_a_result_is_forwarded_whole_when_the_window_is_unknown(run_kwargs):
+ messages = _forwarded([_call_turn("c1"), _ANSWER_TURN], **run_kwargs)
+
+ assert _results(messages) == [_LONG]
+
+
+class CompactingTransport(FakeTransport):
+ def stream(self, *, messages, **kwargs):
+ self.sent_messages = messages[-1:] if messages[-1]["role"] == "user" else messages[-3:]
+ return super().stream(messages = self.sent_messages, **kwargs)
+
+
+def test_a_result_is_sized_to_the_history_left_after_compaction():
+ history = [
+ {"role": role, "content": f"turn {i} " + "x" * 2000}
+ for i in range(20)
+ for role in ("user", "assistant")
+ ]
+ transport = CompactingTransport([_call_turn("c1"), _ANSWER_TURN])
+
+ messages = _forwarded(transport, history = history, served_window = _serves(4096))
+
+ (result,) = _results(messages)
+ assert result.startswith(_LONG[:200])
+ assert estimate_messages_tokens_conservative(messages) < 4096
+
+
[email protected]("escaped", [False, True])
+def test_non_ascii_call_arguments_do_not_use_up_the_result_room(monkeypatch, escaped):
+ reply = "字" * 600
+ monkeypatch.setattr(tools, "_web_search", lambda *a, **k: reply)
+
+ messages = _forwarded(
+ [_call_turn("中" * 1000, escaped = escaped), _ANSWER_TURN], served_window = _serves(4096)
+ )
+
+ (result,) = _results(messages)
+ assert result.startswith(reply[:200])
+ assert estimate_messages_tokens_conservative(messages) < 4096
+
+
+def test_tool_schemas_found_by_search_count_against_the_result_room(monkeypatch):
+ from core.inference import tool_loop_controller as controller_mod
+
+ def _spec(name, description = ""):
+ return {
+ "type": "function",
+ "function": {**WEB["function"], "name": name, "description": description},
+ }
+
+ search = _spec("mcp_tool_search")
+ found = _spec("mcp__srv1__ping", "pings the server " * 300)
+ monkeypatch.setattr(controller_mod, "_mcp_tool_search_specs", lambda result: [found])
+ monkeypatch.setattr(controller_mod, "_mcp_tool_spec", lambda name: found)
+ monkeypatch.setattr(tools, "_web_search", lambda *a, **k: str(list(range(4000))))
+ transport = FakeTransport(
+ [_call_turn("s1", name = "mcp_tool_search"), _call_turn("c1"), _ANSWER_TURN]
+ )
+
+ messages = _forwarded(transport, catalog = (WEB, search), served_window = _serves(8192))
+
+ assert found in transport.tools
+ result = _results(messages)[-1]
+ rest = [message for message in messages if message.get("content") != result]
+ prompt = estimate_messages_tokens_conservative(rest + transport.tools)
+ assert len(result) + prompt < 8192
+
+
+def test_a_local_model_at_the_same_window_does_not_price_the_connection(monkeypatch):
+ import sys
+ import types
+
+ class _LocalGguf:
+ is_loaded = True
+ context_length = 8192
+
+ def count_chat_tokens(self, messages, *args, **kwargs):
+ # A tokenizer that packs digits several to a token, unlike the served model's.
+ return max(1, len(json.dumps(messages)) // 8)
+
+ stub = sys.modules.get("routes.inference") or types.ModuleType("routes.inference")
+ monkeypatch.setitem(sys.modules, "routes.inference", stub)
+ monkeypatch.setattr(stub, "get_llama_cpp_backend", lambda: _LocalGguf(), raising = False)
+ digits = str(list(range(4000)))
+ monkeypatch.setattr(tools, "_web_search", lambda *a, **k: digits)
+
+ messages = _forwarded([_call_turn("c1"), _ANSWER_TURN], served_window = _serves(8192))
+
+ (result,) = _results(messages)
+ rest = [message for message in messages if message.get("role") != "tool"]
+ assert len(result) + estimate_messages_tokens_conservative(rest) < 8192