Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fit max_tokens to the window on vLLM and SGLang tool requests (#17426)

Nilayb347d06cf4b2
diff --git a/studio/backend/core/inference/managed_engine.py b/studio/backend/core/inference/managed_engine.py
index 9e292a28e8a..f3dd94d0f4c 100644
--- a/studio/backend/core/inference/managed_engine.py
+++ b/studio/backend/core/inference/managed_engine.py
@@ -1128,12 +1128,20 @@ class ManagedEngine:
for message in messages
]
- def _reply_cap_in_window(self, messages, system_prompt, limit, params):
+ def _reply_cap_in_window(
+ self,
+ messages,
+ system_prompt,
+ limit,
+ params,
+ tools = None,
+ ):
"""vLLM and SGLang refuse prompt + max_tokens past the window, where llama-server stops at it."""
try:
prompt = self.count_tokens(
messages,
system_prompt,
+ tools = tools,
enable_thinking = params.get("enable_thinking"),
reasoning_effort = params.get("reasoning_effort"),
preserve_thinking = params.get("preserve_thinking"),
@@ -1143,14 +1151,28 @@ class ManagedEngine:
return limit
return max(1, min(limit, self.context - prompt))
+ def _completion_cap_in_window(self, prompt, limit):
+ try:
+ return max(1, min(limit, self.context - self._tokenize({"prompt": prompt})))
+ except Exception:
+ return limit
+
+ def _tokenize(self, body):
+ with httpx.Client(trust_env = False, timeout = 30) as client:
+ response = client.post(
+ self.base_url + "/tokenize",
+ headers = self.headers,
+ json = {"model": self.model, **body},
+ )
+ response.raise_for_status()
+ return int(response.json()["count"])
+
def count_tokens(
self,
messages,
system_prompt = "",
**kwargs,
):
- if kwargs.get("tools"):
- raise ValueError("Tools are unavailable for this optional engine profile.")
if not self.adapter.exact_token_count:
raise RuntimeError("Exact prompt token counting is unavailable for this engine.")
if system_prompt:
@@ -1167,19 +1189,14 @@ class ManagedEngine:
template_kwargs = fields["chat_template_kwargs"]
template_kwargs["reasoning_effort"] = effort
template_kwargs.setdefault("enable_thinking", effort != "none")
- with httpx.Client(trust_env = False, timeout = 30) as client:
- response = client.post(
- self.base_url + "/tokenize",
- headers = self.headers,
- json = {
- "model": self.model,
- "messages": self.replay_reasoning(messages, kwargs.get("preserve_thinking")),
- "add_generation_prompt": True,
- **fields,
- },
- )
- response.raise_for_status()
- return int(response.json()["count"]), self.model
+ body = {
+ "messages": self.replay_reasoning(messages, kwargs.get("preserve_thinking")),
+ "add_generation_prompt": True,
+ **fields,
+ }
+ if kwargs.get("tools"):
+ body["tools"] = kwargs["tools"]
+ return self._tokenize(body), self.model
def generate(
self,
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 8f5d7ded25d..188eb2175f9 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -3050,10 +3050,11 @@ class InferenceOrchestrator:
instead."""
managed = getattr(self, "_managed_engine", None)
if managed is not None:
+ if tools:
+ raise ValueError("Tools are unavailable for this optional engine profile.")
return managed.count_tokens(
messages,
system_prompt,
- tools = tools,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
diff --git a/studio/backend/routes/managed_engine_chat.py b/studio/backend/routes/managed_engine_chat.py
index 16288dab925..71f57d1d797 100644
--- a/studio/backend/routes/managed_engine_chat.py
+++ b/studio/backend/routes/managed_engine_chat.py
@@ -35,11 +35,13 @@ class ManagedToolTransport:
preserve_thinking = None,
message_fitter = None,
continuing = False,
+ reply_cap = None,
):
self.client, self.engine, self.body = client, engine, body
self._continuing = continuing
self._preserve_thinking = preserve_thinking
self._message_fitter = message_fitter
+ self._reply_cap = reply_cap
# The tool loop keeps a turn's reasoning_content on the assistant message only when asked.
self.preserves_reasoning = bool(engine.reasoning_parser and preserve_thinking)
@@ -70,6 +72,8 @@ class ManagedToolTransport:
else:
body.pop("tools", None)
body.pop("tool_choice", None)
+ if self._reply_cap is not None:
+ await self._reply_cap(body)
if cancel_event.is_set():
return
async with self.client.stream(
@@ -177,6 +181,15 @@ def _message_fitter(
return fit
+def _has_image(messages) -> bool:
+ return any(
+ isinstance(part, dict) and part.get("type") == "image_url"
+ for message in messages
+ if isinstance(message, dict) and isinstance(message.get("content"), list)
+ for part in message["content"]
+ )
+
+
def _with_model(line: str, public: str) -> str:
if not line.startswith("data:") or '"model"' not in line:
return line
@@ -329,6 +342,23 @@ async def managed_tool_chat(
if response_format is not None:
body["response_format"] = response_format
continuing = api._continue_final_message(payload)
+ thinking = {
+ "enable_thinking": payload.enable_thinking,
+ "reasoning_effort": payload.reasoning_effort,
+ "preserve_thinking": payload.preserve_thinking,
+ }
+
+ async def cap_reply(request_body):
+ # As in ManagedEngine.generate, an image prompt is left to the engine to judge.
+ if request_body.get("max_tokens") and not _has_image(request_body["messages"]):
+ request_body["max_tokens"] = await asyncio.to_thread(
+ engine._reply_cap_in_window,
+ request_body["messages"],
+ "",
+ request_body["max_tokens"],
+ thinking,
+ request_body.get("tools"),
+ )
cancel = api._chat_cancel_event(request)
completion_id = "chatcmpl-" + uuid.uuid4().hex[:12]
@@ -353,6 +383,7 @@ async def managed_tool_chat(
payload, engine, public, tools or payload.tools, studio_loop = bool(tools)
),
continuing = continuing,
+ reply_cap = cap_reply,
)
stripper = api.ServerToolCallStripper()
stream = None
@@ -402,6 +433,7 @@ async def managed_tool_chat(
)
if max_tokens:
request_body["max_tokens"] = max_tokens
+ await cap_reply(request_body)
if continuing:
engine.continue_final_message(request_body)
response = await client.post(
@@ -559,6 +591,11 @@ async def managed_engine_completions(request, current_subject, backend, body):
stream = bool(body.get("stream"))
include_usage = bool((body.get("stream_options") or {}).get("include_usage"))
upstream = {**body, "model": engine.model}
+ limit = body.get("max_tokens")
+ if isinstance(limit, int) and limit > 0 and isinstance(body.get("prompt"), str):
+ upstream["max_tokens"] = await asyncio.to_thread(
+ engine._completion_cap_in_window, body["prompt"], limit
+ )
if stream:
# Always asked for so the monitor counts tokens; dropped again unless the caller opted in.
upstream["stream_options"] = {**(body.get("stream_options") or {}), "include_usage": True}
diff --git a/studio/backend/tests/test_managed_engine_completions.py b/studio/backend/tests/test_managed_engine_completions.py
index aa8cc2ba1cd..ef6e3179432 100644
--- a/studio/backend/tests/test_managed_engine_completions.py
+++ b/studio/backend/tests/test_managed_engine_completions.py
@@ -51,6 +51,7 @@ def _orchestrator(engine = "vllm", alive = True):
headers = {"Authorization": "Bearer private"},
engine = engine,
alive = lambda: alive,
+ _completion_cap_in_window = lambda prompt, limit: limit,
),
)
return backend
@@ -163,6 +164,40 @@ def test_stream_relays_logprobs_and_honours_usage_opt_in(engine, include_usage):
assert engine.monitor.active_count() == 0
[email protected]("stream", [False, True])
+def test_a_completion_cap_is_fitted_to_the_window(engine, monkeypatch, stream):
+ from core.inference import managed_engine
+
+ class _Client:
+ def __init__(self, **kwargs):
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ return False
+
+ def post(
+ self,
+ url,
+ headers = None,
+ json = None,
+ ):
+ assert (url, json["prompt"]) == ("http://engine.test/tokenize", "2+2=")
+ return SimpleNamespace(raise_for_status = lambda: None, json = lambda: {"count": 3000})
+
+ monkeypatch.setattr(managed_engine.httpx, "Client", _Client)
+ managed = managed_engine.ManagedEngine("vllm")
+ managed.model, managed.context, managed.key = "/models/qwen3", 4096, "private"
+ managed.base_url, managed.alive = "http://engine.test", lambda: True
+ engine.backend._managed_engine = managed
+ _call({"prompt": "2+2=", "max_tokens": 2000, "stream": stream})
+ assert engine.requests[-1]["max_tokens"] == 1096
+ _call({"prompt": "2+2=", "max_tokens": 64, "stream": stream})
+ assert engine.requests[-1]["max_tokens"] == 64
+
+
def test_engine_rejection_keeps_its_status(engine):
with pytest.raises(HTTPException) as caught:
_call({"prompt": "bad"})
diff --git a/studio/backend/tests/test_managed_engine_tools.py b/studio/backend/tests/test_managed_engine_tools.py
index 4ed30807cc7..ee0aaf86adf 100644
--- a/studio/backend/tests/test_managed_engine_tools.py
+++ b/studio/backend/tests/test_managed_engine_tools.py
@@ -241,6 +241,52 @@ def test_client_tools_and_history_reach_native_engine(native, stream):
assert api.active_generations.count() == 0
[email protected]("stream", [False, True])
+def test_a_tool_request_cap_is_fitted_to_the_window(native, monkeypatch, stream):
+ from core.inference import managed_engine
+
+ backend, requests = native
+ counted = []
+
+ class _Client:
+ def __init__(self, **kwargs):
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ return False
+
+ def post(
+ self,
+ url,
+ headers = None,
+ json = None,
+ ):
+ counted.append(json)
+ return SimpleNamespace(raise_for_status = lambda: None, json = lambda: {"count": 3000})
+
+ monkeypatch.setattr(managed_engine.httpx, "Client", _Client)
+ engine = managed_engine.ManagedEngine("vllm")
+ engine.model, engine.context, engine.key = "sf-model", 4096, "private"
+ engine.base_url = "http://native.test"
+ backend._managed_engine = engine
+ request = dict(tools = [route_test.LOOKUP_TOOL], stream = stream, enable_tools = False)
+ run(route_test._request(max_tokens = 2000, **request))
+ assert requests[-1]["max_tokens"] == 1096
+ assert counted[-1]["tools"] == [route_test.LOOKUP_TOOL]
+ run(route_test._request(max_tokens = 64, **request))
+ assert requests[-1]["max_tokens"] == 64
+ counts = len(counted)
+ backend.models["sf-model"]["is_vision"] = True
+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}
+ parts = [{"type": "text", "text": "hi"}, image]
+ looking = [route_test.ChatMessage(role = "user", content = parts)]
+ run(route_test._request(max_tokens = 2000, messages = looking, **request))
+ assert requests[-1]["max_tokens"] == 2000 and len(counted) == counts
+
+
def test_studio_loop_executes_and_replays_native_tool_calls(native, monkeypatch):
from core.inference import studio_tool_loop