Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: fit max_tokens to the window on vLLM and SGLang tool requests (#17426)
Nilayb347d06cf4b2
diff --git a/studio/backend/core/inference/managed_engine.py b/studio/backend/core/inference/managed_engine.pyindex 9e292a28e8a..f3dd94d0f4c 100644--- a/studio/backend/core/inference/managed_engine.py+++ b/studio/backend/core/inference/managed_engine.py@@ -1128,12 +1128,20 @@ class ManagedEngine:for message in messages]- def _reply_cap_in_window(self, messages, system_prompt, limit, params):+ def _reply_cap_in_window(+ self,+ messages,+ system_prompt,+ limit,+ params,+ tools = None,+ ):"""vLLM and SGLang refuse prompt + max_tokens past the window, where llama-server stops at it."""try:prompt = self.count_tokens(messages,system_prompt,+ tools = tools,enable_thinking = params.get("enable_thinking"),reasoning_effort = params.get("reasoning_effort"),preserve_thinking = params.get("preserve_thinking"),@@ -1143,14 +1151,28 @@ class ManagedEngine:return limitreturn max(1, min(limit, self.context - prompt))+ def _completion_cap_in_window(self, prompt, limit):+ try:+ return max(1, min(limit, self.context - self._tokenize({"prompt": prompt})))+ except Exception:+ return limit++ def _tokenize(self, body):+ with httpx.Client(trust_env = False, timeout = 30) as client:+ response = client.post(+ self.base_url + "/tokenize",+ headers = self.headers,+ json = {"model": self.model, **body},+ )+ response.raise_for_status()+ return int(response.json()["count"])+def count_tokens(self,messages,system_prompt = "",**kwargs,):- if kwargs.get("tools"):- raise ValueError("Tools are unavailable for this optional engine profile.")if not self.adapter.exact_token_count:raise RuntimeError("Exact prompt token counting is unavailable for this engine.")if system_prompt:@@ -1167,19 +1189,14 @@ class ManagedEngine:template_kwargs = fields["chat_template_kwargs"]template_kwargs["reasoning_effort"] = efforttemplate_kwargs.setdefault("enable_thinking", effort != "none")- with httpx.Client(trust_env = False, timeout = 30) as client:- response = client.post(- self.base_url + "/tokenize",- headers = self.headers,- json = {- "model": self.model,- "messages": self.replay_reasoning(messages, kwargs.get("preserve_thinking")),- "add_generation_prompt": True,- **fields,- },- )- response.raise_for_status()- return int(response.json()["count"]), self.model+ body = {+ "messages": self.replay_reasoning(messages, kwargs.get("preserve_thinking")),+ "add_generation_prompt": True,+ **fields,+ }+ if kwargs.get("tools"):+ body["tools"] = kwargs["tools"]+ return self._tokenize(body), self.modeldef generate(self,diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 8f5d7ded25d..188eb2175f9 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -3050,10 +3050,11 @@ class InferenceOrchestrator:instead."""managed = getattr(self, "_managed_engine", None)if managed is not None:+ if tools:+ raise ValueError("Tools are unavailable for this optional engine profile.")return managed.count_tokens(messages,system_prompt,- tools = tools,enable_thinking = enable_thinking,reasoning_effort = reasoning_effort,preserve_thinking = preserve_thinking,diff --git a/studio/backend/routes/managed_engine_chat.py b/studio/backend/routes/managed_engine_chat.pyindex 16288dab925..71f57d1d797 100644--- a/studio/backend/routes/managed_engine_chat.py+++ b/studio/backend/routes/managed_engine_chat.py@@ -35,11 +35,13 @@ class ManagedToolTransport:preserve_thinking = None,message_fitter = None,continuing = False,+ reply_cap = None,):self.client, self.engine, self.body = client, engine, bodyself._continuing = continuingself._preserve_thinking = preserve_thinkingself._message_fitter = message_fitter+ self._reply_cap = reply_cap# The tool loop keeps a turn's reasoning_content on the assistant message only when asked.self.preserves_reasoning = bool(engine.reasoning_parser and preserve_thinking)@@ -70,6 +72,8 @@ class ManagedToolTransport:else:body.pop("tools", None)body.pop("tool_choice", None)+ if self._reply_cap is not None:+ await self._reply_cap(body)if cancel_event.is_set():returnasync with self.client.stream(@@ -177,6 +181,15 @@ def _message_fitter(return fit+def _has_image(messages) -> bool:+ return any(+ isinstance(part, dict) and part.get("type") == "image_url"+ for message in messages+ if isinstance(message, dict) and isinstance(message.get("content"), list)+ for part in message["content"]+ )++def _with_model(line: str, public: str) -> str:if not line.startswith("data:") or '"model"' not in line:return line@@ -329,6 +342,23 @@ async def managed_tool_chat(if response_format is not None:body["response_format"] = response_formatcontinuing = api._continue_final_message(payload)+ thinking = {+ "enable_thinking": payload.enable_thinking,+ "reasoning_effort": payload.reasoning_effort,+ "preserve_thinking": payload.preserve_thinking,+ }++ async def cap_reply(request_body):+ # As in ManagedEngine.generate, an image prompt is left to the engine to judge.+ if request_body.get("max_tokens") and not _has_image(request_body["messages"]):+ request_body["max_tokens"] = await asyncio.to_thread(+ engine._reply_cap_in_window,+ request_body["messages"],+ "",+ request_body["max_tokens"],+ thinking,+ request_body.get("tools"),+ )cancel = api._chat_cancel_event(request)completion_id = "chatcmpl-" + uuid.uuid4().hex[:12]@@ -353,6 +383,7 @@ async def managed_tool_chat(payload, engine, public, tools or payload.tools, studio_loop = bool(tools)),continuing = continuing,+ reply_cap = cap_reply,)stripper = api.ServerToolCallStripper()stream = None@@ -402,6 +433,7 @@ async def managed_tool_chat()if max_tokens:request_body["max_tokens"] = max_tokens+ await cap_reply(request_body)if continuing:engine.continue_final_message(request_body)response = await client.post(@@ -559,6 +591,11 @@ async def managed_engine_completions(request, current_subject, backend, body):stream = bool(body.get("stream"))include_usage = bool((body.get("stream_options") or {}).get("include_usage"))upstream = {**body, "model": engine.model}+ limit = body.get("max_tokens")+ if isinstance(limit, int) and limit > 0 and isinstance(body.get("prompt"), str):+ upstream["max_tokens"] = await asyncio.to_thread(+ engine._completion_cap_in_window, body["prompt"], limit+ )if stream:# Always asked for so the monitor counts tokens; dropped again unless the caller opted in.upstream["stream_options"] = {**(body.get("stream_options") or {}), "include_usage": True}diff --git a/studio/backend/tests/test_managed_engine_completions.py b/studio/backend/tests/test_managed_engine_completions.pyindex aa8cc2ba1cd..ef6e3179432 100644--- a/studio/backend/tests/test_managed_engine_completions.py+++ b/studio/backend/tests/test_managed_engine_completions.py@@ -51,6 +51,7 @@ def _orchestrator(engine = "vllm", alive = True):headers = {"Authorization": "Bearer private"},engine = engine,alive = lambda: alive,+ _completion_cap_in_window = lambda prompt, limit: limit,),)return backend@@ -163,6 +164,40 @@ def test_stream_relays_logprobs_and_honours_usage_opt_in(engine, include_usage):assert engine.monitor.active_count() == 0[email protected]("stream", [False, True])+def test_a_completion_cap_is_fitted_to_the_window(engine, monkeypatch, stream):+ from core.inference import managed_engine++ class _Client:+ def __init__(self, **kwargs):+ pass++ def __enter__(self):+ return self++ def __exit__(self, *exc):+ return False++ def post(+ self,+ url,+ headers = None,+ json = None,+ ):+ assert (url, json["prompt"]) == ("http://engine.test/tokenize", "2+2=")+ return SimpleNamespace(raise_for_status = lambda: None, json = lambda: {"count": 3000})++ monkeypatch.setattr(managed_engine.httpx, "Client", _Client)+ managed = managed_engine.ManagedEngine("vllm")+ managed.model, managed.context, managed.key = "/models/qwen3", 4096, "private"+ managed.base_url, managed.alive = "http://engine.test", lambda: True+ engine.backend._managed_engine = managed+ _call({"prompt": "2+2=", "max_tokens": 2000, "stream": stream})+ assert engine.requests[-1]["max_tokens"] == 1096+ _call({"prompt": "2+2=", "max_tokens": 64, "stream": stream})+ assert engine.requests[-1]["max_tokens"] == 64++def test_engine_rejection_keeps_its_status(engine):with pytest.raises(HTTPException) as caught:_call({"prompt": "bad"})diff --git a/studio/backend/tests/test_managed_engine_tools.py b/studio/backend/tests/test_managed_engine_tools.pyindex 4ed30807cc7..ee0aaf86adf 100644--- a/studio/backend/tests/test_managed_engine_tools.py+++ b/studio/backend/tests/test_managed_engine_tools.py@@ -241,6 +241,52 @@ def test_client_tools_and_history_reach_native_engine(native, stream):assert api.active_generations.count() == 0[email protected]("stream", [False, True])+def test_a_tool_request_cap_is_fitted_to_the_window(native, monkeypatch, stream):+ from core.inference import managed_engine++ backend, requests = native+ counted = []++ class _Client:+ def __init__(self, **kwargs):+ pass++ def __enter__(self):+ return self++ def __exit__(self, *exc):+ return False++ def post(+ self,+ url,+ headers = None,+ json = None,+ ):+ counted.append(json)+ return SimpleNamespace(raise_for_status = lambda: None, json = lambda: {"count": 3000})++ monkeypatch.setattr(managed_engine.httpx, "Client", _Client)+ engine = managed_engine.ManagedEngine("vllm")+ engine.model, engine.context, engine.key = "sf-model", 4096, "private"+ engine.base_url = "http://native.test"+ backend._managed_engine = engine+ request = dict(tools = [route_test.LOOKUP_TOOL], stream = stream, enable_tools = False)+ run(route_test._request(max_tokens = 2000, **request))+ assert requests[-1]["max_tokens"] == 1096+ assert counted[-1]["tools"] == [route_test.LOOKUP_TOOL]+ run(route_test._request(max_tokens = 64, **request))+ assert requests[-1]["max_tokens"] == 64+ counts = len(counted)+ backend.models["sf-model"]["is_vision"] = True+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}+ parts = [{"type": "text", "text": "hi"}, image]+ looking = [route_test.ChatMessage(role = "user", content = parts)]+ run(route_test._request(max_tokens = 2000, messages = looking, **request))+ assert requests[-1]["max_tokens"] == 2000 and len(counted) == counts++def test_studio_loop_executes_and_replays_native_tool_calls(native, monkeypatch):from core.inference import studio_tool_loop