Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: leave room for the whole reply cap in the estimated managed-engine fit (#15610)
Daniel Hana665aa08ff55
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex bbac9d6f38a..9e2d26c4fe0 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -2844,13 +2844,19 @@ class InferenceOrchestrator:}# SGLang 0.5.17's /tokenize returns 500 when model_max_length is the 1e30 sentinel.estimated = False+ fit_max_tokens = max_tokensif getattr(self, "_managed_engine", None) is not None:try:self.count_chat_tokens(conversation[-1:], "", **count_kwargs)except Exception as exc:logger.info("Fitting the managed engine prompt by estimate: %s", exc)estimated = True- context_length = int(context_length * (1 - _ESTIMATED_FIT_HEADROOM))+ if max_tokens and 0 < max_tokens < context_length:+ # The cap is sent whole and the bound at most halves the count: fit to half what it leaves.+ context_length = (context_length - max_tokens) * 2 // 3+ fit_max_tokens = None+ else:+ context_length = int(context_length * (1 - _ESTIMATED_FIT_HEADROOM))def _count(fitted):if cancel_event is not None and cancel_event.is_set():@@ -2888,7 +2894,7 @@ class InferenceOrchestrator:))fit_kwargs = dict(- max_tokens = max_tokens,+ max_tokens = fit_max_tokens,count_tokens = _count,anchor_ids = anchor_ids,keeps_boundary = _keeps_compaction_boundary(thread_id),@@ -2936,7 +2942,7 @@ class InferenceOrchestrator:recall_done = recall_done or not truncation["fits"],recall_budget_tokens = retrieval_budget(context_length,- max_tokens,+ fit_max_tokens,truncation.get("prompt_tokens_after") or 0,reply_returns = calls_tools,),diff --git a/studio/backend/tests/test_managed_engine_reasoning.py b/studio/backend/tests/test_managed_engine_reasoning.pyindex 3c2da3c3059..762714efce5 100644--- a/studio/backend/tests/test_managed_engine_reasoning.py+++ b/studio/backend/tests/test_managed_engine_reasoning.py@@ -378,6 +378,35 @@ def test_a_latest_turn_the_strict_estimate_refuses_still_drops_old_history():assert len(fitted["messages"]) < len(messages)+def test_an_estimated_fit_leaves_room_for_the_whole_reply_cap():+ # The cap is sent unchanged when the count fails, so prompt + cap must fit the real window.+ import json++ from core.inference.orchestrator import InferenceOrchestrator++ orchestrator = InferenceOrchestrator.__new__(InferenceOrchestrator)+ orchestrator.active_model_name = "m"+ orchestrator.models = {"m": {"engine": "sglang", "context_length": 2048}}+ orchestrator._managed_engine = SimpleNamespace(adapter = ADAPTERS["sglang"])++ def count(*args, **kwargs):+ raise RuntimeError("500 Server Error: Integer exceeds 64-bit range")++ orchestrator.count_chat_tokens = count+ messages = []+ for index in range(10):+ messages += [+ {"role": "user", "content": f"question {index}: " + "1,2,3,4,5,6,7,8,9,0," * 70},+ {"role": "assistant", "content": "ok"},+ ]+ messages.append({"role": "user", "content": "Continue."})+ fitted = orchestrator.compact_chat_context(+ messages, context_overflow = "truncate_oldest", max_tokens = 1024+ )+ # A token spans at least one UTF-8 byte, so the serialized bytes bound the real count.+ assert sum(len(json.dumps(m).encode()) for m in fitted["messages"]) + 1024 <= 2048++def test_tool_stream_reasoning_reaches_studio_as_reasoning_content():from routes.managed_engine_chat import _with_reasoning_content