Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: leave room for the whole reply cap in the estimated managed-engine fit (#15610)

Daniel Hana665aa08ff55
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index bbac9d6f38a..9e2d26c4fe0 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -2844,13 +2844,19 @@ class InferenceOrchestrator:
}
# SGLang 0.5.17's /tokenize returns 500 when model_max_length is the 1e30 sentinel.
estimated = False
+ fit_max_tokens = max_tokens
if getattr(self, "_managed_engine", None) is not None:
try:
self.count_chat_tokens(conversation[-1:], "", **count_kwargs)
except Exception as exc:
logger.info("Fitting the managed engine prompt by estimate: %s", exc)
estimated = True
- context_length = int(context_length * (1 - _ESTIMATED_FIT_HEADROOM))
+ if max_tokens and 0 < max_tokens < context_length:
+ # The cap is sent whole and the bound at most halves the count: fit to half what it leaves.
+ context_length = (context_length - max_tokens) * 2 // 3
+ fit_max_tokens = None
+ else:
+ context_length = int(context_length * (1 - _ESTIMATED_FIT_HEADROOM))
def _count(fitted):
if cancel_event is not None and cancel_event.is_set():
@@ -2888,7 +2894,7 @@ class InferenceOrchestrator:
)
)
fit_kwargs = dict(
- max_tokens = max_tokens,
+ max_tokens = fit_max_tokens,
count_tokens = _count,
anchor_ids = anchor_ids,
keeps_boundary = _keeps_compaction_boundary(thread_id),
@@ -2936,7 +2942,7 @@ class InferenceOrchestrator:
recall_done = recall_done or not truncation["fits"],
recall_budget_tokens = retrieval_budget(
context_length,
- max_tokens,
+ fit_max_tokens,
truncation.get("prompt_tokens_after") or 0,
reply_returns = calls_tools,
),
diff --git a/studio/backend/tests/test_managed_engine_reasoning.py b/studio/backend/tests/test_managed_engine_reasoning.py
index 3c2da3c3059..762714efce5 100644
--- a/studio/backend/tests/test_managed_engine_reasoning.py
+++ b/studio/backend/tests/test_managed_engine_reasoning.py
@@ -378,6 +378,35 @@ def test_a_latest_turn_the_strict_estimate_refuses_still_drops_old_history():
assert len(fitted["messages"]) < len(messages)
+def test_an_estimated_fit_leaves_room_for_the_whole_reply_cap():
+ # The cap is sent unchanged when the count fails, so prompt + cap must fit the real window.
+ import json
+
+ from core.inference.orchestrator import InferenceOrchestrator
+
+ orchestrator = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ orchestrator.active_model_name = "m"
+ orchestrator.models = {"m": {"engine": "sglang", "context_length": 2048}}
+ orchestrator._managed_engine = SimpleNamespace(adapter = ADAPTERS["sglang"])
+
+ def count(*args, **kwargs):
+ raise RuntimeError("500 Server Error: Integer exceeds 64-bit range")
+
+ orchestrator.count_chat_tokens = count
+ messages = []
+ for index in range(10):
+ messages += [
+ {"role": "user", "content": f"question {index}: " + "1,2,3,4,5,6,7,8,9,0," * 70},
+ {"role": "assistant", "content": "ok"},
+ ]
+ messages.append({"role": "user", "content": "Continue."})
+ fitted = orchestrator.compact_chat_context(
+ messages, context_overflow = "truncate_oldest", max_tokens = 1024
+ )
+ # A token spans at least one UTF-8 byte, so the serialized bytes bound the real count.
+ assert sum(len(json.dumps(m).encode()) for m in fitted["messages"]) + 1024 <= 2048
+
+
def test_tool_stream_reasoning_reaches_studio_as_reasoning_content():
from routes.managed_engine_chat import _with_reasoning_content