Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: say a chat is too long on safetensors models (#15616)

Nilay02a98b053ccf
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 2735d5b0341..791cfce1308 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -30,6 +30,7 @@ from utils.hardware import (
get_visible_gpu_count,
)
from core.inference.audio_codecs import AudioCodecManager
+from core.inference.context_refusal import ContextBudgetExceeded
from core.inference.runtime_context import (
generation_budget_within_context,
runtime_context_length,
@@ -623,6 +624,14 @@ class _StopSequenceStreamer:
abort()
+# LlamaModel_fast_forward cuts input_ids at max_seq_length, so decoding past it crashes; FastModel's stock forward does not.
+def _truncates_past_window(model) -> bool:
+ return any(
+ getattr(getattr(type(module), "forward", None), "__name__", "") == "LlamaModel_fast_forward"
+ for module in model.modules()
+ )
+
+
def _prompt_already_has_bos(tokenizer, prompt):
"""Most templates render BOS, so no second one is added; zephyr and tinyllama-chat do not."""
tok = getattr(tokenizer, "tokenizer", tokenizer)
@@ -2738,6 +2747,12 @@ class InferenceBackend:
)
prompt_len = int(inputs["input_ids"].shape[1])
+ window = runtime_context_length(model)
+ if window and prompt_len > window:
+ if "truncates_past_window" not in model_info:
+ model_info["truncates_past_window"] = _truncates_past_window(model)
+ if model_info["truncates_past_window"]:
+ raise ContextBudgetExceeded(prompt_len, window)
max_new_tokens = generation_budget_within_context(model, prompt_len, max_new_tokens)
generation_kwargs = dict(
diff --git a/studio/backend/tests/test_generation_budget.py b/studio/backend/tests/test_generation_budget.py
index d770d90dcb2..1bd692ee5c6 100644
--- a/studio/backend/tests/test_generation_budget.py
+++ b/studio/backend/tests/test_generation_budget.py
@@ -108,14 +108,36 @@ class _FakeTokenizer:
return _FakeEncoding(input_ids = _FakeTensor(_PROMPT_LEN))
+class _FastDecoder:
+ def forward(self):
+ pass
+
+
+_FastDecoder.forward.__name__ = "LlamaModel_fast_forward"
+
+
+class _StockDecoder:
+ def forward(self):
+ pass
+
+
class _FakeModel:
"""Carries the one guard unsloth_fast_generate applies before generating."""
- def __init__(self, window):
+ def __init__(
+ self,
+ window,
+ decoder = _FastDecoder,
+ native = None,
+ ):
self.max_seq_length = window
- self.config = SimpleNamespace(max_position_embeddings = window)
+ self.config = SimpleNamespace(max_position_embeddings = native or window)
self.device = "cpu"
self.calls = []
+ self._decoder = decoder()
+
+ def modules(self):
+ return iter((self, self._decoder))
def generate(self, **kwargs):
ids, budget = kwargs.get("input_ids"), kwargs.get("max_new_tokens")
@@ -141,7 +163,11 @@ class _EmptyStreamer:
pass
-def _streaming_backend(monkeypatch, window = _WINDOW):
+def _streaming_backend(
+ monkeypatch,
+ window = _WINDOW,
+ **model_kwargs,
+):
try:
from core.inference.inference import InferenceBackend
except (ImportError, RuntimeError) as exc: # pragma: no cover - env-dependent
@@ -152,7 +178,7 @@ def _streaming_backend(monkeypatch, window = _WINDOW):
backend.active_model_name = "unsloth/tinyllama-chat-bnb-4bit"
backend.last_generation_stats = None
backend._generation_lock = threading.Lock()
- model = _FakeModel(window)
+ model = _FakeModel(window, **model_kwargs)
backend.models = {
backend.active_model_name: {
"model": model,
@@ -195,6 +221,31 @@ def test_generate_stream_passes_a_fitting_budget_through(monkeypatch):
assert model.calls[0]["max_new_tokens"] == 512
+def test_generate_stream_refuses_a_prompt_longer_than_the_window(monkeypatch):
+ from core.inference.context_refusal import ContextBudgetExceeded
+
+ backend, model = _streaming_backend(monkeypatch, window = _PROMPT_LEN - 1)
+
+ with pytest.raises(ContextBudgetExceeded) as refused:
+ _run(backend, 64)
+
+ assert (refused.value.request_tokens, refused.value.context_tokens) == (
+ _PROMPT_LEN,
+ _PROMPT_LEN - 1,
+ )
+ assert not model.calls
+
+
+def test_generate_stream_lets_a_stock_forward_model_run_past_the_window(monkeypatch):
+ backend, model = _streaming_backend(
+ monkeypatch, window = _PROMPT_LEN - 1, decoder = _StockDecoder, native = 8192
+ )
+
+ _run(backend, 64)
+
+ assert model.calls[0]["max_new_tokens"] == 64
+
+
def _vision_backend(monkeypatch, window = _WINDOW):
try:
from core.inference.inference import InferenceBackend