Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: say a chat is too long on safetensors models (#15616)
Nilay02a98b053ccf
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex 2735d5b0341..791cfce1308 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -30,6 +30,7 @@ from utils.hardware import (get_visible_gpu_count,)from core.inference.audio_codecs import AudioCodecManager+from core.inference.context_refusal import ContextBudgetExceededfrom core.inference.runtime_context import (generation_budget_within_context,runtime_context_length,@@ -623,6 +624,14 @@ class _StopSequenceStreamer:abort()+# LlamaModel_fast_forward cuts input_ids at max_seq_length, so decoding past it crashes; FastModel's stock forward does not.+def _truncates_past_window(model) -> bool:+ return any(+ getattr(getattr(type(module), "forward", None), "__name__", "") == "LlamaModel_fast_forward"+ for module in model.modules()+ )++def _prompt_already_has_bos(tokenizer, prompt):"""Most templates render BOS, so no second one is added; zephyr and tinyllama-chat do not."""tok = getattr(tokenizer, "tokenizer", tokenizer)@@ -2738,6 +2747,12 @@ class InferenceBackend:)prompt_len = int(inputs["input_ids"].shape[1])+ window = runtime_context_length(model)+ if window and prompt_len > window:+ if "truncates_past_window" not in model_info:+ model_info["truncates_past_window"] = _truncates_past_window(model)+ if model_info["truncates_past_window"]:+ raise ContextBudgetExceeded(prompt_len, window)max_new_tokens = generation_budget_within_context(model, prompt_len, max_new_tokens)generation_kwargs = dict(diff --git a/studio/backend/tests/test_generation_budget.py b/studio/backend/tests/test_generation_budget.pyindex d770d90dcb2..1bd692ee5c6 100644--- a/studio/backend/tests/test_generation_budget.py+++ b/studio/backend/tests/test_generation_budget.py@@ -108,14 +108,36 @@ class _FakeTokenizer:return _FakeEncoding(input_ids = _FakeTensor(_PROMPT_LEN))+class _FastDecoder:+ def forward(self):+ pass+++_FastDecoder.forward.__name__ = "LlamaModel_fast_forward"+++class _StockDecoder:+ def forward(self):+ pass++class _FakeModel:"""Carries the one guard unsloth_fast_generate applies before generating."""- def __init__(self, window):+ def __init__(+ self,+ window,+ decoder = _FastDecoder,+ native = None,+ ):self.max_seq_length = window- self.config = SimpleNamespace(max_position_embeddings = window)+ self.config = SimpleNamespace(max_position_embeddings = native or window)self.device = "cpu"self.calls = []+ self._decoder = decoder()++ def modules(self):+ return iter((self, self._decoder))def generate(self, **kwargs):ids, budget = kwargs.get("input_ids"), kwargs.get("max_new_tokens")@@ -141,7 +163,11 @@ class _EmptyStreamer:pass-def _streaming_backend(monkeypatch, window = _WINDOW):+def _streaming_backend(+ monkeypatch,+ window = _WINDOW,+ **model_kwargs,+):try:from core.inference.inference import InferenceBackendexcept (ImportError, RuntimeError) as exc: # pragma: no cover - env-dependent@@ -152,7 +178,7 @@ def _streaming_backend(monkeypatch, window = _WINDOW):backend.active_model_name = "unsloth/tinyllama-chat-bnb-4bit"backend.last_generation_stats = Nonebackend._generation_lock = threading.Lock()- model = _FakeModel(window)+ model = _FakeModel(window, **model_kwargs)backend.models = {backend.active_model_name: {"model": model,@@ -195,6 +221,31 @@ def test_generate_stream_passes_a_fitting_budget_through(monkeypatch):assert model.calls[0]["max_new_tokens"] == 512+def test_generate_stream_refuses_a_prompt_longer_than_the_window(monkeypatch):+ from core.inference.context_refusal import ContextBudgetExceeded++ backend, model = _streaming_backend(monkeypatch, window = _PROMPT_LEN - 1)++ with pytest.raises(ContextBudgetExceeded) as refused:+ _run(backend, 64)++ assert (refused.value.request_tokens, refused.value.context_tokens) == (+ _PROMPT_LEN,+ _PROMPT_LEN - 1,+ )+ assert not model.calls+++def test_generate_stream_lets_a_stock_forward_model_run_past_the_window(monkeypatch):+ backend, model = _streaming_backend(+ monkeypatch, window = _PROMPT_LEN - 1, decoder = _StockDecoder, native = 8192+ )++ _run(backend, 64)++ assert model.calls[0]["max_new_tokens"] == 64++def _vision_backend(monkeypatch, window = _WINDOW):try:from core.inference.inference import InferenceBackend