Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: refuse MLX prompts longer than the served context (#16821)
Long Yixing692fcd6bd3af
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex dddea6c68ea..2ef224d898a 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -4821,6 +4821,8 @@ class _TextBatchSession:prefix, remainder, cache_key = all_tokens[:-1], all_tokens[-1:], Noneif row_kwargs["max_new_tokens"] is None:plan.max_tokens = backend._unset_generation_budget(plan.prompt, all_tokens)+ else:+ backend._prompt_within_window(plan.prompt, all_tokens)row = _TextBatchRow(handle = handle,plan = plan,@@ -5716,6 +5718,20 @@ class MLXInferenceBackend:videos = None,):"""Free context for an unset limit, or the default if the prompt cannot be counted."""+ prompt_n = self._prompt_within_window(prompt, prompt_tokens, images, audio, videos)+ if prompt_n is None:+ return UNSET_GENERATION_BUDGET+ return generation_budget_for_window(self._served_context, prompt_n, None)++ def _prompt_within_window(+ self,+ prompt,+ prompt_tokens = None,+ images = None,+ audio = None,+ videos = None,+ ):+ """The prompt's length, refused when it leaves no room in the served context; None if uncountable."""try:prompt_n = (len(prompt_tokens)@@ -5723,9 +5739,13 @@ class MLXInferenceBackend:else self._count_prompt_tokens(prompt, images, audio, videos))except Exception as exc:- logger.debug("MLX prompt count for an unset budget failed: %s", exc)- return UNSET_GENERATION_BUDGET- return generation_budget_for_window(self._served_context, prompt_n, None)+ logger.debug("MLX prompt count failed: %s", exc)+ return None+ window = self._served_context+ # Admitted, a bounded cache would rotate the prompt's start away during prefill.+ if window and prompt_n >= int(window):+ raise context_refusal.ContextBudgetExceeded(prompt_n + 1, int(window))+ return prompt_ndef _generation_limit(self,@@ -5746,6 +5766,8 @@ class MLXInferenceBackend:max_new_tokens = self._unset_generation_budget(prompt, prompt_tokens)if cap is not None:max_new_tokens = min(max_new_tokens, cap)+ else:+ self._prompt_within_window(prompt, prompt_tokens)return max_new_tokens# Max Tokens == window is the UI's "no cap"; admitting it as-is would refuse every prompt.if max_new_tokens is not None and int(max_new_tokens) == int(budget):@@ -7684,6 +7706,11 @@ class MLXInferenceBackend:),),)+ if max_new_tokens is None:+ max_tokens = self._unset_generation_budget(prompt, prompt_ids)+ else:+ self._prompt_within_window(prompt, prompt_ids)+ max_tokens = max_new_tokensrow = _RowStream(prefix = think_prefix,sequences = _mlx_stop_sequences(stop),@@ -7697,11 +7724,7 @@ class MLXInferenceBackend:streamed = streamed,sampler = sampler,processors = logits_processors,- max_tokens = (- self._unset_generation_budget(prompt, prompt_ids)- if max_new_tokens is None- else max_new_tokens- ),+ max_tokens = max_tokens,)def _render_vlm_prompt(diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex d72cfd9ce40..69466c70c94 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -3602,6 +3602,48 @@ def test_an_image_row_through_mlx_lm_joins_the_batch_at_its_last_token(monkeypatsession.close()[email protected]("limit", [None, 16])+def test_an_image_row_through_mlx_lm_is_refused_when_its_expanded_prompt_fills_the_window(+ monkeypatch, limit+):+ from core.inference import mlx_inference+ from core.inference.context_refusal import ContextBudgetExceeded++ inserted = []+ generator = SimpleNamespace(+ insert = lambda prompts, **kw: inserted.append(kw) or [len(inserted)],+ remove = lambda *_a, **_k: {},+ close = lambda: None,+ )+ generate = types.ModuleType("mlx_lm.generate")+ generate.BatchGenerator = lambda *_a, **_k: generator+ _zoo_image_prefill(monkeypatch, prefill_image_prompt = lambda *_a: [1, 5, 5, 2])+ backend = mlx_inference.MLXInferenceBackend()+ backend._model, backend._tokenizer = SimpleNamespace(), SimpleNamespace(eos_token_ids = [2])+ backend._is_vlm, backend._reads_vision = False, True+ backend._make_row_cache = lambda: ["cache"]+ plan = SimpleNamespace(+ prompt = "P", max_tokens = limit, sampler = None, processors = (), streamed = None, think_prefix = ""+ )+ backend._plan_text_row = lambda messages, **_kw: plan+ with monkeypatch.context() as patch:+ patch.setitem(sys.modules, "mlx_lm.generate", generate)+ session = mlx_inference._TextBatchSession(backend, width = 2)+ request = {+ "messages": [{"role": "user", "content": "hi"}],+ "image": object(),+ "max_new_tokens": limit,+ }+ backend._served_context = 4+ with pytest.raises(ContextBudgetExceeded):+ session.admit(request, 0)+ assert inserted == []+ backend._served_context = 5+ session.admit(request, 1)+ assert [kw["max_tokens"] for kw in inserted] == [[limit or 1]]+ session.close()++@pytest.mark.parametrize("ends", [None, "length", "stop"])def test_a_row_whose_own_processor_fails_leaves_and_its_neighbour_decodes_on(monkeypatch, ends):import numpy@@ -11114,6 +11156,37 @@ def test_mlx_passes_an_explicit_budget_through_both_paths(monkeypatch):assert _run_vlm_budget(monkeypatch, _IMAGE_TURN, object(), 4096) == 4096[email protected]("limit", [None, 16])+def test_mlx_refuses_a_prompt_that_fills_the_served_window(monkeypatch, limit):+ from core.inference.context_refusal import ContextBudgetExceeded++ fits = _BUDGET_PROMPT_N + 1+ assert _run_text_budget(monkeypatch, limit, served = fits) == (limit or 1)+ assert _run_vlm_budget(monkeypatch, _TEXT_TURN, None, limit, served = fits) == (limit or 1)+ with pytest.raises(ContextBudgetExceeded) as refused:+ _run_text_budget(monkeypatch, limit, served = _BUDGET_PROMPT_N)+ assert (refused.value.request_tokens, refused.value.context_tokens) == (fits, _BUDGET_PROMPT_N)+ with pytest.raises(ContextBudgetExceeded):+ _run_vlm_budget(monkeypatch, _TEXT_TURN, None, limit, served = _BUDGET_PROMPT_N)+ with pytest.raises(ContextBudgetExceeded):+ _run_text_budget(monkeypatch, limit, served = 300, cached = list(range(300)))++[email protected]("limit", [None, 16])+def test_mlx_batch_rows_refuse_a_prompt_that_fills_the_served_window(monkeypatch, limit):+ from core.inference import mlx_inference+ from core.inference.context_refusal import ContextBudgetExceeded++ backend = _install_fake_text_stack(monkeypatch, {"generated": [7]}, [])+ monkeypatch.setattr(mlx_inference, "_mlx_sampling_processors", lambda **_kw: None)+ row = mlx_inference._row_kwargs({"prompt_ids": [1, 2, 3], "max_new_tokens": limit})+ backend._served_context = 4+ assert backend._plan_text_row([], **row).max_tokens == (limit or 1)+ backend._served_context = 3+ with pytest.raises(ContextBudgetExceeded):+ backend._plan_text_row([], **row)++def test_mlx_unset_budget_falls_back_when_the_prompt_cannot_be_counted(monkeypatch):from core.inference.runtime_context import UNSET_GENERATION_BUDGET