Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: refuse MLX prompts longer than the served context (#16821)

Long Yixing692fcd6bd3af
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index dddea6c68ea..2ef224d898a 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -4821,6 +4821,8 @@ class _TextBatchSession:
prefix, remainder, cache_key = all_tokens[:-1], all_tokens[-1:], None
if row_kwargs["max_new_tokens"] is None:
plan.max_tokens = backend._unset_generation_budget(plan.prompt, all_tokens)
+ else:
+ backend._prompt_within_window(plan.prompt, all_tokens)
row = _TextBatchRow(
handle = handle,
plan = plan,
@@ -5716,6 +5718,20 @@ class MLXInferenceBackend:
videos = None,
):
"""Free context for an unset limit, or the default if the prompt cannot be counted."""
+ prompt_n = self._prompt_within_window(prompt, prompt_tokens, images, audio, videos)
+ if prompt_n is None:
+ return UNSET_GENERATION_BUDGET
+ return generation_budget_for_window(self._served_context, prompt_n, None)
+
+ def _prompt_within_window(
+ self,
+ prompt,
+ prompt_tokens = None,
+ images = None,
+ audio = None,
+ videos = None,
+ ):
+ """The prompt's length, refused when it leaves no room in the served context; None if uncountable."""
try:
prompt_n = (
len(prompt_tokens)
@@ -5723,9 +5739,13 @@ class MLXInferenceBackend:
else self._count_prompt_tokens(prompt, images, audio, videos)
)
except Exception as exc:
- logger.debug("MLX prompt count for an unset budget failed: %s", exc)
- return UNSET_GENERATION_BUDGET
- return generation_budget_for_window(self._served_context, prompt_n, None)
+ logger.debug("MLX prompt count failed: %s", exc)
+ return None
+ window = self._served_context
+ # Admitted, a bounded cache would rotate the prompt's start away during prefill.
+ if window and prompt_n >= int(window):
+ raise context_refusal.ContextBudgetExceeded(prompt_n + 1, int(window))
+ return prompt_n
def _generation_limit(
self,
@@ -5746,6 +5766,8 @@ class MLXInferenceBackend:
max_new_tokens = self._unset_generation_budget(prompt, prompt_tokens)
if cap is not None:
max_new_tokens = min(max_new_tokens, cap)
+ else:
+ self._prompt_within_window(prompt, prompt_tokens)
return max_new_tokens
# Max Tokens == window is the UI's "no cap"; admitting it as-is would refuse every prompt.
if max_new_tokens is not None and int(max_new_tokens) == int(budget):
@@ -7684,6 +7706,11 @@ class MLXInferenceBackend:
),
),
)
+ if max_new_tokens is None:
+ max_tokens = self._unset_generation_budget(prompt, prompt_ids)
+ else:
+ self._prompt_within_window(prompt, prompt_ids)
+ max_tokens = max_new_tokens
row = _RowStream(
prefix = think_prefix,
sequences = _mlx_stop_sequences(stop),
@@ -7697,11 +7724,7 @@ class MLXInferenceBackend:
streamed = streamed,
sampler = sampler,
processors = logits_processors,
- max_tokens = (
- self._unset_generation_budget(prompt, prompt_ids)
- if max_new_tokens is None
- else max_new_tokens
- ),
+ max_tokens = max_tokens,
)
def _render_vlm_prompt(
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index d72cfd9ce40..69466c70c94 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -3602,6 +3602,48 @@ def test_an_image_row_through_mlx_lm_joins_the_batch_at_its_last_token(monkeypat
session.close()
[email protected]("limit", [None, 16])
+def test_an_image_row_through_mlx_lm_is_refused_when_its_expanded_prompt_fills_the_window(
+ monkeypatch, limit
+):
+ from core.inference import mlx_inference
+ from core.inference.context_refusal import ContextBudgetExceeded
+
+ inserted = []
+ generator = SimpleNamespace(
+ insert = lambda prompts, **kw: inserted.append(kw) or [len(inserted)],
+ remove = lambda *_a, **_k: {},
+ close = lambda: None,
+ )
+ generate = types.ModuleType("mlx_lm.generate")
+ generate.BatchGenerator = lambda *_a, **_k: generator
+ _zoo_image_prefill(monkeypatch, prefill_image_prompt = lambda *_a: [1, 5, 5, 2])
+ backend = mlx_inference.MLXInferenceBackend()
+ backend._model, backend._tokenizer = SimpleNamespace(), SimpleNamespace(eos_token_ids = [2])
+ backend._is_vlm, backend._reads_vision = False, True
+ backend._make_row_cache = lambda: ["cache"]
+ plan = SimpleNamespace(
+ prompt = "P", max_tokens = limit, sampler = None, processors = (), streamed = None, think_prefix = ""
+ )
+ backend._plan_text_row = lambda messages, **_kw: plan
+ with monkeypatch.context() as patch:
+ patch.setitem(sys.modules, "mlx_lm.generate", generate)
+ session = mlx_inference._TextBatchSession(backend, width = 2)
+ request = {
+ "messages": [{"role": "user", "content": "hi"}],
+ "image": object(),
+ "max_new_tokens": limit,
+ }
+ backend._served_context = 4
+ with pytest.raises(ContextBudgetExceeded):
+ session.admit(request, 0)
+ assert inserted == []
+ backend._served_context = 5
+ session.admit(request, 1)
+ assert [kw["max_tokens"] for kw in inserted] == [[limit or 1]]
+ session.close()
+
+
@pytest.mark.parametrize("ends", [None, "length", "stop"])
def test_a_row_whose_own_processor_fails_leaves_and_its_neighbour_decodes_on(monkeypatch, ends):
import numpy
@@ -11114,6 +11156,37 @@ def test_mlx_passes_an_explicit_budget_through_both_paths(monkeypatch):
assert _run_vlm_budget(monkeypatch, _IMAGE_TURN, object(), 4096) == 4096
[email protected]("limit", [None, 16])
+def test_mlx_refuses_a_prompt_that_fills_the_served_window(monkeypatch, limit):
+ from core.inference.context_refusal import ContextBudgetExceeded
+
+ fits = _BUDGET_PROMPT_N + 1
+ assert _run_text_budget(monkeypatch, limit, served = fits) == (limit or 1)
+ assert _run_vlm_budget(monkeypatch, _TEXT_TURN, None, limit, served = fits) == (limit or 1)
+ with pytest.raises(ContextBudgetExceeded) as refused:
+ _run_text_budget(monkeypatch, limit, served = _BUDGET_PROMPT_N)
+ assert (refused.value.request_tokens, refused.value.context_tokens) == (fits, _BUDGET_PROMPT_N)
+ with pytest.raises(ContextBudgetExceeded):
+ _run_vlm_budget(monkeypatch, _TEXT_TURN, None, limit, served = _BUDGET_PROMPT_N)
+ with pytest.raises(ContextBudgetExceeded):
+ _run_text_budget(monkeypatch, limit, served = 300, cached = list(range(300)))
+
+
[email protected]("limit", [None, 16])
+def test_mlx_batch_rows_refuse_a_prompt_that_fills_the_served_window(monkeypatch, limit):
+ from core.inference import mlx_inference
+ from core.inference.context_refusal import ContextBudgetExceeded
+
+ backend = _install_fake_text_stack(monkeypatch, {"generated": [7]}, [])
+ monkeypatch.setattr(mlx_inference, "_mlx_sampling_processors", lambda **_kw: None)
+ row = mlx_inference._row_kwargs({"prompt_ids": [1, 2, 3], "max_new_tokens": limit})
+ backend._served_context = 4
+ assert backend._plan_text_row([], **row).max_tokens == (limit or 1)
+ backend._served_context = 3
+ with pytest.raises(ContextBudgetExceeded):
+ backend._plan_text_row([], **row)
+
+
def test_mlx_unset_budget_falls_back_when_the_prompt_cannot_be_counted(monkeypatch):
from core.inference.runtime_context import UNSET_GENERATION_BUDGET