Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: make Stop work while an MLX model is still reading the prompt (#15615)

Nilay500f6db072f4
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 12d5f1f2941..498af28c413 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -3557,6 +3557,21 @@ def _raw_prompt_constraint(grammar, tokenizer):
return GrammarConstraint(grammar, tokenizer, strips_special_tokens = True)
+class _PrefillStopped(Exception):
+ pass
+
+
+def _stream_until_cancelled(stream_generate, cancel_event, *args, **kwargs):
+ def check(*_):
+ if cancel_event is not None and cancel_event.is_set():
+ raise _PrefillStopped
+
+ try:
+ yield from stream_generate(*args, prompt_progress_callback = check, **kwargs)
+ except _PrefillStopped:
+ return
+
+
class LogprobsUnsupported(ValueError):
"""``n_probs`` asked of a model that does not sample one token per step."""
@@ -6387,7 +6402,9 @@ class MLXInferenceBackend:
gen_kwargs["input_embeddings"] = input_embeddings
if logits_processors is not None:
gen_kwargs["logits_processors"] = logits_processors
- for response in stream_generate(
+ for response in _stream_until_cancelled(
+ stream_generate,
+ cancel_event,
self._model,
self._tokenizer,
**gen_kwargs,
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 7d5eb3cec92..8edf0d75e88 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -2482,6 +2482,38 @@ def test_mlx_plain_stream_keeps_characters_split_across_tokens(monkeypatch, piec
assert _delivered(snapshots) == expected
+def test_mlx_stop_during_prefill_ends_the_reply_before_the_prompt_is_read(monkeypatch):
+ _install_fake_mlx(monkeypatch)
+ backend = _byte_level_mlx_backend(monkeypatch, [b"a", b"b"])
+ cancel, chunks = threading.Event(), []
+
+ def _stream_generate(
+ _model,
+ _tokenizer,
+ prompt_progress_callback = None,
+ **_kwargs,
+ ):
+ progress = prompt_progress_callback or (lambda *_: None)
+ progress(0, 8)
+ for done in range(2, 8, 2):
+ chunks.append(done)
+ cancel.set()
+ progress(done, 8)
+ for token_id in range(2):
+ yield SimpleNamespace(token = token_id)
+
+ monkeypatch.setattr(sys.modules["mlx_lm"], "stream_generate", _stream_generate)
+ snapshots = list(
+ backend.generate_chat_response(
+ messages = [{"role": "user", "content": "long prompt"}],
+ max_new_tokens = 2,
+ cancel_event = cancel,
+ )
+ )
+ assert chunks == [2]
+ assert _delivered(snapshots) == ""
+
+
@pytest.mark.parametrize(
"pieces, expected",
[
diff --git a/studio/backend/tests/test_mlx_prefill_stop_apple.py b/studio/backend/tests/test_mlx_prefill_stop_apple.py
new file mode 100644
index 00000000000..a4be2495c66
--- /dev/null
+++ b/studio/backend/tests/test_mlx_prefill_stop_apple.py
@@ -0,0 +1,104 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Stop during prefill on real mlx-lm (random init, no download), on Apple Silicon only."""
+
+from __future__ import annotations
+
+import platform
+import sys
+import threading
+
+import pytest
+
+pytestmark = pytest.mark.skipif(
+ sys.platform != "darwin" or platform.machine() != "arm64",
+ reason = "MLX runs on Apple Silicon only",
+)
+
+_VOCAB = 320
+_PROMPT = list(range(2, 202))
+_STEP = 16
+# 199 prompt tokens go through chunked prefill before the last one is sampled from.
+_PREFILL_CHUNKS = -(-(len(_PROMPT) - 1) // _STEP)
+
+
+def _tiny_llama():
+ pytest.importorskip("mlx_lm")
+ from mlx_lm.models import llama
+
+ class Counting(llama.Model):
+ def __call__(self, *args, **kwargs):
+ self.calls += 1
+ if self.on_call is not None:
+ self.on_call(self.calls)
+ return super().__call__(*args, **kwargs)
+
+ model = Counting(
+ llama.ModelArgs(
+ model_type = "llama",
+ hidden_size = 64,
+ num_hidden_layers = 1,
+ intermediate_size = 128,
+ num_attention_heads = 4,
+ rms_norm_eps = 1e-5,
+ vocab_size = _VOCAB,
+ num_key_value_heads = 4,
+ )
+ )
+ model.calls, model.on_call = 0, None
+ return model
+
+
+def _tokenizer():
+ pytest.importorskip("tokenizers")
+ from tokenizers import Tokenizer, models, pre_tokenizers
+ from transformers import PreTrainedTokenizerFast
+
+ vocab = {f"t{i}": i for i in range(_VOCAB)}
+ raw = Tokenizer(models.WordLevel(vocab, unk_token = "t0"))
+ raw.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
+ # No eos: a sampled id cannot end the reply early.
+ return PreTrainedTokenizerFast(tokenizer_object = raw, unk_token = "t0")
+
+
+def _stream(model, tokenizer, cancel):
+ from core.inference.mlx_inference import _stream_until_cancelled
+ from mlx_lm import stream_generate
+ from mlx_lm.sample_utils import make_sampler
+
+ kwargs = dict(
+ prompt = list(_PROMPT),
+ max_tokens = 4,
+ sampler = make_sampler(temp = 0.0),
+ prefill_step_size = _STEP,
+ )
+ if cancel is None:
+ return list(stream_generate(model, tokenizer, **kwargs))
+ return list(_stream_until_cancelled(stream_generate, cancel, model, tokenizer, **kwargs))
+
+
+def test_stop_set_during_prefill_ends_after_the_current_chunk():
+ model, tokenizer, cancel = _tiny_llama(), _tokenizer(), threading.Event()
+ model.on_call = lambda n: cancel.set()
+
+ assert _stream(model, tokenizer, cancel) == []
+ assert model.calls == 1
+
+
+def test_a_plain_stream_reads_the_whole_prompt_before_a_stop_can_land():
+ model, tokenizer = _tiny_llama(), _tokenizer()
+ responses = _stream(model, tokenizer, None)
+
+ assert responses
+ assert model.calls > _PREFILL_CHUNKS
+
+
+def test_the_runtime_generates_again_after_a_stopped_prefill():
+ model, tokenizer, cancel = _tiny_llama(), _tokenizer(), threading.Event()
+ model.on_call = lambda n: cancel.set()
+ assert _stream(model, tokenizer, cancel) == []
+
+ model.on_call, model.calls = None, 0
+ assert len(_stream(model, tokenizer, threading.Event())) == 4
+ assert model.calls > _PREFILL_CHUNKS