Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: make Stop work while an MLX model is still reading the prompt (#15615)
Nilay500f6db072f4
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 12d5f1f2941..498af28c413 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -3557,6 +3557,21 @@ def _raw_prompt_constraint(grammar, tokenizer):return GrammarConstraint(grammar, tokenizer, strips_special_tokens = True)+class _PrefillStopped(Exception):+ pass+++def _stream_until_cancelled(stream_generate, cancel_event, *args, **kwargs):+ def check(*_):+ if cancel_event is not None and cancel_event.is_set():+ raise _PrefillStopped++ try:+ yield from stream_generate(*args, prompt_progress_callback = check, **kwargs)+ except _PrefillStopped:+ return++class LogprobsUnsupported(ValueError):"""``n_probs`` asked of a model that does not sample one token per step."""@@ -6387,7 +6402,9 @@ class MLXInferenceBackend:gen_kwargs["input_embeddings"] = input_embeddingsif logits_processors is not None:gen_kwargs["logits_processors"] = logits_processors- for response in stream_generate(+ for response in _stream_until_cancelled(+ stream_generate,+ cancel_event,self._model,self._tokenizer,**gen_kwargs,diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex 7d5eb3cec92..8edf0d75e88 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -2482,6 +2482,38 @@ def test_mlx_plain_stream_keeps_characters_split_across_tokens(monkeypatch, piecassert _delivered(snapshots) == expected+def test_mlx_stop_during_prefill_ends_the_reply_before_the_prompt_is_read(monkeypatch):+ _install_fake_mlx(monkeypatch)+ backend = _byte_level_mlx_backend(monkeypatch, [b"a", b"b"])+ cancel, chunks = threading.Event(), []++ def _stream_generate(+ _model,+ _tokenizer,+ prompt_progress_callback = None,+ **_kwargs,+ ):+ progress = prompt_progress_callback or (lambda *_: None)+ progress(0, 8)+ for done in range(2, 8, 2):+ chunks.append(done)+ cancel.set()+ progress(done, 8)+ for token_id in range(2):+ yield SimpleNamespace(token = token_id)++ monkeypatch.setattr(sys.modules["mlx_lm"], "stream_generate", _stream_generate)+ snapshots = list(+ backend.generate_chat_response(+ messages = [{"role": "user", "content": "long prompt"}],+ max_new_tokens = 2,+ cancel_event = cancel,+ )+ )+ assert chunks == [2]+ assert _delivered(snapshots) == ""++@pytest.mark.parametrize("pieces, expected",[diff --git a/studio/backend/tests/test_mlx_prefill_stop_apple.py b/studio/backend/tests/test_mlx_prefill_stop_apple.pynew file mode 100644index 00000000000..a4be2495c66--- /dev/null+++ b/studio/backend/tests/test_mlx_prefill_stop_apple.py@@ -0,0 +1,104 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Stop during prefill on real mlx-lm (random init, no download), on Apple Silicon only."""++from __future__ import annotations++import platform+import sys+import threading++import pytest++pytestmark = pytest.mark.skipif(+ sys.platform != "darwin" or platform.machine() != "arm64",+ reason = "MLX runs on Apple Silicon only",+)++_VOCAB = 320+_PROMPT = list(range(2, 202))+_STEP = 16+# 199 prompt tokens go through chunked prefill before the last one is sampled from.+_PREFILL_CHUNKS = -(-(len(_PROMPT) - 1) // _STEP)+++def _tiny_llama():+ pytest.importorskip("mlx_lm")+ from mlx_lm.models import llama++ class Counting(llama.Model):+ def __call__(self, *args, **kwargs):+ self.calls += 1+ if self.on_call is not None:+ self.on_call(self.calls)+ return super().__call__(*args, **kwargs)++ model = Counting(+ llama.ModelArgs(+ model_type = "llama",+ hidden_size = 64,+ num_hidden_layers = 1,+ intermediate_size = 128,+ num_attention_heads = 4,+ rms_norm_eps = 1e-5,+ vocab_size = _VOCAB,+ num_key_value_heads = 4,+ )+ )+ model.calls, model.on_call = 0, None+ return model+++def _tokenizer():+ pytest.importorskip("tokenizers")+ from tokenizers import Tokenizer, models, pre_tokenizers+ from transformers import PreTrainedTokenizerFast++ vocab = {f"t{i}": i for i in range(_VOCAB)}+ raw = Tokenizer(models.WordLevel(vocab, unk_token = "t0"))+ raw.pre_tokenizer = pre_tokenizers.WhitespaceSplit()+ # No eos: a sampled id cannot end the reply early.+ return PreTrainedTokenizerFast(tokenizer_object = raw, unk_token = "t0")+++def _stream(model, tokenizer, cancel):+ from core.inference.mlx_inference import _stream_until_cancelled+ from mlx_lm import stream_generate+ from mlx_lm.sample_utils import make_sampler++ kwargs = dict(+ prompt = list(_PROMPT),+ max_tokens = 4,+ sampler = make_sampler(temp = 0.0),+ prefill_step_size = _STEP,+ )+ if cancel is None:+ return list(stream_generate(model, tokenizer, **kwargs))+ return list(_stream_until_cancelled(stream_generate, cancel, model, tokenizer, **kwargs))+++def test_stop_set_during_prefill_ends_after_the_current_chunk():+ model, tokenizer, cancel = _tiny_llama(), _tokenizer(), threading.Event()+ model.on_call = lambda n: cancel.set()++ assert _stream(model, tokenizer, cancel) == []+ assert model.calls == 1+++def test_a_plain_stream_reads_the_whole_prompt_before_a_stop_can_land():+ model, tokenizer = _tiny_llama(), _tokenizer()+ responses = _stream(model, tokenizer, None)++ assert responses+ assert model.calls > _PREFILL_CHUNKS+++def test_the_runtime_generates_again_after_a_stopped_prefill():+ model, tokenizer, cancel = _tiny_llama(), _tokenizer(), threading.Event()+ model.on_call = lambda n: cancel.set()+ assert _stream(model, tokenizer, cancel) == []++ model.on_call, model.calls = None, 0+ assert len(_stream(model, tokenizer, threading.Event())) == 4+ assert model.calls > _PREFILL_CHUNKS