Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
perf(studio): stream MLX VLM SentencePiece tokens as they are sampled (#16684)
Long Yixinga34d76602938
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 11fdf8c30f1..f36f34ef2ed 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -2266,6 +2266,69 @@ def _mlx_stream_detokenizer(source):return independent+def _eager_spm_detokenizer_class():+ from mlx_lm.tokenizer_utils import SPMStreamingDetokenizer+ class EagerSPMStreamingDetokenizer(SPMStreamingDetokenizer):+ def __init__(+ self,+ tokenizer,+ trim_space = True,+ ):+ self._trims_first_space = trim_space+ super().__init__(tokenizer, trim_space = trim_space)++ def reset(self):+ super().reset()+ self.trim_space = self._trims_first_space++ # mlx-lm trims every flush while the text is still empty; only the first may be trimmed.+ def _try_flush(self, force = False):+ pending = self._unflushed+ super()._try_flush(force)+ if pending and not self._unflushed:+ self.trim_space = False++ def add_token(+ self,+ token,+ skip_special_token_ids = (),+ ):+ if token not in skip_special_token_ids:+ super().add_token(token)++ return EagerSPMStreamingDetokenizer+++class _SpmPieces(list):+ @property+ def vocab(self): # mlx-lm 0.31.x+ return {piece: index for index, piece in enumerate(self)}++ def convert_ids_to_tokens(self, ids): # mlx-lm >= 0.32+ return [self[index] for index in ids]+++def _stream_spm_tokens_eagerly(processor):+ """mlx-vlm's SPM detokenizer holds each piece until the next word starts; mlx-lm's does not."""+ held = getattr(processor, "detokenizer", None)+ if (type(held).__module__, type(held).__name__) != (+ "mlx_vlm.tokenizer_utils",+ "SPMStreamingDetokenizer",+ ):+ return+ # Reusing mlx-vlm's pieces skips a second ~1 s walk of a 262k vocab.+ pieces = getattr(held, "tokenmap", None)+ if pieces and None not in pieces:+ source = _SpmPieces(pieces)+ else:+ source = getattr(processor, "tokenizer", processor)+ try:+ processor.detokenizer = _eager_spm_detokenizer_class()(source, trim_space = held.trim_space)+ except Exception as exc: # noqa: BLE001 -- mlx-vlm's still streams the same text, only later+ logger.debug("MLX streaming detokenizer kept as loaded (%s)", exc)++class _MlxStreamedText:"""A reply rebuilt from the ids the runtime sampled, for turns carrying native controls.@@ -6228,6 +6291,7 @@ class MLXInferenceBackend:processor = tokenizer_or_processorif is_lora:processor = load_mlx_adapter_processor(processor, model_name, hf_token)+ _stream_spm_tokens_eagerly(processor)self._model = modelself._processor = processorself._tokenizer = getattr(processor, "tokenizer", processor)diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex e6ed2e02b8f..0adeb6248e7 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -1763,6 +1763,83 @@ def test_mlx_inference_distributed_vlm_forwards_group_to_fast_mlx(monkeypatch):MLXInferenceBackend().load_model(config, parallel_mode = "tensor", distributed_group = group)+def test_vlm_load_streams_each_spm_token_as_it_arrives(monkeypatch):+ detokenizers = pytest.importorskip("mlx_vlm.tokenizer_utils")+ pytest.importorskip("mlx_lm.tokenizer_utils")+ from core.inference.mlx_inference import MLXInferenceBackend++ pieces = ["▁#", "▁A", "▁head", "ing", "<0xE2>", "<0x9C>", "<0x93>", "<eos>"]++ class _SpmTokenizer(_DummyTokenizer):+ vocab = {piece: index for index, piece in enumerate(pieces)}++ def __len__(self):+ return len(pieces)++ def convert_ids_to_tokens(self, ids):+ return [pieces[index] for index in ids]++ def stream(detokenizer):+ detokenizer.reset()+ segments = []+ for token in range(len(pieces)):+ detokenizer.add_token(token, skip_special_token_ids = [len(pieces) - 1])+ segments.append(detokenizer.last_segment)+ detokenizer.finalize()+ return segments + [detokenizer.last_segment]++ tokenizer = _SpmTokenizer()+ held = detokenizers.SPMStreamingDetokenizer(tokenizer, trim_space = False)+ monkeypatch.setattr(_DummyProcessor, "tokenizer", tokenizer)+ monkeypatch.setattr(_DummyProcessor, "detokenizer", held, raising = False)+ _install_fake_mlx(monkeypatch)+ _install_fake_fast_mlx(monkeypatch, [])+ backend = MLXInferenceBackend()+ config = SimpleNamespace(identifier = "fake/vlm", is_vision = True, is_lora = False)+ assert backend.load_model(config)++ late = stream(held)+ eager = stream(backend._processor.detokenizer)+ assert late[:2] == ["", " #"]+ assert eager == [" #", " A", " head", "ing", "", "", "✓", "", ""]+ assert "".join(eager) == "".join(late) == " # A heading✓"+++def test_vlm_load_trims_only_the_reply_s_first_space(monkeypatch):+ detokenizers = pytest.importorskip("mlx_vlm.tokenizer_utils")+ pytest.importorskip("mlx_lm.tokenizer_utils")+ from core.inference.mlx_inference import MLXInferenceBackend++ pieces = ["▁", "<0x20>", "▁world"]++ class _SpmTokenizer(_DummyTokenizer):+ vocab = {piece: index for index, piece in enumerate(pieces)}++ def __len__(self):+ return len(pieces)++ def convert_ids_to_tokens(self, ids):+ return [pieces[index] for index in ids]++ tokenizer = _SpmTokenizer()+ held = detokenizers.SPMStreamingDetokenizer(tokenizer, trim_space = True)+ monkeypatch.setattr(_DummyProcessor, "tokenizer", tokenizer)+ monkeypatch.setattr(_DummyProcessor, "detokenizer", held, raising = False)+ _install_fake_mlx(monkeypatch)+ _install_fake_fast_mlx(monkeypatch, [])+ backend = MLXInferenceBackend()+ config = SimpleNamespace(identifier = "fake/vlm", is_vision = True, is_lora = False)+ assert backend.load_model(config)++ eager = backend._processor.detokenizer+ for _ in range(2):+ eager.reset()+ for token in range(len(pieces)):+ eager.add_token(token)+ eager.finalize()+ assert eager.text == " world"++@pytest.mark.parametrize("accepts_backend", (True, False))def test_mlx_distributed_init_selects_jaccl_backend(monkeypatch, accepts_backend):_install_fake_mlx(monkeypatch)