Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

perf(studio): stream MLX VLM SentencePiece tokens as they are sampled (#16684)

Long Yixinga34d76602938
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 11fdf8c30f1..f36f34ef2ed 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -2266,6 +2266,69 @@ def _mlx_stream_detokenizer(source):
return independent
+def _eager_spm_detokenizer_class():
+ from mlx_lm.tokenizer_utils import SPMStreamingDetokenizer
+ class EagerSPMStreamingDetokenizer(SPMStreamingDetokenizer):
+ def __init__(
+ self,
+ tokenizer,
+ trim_space = True,
+ ):
+ self._trims_first_space = trim_space
+ super().__init__(tokenizer, trim_space = trim_space)
+
+ def reset(self):
+ super().reset()
+ self.trim_space = self._trims_first_space
+
+ # mlx-lm trims every flush while the text is still empty; only the first may be trimmed.
+ def _try_flush(self, force = False):
+ pending = self._unflushed
+ super()._try_flush(force)
+ if pending and not self._unflushed:
+ self.trim_space = False
+
+ def add_token(
+ self,
+ token,
+ skip_special_token_ids = (),
+ ):
+ if token not in skip_special_token_ids:
+ super().add_token(token)
+
+ return EagerSPMStreamingDetokenizer
+
+
+class _SpmPieces(list):
+ @property
+ def vocab(self): # mlx-lm 0.31.x
+ return {piece: index for index, piece in enumerate(self)}
+
+ def convert_ids_to_tokens(self, ids): # mlx-lm >= 0.32
+ return [self[index] for index in ids]
+
+
+def _stream_spm_tokens_eagerly(processor):
+ """mlx-vlm's SPM detokenizer holds each piece until the next word starts; mlx-lm's does not."""
+ held = getattr(processor, "detokenizer", None)
+ if (type(held).__module__, type(held).__name__) != (
+ "mlx_vlm.tokenizer_utils",
+ "SPMStreamingDetokenizer",
+ ):
+ return
+ # Reusing mlx-vlm's pieces skips a second ~1 s walk of a 262k vocab.
+ pieces = getattr(held, "tokenmap", None)
+ if pieces and None not in pieces:
+ source = _SpmPieces(pieces)
+ else:
+ source = getattr(processor, "tokenizer", processor)
+ try:
+ processor.detokenizer = _eager_spm_detokenizer_class()(source, trim_space = held.trim_space)
+ except Exception as exc: # noqa: BLE001 -- mlx-vlm's still streams the same text, only later
+ logger.debug("MLX streaming detokenizer kept as loaded (%s)", exc)
+
+
class _MlxStreamedText:
"""A reply rebuilt from the ids the runtime sampled, for turns carrying native controls.
@@ -6228,6 +6291,7 @@ class MLXInferenceBackend:
processor = tokenizer_or_processor
if is_lora:
processor = load_mlx_adapter_processor(processor, model_name, hf_token)
+ _stream_spm_tokens_eagerly(processor)
self._model = model
self._processor = processor
self._tokenizer = getattr(processor, "tokenizer", processor)
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index e6ed2e02b8f..0adeb6248e7 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -1763,6 +1763,83 @@ def test_mlx_inference_distributed_vlm_forwards_group_to_fast_mlx(monkeypatch):
MLXInferenceBackend().load_model(config, parallel_mode = "tensor", distributed_group = group)
+def test_vlm_load_streams_each_spm_token_as_it_arrives(monkeypatch):
+ detokenizers = pytest.importorskip("mlx_vlm.tokenizer_utils")
+ pytest.importorskip("mlx_lm.tokenizer_utils")
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ pieces = ["▁#", "▁A", "▁head", "ing", "<0xE2>", "<0x9C>", "<0x93>", "<eos>"]
+
+ class _SpmTokenizer(_DummyTokenizer):
+ vocab = {piece: index for index, piece in enumerate(pieces)}
+
+ def __len__(self):
+ return len(pieces)
+
+ def convert_ids_to_tokens(self, ids):
+ return [pieces[index] for index in ids]
+
+ def stream(detokenizer):
+ detokenizer.reset()
+ segments = []
+ for token in range(len(pieces)):
+ detokenizer.add_token(token, skip_special_token_ids = [len(pieces) - 1])
+ segments.append(detokenizer.last_segment)
+ detokenizer.finalize()
+ return segments + [detokenizer.last_segment]
+
+ tokenizer = _SpmTokenizer()
+ held = detokenizers.SPMStreamingDetokenizer(tokenizer, trim_space = False)
+ monkeypatch.setattr(_DummyProcessor, "tokenizer", tokenizer)
+ monkeypatch.setattr(_DummyProcessor, "detokenizer", held, raising = False)
+ _install_fake_mlx(monkeypatch)
+ _install_fake_fast_mlx(monkeypatch, [])
+ backend = MLXInferenceBackend()
+ config = SimpleNamespace(identifier = "fake/vlm", is_vision = True, is_lora = False)
+ assert backend.load_model(config)
+
+ late = stream(held)
+ eager = stream(backend._processor.detokenizer)
+ assert late[:2] == ["", " #"]
+ assert eager == [" #", " A", " head", "ing", "", "", "✓", "", ""]
+ assert "".join(eager) == "".join(late) == " # A heading✓"
+
+
+def test_vlm_load_trims_only_the_reply_s_first_space(monkeypatch):
+ detokenizers = pytest.importorskip("mlx_vlm.tokenizer_utils")
+ pytest.importorskip("mlx_lm.tokenizer_utils")
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ pieces = ["▁", "<0x20>", "▁world"]
+
+ class _SpmTokenizer(_DummyTokenizer):
+ vocab = {piece: index for index, piece in enumerate(pieces)}
+
+ def __len__(self):
+ return len(pieces)
+
+ def convert_ids_to_tokens(self, ids):
+ return [pieces[index] for index in ids]
+
+ tokenizer = _SpmTokenizer()
+ held = detokenizers.SPMStreamingDetokenizer(tokenizer, trim_space = True)
+ monkeypatch.setattr(_DummyProcessor, "tokenizer", tokenizer)
+ monkeypatch.setattr(_DummyProcessor, "detokenizer", held, raising = False)
+ _install_fake_mlx(monkeypatch)
+ _install_fake_fast_mlx(monkeypatch, [])
+ backend = MLXInferenceBackend()
+ config = SimpleNamespace(identifier = "fake/vlm", is_vision = True, is_lora = False)
+ assert backend.load_model(config)
+
+ eager = backend._processor.detokenizer
+ for _ in range(2):
+ eager.reset()
+ for token in range(len(pieces)):
+ eager.add_token(token)
+ eager.finalize()
+ assert eager.text == " world"
+
+
@pytest.mark.parametrize("accepts_backend", (True, False))
def test_mlx_distributed_init_selects_jaccl_backend(monkeypatch, accepts_backend):
_install_fake_mlx(monkeypatch)