Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep emoji split across tokens in MLX replies (#13416)
Daniel Hanfb35485febc9
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex d90294a907c..b63a6746f8e 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -3399,7 +3399,9 @@ class _RowStream:if self._normalizer is not None:return self._normalize(self.sampled[self._released : self._cut])if not self._sequences:- return [self._prefix + self.sampled]+ # Re-decoded snapshots hold back an unfinished character, as in _generate_text.+ held = self.sampled.rstrip("\ufffd") if self._decode is not None else self.sampled+ return [self._prefix + held]if self._decode is not None:return []if self._cut > self._released:@@ -3409,6 +3411,8 @@ class _RowStream:def settle(self):if not self._sequences:+ if self._decode is not None and self.sampled.endswith("\ufffd"):+ return [self._prefix + self.sampled]return []if not self.stopped:self._cut = len(self.sampled)@@ -5667,7 +5671,9 @@ class MLXInferenceBackend:),)if not sequences:- yield think_prefix + sampled+ # A character whose bytes span tokens decodes to U+FFFD until its last token lands, and+ # consumers diff snapshots by length, so yielding that would replace it for good (#8594).+ yield think_prefix + sampled.rstrip("\ufffd")else:# Matched every step, delivered once at the end: this decode revises earlier snapshots,# and consumers diff them by length, so a revised one splices two renderings into text@@ -5715,6 +5721,8 @@ class MLXInferenceBackend:sampled += streamed_text.finish(sampled)if sequences:cut, stopped = _mlx_stop_cut(sampled, sequences)+ if not sequences and streamed_text is None and sampled.endswith("\ufffd"):+ yield think_prefix + sampledif sequences or streamed_text is not None:if not stopped:cut = len(sampled)diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex b22dffbaf87..6f13121726f 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -1612,6 +1612,102 @@ def test_mlx_generate_text_forwards_kwargs_into_template_helper(assert render["kwargs"]["preserve_thinking"] is True+def _byte_level_mlx_backend(monkeypatch, pieces):+ from core.inference import mlx_inference++ monkeypatch.setattr(+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",+ lambda *_args, **_kwargs: "prompt",+ raising = True,+ )+ monkeypatch.setattr(+ "core.inference.chat_template_helpers.render_with_native_template_fallback",+ lambda formatted_prompt, **_kwargs: SimpleNamespace(+ prompt = formatted_prompt,+ reasoning_channel_markers = None,+ ),+ raising = True,+ )+ mlx_lm_pkg = types.ModuleType("mlx_lm")+ mlx_lm_sample = types.ModuleType("mlx_lm.sample_utils")+ mlx_lm_sample.make_sampler = lambda **_kwargs: object()+ mlx_lm_sample.make_logits_processors = lambda **_kwargs: None++ def _stream_generate(_model, _tokenizer, **_kwargs):+ for token_id in range(len(pieces)):+ yield SimpleNamespace(token = token_id)++ mlx_lm_pkg.stream_generate = _stream_generate+ monkeypatch.setitem(sys.modules, "mlx_lm", mlx_lm_pkg)+ monkeypatch.setitem(sys.modules, "mlx_lm.sample_utils", mlx_lm_sample)++ class _ByteLevelTokenizer:+ chat_template = "x"+ all_special_ids = []++ def decode(self, ids, **_kwargs):+ return b"".join(pieces[int(i)] for i in ids).decode("utf-8", "replace")++ backend = mlx_inference.MLXInferenceBackend()+ backend._model = object()+ backend._tokenizer = _ByteLevelTokenizer()+ backend._is_vlm = False+ return backend+++def _delivered(snapshots):+ # What the chat routes stream: each snapshot's growth past the previous one's length.+ text = prev = ""+ for snapshot in snapshots:+ text += snapshot[len(prev) :]+ prev = snapshot+ return text+++ "pieces, expected",+ [+ # 🎉 and ✨ are byte-fallback pairs in Qwen3's vocab (unslothai/unsloth#8594).+ (+ [b"Party ", b"\xf0\x9f\x8e", b"\x89", b" sparkle ", b"\xe2\x9c", b"\xa8", b" done"],+ "Party \U0001f389 sparkle \u2728 done",+ ),+ ([b"\xf0\x9f", b"\x8e", b"\x89"], "\U0001f389"),+ # A turn cut mid-character still delivers what it decoded to.+ ([b"ok ", b"\xf0\x9f\x8e"], "ok \ufffd"),+ ],+)+def test_mlx_plain_stream_keeps_characters_split_across_tokens(monkeypatch, pieces, expected):+ _install_fake_mlx(monkeypatch)+ backend = _byte_level_mlx_backend(monkeypatch, pieces)+ snapshots = list(+ backend.generate_chat_response(+ messages = [{"role": "user", "content": "emoji please"}],+ max_new_tokens = len(pieces),+ )+ )+ assert _delivered(snapshots) == expected+++ "pieces, expected",+ [+ ([b"Party ", b"\xf0\x9f\x8e", b"\x89", b" done"], "Party \U0001f389 done"),+ ([b"ok ", b"\xf0\x9f\x8e"], "ok \ufffd"),+ ],+)+def test_mlx_batched_row_keeps_characters_split_across_tokens(monkeypatch, pieces, expected):+ _install_fake_mlx(monkeypatch)+ from core.inference import mlx_inference++ stream = mlx_inference._RowStream(+ decode = lambda ids: b"".join(pieces[i] for i in ids).decode("utf-8", "replace"),+ )+ snapshots = [snap for i in range(len(pieces)) for snap in stream.feed(token = i)]+ snapshots += stream.settle()+ assert _delivered(snapshots) == expected++def test_mlx_tool_turn_preserves_native_gemma_special_token_ids(monkeypatch):_install_fake_mlx(monkeypatch)from core.inference import mlx_inference