Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: decode the Transformers chat stream incrementally instead of re-decoding the current line per token (#16800)
Daniel Han91c2e807bdb6
diff --git a/studio/backend/core/inference/incremental_decode.py b/studio/backend/core/inference/incremental_decode.pyindex d53485f1831..d95794f2c28 100644--- a/studio/backend/core/inference/incremental_decode.py+++ b/studio/backend/core/inference/incremental_decode.py@@ -141,7 +141,12 @@ def split_profile(tokenizer):if inner is None:return Nonetry:- size = len(inner)+ # Not len(inner): tokenizers < 0.23 rebuilds the whole vocab for it.+ backend = inner.backend_tokenizer+ size = (+ backend.get_vocab_size(with_added_tokens = False),+ len(backend.get_added_tokens_decoder()),+ )except Exception: # noqa: BLE001return Nonewith _PROFILES_LOCK:diff --git a/studio/backend/core/inference/incremental_streamer.py b/studio/backend/core/inference/incremental_streamer.pynew file mode 100644index 00000000000..891765c6fe3--- /dev/null+++ b/studio/backend/core/inference/incremental_streamer.py@@ -0,0 +1,74 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""``TextIteratorStreamer`` that decodes long lines incrementally instead of re-decoding the whole+line per token; chunks equal upstream ``put``, which short lines and non-splitting tokenizers keep."""++from __future__ import annotations++from transformers import TextIteratorStreamer++from core.inference.incremental_decode import IncrementalDecode, split_profile+from core.inference.native_tool_tokens import NativeToolTokenDecoder++__all__ = ["IncrementalTextIteratorStreamer"]++# Below this many ids upstream's whole-line decode is cheaper than feeding a decoder.+_MIN_LINE_IDS = 64+++def _line_decoder(tokenizer, decode_kwargs):+ if isinstance(tokenizer, NativeToolTokenDecoder):+ return tokenizer.incremental() if split_profile(tokenizer._tokenizer) else None+ return IncrementalDecode(tokenizer, decode_kwargs) if split_profile(tokenizer) else None+++class IncrementalTextIteratorStreamer(TextIteratorStreamer):+ _decoder = False # False = unresolved, None = never splits+ _line_ids = None # token_cache list _decoder consumed; upstream rebinds it on every flush++ def _attach(self) -> bool:+ if len(self.token_cache) < _MIN_LINE_IDS:+ return False+ if self._decoder is False:+ self._decoder = _line_decoder(self.tokenizer, self.decode_kwargs)+ if self._decoder is None:+ return False+ self._decoder.reset()+ for token_id in self.token_cache:+ self._decoder.feed(token_id)+ self._line_ids = self.token_cache+ return True++ def put(self, value):+ if self._line_ids is not self.token_cache and not self._attach():+ return super().put(value)++ if len(value.shape) > 1 and value.shape[0] > 1:+ raise ValueError("TextStreamer only supports batch size 1")+ elif len(value.shape) > 1:+ value = value[0]++ if self.skip_prompt and self.next_tokens_are_prompt:+ self.next_tokens_are_prompt = False+ return++ new_ids = value.tolist()+ self.token_cache.extend(new_ids)+ for token_id in new_ids:+ self._decoder.feed(token_id)+ text = self._decoder.text++ if text.endswith("\n"):+ printable_text = text[self.print_len :]+ self.token_cache = []+ self.print_len = 0+ elif len(text) > 0 and self._is_chinese_char(ord(text[-1])):+ printable_text = text[self.print_len :]+ self.print_len += len(printable_text)+ else:+ # rfind from print_len slices the same text as upstream's whole-line rfind.+ printable_text = text[self.print_len : text.rfind(" ", self.print_len) + 1]+ self.print_len += len(printable_text)++ self.on_finalized_text(printable_text)diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex 1e51b84b0b6..8dbc7733a2d 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -63,6 +63,7 @@ from core.inference.generation_timing import (build_generation_timings,with_prefill_boundary_processor,)+from core.inference.incremental_streamer import IncrementalTextIteratorStreamerfrom core.inference.native_tool_tokens import (NativeToolTokenDecoder,closes_an_open_envelope,@@ -473,7 +474,7 @@ class HarmonyTextStreamer:self._final_emitted = end - start-class ReasoningTextIteratorStreamer(TextIteratorStreamer):+class ReasoningTextIteratorStreamer(IncrementalTextIteratorStreamer):"""TextIteratorStreamer that preserves native channel tokens until parsed."""def __init__(@@ -2649,11 +2650,10 @@ class InferenceBackend:).to(model.device)try:- from transformers import TextIteratorStreamerfrom queue import Empty# As the text path: a rendered catalogue keeps native tool-call tokens for the healer.- streamer = TextIteratorStreamer(+ streamer = IncrementalTextIteratorStreamer(NativeToolTokenDecoder(raw_tokenizer) if tools else raw_tokenizer,skip_prompt = True,skip_special_tokens = not tools,@@ -2830,7 +2830,7 @@ class InferenceBackend:)except Exception as e:logger.warning(f"HarmonyTextStreamer init failed, falling back: {e}")- return TextIteratorStreamer(+ return IncrementalTextIteratorStreamer(tokenizer,skip_prompt = skip_prompt,skip_special_tokens = True,@@ -2853,7 +2853,7 @@ class InferenceBackend:in_reasoning = prompt_opens_reasoning_channel(prompt, markers, continued),)stream_tokenizer = NativeToolTokenDecoder(tokenizer) if preserve_tool_tokens else tokenizer- return TextIteratorStreamer(+ return IncrementalTextIteratorStreamer(stream_tokenizer,skip_prompt = skip_prompt,skip_special_tokens = not preserve_tool_tokens,diff --git a/studio/backend/core/inference/native_tool_tokens.py b/studio/backend/core/inference/native_tool_tokens.pyindex 1aeab9ae809..3ff4a7cff9b 100644--- a/studio/backend/core/inference/native_tool_tokens.py+++ b/studio/backend/core/inference/native_tool_tokens.py@@ -310,6 +310,9 @@ class _IncrementalNativeDecode:def text(self) -> str:return self._text.text+ def reset(self):+ self._text.reset()+def feed(self, token_id) -> str:if self._decoder._special_ids and self._decoder.suppresses(token_id):return self._text.textdiff --git a/studio/backend/tests/test_incremental_streamer.py b/studio/backend/tests/test_incremental_streamer.pynew file mode 100644index 00000000000..8d08abdd528--- /dev/null+++ b/studio/backend/tests/test_incremental_streamer.py@@ -0,0 +1,141 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""IncrementalTextIteratorStreamer emits exactly upstream TextIteratorStreamer's chunks while+decoding only the ids since the last split. In-memory tokenizers, no download."""++import random+import sys+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("tokenizers")+pytest.importorskip("transformers")++from transformers import TextIteratorStreamer++_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)++import core.inference.incremental_streamer as incremental_streamer+from core.inference.incremental_streamer import IncrementalTextIteratorStreamer+from core.inference.native_tool_tokens import NativeToolTokenDecoder+from test_mlx_incremental_decode import ( # noqa: E402+ _CORPUS,+ _TOKENIZERS,+ _byte_level,+ _counting,+ _streams,+)+++class _Recording:+ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self.chunks = []++ def on_finalized_text(+ self,+ text,+ stream_end = False,+ ):+ self.chunks.append((text, stream_end))+++class _Upstream(_Recording, TextIteratorStreamer):+ pass+++class _Incremental(_Recording, IncrementalTextIteratorStreamer):+ pass+++def _drive(cls, source, ids, *, skip_prompt, sizes, generations, **decode_kwargs):+ streamer = cls(source, skip_prompt = skip_prompt, **decode_kwargs)+ for _ in range(generations):+ streamer.put(torch.tensor([ids[:3] or [0]]))+ index = 0+ for size in sizes:+ if index >= len(ids):+ break+ streamer.put(torch.tensor(ids[index : index + size]))+ index += size+ streamer.end()+ return streamer++[email protected](scope = "module", params = sorted(_TOKENIZERS))+def tokenizer(request):+ return _TOKENIZERS[request.param]()++[email protected]("mode", ["plain", "keep-special", "native-tool", "reasoning"])+def test_chunks_equal_upstream(tokenizer, mode):+ rng = random.Random(mode)+ newline_ids = {i for i in range(len(tokenizer)) if "\n" in tokenizer.decode([i])}+ streams = dict(_streams(tokenizer, seed = 2))+ for name, ids in list(streams.items()):+ streams[f"{name}-one-line"] = [i for i in ids if i not in newline_ids]+ for name, ids in streams.items():+ if mode == "plain":+ make, kwargs = (lambda: tokenizer), {"skip_special_tokens": True}+ elif mode == "keep-special":+ make, kwargs = (lambda: tokenizer), {"skip_special_tokens": False}+ elif mode == "native-tool":+ make, kwargs = (lambda: NativeToolTokenDecoder(tokenizer)), {}+ else:+ make = lambda: NativeToolTokenDecoder(+ tokenizer, preserved_tokens = ("<think>", "</think>")+ )+ kwargs = {"skip_special_tokens": False}+ sizes = [1 if rng.random() < 0.9 else rng.randint(2, 5) for _ in ids]+ for skip_prompt in (True, False):+ options = dict(skip_prompt = skip_prompt, sizes = sizes, generations = 2, **kwargs)+ expected = _drive(_Upstream, make(), ids, **options).chunks+ streamer = _drive(_Incremental, make(), ids, **options)+ assert streamer.chunks == expected, (name, skip_prompt)+ if name.endswith("-one-line") and len(ids) > 2 * incremental_streamer._MIN_LINE_IDS:+ assert streamer._decoder, name+++def test_work_per_token_stays_bounded():+ tokenizer = _byte_level(cleanup = True)+ ids = tokenizer.encode(" ".join(_CORPUS[:1]) * 120, add_special_tokens = False)+ assert len(ids) > 2000 and "\n" not in tokenizer.decode(ids)+ counter = _counting(tokenizer)+ streamer = _Incremental(tokenizer, skip_prompt = False, skip_special_tokens = True)+ for token_id in ids:+ streamer.put(torch.tensor([token_id]))+ # Upstream re-decodes the whole line every token: about len(ids) ** 2 / 2 ids.+ assert counter.decoded <= len(ids) * 32+++def test_decoder_resolved_once_and_only_for_long_lines(monkeypatch):+ tokenizer = _byte_level(cleanup = False)+ calls = []+ real = incremental_streamer._line_decoder+ monkeypatch.setattr(+ incremental_streamer,+ "_line_decoder",+ lambda *args: calls.append(1) or real(*args),+ )+ word = tokenizer.encode(" word", add_special_tokens = False)+ newline = tokenizer.encode("\n", add_special_tokens = False)+ short_line = word * 8 + newline+ long_line = word * (2 * incremental_streamer._MIN_LINE_IDS) + newline+ streamer = _Incremental(tokenizer, skip_prompt = False)+ for token_id in short_line * 20:+ streamer.put(torch.tensor([token_id]))+ assert calls == []+ for token_id in long_line * 20:+ streamer.put(torch.tensor([token_id]))+ streamer.end()+ assert calls == [1]+ upstream = _Upstream(tokenizer, skip_prompt = False)+ for token_id in short_line * 20 + long_line * 20:+ upstream.put(torch.tensor([token_id]))+ upstream.end()+ assert streamer.chunks == upstream.chunks