Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: decode the Transformers chat stream incrementally instead of re-decoding the current line per token (#16800)

Daniel Han91c2e807bdb6
diff --git a/studio/backend/core/inference/incremental_decode.py b/studio/backend/core/inference/incremental_decode.py
index d53485f1831..d95794f2c28 100644
--- a/studio/backend/core/inference/incremental_decode.py
+++ b/studio/backend/core/inference/incremental_decode.py
@@ -141,7 +141,12 @@ def split_profile(tokenizer):
if inner is None:
return None
try:
- size = len(inner)
+ # Not len(inner): tokenizers < 0.23 rebuilds the whole vocab for it.
+ backend = inner.backend_tokenizer
+ size = (
+ backend.get_vocab_size(with_added_tokens = False),
+ len(backend.get_added_tokens_decoder()),
+ )
except Exception: # noqa: BLE001
return None
with _PROFILES_LOCK:
diff --git a/studio/backend/core/inference/incremental_streamer.py b/studio/backend/core/inference/incremental_streamer.py
new file mode 100644
index 00000000000..891765c6fe3
--- /dev/null
+++ b/studio/backend/core/inference/incremental_streamer.py
@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""``TextIteratorStreamer`` that decodes long lines incrementally instead of re-decoding the whole
+line per token; chunks equal upstream ``put``, which short lines and non-splitting tokenizers keep."""
+
+from __future__ import annotations
+
+from transformers import TextIteratorStreamer
+
+from core.inference.incremental_decode import IncrementalDecode, split_profile
+from core.inference.native_tool_tokens import NativeToolTokenDecoder
+
+__all__ = ["IncrementalTextIteratorStreamer"]
+
+# Below this many ids upstream's whole-line decode is cheaper than feeding a decoder.
+_MIN_LINE_IDS = 64
+
+
+def _line_decoder(tokenizer, decode_kwargs):
+ if isinstance(tokenizer, NativeToolTokenDecoder):
+ return tokenizer.incremental() if split_profile(tokenizer._tokenizer) else None
+ return IncrementalDecode(tokenizer, decode_kwargs) if split_profile(tokenizer) else None
+
+
+class IncrementalTextIteratorStreamer(TextIteratorStreamer):
+ _decoder = False # False = unresolved, None = never splits
+ _line_ids = None # token_cache list _decoder consumed; upstream rebinds it on every flush
+
+ def _attach(self) -> bool:
+ if len(self.token_cache) < _MIN_LINE_IDS:
+ return False
+ if self._decoder is False:
+ self._decoder = _line_decoder(self.tokenizer, self.decode_kwargs)
+ if self._decoder is None:
+ return False
+ self._decoder.reset()
+ for token_id in self.token_cache:
+ self._decoder.feed(token_id)
+ self._line_ids = self.token_cache
+ return True
+
+ def put(self, value):
+ if self._line_ids is not self.token_cache and not self._attach():
+ return super().put(value)
+
+ if len(value.shape) > 1 and value.shape[0] > 1:
+ raise ValueError("TextStreamer only supports batch size 1")
+ elif len(value.shape) > 1:
+ value = value[0]
+
+ if self.skip_prompt and self.next_tokens_are_prompt:
+ self.next_tokens_are_prompt = False
+ return
+
+ new_ids = value.tolist()
+ self.token_cache.extend(new_ids)
+ for token_id in new_ids:
+ self._decoder.feed(token_id)
+ text = self._decoder.text
+
+ if text.endswith("\n"):
+ printable_text = text[self.print_len :]
+ self.token_cache = []
+ self.print_len = 0
+ elif len(text) > 0 and self._is_chinese_char(ord(text[-1])):
+ printable_text = text[self.print_len :]
+ self.print_len += len(printable_text)
+ else:
+ # rfind from print_len slices the same text as upstream's whole-line rfind.
+ printable_text = text[self.print_len : text.rfind(" ", self.print_len) + 1]
+ self.print_len += len(printable_text)
+
+ self.on_finalized_text(printable_text)
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 1e51b84b0b6..8dbc7733a2d 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -63,6 +63,7 @@ from core.inference.generation_timing import (
build_generation_timings,
with_prefill_boundary_processor,
)
+from core.inference.incremental_streamer import IncrementalTextIteratorStreamer
from core.inference.native_tool_tokens import (
NativeToolTokenDecoder,
closes_an_open_envelope,
@@ -473,7 +474,7 @@ class HarmonyTextStreamer:
self._final_emitted = end - start
-class ReasoningTextIteratorStreamer(TextIteratorStreamer):
+class ReasoningTextIteratorStreamer(IncrementalTextIteratorStreamer):
"""TextIteratorStreamer that preserves native channel tokens until parsed."""
def __init__(
@@ -2649,11 +2650,10 @@ class InferenceBackend:
).to(model.device)
try:
- from transformers import TextIteratorStreamer
from queue import Empty
# As the text path: a rendered catalogue keeps native tool-call tokens for the healer.
- streamer = TextIteratorStreamer(
+ streamer = IncrementalTextIteratorStreamer(
NativeToolTokenDecoder(raw_tokenizer) if tools else raw_tokenizer,
skip_prompt = True,
skip_special_tokens = not tools,
@@ -2830,7 +2830,7 @@ class InferenceBackend:
)
except Exception as e:
logger.warning(f"HarmonyTextStreamer init failed, falling back: {e}")
- return TextIteratorStreamer(
+ return IncrementalTextIteratorStreamer(
tokenizer,
skip_prompt = skip_prompt,
skip_special_tokens = True,
@@ -2853,7 +2853,7 @@ class InferenceBackend:
in_reasoning = prompt_opens_reasoning_channel(prompt, markers, continued),
)
stream_tokenizer = NativeToolTokenDecoder(tokenizer) if preserve_tool_tokens else tokenizer
- return TextIteratorStreamer(
+ return IncrementalTextIteratorStreamer(
stream_tokenizer,
skip_prompt = skip_prompt,
skip_special_tokens = not preserve_tool_tokens,
diff --git a/studio/backend/core/inference/native_tool_tokens.py b/studio/backend/core/inference/native_tool_tokens.py
index 1aeab9ae809..3ff4a7cff9b 100644
--- a/studio/backend/core/inference/native_tool_tokens.py
+++ b/studio/backend/core/inference/native_tool_tokens.py
@@ -310,6 +310,9 @@ class _IncrementalNativeDecode:
def text(self) -> str:
return self._text.text
+ def reset(self):
+ self._text.reset()
+
def feed(self, token_id) -> str:
if self._decoder._special_ids and self._decoder.suppresses(token_id):
return self._text.text
diff --git a/studio/backend/tests/test_incremental_streamer.py b/studio/backend/tests/test_incremental_streamer.py
new file mode 100644
index 00000000000..8d08abdd528
--- /dev/null
+++ b/studio/backend/tests/test_incremental_streamer.py
@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""IncrementalTextIteratorStreamer emits exactly upstream TextIteratorStreamer's chunks while
+decoding only the ids since the last split. In-memory tokenizers, no download."""
+
+import random
+import sys
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("tokenizers")
+pytest.importorskip("transformers")
+
+from transformers import TextIteratorStreamer
+
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+
+import core.inference.incremental_streamer as incremental_streamer
+from core.inference.incremental_streamer import IncrementalTextIteratorStreamer
+from core.inference.native_tool_tokens import NativeToolTokenDecoder
+from test_mlx_incremental_decode import ( # noqa: E402
+ _CORPUS,
+ _TOKENIZERS,
+ _byte_level,
+ _counting,
+ _streams,
+)
+
+
+class _Recording:
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.chunks = []
+
+ def on_finalized_text(
+ self,
+ text,
+ stream_end = False,
+ ):
+ self.chunks.append((text, stream_end))
+
+
+class _Upstream(_Recording, TextIteratorStreamer):
+ pass
+
+
+class _Incremental(_Recording, IncrementalTextIteratorStreamer):
+ pass
+
+
+def _drive(cls, source, ids, *, skip_prompt, sizes, generations, **decode_kwargs):
+ streamer = cls(source, skip_prompt = skip_prompt, **decode_kwargs)
+ for _ in range(generations):
+ streamer.put(torch.tensor([ids[:3] or [0]]))
+ index = 0
+ for size in sizes:
+ if index >= len(ids):
+ break
+ streamer.put(torch.tensor(ids[index : index + size]))
+ index += size
+ streamer.end()
+ return streamer
+
+
[email protected](scope = "module", params = sorted(_TOKENIZERS))
+def tokenizer(request):
+ return _TOKENIZERS[request.param]()
+
+
[email protected]("mode", ["plain", "keep-special", "native-tool", "reasoning"])
+def test_chunks_equal_upstream(tokenizer, mode):
+ rng = random.Random(mode)
+ newline_ids = {i for i in range(len(tokenizer)) if "\n" in tokenizer.decode([i])}
+ streams = dict(_streams(tokenizer, seed = 2))
+ for name, ids in list(streams.items()):
+ streams[f"{name}-one-line"] = [i for i in ids if i not in newline_ids]
+ for name, ids in streams.items():
+ if mode == "plain":
+ make, kwargs = (lambda: tokenizer), {"skip_special_tokens": True}
+ elif mode == "keep-special":
+ make, kwargs = (lambda: tokenizer), {"skip_special_tokens": False}
+ elif mode == "native-tool":
+ make, kwargs = (lambda: NativeToolTokenDecoder(tokenizer)), {}
+ else:
+ make = lambda: NativeToolTokenDecoder(
+ tokenizer, preserved_tokens = ("<think>", "</think>")
+ )
+ kwargs = {"skip_special_tokens": False}
+ sizes = [1 if rng.random() < 0.9 else rng.randint(2, 5) for _ in ids]
+ for skip_prompt in (True, False):
+ options = dict(skip_prompt = skip_prompt, sizes = sizes, generations = 2, **kwargs)
+ expected = _drive(_Upstream, make(), ids, **options).chunks
+ streamer = _drive(_Incremental, make(), ids, **options)
+ assert streamer.chunks == expected, (name, skip_prompt)
+ if name.endswith("-one-line") and len(ids) > 2 * incremental_streamer._MIN_LINE_IDS:
+ assert streamer._decoder, name
+
+
+def test_work_per_token_stays_bounded():
+ tokenizer = _byte_level(cleanup = True)
+ ids = tokenizer.encode(" ".join(_CORPUS[:1]) * 120, add_special_tokens = False)
+ assert len(ids) > 2000 and "\n" not in tokenizer.decode(ids)
+ counter = _counting(tokenizer)
+ streamer = _Incremental(tokenizer, skip_prompt = False, skip_special_tokens = True)
+ for token_id in ids:
+ streamer.put(torch.tensor([token_id]))
+ # Upstream re-decodes the whole line every token: about len(ids) ** 2 / 2 ids.
+ assert counter.decoded <= len(ids) * 32
+
+
+def test_decoder_resolved_once_and_only_for_long_lines(monkeypatch):
+ tokenizer = _byte_level(cleanup = False)
+ calls = []
+ real = incremental_streamer._line_decoder
+ monkeypatch.setattr(
+ incremental_streamer,
+ "_line_decoder",
+ lambda *args: calls.append(1) or real(*args),
+ )
+ word = tokenizer.encode(" word", add_special_tokens = False)
+ newline = tokenizer.encode("\n", add_special_tokens = False)
+ short_line = word * 8 + newline
+ long_line = word * (2 * incremental_streamer._MIN_LINE_IDS) + newline
+ streamer = _Incremental(tokenizer, skip_prompt = False)
+ for token_id in short_line * 20:
+ streamer.put(torch.tensor([token_id]))
+ assert calls == []
+ for token_id in long_line * 20:
+ streamer.put(torch.tensor([token_id]))
+ streamer.end()
+ assert calls == [1]
+ upstream = _Upstream(tokenizer, skip_prompt = False)
+ for token_id in short_line * 20 + long_line * 20:
+ upstream.put(torch.tensor([token_id]))
+ upstream.end()
+ assert streamer.chunks == upstream.chunks