Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: decode and parse gpt-oss Harmony streams incrementally (#15589)
Daniel Han6a3ecd6d8746
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex 791cfce1308..5dc4b7e8793 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -271,6 +271,29 @@ def _exact_model_name_for_load(config: ModelConfig, load_in_4bit: bool) -> Optioreturn config.path+def _decodes_by_concatenation(tokenizer) -> bool:+ """True when decode(a + b) == decode(a) + decode(b) whenever decode(a) ends on a whole character:+ a byte-level BPE (bytes joined, then lossy UTF-8) with no clean-up pass."""+ if getattr(tokenizer, "clean_up_tokenization_spaces", True) is not False:+ return False+ try:+ from tokenizers import decoders+ from transformers import PreTrainedTokenizerFast++ if not isinstance(tokenizer, PreTrainedTokenizerFast):+ return False+ # A subclass or instance that overrides decode / _decode may post-process the whole text.+ for name in ("decode", "_decode"):+ method = getattr(tokenizer, name, None)+ if getattr(method, "__func__", None) is not getattr(+ PreTrainedTokenizerFast, name, None+ ):+ return False+ return type(tokenizer.backend_tokenizer.decoder) is decoders.ByteLevel+ except Exception:+ return False++class HarmonyTextStreamer:"""Streaming text decoder for the gpt-oss harmony channel protocol.@@ -314,6 +337,15 @@ class HarmonyTextStreamer:self._analysis_emitted: int = 0self._final_emitted: int = 0+ # Incremental decode: text of the first _settled_n generated ids, ending on a whole character.+ self._concat_decode: bool = _decodes_by_concatenation(tokenizer)+ self._settled_text: str = ""+ self._settled_n: int = 0+ self._scan_raw: str = ""+ self._scan_pos: int = 0+ self._open_match = None+ self._stable_final: bool = False+def put(self, value):import torch@@ -332,8 +364,17 @@ class HarmonyTextStreamer:self._token_ids.extend(ids)- gen_ids = self._token_ids[self._prompt_len :]- raw = self.tokenizer.decode(gen_ids, skip_special_tokens = False)+ if self._concat_decode:+ # Re-decoding the whole reply each token is quadratic; only ids after the settled text are decoded.+ start = self._prompt_len + self._settled_n+ tail = self.tokenizer.decode(self._token_ids[start:], skip_special_tokens = False)+ raw = self._settled_text + tail+ if tail and not tail.endswith("\ufffd"):+ self._settled_text = raw+ self._settled_n = len(self._token_ids) - self._prompt_len+ else:+ gen_ids = self._token_ids[self._prompt_len :]+ raw = self.tokenizer.decode(gen_ids, skip_special_tokens = False)# A trailing U+FFFD may be a character whose bytes are still arriving; end() emits it.self._process_incremental(raw.rstrip("\ufffd"))@@ -367,39 +408,67 @@ class HarmonyTextStreamer:return valdef _process_incremental(self, raw: str) -> None:- """Parse harmony channels and emit per-channel deltas (tracked by length, not whole-text diff)."""- has_channel_token = "<|channel|>" in raw- matches = list(self._HARMONY_RE.finditer(raw))-- if has_channel_token and not matches:- return+ """Emit per-channel deltas; same output as re-matching all of ``raw``, but only the open channel is+ rescanned: a closed channel never changes, and replaying one can only close ``<think>`` (final)."""+ if not raw.startswith(self._scan_raw):+ self._scan_pos = 0+ self._open_match = None+ self._stable_final = False+ old_len = 0+ else:+ old_len = len(self._scan_raw)+ self._scan_raw = raw+ if self._stable_final and self._emitted_think_open and not self._emitted_think_close:+ self._queue.put("</think>")+ self._emitted_think_close = True- if not has_channel_token and not matches:- return+ pos = self._scan_pos+ if self._open_match is not None:+ channel, start = self._open_match+ # The open channel ran to the old end, so a terminator can only start in the new tail.+ end = len(raw)+ for marker in ("<|end|>", "<|channel|>"):+ found = raw.find(marker, max(start, old_len - len(marker) + 1))+ if 0 <= found < end:+ end = found+ self._emit_channel(channel, raw, start, end)+ if end == len(raw):+ return+ self._close_channel(channel, end)+ pos = end- for m in matches:+ for m in self._HARMONY_RE.finditer(raw, pos):channel = m.group(1).lower()- content = m.group(2)-- if channel == "analysis":- if not self._emitted_think_open:- self._queue.put("<think>")- self._emitted_think_open = True-- new_content = content[self._analysis_emitted :]- if new_content:- self._analysis_emitted = len(content)- self._queue.put(new_content)-- elif channel in ("final", "assistant"):- if self._emitted_think_open and not self._emitted_think_close:- self._queue.put("</think>")- self._emitted_think_close = True-- new_content = content[self._final_emitted :]- if new_content:- self._final_emitted = len(content)- self._queue.put(new_content)+ self._emit_channel(channel, raw, m.start(2), m.end(2))+ if m.end() == len(raw):+ self._open_match = (channel, m.start(2))+ else:+ self._close_channel(channel, m.end())++ def _close_channel(self, channel: str, end: int) -> None:+ self._open_match = None+ self._scan_pos = end+ if channel in ("final", "assistant"):+ self._stable_final = True++ def _emit_channel(self, channel: str, raw: str, start: int, end: int) -> None:+ if channel == "analysis":+ if not self._emitted_think_open:+ self._queue.put("<think>")+ self._emitted_think_open = True++ if end - start > self._analysis_emitted:+ self._queue.put(raw[start + self._analysis_emitted : end])+ self._analysis_emitted = end - start++ elif channel in ("final", "assistant"):+ if self._emitted_think_open and not self._emitted_think_close:+ self._queue.put("</think>")+ self._emitted_think_close = True++ if end - start > self._final_emitted:+ self._queue.put(raw[start + self._final_emitted : end])+ self._final_emitted = end - startclass ReasoningTextIteratorStreamer(TextIteratorStreamer):diff --git a/studio/backend/tests/test_harmony_stream_incremental.py b/studio/backend/tests/test_harmony_stream_incremental.pynew file mode 100644index 00000000000..8db44b4c535--- /dev/null+++ b/studio/backend/tests/test_harmony_stream_incremental.py@@ -0,0 +1,248 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""HarmonyTextStreamer decodes and parses incrementally yet emits exactly what re-decoding and+re-parsing the whole reply on every token emits."""++import importlib+import importlib.machinery+import queue+import random+import re+import sys+import types+from pathlib import Path+from unittest.mock import MagicMock++import pytest++_BACKEND = Path(__file__).resolve().parent.parent+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++_STUBBED: list[str] = []+++def _stub_if_missing(+ name,+ attrs = (),+ named_spec = False,+):+ """Stub a missing dep so CI collection passes without unsloth and trl; real installs are kept."""+ if name in sys.modules:+ return+ try:+ importlib.import_module(name)+ return+ except Exception: # noqa: BLE001 - unusable here either way, so stub it+ pass+ _STUBBED.append(name)+ module = types.ModuleType(name)+ module.__spec__ = importlib.machinery.ModuleSpec(name, None) if named_spec else None+ module.__version__ = "0.0.0"+ module.__getattr__ = lambda _attr: MagicMock()+ for attr in attrs:+ setattr(module, attr, MagicMock())+ sys.modules[name] = module+ parent, _, child = name.rpartition(".")+ if parent and parent in sys.modules:+ setattr(sys.modules[parent], child, module)+++# torchao installed but unusable against the local torch poisons transformers for every later module.+for _torchao in (+ "torchao",+ "torchao.prototype",+ "torchao.prototype.safetensors",+ "torchao.prototype.safetensors.safetensors_support",+ "torchao.prototype.safetensors.safetensors_utils",+ "torchao.quantization",+ "torchao.dtypes",+ "torchao.float8",+ "torchao.utils",+):+ _stub_if_missing(_torchao, named_spec = True)++_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))+_stub_if_missing("unsloth_zoo")+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))++from core.inference import inference as inf # noqa: E402++# Drop the stubs now the name is bound; one left behind outlives this module.+for _name in reversed(_STUBBED):+ sys.modules.pop(_name, None)++torch = pytest.importorskip("torch")+tokenizers = pytest.importorskip("tokenizers")+transformers = pytest.importorskip("transformers")++_SPECIAL = ["<|channel|>", "<|message|>", "<|end|>", "<|start|>", "<|return|>"]+_RE = re.compile(r"<\|channel\|>(\w+)<\|message\|>(.*?)(?=<\|end\|>|<\|channel\|>|\Z)", re.DOTALL)+++class _Reference:+ """The whole-reply algorithm: decode every generated id and re-match every channel per token."""++ def __init__(self, tokenizer):+ self.tokenizer = tokenizer+ self.out = []+ self.ids = []+ self.open = self.close = False+ self.analysis = self.final = 0++ def put(self, ids):+ self.ids.extend(ids)+ self.parse(self.tokenizer.decode(self.ids, skip_special_tokens = False).rstrip("\ufffd"))++ def end(self):+ if self.ids:+ self.parse(self.tokenizer.decode(self.ids, skip_special_tokens = False))+ if self.open and not self.close:+ self.out.append("</think>")+ return self.out++ def parse(self, raw):+ for m in _RE.finditer(raw):+ channel, content = m.group(1).lower(), m.group(2)+ if channel == "analysis":+ if not self.open:+ self.out.append("<think>")+ self.open = True+ if content[self.analysis :]:+ self.out.append(content[self.analysis :])+ self.analysis = len(content)+ elif channel in ("final", "assistant"):+ if self.open and not self.close:+ self.out.append("</think>")+ self.close = True+ if content[self.final :]:+ self.out.append(content[self.final :])+ self.final = len(content)+++def _byte_level_tokenizer():+ """One token per byte, harmony markers as special tokens: splits every multibyte character."""+ from tokenizers import Tokenizer, decoders, models, pre_tokenizers+ from transformers.convert_slow_tokenizer import bytes_to_unicode++ vocab = {ch: i for i, ch in enumerate(bytes_to_unicode().values())}+ tok = Tokenizer(models.BPE(vocab = vocab, merges = []))+ tok.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space = False)+ tok.decoder = decoders.ByteLevel()+ tok.add_special_tokens(_SPECIAL)+ return transformers.PreTrainedTokenizerFast(+ tokenizer_object = tok, clean_up_tokenization_spaces = False+ )+++def _harmony_text(rng):+ words = ["the", " quick", " 饺子", " 🦥", " é", "\n", " <|end", " ü", " 中文", " ok", "\ufffd"]+ parts = []+ for _ in range(rng.randint(0, 5)):+ channel = rng.choice(["analysis", "final", "commentary", "assistant", "Final"])+ parts.append(f"<|channel|>{channel}<|message|>" if rng.random() < 0.9 else "<|channel|>ana")+ parts.append("".join(rng.choice(words) for _ in range(rng.randint(0, 40))))+ if rng.random() < 0.5:+ parts.append(rng.choice(["<|end|><|start|>assistant", "<|end|>"]))+ return "".join(parts)+++def _drain(streamer):+ out = []+ while True:+ try:+ out.append(next(streamer))+ except StopIteration:+ return out+ except queue.Empty:+ raise AssertionError("streamer did not finish")+++def test_byte_level_tokenizer_takes_the_incremental_decode():+ assert inf._decodes_by_concatenation(_byte_level_tokenizer())++ class Cleaning:+ clean_up_tokenization_spaces = True++ assert not inf._decodes_by_concatenation(Cleaning())+ assert not inf._decodes_by_concatenation(object())+++def _stripping_tokenizers():+ base = _byte_level_tokenizer()++ class StripDecode(transformers.PreTrainedTokenizerFast):+ def decode(self, *args, **kwargs):+ return super().decode(*args, **kwargs).lstrip()++ class StripPrivateDecode(transformers.PreTrainedTokenizerFast):+ def _decode(self, *args, **kwargs):+ return super()._decode(*args, **kwargs).lstrip()++ out = []+ for cls in (StripDecode, StripPrivateDecode):+ out.append(cls(tokenizer_object = base.backend_tokenizer, clean_up_tokenization_spaces = False))+ instance = _byte_level_tokenizer()+ plain = instance.decode+ instance.decode = lambda *args, **kwargs: plain(*args, **kwargs).lstrip()+ out.append(instance)+ return out+++def test_overridden_decode_takes_the_whole_reply_path():+ rng = random.Random(11)+ for tokenizer in _stripping_tokenizers():+ assert not inf._decodes_by_concatenation(tokenizer), type(tokenizer).__name__+ for _ in range(40):+ ids = tokenizer.encode(_harmony_text(rng), add_special_tokens = False)+ reference = _Reference(tokenizer)+ streamer = inf.HarmonyTextStreamer(tokenizer, skip_prompt = False)+ for token in ids:+ reference.put([token])+ streamer.put(torch.tensor([token]))+ streamer.end()+ assert _drain(streamer) == reference.end()++[email protected]("seed", range(4))+def test_stream_matches_whole_reply_decode(seed):+ tokenizer = _byte_level_tokenizer()+ vocab = len(tokenizer)+ rng = random.Random(seed)+ for _ in range(60):+ ids = tokenizer.encode(_harmony_text(rng), add_special_tokens = False)+ for _ in range(rng.randint(0, 4)):+ ids.insert(rng.randint(0, len(ids)), rng.randrange(vocab))+ reference = _Reference(tokenizer)+ streamer = inf.HarmonyTextStreamer(tokenizer, skip_prompt = True)+ streamer.put(torch.tensor([[1, 2]]))+ i = 0+ while i < len(ids):+ step = ids[i : i + (1 if rng.random() < 0.8 else rng.randint(1, 4))]+ i += len(step)+ reference.put(step)+ streamer.put(torch.tensor(step))+ streamer.end()+ assert _drain(streamer) == reference.end()+++def test_parse_matches_reference_when_text_is_rewritten():+ tokenizer = _byte_level_tokenizer()+ rng = random.Random(7)+ for _ in range(300):+ text = _harmony_text(rng)+ seen = []+ for _ in range(rng.randint(1, 20)):+ if seen and rng.random() < 0.2:+ seen.append(seen[-1][: rng.randint(0, len(seen[-1]))])+ else:+ seen.append(text[: rng.randint(len(seen[-1]) if seen else 0, len(text))])+ reference = _Reference(tokenizer)+ streamer = inf.HarmonyTextStreamer(tokenizer, skip_prompt = False)+ for raw in seen:+ reference.parse(raw)+ streamer._process_incremental(raw)+ streamer.end()+ assert _drain(streamer) == reference.end()