Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: decode and parse gpt-oss Harmony streams incrementally (#15589)

Daniel Han6a3ecd6d8746
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 791cfce1308..5dc4b7e8793 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -271,6 +271,29 @@ def _exact_model_name_for_load(config: ModelConfig, load_in_4bit: bool) -> Optio
return config.path
+def _decodes_by_concatenation(tokenizer) -> bool:
+ """True when decode(a + b) == decode(a) + decode(b) whenever decode(a) ends on a whole character:
+ a byte-level BPE (bytes joined, then lossy UTF-8) with no clean-up pass."""
+ if getattr(tokenizer, "clean_up_tokenization_spaces", True) is not False:
+ return False
+ try:
+ from tokenizers import decoders
+ from transformers import PreTrainedTokenizerFast
+
+ if not isinstance(tokenizer, PreTrainedTokenizerFast):
+ return False
+ # A subclass or instance that overrides decode / _decode may post-process the whole text.
+ for name in ("decode", "_decode"):
+ method = getattr(tokenizer, name, None)
+ if getattr(method, "__func__", None) is not getattr(
+ PreTrainedTokenizerFast, name, None
+ ):
+ return False
+ return type(tokenizer.backend_tokenizer.decoder) is decoders.ByteLevel
+ except Exception:
+ return False
+
+
class HarmonyTextStreamer:
"""Streaming text decoder for the gpt-oss harmony channel protocol.
@@ -314,6 +337,15 @@ class HarmonyTextStreamer:
self._analysis_emitted: int = 0
self._final_emitted: int = 0
+ # Incremental decode: text of the first _settled_n generated ids, ending on a whole character.
+ self._concat_decode: bool = _decodes_by_concatenation(tokenizer)
+ self._settled_text: str = ""
+ self._settled_n: int = 0
+ self._scan_raw: str = ""
+ self._scan_pos: int = 0
+ self._open_match = None
+ self._stable_final: bool = False
+
def put(self, value):
import torch
@@ -332,8 +364,17 @@ class HarmonyTextStreamer:
self._token_ids.extend(ids)
- gen_ids = self._token_ids[self._prompt_len :]
- raw = self.tokenizer.decode(gen_ids, skip_special_tokens = False)
+ if self._concat_decode:
+ # Re-decoding the whole reply each token is quadratic; only ids after the settled text are decoded.
+ start = self._prompt_len + self._settled_n
+ tail = self.tokenizer.decode(self._token_ids[start:], skip_special_tokens = False)
+ raw = self._settled_text + tail
+ if tail and not tail.endswith("\ufffd"):
+ self._settled_text = raw
+ self._settled_n = len(self._token_ids) - self._prompt_len
+ else:
+ gen_ids = self._token_ids[self._prompt_len :]
+ raw = self.tokenizer.decode(gen_ids, skip_special_tokens = False)
# A trailing U+FFFD may be a character whose bytes are still arriving; end() emits it.
self._process_incremental(raw.rstrip("\ufffd"))
@@ -367,39 +408,67 @@ class HarmonyTextStreamer:
return val
def _process_incremental(self, raw: str) -> None:
- """Parse harmony channels and emit per-channel deltas (tracked by length, not whole-text diff)."""
- has_channel_token = "<|channel|>" in raw
- matches = list(self._HARMONY_RE.finditer(raw))
-
- if has_channel_token and not matches:
- return
+ """Emit per-channel deltas; same output as re-matching all of ``raw``, but only the open channel is
+ rescanned: a closed channel never changes, and replaying one can only close ``<think>`` (final)."""
+ if not raw.startswith(self._scan_raw):
+ self._scan_pos = 0
+ self._open_match = None
+ self._stable_final = False
+ old_len = 0
+ else:
+ old_len = len(self._scan_raw)
+ self._scan_raw = raw
+ if self._stable_final and self._emitted_think_open and not self._emitted_think_close:
+ self._queue.put("</think>")
+ self._emitted_think_close = True
- if not has_channel_token and not matches:
- return
+ pos = self._scan_pos
+ if self._open_match is not None:
+ channel, start = self._open_match
+ # The open channel ran to the old end, so a terminator can only start in the new tail.
+ end = len(raw)
+ for marker in ("<|end|>", "<|channel|>"):
+ found = raw.find(marker, max(start, old_len - len(marker) + 1))
+ if 0 <= found < end:
+ end = found
+ self._emit_channel(channel, raw, start, end)
+ if end == len(raw):
+ return
+ self._close_channel(channel, end)
+ pos = end
- for m in matches:
+ for m in self._HARMONY_RE.finditer(raw, pos):
channel = m.group(1).lower()
- content = m.group(2)
-
- if channel == "analysis":
- if not self._emitted_think_open:
- self._queue.put("<think>")
- self._emitted_think_open = True
-
- new_content = content[self._analysis_emitted :]
- if new_content:
- self._analysis_emitted = len(content)
- self._queue.put(new_content)
-
- elif channel in ("final", "assistant"):
- if self._emitted_think_open and not self._emitted_think_close:
- self._queue.put("</think>")
- self._emitted_think_close = True
-
- new_content = content[self._final_emitted :]
- if new_content:
- self._final_emitted = len(content)
- self._queue.put(new_content)
+ self._emit_channel(channel, raw, m.start(2), m.end(2))
+ if m.end() == len(raw):
+ self._open_match = (channel, m.start(2))
+ else:
+ self._close_channel(channel, m.end())
+
+ def _close_channel(self, channel: str, end: int) -> None:
+ self._open_match = None
+ self._scan_pos = end
+ if channel in ("final", "assistant"):
+ self._stable_final = True
+
+ def _emit_channel(self, channel: str, raw: str, start: int, end: int) -> None:
+ if channel == "analysis":
+ if not self._emitted_think_open:
+ self._queue.put("<think>")
+ self._emitted_think_open = True
+
+ if end - start > self._analysis_emitted:
+ self._queue.put(raw[start + self._analysis_emitted : end])
+ self._analysis_emitted = end - start
+
+ elif channel in ("final", "assistant"):
+ if self._emitted_think_open and not self._emitted_think_close:
+ self._queue.put("</think>")
+ self._emitted_think_close = True
+
+ if end - start > self._final_emitted:
+ self._queue.put(raw[start + self._final_emitted : end])
+ self._final_emitted = end - start
class ReasoningTextIteratorStreamer(TextIteratorStreamer):
diff --git a/studio/backend/tests/test_harmony_stream_incremental.py b/studio/backend/tests/test_harmony_stream_incremental.py
new file mode 100644
index 00000000000..8db44b4c535
--- /dev/null
+++ b/studio/backend/tests/test_harmony_stream_incremental.py
@@ -0,0 +1,248 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""HarmonyTextStreamer decodes and parses incrementally yet emits exactly what re-decoding and
+re-parsing the whole reply on every token emits."""
+
+import importlib
+import importlib.machinery
+import queue
+import random
+import re
+import sys
+import types
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parent.parent
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+_STUBBED: list[str] = []
+
+
+def _stub_if_missing(
+ name,
+ attrs = (),
+ named_spec = False,
+):
+ """Stub a missing dep so CI collection passes without unsloth and trl; real installs are kept."""
+ if name in sys.modules:
+ return
+ try:
+ importlib.import_module(name)
+ return
+ except Exception: # noqa: BLE001 - unusable here either way, so stub it
+ pass
+ _STUBBED.append(name)
+ module = types.ModuleType(name)
+ module.__spec__ = importlib.machinery.ModuleSpec(name, None) if named_spec else None
+ module.__version__ = "0.0.0"
+ module.__getattr__ = lambda _attr: MagicMock()
+ for attr in attrs:
+ setattr(module, attr, MagicMock())
+ sys.modules[name] = module
+ parent, _, child = name.rpartition(".")
+ if parent and parent in sys.modules:
+ setattr(sys.modules[parent], child, module)
+
+
+# torchao installed but unusable against the local torch poisons transformers for every later module.
+for _torchao in (
+ "torchao",
+ "torchao.prototype",
+ "torchao.prototype.safetensors",
+ "torchao.prototype.safetensors.safetensors_support",
+ "torchao.prototype.safetensors.safetensors_utils",
+ "torchao.quantization",
+ "torchao.dtypes",
+ "torchao.float8",
+ "torchao.utils",
+):
+ _stub_if_missing(_torchao, named_spec = True)
+
+_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
+_stub_if_missing("unsloth_zoo")
+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))
+
+from core.inference import inference as inf # noqa: E402
+
+# Drop the stubs now the name is bound; one left behind outlives this module.
+for _name in reversed(_STUBBED):
+ sys.modules.pop(_name, None)
+
+torch = pytest.importorskip("torch")
+tokenizers = pytest.importorskip("tokenizers")
+transformers = pytest.importorskip("transformers")
+
+_SPECIAL = ["<|channel|>", "<|message|>", "<|end|>", "<|start|>", "<|return|>"]
+_RE = re.compile(r"<\|channel\|>(\w+)<\|message\|>(.*?)(?=<\|end\|>|<\|channel\|>|\Z)", re.DOTALL)
+
+
+class _Reference:
+ """The whole-reply algorithm: decode every generated id and re-match every channel per token."""
+
+ def __init__(self, tokenizer):
+ self.tokenizer = tokenizer
+ self.out = []
+ self.ids = []
+ self.open = self.close = False
+ self.analysis = self.final = 0
+
+ def put(self, ids):
+ self.ids.extend(ids)
+ self.parse(self.tokenizer.decode(self.ids, skip_special_tokens = False).rstrip("\ufffd"))
+
+ def end(self):
+ if self.ids:
+ self.parse(self.tokenizer.decode(self.ids, skip_special_tokens = False))
+ if self.open and not self.close:
+ self.out.append("</think>")
+ return self.out
+
+ def parse(self, raw):
+ for m in _RE.finditer(raw):
+ channel, content = m.group(1).lower(), m.group(2)
+ if channel == "analysis":
+ if not self.open:
+ self.out.append("<think>")
+ self.open = True
+ if content[self.analysis :]:
+ self.out.append(content[self.analysis :])
+ self.analysis = len(content)
+ elif channel in ("final", "assistant"):
+ if self.open and not self.close:
+ self.out.append("</think>")
+ self.close = True
+ if content[self.final :]:
+ self.out.append(content[self.final :])
+ self.final = len(content)
+
+
+def _byte_level_tokenizer():
+ """One token per byte, harmony markers as special tokens: splits every multibyte character."""
+ from tokenizers import Tokenizer, decoders, models, pre_tokenizers
+ from transformers.convert_slow_tokenizer import bytes_to_unicode
+
+ vocab = {ch: i for i, ch in enumerate(bytes_to_unicode().values())}
+ tok = Tokenizer(models.BPE(vocab = vocab, merges = []))
+ tok.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space = False)
+ tok.decoder = decoders.ByteLevel()
+ tok.add_special_tokens(_SPECIAL)
+ return transformers.PreTrainedTokenizerFast(
+ tokenizer_object = tok, clean_up_tokenization_spaces = False
+ )
+
+
+def _harmony_text(rng):
+ words = ["the", " quick", " 饺子", " 🦥", " é", "\n", " <|end", " ü", " 中文", " ok", "\ufffd"]
+ parts = []
+ for _ in range(rng.randint(0, 5)):
+ channel = rng.choice(["analysis", "final", "commentary", "assistant", "Final"])
+ parts.append(f"<|channel|>{channel}<|message|>" if rng.random() < 0.9 else "<|channel|>ana")
+ parts.append("".join(rng.choice(words) for _ in range(rng.randint(0, 40))))
+ if rng.random() < 0.5:
+ parts.append(rng.choice(["<|end|><|start|>assistant", "<|end|>"]))
+ return "".join(parts)
+
+
+def _drain(streamer):
+ out = []
+ while True:
+ try:
+ out.append(next(streamer))
+ except StopIteration:
+ return out
+ except queue.Empty:
+ raise AssertionError("streamer did not finish")
+
+
+def test_byte_level_tokenizer_takes_the_incremental_decode():
+ assert inf._decodes_by_concatenation(_byte_level_tokenizer())
+
+ class Cleaning:
+ clean_up_tokenization_spaces = True
+
+ assert not inf._decodes_by_concatenation(Cleaning())
+ assert not inf._decodes_by_concatenation(object())
+
+
+def _stripping_tokenizers():
+ base = _byte_level_tokenizer()
+
+ class StripDecode(transformers.PreTrainedTokenizerFast):
+ def decode(self, *args, **kwargs):
+ return super().decode(*args, **kwargs).lstrip()
+
+ class StripPrivateDecode(transformers.PreTrainedTokenizerFast):
+ def _decode(self, *args, **kwargs):
+ return super()._decode(*args, **kwargs).lstrip()
+
+ out = []
+ for cls in (StripDecode, StripPrivateDecode):
+ out.append(cls(tokenizer_object = base.backend_tokenizer, clean_up_tokenization_spaces = False))
+ instance = _byte_level_tokenizer()
+ plain = instance.decode
+ instance.decode = lambda *args, **kwargs: plain(*args, **kwargs).lstrip()
+ out.append(instance)
+ return out
+
+
+def test_overridden_decode_takes_the_whole_reply_path():
+ rng = random.Random(11)
+ for tokenizer in _stripping_tokenizers():
+ assert not inf._decodes_by_concatenation(tokenizer), type(tokenizer).__name__
+ for _ in range(40):
+ ids = tokenizer.encode(_harmony_text(rng), add_special_tokens = False)
+ reference = _Reference(tokenizer)
+ streamer = inf.HarmonyTextStreamer(tokenizer, skip_prompt = False)
+ for token in ids:
+ reference.put([token])
+ streamer.put(torch.tensor([token]))
+ streamer.end()
+ assert _drain(streamer) == reference.end()
+
+
[email protected]("seed", range(4))
+def test_stream_matches_whole_reply_decode(seed):
+ tokenizer = _byte_level_tokenizer()
+ vocab = len(tokenizer)
+ rng = random.Random(seed)
+ for _ in range(60):
+ ids = tokenizer.encode(_harmony_text(rng), add_special_tokens = False)
+ for _ in range(rng.randint(0, 4)):
+ ids.insert(rng.randint(0, len(ids)), rng.randrange(vocab))
+ reference = _Reference(tokenizer)
+ streamer = inf.HarmonyTextStreamer(tokenizer, skip_prompt = True)
+ streamer.put(torch.tensor([[1, 2]]))
+ i = 0
+ while i < len(ids):
+ step = ids[i : i + (1 if rng.random() < 0.8 else rng.randint(1, 4))]
+ i += len(step)
+ reference.put(step)
+ streamer.put(torch.tensor(step))
+ streamer.end()
+ assert _drain(streamer) == reference.end()
+
+
+def test_parse_matches_reference_when_text_is_rewritten():
+ tokenizer = _byte_level_tokenizer()
+ rng = random.Random(7)
+ for _ in range(300):
+ text = _harmony_text(rng)
+ seen = []
+ for _ in range(rng.randint(1, 20)):
+ if seen and rng.random() < 0.2:
+ seen.append(seen[-1][: rng.randint(0, len(seen[-1]))])
+ else:
+ seen.append(text[: rng.randint(len(seen[-1]) if seen else 0, len(text))])
+ reference = _Reference(tokenizer)
+ streamer = inf.HarmonyTextStreamer(tokenizer, skip_prompt = False)
+ for raw in seen:
+ reference.parse(raw)
+ streamer._process_incremental(raw)
+ streamer.end()
+ assert _drain(streamer) == reference.end()