Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: strip only harmony protocol tokens from gpt-oss answers (#16004)
Daniel Han21904ceaf19f
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex 3d5c7d5aad0..1a2c3b80043 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -12,6 +12,7 @@ from peft import PeftModel, PeftModelForCausalLMimport contextlibimport json+import reimport sysimport threadingimport torch@@ -854,6 +855,12 @@ def _diffusion_frame_relay(return on_frame+# o200k_harmony's non-reserved special tokens (tokenizer_config.json added_tokens_decoder).+_HARMONY_PROTOCOL_TOKEN_RE = re.compile(+ r"<\|(?:start|end|message|channel|constrain|return|call|startoftext|endoftext|endofprompt)\|>"+)++class InferenceBackend:"""Unified inference backend supporting text, vision, and LoRA models"""@@ -3685,9 +3692,8 @@ class InferenceBackend:) -> str:"""Strip leaked response-boundary tokens after streaming."""if self._is_gpt_oss_model():- # Strip harmony protocol tokens (e.g. <|return|>) that leak past HarmonyTextStreamer.- import re- text = re.sub(r"<\|[a-z_]+\|>", "", text)+ # Harmony vocabulary only: a literal <|im_end|> in an answer is content.+ text = _HARMONY_PROTOCOL_TOKEN_RE.sub("", text)return text.strip()tokenizer = self.models.get(self.active_model_name, {}).get("tokenizer")diff --git a/studio/backend/tests/test_gpt_oss_harmony_token_cleanup.py b/studio/backend/tests/test_gpt_oss_harmony_token_cleanup.pynew file mode 100644index 00000000000..e8eec37f5cb--- /dev/null+++ b/studio/backend/tests/test_gpt_oss_harmony_token_cleanup.py@@ -0,0 +1,63 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""gpt-oss cleanup drops only harmony protocol tokens; a literal <|im_end|> in an answer is kept."""++from __future__ import annotations++import ast+import pathlib++import pytest++_SRC = pathlib.Path(__file__).resolve().parents[1] / "core/inference/inference.py"+++def _cleanup():+ """The module's `import re`, regex and _clean_generated_text, without importing torch/unsloth."""+ tree = ast.parse(_SRC.read_text(encoding = "utf-8"))+ nodes = [+ node+ for node in tree.body+ if isinstance(node, ast.Import) and any(alias.name == "re" for alias in node.names)+ ]+ for node in ast.walk(tree):+ if isinstance(node, ast.Assign) and any(+ isinstance(t, ast.Name) and t.id == "_HARMONY_PROTOCOL_TOKEN_RE" for t in node.targets+ ):+ nodes.append(node)+ if isinstance(node, ast.FunctionDef) and node.name == "_clean_generated_text":+ nodes.append(node)+ namespace = {}+ exec(compile(ast.Module(body = nodes, type_ignores = []), str(_SRC), "exec"), namespace)++ class _GptOss:+ def _is_gpt_oss_model(self):+ return True++ return lambda text: namespace["_clean_generated_text"](_GptOss(), text)+++ "text, expected",+ [+ ("The answer is 4.<|return|>", "The answer is 4."),+ ("<|start|>assistant<|channel|>final<|message|>Hi<|end|>", "assistantfinalHi"),+ ("Done<|call|>", "Done"),+ ("x<|endoftext|>", "x"),+ ],+)+def test_harmony_protocol_tokens_are_still_removed(text, expected):+ assert _cleanup()(text) == expected+++ "text",+ [+ "parts = text.split('<|im_end|>')",+ "Llama 3 ends a turn with <|eot_id|> and calls tools after <|python_tag|>.",+ "ChatML opens a turn with <|im_start|>user",+ ],+)+def test_other_models_markers_in_an_answer_are_kept(text):+ assert _cleanup()(text) == text