Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: strip only harmony protocol tokens from gpt-oss answers (#16004)

Daniel Han21904ceaf19f
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 3d5c7d5aad0..1a2c3b80043 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -12,6 +12,7 @@ from peft import PeftModel, PeftModelForCausalLM
import contextlib
import json
+import re
import sys
import threading
import torch
@@ -854,6 +855,12 @@ def _diffusion_frame_relay(
return on_frame
+# o200k_harmony's non-reserved special tokens (tokenizer_config.json added_tokens_decoder).
+_HARMONY_PROTOCOL_TOKEN_RE = re.compile(
+ r"<\|(?:start|end|message|channel|constrain|return|call|startoftext|endoftext|endofprompt)\|>"
+)
+
+
class InferenceBackend:
"""Unified inference backend supporting text, vision, and LoRA models"""
@@ -3685,9 +3692,8 @@ class InferenceBackend:
) -> str:
"""Strip leaked response-boundary tokens after streaming."""
if self._is_gpt_oss_model():
- # Strip harmony protocol tokens (e.g. <|return|>) that leak past HarmonyTextStreamer.
- import re
- text = re.sub(r"<\|[a-z_]+\|>", "", text)
+ # Harmony vocabulary only: a literal <|im_end|> in an answer is content.
+ text = _HARMONY_PROTOCOL_TOKEN_RE.sub("", text)
return text.strip()
tokenizer = self.models.get(self.active_model_name, {}).get("tokenizer")
diff --git a/studio/backend/tests/test_gpt_oss_harmony_token_cleanup.py b/studio/backend/tests/test_gpt_oss_harmony_token_cleanup.py
new file mode 100644
index 00000000000..e8eec37f5cb
--- /dev/null
+++ b/studio/backend/tests/test_gpt_oss_harmony_token_cleanup.py
@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""gpt-oss cleanup drops only harmony protocol tokens; a literal <|im_end|> in an answer is kept."""
+
+from __future__ import annotations
+
+import ast
+import pathlib
+
+import pytest
+
+_SRC = pathlib.Path(__file__).resolve().parents[1] / "core/inference/inference.py"
+
+
+def _cleanup():
+ """The module's `import re`, regex and _clean_generated_text, without importing torch/unsloth."""
+ tree = ast.parse(_SRC.read_text(encoding = "utf-8"))
+ nodes = [
+ node
+ for node in tree.body
+ if isinstance(node, ast.Import) and any(alias.name == "re" for alias in node.names)
+ ]
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Assign) and any(
+ isinstance(t, ast.Name) and t.id == "_HARMONY_PROTOCOL_TOKEN_RE" for t in node.targets
+ ):
+ nodes.append(node)
+ if isinstance(node, ast.FunctionDef) and node.name == "_clean_generated_text":
+ nodes.append(node)
+ namespace = {}
+ exec(compile(ast.Module(body = nodes, type_ignores = []), str(_SRC), "exec"), namespace)
+
+ class _GptOss:
+ def _is_gpt_oss_model(self):
+ return True
+
+ return lambda text: namespace["_clean_generated_text"](_GptOss(), text)
+
+
+ "text, expected",
+ [
+ ("The answer is 4.<|return|>", "The answer is 4."),
+ ("<|start|>assistant<|channel|>final<|message|>Hi<|end|>", "assistantfinalHi"),
+ ("Done<|call|>", "Done"),
+ ("x<|endoftext|>", "x"),
+ ],
+)
+def test_harmony_protocol_tokens_are_still_removed(text, expected):
+ assert _cleanup()(text) == expected
+
+
+ "text",
+ [
+ "parts = text.split('<|im_end|>')",
+ "Llama 3 ends a turn with <|eot_id|> and calls tools after <|python_tag|>.",
+ "ChatML opens a turn with <|im_start|>user",
+ ],
+)
+def test_other_models_markers_in_an_answer_are_kept(text):
+ assert _cleanup()(text) == text