Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: strip tool markup from streamed text without rescanning the whole turn per token (#14143)

Daniel Han1de12ab3091c
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index ed9e66dbe76..42784f7ea05 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -7335,6 +7335,82 @@ def _strip_tool_xml_for_display(
return restored if restored is not None else strip_outside_think(text, _strip_segment)
+# Every literal some arm of ``_strip_tool_xml_for_display`` needs before it can edit text. Markerless Gemma has
+# none and is caught by ``_DISPLAY_GEMMA_CALL_RE``, a superset of ``_GEMMA_BARE_TC_RE``.
+_DISPLAY_STRIP_SENTINELS = (
+ "<tool_call",
+ "</tool_call",
+ "<|tool_call",
+ "<function",
+ "</function",
+ "[TOOL_CALLS]",
+ "[/TOOL_CALLS]",
+ "[ARGS]",
+ "<|python_tag|>",
+ "<|",
+ "</param",
+)
+_DISPLAY_SENTINEL_OVERLAP = max(len(sentinel) for sentinel in _DISPLAY_STRIP_SENTINELS) - 1
+_DISPLAY_GEMMA_CALL_RE = _re.compile(r"(?<!\w)call\s*:")
+# A trailing ``call`` (or a prefix of one) that later text could still turn into ``call:``.
+_DISPLAY_GEMMA_TAIL_RE = _re.compile(r"(?<!\w)c(?:a(?:l(?:l\s*)?)?)?\Z")
+
+
+class _StreamingDisplayStrip:
+ """``_strip_tool_xml_for_display`` for a cumulative buffer that grows one token at a time.
+
+ The full strip rescans the whole buffer per call, quadratic over a turn on the event loop. No arm can edit
+ text holding none of ``_DISPLAY_STRIP_SENTINELS`` and no ``call:``, so such a buffer is returned as is and the
+ scan resumes where it stopped. After a sentinel the full strip runs on every call, as before.
+ """
+
+ __slots__ = ("_auto_heal", "_names", "_prev", "_scanned", "_gemma_from", "_marked")
+
+ def __init__(
+ self,
+ *,
+ auto_heal_tool_calls: bool,
+ enabled_tool_names: Optional[set] = None,
+ ):
+ self._auto_heal = auto_heal_tool_calls
+ self._names = enabled_tool_names
+ self._reset()
+
+ def _reset(self) -> None:
+ self._prev = ""
+ self._scanned = 0
+ self._gemma_from = 0
+ self._marked = False
+
+ def _has_sentinel(self, text: str) -> bool:
+ start = max(0, self._scanned - _DISPLAY_SENTINEL_OVERLAP)
+ if any(text.find(sentinel, start) >= 0 for sentinel in _DISPLAY_STRIP_SENTINELS):
+ return True
+ # Searched on the whole string from an offset, so ``(?<!\w)`` still reads the character before it.
+ if _DISPLAY_GEMMA_CALL_RE.search(text, self._gemma_from) is not None:
+ return True
+ tail = _DISPLAY_GEMMA_TAIL_RE.search(text, max(self._gemma_from, len(text.rstrip()) - 4))
+ self._gemma_from = tail.start() if tail is not None else len(text)
+ self._scanned = len(text)
+ return False
+
+ def strip(self, text: str) -> str:
+ if not self._auto_heal:
+ return text
+ if not text.startswith(self._prev):
+ self._reset()
+ self._prev = text
+ if not self._marked and self._has_sentinel(text):
+ self._marked = True
+ if not self._marked:
+ return text
+ return _strip_tool_xml_for_display(
+ text,
+ auto_heal_tool_calls = True,
+ enabled_tool_names = self._names,
+ )
+
+
class _ReasoningSpanGuard:
"""Display strip that never edits a provenance-recorded reasoning trace.
@@ -33896,6 +33972,10 @@ async def produce_openai_chat_completions(
# stays free for disconnect detection.
gen = gguf_generate_with_tools()
prev_text = ""
+ _gguf_display_strip = _StreamingDisplayStrip(
+ auto_heal_tool_calls = _gguf_auto_heal_tool_calls,
+ enabled_tool_names = _gguf_display_tool_names,
+ )
reasoning_extractor = _new_chat_reasoning_extractor()
_stream_usage = None
_stream_timings = None
@@ -34086,11 +34166,7 @@ async def produce_openai_chat_completions(
# cumulative then diff against the last sanitized
# snapshot so cross-chunk XML tags are handled correctly.
raw_cumulative = event.get("text", "")
- clean_cumulative = _strip_tool_xml_for_display(
- raw_cumulative,
- auto_heal_tool_calls = _gguf_auto_heal_tool_calls,
- enabled_tool_names = _gguf_display_tool_names,
- )
+ clean_cumulative = _gguf_display_strip.strip(raw_cumulative)
new_text = clean_cumulative[len(prev_text) :]
prev_text = clean_cumulative
if not new_text:
@@ -34333,6 +34409,7 @@ async def produce_openai_chat_completions(
timings = None
context_truncation = None
gen = gguf_generate_with_tools()
+ raw_text = None
try:
for event in gen:
if cancel_event.is_set():
@@ -34351,11 +34428,14 @@ async def produce_openai_chat_completions(
# turn's text. As in the safetensors drain, a visible
# preamble emitted before a tool call (its own earlier
# turn) isn't carried -- only the final turn is.
- full_text = _strip_tool_xml_for_display(
- event.get("text", ""),
- auto_heal_tool_calls = _gguf_auto_heal_tool_calls,
- enabled_tool_names = _gguf_display_tool_names,
- )
+ raw_text = event.get("text", "")
+ if raw_text is not None:
+ # Stripped once: stripping every cumulative event is quadratic in the turn.
+ full_text = _strip_tool_xml_for_display(
+ raw_text,
+ auto_heal_tool_calls = _gguf_auto_heal_tool_calls,
+ enabled_tool_names = _gguf_display_tool_names,
+ )
return full_text, usage, finish, timings, context_truncation
finally:
# Close the generator on early break/cancel so the underlying
@@ -35907,6 +35987,10 @@ async def produce_openai_chat_completions(
gen = sf_generate_with_tools()
prev_text = ""
+ _sf_display_strip = _StreamingDisplayStrip(
+ auto_heal_tool_calls = _sf_auto_heal_tool_calls,
+ enabled_tool_names = _sf_display_tool_names,
+ )
reasoning_extractor = _new_sf_reasoning_extractor()
approval_flush_pending = False
@@ -36041,11 +36125,7 @@ async def produce_openai_chat_completions(
# Diff cumulative cleaned text against last snapshot.
raw_cumulative = event.get("text", "")
- clean_cumulative = _strip_tool_xml_for_display(
- raw_cumulative,
- auto_heal_tool_calls = _sf_auto_heal_tool_calls,
- enabled_tool_names = _sf_display_tool_names,
- )
+ clean_cumulative = _sf_display_strip.strip(raw_cumulative)
new_text = clean_cumulative[len(prev_text) :]
prev_text = clean_cumulative
if not new_text:
@@ -36161,6 +36241,7 @@ async def produce_openai_chat_completions(
continued = _sf_continue
prefilled = _sf_resumes_thought if continued else _sf_reasoning_prefilled
gen = sf_generate_with_tools()
+ raw_text = None
for event in gen:
if cancel_event.is_set():
break
@@ -36175,11 +36256,7 @@ async def produce_openai_chat_completions(
)
_event_type = event.get("type")
if _event_type == "content":
- full_text = _strip_tool_xml_for_display(
- event.get("text", ""),
- auto_heal_tool_calls = _sf_auto_heal_tool_calls,
- enabled_tool_names = _sf_display_tool_names,
- )
+ raw_text = event.get("text", "")
prefilled = _sf_resumes_thought if continued else _sf_reasoning_prefilled
elif _event_type == "tool_start" or (
_event_type == "status" and not event.get("text")
@@ -36189,6 +36266,12 @@ async def produce_openai_chat_completions(
_sf_context_truncation = _accumulate_context_truncation(
_sf_context_truncation, event
)
+ if raw_text is not None:
+ full_text = _strip_tool_xml_for_display(
+ raw_text,
+ auto_heal_tool_calls = _sf_auto_heal_tool_calls,
+ enabled_tool_names = _sf_display_tool_names,
+ )
return full_text, prefilled
content_text, _sf_drain_prefilled = await asyncio.to_thread(_drain_to_text)
diff --git a/studio/backend/tests/test_streaming_display_strip.py b/studio/backend/tests/test_streaming_display_strip.py
new file mode 100644
index 00000000000..4dac01640e4
--- /dev/null
+++ b/studio/backend/tests/test_streaming_display_strip.py
@@ -0,0 +1,94 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""_StreamingDisplayStrip must return exactly what _strip_tool_xml_for_display returns for every prefix of a
+streamed turn, while skipping the full rescan on turns with no tool markup."""
+
+import random
+
+import pytest
+
+from routes import inference
+from routes.inference import _StreamingDisplayStrip, _strip_tool_xml_for_display
+
+_NAMES = {"web_search", "python", "terminal"}
+_PROSE = (
+ "A hash map stores keys in buckets chosen by a hash function. Collisions chain or probe; when the load factor "
+ "grows past a threshold the table resizes and rehashes. `dict[str, int]` lookups stay O(1) on average. "
+)
+_TRANSCRIPTS = [
+ "<think>" + _PROSE * 3 + "</think>" + _PROSE * 4,
+ _PROSE
+ + '<tool_call>{"name": "python", "arguments": {"code": "print(1)"}}</tool_call>'
+ + _PROSE,
+ _PROSE + "<function=python><parameter=code>print(2)</parameter></function>" + _PROSE,
+ _PROSE + '[TOOL_CALLS]python[ARGS]{"code": "print(3)"}' + _PROSE,
+ _PROSE + "<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>python" + _PROSE,
+ _PROSE
+ + '<|tool_calls_section_begin|><|tool_call_begin|>functions.python:0 {"code": "x"}'
+ + _PROSE,
+ _PROSE + "<|python_tag|>print(4)" + _PROSE,
+ _PROSE + 'call:python{code: "print(5)"}' + _PROSE,
+ _PROSE + "so we call : python with args" + _PROSE,
+ _PROSE + "recall: this is prose, not a call" + _PROSE,
+ "<think>" + _PROSE + '<tool_call>{"name": "python"}</tool_call></think>' + _PROSE,
+]
+
+
+def _replay(strip, text, rng):
+ cut = 0
+ while cut < len(text):
+ cut = min(len(text), cut + rng.randint(1, 16))
+ yield text[:cut], strip.strip(text[:cut])
+
+
[email protected]("names", [None, set(), _NAMES])
[email protected]("auto_heal", [True, False])
[email protected]("seed", range(5))
+def test_every_prefix_matches_the_full_strip(names, auto_heal, seed):
+ rng = random.Random(seed)
+ strip = _StreamingDisplayStrip(auto_heal_tool_calls = auto_heal, enabled_tool_names = names)
+ # One instance across turns, as a tool loop restarts its cumulative text every iteration.
+ for text in rng.sample(_TRANSCRIPTS, len(_TRANSCRIPTS)):
+ for prefix, got in _replay(strip, text, rng):
+ assert got == _strip_tool_xml_for_display(
+ prefix, auto_heal_tool_calls = auto_heal, enabled_tool_names = names
+ ), prefix
+
+
+def test_a_call_split_across_tokens_is_still_stripped():
+ strip = _StreamingDisplayStrip(auto_heal_tool_calls = True, enabled_tool_names = _NAMES)
+ text = _PROSE + 'call:python{code: "print(5)"}'
+ for prefix in (_PROSE + "ca", _PROSE + "call", _PROSE + "call :", text):
+ assert strip.strip(prefix) == _strip_tool_xml_for_display(
+ prefix, auto_heal_tool_calls = True, enabled_tool_names = _NAMES
+ )
+
+
[email protected]("gap", ["", " " * 70, "\n" * 70], ids = ["no-gap", "spaces", "newlines"])
+def test_a_call_with_a_long_gap_before_its_colon_is_still_stripped(gap):
+ strip = _StreamingDisplayStrip(auto_heal_tool_calls = True, enabled_tool_names = _NAMES)
+ text = "call" + gap + ":web_search{query: x} tail"
+ for cut in range(1, len(text) + 1):
+ assert strip.strip(text[:cut]) == _strip_tool_xml_for_display(
+ text[:cut], auto_heal_tool_calls = True, enabled_tool_names = _NAMES
+ ), cut
+
+
+def test_prose_turns_skip_the_full_rescan(monkeypatch):
+ calls = []
+ original = inference._strip_tool_xml_for_display
+
+ def counting(*args, **kwargs):
+ calls.append(1)
+ return original(*args, **kwargs)
+
+ monkeypatch.setattr(inference, "_strip_tool_xml_for_display", counting)
+ strip = _StreamingDisplayStrip(auto_heal_tool_calls = True, enabled_tool_names = _NAMES)
+ text = _TRANSCRIPTS[0]
+ for cut in range(1, len(text) + 1, 4):
+ assert strip.strip(text[:cut]) == text[:cut]
+ assert calls == []
+ marked = _TRANSCRIPTS[1]
+ strip.strip(marked)
+ assert len(calls) == 1