Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: strip tool markup from streamed text without rescanning the whole turn per token (#14143)
Daniel Han1de12ab3091c
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex ed9e66dbe76..42784f7ea05 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -7335,6 +7335,82 @@ def _strip_tool_xml_for_display(return restored if restored is not None else strip_outside_think(text, _strip_segment)+# Every literal some arm of ``_strip_tool_xml_for_display`` needs before it can edit text. Markerless Gemma has+# none and is caught by ``_DISPLAY_GEMMA_CALL_RE``, a superset of ``_GEMMA_BARE_TC_RE``.+_DISPLAY_STRIP_SENTINELS = (+ "<tool_call",+ "</tool_call",+ "<|tool_call",+ "<function",+ "</function",+ "[TOOL_CALLS]",+ "[/TOOL_CALLS]",+ "[ARGS]",+ "<|python_tag|>",+ "<|",+ "</param",+)+_DISPLAY_SENTINEL_OVERLAP = max(len(sentinel) for sentinel in _DISPLAY_STRIP_SENTINELS) - 1+_DISPLAY_GEMMA_CALL_RE = _re.compile(r"(?<!\w)call\s*:")+# A trailing ``call`` (or a prefix of one) that later text could still turn into ``call:``.+_DISPLAY_GEMMA_TAIL_RE = _re.compile(r"(?<!\w)c(?:a(?:l(?:l\s*)?)?)?\Z")+++class _StreamingDisplayStrip:+ """``_strip_tool_xml_for_display`` for a cumulative buffer that grows one token at a time.++ The full strip rescans the whole buffer per call, quadratic over a turn on the event loop. No arm can edit+ text holding none of ``_DISPLAY_STRIP_SENTINELS`` and no ``call:``, so such a buffer is returned as is and the+ scan resumes where it stopped. After a sentinel the full strip runs on every call, as before.+ """++ __slots__ = ("_auto_heal", "_names", "_prev", "_scanned", "_gemma_from", "_marked")++ def __init__(+ self,+ *,+ auto_heal_tool_calls: bool,+ enabled_tool_names: Optional[set] = None,+ ):+ self._auto_heal = auto_heal_tool_calls+ self._names = enabled_tool_names+ self._reset()++ def _reset(self) -> None:+ self._prev = ""+ self._scanned = 0+ self._gemma_from = 0+ self._marked = False++ def _has_sentinel(self, text: str) -> bool:+ start = max(0, self._scanned - _DISPLAY_SENTINEL_OVERLAP)+ if any(text.find(sentinel, start) >= 0 for sentinel in _DISPLAY_STRIP_SENTINELS):+ return True+ # Searched on the whole string from an offset, so ``(?<!\w)`` still reads the character before it.+ if _DISPLAY_GEMMA_CALL_RE.search(text, self._gemma_from) is not None:+ return True+ tail = _DISPLAY_GEMMA_TAIL_RE.search(text, max(self._gemma_from, len(text.rstrip()) - 4))+ self._gemma_from = tail.start() if tail is not None else len(text)+ self._scanned = len(text)+ return False++ def strip(self, text: str) -> str:+ if not self._auto_heal:+ return text+ if not text.startswith(self._prev):+ self._reset()+ self._prev = text+ if not self._marked and self._has_sentinel(text):+ self._marked = True+ if not self._marked:+ return text+ return _strip_tool_xml_for_display(+ text,+ auto_heal_tool_calls = True,+ enabled_tool_names = self._names,+ )++class _ReasoningSpanGuard:"""Display strip that never edits a provenance-recorded reasoning trace.@@ -33896,6 +33972,10 @@ async def produce_openai_chat_completions(# stays free for disconnect detection.gen = gguf_generate_with_tools()prev_text = ""+ _gguf_display_strip = _StreamingDisplayStrip(+ auto_heal_tool_calls = _gguf_auto_heal_tool_calls,+ enabled_tool_names = _gguf_display_tool_names,+ )reasoning_extractor = _new_chat_reasoning_extractor()_stream_usage = None_stream_timings = None@@ -34086,11 +34166,7 @@ async def produce_openai_chat_completions(# cumulative then diff against the last sanitized# snapshot so cross-chunk XML tags are handled correctly.raw_cumulative = event.get("text", "")- clean_cumulative = _strip_tool_xml_for_display(- raw_cumulative,- auto_heal_tool_calls = _gguf_auto_heal_tool_calls,- enabled_tool_names = _gguf_display_tool_names,- )+ clean_cumulative = _gguf_display_strip.strip(raw_cumulative)new_text = clean_cumulative[len(prev_text) :]prev_text = clean_cumulativeif not new_text:@@ -34333,6 +34409,7 @@ async def produce_openai_chat_completions(timings = Nonecontext_truncation = Nonegen = gguf_generate_with_tools()+ raw_text = Nonetry:for event in gen:if cancel_event.is_set():@@ -34351,11 +34428,14 @@ async def produce_openai_chat_completions(# turn's text. As in the safetensors drain, a visible# preamble emitted before a tool call (its own earlier# turn) isn't carried -- only the final turn is.- full_text = _strip_tool_xml_for_display(- event.get("text", ""),- auto_heal_tool_calls = _gguf_auto_heal_tool_calls,- enabled_tool_names = _gguf_display_tool_names,- )+ raw_text = event.get("text", "")+ if raw_text is not None:+ # Stripped once: stripping every cumulative event is quadratic in the turn.+ full_text = _strip_tool_xml_for_display(+ raw_text,+ auto_heal_tool_calls = _gguf_auto_heal_tool_calls,+ enabled_tool_names = _gguf_display_tool_names,+ )return full_text, usage, finish, timings, context_truncationfinally:# Close the generator on early break/cancel so the underlying@@ -35907,6 +35987,10 @@ async def produce_openai_chat_completions(gen = sf_generate_with_tools()prev_text = ""+ _sf_display_strip = _StreamingDisplayStrip(+ auto_heal_tool_calls = _sf_auto_heal_tool_calls,+ enabled_tool_names = _sf_display_tool_names,+ )reasoning_extractor = _new_sf_reasoning_extractor()approval_flush_pending = False@@ -36041,11 +36125,7 @@ async def produce_openai_chat_completions(# Diff cumulative cleaned text against last snapshot.raw_cumulative = event.get("text", "")- clean_cumulative = _strip_tool_xml_for_display(- raw_cumulative,- auto_heal_tool_calls = _sf_auto_heal_tool_calls,- enabled_tool_names = _sf_display_tool_names,- )+ clean_cumulative = _sf_display_strip.strip(raw_cumulative)new_text = clean_cumulative[len(prev_text) :]prev_text = clean_cumulativeif not new_text:@@ -36161,6 +36241,7 @@ async def produce_openai_chat_completions(continued = _sf_continueprefilled = _sf_resumes_thought if continued else _sf_reasoning_prefilledgen = sf_generate_with_tools()+ raw_text = Nonefor event in gen:if cancel_event.is_set():break@@ -36175,11 +36256,7 @@ async def produce_openai_chat_completions()_event_type = event.get("type")if _event_type == "content":- full_text = _strip_tool_xml_for_display(- event.get("text", ""),- auto_heal_tool_calls = _sf_auto_heal_tool_calls,- enabled_tool_names = _sf_display_tool_names,- )+ raw_text = event.get("text", "")prefilled = _sf_resumes_thought if continued else _sf_reasoning_prefilledelif _event_type == "tool_start" or (_event_type == "status" and not event.get("text")@@ -36189,6 +36266,12 @@ async def produce_openai_chat_completions(_sf_context_truncation = _accumulate_context_truncation(_sf_context_truncation, event)+ if raw_text is not None:+ full_text = _strip_tool_xml_for_display(+ raw_text,+ auto_heal_tool_calls = _sf_auto_heal_tool_calls,+ enabled_tool_names = _sf_display_tool_names,+ )return full_text, prefilledcontent_text, _sf_drain_prefilled = await asyncio.to_thread(_drain_to_text)diff --git a/studio/backend/tests/test_streaming_display_strip.py b/studio/backend/tests/test_streaming_display_strip.pynew file mode 100644index 00000000000..4dac01640e4--- /dev/null+++ b/studio/backend/tests/test_streaming_display_strip.py@@ -0,0 +1,94 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""_StreamingDisplayStrip must return exactly what _strip_tool_xml_for_display returns for every prefix of a+streamed turn, while skipping the full rescan on turns with no tool markup."""++import random++import pytest++from routes import inference+from routes.inference import _StreamingDisplayStrip, _strip_tool_xml_for_display++_NAMES = {"web_search", "python", "terminal"}+_PROSE = (+ "A hash map stores keys in buckets chosen by a hash function. Collisions chain or probe; when the load factor "+ "grows past a threshold the table resizes and rehashes. `dict[str, int]` lookups stay O(1) on average. "+)+_TRANSCRIPTS = [+ "<think>" + _PROSE * 3 + "</think>" + _PROSE * 4,+ _PROSE+ + '<tool_call>{"name": "python", "arguments": {"code": "print(1)"}}</tool_call>'+ + _PROSE,+ _PROSE + "<function=python><parameter=code>print(2)</parameter></function>" + _PROSE,+ _PROSE + '[TOOL_CALLS]python[ARGS]{"code": "print(3)"}' + _PROSE,+ _PROSE + "<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>python" + _PROSE,+ _PROSE+ + '<|tool_calls_section_begin|><|tool_call_begin|>functions.python:0 {"code": "x"}'+ + _PROSE,+ _PROSE + "<|python_tag|>print(4)" + _PROSE,+ _PROSE + 'call:python{code: "print(5)"}' + _PROSE,+ _PROSE + "so we call : python with args" + _PROSE,+ _PROSE + "recall: this is prose, not a call" + _PROSE,+ "<think>" + _PROSE + '<tool_call>{"name": "python"}</tool_call></think>' + _PROSE,+]+++def _replay(strip, text, rng):+ cut = 0+ while cut < len(text):+ cut = min(len(text), cut + rng.randint(1, 16))+ yield text[:cut], strip.strip(text[:cut])++[email protected]("names", [None, set(), _NAMES])[email protected]("auto_heal", [True, False])[email protected]("seed", range(5))+def test_every_prefix_matches_the_full_strip(names, auto_heal, seed):+ rng = random.Random(seed)+ strip = _StreamingDisplayStrip(auto_heal_tool_calls = auto_heal, enabled_tool_names = names)+ # One instance across turns, as a tool loop restarts its cumulative text every iteration.+ for text in rng.sample(_TRANSCRIPTS, len(_TRANSCRIPTS)):+ for prefix, got in _replay(strip, text, rng):+ assert got == _strip_tool_xml_for_display(+ prefix, auto_heal_tool_calls = auto_heal, enabled_tool_names = names+ ), prefix+++def test_a_call_split_across_tokens_is_still_stripped():+ strip = _StreamingDisplayStrip(auto_heal_tool_calls = True, enabled_tool_names = _NAMES)+ text = _PROSE + 'call:python{code: "print(5)"}'+ for prefix in (_PROSE + "ca", _PROSE + "call", _PROSE + "call :", text):+ assert strip.strip(prefix) == _strip_tool_xml_for_display(+ prefix, auto_heal_tool_calls = True, enabled_tool_names = _NAMES+ )++[email protected]("gap", ["", " " * 70, "\n" * 70], ids = ["no-gap", "spaces", "newlines"])+def test_a_call_with_a_long_gap_before_its_colon_is_still_stripped(gap):+ strip = _StreamingDisplayStrip(auto_heal_tool_calls = True, enabled_tool_names = _NAMES)+ text = "call" + gap + ":web_search{query: x} tail"+ for cut in range(1, len(text) + 1):+ assert strip.strip(text[:cut]) == _strip_tool_xml_for_display(+ text[:cut], auto_heal_tool_calls = True, enabled_tool_names = _NAMES+ ), cut+++def test_prose_turns_skip_the_full_rescan(monkeypatch):+ calls = []+ original = inference._strip_tool_xml_for_display++ def counting(*args, **kwargs):+ calls.append(1)+ return original(*args, **kwargs)++ monkeypatch.setattr(inference, "_strip_tool_xml_for_display", counting)+ strip = _StreamingDisplayStrip(auto_heal_tool_calls = True, enabled_tool_names = _NAMES)+ text = _TRANSCRIPTS[0]+ for cut in range(1, len(text) + 1, 4):+ assert strip.strip(text[:cut]) == text[:cut]+ assert calls == []+ marked = _TRANSCRIPTS[1]+ strip.strip(marked)+ assert len(calls) == 1