Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop re-stripping an open tool call body on every streamed token (#15707)

Daniel Han7945fb8c160d
diff --git a/studio/backend/core/inference/tool_call_parser.py b/studio/backend/core/inference/tool_call_parser.py
index 3c2ce4eb0d3..a07e5e3d165 100644
--- a/studio/backend/core/inference/tool_call_parser.py
+++ b/studio/backend/core/inference/tool_call_parser.py
@@ -1405,6 +1405,24 @@ _SENTINEL_MAX_LEN = max(len(sentinel) for sentinel in _STRIP_SENTINELS)
# Bytes sampled from each end when checking that a buffer continues the previous one.
_EXTENSION_SAMPLE = 64
+# Literals that can change the strip of an open ``prefix + <tool_call>{...`` body (see ``_gate_try_arm``).
+_OPEN_CALL_OPENER = "<tool_call>"
+_OPEN_CALL_TRIGGERS = (
+ "<tool_call",
+ "</tool_call",
+ "<|tool_call",
+ "<function",
+ "[TOOL_CALLS]",
+ "[ARGS]",
+ "<|",
+ "<think",
+ "</think",
+ "[THINK",
+ "[/THINK",
+)
+_OPEN_CALL_TRIGGER_MAX_LEN = max(len(trigger) for trigger in _OPEN_CALL_TRIGGERS)
+_OPEN_CALL_WS_RE = re.compile(r"\s*")
+
def _promotable_gemma_call_pos(text: str, start: int, enabled_tool_names) -> int:
"""The sentinel find is exact, not a heuristic: sweeping call-free prose per chunk was quadratic."""
@@ -1524,6 +1542,12 @@ class StreamingMarkupStripper:
"_seen_head",
"_seen_mid",
"_seen_tail",
+ "_gate_on",
+ "_gate_out",
+ "_gate_dead",
+ "_gate_from",
+ "_gate_scanned",
+ "_gate_call_ws",
)
def __init__(
@@ -1554,6 +1578,98 @@ class StreamingMarkupStripper:
self._seen_head = ""
self._seen_mid = ""
self._seen_tail = ""
+ self._gate_on = False
+ self._gate_out = ""
+ # (floor, first) the gate can never arm for; append-only growth cannot undo its reason.
+ self._gate_dead = None
+ self._gate_from = 0
+ self._gate_scanned = 0
+ # Where a ``call`` followed only by whitespace so far resumes its colon check, else -1.
+ self._gate_call_ws = -1
+
+ def _gate_clean(self, text: str) -> bool:
+ """Scan only text appended since the last call (plus an overlap) for anything that could change the strip."""
+ n = len(text)
+ lo = self._gate_scanned - _OPEN_CALL_TRIGGER_MAX_LEN + 1
+ if lo < self._gate_from:
+ lo = self._gate_from
+ for trigger in _OPEN_CALL_TRIGGERS:
+ if text.find(trigger, lo) >= 0:
+ return False
+ at = self._gate_call_ws
+ if at >= 0:
+ at = _OPEN_CALL_WS_RE.match(text, at).end()
+ if at < n:
+ if text[at] == ":":
+ return False
+ at = -1
+ self._gate_call_ws = at
+ lo = self._gate_scanned - len(_GEMMA_BARE_SENTINEL) + 1
+ found = text.find(_GEMMA_BARE_SENTINEL, lo if lo > self._gate_from else self._gate_from)
+ while found >= 0:
+ at = _OPEN_CALL_WS_RE.match(text, found + len(_GEMMA_BARE_SENTINEL)).end()
+ if at == n:
+ self._gate_call_ws = at
+ break
+ if text[at] == ":":
+ return False
+ found = text.find(_GEMMA_BARE_SENTINEL, found + 1)
+ self._gate_scanned = n
+ return True
+
+ def _gate_try_arm(self, text: str, seg_start: int, out: str, seg_out: str):
+ """Arm when ``text[seg_start:]`` is ``P + <tool_call> + B``: P without ``<``, ``[``, ``{``, ``call\\s*:``;
+ B starting ``\\s*{`` without a trigger or ``call\\s*:``. Then only ``<tool_call>.*$`` matches (the masking
+ walk stops at the opener, GLM needs a name, every other arm needs a ``<`` or ``[`` in P), so the segment
+ strips to P (``seg_final``) or itself, and so do the cut decisions, until a trigger arrives."""
+ key = (self._floor, self._first)
+ if self._gate_dead == key:
+ return
+ start = self._floor + self._first
+ prefix = text[seg_start:start]
+ body = start + len(_OPEN_CALL_OPENER)
+ if (
+ not text.startswith(_OPEN_CALL_OPENER, start)
+ # ``<param`` in the kept head makes the cut depend on later text (``_inside_open_parameter``).
+ or text.find("<param", self._floor, seg_start) >= 0
+ or "<" in prefix
+ or "[" in prefix
+ or "{" in prefix
+ or _has_gemma_bare_trigger(prefix)
+ or len(text) - self._floor > _tool_healing._MAX_BRACKET_SCAN_CHARS
+ # Holding the caller's buffer would make each of its appends a copy; only the final strip is cached.
+ or (self._seg_final and out is text)
+ ):
+ self._gate_dead = key
+ return
+ brace = _OPEN_CALL_WS_RE.match(text, body).end()
+ if brace == len(text):
+ return
+ # Fail closed if the full strip just run disagrees with the reasoning above.
+ if text[brace] != "{" or seg_out != (prefix if self._seg_final else text[seg_start:]):
+ self._gate_dead = key
+ return
+ self._gate_from = body
+ self._gate_scanned = body
+ self._gate_call_ws = -1
+ if not self._gate_clean(text):
+ self._gate_dead = key
+ return
+ self._gate_on = True
+ self._gate_out = out if self._seg_final else ""
+
+ def _gate_hold(self, text: str):
+ """The cached result while the gate holds, else None (and the gate is dropped for this call)."""
+ if len(text) - self._floor > _tool_healing._MAX_BRACKET_SCAN_CHARS or not self._gate_clean(
+ text
+ ):
+ self._gate_on = False
+ self._gate_out = ""
+ self._gate_dead = (self._floor, self._first)
+ return None
+ if self._seg_final:
+ return self._gate_out
+ return text if self._floor_identity else self._floor_out + text[self._floor :]
def _think_block(self, text: str, first: int) -> str:
"""Returns settled, open, or empty (full strip); fires only if the opener is the first sentinel and the body has no other markup, as a closer in args is data."""
@@ -1651,13 +1767,22 @@ class StreamingMarkupStripper:
if not self._is_extension(text):
self._reset()
+ if self._gate_on:
+ out = self._gate_hold(text)
+ if out is not None:
+ self._note(text)
+ return out
+
if self._degenerate:
# Guard on ``_floor``: with nothing settled both arms match, so the check would only add two scans.
- out = (
- self._floor_out + self._full_strip(text[self._floor :])
- if self._floor and not self._needs_whole_buffer(text)
- else self._full_strip(text)
- )
+ if self._floor and not self._needs_whole_buffer(text):
+ seg = self._full_strip(text[self._floor :])
+ out = self._floor_out + seg
+ self._gate_try_arm(text, self._floor, out, seg)
+ else:
+ out = self._full_strip(text)
+ if not self._floor:
+ self._gate_try_arm(text, 0, out, out)
self._note(text)
return out
@@ -1694,8 +1819,10 @@ class StreamingMarkupStripper:
out = self._full_strip(text)
self._note(text)
return out
- stripped = tail[:cut] + self._full_strip(tail[cut:]) if cut else self._full_strip(tail)
+ seg = self._full_strip(tail[cut:]) if cut else self._full_strip(tail)
+ stripped = tail[:cut] + seg if cut else seg
out = self._floor_out + stripped if self._floor else stripped
+ self._gate_try_arm(text, self._floor + cut, out, seg)
self._note(text)
return out
diff --git a/studio/backend/tests/data/refactor_guard/ast_inventory.json b/studio/backend/tests/data/refactor_guard/ast_inventory.json
index 70f3d28f6fc..19eaf061272 100644
--- a/studio/backend/tests/data/refactor_guard/ast_inventory.json
+++ b/studio/backend/tests/data/refactor_guard/ast_inventory.json
@@ -1129,6 +1129,7 @@
"_LLAMA3_PY_CALL_RE": "'<\\\\|python_tag\\\\|>\\\\s*([\\\\w\\\\.\\\\-]+)\\\\s*\\\\.\\\\s*call\\\\s*\\\\(' flags=32",
"_LLAMA3_WS_RE": "'\\\\s*' flags=32",
"_MISTRAL_V11_NAME_RE": "'\\\\s*([\\\\w\\\\.\\\\-]+)\\\\s*' flags=32",
+ "_OPEN_CALL_WS_RE": "'\\\\s*' flags=32",
"_OUTER_ENVELOPE_CLOSED_PATS[0]": "'<tool_call>(?:(?!<tool_call>).)*</tool_call>' flags=48",
"_OUTER_ENVELOPE_CLOSED_PATS[1]": "'<function(?:=[\\\\w.\\\\-]+|\\\\s+name=\"[\\\\w.\\\\-]+\")>(?:(?!<function(?:=[\\\\w.\\\\-]+|\\\\s+name=\"[\\\\w.\\\\-]+\")>).)*</function>' flags=48",
"_OUTER_ENVELOPE_CLOSED_PATS[2]": "'<\\\\|tool_call>.*?<tool_call\\\\|>' flags=48",
@@ -1200,7 +1201,7 @@
"re.compile('\\\\[TOOL_CALLS\\\\](?=\\\\s*(?:[\\\\[{]|[A-Za-z_][\\\\w.\\\\-]*(?:[\\\\[{]|\\\\s*$))|\\\\s*$).*$', re.DOTALL)": 1,
"re.compile('\\\\[TOOL_CALLS\\\\]\\\\s*[\\\\w\\\\.\\\\-]+\\\\s*(?:\\\\[ARGS\\\\])?\\\\s*\\\\{.*?\\\\}', re.DOTALL)": 1,
"re.compile('\\\\[TOOL_CALLS\\\\]\\\\s*\\\\[.*?\\\\](?:\\\\s*</s>)?', re.DOTALL)": 1,
- "re.compile('\\\\s*')": 1,
+ "re.compile('\\\\s*')": 2,
"re.compile('\\\\s*([A-Za-z_][\\\\w.\\\\-]*)\\\\s*:')": 1,
"re.compile('\\\\s*([\\\\w\\\\.\\\\-]+)\\\\s*')": 1,
"re.compile('\\\\s*([\\\\w\\\\.\\\\-]+)\\\\s*\\\\.\\\\s*call\\\\s*\\\\(')": 1,
@@ -1262,6 +1263,18 @@
"decorators": [],
"signature": "self, text: str"
},
+ "_gate_clean": {
+ "decorators": [],
+ "signature": "self, text: str"
+ },
+ "_gate_hold": {
+ "decorators": [],
+ "signature": "self, text: str"
+ },
+ "_gate_try_arm": {
+ "decorators": [],
+ "signature": "self, text: str, seg_start: int, out: str, seg_out: str"
+ },
"_is_extension": {
"decorators": [],
"signature": "self, text: str"
@@ -1518,6 +1531,18 @@
"_MISTRAL_V11_NAME_RE": {
"kind": "assign"
},
+ "_OPEN_CALL_OPENER": {
+ "kind": "assign"
+ },
+ "_OPEN_CALL_TRIGGERS": {
+ "kind": "assign"
+ },
+ "_OPEN_CALL_TRIGGER_MAX_LEN": {
+ "kind": "assign"
+ },
+ "_OPEN_CALL_WS_RE": {
+ "kind": "assign"
+ },
"_OUTER_ENVELOPE_CLOSED_PATS": {
"kind": "assign"
},
diff --git a/studio/backend/tests/data/refactor_guard/runtime_inventory.json b/studio/backend/tests/data/refactor_guard/runtime_inventory.json
index de3dc27c9e0..a927da46a2c 100644
--- a/studio/backend/tests/data/refactor_guard/runtime_inventory.json
+++ b/studio/backend/tests/data/refactor_guard/runtime_inventory.json
@@ -88,6 +88,10 @@
"_MISTRAL_THINK_OPEN",
"_MISTRAL_TRIGGER",
"_MISTRAL_V11_NAME_RE",
+ "_OPEN_CALL_OPENER",
+ "_OPEN_CALL_TRIGGERS",
+ "_OPEN_CALL_TRIGGER_MAX_LEN",
+ "_OPEN_CALL_WS_RE",
"_OUTER_ENVELOPE_CLOSED_PATS",
"_OUTER_ENVELOPE_OPEN_RE",
"_REPEAT_LEAD_PUNCT",
diff --git a/studio/backend/tests/test_streaming_stripper_open_call.py b/studio/backend/tests/test_streaming_stripper_open_call.py
new file mode 100644
index 00000000000..8f863deff84
--- /dev/null
+++ b/studio/backend/tests/test_streaming_stripper_open_call.py
@@ -0,0 +1,241 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""An open ``<tool_call>`` body is not re-stripped per token, and the strip results do not change."""
+
+import json
+import random
+import sys
+from pathlib import Path
+
+import pytest
+
+BACKEND_ROOT = Path(__file__).resolve().parents[1]
+if str(BACKEND_ROOT) not in sys.path:
+ sys.path.insert(0, str(BACKEND_ROOT))
+
+from core.inference.tool_call_parser import StreamingMarkupStripper # noqa: E402
+
+ENABLED = {"web_search", "python", "terminal"}
+
+
+class _FullStripEveryToken(StreamingMarkupStripper):
+ """The stripper before the open-call gate: the gate never arms, so every token strips the open tail."""
+
+ __slots__ = ()
+
+ def _gate_try_arm(self, *args):
+ return None
+
+
+def _code_tokens(count):
+ line = 'def f(x):\n return {"a": [x, "b\\\\c"], "d": f"{x}"}\n'
+ escaped = json.dumps(line)[1:-1]
+ return [escaped[i : i + 4] for i in range(0, len(escaped), 4)] * (count // 16 + 1)
+
+
+def _replay(stripper, chunks):
+ text, outs = "", []
+ for chunk in chunks:
+ text += chunk
+ outs.append(stripper.strip(text))
+ return outs
+
+
+class _CountingStripper(StreamingMarkupStripper):
+ __slots__ = ()
+ seen = [0, 0]
+
+ def _full_strip(self, text):
+ _CountingStripper.seen[0] += 1
+ _CountingStripper.seen[1] += len(text)
+ return super()._full_strip(text)
+
+
[email protected]("seg_final", [True, False])
+ "preamble", ["I'll search. ", "", "Running it:\n", "<think>plan</think>\n\n"]
+)
+def test_open_call_work_does_not_grow_with_the_body(seg_final, preamble):
+ """Counts characters handed to the full strip, not time: re-stripping the open tail per token is quadratic."""
+ opener = preamble + '<tool_call>{"name": "python", "arguments": {"code": "'
+ seen = {}
+ for count in (250, 2000):
+ chunks = [opener] + _code_tokens(count)[:count]
+ _CountingStripper.seen[:] = [0, 0]
+ outs = _replay(_CountingStripper(ENABLED, seg_final = seg_final), chunks)
+ seen[count] = tuple(_CountingStripper.seen)
+ assert outs == _replay(_FullStripEveryToken(ENABLED, seg_final = seg_final), chunks)
+ # The few calls left fall in the first ~128 characters, before ``_is_extension`` samples a middle.
+ assert seen[250] == seen[2000], seen
+ assert seen[2000][0] < 40, seen
+
+
+def test_open_call_engages_and_matches_on_close(monkeypatch):
+ """The close tag, split at every offset, still ends the gate on the token that completes it."""
+ body = (
+ 'Searching.\n<tool_call>{"name": "web_search", "arguments": {"query": "a \\"quoted\\" {x}'
+ )
+ close = '"}}</tool_call>\nDone.'
+ for split in range(1, len(close)):
+ chunks = [body] + list("pad words ") + [close[:split], close[split:]]
+ for names in (ENABLED, None):
+ got = _replay(StreamingMarkupStripper(names), chunks)
+ want = _replay(_FullStripEveryToken(names), chunks)
+ assert got == want, split
+ assert got[-1] == want[-1] == "Searching.\n\nDone."
+
+
+_TRIGGERS = (
+ "</tool_call>",
+ "<tool_call>",
+ "<|tool_call>",
+ "<function=python>",
+ "[TOOL_CALLS]",
+ "x[ARGS]{",
+ "web_search[ARGS]{}",
+ "<|tool▁calls▁begin|>",
+ "<think>",
+ "</think>",
+ "[THINK]",
+ "[/THINK]",
+ "call:web_search{q:1}",
+ "call \n\t :python{code:1}",
+ "<|python_tag|>",
+ "<param",
+ # Closed forms: the non-final strip removes these, so a missed trigger changes its result.
+ "<function=web_search>x</function>",
+ '[TOOL_CALLS]web_search{"q": 1}',
+ "[TOOL_CALLS][]",
+ "<|tool_call>call:web_search{q:1}<tool_call|>",
+ "<|tool▁calls▁begin|>x<|tool▁calls▁end|>",
+ # Kimi K2: only ``<|tool_call`` sees these, as neither holds ``<tool_call``.
+ "<|tool_calls_section_begin|><|tool_call_begin|>x<|tool_call_end|><|tool_calls_section_end|>",
+ "<|tool_call_begin|>x<|tool_call_end|>",
+ "<think>t</think>",
+ "\n</think>\n",
+ "\ncall:web_search{q:1}\n",
+)
+
+
+def test_every_strip_sentinel_is_an_open_call_trigger():
+ """A new strip arm adds its literal to ``_STRIP_SENTINELS``; the gate must see it too. The two
+ left out have no arm a non-final strip runs, and a final strip cuts them with the open call."""
+ from core.inference import tool_call_parser as parser
+
+ missing = (
+ set(parser._STRIP_SENTINELS)
+ - {parser._GEMMA_BARE_SENTINEL}
+ - set(parser._OPEN_CALL_TRIGGERS)
+ )
+ assert missing == {"<|python_tag|>", "<|content_invoke_tool_json|>"}, missing
+
+
[email protected]("trigger", _TRIGGERS)
[email protected]("seg_final", [True, False])
+def test_every_trigger_split_one_char_at_a_time_matches(trigger, seg_final):
+ # Past the ~128 characters ``_is_extension`` needs, so the gate answers later tokens.
+ prefix = 'ok <tool_call>{"name": "web_search", "arguments": {"query": "' + "words " * 40
+ chunks = [prefix] + list(trigger + ' more" }}') + ["</tool_call>", " after"]
+ for names in (ENABLED, None):
+ assert _replay(StreamingMarkupStripper(names, seg_final = seg_final), chunks) == _replay(
+ _FullStripEveryToken(names, seg_final = seg_final), chunks
+ )
+
+
+_PREFIXES = (
+ "",
+ "I'll search. ",
+ "a [link](x) ",
+ "brace {x} ",
+ "call me ",
+ "x `y` z\n",
+ "<think>p</think>\n",
+ "</think>stray ",
+ "<parameter=x>v\n",
+ "call :",
+ "  ",
+ "line\n",
+)
+_PIECES = (
+ "word ",
+ '\\"',
+ "\\\\",
+ "\\n",
+ "{",
+ "}",
+ "[",
+ "]",
+ ":",
+ "call",
+ "call ",
+ " :",
+ "recall:",
+ "<",
+ "</",
+ "<tool_call>",
+ "</tool_call>",
+ "<|tool_call>",
+ "<function=python>",
+ "[ARGS]",
+ "[TOOL_CALLS]",
+ "<think>",
+ "</think>",
+ "[/THINK]",
+ "<|",
+ " ",
+ "\n",
+ "`",
+ "<param",
+ "中文",
+)
+
+
+def test_gate_matches_full_strip_on_seeded_open_calls():
+ rng = random.Random(20261027)
+ engaged = 0
+ for _ in range(250):
+ text = rng.choice(_PREFIXES) + rng.choice(
+ ("<tool_call>", "<tool_call>\n", "<tool_call>name\n")
+ )
+ text += (
+ '{"name": "' + rng.choice(("web_search", "python", "nope")) + '", "arguments": {"q": "'
+ )
+ text += "filler " * 25
+ plain = rng.random()
+ text += "".join(
+ rng.choice(_PIECES) if rng.random() > plain else "word "
+ for _ in range(rng.randint(0, 40))
+ )
+ text += rng.choice(
+ ("", '"}}', '"}}</tool_call>', '"}}</tool_call> then <tool_call>{"name": "x", "a')
+ )
+ for names in (ENABLED, None):
+ for seg_final in (True, False):
+ gated = StreamingMarkupStripper(names, seg_final = seg_final)
+ plain_stripper = _FullStripEveryToken(names, seg_final = seg_final)
+ pos = 0
+ while pos < len(text):
+ pos = min(len(text), pos + rng.randint(1, 4))
+ was_on = gated._gate_on
+ assert gated.strip(text[:pos]) == plain_stripper.strip(text[:pos]), (text, pos)
+ engaged += was_on and gated._gate_on
+ # The gate must actually answer calls here, or this test proves nothing.
+ assert engaged > 1000, engaged
+
+
+def test_open_call_keeps_no_reference_to_the_buffer():
+ stripper = StreamingMarkupStripper(ENABLED)
+ text = 'Sure. <tool_call>{"name": "web_search", "arguments": {"query": "' + "w " * 50
+ stripper.strip(text)
+ text += "more"
+ stripper.strip(text)
+ assert stripper._gate_on
+ assert all(getattr(stripper, slot) is not text for slot in StreamingMarkupStripper.__slots__)
+
+
+def test_repeated_open_call_strip_is_idempotent():
+ stripper = StreamingMarkupStripper(ENABLED)
+ text = 'Sure. <tool_call>{"name": "web_search", "arguments": {"query": "x'
+ first = stripper.strip(text)
+ assert stripper.strip(text) == stripper.strip(text) == first == "Sure. "