Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop re-stripping an open tool call body on every streamed token (#15707)
Daniel Han7945fb8c160d
diff --git a/studio/backend/core/inference/tool_call_parser.py b/studio/backend/core/inference/tool_call_parser.pyindex 3c2ce4eb0d3..a07e5e3d165 100644--- a/studio/backend/core/inference/tool_call_parser.py+++ b/studio/backend/core/inference/tool_call_parser.py@@ -1405,6 +1405,24 @@ _SENTINEL_MAX_LEN = max(len(sentinel) for sentinel in _STRIP_SENTINELS)# Bytes sampled from each end when checking that a buffer continues the previous one._EXTENSION_SAMPLE = 64+# Literals that can change the strip of an open ``prefix + <tool_call>{...`` body (see ``_gate_try_arm``).+_OPEN_CALL_OPENER = "<tool_call>"+_OPEN_CALL_TRIGGERS = (+ "<tool_call",+ "</tool_call",+ "<|tool_call",+ "<function",+ "[TOOL_CALLS]",+ "[ARGS]",+ "<|",+ "<think",+ "</think",+ "[THINK",+ "[/THINK",+)+_OPEN_CALL_TRIGGER_MAX_LEN = max(len(trigger) for trigger in _OPEN_CALL_TRIGGERS)+_OPEN_CALL_WS_RE = re.compile(r"\s*")+def _promotable_gemma_call_pos(text: str, start: int, enabled_tool_names) -> int:"""The sentinel find is exact, not a heuristic: sweeping call-free prose per chunk was quadratic."""@@ -1524,6 +1542,12 @@ class StreamingMarkupStripper:"_seen_head","_seen_mid","_seen_tail",+ "_gate_on",+ "_gate_out",+ "_gate_dead",+ "_gate_from",+ "_gate_scanned",+ "_gate_call_ws",)def __init__(@@ -1554,6 +1578,98 @@ class StreamingMarkupStripper:self._seen_head = ""self._seen_mid = ""self._seen_tail = ""+ self._gate_on = False+ self._gate_out = ""+ # (floor, first) the gate can never arm for; append-only growth cannot undo its reason.+ self._gate_dead = None+ self._gate_from = 0+ self._gate_scanned = 0+ # Where a ``call`` followed only by whitespace so far resumes its colon check, else -1.+ self._gate_call_ws = -1++ def _gate_clean(self, text: str) -> bool:+ """Scan only text appended since the last call (plus an overlap) for anything that could change the strip."""+ n = len(text)+ lo = self._gate_scanned - _OPEN_CALL_TRIGGER_MAX_LEN + 1+ if lo < self._gate_from:+ lo = self._gate_from+ for trigger in _OPEN_CALL_TRIGGERS:+ if text.find(trigger, lo) >= 0:+ return False+ at = self._gate_call_ws+ if at >= 0:+ at = _OPEN_CALL_WS_RE.match(text, at).end()+ if at < n:+ if text[at] == ":":+ return False+ at = -1+ self._gate_call_ws = at+ lo = self._gate_scanned - len(_GEMMA_BARE_SENTINEL) + 1+ found = text.find(_GEMMA_BARE_SENTINEL, lo if lo > self._gate_from else self._gate_from)+ while found >= 0:+ at = _OPEN_CALL_WS_RE.match(text, found + len(_GEMMA_BARE_SENTINEL)).end()+ if at == n:+ self._gate_call_ws = at+ break+ if text[at] == ":":+ return False+ found = text.find(_GEMMA_BARE_SENTINEL, found + 1)+ self._gate_scanned = n+ return True++ def _gate_try_arm(self, text: str, seg_start: int, out: str, seg_out: str):+ """Arm when ``text[seg_start:]`` is ``P + <tool_call> + B``: P without ``<``, ``[``, ``{``, ``call\\s*:``;+ B starting ``\\s*{`` without a trigger or ``call\\s*:``. Then only ``<tool_call>.*$`` matches (the masking+ walk stops at the opener, GLM needs a name, every other arm needs a ``<`` or ``[`` in P), so the segment+ strips to P (``seg_final``) or itself, and so do the cut decisions, until a trigger arrives."""+ key = (self._floor, self._first)+ if self._gate_dead == key:+ return+ start = self._floor + self._first+ prefix = text[seg_start:start]+ body = start + len(_OPEN_CALL_OPENER)+ if (+ not text.startswith(_OPEN_CALL_OPENER, start)+ # ``<param`` in the kept head makes the cut depend on later text (``_inside_open_parameter``).+ or text.find("<param", self._floor, seg_start) >= 0+ or "<" in prefix+ or "[" in prefix+ or "{" in prefix+ or _has_gemma_bare_trigger(prefix)+ or len(text) - self._floor > _tool_healing._MAX_BRACKET_SCAN_CHARS+ # Holding the caller's buffer would make each of its appends a copy; only the final strip is cached.+ or (self._seg_final and out is text)+ ):+ self._gate_dead = key+ return+ brace = _OPEN_CALL_WS_RE.match(text, body).end()+ if brace == len(text):+ return+ # Fail closed if the full strip just run disagrees with the reasoning above.+ if text[brace] != "{" or seg_out != (prefix if self._seg_final else text[seg_start:]):+ self._gate_dead = key+ return+ self._gate_from = body+ self._gate_scanned = body+ self._gate_call_ws = -1+ if not self._gate_clean(text):+ self._gate_dead = key+ return+ self._gate_on = True+ self._gate_out = out if self._seg_final else ""++ def _gate_hold(self, text: str):+ """The cached result while the gate holds, else None (and the gate is dropped for this call)."""+ if len(text) - self._floor > _tool_healing._MAX_BRACKET_SCAN_CHARS or not self._gate_clean(+ text+ ):+ self._gate_on = False+ self._gate_out = ""+ self._gate_dead = (self._floor, self._first)+ return None+ if self._seg_final:+ return self._gate_out+ return text if self._floor_identity else self._floor_out + text[self._floor :]def _think_block(self, text: str, first: int) -> str:"""Returns settled, open, or empty (full strip); fires only if the opener is the first sentinel and the body has no other markup, as a closer in args is data."""@@ -1651,13 +1767,22 @@ class StreamingMarkupStripper:if not self._is_extension(text):self._reset()+ if self._gate_on:+ out = self._gate_hold(text)+ if out is not None:+ self._note(text)+ return out+if self._degenerate:# Guard on ``_floor``: with nothing settled both arms match, so the check would only add two scans.- out = (- self._floor_out + self._full_strip(text[self._floor :])- if self._floor and not self._needs_whole_buffer(text)- else self._full_strip(text)- )+ if self._floor and not self._needs_whole_buffer(text):+ seg = self._full_strip(text[self._floor :])+ out = self._floor_out + seg+ self._gate_try_arm(text, self._floor, out, seg)+ else:+ out = self._full_strip(text)+ if not self._floor:+ self._gate_try_arm(text, 0, out, out)self._note(text)return out@@ -1694,8 +1819,10 @@ class StreamingMarkupStripper:out = self._full_strip(text)self._note(text)return out- stripped = tail[:cut] + self._full_strip(tail[cut:]) if cut else self._full_strip(tail)+ seg = self._full_strip(tail[cut:]) if cut else self._full_strip(tail)+ stripped = tail[:cut] + seg if cut else segout = self._floor_out + stripped if self._floor else stripped+ self._gate_try_arm(text, self._floor + cut, out, seg)self._note(text)return outdiff --git a/studio/backend/tests/data/refactor_guard/ast_inventory.json b/studio/backend/tests/data/refactor_guard/ast_inventory.jsonindex 70f3d28f6fc..19eaf061272 100644--- a/studio/backend/tests/data/refactor_guard/ast_inventory.json+++ b/studio/backend/tests/data/refactor_guard/ast_inventory.json@@ -1129,6 +1129,7 @@"_LLAMA3_PY_CALL_RE": "'<\\\\|python_tag\\\\|>\\\\s*([\\\\w\\\\.\\\\-]+)\\\\s*\\\\.\\\\s*call\\\\s*\\\\(' flags=32","_LLAMA3_WS_RE": "'\\\\s*' flags=32","_MISTRAL_V11_NAME_RE": "'\\\\s*([\\\\w\\\\.\\\\-]+)\\\\s*' flags=32",+ "_OPEN_CALL_WS_RE": "'\\\\s*' flags=32","_OUTER_ENVELOPE_CLOSED_PATS[0]": "'<tool_call>(?:(?!<tool_call>).)*</tool_call>' flags=48","_OUTER_ENVELOPE_CLOSED_PATS[1]": "'<function(?:=[\\\\w.\\\\-]+|\\\\s+name=\"[\\\\w.\\\\-]+\")>(?:(?!<function(?:=[\\\\w.\\\\-]+|\\\\s+name=\"[\\\\w.\\\\-]+\")>).)*</function>' flags=48","_OUTER_ENVELOPE_CLOSED_PATS[2]": "'<\\\\|tool_call>.*?<tool_call\\\\|>' flags=48",@@ -1200,7 +1201,7 @@"re.compile('\\\\[TOOL_CALLS\\\\](?=\\\\s*(?:[\\\\[{]|[A-Za-z_][\\\\w.\\\\-]*(?:[\\\\[{]|\\\\s*$))|\\\\s*$).*$', re.DOTALL)": 1,"re.compile('\\\\[TOOL_CALLS\\\\]\\\\s*[\\\\w\\\\.\\\\-]+\\\\s*(?:\\\\[ARGS\\\\])?\\\\s*\\\\{.*?\\\\}', re.DOTALL)": 1,"re.compile('\\\\[TOOL_CALLS\\\\]\\\\s*\\\\[.*?\\\\](?:\\\\s*</s>)?', re.DOTALL)": 1,- "re.compile('\\\\s*')": 1,+ "re.compile('\\\\s*')": 2,"re.compile('\\\\s*([A-Za-z_][\\\\w.\\\\-]*)\\\\s*:')": 1,"re.compile('\\\\s*([\\\\w\\\\.\\\\-]+)\\\\s*')": 1,"re.compile('\\\\s*([\\\\w\\\\.\\\\-]+)\\\\s*\\\\.\\\\s*call\\\\s*\\\\(')": 1,@@ -1262,6 +1263,18 @@"decorators": [],"signature": "self, text: str"},+ "_gate_clean": {+ "decorators": [],+ "signature": "self, text: str"+ },+ "_gate_hold": {+ "decorators": [],+ "signature": "self, text: str"+ },+ "_gate_try_arm": {+ "decorators": [],+ "signature": "self, text: str, seg_start: int, out: str, seg_out: str"+ },"_is_extension": {"decorators": [],"signature": "self, text: str"@@ -1518,6 +1531,18 @@"_MISTRAL_V11_NAME_RE": {"kind": "assign"},+ "_OPEN_CALL_OPENER": {+ "kind": "assign"+ },+ "_OPEN_CALL_TRIGGERS": {+ "kind": "assign"+ },+ "_OPEN_CALL_TRIGGER_MAX_LEN": {+ "kind": "assign"+ },+ "_OPEN_CALL_WS_RE": {+ "kind": "assign"+ },"_OUTER_ENVELOPE_CLOSED_PATS": {"kind": "assign"},diff --git a/studio/backend/tests/data/refactor_guard/runtime_inventory.json b/studio/backend/tests/data/refactor_guard/runtime_inventory.jsonindex de3dc27c9e0..a927da46a2c 100644--- a/studio/backend/tests/data/refactor_guard/runtime_inventory.json+++ b/studio/backend/tests/data/refactor_guard/runtime_inventory.json@@ -88,6 +88,10 @@"_MISTRAL_THINK_OPEN","_MISTRAL_TRIGGER","_MISTRAL_V11_NAME_RE",+ "_OPEN_CALL_OPENER",+ "_OPEN_CALL_TRIGGERS",+ "_OPEN_CALL_TRIGGER_MAX_LEN",+ "_OPEN_CALL_WS_RE","_OUTER_ENVELOPE_CLOSED_PATS","_OUTER_ENVELOPE_OPEN_RE","_REPEAT_LEAD_PUNCT",diff --git a/studio/backend/tests/test_streaming_stripper_open_call.py b/studio/backend/tests/test_streaming_stripper_open_call.pynew file mode 100644index 00000000000..8f863deff84--- /dev/null+++ b/studio/backend/tests/test_streaming_stripper_open_call.py@@ -0,0 +1,241 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""An open ``<tool_call>`` body is not re-stripped per token, and the strip results do not change."""++import json+import random+import sys+from pathlib import Path++import pytest++BACKEND_ROOT = Path(__file__).resolve().parents[1]+if str(BACKEND_ROOT) not in sys.path:+ sys.path.insert(0, str(BACKEND_ROOT))++from core.inference.tool_call_parser import StreamingMarkupStripper # noqa: E402++ENABLED = {"web_search", "python", "terminal"}+++class _FullStripEveryToken(StreamingMarkupStripper):+ """The stripper before the open-call gate: the gate never arms, so every token strips the open tail."""++ __slots__ = ()++ def _gate_try_arm(self, *args):+ return None+++def _code_tokens(count):+ line = 'def f(x):\n return {"a": [x, "b\\\\c"], "d": f"{x}"}\n'+ escaped = json.dumps(line)[1:-1]+ return [escaped[i : i + 4] for i in range(0, len(escaped), 4)] * (count // 16 + 1)+++def _replay(stripper, chunks):+ text, outs = "", []+ for chunk in chunks:+ text += chunk+ outs.append(stripper.strip(text))+ return outs+++class _CountingStripper(StreamingMarkupStripper):+ __slots__ = ()+ seen = [0, 0]++ def _full_strip(self, text):+ _CountingStripper.seen[0] += 1+ _CountingStripper.seen[1] += len(text)+ return super()._full_strip(text)++[email protected]("seg_final", [True, False])+ "preamble", ["I'll search. ", "", "Running it:\n", "<think>plan</think>\n\n"]+)+def test_open_call_work_does_not_grow_with_the_body(seg_final, preamble):+ """Counts characters handed to the full strip, not time: re-stripping the open tail per token is quadratic."""+ opener = preamble + '<tool_call>{"name": "python", "arguments": {"code": "'+ seen = {}+ for count in (250, 2000):+ chunks = [opener] + _code_tokens(count)[:count]+ _CountingStripper.seen[:] = [0, 0]+ outs = _replay(_CountingStripper(ENABLED, seg_final = seg_final), chunks)+ seen[count] = tuple(_CountingStripper.seen)+ assert outs == _replay(_FullStripEveryToken(ENABLED, seg_final = seg_final), chunks)+ # The few calls left fall in the first ~128 characters, before ``_is_extension`` samples a middle.+ assert seen[250] == seen[2000], seen+ assert seen[2000][0] < 40, seen+++def test_open_call_engages_and_matches_on_close(monkeypatch):+ """The close tag, split at every offset, still ends the gate on the token that completes it."""+ body = (+ 'Searching.\n<tool_call>{"name": "web_search", "arguments": {"query": "a \\"quoted\\" {x}'+ )+ close = '"}}</tool_call>\nDone.'+ for split in range(1, len(close)):+ chunks = [body] + list("pad words ") + [close[:split], close[split:]]+ for names in (ENABLED, None):+ got = _replay(StreamingMarkupStripper(names), chunks)+ want = _replay(_FullStripEveryToken(names), chunks)+ assert got == want, split+ assert got[-1] == want[-1] == "Searching.\n\nDone."+++_TRIGGERS = (+ "</tool_call>",+ "<tool_call>",+ "<|tool_call>",+ "<function=python>",+ "[TOOL_CALLS]",+ "x[ARGS]{",+ "web_search[ARGS]{}",+ "<|tool▁calls▁begin|>",+ "<think>",+ "</think>",+ "[THINK]",+ "[/THINK]",+ "call:web_search{q:1}",+ "call \n\t :python{code:1}",+ "<|python_tag|>",+ "<param",+ # Closed forms: the non-final strip removes these, so a missed trigger changes its result.+ "<function=web_search>x</function>",+ '[TOOL_CALLS]web_search{"q": 1}',+ "[TOOL_CALLS][]",+ "<|tool_call>call:web_search{q:1}<tool_call|>",+ "<|tool▁calls▁begin|>x<|tool▁calls▁end|>",+ # Kimi K2: only ``<|tool_call`` sees these, as neither holds ``<tool_call``.+ "<|tool_calls_section_begin|><|tool_call_begin|>x<|tool_call_end|><|tool_calls_section_end|>",+ "<|tool_call_begin|>x<|tool_call_end|>",+ "<think>t</think>",+ "\n</think>\n",+ "\ncall:web_search{q:1}\n",+)+++def test_every_strip_sentinel_is_an_open_call_trigger():+ """A new strip arm adds its literal to ``_STRIP_SENTINELS``; the gate must see it too. The two+ left out have no arm a non-final strip runs, and a final strip cuts them with the open call."""+ from core.inference import tool_call_parser as parser++ missing = (+ set(parser._STRIP_SENTINELS)+ - {parser._GEMMA_BARE_SENTINEL}+ - set(parser._OPEN_CALL_TRIGGERS)+ )+ assert missing == {"<|python_tag|>", "<|content_invoke_tool_json|>"}, missing++[email protected]("trigger", _TRIGGERS)[email protected]("seg_final", [True, False])+def test_every_trigger_split_one_char_at_a_time_matches(trigger, seg_final):+ # Past the ~128 characters ``_is_extension`` needs, so the gate answers later tokens.+ prefix = 'ok <tool_call>{"name": "web_search", "arguments": {"query": "' + "words " * 40+ chunks = [prefix] + list(trigger + ' more" }}') + ["</tool_call>", " after"]+ for names in (ENABLED, None):+ assert _replay(StreamingMarkupStripper(names, seg_final = seg_final), chunks) == _replay(+ _FullStripEveryToken(names, seg_final = seg_final), chunks+ )+++_PREFIXES = (+ "",+ "I'll search. ",+ "a [link](x) ",+ "brace {x} ",+ "call me ",+ "x `y` z\n",+ "<think>p</think>\n",+ "</think>stray ",+ "<parameter=x>v\n",+ "call :",+ " ",+ "line\n",+)+_PIECES = (+ "word ",+ '\\"',+ "\\\\",+ "\\n",+ "{",+ "}",+ "[",+ "]",+ ":",+ "call",+ "call ",+ " :",+ "recall:",+ "<",+ "</",+ "<tool_call>",+ "</tool_call>",+ "<|tool_call>",+ "<function=python>",+ "[ARGS]",+ "[TOOL_CALLS]",+ "<think>",+ "</think>",+ "[/THINK]",+ "<|",+ " ",+ "\n",+ "`",+ "<param",+ "中文",+)+++def test_gate_matches_full_strip_on_seeded_open_calls():+ rng = random.Random(20261027)+ engaged = 0+ for _ in range(250):+ text = rng.choice(_PREFIXES) + rng.choice(+ ("<tool_call>", "<tool_call>\n", "<tool_call>name\n")+ )+ text += (+ '{"name": "' + rng.choice(("web_search", "python", "nope")) + '", "arguments": {"q": "'+ )+ text += "filler " * 25+ plain = rng.random()+ text += "".join(+ rng.choice(_PIECES) if rng.random() > plain else "word "+ for _ in range(rng.randint(0, 40))+ )+ text += rng.choice(+ ("", '"}}', '"}}</tool_call>', '"}}</tool_call> then <tool_call>{"name": "x", "a')+ )+ for names in (ENABLED, None):+ for seg_final in (True, False):+ gated = StreamingMarkupStripper(names, seg_final = seg_final)+ plain_stripper = _FullStripEveryToken(names, seg_final = seg_final)+ pos = 0+ while pos < len(text):+ pos = min(len(text), pos + rng.randint(1, 4))+ was_on = gated._gate_on+ assert gated.strip(text[:pos]) == plain_stripper.strip(text[:pos]), (text, pos)+ engaged += was_on and gated._gate_on+ # The gate must actually answer calls here, or this test proves nothing.+ assert engaged > 1000, engaged+++def test_open_call_keeps_no_reference_to_the_buffer():+ stripper = StreamingMarkupStripper(ENABLED)+ text = 'Sure. <tool_call>{"name": "web_search", "arguments": {"query": "' + "w " * 50+ stripper.strip(text)+ text += "more"+ stripper.strip(text)+ assert stripper._gate_on+ assert all(getattr(stripper, slot) is not text for slot in StreamingMarkupStripper.__slots__)+++def test_repeated_open_call_strip_is_idempotent():+ stripper = StreamingMarkupStripper(ENABLED)+ text = 'Sure. <tool_call>{"name": "web_search", "arguments": {"query": "x'+ first = stripper.strip(text)+ assert stripper.strip(text) == stripper.strip(text) == first == "Sure. "