Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(studio): heal tool calls in the model's own format on the in-process route (#16824)

Long Yixinge60825ffbd99
diff --git a/studio/backend/core/inference/passthrough_healing.py b/studio/backend/core/inference/passthrough_healing.py
index 757c35db30a..03bf57cf1c3 100644
--- a/studio/backend/core/inference/passthrough_healing.py
+++ b/studio/backend/core/inference/passthrough_healing.py
@@ -10,8 +10,20 @@ import json
import os
import re
from collections.abc import Mapping
-from typing import Any, Optional
-
+from typing import Any, Callable, NamedTuple, Optional
+
+from core.inference.chat_template_helpers import _template_strings
+from core.inference.tool_call_parser import (
+ _ATTR_FUNC_OPEN_RE,
+ _MINIMAX_INVOKE_CLOSE,
+ _MINIMAX_INVOKE_OPEN_RE,
+ TOOL_XML_SIGNALS,
+ _MinimaxValueIndex,
+ _parse_deepseek_tool_calls,
+ _parse_glm_tool_calls,
+ _parse_kimi_tool_calls,
+ _parse_minimax_tool_calls,
+)
from core.inference.tool_loop_controller import (
coerce_arguments_by_schema,
coerce_tool_arguments,
@@ -19,6 +31,8 @@ from core.inference.tool_loop_controller import (
from core.tool_healing import (
_MISTRAL_ARRAY_RE,
_MISTRAL_BRACKET_RE,
+ _THINK_OPEN_RE,
+ _THINK_TAG_RE,
parse_tool_calls_from_text,
)
@@ -56,6 +70,8 @@ def nudge_enabled(request_flag: Optional[bool], *, response_format: Any = None)
_MAX_SIGNAL_LEN = max(len(s) for s in _HEAL_SIGNALS)
# Hold limit: a suspected tool block longer than this is declared a false alarm and flushed as text, bounding memory on a rambling model.
_MAX_HOLD_CHARS = 64 * 1024
+# Each closer a native block fails to parse at costs a reparse of the whole block.
+_MAX_NATIVE_CLOSERS = 16
def heal_gate(
@@ -179,16 +195,278 @@ def _promote(
return promoted
+class NativeToolFormat(NamedTuple):
+ """A model's own tool-call block format. llama-server parses these itself, but an in-process
+ model's output reaches the client healer raw; as there, only the template's format is healed."""
+
+ open: str
+ close: str
+ call: re.Pattern
+ parse: Callable[..., list]
+ # GLM shares <tool_call> with Hermes; its own blocks have this shape.
+ shape: Optional[re.Pattern] = None
+
+
+def _loaded_arguments(call: dict) -> Any:
+ arguments = call["function"].get("arguments")
+ try:
+ return json.loads(arguments) if isinstance(arguments, str) else arguments
+ except ValueError:
+ return arguments
+
+
+def _string_leaves(value: Any):
+ if isinstance(value, str):
+ yield value
+ elif isinstance(value, Mapping):
+ for key, item in value.items():
+ yield from _string_leaves(key)
+ yield from _string_leaves(item)
+ elif isinstance(value, (list, tuple)):
+ for item in value:
+ yield from _string_leaves(item)
+
+
+_SEED_OPEN = "<seed:tool_call>"
+_FOREIGN_SIGNALS = frozenset({*_HEAL_SIGNALS, *TOOL_XML_SIGNALS, _SEED_OPEN})
+
+
+def _parse_seed_block(
+ block: str,
+ *,
+ id_offset: int,
+ allow_incomplete: bool = True,
+) -> list:
+ body = block[len(_SEED_OPEN) :]
+ calls = parse_tool_calls_from_text(body, id_offset = id_offset, allow_incomplete = allow_incomplete)
+ # Strict parsing reads a JSON function body as no arguments, so a disagreement is a misread.
+ lenient = parse_tool_calls_from_text(body, id_offset = id_offset, allow_incomplete = True)
+ if [_loaded_arguments(c) for c in calls] != [_loaded_arguments(c) for c in lenient]:
+ return []
+ return calls
+
+
+def _parse_minimax_block(
+ block: str,
+ *,
+ id_offset: int,
+ allow_incomplete: bool = True,
+) -> list:
+ calls = _parse_minimax_tool_calls(block, id_offset = id_offset, allow_incomplete = allow_incomplete)
+ # The parser skips a body it cannot read, so a call with no arguments must have had none.
+ inside = _MinimaxValueIndex(block).inside
+ invokes = [m for m in _MINIMAX_INVOKE_OPEN_RE.finditer(block) if not inside(m.start())]
+ for call, invoke in zip(calls, invokes) if len(calls) == len(invokes) else ():
+ close = block.find(_MINIMAX_INVOKE_CLOSE, invoke.end())
+ if (
+ not _loaded_arguments(call)
+ and block[invoke.end() : close if close >= 0 else None].strip()
+ ):
+ return []
+ return calls
+
+
+def _parse_glm_block(
+ block: str,
+ *,
+ id_offset: int,
+ allow_incomplete: bool = True,
+) -> list:
+ # Raw, so the client's schema decides a value's type: a string parameter keeps JSON-looking text.
+ return _parse_glm_tool_calls(
+ block, id_offset = id_offset, allow_incomplete = allow_incomplete, raw_values = True
+ )
+
+
+# Most specific marker first.
+_NATIVE_TOOL_FORMATS = (
+ (
+ "<minimax:tool_call>",
+ NativeToolFormat(
+ "<minimax:tool_call>",
+ "</minimax:tool_call>",
+ re.compile(r"<invoke\s"),
+ _parse_minimax_block,
+ ),
+ ),
+ (
+ _SEED_OPEN,
+ NativeToolFormat(
+ _SEED_OPEN, "</seed:tool_call>", re.compile("<function="), _parse_seed_block
+ ),
+ ),
+ (
+ "<|tool_calls_section_begin|>",
+ NativeToolFormat(
+ "<|tool_calls_section_begin|>",
+ "<|tool_calls_section_end|>",
+ re.compile(re.escape("<|tool_call_begin|>")),
+ _parse_kimi_tool_calls,
+ ),
+ ),
+ (
+ "<|tool▁calls▁begin|>",
+ NativeToolFormat(
+ "<|tool▁calls▁begin|>",
+ "<|tool▁calls▁end|>",
+ re.compile("<|tool▁call▁begin|>"),
+ _parse_deepseek_tool_calls,
+ ),
+ ),
+ (
+ "<arg_key>",
+ NativeToolFormat(
+ "<tool_call>",
+ "</tool_call>",
+ re.compile("<tool_call>"),
+ _parse_glm_block,
+ # Anything but arg pairs after the name is a body the parser would skip.
+ re.compile(r"<tool_call>\s*[\w.\-]+\s*(?:<arg_key>|</tool_call>)"),
+ ),
+ ),
+)
+
+
+def native_tool_format(*templates: Any) -> Optional[NativeToolFormat]:
+ """The block format the model's chat template writes tool calls in, if it is one of these."""
+ bodies = [body for template in templates for body in _template_strings(template)]
+ for marker, fmt in _NATIVE_TOOL_FORMATS:
+ if any(marker in body for body in bodies):
+ return fmt
+ return None
+
+
+def _native_block_calls(
+ fmt: NativeToolFormat, block: str, allow_incomplete: bool
+) -> Optional[list]:
+ """The block's calls when each of its call markers parsed to exactly one call."""
+ if fmt.shape is not None and not fmt.shape.match(block):
+ return None
+ try:
+ calls = fmt.parse(block, id_offset = 0, allow_incomplete = allow_incomplete)
+ leaves = [leaf for call in calls for leaf in _string_leaves(_loaded_arguments(call))]
+ except RecursionError:
+ return None
+
+ # Markers inside a parsed argument are that argument's data; the rest must be one envelope.
+ def bare(count: Callable[[str], int]) -> int:
+ return count(block) - sum(map(count, leaves))
+
+ if (
+ not calls
+ or bare(lambda s: len(fmt.call.findall(s))) != len(calls)
+ or bare(lambda s: s.count(fmt.close)) != block.endswith(fmt.close)
+ # Other formats' markup is refused even in arguments: decoded text does not locate it.
+ or _ATTR_FUNC_OPEN_RE.search(block)
+ or any(m in block for m in _FOREIGN_SIGNALS if m != fmt.open and not fmt.call.fullmatch(m))
+ ):
+ return None
+ return calls
+
+
+def _native_block(
+ text: str, begin: int, fmt: NativeToolFormat, allow_incomplete: bool
+) -> Optional[tuple]:
+ close = text.find(fmt.close, begin + len(fmt.open))
+ if close < 0:
+ calls = _native_block_calls(fmt, text[begin:], True) if allow_incomplete else None
+ return None if calls is None else (begin, len(text), calls)
+ for _attempt in range(_MAX_NATIVE_CLOSERS):
+ end = close + len(fmt.close)
+ calls = _native_block_calls(fmt, text[begin:end], False)
+ if calls is not None:
+ return begin, end, calls
+ # An argument may hold the closer, so the block may end at a later one.
+ close = text.find(fmt.close, end)
+ if close < 0:
+ return None
+ return None
+
+
+def _heal_native_text(
+ text: str,
+ fmt: NativeToolFormat,
+ allowed_tools: set,
+ tool_schemas: Optional[dict],
+ *,
+ allow_incomplete: bool,
+ start: int = 0,
+) -> Optional[tuple[list, dict]]:
+ """Events for ``text[start:]`` with every block promoted, and each call's source; None keeps the
+ text. All or nothing: other-format markup outside the blocks could quote a block that would run."""
+ blocks: list = []
+ # Markup quoted in a leading JSON answer is data, as on the generic path.
+ lead = len(text) - len(text.lstrip())
+ try:
+ lead = (
+ json.JSONDecoder().raw_decode(text, lead)[1] if text.startswith(("{", "["), lead) else 0
+ )
+ except (ValueError, RecursionError):
+ lead = 0
+ pos = lead
+ while (begin := text.find(fmt.open, pos)) >= 0:
+ think = _THINK_OPEN_RE.search(text, pos, begin)
+ if think is not None:
+ pos = _THINK_TAG_RE.match(text, think.start()).end()
+ continue
+ block = _native_block(text, begin, fmt, allow_incomplete)
+ if block is None:
+ return None
+ blocks.append(block)
+ pos = block[1]
+ if not blocks:
+ return None
+ bounds = [lead, *(edge for begin, end, _calls in blocks for edge in (begin, end)), len(text)]
+ outside = "".join(text[a:b] for a, b in zip(bounds[::2], bounds[1::2]))
+ foreign = _FOREIGN_SIGNALS - {fmt.open}
+ if any(m in outside for m in foreign) or _ATTR_FUNC_OPEN_RE.search(outside):
+ return None
+ calls = [call for _begin, _end, block_calls in blocks for call in block_calls]
+ promoted = _promote(calls, allowed_tools, tool_schemas = tool_schemas)
+ if len(promoted) != len(calls):
+ return None
+ ids = [call["id"] for call in calls]
+ if len(set(ids)) == len(ids):
+ # Kimi's functions.NAME:IDX ids replay through its template; generated ones restart per block.
+ for call, call_id in zip(promoted, ids):
+ call["id"] = call_id
+ events: list = []
+ sources: dict = {}
+ order = iter(promoted)
+ pos = start
+ for begin, end, block_calls in blocks:
+ if text[pos:begin]:
+ events.append(("text", text[pos:begin]))
+ for index in range(len(block_calls)):
+ call = next(order)
+ events.append(("tool_call", call))
+ # One source per block, so un-promoting every call relays it once.
+ sources[call["id"]] = "" if index else text[begin:end]
+ pos = end
+ if text[pos:]:
+ events.append(("text", text[pos:]))
+ return events, sources
+
+
def heal_openai_message_events(
msg: dict,
allowed_tools: set,
tools: Optional[list] = None,
allow_incomplete: bool = True,
+ native: Optional[NativeToolFormat] = None,
) -> Optional[list]:
if not isinstance(msg, dict) or msg.get("tool_calls"):
return None
content = msg.get("content")
- if not isinstance(content, str) or not _has_heal_signal(content):
+ if not isinstance(content, str):
+ return None
+ if native is not None:
+ tool_schemas = _tool_schemas_by_name(tools) if tools is not None else None
+ healed = _heal_native_text(
+ content, native, allowed_tools, tool_schemas, allow_incomplete = allow_incomplete
+ )
+ return healed[0] if healed else None
+ if not _has_heal_signal(content):
return None
parsed, spans = parse_tool_calls_from_text(content, allow_incomplete = True, with_spans = True)
# A strict parse alone would promote calls quoted inside a rejected unfinished call.
@@ -227,9 +505,10 @@ def heal_openai_message(
allowed_tools: set,
tools: Optional[list] = None,
allow_incomplete: bool = True,
+ native: Optional[NativeToolFormat] = None,
) -> bool:
"""Edits in place; only promoted calls' markup is removed, undeclared calls stay byte-intact."""
- events = heal_openai_message_events(msg, allowed_tools, tools, allow_incomplete)
+ events = heal_openai_message_events(msg, allowed_tools, tools, allow_incomplete, native)
if not events:
return False
calls = [value for kind, value in events if kind == "tool_call"]
@@ -431,11 +710,15 @@ class _StrictParseGate:
return False
-def _partial_signal_suffix(buffer: str) -> int:
+def _partial_signal_suffix(
+ buffer: str,
+ signals: tuple = _HEAL_SIGNALS,
+ longest: int = _MAX_SIGNAL_LEN,
+) -> int:
"""Length of the longest buffer suffix that is a proper prefix of a signal."""
- for length in range(min(len(buffer), _MAX_SIGNAL_LEN - 1), 0, -1):
+ for length in range(min(len(buffer), longest - 1), 0, -1):
tail = buffer[-length:]
- if any(signal.startswith(tail) for signal in _HEAL_SIGNALS):
+ if any(signal.startswith(tail) for signal in signals):
return length
return 0
@@ -456,8 +739,12 @@ class StreamToolCallHealer:
self,
allowed_tools: set,
tools: Optional[list] = None,
+ native: Optional[NativeToolFormat] = None,
) -> None:
self._allowed = set(allowed_tools)
+ # With a native format, text from its first opener is held to the end and healed whole.
+ self._native = native
+ self._relayed: Optional[str] = ""
self._tool_schemas = _tool_schemas_by_name(tools) if tools is not None else None
self._buffer = ""
@@ -487,9 +774,34 @@ class StreamToolCallHealer:
def feed(self, text: str) -> list:
if self.dormant:
return [("text", text)] if text else []
+ if self._native is not None:
+ return self._feed_native(text)
self._buffer += text
return self._drain()
+ def _feed_native(self, text: str) -> list:
+ if self._relayed is None:
+ return [("text", text)] if text else []
+ self._buffer += text
+ events: list = []
+ if not self._holding:
+ start = self._buffer.find(self._native.open)
+ cut = (
+ start
+ if start >= 0
+ else len(self._buffer)
+ - _partial_signal_suffix(self._buffer, (self._native.open,), len(self._native.open))
+ )
+ if cut:
+ events.append(("text", self._buffer[:cut]))
+ self._relayed += self._buffer[:cut]
+ self._buffer = self._buffer[cut:]
+ self._holding = start >= 0
+ if self._holding and len(self._buffer) > _MAX_HOLD_CHARS:
+ events.append(("text", self._buffer))
+ self._buffer, self._holding, self._relayed = "", False, None
+ return events
+
def _drain(self) -> list:
events: list = []
while True:
@@ -585,7 +897,25 @@ class StreamToolCallHealer:
return []
residue, self._buffer = self._buffer, ""
holding, self._holding = self._holding, False
- if self.dormant or not holding or not promote:
+ if self.dormant or not holding:
+ return [("text", residue)]
+ if self._native is not None:
+ # Held whole, so its closed blocks promote even when cut-off calls may not.
+ healed = _heal_native_text(
+ self._relayed + residue,
+ self._native,
+ self._allowed,
+ self._tool_schemas,
+ allow_incomplete = promote,
+ start = len(self._relayed),
+ )
+ if healed is None:
+ return [("text", residue)]
+ events, sources = healed
+ self._promoted_spans.update(sources)
+ self._id_offset += len(sources)
+ return events
+ if not promote:
return [("text", residue)]
parsed, spans = parse_tool_calls_from_text(
residue,
diff --git a/studio/backend/core/inference/tool_call_parser.py b/studio/backend/core/inference/tool_call_parser.py
index a2e6c17b645..98d7d540270 100644
--- a/studio/backend/core/inference/tool_call_parser.py
+++ b/studio/backend/core/inference/tool_call_parser.py
@@ -4146,8 +4146,10 @@ def _parse_glm_tool_calls(
*,
id_offset: int,
allow_incomplete: bool = True,
+ raw_values: bool = False,
) -> list[dict]:
- """GLM 4.x: back-to-back <tool_call> blocks, no envelope, each with arg_key/arg_value pairs."""
+ """GLM 4.x: back-to-back <tool_call> blocks, no envelope, each with arg_key/arg_value pairs.
+ ``raw_values`` leaves every value a string, for a caller that types them by the tool's schema."""
out: list[dict] = []
pos = 0
while pos < len(content):
@@ -4204,7 +4206,7 @@ def _parse_glm_tool_calls(
apos = ve + len(_GLM_ARG_VAL_CLOSE)
# Keep the value raw so string whitespace survives (matches vLLM glm4_moe).
probe = raw_val.strip()
- if (
+ if not raw_values and (
probe[:1] in "{["
or probe in ("true", "false", "null")
or _GLM_JSON_NUMERIC_RE.fullmatch(probe)
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index d5804535a1d..9aedfcd63d0 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -3703,6 +3703,7 @@ from core.inference.passthrough_healing import (
heal_gate,
heal_openai_message,
heal_openai_message_events,
+ native_tool_format,
nudge_enabled,
nudge_messages,
nudge_should_retry,
@@ -34965,6 +34966,16 @@ async def produce_openai_chat_completions(
if _sf_client_tools
else None
)
+ # llama-server parses the model's own call markup; here the healer must. The load-time template counts: a mapped template does not change what the model writes.
+ _sf_native_tools = (
+ native_tool_format(
+ _sf_image_tpl,
+ (_sf_model_info.get("chat_template_info") or {}).get("template"),
+ *(getattr(target, "chat_template", None) for target in _sf_chat_targets),
+ )
+ if _sf_heal
+ else None
+ )
if _sf_client_tools:
# Re-derive from messages to keep tool history; fold system and developer into one system message; tool_choice none drops tools or markup leaks as prose
_sf_tc = payload.tool_choice
@@ -35180,7 +35191,11 @@ async def produce_openai_chat_completions(
yield _chat_role_chunk(completion_id, created, model_name)
# Text-form tool calls are healed on the fly; with healing off (None) the stream is relayed verbatim.
- healer = StreamToolCallHealer(_sf_heal, _sf_healing_tools) if _sf_heal else None
+ healer = (
+ StreamToolCallHealer(_sf_heal, _sf_healing_tools, native = _sf_native_tools)
+ if _sf_heal
+ else None
+ )
heal_state = {"idx": 0}
prev_text = ""
@@ -35547,7 +35562,11 @@ async def produce_openai_chat_completions(
if _sf_heal:
# Truncation wins: a call open at the cut stays text and the reason stays 'length'.
if heal_openai_message(
- _msg, _sf_heal, _sf_healing_tools, allow_incomplete = _finish != "length"
+ _msg,
+ _sf_heal,
+ _sf_healing_tools,
+ allow_incomplete = _finish != "length",
+ native = _sf_native_tools,
):
if _finish != "length":
_finish = "tool_calls"
@@ -35605,6 +35624,7 @@ async def produce_openai_chat_completions(
_sf_heal,
_sf_healing_tools,
allow_incomplete = _retry_finish != "length",
+ native = _sf_native_tools,
):
_visible_text, _msg, _finish = (
_retry_visible,
diff --git a/studio/backend/tests/data/refactor_guard/ast_inventory.json b/studio/backend/tests/data/refactor_guard/ast_inventory.json
index 7a6017d06c4..83e0bfce8aa 100644
--- a/studio/backend/tests/data/refactor_guard/ast_inventory.json
+++ b/studio/backend/tests/data/refactor_guard/ast_inventory.json
@@ -1946,7 +1946,7 @@
"_parse_glm_tool_calls": {
"decorators": [],
"kind": "def",
- "signature": "content: str, *, id_offset: int, allow_incomplete: bool=True"
+ "signature": "content: str, *, id_offset: int, allow_incomplete: bool=True, raw_values: bool=False"
},
"_parse_glued_deepseek_envelopes": {
"decorators": [],
diff --git a/studio/backend/tests/data/refactor_guard/runtime_inventory.json b/studio/backend/tests/data/refactor_guard/runtime_inventory.json
index 2a783678257..9ab2cd235b8 100644
--- a/studio/backend/tests/data/refactor_guard/runtime_inventory.json
+++ b/studio/backend/tests/data/refactor_guard/runtime_inventory.json
@@ -303,7 +303,7 @@
"_parse_deepseek_tool_calls": "core.inference.tool_call_parser._parse_deepseek_tool_calls(content: 'str', *, id_offset: 'int', allow_incomplete: 'bool' = True) -> 'list[dict]'",
"_parse_function_xml": "core.inference.tool_call_parser._parse_function_xml(content: 'str', *, id_offset: 'int', allow_incomplete: 'bool' = True) -> 'list[dict]'",
"_parse_gemma_tool_calls": "core.inference.tool_call_parser._parse_gemma_tool_calls(content: 'str', *, id_offset: 'int', allow_incomplete: 'bool' = True, enabled_tool_names: 'Optional[set]' = None) -> 'list[dict]'",
- "_parse_glm_tool_calls": "core.inference.tool_call_parser._parse_glm_tool_calls(content: 'str', *, id_offset: 'int', allow_incomplete: 'bool' = True) -> 'list[dict]'",
+ "_parse_glm_tool_calls": "core.inference.tool_call_parser._parse_glm_tool_calls(content: 'str', *, id_offset: 'int', allow_incomplete: 'bool' = True, raw_values: 'bool' = False) -> 'list[dict]'",
"_parse_glued_deepseek_envelopes": "core.inference.tool_call_parser._parse_glued_deepseek_envelopes(content: 'str', pos: 'int', *, id_offset: 'int') -> 'list[dict]'",
"_parse_kimi_section_body": "core.inference.tool_call_parser._parse_kimi_section_body(body: 'str', *, id_offset: 'int', allow_incomplete: 'bool' = True) -> 'list[dict]'",
"_parse_kimi_tool_calls": "core.inference.tool_call_parser._parse_kimi_tool_calls(content: 'str', *, id_offset: 'int', allow_incomplete: 'bool' = True) -> 'list[dict]'",
diff --git a/studio/backend/tests/test_control_markup_neutralize_7066.py b/studio/backend/tests/test_control_markup_neutralize_7066.py
index af4999f6107..6b65d7b529d 100644
--- a/studio/backend/tests/test_control_markup_neutralize_7066.py
+++ b/studio/backend/tests/test_control_markup_neutralize_7066.py
@@ -4469,7 +4469,7 @@ def test_safetensors_healing_is_gated_on_the_sanitized_catalog():
gate = "heal_gate(payload.auto_heal_tool_calls, _sf_healing_tools, payload.tool_choice"
assert packed.count(gate + ",") + packed.count(gate + ")") == 1
for call in (
- "StreamToolCallHealer(_sf_heal, _sf_healing_tools)",
+ "StreamToolCallHealer(_sf_heal, _sf_healing_tools,",
"heal_openai_message(_msg, _sf_heal, _sf_healing_tools,",
"nudge_should_retry(_data, _sf_heal, _sf_healing_tools)",
"heal_openai_message(retry_msg, _sf_heal, _sf_healing_tools,",
diff --git a/studio/backend/tests/test_passthrough_healing.py b/studio/backend/tests/test_passthrough_healing.py
index cc095586dd4..0917af91db0 100644
--- a/studio/backend/tests/test_passthrough_healing.py
+++ b/studio/backend/tests/test_passthrough_healing.py
@@ -21,6 +21,7 @@ from core.inference.passthrough_healing import ( # noqa: E402
heal_openai_message,
nudge_enabled,
nudge_messages,
+ native_tool_format,
nudge_should_retry,
response_has_promotable_calls,
)
@@ -1730,6 +1731,228 @@ class TestHealerSignalAlignment:
assert healer.healed
+_XML_BASH = "<function=Bash>\n<parameter=command>\nls\n</parameter>\n</function>"
+_GLM = "<tool_call>Bash<arg_key>command</arg_key><arg_value>ls</arg_value></tool_call>"
+_KIMI = (
+ "<|tool_calls_section_begin|><|tool_call_begin|>functions.Bash:0<|tool_call_argument_begin|>"
+ '{"command": "ls"}<|tool_call_end|><|tool_calls_section_end|>'
+)
+_MINIMAX = (
+ '<minimax:tool_call><invoke name="Bash"><parameter name="command">ls</parameter></invoke>'
+ "</minimax:tool_call>"
+)
+_SEED = f"<seed:tool_call>\n{_XML_BASH}\n</seed:tool_call>"
+_DEEPSEEK = (
+ '<|tool▁calls▁begin|><|tool▁call▁begin|>Bash<|tool▁sep|>{"command": "ls"}'
+ "<|tool▁call▁end|><|tool▁calls▁end|>"
+)
+# Format -> (template marker, a block calling Bash(command="ls")).
+_NATIVE = {
+ "glm": ("<arg_key>", _GLM),
+ "minimax": ("<minimax:tool_call>", _MINIMAX),
+ "kimi": ("<|tool_calls_section_begin|>", _KIMI),
+ "deepseek": ("<|tool▁calls▁begin|>", _DEEPSEEK),
+ "seed": ("<seed:tool_call>", _SEED),
+}
+_LS = ("Bash", {"command": "ls"})
+_JSON_LS = '{"command": "ls"}'
+
+
+def _named(calls):
+ return [(c["function"]["name"], json.loads(c["function"]["arguments"])) for c in calls]
+
+
+def _heal_native(
+ name,
+ text,
+ allowed = ("Bash", "Write"),
+ tools = None,
+):
+ """Calls and text from the stream at several chunk sizes and from the message path, which agree."""
+ native = native_tool_format(_NATIVE[name][0])
+ results = []
+ for step in (1, 7, len(text)):
+ healer = StreamToolCallHealer(set(allowed), tools, native = native)
+ events = [e for i in range(0, len(text), step) for e in healer.feed(text[i : i + step])]
+ events += healer.finalize()
+ results.append((_named(_events_calls(events)), _events_text(events)))
+ assert results.count(results[0]) == len(results)
+ calls, relayed = results[0]
+ msg = {"content": text}
+ heal_openai_message(msg, set(allowed), tools, native = native)
+ assert _named(msg.get("tool_calls") or []) == calls
+ assert msg["content"] == ((relayed.strip() or None) if calls else text)
+ return results[0]
+
+
+class TestNativeToolFormats:
+ """In-process models: the template's own call blocks are promoted, as llama-server parses them."""
+
+ def test_detects_the_format_from_the_template(self):
+ for marker, block in _NATIVE.values():
+ fmt = native_tool_format(f"{{% if tools %}}{marker}{{% endif %}}")
+ assert block.startswith(fmt.open) and block.endswith(fmt.close)
+ assert native_tool_format({"tool_use": "<minimax:tool_call>"}).open == "<minimax:tool_call>"
+ assert native_tool_format("<tool_call>\n<function=", None) is None
+
+ @pytest.mark.parametrize("name", _NATIVE)
+ def test_each_format_is_promoted_with_no_markup_left(self, name):
+ block = _NATIVE[name][1]
+ text = f"On it.\n{block}\n{block}\nDone."
+ assert _heal_native(name, text) == ([_LS, _LS], "On it.\n\n\nDone.")
+
+ @pytest.mark.parametrize(
+ "name, text",
+ [
+ ("minimax", _MINIMAX.replace("</invoke>", '</invoke><invoke name="Read"></invoke>')),
+ ("minimax", _MINIMAX.replace("</invoke>", '</invoke><invoke\nname="Bash"><parameter>')),
+ ("kimi", _KIMI.replace(_JSON_LS, "{command: ls}")),
+ ("seed", f"<seed:tool_call><function=Bash>{_JSON_LS}</function></seed:tool_call>"),
+ ("minimax", _MINIMAX.replace('<parameter name="command">ls</parameter>', _JSON_LS)),
+ ("glm", f"<tool_call>Bash\n{_JSON_LS}</tool_call>"),
+ (
+ "glm",
+ f'<tool_call>{json.dumps({"name": "Write", "arguments": {"c": _GLM}})}</tool_call>',
+ ),
+ (
+ "minimax",
+ f"<function=Write>\n<parameter=content>\n{_MINIMAX}\n</parameter>\n</function>",
+ ),
+ ("kimi", f'<|tool_call>call:Write{{c:<|"|>a<tool_call|>{_KIMI}<|"|>}}<tool_call|>'),
+ ("glm", f"<|tool_call_begin|>functions.Write:0<|tool_call_argument_begin|>{_GLM}"),
+ ("glm", f'<function\n name="Write"><parameter name="c">{_GLM}</parameter></function>'),
+ ("kimi", f"{_KIMI} then {_XML_BASH}"),
+ ("kimi", f"see <seed:tool_call> then {_KIMI}"),
+ (
+ "minimax",
+ _MINIMAX.replace(">ls<", '><invoke name="Bash"></invoke><').replace(
+ '"><parameter name="command">',
+ f'">{_JSON_LS}</invoke><invoke name="Bash"><parameter name="command">',
+ 1,
+ ),
+ ),
+ ("kimi", f"<think>{_XML_BASH}</think>{_KIMI}"),
+ ("kimi", "<function=Write><|tool_calls_section_begin|>x"),
+ ("kimi", f"<|tool_calls_section_begin|>x<|tool_calls_section_end|> see {_KIMI[28:]}"),
+ ("kimi", _KIMI.replace(_JSON_LS, '{"x": ' + "[" * 1100 + "]" * 1100 + "}")),
+ ("seed", _SEED.replace("ls", "</seed:tool_call>" * 3200)),
+ *(
+ ("kimi", f"<|tool_calls_section_begin|>{wrapper}{_KIMI[28:]}</parameter>")
+ for wrapper in (
+ "<function=Write><parameter=c>",
+ '<function\n name="Write"><parameter name="c">',
+ )
+ ),
+ (
+ "kimi",
+ f'<|tool_calls_section_begin|><function name="Write"><parameter name="c">'
+ + _KIMI[28:].replace('"ls"', json.dumps('<function name="Write">')),
+ ),
+ ],
+ )
+ def test_a_reply_that_is_not_all_declared_well_formed_blocks_stays_text(self, name, text):
+ assert _heal_native(name, text, ("Bash",)) == ([], text)
+
+ @pytest.mark.parametrize(
+ "name, text, calls",
+ [
+ (
+ "kimi",
+ _KIMI.replace('"ls"', '"<|tool_calls_section_end|>"'),
+ [("Bash", {"command": "<|tool_calls_section_end|>"})],
+ ),
+ (
+ "kimi",
+ _KIMI.replace('"ls"', '"<|tool_call_begin|>"'),
+ [("Bash", {"command": "<|tool_call_begin|>"})],
+ ),
+ (
+ "seed",
+ _SEED.replace("\nls\n", "\n<function=x>\n"),
+ [("Bash", {"command": "<function=x>"})],
+ ),
+ ("minimax", _MINIMAX.replace("<invoke ", "<invoke\n"), [_LS]),
+ (
+ "minimax",
+ _MINIMAX.replace("<invoke", '<invoke name="Bash"></invoke><invoke').replace(
+ "parameter", "param"
+ ),
+ [("Bash", {}), _LS],
+ ),
+ (
+ "minimax",
+ _MINIMAX.replace(">ls<", '><invoke name="x">y</invoke><'),
+ [("Bash", {"command": '<invoke name="x">y</invoke>'})],
+ ),
+ ("glm", f"<think>maybe [/THINK] {_GLM}</think>{_GLM}", [_LS]),
+ (
+ "minimax",
+ f"{_MINIMAX.replace('>ls<', '><think><')}{_MINIMAX}",
+ [("Bash", {"command": "<think>"}), _LS],
+ ),
+ ("minimax", f"[THINK]maybe </think> {_MINIMAX}[/THINK]{_MINIMAX}", [_LS]),
+ ("glm", json.dumps({"a": [_GLM, "<function=x>"]}) + _GLM, [_LS]),
+ ("glm", "[" * 100000 + _GLM, [_LS]),
+ ],
+ )
+ def test_argument_and_reasoning_text_is_data(self, name, text, calls):
+ assert _heal_native(name, text)[0] == calls
+
+ def test_kimi_call_ids_replay_through_its_template(self):
+ native = native_tool_format(_NATIVE["kimi"][0])
+ two = _KIMI + _KIMI.replace(":0", ":1")
+ for text, ids in (
+ (two, ["functions.Bash:0", "functions.Bash:1"]),
+ (_KIMI * 2, ["call_0", "call_1"]),
+ ):
+ msg = {"content": text}
+ heal_openai_message(msg, {"Bash"}, native = native)
+ healer = StreamToolCallHealer({"Bash"}, native = native)
+ events = healer.feed(text) + healer.finalize()
+ assert (
+ [c["id"] for c in msg["tool_calls"]]
+ == [c["id"] for c in _events_calls(events)]
+ == ids
+ )
+
+ def test_a_block_relays_once_if_its_calls_are_un_promoted(self):
+ block = _MINIMAX.replace("</invoke>", '</invoke><invoke name="Bash"></invoke>')
+ healer = StreamToolCallHealer({"Bash"}, native = native_tool_format("<minimax:tool_call>"))
+ assert _events_calls(healer.feed(block) + healer.finalize())
+ assert (healer.promoted_source("call_0"), healer.promoted_source("call_1")) == (block, "")
+
+ def test_glm_values_are_typed_by_the_tools_schema(self):
+ tools = [_client_tool("Write", {"c": {"type": "string"}, "o": {"type": "object"}})]
+ text = '<tool_call>Write<arg_key>c</arg_key><arg_value>{"on":true}</arg_value>'
+ text += '<arg_key>o</arg_key><arg_value>{"a": 1}</arg_value></tool_call>'
+ calls = [("Write", {"c": '{"on":true}', "o": {"a": 1}})]
+ assert _heal_native("glm", text, ("Write",), tools)[0] == calls
+
+ def test_a_cut_off_block_promotes_only_when_incomplete_calls_may(self):
+ cut = _DEEPSEEK[: -len("<|tool▁calls▁end|>")]
+ assert _heal_native("deepseek", cut)[0] == [_LS]
+ native = native_tool_format(_NATIVE["deepseek"][0])
+ generic = f'<tool_call>{{"name": "Bash", "arguments": {_JSON_LS}}}'
+ for native, text, calls in (
+ (native, cut, []),
+ (native, f"{_DEEPSEEK} and the", [_LS]),
+ (None, generic, []),
+ ):
+ msg = {"content": text}
+ heal_openai_message(msg, {"Bash"}, allow_incomplete = False, native = native)
+ healer = StreamToolCallHealer({"Bash"}, native = native)
+ events = healer.feed(text) + healer.finalize(promote = False)
+ assert _named(msg.get("tool_calls") or []) == _named(_events_calls(events)) == calls
+
+ def test_the_stream_holds_no_more_than_the_hold_limit(self):
+ native = native_tool_format("<seed:tool_call>")
+ text = _SEED.replace("\nls\n", "x" * (70 * 1024)).replace("Bash", "Write") + _SEED
+ healer = StreamToolCallHealer({"Bash", "Write"}, native = native)
+ events = [e for i in range(0, len(text), 4096) for e in healer.feed(text[i : i + 4096])]
+ assert _events_text(events + healer.finalize()) == text
+ assert heal_openai_message({"content": text}, {"Bash", "Write"}, native = native)
+
+
def _client_tool(name, properties):
return {
"type": "function",
diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.py
index 549b591cf98..646b373740d 100644
--- a/studio/backend/tests/test_sf_client_tools_passthrough.py
+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py
@@ -699,6 +699,30 @@ def test_streaming_heals_split_call_into_one_delta(monkeypatch):
assert finishes == ["tool_calls"]
+_MINIMAX_CALL = (
+ '<minimax:tool_call>\n<invoke name="lookup">\n<parameter name="q">cats</parameter>\n'
+ "</invoke>\n</minimax:tool_call>"
+)
+
+
[email protected]("stream", [False, True])
+def test_the_models_own_call_format_is_healed(monkeypatch, stream):
+ backend = _ScriptedBackend(_fixed("Looking.\n", f"Looking.\n{_MINIMAX_CALL}"))
+ backend.models["sf-model"]["chat_template_info"]["template"] = "{{ '<minimax:tool_call>' }}"
+ response = _call(_request(tools = [LOOKUP_TOOL], stream = stream), monkeypatch, backend)
+ if stream:
+ objs = _sse_objects(_collect_sse(response))
+ deltas = [(o.get("choices") or [{}])[0].get("delta") or {} for o in objs]
+ calls = [tc["function"] for d in deltas for tc in d.get("tool_calls") or []]
+ text = "".join(d.get("content") or "" for d in deltas)
+ else:
+ message = _json_body(response)["choices"][0]["message"]
+ calls = [tc["function"] for tc in message["tool_calls"]]
+ text = message["content"]
+ assert [(c["name"], json.loads(c["arguments"])) for c in calls] == [("lookup", {"q": "cats"})]
+ assert text.strip() == "Looking."
+
+
def test_what_this_backend_can_serve_reaches_it_rather_than_being_refused(monkeypatch):
"""An empty stop sequence is dropped, since it would match at position 0 and end every turn."""
backend = _ScriptedBackend(_fixed("hi"), stats = {"usage": {"prompt_tokens": 7}})