Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: do not run Gemma tool call examples written inside code blocks (#11153)
Nilayf10b3def75b0
diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.pyindex 219dd9b439b..576c0cbe77f 100644--- a/studio/backend/core/inference/safetensors_agentic.py+++ b/studio/backend/core/inference/safetensors_agentic.py@@ -198,6 +198,7 @@ def _earliest_tool_signal(*,unrestricted: bool = False,start: int = 0,+ streaming: bool = False,) -> int:"""Index where the turn's first genuine tool-call boundary begins, or -1.@@ -242,6 +243,7 @@ def _earliest_tool_signal(None if unrestricted else (lambda: _active_tool_names(active_tools)),start,floor = floor,+ streaming = streaming,)if gemma >= floor and (best < 0 or gemma < best):best = gemma@@ -904,6 +906,7 @@ def run_safetensors_tool_loop(_detect_tools,unrestricted = unrestricted_tools,start = max(0, _tool_signal_scanned_upto - _TOOL_SIGNAL_OVERLAP),+ streaming = True,)if signal_pos >= 0:before_tool = candidate[:signal_pos]diff --git a/studio/backend/core/inference/tool_call_parser.py b/studio/backend/core/inference/tool_call_parser.pyindex 93e14a60f65..92538aac54f 100644--- a/studio/backend/core/inference/tool_call_parser.py+++ b/studio/backend/core/inference/tool_call_parser.py@@ -1295,8 +1295,8 @@ def _unmask_blocked_bodies(text: str, bodies: list) -> Optional[str]:def _strip_gemma_wrapperless_calls(text: str, enabled_tool_names: Optional[set] = None) -> str:"""Strip closed wrapper-less Gemma ``call:NAME{...}`` calls with balanced brace scanning (nestedarguments are removed whole). Gated like the parser: a name that is not markerless-promotable- stays visible, as does an unanchored mid-sentence call. ``None`` strips every anchored closed- non-execution call."""+ stays visible, as does a call quoted in markdown code or an unanchored mid-sentence one.+ ``None`` strips every anchored closed non-execution call."""if _whole_content_is_json_value(text):return textn = len(text)@@ -1310,6 +1310,7 @@ def _strip_gemma_wrapperless_calls(text: str, enabled_tool_names: Optional[set]# A blocked prefix in EITHER markerless format anchors, or the peer behind it is promoted# while its raw text stays in the content and the next iteration replays the call.floor = blocked_markerless_prefix_end(text, cursor, enabled_tool_names)+ code_spans, code_from = None, 0while cursor < n:m = _GEMMA_BARE_TC_RE.search(text, cursor)if not m:@@ -1318,9 +1319,14 @@ def _strip_gemma_wrapperless_calls(text: str, enabled_tool_names: Optional[set]# A blocked call is skipped markup, not a sentence: it holds its position so the# promotable call beside it stays anchored, and is still stripped.blocked = _markerless_blocked_execution(m.group(1), enabled_tool_names)- keep_as_prose = not _markerless_promotable(m.group(1), enabled_tool_names) or (- not _gemma_call_is_anchored(text, m.start(), floor)- )+ promotable = _markerless_promotable(m.group(1), enabled_tool_names)+ quoted = False+ if promotable:+ if code_spans is None:+ code_spans = _tool_healing._code_spans(text, code_from)+ quoted = _tool_healing._in_code(code_spans, m.start())+ parsed_as_call = promotable and not quoted+ keep_as_prose = not parsed_as_call or not _gemma_call_is_anchored(text, m.start(), floor)brace = m.end() - 1# Same boundary scanner as the parser: strip exactly what it consumed.end = _gemma_body_brace_end(text, brace)@@ -1338,6 +1344,9 @@ def _strip_gemma_wrapperless_calls(text: str, enabled_tool_names: Optional[set]else:out.append(text[cursor : m.start()])floor = blocked_markerless_prefix_end(text, next_index, enabled_tool_names)+ # Rescan past every call the parser promotes, kept unanchored or not, as the parser does.+ if parsed_as_call and _tool_healing._in_code(code_spans, next_index):+ code_spans, code_from = None, next_indexcursor = next_indexreturn "".join(out)@@ -3214,9 +3223,9 @@ def _parse_gemma_tool_calls(``skip_special_tokens`` stream where the wrapper and string markers were stripped (bare``call:NAME{k:v, ...}``). ``enabled_tool_names`` gates on the parsed name: the wrapper-lessshape is indistinguishable from prose documenting the syntax, so a disabled/example name must- not be stolen as a call (``None`` keeps the name-agnostic behaviour). Execution-class and MCP- names are never promoted here whatever the gate, since a bare call may be attacker-quoted prose;- they must carry the ``<|tool_call>`` wrapper."""+ not be stolen as a call (``None`` keeps the name-agnostic behaviour), nor may one quoted in+ markdown code. Execution-class and MCP names are never promoted here whatever the gate, since+ a bare call may be attacker-quoted prose; they must carry the ``<|tool_call>`` wrapper."""out: list[dict] = []# The WRAPPED form (strict + nested-marker handling) is tool_healing's, which runs first: defer content with a# wrapped opener. A marker literal alone is not enough -- a wrapper-less call mentioning ``<|tool_call>`` would be@@ -3255,6 +3264,7 @@ def _parse_gemma_tool_calls(_reh_cursor = _reh_end + 1# Monotonic index into the sorted spans: re-testing every span per match is quadratic.blocked_i = 0+ code_spans, code_from = None, 0while True:m = _GEMMA_BARE_TC_RE.search(content, cursor)if m is None:@@ -3274,6 +3284,13 @@ def _parse_gemma_tool_calls(cursor = end + 1if not _markerless_promotable(name, enabled_tool_names):continue+ if code_spans is None:+ code_spans = _tool_healing._code_spans(content, code_from)+ if _tool_healing._in_code(code_spans, m.start()):+ continue+ # Its arguments are data: a code span they open must not hide the sibling call behind it.+ if _tool_healing._in_code(code_spans, cursor):+ code_spans, code_from = None, cursorbody = content[body_start + 1 : end]try:args = _gemma_parse_stripped_body(body)@@ -3581,6 +3598,7 @@ def promotable_gemma_call_pos(start: int = 0,*,floor: int = 0,+ streaming: bool = False,) -> int:"""Offset of the first bare ``call:NAME{`` the parser would promote, or -1. Bare Gemma hasno ``TOOL_XML_SIGNALS`` entry, so without this the streaming detectors miss a mid-prose@@ -3593,25 +3611,48 @@ def promotable_gemma_call_pos(covers 4x the 64-character cap every provider enforces (OpenAI/Bedrock``^[a-zA-Z0-9_-]{1,64}$``, MCP SEP-986). Sentinel-gated like ``_promotable_gemma_call_pos``.``enabled_tool_names`` may be a zero-argument callable, resolved only once a candidate- exists, so an ordinary completion never materializes a large MCP catalogue per chunk."""+ exists, so an ordinary completion never materializes a large MCP catalogue per chunk.+ ``streaming`` also skips a call behind an inline backtick still open on the last line, whose+ closer may be the next token."""# Widen, do not seek: an rfind window has to contain the whole sentinel, so a ``call``# straddling the boundary was missed and a short-named call went unseen.start = text.find(_GEMMA_BARE_SENTINEL, max(0, start - _MAX_GEMMA_PREFIX_TAIL))if start < 0:return -1names = enabled_tool_names() if callable(enabled_tool_names) else enabled_tool_names+ code_spans = Nonefor m in _GEMMA_BARE_TC_RE.finditer(text, start):# Skip and keep scanning, never give up: the widening above can re-find a call the# caller has already stepped past (one rehearsed inside a ``<think>`` block), and# returning it made the caller treat "below my floor" as "no call anywhere", so a# real call after the block went undetected while streaming.- if m.start() < floor:+ if m.start() < floor or not _markerless_promotable(m.group(1), names):continue- if _markerless_promotable(m.group(1), names):- return m.start()+ if code_spans is None:+ code_spans = _tool_healing._code_spans(text)+ if _tool_healing._in_code(code_spans, m.start()):+ continue+ if streaming and "\n" not in text[m.start() :] and _open_inline_code_run(text, m.start()):+ continue+ return m.start()return -1+_BACKTICK_RUN_RE = re.compile(r"`+")+++def _open_inline_code_run(text: str, pos: int) -> bool:+ """Whether a backtick run earlier on ``pos``'s line is still waiting for its equal-length closer."""+ open_len = 0+ for run in _BACKTICK_RUN_RE.finditer(text, text.rfind("\n", 0, pos) + 1, pos):+ n = len(run.group())+ if not open_len:+ open_len = n+ elif n == open_len:+ open_len = 0+ return open_len > 0++# ``call`` plus separators and any name worth holding; the candidate can only sit at the end._MAX_GEMMA_PREFIX_TAIL = 256diff --git a/studio/backend/core/tool_healing.py b/studio/backend/core/tool_healing.pyindex 99a550362d1..2c1ccf4cf2f 100644--- a/studio/backend/core/tool_healing.py+++ b/studio/backend/core/tool_healing.py@@ -285,12 +285,14 @@ _REHEARSAL_RE = re.compile(r"(?<!\[CALL_ID\])\b([\w-]+)\[ARGS\]\s*(?=\{)")# Gates the markerless rehearsal form only, so quoting the syntax as documentation stays text# instead of executing. A fence's info string cannot contain backticks, so a triple-backtick run# opening a line is an inline span and not a fence running to EOF; the closer tolerates a CR so a-# CRLF block still closes. Inline runs are enumerated by length rather than backreferenced: a-# (`+)..\1 form backtracks over every candidate length and turned 21 KB of unmatched runs into a-# 1.8s stall.+# CRLF block still closes. A fence closes only on a run of its own character at least as long as+# its opener, so a ```` block can quote a ``` one. Inline runs are enumerated by length rather than+# backreferenced: a (`+)..\1 form backtracks over every candidate length and turned 21 KB of+# unmatched runs into a 1.8s stall._CODE_SPAN_RE = re.compile(- r"^[ \t]*(?:>[ \t]*)*(?:```+[^`\n]*|~~~+[^\n]*)$"- r".*?(?:^[ \t]*(?:>[ \t]*)*(?:```+|~~~+)[ \t\r]*$|\Z)"+ r"^[ \t]*(?:>[ \t]*)*(?:"+ r"(?P<bt>```+)[^`\n]*$.*?(?:^[ \t]*(?:>[ \t]*)*(?P=bt)`*[ \t\r]*$|\Z)"+ r"|(?P<tl>~~~+)[^\n]*$.*?(?:^[ \t]*(?:>[ \t]*)*(?P=tl)~*[ \t\r]*$|\Z))"r"|``(?:[^`]|`(?!`))*?``|`[^`\n]*`",re.DOTALL | re.MULTILINE,)@@ -453,17 +455,18 @@ def _decode_array_items(text: str, body_start: int, body_end: int):return objs, ends-def _code_spans(text: str) -> list[tuple[int, int]]:- """Markdown code spans, used to keep a quoted rehearsal out of the call set.+def _code_spans(text: str, start: int = 0) -> list[tuple[int, int]]:+ """Markdown code spans from ``start`` on, used to keep a quoted rehearsal out of the call set.A line-start fence cannot occur inside a call's JSON body (a raw newline is- invalid JSON there), so no tool-markup exclusion is needed; a stray inline span- can at worst hide a same-line rehearsal, which drops a call rather than- inventing one. Spans are ordered and non-overlapping, so ``_in_code`` bisects- them instead of rescanning from the front per candidate."""+ invalid JSON there), so no tool-markup exclusion is needed; a raw Gemma body can+ hold one, so a caller may ``start`` past such a body. A stray inline span can at+ worst hide a same-line rehearsal, which drops a call rather than inventing one.+ Spans are ordered and non-overlapping, so ``_in_code`` bisects them instead of+ rescanning from the front per candidate."""if "`" not in text and "~~~" not in text:return []- return [m.span() for m in _CODE_SPAN_RE.finditer(text)]+ return [m.span() for m in _CODE_SPAN_RE.finditer(text, start)]def _in_code(spans, pos: int) -> bool:diff --git a/studio/backend/tests/data/refactor_guard/ast_inventory.json b/studio/backend/tests/data/refactor_guard/ast_inventory.jsonindex 3944ad2730f..9b1c0dbb1c9 100644--- a/studio/backend/tests/data/refactor_guard/ast_inventory.json+++ b/studio/backend/tests/data/refactor_guard/ast_inventory.json@@ -1107,6 +1107,7 @@"compiled": {"INTENT_SIGNAL": "\"(?im)(\\\\b(i['\\\\u2019](ll|m going to|m gonna)|i am (going to|gonna)|i will|i shall)\\\\b(?!\\\\s+(?:not|never)\\\\b)(?!\\\\s+(?:do(?:ing)?\\\\s+my\\\\s+best|try\\\\s+my\\\\s+best|be\\\\s+(?:able|happy|glad)\\\\s+to\\\\b|assist\\\\b|help\\\\s+you\\\\b(?!\\\\s+(?:search|check|look|find|fetch|get|call|use|run|query|invoke|analy[sz]e|review|inspect|read|gather|examine|retrieve|browse|consult|verify|confirm|compute|calculate|determine|identify|render))|give\\\\s+you\\\\s+accurate\\\\b|(?:dig\\\\s+in|help\\\\s+analy[sz]e)\\\\b(?=[^\\\\w]*\\\\Z)))|\\\\b(?:let me|allow me)\\\\b(?!\\\\s+(?:not|never|know)\\\\b)(?!\\\\s+(?:dig\\\\s+in|help\\\\s+analy[sz]e)\\\\b(?=[^\\\\w]*\\\\Z))(?!\\\\s+to\\\\s+(?:do(?:ing)?\\\\s+my\\\\s+best|try\\\\s+my\\\\s+best|be\\\\s+(?:able|happy|glad)\\\\s+to\\\\b|assist\\\\b|help\\\\s+you\\\\b(?!\\\\s+(?:search|check|look|find|fetch|get|call|use|run|query|invoke|analy[sz]e|review|inspect|read|gather|examine|retrieve|browse|consult|verify|confirm|compute|calculate|determine|identify|render))|give\\\\s+you\\\\s+accurate\\\\b|(?:dig\\\\s+in|help\\\\s+analy[sz]e)\\\\b(?=[^\\\\w]*\\\\Z)))|(?:^|[.!?]\\\\s+)\\\\s*(?:the\\\\s+)?first\\\\s+step\\\\b|(?:^|[.!?]\\\\s+)\\\\s*first\\\\s*[,:–—-]?\\\\s+(?:my|our)\\\\s+(?:plan|approach|step)\\\\b|(?:^|[.!?]\\\\s+)\\\\s*first\\\\s*[,:–—-]?\\\\s+(?:i|we|let['’]?s|let us)\\\\b|(?:^|[.!?]\\\\s+)\\\\s*first\\\\s*[,:–—-]?\\\\s+(?:search|check|look|find|fetch|get|call|use|run|query|invoke|analy[sz]e|review|inspect|read|gather|examine|retrieve|browse|consult|verify|confirm|compute|calculate|determine|identify|render)\\\\b|\\\\b(?:step \\\\d+:?|here['\\\\u2019]?s (?:my |the |a )?(?:plan|approach))|\\\\b(?:now i|next i)\\\\b)\" flags=42","_ATTR_FUNC_OPEN_RE": "'<function\\\\s+name=\"' flags=32",+ "_BACKTICK_RUN_RE": "'`+' flags=32","_BARE_JSON_NAME_RE": "'\"name\"\\\\s*:\\\\s*\"([^\"]+)\"' flags=32","_BLOCKED_BODY_MASK_RUN_RE": "'\\ue000+' flags=32","_DEEPSEEK_BEGIN_RE": "'<|(?:tool▁calls▁begin|tool_calls_begin|tool calls begin|tool\\\\\\\\_calls\\\\\\\\_begin|tool▁calls)|>' flags=32",@@ -1210,6 +1211,7 @@"re.compile('\\ue000+')": 1,"re.compile('^(?:functions\\\\.)?([\\\\w\\\\.\\\\-]+)(?::(\\\\d+))?$')": 1,"re.compile('^`{3,}[a-zA-Z0-9_+-]*[ \\\\t]*\\\\r?\\\\n')": 1,+ "re.compile('`+')": 1,"re.compile('```[\\\\s\\\\r\\\\n]*' + re.escape(_DEEPSEEK_CALL_END))": 1,"re.compile('true|false|null')": 1,"re.compile(_DEEPSEEK_OPEN_RE_SRC + '(?=\\\\s*(?:<|tool▁call▁begin|>|function)|\\\\s*$).*$', re.DOTALL)": 1,@@ -1310,6 +1312,9 @@"_ATTR_FUNC_OPEN_RE": {"kind": "assign"},+ "_BACKTICK_RUN_RE": {+ "kind": "assign"+ },"_BARE_JSON_ARGS_KEYS": {"kind": "assign"},@@ -1753,6 +1758,11 @@"kind": "def","signature": "between: str"},+ "_open_inline_code_run": {+ "decorators": [],+ "kind": "def",+ "signature": "text: str, pos: int"+ },"_parse_bare_rehearsals": {"decorators": [],"kind": "def",@@ -1989,7 +1999,7 @@"promotable_gemma_call_pos": {"decorators": [],"kind": "def",- "signature": "text: str, enabled_tool_names, start: int=0, *, floor: int=0"+ "signature": "text: str, enabled_tool_names, start: int=0, *, floor: int=0, streaming: bool=False"},"reasoning_cap_spent_message": {"decorators": [],@@ -2045,7 +2055,7 @@},"core.tool_healing": {"compiled": {- "_CODE_SPAN_RE": "'^[ \\\\t]*(?:>[ \\\\t]*)*(?:```+[^`\\\\n]*|~~~+[^\\\\n]*)$.*?(?:^[ \\\\t]*(?:>[ \\\\t]*)*(?:```+|~~~+)[ \\\\t\\\\r]*$|\\\\Z)|``(?:[^`]|`(?!`))*?``|`[^`\\\\n]*`' flags=56",+ "_CODE_SPAN_RE": "'^[ \\\\t]*(?:>[ \\\\t]*)*(?:(?P<bt>```+)[^`\\\\n]*$.*?(?:^[ \\\\t]*(?:>[ \\\\t]*)*(?P=bt)`*[ \\\\t\\\\r]*$|\\\\Z)|(?P<tl>~~~+)[^\\\\n]*$.*?(?:^[ \\\\t]*(?:>[ \\\\t]*)*(?P=tl)~*[ \\\\t\\\\r]*$|\\\\Z))|``(?:[^`]|`(?!`))*?``|`[^`\\\\n]*`' flags=56","_GEMMA_NEXT_KEY_RE": "'\\\\s*[A-Za-z_][\\\\w.\\\\-]*\\\\s*:' flags=32","_MISTRAL_ARRAY_RE": "'\\\\[TOOL_CALLS\\\\]\\\\s*(?=\\\\[)' flags=32","_MISTRAL_BRACKET_RE": "'\\\\[TOOL_CALLS\\\\]\\\\s*([\\\\w-]+)(?:\\\\[CALL_ID\\\\][\\\\w-]+)?(?:\\\\[ARGS\\\\])?\\\\s*(?=\\\\{)' flags=32",@@ -2118,7 +2128,7 @@"re.compile('\\\\s*</function>\\\\s*$')": 1,"re.compile('\\\\s*</parameter>\\\\s*$')": 1,"re.compile('\\\\s*[A-Za-z_][\\\\w.\\\\-]*\\\\s*:')": 1,- "re.compile('^[ \\\\t]*(?:>[ \\\\t]*)*(?:```+[^`\\\\n]*|~~~+[^\\\\n]*)$.*?(?:^[ \\\\t]*(?:>[ \\\\t]*)*(?:```+|~~~+)[ \\\\t\\\\r]*$|\\\\Z)|``(?:[^`]|`(?!`))*?``|`[^`\\\\n]*`', re.DOTALL | re.MULTILINE)": 1,+ "re.compile('^[ \\\\t]*(?:>[ \\\\t]*)*(?:(?P<bt>```+)[^`\\\\n]*$.*?(?:^[ \\\\t]*(?:>[ \\\\t]*)*(?P=bt)`*[ \\\\t\\\\r]*$|\\\\Z)|(?P<tl>~~~+)[^\\\\n]*$.*?(?:^[ \\\\t]*(?:>[ \\\\t]*)*(?P=tl)~*[ \\\\t\\\\r]*$|\\\\Z))|``(?:[^`]|`(?!`))*?``|`[^`\\\\n]*`', re.DOTALL | re.MULTILINE)": 1,"re.match('\\\\s*</tool_call>', content[span_end:])": 1,"re.match('\\\\s*\\\\[/TOOL_CALLS\\\\]', content[end:])": 1,"re.search('<tool_call>\\\\s*$', content[:span_start])": 1@@ -2249,7 +2259,7 @@"_code_spans": {"decorators": [],"kind": "def",- "signature": "text: str"+ "signature": "text: str, start: int=0"},"_decode_array_items": {"decorators": [],diff --git a/studio/backend/tests/data/refactor_guard/golden_outputs.json b/studio/backend/tests/data/refactor_guard/golden_outputs.jsonindex 6f5ba8ba477..7f159d7fbb1 100644--- a/studio/backend/tests/data/refactor_guard/golden_outputs.json+++ b/studio/backend/tests/data/refactor_guard/golden_outputs.json@@ -34,10 +34,11 @@"_next_top_level_comma": "1f44b94c514b2189665de34a043f107dbeee0324c27db809594beb851e17b646","_normalize_for_repeat": "2548cc8fc8483a98540456d2d6fb138a2a168a84cdc780c38f4a71c34ec95f3a","_only_a_code_fence": "986a0b8d0c6d4abe8041da540909357131977dd7ba6a72d537440e44b383a32b",+ "_open_inline_code_run": "2f27bf94223e8b14b8faf656fbe7b59f75a087118c516ff26366865179666517","_parse_bare_rehearsals": "c809e331067827ef1f773b2613bbcb9c9c1138f55e12bd37cbfe35919cceba54","_parse_deepseek_tool_calls": "b16631fa1f86628069da6662060cd78f6c2b1edc87e15011bcec5103a450cabd","_parse_function_xml": "8e47d9a73bea635e90c3a8c56b8be91b3e42d1bbf75c36795fad2bdefeb71696",- "_parse_gemma_tool_calls": "405b8f770c78bae4a20ffab6c550107fec66843125e352447a21f45cb24ce4c3",+ "_parse_gemma_tool_calls": "2f1fc545935ef45a31aa2dc32870ee4fc8cfa93d0121f0bcf714c54ac469329f","_parse_glm_tool_calls": "a59501129afac2b10bf29a43097954b5f9cbcebbcf6202982e43f37c7510a190","_parse_kimi_section_body": "5dc402646fc4b4a4fa5c3221f341502911f8904b4248d3108e3c3d028ed85e0f","_parse_kimi_tool_calls": "5dc402646fc4b4a4fa5c3221f341502911f8904b4248d3108e3c3d028ed85e0f",@@ -54,7 +55,7 @@"_skip_mistral_call_id": "df19f4198fa8e5f7472a47ae59a8efe5aadf15737dacf7c8c0f2d8f367b45f34","_string_content_spans": "87ab6652018fa4bd7b4e986eadaa46f29362693f1eca52cb7cae130dd74ff4f1","_strip_function_xml_calls": "88b342ae0b8dad25cfbbaeaf2f77b5f3a1fea9c2553d03f7cad2e19b5230594a",- "_strip_gemma_wrapperless_calls": "002488dbf57e532e01488bdf281eddb0c2dbbc82bc4c86f1ea5cbe4d8af38a84",+ "_strip_gemma_wrapperless_calls": "16f797057d8fac12dd00950f3c172ba8a9ab7bad5918e9b905cbf2ffd25382ce","_strip_glm_calls": "5d2c536c4a3fbbc2e25525b266b06da99e469c10cd0551020f341c9c6f6d01a6","_strip_leading_code_fence": "f0d73fa03d83fd4a7b2cc11df48f2ebe922e968fee683eedb7833f3234e5bc63","_strip_mistral_closed_calls": "874d69e07bf2c7007ed945493c4e065bbdea740d4d9e1f64cbeda382cc8c90c8",@@ -77,15 +78,15 @@"is_reprompt_restatement": "5a3fe224ba657bf9ea1706ef997b81e5c97535b0e03d6699962f386ecbaf15a0","is_short_intent_without_action": "1ed247c04b2b2bae560ef52224328500d36644b68274015b8efd2c2b9072ac4b","leading_bare_gemma_call_is_promotable": "f97184954908b5ae23160649f0aadf26a8f3f1b65534d9cc350d91504f478a94",- "parse_tool_calls_from_text": "504e5ea7ae128e882236ce50ac5f24a3d1a793d6a47c0664534e4cc6a1b8db40",- "promotable_gemma_call_pos": "70c768036fb37edebc476d8136d5e47e1fb5e6b6691b0cd47becc687552f3071",+ "parse_tool_calls_from_text": "d577ff6ffb72d4a79cde3cac59149ba0a3d923f155ec43c54596410fc8e4d9f4",+ "promotable_gemma_call_pos": "9439dc754e212c8a7a203c6740a782e4885c2c61ba413fc4ff0948ea7200e966","repeated_result_message": "d6e89e731201e216a32e40ccaadc19eaf0f796cee28f4af73675b731666f7e27","reprompt_to_act_message": "27b5a6e3694684acde1db0e63b2415462f24ab5a5825e37a30f00912ebcac01f","starved_result_message": "0970549670906c513a1de39c5fb073a1b08b06919c14ba2b7dd1145cbd9caa51","strip_leading_bare_json_call": "99eb64662edae2a7404834def44076a021999fc257ef387d0032f1ccc69709b9","strip_llama3_leading_sentinels": "de12417a3603b3fca509343aae9c28b8ac19ae4f7ce440c42d2bdd766ef0cf15",- "strip_segment": "bbd87de97063c8ef0201d80af27da3f39cb1f1069756d758bfa6932ded58f668",- "strip_tool_markup": "6daead708970bc2294e207ded2115c6b37661512be2df2f5309b5b6bc65b7ca3",+ "strip_segment": "b147d755b81e75d1d1413fdc6adc58f9f4624200271f07e2c4ffe7b14b6a6b3b",+ "strip_tool_markup": "138b118039304fc4494831a8d5f98a768232c3c0fff7d2f2029572d0f45825b2","unfinished_thought_progress": "9d557ae4ed87151307746a8c62575b4479aeb7130f1f5ebb78dd60c710e66f71"},"core.tool_healing": {diff --git a/studio/backend/tests/data/refactor_guard/runtime_inventory.json b/studio/backend/tests/data/refactor_guard/runtime_inventory.jsonindex c990b9b26fd..f5e38e32e6e 100644--- a/studio/backend/tests/data/refactor_guard/runtime_inventory.json+++ b/studio/backend/tests/data/refactor_guard/runtime_inventory.json@@ -19,6 +19,7 @@"TOOL_XML_SIGNALS","_ACTION_VERB","_ATTR_FUNC_OPEN_RE",+ "_BACKTICK_RUN_RE","_BARE_JSON_ARGS_KEYS","_BARE_JSON_CLASSIFY_KEYS","_BARE_JSON_NAME_RE",@@ -142,6 +143,7 @@"_next_top_level_comma","_normalize_for_repeat","_only_a_code_fence",+ "_open_inline_code_run","_parse_bare_rehearsals","_parse_deepseek_tool_calls","_parse_function_xml",@@ -248,6 +250,7 @@"_next_top_level_comma": "core.inference.tool_call_parser._next_top_level_comma(text: 'str', start: 'int', end: 'int') -> \"'int | None'\"","_normalize_for_repeat": "core.inference.tool_call_parser._normalize_for_repeat(text: 'str') -> 'str'","_only_a_code_fence": "core.inference.tool_call_parser._only_a_code_fence(between: 'str') -> 'bool'",+ "_open_inline_code_run": "core.inference.tool_call_parser._open_inline_code_run(text: 'str', pos: 'int') -> 'bool'","_parse_bare_rehearsals": "core.inference.tool_call_parser._parse_bare_rehearsals(content: 'str', *, id_offset: 'int' = 0, enabled_tool_names: 'Optional[set]' = None) -> 'list[dict]'","_parse_deepseek_tool_calls": "core.inference.tool_call_parser._parse_deepseek_tool_calls(content: 'str', *, id_offset: 'int', allow_incomplete: 'bool' = True) -> 'list[dict]'","_parse_function_xml": "core.inference.tool_call_parser._parse_function_xml(content: 'str', *, id_offset: 'int', allow_incomplete: 'bool' = True) -> 'list[dict]'",@@ -295,7 +298,7 @@"is_short_intent_without_action": "core.inference.tool_call_parser.is_short_intent_without_action(text: 'str') -> 'bool'","leading_bare_gemma_call_is_promotable": "core.inference.tool_call_parser.leading_bare_gemma_call_is_promotable(stripped: 'str', enabled_tool_names) -> 'bool'","parse_tool_calls_from_text": "core.inference.tool_call_parser.parse_tool_calls_from_text(content: 'str', *, id_offset: 'int' = 0, allow_incomplete: 'bool' = True, enabled_tool_names: 'Optional[set]' = None) -> 'list[dict]'",- "promotable_gemma_call_pos": "core.inference.tool_call_parser.promotable_gemma_call_pos(text: 'str', enabled_tool_names, start: 'int' = 0, *, floor: 'int' = 0) -> 'int'",+ "promotable_gemma_call_pos": "core.inference.tool_call_parser.promotable_gemma_call_pos(text: 'str', enabled_tool_names, start: 'int' = 0, *, floor: 'int' = 0, streaming: 'bool' = False) -> 'int'","reasoning_cap_spent_message": "core.inference.tool_call_parser.reasoning_cap_spent_message(max_tokens: 'Optional[int]' = None) -> 'str'","repeated_result_message": "core.inference.tool_call_parser.repeated_result_message(tool_name: 'str', times: 'int', last_result: 'str') -> 'str'","reprompt_to_act_message": "core.inference.tool_call_parser.reprompt_to_act_message(tool_hint: 'str') -> 'str'",@@ -389,7 +392,7 @@"_balanced_bracket_end": "core.tool_healing._balanced_bracket_end(src: 'str', start: 'int', *, braces_count: 'bool' = True) -> 'int | None'","_balanced_json_span": "core.tool_healing._balanced_json_span(text: 'str', start: 'int') -> 'int | None'","_build_markers": "core.tool_healing._build_markers(content: 'str')",- "_code_spans": "core.tool_healing._code_spans(text: 'str') -> 'list[tuple[int, int]]'",+ "_code_spans": "core.tool_healing._code_spans(text: 'str', start: 'int' = 0) -> 'list[tuple[int, int]]'","_decode_array_items": "core.tool_healing._decode_array_items(text: 'str', body_start: 'int', body_end: 'int')","_func_close_index": "core.tool_healing._func_close_index(content: 'str', body_start: 'int', body: 'str') -> 'int'","_gemma_arguments_to_json": "core.tool_healing._gemma_arguments_to_json(args_src: 'str') -> 'dict'",diff --git a/studio/backend/tests/test_gemma_tool_parse_edge_cases.py b/studio/backend/tests/test_gemma_tool_parse_edge_cases.pyindex 02a4d70cbee..8a469c85920 100644--- a/studio/backend/tests/test_gemma_tool_parse_edge_cases.py+++ b/studio/backend/tests/test_gemma_tool_parse_edge_cases.py@@ -10,13 +10,17 @@ import jsonimport sysfrom pathlib import Path+import pytest+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)if _BACKEND_DIR not in sys.path:sys.path.insert(0, _BACKEND_DIR)from core.inference.tool_call_parser import (+ StreamingMarkupStripper,_gemma_parse_value,parse_tool_calls_from_text,+ promotable_gemma_call_pos,)from core.tool_healing import strip_tool_call_markup@@ -462,3 +466,127 @@ def test_streaming_display_of_prose_call_never_shrinks():assert len(out) >= len(seen), (i, out, seen)seen = outassert seen == prose+++_CODE_QUOTED_EXAMPLES = [+ '```\ncall:web_search{query: "cats"}\n```',+ "```text\ncall:web_search{query:cats}\n```",+ "Here is the syntax:\n~~~\ncall:web_search{query:cats}\n~~~",+ "Use `call:web_search{query:cats}` to search.",+]++[email protected]("text", _CODE_QUOTED_EXAMPLES)+def test_wrapperless_call_quoted_in_markdown_code_is_documentation(text):+ en = {"web_search"}+ assert parse_tool_calls_from_text(text, enabled_tool_names = en) == []+ assert promotable_gemma_call_pos(text, en) == -1+ assert _strip(text, en) == text+++def test_streamed_fenced_example_never_turns_into_a_call():+ text = 'Example:\n```\ncall:web_search{query: "cats"}\n```\nDone.'+ en = {"web_search"}+ stripper = StreamingMarkupStripper(en)+ seen = ""+ for i in range(1, len(text) + 1):+ out = stripper.strip(text[:i])+ assert out.startswith(seen), (i, out, seen)+ assert parse_tool_calls_from_text(text[:i], enabled_tool_names = en) == [], i+ assert promotable_gemma_call_pos(text[:i], en) == -1, i+ seen = out+ assert seen == text+++def test_unfenced_wrapperless_call_is_still_promoted_beside_a_fence():+ text = "```\ncall:web_search{query:dogs}\n```\ncall:web_search{query:cats}"+ en = {"web_search"}+ calls = parse_tool_calls_from_text(text, enabled_tool_names = en)+ assert [_args(c) for c in calls] == [{"query": "cats"}]+ assert _strip(text, en) == "```\ncall:web_search{query:dogs}\n```"+++ "text",+ [+ 'call:web_search{query:"intro\n```py\nprint(1)"}\ncall:web_search{query:"y"}',+ 'call:web_search{query:"a``b"}\ncall:web_search{query:"y"} and ``tail``',+ 'Sure call:web_search{query:"a\n```\nx"}\ncall:web_search{query:"y"}',+ ],+)+def test_code_opened_inside_a_call_argument_does_not_hide_the_next_call(text):+ en = {"web_search"}+ calls = parse_tool_calls_from_text(text, enabled_tool_names = en)+ assert len(calls) == 2 and _args(calls[1]) == {"query": "y"}, calls+ assert 'query:"y"' not in _strip(text, en)+++def test_native_token_gemma_call_inside_a_fence_is_still_a_call():+ text = '```\n<|tool_call>call:web_search{query:<|"|>cats<|"|>}<tool_call|>\n```'+ en = {"web_search"}+ calls = parse_tool_calls_from_text(text, enabled_tool_names = en)+ assert len(calls) == 1, calls+ assert _args(calls[0]) == {"query": "cats"}+ assert "call:web_search" not in _strip(text, en)+++ "fenced",+ [+ "````md\n```py\nx=1\n```\n````",+ "~~~~md\n~~~\nx=1\n~~~~",+ "```md\n~~~\nx=1\n```",+ "~~~md\n```\nx=1\n~~~",+ ],+)+def test_wrapperless_call_after_a_fence_quoting_another_fence_is_still_promoted(fenced):+ text = fenced + '\ncall:web_search{query:"b"}'+ en = {"web_search"}+ calls = parse_tool_calls_from_text(text, enabled_tool_names = en)+ assert [_args(c) for c in calls] == [{"query": "b"}]+ assert promotable_gemma_call_pos(text, en) == text.index("call:web_search")+ assert _strip(text, en) == fenced+++def test_inline_code_example_keeps_streaming_on_the_safetensors_loop():+ from core.inference.safetensors_agentic import run_safetensors_tool_loop++ en = {"web_search"}+ for quoted in ('`call:web_search{query:"x"}`', '``call:web_search{query:"x"}``'):+ prose = f"In Gemma syntax: {quoted}. " + "More explanation follows. " * 40+ for i in range(1, len(prose) + 1):+ assert promotable_gemma_call_pos(prose[:i], en, streaming = True) == -1, (quoted, i)++ text = 'In Gemma syntax: ``call:web_search{query:"x"}``. ' + "More explanation follows. " * 40++ calls = []++ def _gen(_messages):+ acc = ""+ for i in range(0, len(text), 4):+ acc += text[i : i + 4]+ yield acc++ events = list(+ run_safetensors_tool_loop(+ single_turn = _gen,+ messages = [{"role": "user", "content": "hi"}],+ tools = [{"type": "function", "function": {"name": "web_search"}}],+ execute_tool = lambda name, arguments, **_: calls.append(name) or "RESULT",+ )+ )+ contents = [e["text"] for e in events if e["type"] == "content"]+ assert calls == []+ assert contents[-1] == text+ assert len(contents[-2]) > text.index("``.") + 2, "stopped streaming at the inline example"+++def test_streaming_hold_lasts_only_while_an_inline_run_is_open_on_the_last_line():+ text = 'Use `call:web_search{query:"x"}\n'+ en = {"web_search"}+ assert promotable_gemma_call_pos(text[:-1], en, streaming = True) == -1+ assert promotable_gemma_call_pos(text, en, streaming = True) == text.index("call:")+ shorter_run = 'Use ``a`b call:web_search{query:"x"}'+ assert promotable_gemma_call_pos(shorter_run, en, streaming = True) == -1+ closed = 'See `code` then call:web_search{query:"x"}'+ assert promotable_gemma_call_pos(closed, en, streaming = True) == closed.index("call:")