Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: do not hand agents a tool call cut off at the token limit (#16035)

Nilay9d195d44246b
diff --git a/studio/backend/core/inference/anthropic_compat.py b/studio/backend/core/inference/anthropic_compat.py
index ce120e238df..c2198227800 100644
--- a/studio/backend/core/inference/anthropic_compat.py
+++ b/studio/backend/core/inference/anthropic_compat.py
@@ -1256,7 +1256,7 @@ class AnthropicPassthroughEmitter:
events.extend(self._emit_text_delta("</think>"))
if self._healer is not None:
# Last-chance heal of any held residue (e.g. an unclosed tool block).
- for kind, value in self._healer.finalize():
+ for kind, value in self._healer.finalize(promote = self._stop_reason != "max_tokens"):
if kind == "text" and value:
events.extend(self._emit_text_delta(value))
elif kind == "tool_call":
diff --git a/studio/backend/core/inference/passthrough_healing.py b/studio/backend/core/inference/passthrough_healing.py
index 7e4c255c75f..948bad2bf38 100644
--- a/studio/backend/core/inference/passthrough_healing.py
+++ b/studio/backend/core/inference/passthrough_healing.py
@@ -183,6 +183,7 @@ def heal_openai_message_events(
msg: dict,
allowed_tools: set,
tools: Optional[list] = None,
+ allow_incomplete: bool = True,
) -> Optional[list]:
if not isinstance(msg, dict) or msg.get("tool_calls"):
return None
@@ -190,12 +191,22 @@ def heal_openai_message_events(
if not isinstance(content, str) or not _has_heal_signal(content):
return None
parsed, spans = parse_tool_calls_from_text(content, allow_incomplete = True, with_spans = True)
+ # A strict parse alone would promote calls quoted inside a rejected unfinished call.
+ closed = (
+ None
+ if allow_incomplete
+ else set(parse_tool_calls_from_text(content, allow_incomplete = False, with_spans = True)[1])
+ )
tool_schemas = _tool_schemas_by_name(tools) if tools is not None else None
events: list = []
pos = 0
call_count = 0
for call, (start, end) in zip(parsed, spans):
- promoted = _promote([call], allowed_tools, id_offset = call_count, tool_schemas = tool_schemas)
+ promoted = (
+ _promote([call], allowed_tools, id_offset = call_count, tool_schemas = tool_schemas)
+ if closed is None or (start, end) in closed
+ else None
+ )
if promoted:
if content[pos:start]:
events.append(("text", content[pos:start]))
@@ -215,9 +226,10 @@ def heal_openai_message(
msg: dict,
allowed_tools: set,
tools: Optional[list] = None,
+ allow_incomplete: bool = True,
) -> bool:
"""Edits in place; only promoted calls' markup is removed, undeclared calls stay byte-intact."""
- events = heal_openai_message_events(msg, allowed_tools, tools)
+ events = heal_openai_message_events(msg, allowed_tools, tools, allow_incomplete)
if not events:
return False
calls = [value for kind, value in events if kind == "tool_call"]
@@ -500,6 +512,14 @@ class StreamToolCallHealer:
if may_parse
else ([], [])
)
+ # A bracket call after a still-open tag block is that block's argument text.
+ if (
+ parsed
+ and self._buffer.startswith("<")
+ and not self._buffer.startswith("<", spans[0][0])
+ and _closed_signal_span(self._buffer) is None
+ ):
+ parsed, spans = [], []
if not parsed:
# Its close tags are gate tokens, so a closed gate means no closed span.
closed_span = _closed_signal_span(self._buffer) if may_parse else None
@@ -543,14 +563,14 @@ class StreamToolCallHealer:
self._buffer = self._buffer[run_end:]
self._holding = False
- def finalize(self) -> list:
+ def finalize(self, promote: bool = True) -> list:
"""Keeps event order; only promoted call markup is dropped and all other residue flushes as text."""
self._strict_gate.release()
if not self._buffer:
return []
residue, self._buffer = self._buffer, ""
holding, self._holding = self._holding, False
- if self.dormant or not holding:
+ if self.dormant or not holding or not promote:
return [("text", residue)]
parsed, spans = parse_tool_calls_from_text(
residue,
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 44223c4dacd..04d59f4a5fd 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -34630,9 +34630,22 @@ async def produce_openai_chat_completions(
# A cancelled stream must not heal incomplete markup into a tool the user cancelled.
_cancelled = cancel_event.is_set()
+ # Request-scoped holder so concurrent streams cannot read each other's stats.
+ _stats = stats_holder.get("stats")
+ # A cancelled turn is never 'length'.
+ _finish = (
+ "stop"
+ if _cancelled
+ else _safetensors_finish_reason(
+ _stats,
+ payload,
+ is_mlx = bool(_sf_model_info.get("is_mlx", False)),
+ durable_run = not cancel_on_disconnect,
+ )
+ )
if healer is not None and not _cancelled:
for line in _sf_heal_events_to_sse(
- healer.finalize(),
+ healer.finalize(promote = _finish != "length"),
completion_id,
created,
model_name,
@@ -34642,23 +34655,9 @@ async def produce_openai_chat_completions(
):
yield line
- # Request-scoped holder so concurrent streams cannot read each other's stats.
- _stats = stats_holder.get("stats")
- # A healed tool call wins; a cancelled turn is never 'length'.
- _finish = (
- "tool_calls"
- if (healer is not None and not _cancelled and healer.healed)
- else (
- "stop"
- if _cancelled
- else _safetensors_finish_reason(
- _stats,
- payload,
- is_mlx = bool(_sf_model_info.get("is_mlx", False)),
- durable_run = not cancel_on_disconnect,
- )
- )
- )
+ # Truncation wins over a healed call so the client still sees the cut.
+ if healer is not None and not _cancelled and healer.healed and _finish != "length":
+ _finish = "tool_calls"
yield _chat_final_chunk(completion_id, created, model_name, _finish)
# Outside the stats block: a run with unknown token count reports no stats.
api_monitor.set_perf(monitor_id, stop_reason = _finish)
@@ -34877,8 +34876,12 @@ async def produce_openai_chat_completions(
else _stats_finish_reason(stats_holder.get("stats"))
)
if _sf_heal:
- if heal_openai_message(_msg, _sf_heal, _sf_healing_tools):
- _finish = "tool_calls"
+ # Truncation wins: a call open at the cut stays text and the reason stays 'length'.
+ if heal_openai_message(
+ _msg, _sf_heal, _sf_healing_tools, allow_incomplete = _finish != "length"
+ ):
+ if _finish != "length":
+ _finish = "tool_calls"
elif nudge_enabled(
payload.nudge_tool_calls,
response_format = _sf_response_format,
@@ -34927,11 +34930,17 @@ async def produce_openai_chat_completions(
if _retry_reasoning:
retry_msg["reasoning_content"] = _retry_reasoning
# The winning attempt's prompt is reported; completions of both are summed.
- if heal_openai_message(retry_msg, _sf_heal, _sf_healing_tools):
+ _retry_finish = _stats_finish_reason(stats_holder.get("stats"))
+ if heal_openai_message(
+ retry_msg,
+ _sf_heal,
+ _sf_healing_tools,
+ allow_incomplete = _retry_finish != "length",
+ ):
_visible_text, _msg, _finish = (
_retry_visible,
retry_msg,
- "tool_calls",
+ "length" if _retry_finish == "length" else "tool_calls",
)
stats_holder["stats"] = _summed_tool_loop_stats(
_first_stats, stats_holder.get("stats")
@@ -39576,7 +39585,9 @@ class _ResponsesStreamTranslator:
# Heal held residue (an unclosed tool block) before the trailing text, keeping healer order.
healer = self.healer
if healer is not None:
- events = (healer.feed(final_visible) if final_visible else []) + healer.finalize()
+ events = (healer.feed(final_visible) if final_visible else []) + healer.finalize(
+ promote = self.stream_finish_reason != "length"
+ )
final_visible = ""
if events:
# Same as the in-loop stamp: this is where the client first sees
@@ -45205,7 +45216,11 @@ def _anthropic_message_from_openai_completion(
healing_active = bool(allowed_tools)
healed_events = (
- heal_openai_message_events(message, allowed_tools, openai_tools) if healing_active else None
+ heal_openai_message_events(
+ message, allowed_tools, openai_tools, allow_incomplete = finish_reason != "length"
+ )
+ if healing_active
+ else None
)
content_blocks = []
@@ -45249,8 +45264,10 @@ def _anthropic_message_from_openai_completion(
# Keep unpromoted bytes when healing is active; legacy stripping is only for opted-out
# or no-client-tool requests. The protected helper preserves <think> rehearsal and
# balanced [TOOL_CALLS] prose, gated on the declared tools so an inactive
- # NAME[ARGS]{...} example is kept.
- if not healing_active and response_format is None:
+ # NAME[ARGS]{...} example is kept. A client-tool reply cut at the limit keeps its unfinished
+ # call: the in-process route already healed it, and that text is all the client gets of it.
+ keep_cut_call = openai_tools and finish_reason == "length"
+ if not healing_active and response_format is None and not keep_cut_call:
text = _strip_tool_xml_for_display(
text,
auto_heal_tool_calls = True,
@@ -46392,14 +46409,14 @@ async def _openai_passthrough_stream_admitted(
if not finish:
return [raw_line]
# On the finish chunk, heal the residue one last time and rewrite "stop" to "tool_calls" if healed.
- lines = _healer_sse_lines(healer.finalize())
+ lines = _healer_sse_lines(healer.finalize(promote = finish != "length"))
if healer.healed and finish == "stop":
choice["finish_reason"] = "tool_calls"
return lines + ["data: " + json.dumps(chunk_data, ensure_ascii = False)]
return lines + [raw_line]
events = healer.feed(content)
if finish:
- events += healer.finalize()
+ events += healer.finalize(promote = finish != "length")
if not finish and events == [("text", content)]:
# Nothing held or promoted: the healer passed the chunk whole, so keep the verbatim upstream bytes.
return [raw_line]
@@ -47063,9 +47080,14 @@ async def _openai_passthrough_non_streaming_upstream(
if not isinstance(msg, dict):
continue
- # Promote text-form tool calls (declared tools only). Truncation wins: a call cut at max_tokens
- # keeps finish_reason='length'.
- if _allowed_tools and heal_openai_message(msg, _allowed_tools, body.get("tools")):
+ # Promote text-form tool calls (declared tools only). Truncation wins: a reply cut at max_tokens
+ # keeps finish_reason='length', and a call still open at the cut stays text.
+ if _allowed_tools and heal_openai_message(
+ msg,
+ _allowed_tools,
+ body.get("tools"),
+ allow_incomplete = choice.get("finish_reason") != "length",
+ ):
if choice.get("finish_reason") == "stop":
choice["finish_reason"] = "tool_calls"
changed = True
diff --git a/studio/backend/tests/test_control_markup_neutralize_7066.py b/studio/backend/tests/test_control_markup_neutralize_7066.py
index 196c4a53641..dbea5760403 100644
--- a/studio/backend/tests/test_control_markup_neutralize_7066.py
+++ b/studio/backend/tests/test_control_markup_neutralize_7066.py
@@ -4470,11 +4470,11 @@ def test_safetensors_healing_is_gated_on_the_sanitized_catalog():
assert packed.count(gate + ",") + packed.count(gate + ")") == 1
for call in (
"StreamToolCallHealer(_sf_heal, _sf_healing_tools)",
- "heal_openai_message(_msg, _sf_heal, _sf_healing_tools)",
+ "heal_openai_message(_msg, _sf_heal, _sf_healing_tools,",
"nudge_should_retry(_data, _sf_heal, _sf_healing_tools)",
- "heal_openai_message(retry_msg, _sf_heal, _sf_healing_tools)",
+ "heal_openai_message(retry_msg, _sf_heal, _sf_healing_tools,",
):
- assert call in source, call
+ assert call in packed, call
def test_a_dropped_tool_with_a_clean_name_is_not_promotable():
diff --git a/studio/backend/tests/test_passthrough_healing.py b/studio/backend/tests/test_passthrough_healing.py
index b65abc1f309..870bb1fc80b 100644
--- a/studio/backend/tests/test_passthrough_healing.py
+++ b/studio/backend/tests/test_passthrough_healing.py
@@ -336,6 +336,29 @@ class TestStreamHealer:
assert not _events_calls(after)
+def test_stream_keeps_a_bracket_call_inside_an_open_block_in_its_arguments():
+ text = (
+ '<tool_call><function=Bash><parameter=command>run [TOOL_CALLS]Bash[ARGS]{"command": "rm"} '
+ "li</parameter></function></tool_call>"
+ )
+ healer = StreamToolCallHealer({"Bash"})
+ events = []
+ for start in range(0, len(text), 7):
+ events += healer.feed(text[start : start + 7])
+ events += healer.finalize()
+ (call,) = _events_calls(events)
+ assert "rm" in json.loads(call["function"]["arguments"])["command"]
+ assert "run " in json.loads(call["function"]["arguments"])["command"]
+
+
+def test_stream_delivers_a_bracket_call_wrapped_in_a_closed_block():
+ healer = StreamToolCallHealer({"Bash"})
+ events = healer.feed('<tool_call>[TOOL_CALLS]Bash[ARGS]{"command": "ls"}</tool_call>')
+ events += healer.finalize(promote = False)
+ (call,) = _events_calls(events)
+ assert json.loads(call["function"]["arguments"]) == {"command": "ls"}
+
+
class TestNudgeHelpers:
def _resp(
self,
@@ -641,6 +664,22 @@ class TestOpenaiNonStreamingRoute:
asyncio.run(_run())
+ def test_call_cut_at_length_stays_text(self, monkeypatch):
+ async def _run():
+ cut = "<tool_call><function=lookup><parameter=q>line1\nli"
+ _, data = await _drive_non_streaming(
+ monkeypatch,
+ _payload(),
+ [_upstream_message(LOOKUP_XML + cut, finish_reason = "length")],
+ )
+ choice = data["choices"][0]
+ assert choice["finish_reason"] == "length"
+ (call,) = choice["message"]["tool_calls"]
+ assert json.loads(call["function"]["arguments"]) == {"q": "x"}
+ assert choice["message"]["content"] == cut
+
+ asyncio.run(_run())
+
def test_tool_choice_none_relays_verbatim(self, monkeypatch):
async def _run():
_, data = await _drive_non_streaming(
@@ -1093,6 +1132,48 @@ class TestAnthropicPassthroughHealingText:
asyncio.run(_run())
+ def test_call_cut_at_max_tokens_is_not_runnable(self, monkeypatch):
+ async def _run():
+ cut = "<tool_call><function=lookup><parameter=q>line1\nli"
+ data = await self._drive(
+ monkeypatch, _upstream_message(LOOKUP_XML + cut, finish_reason = "length")
+ )
+ (tool_use,) = [b for b in data["content"] if b["type"] == "tool_use"]
+ assert tool_use["input"] == {"q": "x"}
+ text = "".join(b["text"] for b in data["content"] if b["type"] == "text")
+ assert text == cut
+ assert data["stop_reason"] == "max_tokens"
+
+ asyncio.run(_run())
+
+ def test_cut_call_healed_upstream_keeps_its_text(self):
+ import routes.inference as inf
+
+ cut = "<tool_call><function=lookup><parameter=q>line1\nli"
+ response = inf._anthropic_message_from_openai_completion(
+ {"choices": [{"message": {"content": cut}, "finish_reason": "length"}]},
+ "msg_1",
+ "m",
+ openai_tools = [LOOKUP_TOOL],
+ allowed_tools = None,
+ disable_parallel_tool_use = False,
+ response_format = None,
+ parse_think = True,
+ )
+ data = json.loads(response.body)
+ assert [(b["type"], b["text"]) for b in data["content"]] == [("text", cut)]
+ assert data["stop_reason"] == "max_tokens"
+
+ def test_call_quoted_inside_cut_call_is_not_runnable(self, monkeypatch):
+ async def _run():
+ cut = '<tool_call><function=lookup><parameter=q>run [TOOL_CALLS]lookup[ARGS]{"q": "rm"} li'
+ data = await self._drive(monkeypatch, _upstream_message(cut, finish_reason = "length"))
+ assert [b["type"] for b in data["content"]] == ["text"]
+ assert data["content"][0]["text"] == cut
+ assert data["stop_reason"] == "max_tokens"
+
+ asyncio.run(_run())
+
class TestAnthropicEmitterHealing:
def _events(
@@ -1217,6 +1298,38 @@ class TestAnthropicEmitterHealing:
(message_delta,) = [e for e in events if e.get("type") == "message_delta"]
assert message_delta["delta"]["stop_reason"] == "max_tokens"
+ def test_call_cut_at_max_tokens_is_not_runnable(self):
+ cut = (
+ "<tool_call><function=Write><parameter=file_path>app.py</parameter>"
+ "<parameter=content>line1\nline2\nli"
+ )
+ events = self._events(
+ self._emitter(allowed = ("Write",)),
+ [self._chunk(content = cut), self._chunk(finish_reason = "length")],
+ )
+ starts = [e for e in events if e.get("type") == "content_block_start"]
+ assert [e["content_block"]["type"] for e in starts] == ["text"]
+ texts = [
+ e["delta"]["text"]
+ for e in events
+ if e.get("type") == "content_block_delta" and e["delta"]["type"] == "text_delta"
+ ]
+ assert "".join(texts) == cut
+ (message_delta,) = [e for e in events if e.get("type") == "message_delta"]
+ assert message_delta["delta"]["stop_reason"] == "max_tokens"
+
+ def test_call_quoted_inside_cut_call_is_not_runnable(self):
+ cut = (
+ "<tool_call><function=Write><parameter=file_path>app.py</parameter>"
+ '<parameter=content>run [TOOL_CALLS]Write[ARGS]{"file_path": "x"} li'
+ )
+ events = self._events(
+ self._emitter(allowed = ("Write",)),
+ [self._chunk(content = cut), self._chunk(finish_reason = "length")],
+ )
+ starts = [e for e in events if e.get("type") == "content_block_start"]
+ assert [e["content_block"]["type"] for e in starts] == ["text"]
+
def test_structured_deltas_disable_healing_and_flush(self):
structured = [
{
@@ -1435,6 +1548,28 @@ class TestOpenaiStreamingRoute:
asyncio.run(_run())
+ @pytest.mark.parametrize("finish_on_content", [False, True])
+ def test_call_cut_at_length_stays_text(self, monkeypatch, finish_on_content):
+ async def _run():
+ cut = "<tool_call><function=lookup><parameter=q>line1\nli"
+ head = {"id": "c1", "model": "gguf", "created": 1}
+ content = {"index": 0, "delta": {"content": LOOKUP_XML + cut}}
+ finish = {"index": 0, "delta": {}, "finish_reason": "length"}
+ choices = (
+ [[{**content, "finish_reason": "length"}]]
+ if finish_on_content
+ else [[content], [finish]]
+ )
+ lines = ["data: " + json.dumps({**head, "choices": c}) for c in choices]
+ chunks = await _drive_stream(monkeypatch, _payload(), lines + ["data: [DONE]"])
+ found = [c for p in _stream_payloads(chunks) for c in p.get("choices", [])]
+ (call,) = [tc for c in found for tc in (c.get("delta") or {}).get("tool_calls") or []]
+ assert json.loads(call["function"]["arguments"]) == {"q": "x"}
+ assert "".join((c.get("delta") or {}).get("content") or "" for c in found) == cut
+ assert [c["finish_reason"] for c in found if c.get("finish_reason")] == ["length"]
+
+ asyncio.run(_run())
+
def test_parallel_cap_drops_native_after_healed(self, monkeypatch):
async def _run():
# The upstream SSE cap keeps native index 0, so the route must drop it itself.
diff --git a/studio/backend/tests/test_responses_tool_passthrough.py b/studio/backend/tests/test_responses_tool_passthrough.py
index 2a99c7c11f9..7240fc1bfa8 100644
--- a/studio/backend/tests/test_responses_tool_passthrough.py
+++ b/studio/backend/tests/test_responses_tool_passthrough.py
@@ -3825,6 +3825,29 @@ def test_a_healed_truncated_tool_call_remains_incomplete(
assert item_done["item"]["status"] == "incomplete"
+def test_a_call_cut_at_length_stays_text_in_a_responses_stream(monkeypatch):
+ api_monitor, tool, _xml = _shared_setup_6()
+ name = tool["name"]
+ cut = f"<tool_call><function={name}><parameter=q>line1\nli"
+ TestResponsesStreamAdapter._install_stream_mock(
+ monkeypatch,
+ [{"choices": [{"delta": {"content": cut}, "finish_reason": "length"}]}],
+ )
+ messages, monitor_id, payload = _shared_setup_3(api_monitor, tool)
+
+ async def run():
+ response = await _responses_stream(
+ payload, messages, TestResponsesStreamAdapter._Request(), monitor_id
+ )
+ return await TestResponsesStreamAdapter._collect(response)
+
+ lines = asyncio.run(run())
+
+ incomplete = TestResponsesStreamAdapter._payloads(lines, "response.incomplete")[0]
+ assert [item["type"] for item in incomplete["response"]["output"]] == ["message"]
+ assert incomplete["response"]["output"][0]["content"][0]["text"] == cut
+
+
def test_a_complete_responses_stream_still_ends_on_response_completed(monkeypatch):
TestResponsesStreamAdapter._install_stream_mock(
monkeypatch,
diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.py
index 1c7a491c982..60099ba4a98 100644
--- a/studio/backend/tests/test_sf_client_tools_passthrough.py
+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py
@@ -314,6 +314,46 @@ def test_xml_healed_to_tool_calls_non_streaming(monkeypatch):
assert backend.calls[0]["tools"] == [LOOKUP_TOOL]
+_CUT_XML = "<tool_call><function=lookup><parameter=q>line1\nli"
+_LENGTH_STATS = {"finish_reason": "length"}
+
+
+def test_call_cut_at_length_stays_text_non_streaming(monkeypatch):
+ backend = _ScriptedBackend(_fixed(_CALL_XML + _CUT_XML), stats = _LENGTH_STATS)
+ payload = _request(tools = [LOOKUP_TOOL], stream = False)
+ choice = _json_body(_call(payload, monkeypatch, backend))["choices"][0]
+ assert choice["finish_reason"] == "length"
+ (call,) = choice["message"]["tool_calls"]
+ assert json.loads(call["function"]["arguments"]) == {"q": "cats"}
+ assert choice["message"]["content"] == _CUT_XML
+
+
+def test_call_cut_at_length_stays_text_streaming(monkeypatch):
+ backend = _ScriptedBackend(_fixed(_CALL_XML + _CUT_XML), stats = _LENGTH_STATS)
+ payload = _request(tools = [LOOKUP_TOOL], stream = True)
+ objs = _sse_objects(_collect_sse(_call(payload, monkeypatch, backend)))
+ choices = [o["choices"][0] for o in objs if o.get("choices")]
+ deltas = [c.get("delta") or {} for c in choices]
+ (call,) = [tc for d in deltas for tc in d.get("tool_calls") or []]
+ assert json.loads(call["function"]["arguments"]) == {"q": "cats"}
+ assert "".join(d.get("content") or "" for d in deltas) == _CUT_XML
+ assert [c["finish_reason"] for c in choices if c.get("finish_reason")] == ["length"]
+
+
+def test_nudge_retry_cut_at_length_keeps_the_cut(monkeypatch):
+ replies = iter(['<tool_call>{"name": "lookup"', _CALL_XML + _CUT_XML])
+ backend = _ScriptedBackend(
+ lambda messages, tools: [next(replies)], stats = [{"finish_reason": "stop"}, _LENGTH_STATS]
+ )
+ payload = _request(tools = [LOOKUP_TOOL], stream = False, nudge_tool_calls = True)
+ choice = _json_body(_call(payload, monkeypatch, backend))["choices"][0]
+ assert len(backend.calls) == 2
+ assert choice["finish_reason"] == "length"
+ (call,) = choice["message"]["tool_calls"]
+ assert json.loads(call["function"]["arguments"]) == {"q": "cats"}
+ assert choice["message"]["content"] == _CUT_XML
+
+
def test_undeclared_call_stays_text(monkeypatch):
xml = '<tool_call>{"name": "other", "arguments": {}}</tool_call>'
backend = _ScriptedBackend(_fixed(xml))