Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: do not hand agents a tool call cut off at the token limit (#16035)
Nilay9d195d44246b
diff --git a/studio/backend/core/inference/anthropic_compat.py b/studio/backend/core/inference/anthropic_compat.pyindex ce120e238df..c2198227800 100644--- a/studio/backend/core/inference/anthropic_compat.py+++ b/studio/backend/core/inference/anthropic_compat.py@@ -1256,7 +1256,7 @@ class AnthropicPassthroughEmitter:events.extend(self._emit_text_delta("</think>"))if self._healer is not None:# Last-chance heal of any held residue (e.g. an unclosed tool block).- for kind, value in self._healer.finalize():+ for kind, value in self._healer.finalize(promote = self._stop_reason != "max_tokens"):if kind == "text" and value:events.extend(self._emit_text_delta(value))elif kind == "tool_call":diff --git a/studio/backend/core/inference/passthrough_healing.py b/studio/backend/core/inference/passthrough_healing.pyindex 7e4c255c75f..948bad2bf38 100644--- a/studio/backend/core/inference/passthrough_healing.py+++ b/studio/backend/core/inference/passthrough_healing.py@@ -183,6 +183,7 @@ def heal_openai_message_events(msg: dict,allowed_tools: set,tools: Optional[list] = None,+ allow_incomplete: bool = True,) -> Optional[list]:if not isinstance(msg, dict) or msg.get("tool_calls"):return None@@ -190,12 +191,22 @@ def heal_openai_message_events(if not isinstance(content, str) or not _has_heal_signal(content):return Noneparsed, spans = parse_tool_calls_from_text(content, allow_incomplete = True, with_spans = True)+ # A strict parse alone would promote calls quoted inside a rejected unfinished call.+ closed = (+ None+ if allow_incomplete+ else set(parse_tool_calls_from_text(content, allow_incomplete = False, with_spans = True)[1])+ )tool_schemas = _tool_schemas_by_name(tools) if tools is not None else Noneevents: list = []pos = 0call_count = 0for call, (start, end) in zip(parsed, spans):- promoted = _promote([call], allowed_tools, id_offset = call_count, tool_schemas = tool_schemas)+ promoted = (+ _promote([call], allowed_tools, id_offset = call_count, tool_schemas = tool_schemas)+ if closed is None or (start, end) in closed+ else None+ )if promoted:if content[pos:start]:events.append(("text", content[pos:start]))@@ -215,9 +226,10 @@ def heal_openai_message(msg: dict,allowed_tools: set,tools: Optional[list] = None,+ allow_incomplete: bool = True,) -> bool:"""Edits in place; only promoted calls' markup is removed, undeclared calls stay byte-intact."""- events = heal_openai_message_events(msg, allowed_tools, tools)+ events = heal_openai_message_events(msg, allowed_tools, tools, allow_incomplete)if not events:return Falsecalls = [value for kind, value in events if kind == "tool_call"]@@ -500,6 +512,14 @@ class StreamToolCallHealer:if may_parseelse ([], []))+ # A bracket call after a still-open tag block is that block's argument text.+ if (+ parsed+ and self._buffer.startswith("<")+ and not self._buffer.startswith("<", spans[0][0])+ and _closed_signal_span(self._buffer) is None+ ):+ parsed, spans = [], []if not parsed:# Its close tags are gate tokens, so a closed gate means no closed span.closed_span = _closed_signal_span(self._buffer) if may_parse else None@@ -543,14 +563,14 @@ class StreamToolCallHealer:self._buffer = self._buffer[run_end:]self._holding = False- def finalize(self) -> list:+ def finalize(self, promote: bool = True) -> list:"""Keeps event order; only promoted call markup is dropped and all other residue flushes as text."""self._strict_gate.release()if not self._buffer:return []residue, self._buffer = self._buffer, ""holding, self._holding = self._holding, False- if self.dormant or not holding:+ if self.dormant or not holding or not promote:return [("text", residue)]parsed, spans = parse_tool_calls_from_text(residue,diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 44223c4dacd..04d59f4a5fd 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -34630,9 +34630,22 @@ async def produce_openai_chat_completions(# A cancelled stream must not heal incomplete markup into a tool the user cancelled._cancelled = cancel_event.is_set()+ # Request-scoped holder so concurrent streams cannot read each other's stats.+ _stats = stats_holder.get("stats")+ # A cancelled turn is never 'length'.+ _finish = (+ "stop"+ if _cancelled+ else _safetensors_finish_reason(+ _stats,+ payload,+ is_mlx = bool(_sf_model_info.get("is_mlx", False)),+ durable_run = not cancel_on_disconnect,+ )+ )if healer is not None and not _cancelled:for line in _sf_heal_events_to_sse(- healer.finalize(),+ healer.finalize(promote = _finish != "length"),completion_id,created,model_name,@@ -34642,23 +34655,9 @@ async def produce_openai_chat_completions():yield line- # Request-scoped holder so concurrent streams cannot read each other's stats.- _stats = stats_holder.get("stats")- # A healed tool call wins; a cancelled turn is never 'length'.- _finish = (- "tool_calls"- if (healer is not None and not _cancelled and healer.healed)- else (- "stop"- if _cancelled- else _safetensors_finish_reason(- _stats,- payload,- is_mlx = bool(_sf_model_info.get("is_mlx", False)),- durable_run = not cancel_on_disconnect,- )- )- )+ # Truncation wins over a healed call so the client still sees the cut.+ if healer is not None and not _cancelled and healer.healed and _finish != "length":+ _finish = "tool_calls"yield _chat_final_chunk(completion_id, created, model_name, _finish)# Outside the stats block: a run with unknown token count reports no stats.api_monitor.set_perf(monitor_id, stop_reason = _finish)@@ -34877,8 +34876,12 @@ async def produce_openai_chat_completions(else _stats_finish_reason(stats_holder.get("stats")))if _sf_heal:- if heal_openai_message(_msg, _sf_heal, _sf_healing_tools):- _finish = "tool_calls"+ # Truncation wins: a call open at the cut stays text and the reason stays 'length'.+ if heal_openai_message(+ _msg, _sf_heal, _sf_healing_tools, allow_incomplete = _finish != "length"+ ):+ if _finish != "length":+ _finish = "tool_calls"elif nudge_enabled(payload.nudge_tool_calls,response_format = _sf_response_format,@@ -34927,11 +34930,17 @@ async def produce_openai_chat_completions(if _retry_reasoning:retry_msg["reasoning_content"] = _retry_reasoning# The winning attempt's prompt is reported; completions of both are summed.- if heal_openai_message(retry_msg, _sf_heal, _sf_healing_tools):+ _retry_finish = _stats_finish_reason(stats_holder.get("stats"))+ if heal_openai_message(+ retry_msg,+ _sf_heal,+ _sf_healing_tools,+ allow_incomplete = _retry_finish != "length",+ ):_visible_text, _msg, _finish = (_retry_visible,retry_msg,- "tool_calls",+ "length" if _retry_finish == "length" else "tool_calls",)stats_holder["stats"] = _summed_tool_loop_stats(_first_stats, stats_holder.get("stats")@@ -39576,7 +39585,9 @@ class _ResponsesStreamTranslator:# Heal held residue (an unclosed tool block) before the trailing text, keeping healer order.healer = self.healerif healer is not None:- events = (healer.feed(final_visible) if final_visible else []) + healer.finalize()+ events = (healer.feed(final_visible) if final_visible else []) + healer.finalize(+ promote = self.stream_finish_reason != "length"+ )final_visible = ""if events:# Same as the in-loop stamp: this is where the client first sees@@ -45205,7 +45216,11 @@ def _anthropic_message_from_openai_completion(healing_active = bool(allowed_tools)healed_events = (- heal_openai_message_events(message, allowed_tools, openai_tools) if healing_active else None+ heal_openai_message_events(+ message, allowed_tools, openai_tools, allow_incomplete = finish_reason != "length"+ )+ if healing_active+ else None)content_blocks = []@@ -45249,8 +45264,10 @@ def _anthropic_message_from_openai_completion(# Keep unpromoted bytes when healing is active; legacy stripping is only for opted-out# or no-client-tool requests. The protected helper preserves <think> rehearsal and# balanced [TOOL_CALLS] prose, gated on the declared tools so an inactive- # NAME[ARGS]{...} example is kept.- if not healing_active and response_format is None:+ # NAME[ARGS]{...} example is kept. A client-tool reply cut at the limit keeps its unfinished+ # call: the in-process route already healed it, and that text is all the client gets of it.+ keep_cut_call = openai_tools and finish_reason == "length"+ if not healing_active and response_format is None and not keep_cut_call:text = _strip_tool_xml_for_display(text,auto_heal_tool_calls = True,@@ -46392,14 +46409,14 @@ async def _openai_passthrough_stream_admitted(if not finish:return [raw_line]# On the finish chunk, heal the residue one last time and rewrite "stop" to "tool_calls" if healed.- lines = _healer_sse_lines(healer.finalize())+ lines = _healer_sse_lines(healer.finalize(promote = finish != "length"))if healer.healed and finish == "stop":choice["finish_reason"] = "tool_calls"return lines + ["data: " + json.dumps(chunk_data, ensure_ascii = False)]return lines + [raw_line]events = healer.feed(content)if finish:- events += healer.finalize()+ events += healer.finalize(promote = finish != "length")if not finish and events == [("text", content)]:# Nothing held or promoted: the healer passed the chunk whole, so keep the verbatim upstream bytes.return [raw_line]@@ -47063,9 +47080,14 @@ async def _openai_passthrough_non_streaming_upstream(if not isinstance(msg, dict):continue- # Promote text-form tool calls (declared tools only). Truncation wins: a call cut at max_tokens- # keeps finish_reason='length'.- if _allowed_tools and heal_openai_message(msg, _allowed_tools, body.get("tools")):+ # Promote text-form tool calls (declared tools only). Truncation wins: a reply cut at max_tokens+ # keeps finish_reason='length', and a call still open at the cut stays text.+ if _allowed_tools and heal_openai_message(+ msg,+ _allowed_tools,+ body.get("tools"),+ allow_incomplete = choice.get("finish_reason") != "length",+ ):if choice.get("finish_reason") == "stop":choice["finish_reason"] = "tool_calls"changed = Truediff --git a/studio/backend/tests/test_control_markup_neutralize_7066.py b/studio/backend/tests/test_control_markup_neutralize_7066.pyindex 196c4a53641..dbea5760403 100644--- a/studio/backend/tests/test_control_markup_neutralize_7066.py+++ b/studio/backend/tests/test_control_markup_neutralize_7066.py@@ -4470,11 +4470,11 @@ def test_safetensors_healing_is_gated_on_the_sanitized_catalog():assert packed.count(gate + ",") + packed.count(gate + ")") == 1for call in ("StreamToolCallHealer(_sf_heal, _sf_healing_tools)",- "heal_openai_message(_msg, _sf_heal, _sf_healing_tools)",+ "heal_openai_message(_msg, _sf_heal, _sf_healing_tools,","nudge_should_retry(_data, _sf_heal, _sf_healing_tools)",- "heal_openai_message(retry_msg, _sf_heal, _sf_healing_tools)",+ "heal_openai_message(retry_msg, _sf_heal, _sf_healing_tools,",):- assert call in source, call+ assert call in packed, calldef test_a_dropped_tool_with_a_clean_name_is_not_promotable():diff --git a/studio/backend/tests/test_passthrough_healing.py b/studio/backend/tests/test_passthrough_healing.pyindex b65abc1f309..870bb1fc80b 100644--- a/studio/backend/tests/test_passthrough_healing.py+++ b/studio/backend/tests/test_passthrough_healing.py@@ -336,6 +336,29 @@ class TestStreamHealer:assert not _events_calls(after)+def test_stream_keeps_a_bracket_call_inside_an_open_block_in_its_arguments():+ text = (+ '<tool_call><function=Bash><parameter=command>run [TOOL_CALLS]Bash[ARGS]{"command": "rm"} '+ "li</parameter></function></tool_call>"+ )+ healer = StreamToolCallHealer({"Bash"})+ events = []+ for start in range(0, len(text), 7):+ events += healer.feed(text[start : start + 7])+ events += healer.finalize()+ (call,) = _events_calls(events)+ assert "rm" in json.loads(call["function"]["arguments"])["command"]+ assert "run " in json.loads(call["function"]["arguments"])["command"]+++def test_stream_delivers_a_bracket_call_wrapped_in_a_closed_block():+ healer = StreamToolCallHealer({"Bash"})+ events = healer.feed('<tool_call>[TOOL_CALLS]Bash[ARGS]{"command": "ls"}</tool_call>')+ events += healer.finalize(promote = False)+ (call,) = _events_calls(events)+ assert json.loads(call["function"]["arguments"]) == {"command": "ls"}++class TestNudgeHelpers:def _resp(self,@@ -641,6 +664,22 @@ class TestOpenaiNonStreamingRoute:asyncio.run(_run())+ def test_call_cut_at_length_stays_text(self, monkeypatch):+ async def _run():+ cut = "<tool_call><function=lookup><parameter=q>line1\nli"+ _, data = await _drive_non_streaming(+ monkeypatch,+ _payload(),+ [_upstream_message(LOOKUP_XML + cut, finish_reason = "length")],+ )+ choice = data["choices"][0]+ assert choice["finish_reason"] == "length"+ (call,) = choice["message"]["tool_calls"]+ assert json.loads(call["function"]["arguments"]) == {"q": "x"}+ assert choice["message"]["content"] == cut++ asyncio.run(_run())+def test_tool_choice_none_relays_verbatim(self, monkeypatch):async def _run():_, data = await _drive_non_streaming(@@ -1093,6 +1132,48 @@ class TestAnthropicPassthroughHealingText:asyncio.run(_run())+ def test_call_cut_at_max_tokens_is_not_runnable(self, monkeypatch):+ async def _run():+ cut = "<tool_call><function=lookup><parameter=q>line1\nli"+ data = await self._drive(+ monkeypatch, _upstream_message(LOOKUP_XML + cut, finish_reason = "length")+ )+ (tool_use,) = [b for b in data["content"] if b["type"] == "tool_use"]+ assert tool_use["input"] == {"q": "x"}+ text = "".join(b["text"] for b in data["content"] if b["type"] == "text")+ assert text == cut+ assert data["stop_reason"] == "max_tokens"++ asyncio.run(_run())++ def test_cut_call_healed_upstream_keeps_its_text(self):+ import routes.inference as inf++ cut = "<tool_call><function=lookup><parameter=q>line1\nli"+ response = inf._anthropic_message_from_openai_completion(+ {"choices": [{"message": {"content": cut}, "finish_reason": "length"}]},+ "msg_1",+ "m",+ openai_tools = [LOOKUP_TOOL],+ allowed_tools = None,+ disable_parallel_tool_use = False,+ response_format = None,+ parse_think = True,+ )+ data = json.loads(response.body)+ assert [(b["type"], b["text"]) for b in data["content"]] == [("text", cut)]+ assert data["stop_reason"] == "max_tokens"++ def test_call_quoted_inside_cut_call_is_not_runnable(self, monkeypatch):+ async def _run():+ cut = '<tool_call><function=lookup><parameter=q>run [TOOL_CALLS]lookup[ARGS]{"q": "rm"} li'+ data = await self._drive(monkeypatch, _upstream_message(cut, finish_reason = "length"))+ assert [b["type"] for b in data["content"]] == ["text"]+ assert data["content"][0]["text"] == cut+ assert data["stop_reason"] == "max_tokens"++ asyncio.run(_run())+class TestAnthropicEmitterHealing:def _events(@@ -1217,6 +1298,38 @@ class TestAnthropicEmitterHealing:(message_delta,) = [e for e in events if e.get("type") == "message_delta"]assert message_delta["delta"]["stop_reason"] == "max_tokens"+ def test_call_cut_at_max_tokens_is_not_runnable(self):+ cut = (+ "<tool_call><function=Write><parameter=file_path>app.py</parameter>"+ "<parameter=content>line1\nline2\nli"+ )+ events = self._events(+ self._emitter(allowed = ("Write",)),+ [self._chunk(content = cut), self._chunk(finish_reason = "length")],+ )+ starts = [e for e in events if e.get("type") == "content_block_start"]+ assert [e["content_block"]["type"] for e in starts] == ["text"]+ texts = [+ e["delta"]["text"]+ for e in events+ if e.get("type") == "content_block_delta" and e["delta"]["type"] == "text_delta"+ ]+ assert "".join(texts) == cut+ (message_delta,) = [e for e in events if e.get("type") == "message_delta"]+ assert message_delta["delta"]["stop_reason"] == "max_tokens"++ def test_call_quoted_inside_cut_call_is_not_runnable(self):+ cut = (+ "<tool_call><function=Write><parameter=file_path>app.py</parameter>"+ '<parameter=content>run [TOOL_CALLS]Write[ARGS]{"file_path": "x"} li'+ )+ events = self._events(+ self._emitter(allowed = ("Write",)),+ [self._chunk(content = cut), self._chunk(finish_reason = "length")],+ )+ starts = [e for e in events if e.get("type") == "content_block_start"]+ assert [e["content_block"]["type"] for e in starts] == ["text"]+def test_structured_deltas_disable_healing_and_flush(self):structured = [{@@ -1435,6 +1548,28 @@ class TestOpenaiStreamingRoute:asyncio.run(_run())+ @pytest.mark.parametrize("finish_on_content", [False, True])+ def test_call_cut_at_length_stays_text(self, monkeypatch, finish_on_content):+ async def _run():+ cut = "<tool_call><function=lookup><parameter=q>line1\nli"+ head = {"id": "c1", "model": "gguf", "created": 1}+ content = {"index": 0, "delta": {"content": LOOKUP_XML + cut}}+ finish = {"index": 0, "delta": {}, "finish_reason": "length"}+ choices = (+ [[{**content, "finish_reason": "length"}]]+ if finish_on_content+ else [[content], [finish]]+ )+ lines = ["data: " + json.dumps({**head, "choices": c}) for c in choices]+ chunks = await _drive_stream(monkeypatch, _payload(), lines + ["data: [DONE]"])+ found = [c for p in _stream_payloads(chunks) for c in p.get("choices", [])]+ (call,) = [tc for c in found for tc in (c.get("delta") or {}).get("tool_calls") or []]+ assert json.loads(call["function"]["arguments"]) == {"q": "x"}+ assert "".join((c.get("delta") or {}).get("content") or "" for c in found) == cut+ assert [c["finish_reason"] for c in found if c.get("finish_reason")] == ["length"]++ asyncio.run(_run())+def test_parallel_cap_drops_native_after_healed(self, monkeypatch):async def _run():# The upstream SSE cap keeps native index 0, so the route must drop it itself.diff --git a/studio/backend/tests/test_responses_tool_passthrough.py b/studio/backend/tests/test_responses_tool_passthrough.pyindex 2a99c7c11f9..7240fc1bfa8 100644--- a/studio/backend/tests/test_responses_tool_passthrough.py+++ b/studio/backend/tests/test_responses_tool_passthrough.py@@ -3825,6 +3825,29 @@ def test_a_healed_truncated_tool_call_remains_incomplete(assert item_done["item"]["status"] == "incomplete"+def test_a_call_cut_at_length_stays_text_in_a_responses_stream(monkeypatch):+ api_monitor, tool, _xml = _shared_setup_6()+ name = tool["name"]+ cut = f"<tool_call><function={name}><parameter=q>line1\nli"+ TestResponsesStreamAdapter._install_stream_mock(+ monkeypatch,+ [{"choices": [{"delta": {"content": cut}, "finish_reason": "length"}]}],+ )+ messages, monitor_id, payload = _shared_setup_3(api_monitor, tool)++ async def run():+ response = await _responses_stream(+ payload, messages, TestResponsesStreamAdapter._Request(), monitor_id+ )+ return await TestResponsesStreamAdapter._collect(response)++ lines = asyncio.run(run())++ incomplete = TestResponsesStreamAdapter._payloads(lines, "response.incomplete")[0]+ assert [item["type"] for item in incomplete["response"]["output"]] == ["message"]+ assert incomplete["response"]["output"][0]["content"][0]["text"] == cut++def test_a_complete_responses_stream_still_ends_on_response_completed(monkeypatch):TestResponsesStreamAdapter._install_stream_mock(monkeypatch,diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.pyindex 1c7a491c982..60099ba4a98 100644--- a/studio/backend/tests/test_sf_client_tools_passthrough.py+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py@@ -314,6 +314,46 @@ def test_xml_healed_to_tool_calls_non_streaming(monkeypatch):assert backend.calls[0]["tools"] == [LOOKUP_TOOL]+_CUT_XML = "<tool_call><function=lookup><parameter=q>line1\nli"+_LENGTH_STATS = {"finish_reason": "length"}+++def test_call_cut_at_length_stays_text_non_streaming(monkeypatch):+ backend = _ScriptedBackend(_fixed(_CALL_XML + _CUT_XML), stats = _LENGTH_STATS)+ payload = _request(tools = [LOOKUP_TOOL], stream = False)+ choice = _json_body(_call(payload, monkeypatch, backend))["choices"][0]+ assert choice["finish_reason"] == "length"+ (call,) = choice["message"]["tool_calls"]+ assert json.loads(call["function"]["arguments"]) == {"q": "cats"}+ assert choice["message"]["content"] == _CUT_XML+++def test_call_cut_at_length_stays_text_streaming(monkeypatch):+ backend = _ScriptedBackend(_fixed(_CALL_XML + _CUT_XML), stats = _LENGTH_STATS)+ payload = _request(tools = [LOOKUP_TOOL], stream = True)+ objs = _sse_objects(_collect_sse(_call(payload, monkeypatch, backend)))+ choices = [o["choices"][0] for o in objs if o.get("choices")]+ deltas = [c.get("delta") or {} for c in choices]+ (call,) = [tc for d in deltas for tc in d.get("tool_calls") or []]+ assert json.loads(call["function"]["arguments"]) == {"q": "cats"}+ assert "".join(d.get("content") or "" for d in deltas) == _CUT_XML+ assert [c["finish_reason"] for c in choices if c.get("finish_reason")] == ["length"]+++def test_nudge_retry_cut_at_length_keeps_the_cut(monkeypatch):+ replies = iter(['<tool_call>{"name": "lookup"', _CALL_XML + _CUT_XML])+ backend = _ScriptedBackend(+ lambda messages, tools: [next(replies)], stats = [{"finish_reason": "stop"}, _LENGTH_STATS]+ )+ payload = _request(tools = [LOOKUP_TOOL], stream = False, nudge_tool_calls = True)+ choice = _json_body(_call(payload, monkeypatch, backend))["choices"][0]+ assert len(backend.calls) == 2+ assert choice["finish_reason"] == "length"+ (call,) = choice["message"]["tool_calls"]+ assert json.loads(call["function"]["arguments"]) == {"q": "cats"}+ assert choice["message"]["content"] == _CUT_XML++def test_undeclared_call_stays_text(monkeypatch):xml = '<tool_call>{"name": "other", "arguments": {}}</tool_call>'backend = _ScriptedBackend(_fixed(xml))