Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep the prompt cache across long tool chats and follow-up turns (#13456)

Daniel Hanfd30659ebc8b
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index c2167046f7a..02fbcee7212 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -495,6 +495,7 @@ from core.inference.tool_loop_controller import (
awaiting_approval_status,
deferred_nudge_text,
provisional_tool_provenance,
+ restore_replayed_tool_results,
tool_call_limit_nudge,
)
from state.tool_approvals import (
@@ -9075,6 +9076,94 @@ class LlamaCppBackend:
)
return {"enable_thinking": enable_thinking}
+ def _replay_preserve_thinking(
+ self,
+ preserve_thinking: Optional[bool],
+ messages,
+ enable_thinking: Optional[bool] = None,
+ reasoning_effort: Optional[str] = None,
+ ) -> Optional[bool]:
+ """True for a thinking-off follow-up with no reasoning in its history, where the template proves
+ that only preserve_thinking replays each answer's empty think block as generated."""
+ if preserve_thinking is not False or not getattr(
+ self, "_supports_preserve_thinking", False
+ ):
+ return preserve_thinking
+ mode = self._request_reasoning_kwargs(enable_thinking, reasoning_effort, None) or {}
+ if mode.get("enable_thinking") is not False:
+ return preserve_thinking
+ answered = False
+ for message in messages or ():
+ if not isinstance(message, dict) or message.get("role") != "assistant":
+ continue
+ answered = True
+ reasoning = message.get("reasoning_content")
+ if isinstance(reasoning, str) and reasoning.strip():
+ return preserve_thinking
+ # Inline reasoning counts too.
+ content = message.get("content")
+ parts = content if isinstance(content, list) else [content]
+ if any(
+ "</think>" in (part.get("text") if isinstance(part, dict) else part)
+ for part in parts
+ if isinstance(part, str)
+ or (isinstance(part, dict) and isinstance(part.get("text"), str))
+ ):
+ return preserve_thinking
+ if not answered:
+ return preserve_thinking
+ return True if self._empty_think_replays_with_preserve() else preserve_thinking
+
+ def _empty_think_replays_with_preserve(self) -> bool:
+ """Probed once per server: generation prompt + answer prefixes the next render only when kept."""
+ key = (
+ getattr(self, "base_url", None),
+ id(getattr(self, "_chat_template", None)),
+ id(getattr(self, "_chat_template_override", None)),
+ )
+ cached = getattr(self, "_empty_think_preserve_probe", None)
+ if cached is not None and cached[0] == key:
+ return cached[1]
+ result = False
+ try:
+ off = dict(self._request_reasoning_kwargs(False, None, None) or {})
+ answer = "Studio replay probe answer."
+ first = [{"role": "user", "content": "Studio replay probe question."}]
+ history = first + [
+ {"role": "assistant", "content": answer},
+ {"role": "user", "content": "Studio replay probe follow-up."},
+ ]
+ with httpx.Client(
+ timeout = 10,
+ headers = self._auth_headers,
+ trust_env = False,
+ verify = _local_ssl_context(),
+ ) as client:
+
+ def _render(msgs, preserve):
+ r = client.post(
+ f"{self.base_url}/apply-template",
+ json = {
+ "messages": msgs,
+ "chat_template_kwargs": {**off, "preserve_thinking": preserve},
+ },
+ )
+ prompt = r.json().get("prompt") if r.status_code == 200 else None
+ return prompt if isinstance(prompt, str) else None
+
+ generated = _render(first, False)
+ kept = _render(history, True)
+ dropped = _render(history, False)
+ if generated is None or kept is None or dropped is None:
+ return False # Unanswered: not cached, so the next request asks again.
+ result = kept.startswith(generated + answer) and not dropped.startswith(
+ generated + answer
+ )
+ except Exception: # noqa: BLE001 -- an unanswerable probe keeps the caller's setting
+ return False
+ self._empty_think_preserve_probe = (key, result)
+ return result
+
def _request_reasoning_kwargs(
self,
enable_thinking: Optional[bool],
@@ -37597,6 +37686,9 @@ class LlamaCppBackend:
"""
if not self.is_loaded:
raise RuntimeError("llama-server is not loaded")
+ preserve_thinking = self._replay_preserve_thinking(
+ preserve_thinking, messages, enable_thinking, reasoning_effort
+ )
from core.inference.chat_template_helpers import (
neutralize_control_markup_in_messages,
@@ -38065,6 +38157,9 @@ class LlamaCppBackend:
{"type": "content", "text": "token"} -- streamed content tokens (cumulative)
{"type": "reasoning", "text": "token"} -- streamed reasoning tokens (cumulative)
"""
+ preserve_thinking = self._replay_preserve_thinking(
+ preserve_thinking, messages, enable_thinking, reasoning_effort
+ )
from core.inference.tool_stream_exec import (
accepts_kwarg,
accepts_output_callback,
@@ -38110,7 +38205,7 @@ class LlamaCppBackend:
messages = note_attached_image(
messages, mcp_image_targets(_gguf_active_tool_names(tools))
)
- conversation = list(messages)
+ conversation = restore_replayed_tool_results(messages)
# Seeded with tool results an explicit skill load relies on, so fitting keeps them.
_rolling_anchor_ids: set[int] = set(instruction_anchor_ids or ())
# The loop refits on every iteration; recall must fire once per request, or each
@@ -38131,14 +38226,15 @@ class LlamaCppBackend:
# injected into `conversation`. The archive is keyed by thread and the stored rows
# are the whole DAG (Retry keeps the replaced response as a sibling), so recall
# has to be told which branch is live, and the client's messages are the answer.
- _request_branch: list[dict] = list(messages)
+ # Restored dicts are new; both branch lists match by identity.
+ _request_branch: list[dict] = list(conversation)
# ...and everything this request adds to it. A long agent run can evict its own
# earlier tool exchange, which is archived; filtered against the messages the
# client sent, that document looks like an abandoned branch and is refused, so the
# model cannot get back a tool result it still needs to answer. Accumulated rather
# than read off `conversation`, which has already lost what earlier fits evicted.
- _live_branch: list[dict] = list(messages)
- _live_branch_ids: set[int] = {id(message) for message in messages}
+ _live_branch: list[dict] = list(conversation)
+ _live_branch_ids: set[int] = {id(message) for message in conversation}
def _extend_live_branch(current: list[dict]) -> list[dict]:
for message in current:
@@ -38152,6 +38248,10 @@ class LlamaCppBackend:
_rolling_anchor_ids.add(id(message))
break
+ _tool_card_by_message: dict[int, str] = {}
+ # Feedback folded in after tool_end, reported so the client replays it.
+ _model_note_events: list[dict] = []
+
def _attach_internal_feedback_to_tool_result(feedback: str) -> bool:
"""Keep controller instructions inside the current tool exchange."""
if not conversation or conversation[-1].get("role") != "tool":
@@ -38162,7 +38262,18 @@ class LlamaCppBackend:
# Replaced, not mutated: ``conversation`` copies the list, not the dicts, and
# /v1/messages passes a caller's own history through, which may end on a tool
# result. An in-place edit would leave the nudge there and re-send it next turn.
- conversation[-1] = {**conversation[-1], "content": f"{content}\n\n{feedback}"}
+ previous = conversation[-1]
+ conversation[-1] = {**previous, "content": f"{content}\n\n{feedback}"}
+ card_id = _tool_card_by_message.pop(id(previous), None)
+ if card_id:
+ _tool_card_by_message[id(conversation[-1])] = card_id
+ _model_note_events.append(
+ {
+ "type": "tool_model_note",
+ "tool_call_id": card_id,
+ "suffix": f"\n\n{feedback}",
+ }
+ )
return True
# Forced first-pass RAG so a doc question doesn't lose to web_search. Emits
@@ -38585,6 +38696,8 @@ class LlamaCppBackend:
_MAX_CONTINUATION_CREDITS = _MAX_LENGTH_CONTINUATIONS * max(1, max_tool_iterations)
# A fixed seed keeps the prompt cache once this run has sent a request (#10698).
_prompt_sent = False
+ # Resent on the final pass with tool_choice "none": tools head the system prompt.
+ _last_sent_tools = None
iteration = -1
# Outside the loop: the cap is across the whole run, and a per-iteration list
# would only ever see the current batch, leaving every earlier iteration's
@@ -38595,6 +38708,8 @@ class LlamaCppBackend:
loop_mcp_image_parts: list = list(replayed_image_parts)
while True:
iteration += 1
+ while _model_note_events:
+ yield _model_note_events.pop(0)
# Here rather than at each append: six sites grow the conversation and all of
# them pass through this one point before the cache must hold the result.
if on_conversation_grew is not None:
@@ -38832,6 +38947,7 @@ class LlamaCppBackend:
if safe_tools:
payload["tools"] = llama_grammar_tools(safe_tools)
payload["tool_choice"] = requested_choice
+ _last_sent_tools = safe_tools
if _reasoning_kw is not None:
payload["chat_template_kwargs"] = _reasoning_kw
if thinking_budget_tokens is not None:
@@ -40347,6 +40463,8 @@ class LlamaCppBackend:
provisional = provisional_match,
allowed_tool_names = {forced_name} if forced_name else None,
)
+ # The frontend groups a parallel batch into one assistant message by this.
+ decision.provenance["round_id"] = iteration
if not decision.should_execute:
if provisional_match:
@@ -41208,8 +41326,11 @@ class LlamaCppBackend:
# A tool ran this turn, so it counts against the caller's budget.
_turn_executed_real_tool = True
_forced_choice_resolved = True
- yield completion.tool_end_event()
+ _tool_end = completion.tool_end_event()
+ yield _tool_end
conversation.append(completion.tool_message())
+ if _tool_end.get("tool_call_id"):
+ _tool_card_by_message[id(conversation[-1])] = _tool_end["tool_call_id"]
# Parsed only for a result that carries them and a model that will
# be shown them; the marker test first, so a text result never asks.
_completion_images = (
@@ -41469,10 +41590,13 @@ class LlamaCppBackend:
else:
conversation.append({"role": "user", "content": BUDGET_EXHAUSTED_NUDGE})
+ while _model_note_events:
+ yield _model_note_events.pop(0)
# Clear status.
yield {"type": "status", "text": ""}
# Final streaming pass with the full conversation context.
+ _final_tools = _last_sent_tools
from core.inference.chat_template_helpers import neutralize_control_markup_in_messages
_reasoning_kw = self._request_reasoning_kwargs(
@@ -41490,7 +41614,7 @@ class LlamaCppBackend:
try:
_before_final_fit = conversation
# Withheld, not `tools`: this is the synthesised final answer and
- # `stream_payload` below sends no tools array at all, so the request's
+ # `stream_payload` below sends tool_choice "none", so the request's
# catalogue answers a question it does not pose and would let a NEW epoch
# start on the one pass that can never call `search_conversation`.
# Replaying an epoch already in force is unaffected.
@@ -41517,7 +41641,7 @@ class LlamaCppBackend:
messages_without_unpriced_media(fitted), None, self.markup_profile
),
None,
- None,
+ _final_tools,
strict = True,
chat_template_kwargs = _reasoning_kw,
should_abort = lambda: bool(cancel_event and cancel_event.is_set()),
@@ -41538,9 +41662,7 @@ class LlamaCppBackend:
_before_final_fit,
branch_messages = _extend_live_branch(_before_final_fit),
thread_id = thread_id,
- # Inline, not tool: this final-answer request is counted and sent
- # with no tools array, so a forged tool role has no catalogue to
- # match and breaks strict templates.
+ # Inline: under tool_choice "none" a forged tool role breaks strict templates.
style = "inline",
# Checkpoint mode recalls once, on the turn that reset; rolling has no
# such key and keeps recalling on every turn that evicted anything.
@@ -41560,7 +41682,7 @@ class LlamaCppBackend:
self.markup_profile,
),
None,
- None,
+ _final_tools,
strict = True,
chat_template_kwargs = _reasoning_kw,
),
@@ -41617,6 +41739,9 @@ class LlamaCppBackend:
}
if logit_bias:
stream_payload["logit_bias"] = logit_bias
+ if _final_tools:
+ stream_payload["tools"] = llama_grammar_tools(_final_tools)
+ stream_payload["tool_choice"] = "none"
if _reasoning_kw is not None:
stream_payload["chat_template_kwargs"] = _reasoning_kw
if thinking_budget_tokens is not None:
@@ -41679,7 +41804,7 @@ class LlamaCppBackend:
messages_without_unpriced_media(fitted), None, self.markup_profile
),
None,
- None,
+ _final_tools,
strict = True,
chat_template_kwargs = _reasoning_kw,
should_abort = lambda: bool(cancel_event and cancel_event.is_set()),
@@ -41690,7 +41815,7 @@ class LlamaCppBackend:
can_reset = _can_reset_epoch(
thread_id,
_backend_supports_tools(self),
- # The final pass again, so again no tools array is sent.
+ # The final pass again, so again tool_choice is "none".
tools_withheld = True,
),
**_compaction_fit_kwargs(context_policy, compaction_headroom_ratio),
@@ -41739,7 +41864,7 @@ class LlamaCppBackend:
break
_refit_evicted = _evict_until_it_fits(
stream_payload["messages"],
- None,
+ _final_tools,
stream_payload.get("chat_template_kwargs"),
_refit_continue,
_refit_protect,
@@ -41832,7 +41957,7 @@ class LlamaCppBackend:
_tokens = self.count_chat_tokens(
messages_without_unpriced_media(candidate_messages),
None,
- None,
+ _final_tools,
strict = True,
chat_template_kwargs = (
stream_payload.get("chat_template_kwargs")
@@ -42063,7 +42188,7 @@ class LlamaCppBackend:
# gets that eviction rather than being abandoned.
_evicted_f = _evict_until_it_fits(
_candidate_messages,
- None,
+ _final_tools,
stream_payload.get("chat_template_kwargs"),
True,
)
@@ -42187,7 +42312,7 @@ class LlamaCppBackend:
)
if _next_cap_r != 0 and not _served_r:
_evicted_r = _evict_until_it_fits(
- _candidate_r, None, _off_kw, False, _recovery_protect
+ _candidate_r, _final_tools, _off_kw, False, _recovery_protect
)
if _evicted_r is not None and _continuation_would_be_served(
_evicted_r, False, _off_kw
diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.py
index 5858d12ce82..216472a45a7 100644
--- a/studio/backend/core/inference/safetensors_agentic.py
+++ b/studio/backend/core/inference/safetensors_agentic.py
@@ -69,6 +69,7 @@ from core.inference.tool_loop_controller import (
append_deferred_nudges,
awaiting_approval_status,
coerce_tool_arguments,
+ restore_replayed_tool_results,
status_for_tool,
tool_call_limit_nudge,
tool_event_provenance,
@@ -669,7 +670,7 @@ def run_safetensors_tool_loop(
from core.inference.mcp_image import note_attached_image
from core.inference.tools import mcp_image_targets
messages = note_attached_image(messages, mcp_image_targets(_active_tool_names(tools)))
- conversation = list(messages)
+ conversation = restore_replayed_tool_results(messages)
# Where the caller's own attachment sits in the seeded sink. The cap is about what
# the loop RE-SENDS, so that entry is never the one it drops -- but it is not the
# whole seed: a resumed chat seeds replayed pictures too, and exempting those would
@@ -680,7 +681,8 @@ def run_safetensors_tool_loop(
# switch to safetensors, so search_conversation is advertised here too and needs the
# same filtering: the stored rows are the whole DAG, and Retry leaves the replaced
# response in them.
- _live_branch = list(messages)
+ # Restored dicts are new; the branch matches by identity.
+ _live_branch = list(conversation)
# ...and the replies and tool results the loop adds to it: a fit can evict this request's
# own earlier tool exchange into the archive, where the client's messages alone would
# refuse it. Not the loop's user turns: those are its own notices, and recall searches
diff --git a/studio/backend/core/inference/sse_control_frames.py b/studio/backend/core/inference/sse_control_frames.py
index a6a5784930b..6ab1a7b0d7b 100644
--- a/studio/backend/core/inference/sse_control_frames.py
+++ b/studio/backend/core/inference/sse_control_frames.py
@@ -38,6 +38,7 @@ _CONTROL_TYPES = frozenset(
"tool_output",
"tool_args",
"tool_status",
+ "tool_model_note",
"skill_load",
"diffusion_frame",
"reasoning_summary",
diff --git a/studio/backend/core/inference/studio_tool_loop.py b/studio/backend/core/inference/studio_tool_loop.py
index 02732b444b1..8d7b678dd7a 100644
--- a/studio/backend/core/inference/studio_tool_loop.py
+++ b/studio/backend/core/inference/studio_tool_loop.py
@@ -83,6 +83,7 @@ from core.inference.tool_loop_controller import (
canonical_arguments_text,
mcp_display_parts,
provisional_tool_provenance,
+ restore_replayed_tool_results,
strip_result_for_model,
)
from core.inference.tool_stream_exec import (
@@ -1406,7 +1407,9 @@ async def stream_with_studio_tools(
cancel_event: threading.Event,
mcp_image = None,
) -> AsyncIterator[str]:
- conversation = [dict(message) for message in run.messages]
+ # Off the loop: restoring may parse an envelope.
+ restored = await asyncio.to_thread(restore_replayed_tool_results, run.messages)
+ conversation = [dict(message) for message in restored]
if mcp_image is not None:
targets = await asyncio.to_thread(mcp_image_targets, sorted(_tool_names(policy.tools)))
conversation = note_attached_image(conversation, targets)
@@ -1415,7 +1418,7 @@ async def stream_with_studio_tools(
# cap run-owned image parts across the full loop without counting caller attachments.
loop_mcp_image_parts: list = list(run.promoted_image_parts)
# preserve the request branch for tools that search conversation history.
- request_branch = list(run.messages)
+ request_branch = list(restored)
remaining = policy.max_calls
unlimited = remaining >= 9999
session_id = run.session_id
diff --git a/studio/backend/core/inference/tool_loop_controller.py b/studio/backend/core/inference/tool_loop_controller.py
index d0413a46050..3a9eae4496f 100644
--- a/studio/backend/core/inference/tool_loop_controller.py
+++ b/studio/backend/core/inference/tool_loop_controller.py
@@ -861,6 +861,46 @@ def is_tool_error(result: str) -> bool:
return isinstance(result, str) and result.lstrip().startswith(TOOL_ERROR_PREFIXES)
+TOOL_MODEL_SUFFIX_KEY = "studio_model_suffix"
+
+
+def restore_replayed_tool_results(messages: Sequence[Any]) -> list[Any]:
+ """Replayed tool results as the model saw them (error nudge, folded feedback from
+ ``extra_content``, no citation map), so the cached prefix holds. Idempotent."""
+ out = list(messages)
+ for i, message in enumerate(out):
+ if not isinstance(message, dict) or message.get("role") != "tool":
+ continue
+ content = message.get("content")
+ if not isinstance(content, str):
+ continue
+ # The source map is the card's; the live loop never showed it.
+ if message.get("name") in _SOURCE_MAP_TOOLS and "\n__RAG_SOURCES__:" in content:
+ content = _strip_rag_sources_sentinel(content)
+ extra = message.get("extra_content")
+ suffix = extra.get(TOOL_MODEL_SUFFIX_KEY) if isinstance(extra, dict) else None
+ suffix = suffix if isinstance(suffix, str) else ""
+ # `in`: restored output may carry feedback after the nudge.
+ nudge = (
+ TOOL_ERROR_NUDGE if is_tool_error(content) and TOOL_ERROR_NUDGE not in content else ""
+ )
+ restored = {**message, "content": content}
+ if nudge or suffix:
+ # Before any envelope, as live: text after the image array breaks its parse.
+ from core.inference.tools import _split_frontend_suffix
+ body, envelope = _split_frontend_suffix(content, message.get("name"))
+ restored["content"] = body + nudge + suffix + envelope
+ if isinstance(extra, dict) and TOOL_MODEL_SUFFIX_KEY in extra:
+ rest = {k: v for k, v in extra.items() if k != TOOL_MODEL_SUFFIX_KEY}
+ if rest:
+ restored["extra_content"] = rest
+ else:
+ restored.pop("extra_content", None)
+ if restored != message:
+ out[i] = restored
+ return out
+
+
def _strip_mcp_ui_suffix(result: str) -> str:
"""The payload is one JSON line, so the scan stops there; images may follow."""
marker = "\n__MCP_UI__:"
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 469792ccfda..5f9ee46e16c 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -2571,6 +2571,16 @@ def _openai_llama_admission_reserve(
return reservation, config
+def _replay_preserve_thinking(llama_backend, payload, messages):
+ """The preserve_thinking generation will render with, so counts price the same prompt."""
+ resolver = getattr(llama_backend, "_replay_preserve_thinking", None)
+ if resolver is None:
+ return payload.preserve_thinking
+ return resolver(
+ payload.preserve_thinking, messages, payload.enable_thinking, payload.reasoning_effort
+ )
+
+
def _count_gguf_admission_prompt(
llama_backend,
payload,
@@ -2593,7 +2603,9 @@ def _count_gguf_admission_prompt(
strict = True,
prefer_native = images == 0,
chat_template_kwargs = llama_backend._request_reasoning_kwargs(
- payload.enable_thinking, payload.reasoning_effort, payload.preserve_thinking
+ payload.enable_thinking,
+ payload.reasoning_effort,
+ _replay_preserve_thinking(llama_backend, payload, messages),
),
continue_final_message = _continue_final_message(payload, thought = True)
and trailing_assistant_resume_kind(messages) is not None,
@@ -32092,7 +32104,12 @@ async def produce_openai_chat_completions(
yield _OPENAI_TOOL_HEARTBEAT_SSE
continue
- if event["type"] in ("tool_output", "tool_args", "skill_load"):
+ if event["type"] in (
+ "tool_output",
+ "tool_args",
+ "skill_load",
+ "tool_model_note",
+ ):
if (
event["type"] == "skill_load"
and event.get("status") == "awaiting_approval"
@@ -41264,14 +41281,16 @@ async def chat_count_tokens(
_count_messages = payload.messages
if _folds_studio_tool_history(payload, llama_backend):
_count_messages = _folded_studio_tool_messages(_count_messages)
+ _count_dumped = [m.model_dump(exclude_none = True) for m in _count_messages]
+ if not _takes_passthrough:
+ from core.inference.tool_loop_controller import restore_replayed_tool_results
+
+ # As generation sends them; off the loop since restoring may parse an envelope.
+ _count_dumped = await asyncio.to_thread(restore_replayed_tool_results, _count_dumped)
# Awaited: stripping parses the whole envelope, and this recount runs in the
# background on every turn, so it must not do that on the shared loop.
openai_messages = await _promote_mcp_history_images_async(
- _strip_provider_synthetic_tool_history(
- _drop_empty_assistant_sentinels(
- [m.model_dump(exclude_none = True) for m in _count_messages]
- )
- ),
+ _strip_provider_synthetic_tool_history(_drop_empty_assistant_sentinels(_count_dumped)),
# Never as pixels: /apply-template prices a media marker, not the image.
# This drops the base64 the completion path would have promoted, which is
# the one part a count must not price as text.
@@ -41405,7 +41424,12 @@ async def chat_count_tokens(
_template_kwargs = llama_backend._request_reasoning_kwargs(
payload.enable_thinking,
payload.reasoning_effort,
- payload.preserve_thinking,
+ # Off the loop: first use probes the template.
+ payload.preserve_thinking
+ if _takes_passthrough
+ else await asyncio.to_thread(
+ _replay_preserve_thinking, llama_backend, payload, openai_messages
+ ),
)
# Whose tokenizer this is, in the shape /api/inference/status publishes: another tab's load
@@ -41641,6 +41665,17 @@ async def anthropic_count_tokens(
# rendered prompt, so counting the load-time default would under- or
# over-count the request the client is about to send.
_reasoning_args = _anthropic_reasoning_args(payload)
+ _count_preserve = _reasoning_args["preserve_thinking"]
+ _resolve_preserve = getattr(llama_backend, "_replay_preserve_thinking", None)
+ if _resolve_preserve is not None and not _count_client_tools:
+ # Off the loop: first use probes the template.
+ _count_preserve = await asyncio.to_thread(
+ _resolve_preserve,
+ _count_preserve,
+ openai_messages,
+ _reasoning_args["enable_thinking"],
+ _reasoning_args["reasoning_effort"],
+ )
try:
count = await asyncio.to_thread(
lambda: llama_backend.count_chat_tokens(
@@ -41652,7 +41687,7 @@ async def anthropic_count_tokens(
llama_backend,
_reasoning_args["enable_thinking"],
_reasoning_args["reasoning_effort"],
- _reasoning_args["preserve_thinking"],
+ _count_preserve,
),
)
)
@@ -44604,12 +44639,16 @@ def _openai_messages_for_gguf_chat(
per-turn ``image_url`` parts so multi-image chat history keeps each image
attached to its original turn.
"""
+ from core.inference.tool_loop_controller import restore_replayed_tool_results
+
# Coalesce only on the GGUF chat path (strict Jinja template); the tool path
# reuses this via _set_or_prepend_system_message. Passthrough forwards verbatim.
messages = _coalesce_consecutive_user_turns(
_strip_provider_synthetic_tool_history(
_drop_empty_assistant_sentinels(
- [m.model_dump(exclude_none = True) for m in payload.messages]
+ restore_replayed_tool_results(
+ [m.model_dump(exclude_none = True) for m in payload.messages]
+ )
)
)
)
diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.py
index 3819698f230..0b5891b490d 100644
--- a/studio/backend/tests/test_anthropic_messages.py
+++ b/studio/backend/tests/test_anthropic_messages.py
@@ -3411,6 +3411,63 @@ class TestAnthropicMessagesToolRouting:
== 200
)
+ def test_count_tokens_prices_the_preserve_thinking_generation_resolves(self, monkeypatch):
+ from routes.inference import anthropic_count_tokens
+
+ import routes.inference as inf_mod
+
+ seen: list = []
+
+ class _Backend:
+ is_loaded = True
+ is_vision = False
+ model_identifier = "qwen"
+ context_length = 4096
+ supports_tools = False
+ supports_tool_passthrough = False
+
+ def _request_reasoning_kwargs(
+ self, enable_thinking, reasoning_effort, preserve_thinking
+ ):
+ return {"enable_thinking": enable_thinking, "preserve_thinking": preserve_thinking}
+
+ def _replay_preserve_thinking(
+ self,
+ preserve,
+ messages,
+ enable_thinking = None,
+ effort = None,
+ ):
+ return True if preserve is False and enable_thinking is False else preserve
+
+ def count_chat_tokens(
+ self,
+ *args,
+ chat_template_kwargs = None,
+ **kwargs,
+ ):
+ seen.append(chat_template_kwargs)
+ return 2
+
+ monkeypatch.setattr(inf_mod, "current_date_prompt_line", lambda **_kwargs: "")
+ monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: _Backend())
+ payload = _basic_payload(
+ messages = [
+ {"role": "user", "content": "hi"},
+ {"role": "assistant", "content": "hello"},
+ {"role": "user", "content": "again"},
+ ],
+ thinking = {"type": "disabled"},
+ preserve_thinking = False,
+ )
+
+ response = _drive(
+ anthropic_count_tokens(payload, request = self._Request(), current_subject = "t")
+ )
+
+ assert response.status_code == 200
+ assert seen == [{"enable_thinking": False, "preserve_thinking": True}]
+
def test_count_tokens_rejects_the_catalog_messages_rejects(self, monkeypatch):
# The template renders no schemas, so the count matched with and without them
# (27 either way on a real gemma-3-270m) and handed an SDK a budget /messages 400s.
diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.py
index 0d6c9ae9ac8..f712194c8f9 100644
--- a/studio/backend/tests/test_llama_cpp_tool_loop.py
+++ b/studio/backend/tests/test_llama_cpp_tool_loop.py
@@ -1638,7 +1638,10 @@ def test_render_html_success_drops_tool_schema_before_final_pass(monkeypatch):
)
assert len(payloads) == 2
- assert "tools" not in payloads[1]
+ # The final pass may not call tools, but re-renders the same catalogue so the cached
+ # prompt prefix still matches.
+ assert payloads[1]["tool_choice"] == "none"
+ assert payloads[1]["tools"] == payloads[0]["tools"]
assert any(event.get("type") == "content" and event.get("text") == "Done." for event in events)
final_user_messages = [
m.get("content", "") for m in payloads[1]["messages"] if m.get("role") == "user"
@@ -1837,7 +1840,8 @@ def test_repeated_duplicate_noop_transitions_to_final_pass(monkeypatch):
"call_search_1"
]
assert len(payloads) == 4
- assert "tools" not in payloads[-1]
+ assert payloads[-1]["tool_choice"] == "none"
+ assert payloads[-1]["tools"] == payloads[-2]["tools"]
assert any(
event.get("type") == "content" and event.get("text") == "Final answer from first search."
for event in events
@@ -2020,7 +2024,9 @@ def test_same_turn_repeated_render_html_does_not_emit_second_provisional_start(m
and event.get("type") in {"tool_start", "tool_end"}
]
assert len(payloads) == 2
- assert "tools" not in payloads[1]
+ # No more calls, but the same catalogue keeps the cached prompt prefix.
+ assert payloads[1]["tool_choice"] == "none"
+ assert payloads[1]["tools"] == payloads[0]["tools"]
render_feedback = [
message
for message in payloads[1]["messages"]
@@ -5461,7 +5467,7 @@ def test_gguf_over_cap_does_not_ask_for_a_retry_when_the_range_check_ends_the_lo
# Assert the scenario really happened rather than letting the check pass vacuously: the
# loop ended without offering tools again, and it did produce exactly one notice.
- assert not payloads[-1].get("tools")
+ assert payloads[-1]["tool_choice"] == "none"
messages = payloads[-1]["messages"]
notices = [m for m in messages if "more tool call(s)" in (m.get("content") or "")]
assert len(notices) == 1
@@ -5600,9 +5606,9 @@ def test_gguf_valid_tool_calls_respect_max_tool_iterations(monkeypatch):
# Exactly two executed tool rounds, then one final-answer pass.
assert len(calls) == 2, calls
assert len(payloads) == 3, len(payloads)
- # The final pass carries no tool schemas. Its controller feedback stays in
+ # The final pass may not call tools (it keeps the schemas for the cache). Its controller feedback stays in
# the latest tool result rather than opening a newer user turn.
- assert _tool_names(payloads[2]) == [], _tool_names(payloads[2])
+ assert payloads[2]["tool_choice"] == "none", payloads[2].get("tool_choice")
assert any(
m.get("role") == "tool" and "used all available tool calls" in m.get("content", "")
for m in payloads[2]["messages"]
@@ -7330,3 +7336,174 @@ def test_nested_object_schema_is_relaxed_on_the_wire_but_not_in_the_loop(monkeyp
assert calls == [
("mcp__notion__query", {"data": {"view_url": "u", "page_size": 1, "start_cursor": "c"}})
]
+
+
+def test_budget_exhausted_final_pass_keeps_the_cached_tool_catalogue(monkeypatch):
+ # The template renders tools at the top of the system prompt. Dropping them on the final
+ # pass changed the prompt from its first tokens, so llama-server re-processed the whole
+ # conversation on that pass and again on the next turn.
+ first = [_tool_call_sse("web_search", {"query": "flappy bird"}, "call_1"), _done()]
+ final_stream = [_sse({"content": "Summary."}), _done()]
+ backend, payloads = _backend_and_payloads(monkeypatch, [first, final_stream])
+ _record_tool_calls(monkeypatch, "result")
+
+ events = _run_tool_loop(
+ backend,
+ [{"role": "user", "content": "search"}],
+ [_web_search_tool()],
+ max_tool_iterations = 1,
+ )
+
+ assert len(payloads) == 2
+ assert payloads[1]["tools"] == payloads[0]["tools"]
+ assert payloads[1]["tool_choice"] == "none"
+ assert any(e.get("type") == "content" and e.get("text") == "Summary." for e in events)
+
+
+def test_replayed_failed_tool_result_carries_the_nudge_the_model_saw(monkeypatch):
+ from core.inference.tool_call_parser import TOOL_ERROR_NUDGE
+
+ failed = "Exit code 1\nerror: aborting due to previous error"
+ history = [
+ {"role": "user", "content": "build it"},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_old",
+ "type": "function",
+ "function": {"name": "web_search", "arguments": {"query": "x"}},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "call_old", "name": "web_search", "content": failed},
+ {"role": "assistant", "content": "It failed."},
+ {"role": "user", "content": "show me"},
+ ]
+ backend, payloads = _backend_and_payloads(monkeypatch, [[_sse({"content": "Here."}), _done()]])
+
+ _run_tool_loop(backend, copy.deepcopy(history), [_web_search_tool()])
+
+ tool_messages = [m for m in payloads[0]["messages"] if m.get("role") == "tool"]
+ assert [m["content"] for m in tool_messages] == [failed + TOOL_ERROR_NUDGE]
+
+
+def test_parallel_batch_tool_events_share_one_round_id(monkeypatch):
+ # The frontend groups a round's cards into one assistant message by provenance round_id;
+ # without it a batch replayed as separate assistant/tool pairs and broke the cached prefix.
+ batch = [
+ _tool_call_sse("web_search", {"query": "a"}, "call_a", index = 0),
+ _tool_call_sse("web_search", {"query": "b"}, "call_b", index = 1),
+ _done(),
+ ]
+ second = [_tool_call_sse("web_search", {"query": "c"}, "call_c"), _done()]
+ final_stream = [_sse({"content": "Done."}), _done()]
+ backend, payloads = _backend_and_payloads(monkeypatch, [batch, second, final_stream])
+ _record_tool_calls(monkeypatch, "result")
+
+ events = _run_tool_loop(
+ backend,
+ [{"role": "user", "content": "search"}],
+ [_web_search_tool()],
+ max_tool_iterations = 3,
+ )
+
+ rounds = {
+ e["tool_call_id"]: e["provenance"]["round_id"]
+ for e in events
+ if e.get("type") == "tool_end"
+ }
+ assert set(rounds) == {"call_a", "call_b", "call_c"}
+ assert rounds["call_a"] == rounds["call_b"] != rounds["call_c"]
+ batch_assistant = [m for m in payloads[1]["messages"] if m.get("tool_calls")]
+ assert [len(m["tool_calls"]) for m in batch_assistant] == [2]
+
+
+def test_budget_notice_folded_into_a_result_is_reported_for_replay(monkeypatch):
+ from core.inference.tool_call_parser import BUDGET_EXHAUSTED_NUDGE
+ from core.inference.tool_loop_controller import restore_replayed_tool_results
+
+ first = [_tool_call_sse("web_search", {"query": "flappy bird"}, "call_1"), _done()]
+ final_stream = [_sse({"content": "Summary."}), _done()]
+ backend, payloads = _backend_and_payloads(monkeypatch, [first, final_stream])
+ _record_tool_calls(monkeypatch, "Exit code 1\nboom")
+
+ events = _run_tool_loop(
+ backend,
+ [{"role": "user", "content": "search"}],
+ [_web_search_tool()],
+ max_tool_iterations = 1,
+ )
+
+ notes = [e for e in events if e.get("type") == "tool_model_note"]
+ tool_end = next(e for e in events if e.get("type") == "tool_end")
+ assert notes == [
+ {
+ "type": "tool_model_note",
+ "tool_call_id": tool_end["tool_call_id"],
+ "suffix": f"\n\n{BUDGET_EXHAUSTED_NUDGE}",
+ }
+ ]
+ # What the client sends back next turn renders exactly what the final pass sent.
+ replayed = {
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "name": "web_search",
+ "content": tool_end["result"],
+ "extra_content": {"studio_model_suffix": notes[0]["suffix"]},
+ }
+ sent = [m for m in payloads[1]["messages"] if m.get("role") == "tool"][-1]
+ assert restore_replayed_tool_results([replayed])[0]["content"] == sent["content"]
+
+
+def test_restored_replay_is_the_branch_the_tools_are_given(monkeypatch):
+ # Restored results are new dicts and the live branch is accumulated by identity, so seeding it
+ # from the unrestored messages handed a tool the same replayed result twice.
+ history = [
+ {"role": "user", "content": "build it"},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_old",
+ "type": "function",
+ "function": {"name": "web_search", "arguments": {"query": "x"}},
+ }
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_old",
+ "name": "web_search",
+ "content": "Error: offline",
+ },
+ {"role": "assistant", "content": "It failed."},
+ {"role": "user", "content": "try again"},
+ ]
+ backend, _payloads = _backend_and_payloads(
+ monkeypatch,
+ [
+ [_tool_call_sse("web_search", {"query": "y"}, "call_new"), _done()],
+ [_sse({"content": "Done."}), _done()],
+ ],
+ )
+ branches: list[list[dict]] = []
+
+ def execute_tool(
+ name,
+ arguments,
+ conversation_branch = None,
+ **_kwargs,
+ ):
+ branches.append(list(conversation_branch or ()))
+ return "sunny"
+
+ monkeypatch.setattr("core.inference.tools.execute_tool", execute_tool)
+
+ _run_tool_loop(backend, copy.deepcopy(history), [_web_search_tool()])
+
+ assert branches
+ replayed = [m for m in branches[0] if m.get("tool_call_id") == "call_old"]
+ assert len(replayed) == 1
diff --git a/studio/backend/tests/test_preserve_thinking_replay.py b/studio/backend/tests/test_preserve_thinking_replay.py
new file mode 100644
index 00000000000..d358cb6300e
--- /dev/null
+++ b/studio/backend/tests/test_preserve_thinking_replay.py
@@ -0,0 +1,131 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A thinking-off chat replays each earlier answer with the empty think block it was generated
+behind, so the next turn's prompt keeps llama-server's cached prefix."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from core.inference import llama_cpp
+from core.inference.llama_cpp import LlamaCppBackend
+
+QWEN_GEN = "<|im_start|>user\nStudio replay probe question.<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
+
+
+def _backend(supports = True) -> LlamaCppBackend:
+ backend = LlamaCppBackend.__new__(LlamaCppBackend)
+ backend._port = 48847
+ backend._api_key = None
+ backend._supports_reasoning = True
+ backend._reasoning_always_on = False
+ backend._reasoning_style = "enable_thinking"
+ backend._supports_preserve_thinking = supports
+ backend._chat_template = "tpl"
+ return backend
+
+
+def _history(reasoning = None) -> list[dict]:
+ answer = {"role": "assistant", "content": "Pip paints."}
+ if reasoning is not None:
+ answer["reasoning_content"] = reasoning
+ return [{"role": "user", "content": "story"}, answer, {"role": "user", "content": "more"}]
+
+
+def test_no_reasoning_history_keeps_the_empty_think_block(monkeypatch):
+ backend = _backend()
+ monkeypatch.setattr(LlamaCppBackend, "_empty_think_replays_with_preserve", lambda self: True)
+ assert backend._replay_preserve_thinking(False, _history(), False) is True
+ assert backend._replay_preserve_thinking(False, _history(""), False) is True
+
+
+def test_the_users_choice_stands_wherever_there_is_reasoning_or_no_proof(monkeypatch):
+ backend = _backend()
+ monkeypatch.setattr(LlamaCppBackend, "_empty_think_replays_with_preserve", lambda self: True)
+ assert backend._replay_preserve_thinking(False, _history("thought about it"), False) is False
+ assert backend._replay_preserve_thinking(None, _history(), False) is None
+ assert backend._replay_preserve_thinking(True, _history(), False) is True
+ assert _backend(supports = False)._replay_preserve_thinking(False, _history(), False) is False
+ # Thinking on: its reasoning is real, so the caller's choice to drop it stands.
+ assert backend._replay_preserve_thinking(False, _history(), True) is False
+ assert backend._replay_preserve_thinking(False, _history(), None) is False
+ # Reasoning carried inline in the answer is still reasoning.
+ inline = _history()
+ inline[1]["content"] = "<think>\nplan\n</think>\n\nPip paints."
+ assert backend._replay_preserve_thinking(False, inline, False) is False
+ inline[1]["content"] = [{"type": "text", "text": "<think>plan</think>Pip paints."}]
+ assert backend._replay_preserve_thinking(False, inline, False) is False
+ # Nothing answered yet: nothing to replay, so no probe either.
+ assert backend._replay_preserve_thinking(False, _history()[:1], False) is False
+ monkeypatch.setattr(LlamaCppBackend, "_empty_think_replays_with_preserve", lambda self: False)
+ assert backend._replay_preserve_thinking(False, _history(), False) is False
+
+
+class _FakeClient:
+ def __init__(self, render, calls):
+ self._render = render
+ self._calls = calls
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ return False
+
+ def post(self, url, json):
+ self._calls.append(json)
+ prompt = self._render(json["messages"], json["chat_template_kwargs"]["preserve_thinking"])
+ return type("R", (), {"status_code": 200, "json": lambda self: {"prompt": prompt}})()
+
+
+def _qwen_render(messages, preserve):
+ if len(messages) == 1:
+ return QWEN_GEN
+ think = "<think>\n\n</think>\n\n" if preserve else ""
+ return (
+ "<|im_start|>user\nStudio replay probe question.<|im_end|>\n<|im_start|>assistant\n"
+ + think
+ + "Studio replay probe answer.<|im_end|>\n<|im_start|>user\nStudio replay probe follow-up.<|im_end|>\n"
+ )
+
+
+def _plain_render(messages, preserve):
+ # A template that renders the same either way: nothing to gain, so nothing changes.
+ if len(messages) == 1:
+ return "<|im_start|>user\nStudio replay probe question.<|im_end|>\n<|im_start|>assistant\n"
+ return (
+ "<|im_start|>user\nStudio replay probe question.<|im_end|>\n<|im_start|>assistant\n"
+ "Studio replay probe answer.<|im_end|>\n<|im_start|>user\nStudio replay probe follow-up.<|im_end|>\n"
+ )
+
+
+def test_probe_reads_the_templates_own_render_once(monkeypatch):
+ calls: list = []
+ monkeypatch.setattr(llama_cpp.httpx, "Client", lambda **kw: _FakeClient(_qwen_render, calls))
+ backend = _backend()
+ assert backend._empty_think_replays_with_preserve() is True
+ assert backend._empty_think_replays_with_preserve() is True
+ assert len(calls) == 3
+ assert all(c["chat_template_kwargs"]["enable_thinking"] is False for c in calls)
+
+
+def test_probe_refuses_a_template_that_renders_alike(monkeypatch):
+ calls: list = []
+ monkeypatch.setattr(llama_cpp.httpx, "Client", lambda **kw: _FakeClient(_plain_render, calls))
+ assert _backend()._empty_think_replays_with_preserve() is False
+
+
+def test_probe_failure_keeps_the_setting_and_retries(monkeypatch):
+ def boom(**kw):
+ raise OSError("server busy")
+
+ monkeypatch.setattr(llama_cpp.httpx, "Client", boom)
+ backend = _backend()
+ assert backend._empty_think_replays_with_preserve() is False
+ assert getattr(backend, "_empty_think_preserve_probe", None) is None
diff --git a/studio/backend/tests/test_studio_tool_loop.py b/studio/backend/tests/test_studio_tool_loop.py
index 1b3d17c4a9f..591c58a7cf7 100644
--- a/studio/backend/tests/test_studio_tool_loop.py
+++ b/studio/backend/tests/test_studio_tool_loop.py
@@ -336,6 +336,56 @@ def test_a_resumed_reply_does_not_merge_over_the_compaction_it_follows(executed)
assert follow_up[0]["extra_content"]["anthropic"]["content"] == [summary]
+def test_the_searched_branch_carries_replayed_results_as_the_model_saw_them(executed):
+ from core.inference.tool_loop_controller import TOOL_ERROR_NUDGE
+
+ branch = [
+ {"role": "user", "content": "run it"},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "call_a",
+ "type": "function",
+ "function": {"name": "python", "arguments": "{}"},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "call_a", "name": "python", "content": "Error: boom"},
+ {"role": "user", "content": "what failed earlier"},
+ ]
+ transport = FakeTransport(
+ [
+ [
+ _sse(
+ {
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_c",
+ "function": {
+ "name": "search_conversation",
+ "arguments": '{"query":"failed"}',
+ },
+ }
+ ]
+ }
+ ),
+ _sse(finish = "tool_calls"),
+ _DONE,
+ ],
+ [_sse({"content": "boom"}), _sse(finish = "stop"), _DONE],
+ ]
+ )
+
+ _run(transport, tools = [_tool("search_conversation")], messages = branch)
+
+ searched = executed[0]["conversation_branch"]
+ assert searched[2]["content"] == "Error: boom" + TOOL_ERROR_NUDGE
+ assert searched[2]["content"] == transport.requests[0]["messages"][2]["content"]
+
+
def test_a_conversation_search_here_gets_the_active_branch(executed):
"""The provider loops share the local paths' tool catalogue.
diff --git a/studio/backend/tests/test_tool_error_nudge_replay.py b/studio/backend/tests/test_tool_error_nudge_replay.py
new file mode 100644
index 00000000000..e68cce5c20e
--- /dev/null
+++ b/studio/backend/tests/test_tool_error_nudge_replay.py
@@ -0,0 +1,144 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A replayed failed tool result must render exactly as the model saw it, or the next turn's prompt
+stops matching llama-server's cached prefix at that result."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from core.inference.tool_call_parser import TOOL_ERROR_NUDGE
+from core.inference.tool_loop_controller import ToolLoopController, restore_replayed_tool_results
+
+
+def _tool(name: str) -> dict:
+ return {"type": "function", "function": {"name": name, "parameters": {"type": "object"}}}
+
+
+def _completion(result: str):
+ controller = ToolLoopController(tools = [_tool("terminal")])
+ call = {
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "terminal", "arguments": '{"command": "rustc main.rs"}'},
+ }
+ return controller.record_result(controller.prepare_call(call), result)
+
+
+def _replayed(completion) -> dict:
+ # What the frontend persists from tool_end and sends back as role="tool" next turn.
+ return {
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "name": "terminal",
+ "content": completion.tool_end_payload()["result"],
+ }
+
+
+def test_replayed_failed_result_matches_what_the_model_saw():
+ completion = _completion(
+ "Exit code 1\nerror[E0277]: the type `[u8]` cannot be indexed by `i32`"
+ )
+ assert completion.is_error
+ replayed = _replayed(completion)
+ assert TOOL_ERROR_NUDGE not in replayed["content"]
+
+ restored = restore_replayed_tool_results([{"role": "user", "content": "go"}, replayed])
+
+ assert restored[1]["content"] == completion.model_message()["content"]
+ assert restored[1]["tool_call_id"] == "call_1"
+
+
+def test_restore_is_idempotent_and_does_not_mutate_input():
+ replayed = _replayed(_completion("Error: file not found"))
+ original = dict(replayed)
+
+ once = restore_replayed_tool_results([replayed])
+ twice = restore_replayed_tool_results(once)
+
+ assert once == twice
+ assert once[0]["content"].count(TOOL_ERROR_NUDGE) == 1
+ assert replayed == original
+
+
+def test_successful_and_non_tool_messages_are_untouched():
+ messages = [
+ {"role": "user", "content": "Error: this is the user typing"},
+ {"role": "assistant", "content": "Error: quoted by the model"},
+ {"role": "tool", "content": "compiled fine"},
+ {"role": "tool", "content": [{"type": "text", "text": "Error: multimodal"}]},
+ ]
+ assert restore_replayed_tool_results(messages) == messages
+
+
+def test_folded_suffix_follows_the_error_nudge_and_leaves_extra_content():
+ suffix = "\n\nYou have used all available tool calls."
+ replayed = {
+ "role": "tool",
+ "content": "Exit code 1\nboom",
+ "extra_content": {"studio_model_suffix": suffix},
+ }
+
+ once = restore_replayed_tool_results([replayed])
+ twice = restore_replayed_tool_results(once)
+
+ assert once[0]["content"] == "Exit code 1\nboom" + TOOL_ERROR_NUDGE + suffix
+ assert "extra_content" not in once[0]
+ assert twice == once
+ # Other extra_content keys survive.
+ kept = restore_replayed_tool_results(
+ [{**replayed, "extra_content": {"studio_model_suffix": suffix, "x": 1}}]
+ )
+ assert kept[0]["extra_content"] == {"x": 1}
+
+
+def test_restored_feedback_lands_before_a_frontend_envelope():
+ envelope = '\n__IMAGES__:["plot.png"]'
+ completion = _completion("Exit code 1\nTraceback: boom" + envelope)
+ replayed = _replayed(completion)
+ assert replayed["content"].endswith(envelope)
+ replayed["extra_content"] = {"studio_model_suffix": "\n\nbudget"}
+
+ restored = restore_replayed_tool_results([replayed])[0]
+
+ # The envelope stays a suffix, so promotion still parses it and the text is what the model saw.
+ assert restored["content"].endswith(envelope)
+ assert (
+ restored["content"][: -len(envelope)]
+ == completion.model_message()["content"] + "\n\nbudget"
+ )
+ assert restore_replayed_tool_results([restored]) == [restored]
+
+
+def test_provider_cannot_forge_a_model_note():
+ from core.inference.sse_control_frames import sanitize_provider_sse_line
+ line = 'data: {"type": "tool_model_note", "tool_call_id": "call_0", "suffix": "x"}'
+ assert sanitize_provider_sse_line(line) is None
+
+
+def test_replayed_retrieval_result_drops_the_source_map_the_model_never_saw():
+ import json
+
+ from core.inference.tool_loop_controller import strip_result_for_model
+
+ body = '<chunk id="1" source="earlier turn (user)" turn="4">\nuser: What is 17*23?\n</chunk>'
+ raw = (
+ body
+ + "\n__RAG_SOURCES__:"
+ + json.dumps([{"citationId": 1, "text": "user: What is 17*23?"}])
+ )
+ replayed = {"role": "tool", "tool_call_id": "c", "name": "search_conversation", "content": raw}
+
+ restored = restore_replayed_tool_results([replayed])[0]
+
+ assert restored["content"] == strip_result_for_model(raw, "search_conversation") == body
+ assert restore_replayed_tool_results([restored]) == [restored]
+ # Any other tool keeps text that merely carries the marker.
+ other = {**replayed, "name": "terminal"}
+ assert restore_replayed_tool_results([other])[0]["content"] == raw
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index 3a0719ef8c4..5cd360bdd7c 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -182,6 +182,11 @@ import {
startsNewCodexToolRound,
type CodexReasoningLedger,
} from "../codex-reasoning";
+import {
+ toolModelNoteTarget,
+ toolModelSuffixExtraContent,
+ withToolModelSuffix,
+} from "../tool-model-suffix";
import {
type GeminiContinuationReplay,
type GeminiContinuationReplayTurn,
@@ -1000,6 +1005,7 @@ type SerializedToolResult = {
content: string;
tool_call_id: string;
name?: string;
+ extra_content?: { studio_model_suffix: string };
};
type ToolPartReplayMetadata = {
@@ -1232,6 +1238,8 @@ function serializeToolResultPart(
content,
tool_call_id: tc.toolCallId,
...(tc.toolName ? { name: tc.toolName } : {}),
+ // Model-only feedback the backend re-appends on replay.
+ ...toolModelSuffixExtraContent(getToolReplayProvenance(tc)),
};
}
@@ -7569,6 +7577,37 @@ export function createOpenAIStreamAdapter(
);
continue;
}
+ if (toolEvent.type === "tool_model_note") {
+ const at = toolModelNoteTarget(
+ toolCallParts as Parameters<typeof toolModelNoteTarget>[0],
+ (toolEvent.tool_call_id as string) || "",
+ );
+ const suffix =
+ typeof toolEvent.suffix === "string" ? toolEvent.suffix : "";
+ if (at !== -1 && suffix) {
+ const existing = toolCallParts[at] as PositionedToolCallPart;
+ toolCallParts[at] = {
+ ...existing,
+ provenance: withToolModelSuffix(
+ existing.provenance as Record<string, unknown> | undefined,
+ suffix,
+ ),
+ } as PositionedToolCallPart;
+ // A stop before the next chunk saves only what was yielded.
+ yield {
+ content: liveAssistantContent(),
+ metadata: {
+ timing: buildTiming(
+ streamStartTime,
+ totalChunks,
+ firstTokenTime,
+ ),
+ custom: liveCustom(),
+ },
+ };
+ }
+ continue;
+ }
if (toolEvent.type === "tool_output") {
// Incremental stdout from a running tool: append to the live store so the card renders it.
// The final result arrives via tool_end.
diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts
index dc9444ce8fa..1974e649e1d 100644
--- a/studio/frontend/src/features/chat/api/chat-api.ts
+++ b/studio/frontend/src/features/chat/api/chat-api.ts
@@ -2049,7 +2049,8 @@ export async function* streamChatCompletions(
(parsed.type === "tool_start" ||
parsed.type === "tool_end" ||
parsed.type === "tool_output" ||
- parsed.type === "tool_args")
+ parsed.type === "tool_args" ||
+ parsed.type === "tool_model_note")
) {
yield { _toolEvent: parsed } as unknown as OpenAIChatChunk;
separatorIndex = buffer.search(/\r?\n\r?\n/);
diff --git a/studio/frontend/src/features/chat/api/chat-generation-api.ts b/studio/frontend/src/features/chat/api/chat-generation-api.ts
index 21404969ab0..b3485b8e490 100644
--- a/studio/frontend/src/features/chat/api/chat-generation-api.ts
+++ b/studio/frontend/src/features/chat/api/chat-generation-api.ts
@@ -86,7 +86,8 @@ export function normalizeChatGenerationChunkPayload(
frameType === "tool_start" ||
frameType === "tool_end" ||
frameType === "tool_output" ||
- frameType === "tool_args"
+ frameType === "tool_args" ||
+ frameType === "tool_model_note"
) {
// tool_start/end carry full input/output; tool_output streams incremental stdout and
// tool_args streams the call arguments live. The consumer accumulates by tag, so keep the frame whole.
diff --git a/studio/frontend/src/features/chat/tool-model-suffix.ts b/studio/frontend/src/features/chat/tool-model-suffix.ts
new file mode 100644
index 00000000000..a0d685ca50e
--- /dev/null
+++ b/studio/frontend/src/features/chat/tool-model-suffix.ts
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+/** Model-only feedback (the tool budget notice) on a finished card, replayed with its result. */
+
+type NotedPart = {
+ toolCallId: string;
+ backendToolCallId?: string;
+ provenance?: Record<string, unknown>;
+};
+
+/** The LAST match: generated ids restart at call_0 every round. */
+export function toolModelNoteTarget(
+ parts: readonly NotedPart[],
+ backendToolCallId: string,
+): number {
+ if (!backendToolCallId) return -1;
+ for (let i = parts.length - 1; i >= 0; i -= 1) {
+ const part = parts[i];
+ if (
+ part.backendToolCallId === backendToolCallId ||
+ part.toolCallId === backendToolCallId
+ ) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+/** A note delivered twice (a reconnect replay) is kept once. */
+export function withToolModelSuffix(
+ provenance: Record<string, unknown> | undefined,
+ suffix: string,
+): Record<string, unknown> {
+ const previous =
+ typeof provenance?.model_suffix === "string" ? provenance.model_suffix : "";
+ if (suffix && previous.endsWith(suffix)) return { ...(provenance ?? {}) };
+ return {
+ ...(provenance ?? {}),
+ model_suffix: previous + suffix,
+ };
+}
+
+export function toolModelSuffixExtraContent(
+ provenance: Record<string, unknown> | null | undefined,
+): { extra_content: { studio_model_suffix: string } } | Record<string, never> {
+ const suffix = provenance?.model_suffix;
+ return typeof suffix === "string" && suffix
+ ? { extra_content: { studio_model_suffix: suffix } }
+ : {};
+}
diff --git a/studio/frontend/src/features/chat/utils/generation-tool-recovery.ts b/studio/frontend/src/features/chat/utils/generation-tool-recovery.ts
index 13dd48a70fc..b95142d806a 100644
--- a/studio/frontend/src/features/chat/utils/generation-tool-recovery.ts
+++ b/studio/frontend/src/features/chat/utils/generation-tool-recovery.ts
@@ -30,6 +30,7 @@ import {
providerCompactionPart,
providerCompactionReplayToolCallCount,
} from "./provider-compaction";
+import { toolModelNoteTarget, withToolModelSuffix } from "../tool-model-suffix";
function record(value: unknown): Record<string, unknown> | undefined {
return value !== null && typeof value === "object" && !Array.isArray(value)
@@ -369,6 +370,7 @@ export function createGenerationToolRecovery(
if (
event?.type !== "tool_start" &&
event?.type !== "tool_end" &&
+ event?.type !== "tool_model_note" &&
event?.type !== "document_citations" &&
event?.type !== "compaction_block"
) {
@@ -413,6 +415,23 @@ export function createGenerationToolRecovery(
}
return;
}
+ if (event.type === "tool_model_note") {
+ const suffix = typeof event.suffix === "string" ? event.suffix : "";
+ const at = toolModelNoteTarget(
+ carried.map((carriedEntry) => record(carriedEntry.part) ?? {}) as unknown as Parameters<
+ typeof toolModelNoteTarget
+ >[0],
+ backendId,
+ );
+ const part = at === -1 ? undefined : record(carried[at].part);
+ if (part && suffix) {
+ carried[at].part = {
+ ...part,
+ provenance: withToolModelSuffix(record(part.provenance), suffix),
+ } as CarriedPart["part"];
+ }
+ return;
+ }
if (event.type === "document_citations") {
if (Array.isArray(event.citations)) {
event.citations.forEach((value, index) => {
diff --git a/studio/frontend/tests/generation-tool-recovery.test.ts b/studio/frontend/tests/generation-tool-recovery.test.ts
index fc7f594dc8d..92af118956f 100644
--- a/studio/frontend/tests/generation-tool-recovery.test.ts
+++ b/studio/frontend/tests/generation-tool-recovery.test.ts
@@ -1555,3 +1555,21 @@ test("each rebuild yields its own source objects", () => {
assert.equal(second.title, "Docs");
assert.deepEqual(second.metadata, { description: "only" });
});
+
+test("recovery keeps a model note on the finished card it targets", () => {
+ const carried: Carried[] = [];
+ const replay = createGenerationToolRecovery(carried, "run").apply;
+ replay(start(), 12, 1);
+ replay(end(), 20, 2);
+ const note = {
+ type: "tool_model_note",
+ tool_call_id: "call_0",
+ suffix: "\n\nbudget",
+ };
+ replay({ _toolEvent: note }, 20, 3);
+ replay(note, 20, 3);
+ const part = carried[0].part as Record<string, unknown>;
+ assert.equal(carried.length, 1);
+ assert.equal(part.result, "ok");
+ assert.deepEqual(part.provenance, { model_suffix: "\n\nbudget" });
+});
diff --git a/studio/frontend/tests/tool-model-suffix.test.ts b/studio/frontend/tests/tool-model-suffix.test.ts
new file mode 100644
index 00000000000..41848a00db8
--- /dev/null
+++ b/studio/frontend/tests/tool-model-suffix.test.ts
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ toolModelNoteTarget,
+ toolModelSuffixExtraContent,
+ withToolModelSuffix,
+} from "../src/features/chat/tool-model-suffix.ts";
+
+import { readSrc } from "./helpers/kit.ts";
+
+test("a model note targets the latest card for a reused backend id", () => {
+ const parts = [
+ { toolCallId: "call_0:a", backendToolCallId: "call_0" },
+ { toolCallId: "call_1:b", backendToolCallId: "call_1" },
+ { toolCallId: "call_0:c", backendToolCallId: "call_0" },
+ ];
+ assert.equal(toolModelNoteTarget(parts, "call_0"), 2);
+ assert.equal(toolModelNoteTarget(parts, "call_1:b"), 1);
+ assert.equal(toolModelNoteTarget(parts, "missing"), -1);
+ assert.equal(toolModelNoteTarget(parts, ""), -1);
+});
+
+test("suffixes accumulate in order and keep the rest of the provenance", () => {
+ const once = withToolModelSuffix(
+ { source: "local", round_id: 3 },
+ "\n\nfirst",
+ );
+ const twice = withToolModelSuffix(once, "\n\nsecond");
+ assert.deepEqual(twice, {
+ source: "local",
+ round_id: 3,
+ model_suffix: "\n\nfirst\n\nsecond",
+ });
+ // A reconnect that replays the same note keeps it once.
+ assert.deepEqual(withToolModelSuffix(twice, "\n\nsecond"), twice);
+});
+
+test("only a noted card replays extra_content", () => {
+ assert.deepEqual(toolModelSuffixExtraContent({ source: "local" }), {});
+ assert.deepEqual(toolModelSuffixExtraContent(null), {});
+ assert.deepEqual(
+ toolModelSuffixExtraContent({ model_suffix: "\n\nUse what you have." }),
+ {
+ extra_content: { studio_model_suffix: "\n\nUse what you have." },
+ },
+ );
+});
+
+test("the adapter publishes a model note as soon as it lands", () => {
+ const src = readSrc("features/chat/api/chat-adapter.ts");
+ const start = src.indexOf('toolEvent.type === "tool_model_note"');
+ assert.notEqual(start, -1);
+ const branch = src.slice(start, src.indexOf("continue;", start));
+ // A stop before the next chunk saves only the last yield, so the note must reach one.
+ assert.match(branch, /yield \{\s*content: liveAssistantContent\(\)/);
+});