Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep the prompt cache for API clients and coding agents (#13645)

Daniel Hana9dc8ea94cbe
diff --git a/studio/backend/core/inference/anthropic_compat.py b/studio/backend/core/inference/anthropic_compat.py
index a47e0a558bb..b4124e49575 100644
--- a/studio/backend/core/inference/anthropic_compat.py
+++ b/studio/backend/core/inference/anthropic_compat.py
@@ -181,7 +181,7 @@ def anthropic_messages_to_openai(
``preserve_thinking`` keeps replayed assistant ``thinking`` blocks as
``reasoning_content`` on the converted message, so templates that render
historical reasoning (Qwen3.6-style ``preserve_thinking``) actually receive
- it; otherwise thinking is dropped from the prompt. ``redacted_thinking``
+ it; otherwise only the current turn's thinking (after the last real query) is kept. ``redacted_thinking``
carries only ciphertext and is always dropped.
"""
result: list[dict] = []
@@ -199,9 +199,26 @@ def anthropic_messages_to_openai(
if parts:
result.append({"role": "system", "content": "\n".join(parts)})
- for msg in messages:
+ # The current turn's thinking stays even with preservation off: templates render reasoning after
+ # the last real query (a user message that is not only tool results), as it was generated.
+ last_query = -1
+ for index, msg in enumerate(messages):
+ role = msg["role"] if isinstance(msg, dict) else msg.role
+ content = msg["content"] if isinstance(msg, dict) else msg.content
+ if role == "user" and (
+ isinstance(content, str)
+ or any(
+ (block.get("type") if isinstance(block, dict) else getattr(block, "type", None))
+ != "tool_result"
+ for block in content
+ )
+ ):
+ last_query = index
+
+ for index, msg in enumerate(messages):
role = msg["role"] if isinstance(msg, dict) else msg.role
content = msg["content"] if isinstance(msg, dict) else msg.content
+ keep_thinking = preserve_thinking or index > last_query
if isinstance(content, str):
result.append({"role": role, "content": content})
@@ -235,7 +252,7 @@ def anthropic_messages_to_openai(
btype = b.get("type", "")
if btype == "text":
text_parts.append(b["text"])
- elif btype == "thinking" and preserve_thinking:
+ elif btype == "thinking" and keep_thinking:
t = b.get("thinking") or ""
if t:
thinking_parts.append(t)
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 29c25acc8e6..de45e984860 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -496,6 +496,7 @@ from core.inference.tool_loop_controller import (
deferred_nudge_text,
provisional_tool_provenance,
restore_replayed_tool_results,
+ with_appended_text,
tool_call_limit_nudge,
)
from state.tool_approvals import (
@@ -795,6 +796,9 @@ _SPEC_KIND_CAPABILITY: dict[str, str] = {
_LLAMA_RANDOM_SEED = 0xFFFFFFFF
+_EMPTY_THINK_PROBE_LOCK = threading.Lock()
+
+
def _apply_seeded_llama_request(
payload: dict,
seed: Optional[int],
@@ -9187,12 +9191,15 @@ class LlamaCppBackend:
) -> Optional[bool]:
"""True for a thinking-off follow-up with no reasoning in its history, where the template proves
that only preserve_thinking replays each answer's empty think block as generated."""
- if preserve_thinking is not False or not getattr(
- self, "_supports_preserve_thinking", False
- ):
+ # Omitted reads as off; the probe proves what replays.
+ if preserve_thinking is True or not getattr(self, "_supports_preserve_thinking", False):
return preserve_thinking
mode = self._request_reasoning_kwargs(enable_thinking, reasoning_effort, None) or {}
- if mode.get("enable_thinking") is not False:
+ thinking = mode.get("enable_thinking")
+ if thinking is None and enable_thinking is None and reasoning_effort is None:
+ # Agents rarely send either: llama-server then renders the load-time default.
+ thinking = getattr(self, "_reasoning_default", None)
+ if thinking is not False:
return preserve_thinking
answered = False
for message in messages or ():
@@ -9216,16 +9223,26 @@ class LlamaCppBackend:
return preserve_thinking
return True if self._empty_think_replays_with_preserve() else preserve_thinking
- def _empty_think_replays_with_preserve(self) -> bool:
- """Probed once per server: generation prompt + answer prefixes the next render only when kept."""
- key = (
+ def _empty_think_probe_key(self) -> tuple:
+ return (
getattr(self, "base_url", None),
id(getattr(self, "_chat_template", None)),
id(getattr(self, "_chat_template_override", None)),
)
+
+ def _empty_think_replays_with_preserve(self) -> bool:
+ """Probed once per server: generation prompt + answer prefixes the next render only when kept."""
+ key = self._empty_think_probe_key()
cached = getattr(self, "_empty_think_preserve_probe", None)
if cached is not None and cached[0] == key:
return cached[1]
+ with _EMPTY_THINK_PROBE_LOCK:
+ cached = getattr(self, "_empty_think_preserve_probe", None)
+ if cached is not None and cached[0] == key:
+ return cached[1]
+ return self._probe_empty_think_replay(key)
+
+ def _probe_empty_think_replay(self, key: tuple) -> bool:
result = False
try:
off = dict(self._request_reasoning_kwargs(False, None, None) or {})
@@ -9241,6 +9258,7 @@ class LlamaCppBackend:
trust_env = False,
verify = _local_ssl_context(),
) as client:
+ refused = []
def _render(msgs, preserve):
r = client.post(
@@ -9250,6 +9268,9 @@ class LlamaCppBackend:
"chat_template_kwargs": {**off, "preserve_thinking": preserve},
},
)
+ # A 4xx is the template's answer, cached; 408 / 429 mean busy.
+ if 400 <= r.status_code < 500 and r.status_code not in (408, 429):
+ refused.append(r.status_code)
prompt = r.json().get("prompt") if r.status_code == 200 else None
return prompt if isinstance(prompt, str) else None
@@ -9257,7 +9278,9 @@ class LlamaCppBackend:
kept = _render(history, True)
dropped = _render(history, False)
if generated is None or kept is None or dropped is None:
- return False # Unanswered: not cached, so the next request asks again.
+ if refused:
+ self._empty_think_preserve_probe = (key, False)
+ return False
result = kept.startswith(generated + answer) and not dropped.startswith(
generated + answer
)
@@ -38220,6 +38243,7 @@ class LlamaCppBackend:
enable_thinking: Optional[bool] = None,
reasoning_effort: Optional[str] = None,
preserve_thinking: Optional[bool] = None,
+ replay_preserve_thinking: bool = True,
continue_final_message: bool = False,
seed: Optional[int] = None,
promote_reasoning_only: bool = True,
@@ -38244,9 +38268,10 @@ class LlamaCppBackend:
"""
if not self.is_loaded:
raise RuntimeError("llama-server is not loaded")
- preserve_thinking = self._replay_preserve_thinking(
- preserve_thinking, messages, enable_thinking, reasoning_effort
- )
+ if replay_preserve_thinking:
+ preserve_thinking = self._replay_preserve_thinking(
+ preserve_thinking, messages, enable_thinking, reasoning_effort
+ )
from core.inference.chat_template_helpers import (
neutralize_control_markup_in_messages,
@@ -38666,6 +38691,8 @@ class LlamaCppBackend:
enable_thinking: Optional[bool] = None,
reasoning_effort: Optional[str] = None,
preserve_thinking: Optional[bool] = None,
+ # False when the caller knows the history held reasoning its conversion dropped.
+ replay_preserve_thinking: bool = True,
continue_final_message: bool = False,
max_tool_iterations: int = 25,
auto_heal_tool_calls: bool = True,
@@ -38715,9 +38742,10 @@ class LlamaCppBackend:
{"type": "content", "text": "token"} -- streamed content tokens (cumulative)
{"type": "reasoning", "text": "token"} -- streamed reasoning tokens (cumulative)
"""
- preserve_thinking = self._replay_preserve_thinking(
- preserve_thinking, messages, enable_thinking, reasoning_effort
- )
+ if replay_preserve_thinking:
+ preserve_thinking = self._replay_preserve_thinking(
+ preserve_thinking, messages, enable_thinking, reasoning_effort
+ )
from core.inference.tool_stream_exec import (
accepts_kwarg,
accepts_output_callback,
@@ -39061,7 +39089,6 @@ class LlamaCppBackend:
# Hold a leading ``{`` well past the 32-char XML cap until it balances (mirrors safetensors).
_MAX_BARE_JSON_BUFFER = 16384
_append_budget_exhausted_nudge = True
- _limit_notice_in_user_turn = False
# RAG: cap knowledge-base searches per assistant turn. The controller is
# tool-agnostic, so this gate stays in the loop.
_kb_search_count = 0
@@ -40913,7 +40940,6 @@ class LlamaCppBackend:
# Collapse exact-duplicate calls and cap the count for the TEXTUAL
# fallback (mirrors the safetensors loop; see _MAX_TOOL_CALLS_PER_TURN).
_over_cap: list = []
- _limit_notice_in_user_turn = False
if tool_calls and not has_structured_tc and len(tool_calls) > 1:
_seen_keys: set = set()
_last_workspace_key = None
@@ -41157,6 +41183,7 @@ class LlamaCppBackend:
}
if decision.tool_call_id:
denied_message["tool_call_id"] = decision.tool_call_id
+ _tool_card_by_message[id(denied_message)] = decision.tool_call_id
conversation.append(denied_message)
# The denial is itself the `tool` message that makes the
# template render the call it refused, so declining an
@@ -41384,6 +41411,7 @@ class LlamaCppBackend:
}
if decision.tool_call_id:
_refused_message["tool_call_id"] = decision.tool_call_id
+ _tool_card_by_message[id(_refused_message)] = decision.tool_call_id
conversation.append(_refused_message)
# The refusal is itself the `tool` message that makes this call's
# arguments start rendering, so declining without this leaves the
@@ -42045,7 +42073,6 @@ class LlamaCppBackend:
}
else:
conversation.append({"role": "user", "content": _limit_text})
- _limit_notice_in_user_turn = True
if batch_mcp_images and self.is_vision:
# One block after the whole batch. With a single result "the tool
@@ -42139,11 +42166,13 @@ class LlamaCppBackend:
# continuing to request tools.
if max_tool_iterations > 0 and _append_budget_exhausted_nudge:
if not _attach_internal_feedback_to_tool_result(BUDGET_EXHAUSTED_NUDGE):
- # Two user turns in a row break strict templates (Gemma), so ride the notice's turn.
- if _limit_notice_in_user_turn and conversation[-1].get("role") == "user":
+ # Ride the trailing user turn (over-cap notice or promoted image): two in a row break Gemma.
+ if conversation[-1].get("role") == "user":
conversation[-1] = {
**conversation[-1],
- "content": f"{conversation[-1]['content']}\n\n{BUDGET_EXHAUSTED_NUDGE}",
+ "content": with_appended_text(
+ conversation[-1]["content"], f"\n\n{BUDGET_EXHAUSTED_NUDGE}"
+ ),
}
else:
conversation.append({"role": "user", "content": BUDGET_EXHAUSTED_NUDGE})
diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.py
index 216472a45a7..b56226b6f44 100644
--- a/studio/backend/core/inference/safetensors_agentic.py
+++ b/studio/backend/core/inference/safetensors_agentic.py
@@ -70,6 +70,7 @@ from core.inference.tool_loop_controller import (
awaiting_approval_status,
coerce_tool_arguments,
restore_replayed_tool_results,
+ with_appended_text,
status_for_tool,
tool_call_limit_nudge,
tool_event_provenance,
@@ -1879,10 +1880,13 @@ def run_safetensors_tool_loop(
last_reprompt_text = ""
if _executed_tool_iters >= max_tool_iterations and not final_attempt_done:
final_attempt_done = True
- if over_cap:
+ # Ride a trailing user turn, as the GGUF loop does.
+ if over_cap or conversation[-1].get("role") == "user":
conversation[-1] = {
**conversation[-1],
- "content": f"{conversation[-1]['content']}\n\n{BUDGET_EXHAUSTED_NUDGE}",
+ "content": with_appended_text(
+ conversation[-1]["content"], f"\n\n{BUDGET_EXHAUSTED_NUDGE}"
+ ),
}
else:
conversation.append({"role": "user", "content": BUDGET_EXHAUSTED_NUDGE})
diff --git a/studio/backend/core/inference/studio_tool_loop.py b/studio/backend/core/inference/studio_tool_loop.py
index d9410eac70e..d42ebb800ca 100644
--- a/studio/backend/core/inference/studio_tool_loop.py
+++ b/studio/backend/core/inference/studio_tool_loop.py
@@ -84,6 +84,7 @@ from core.inference.tool_loop_controller import (
mcp_display_parts,
provisional_tool_provenance,
restore_replayed_tool_results,
+ with_appended_text,
strip_result_for_model,
)
from core.inference.tool_stream_exec import (
@@ -1353,12 +1354,11 @@ def _append_user_turn(conversation: list[dict[str, Any]], content: str) -> None:
if not content:
return
last = conversation[-1] if conversation else None
- if (
- isinstance(last, dict)
- and last.get("role") == "user"
- and isinstance(last.get("content"), str)
- ):
- conversation[-1] = {**last, "content": f"{last['content']}\n\n{content}"}
+ if isinstance(last, dict) and last.get("role") == "user":
+ conversation[-1] = {
+ **last,
+ "content": with_appended_text(last.get("content") or "", f"\n\n{content}"),
+ }
return
conversation.append({"role": "user", "content": content})
diff --git a/studio/backend/core/inference/tool_loop_controller.py b/studio/backend/core/inference/tool_loop_controller.py
index 3a9eae4496f..5ab6a5cfd26 100644
--- a/studio/backend/core/inference/tool_loop_controller.py
+++ b/studio/backend/core/inference/tool_loop_controller.py
@@ -864,6 +864,14 @@ def is_tool_error(result: str) -> bool:
TOOL_MODEL_SUFFIX_KEY = "studio_model_suffix"
+def with_appended_text(content, text: str):
+ """``content`` with ``text`` at the end. A part list (an image a tool returned may have joined
+ the turn) gets a text part; formatting it as a string ships the image as base64 prose."""
+ if isinstance(content, list):
+ return [*content, {"type": "text", "text": text}]
+ return f"{content}{text}"
+
+
def restore_replayed_tool_results(messages: Sequence[Any]) -> list[Any]:
"""Replayed tool results as the model saw them (error nudge, folded feedback from
``extra_content``, no citation map), so the cached prefix holds. Idempotent."""
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index 3d255ec85ea..f2b9014c6b8 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -14926,6 +14926,8 @@ def build_synthetic_search_exchange(
call_id = call_prefix + _uuid.uuid4().hex[:12]
args = {"query": query}
full_result = text + RAG_SOURCES_SENTINEL + _json.dumps(sources, ensure_ascii = False)
+ # Local and round-less: the client replays it as its own exchange.
+ provenance = {"source": "local"}
events = [
{"type": "status", "text": f"{status_label}: {query[:60]}"},
{
@@ -14933,12 +14935,14 @@ def build_synthetic_search_exchange(
"tool_name": tool_name,
"tool_call_id": call_id,
"arguments": args,
+ "provenance": provenance,
},
{
"type": "tool_end",
"tool_name": tool_name,
"tool_call_id": call_id,
"result": full_result,
+ "provenance": provenance,
},
{"type": "status", "text": ""},
]
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 2179d0f8638..480ddb620ed 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -2586,10 +2586,20 @@ def _openai_llama_admission_reserve(
return reservation, config
+def _preserve_probe_pending(llama_backend) -> bool:
+ """Whether resolving preserve_thinking may still make the one-off template probe (HTTP)."""
+ if not getattr(llama_backend, "_supports_preserve_thinking", False):
+ return False
+ cached = getattr(llama_backend, "_empty_think_preserve_probe", None)
+ key = getattr(llama_backend, "_empty_think_probe_key", None)
+ return key is not None and (cached is None or cached[0] != key())
+
+
def _replay_preserve_thinking(llama_backend, payload, messages):
"""The preserve_thinking generation will render with, so counts price the same prompt."""
resolver = getattr(llama_backend, "_replay_preserve_thinking", None)
- if resolver is None:
+ # An Anthropic recount: conversion dropped replayed thinking the resolver would miss.
+ if resolver is None or _anthropic_history_has_thinking(payload):
return payload.preserve_thinking
return resolver(
payload.preserve_thinking, messages, payload.enable_thinking, payload.reasoning_effort
@@ -5557,9 +5567,46 @@ def _think_parsing_expected(llama_backend, payload) -> bool:
return _resolved_kwargs_think(llama_backend, resolved)
-def _anthropic_count_template_kwargs(llama_backend, payload):
- """Resolved reasoning chat_template_kwargs for prompt-token counting."""
+def _anthropic_history_has_thinking(payload) -> bool:
+ for message in payload.messages or ():
+ role = message.get("role") if isinstance(message, dict) else getattr(message, "role", None)
+ content = (
+ message.get("content")
+ if isinstance(message, dict)
+ else getattr(message, "content", None)
+ )
+ if role != "assistant" or isinstance(content, str):
+ continue
+ for block in content or ():
+ kind = block.get("type") if isinstance(block, dict) else getattr(block, "type", None)
+ if kind in ("thinking", "redacted_thinking"):
+ return True
+ return False
+
+
+async def _resolved_anthropic_reasoning_args_async(llama_backend, payload, messages) -> dict:
+ """Off the loop only while resolving may still probe the template over HTTP."""
+ if _preserve_probe_pending(llama_backend):
+ return await asyncio.to_thread(
+ _resolved_anthropic_reasoning_args, llama_backend, payload, messages
+ )
+ return _resolved_anthropic_reasoning_args(llama_backend, payload, messages)
+
+
+def _resolved_anthropic_reasoning_args(llama_backend, payload, messages) -> dict:
+ """``_anthropic_reasoning_args`` with preserve_thinking resolved as generation resolves it."""
args = _anthropic_reasoning_args(payload)
+ resolver = getattr(llama_backend, "_replay_preserve_thinking", None)
+ # Conversion drops thinking it does not preserve, so ask the payload whether any was replayed.
+ if resolver is not None and not _anthropic_history_has_thinking(payload):
+ args["preserve_thinking"] = resolver(
+ args["preserve_thinking"], messages, args["enable_thinking"], args["reasoning_effort"]
+ )
+ return args
+
+
+def _anthropic_count_template_kwargs(llama_backend, args):
+ """Reasoning chat_template_kwargs for prompt-token counting, from resolved args."""
return _reasoning_template_kwargs(
llama_backend, args["enable_thinking"], args["reasoning_effort"], args["preserve_thinking"]
)
@@ -27800,6 +27847,31 @@ def _replay_tool_call_ids(tool_calls: Any, replay_ids: dict[str, str]) -> Any:
return out
+def _live_tool_call_ids(messages: list[dict]) -> list[dict]:
+ """GGUF history with each tool-call id as the model generated it: the client suffixes every card id
+ with a minted uuid, which templates that render ids (Kimi, Mistral) would otherwise show, and Mistral
+ rejects outright. Calls and results lose the same suffix, so they stay paired."""
+ out = []
+ for message in messages:
+ calls = message.get("tool_calls")
+ call_id = message.get("tool_call_id")
+ if isinstance(calls, list):
+ calls = [
+ {**call, "id": _MINTED_TOOL_CALL_ID_SUFFIX.sub("", call["id"]) or call["id"]}
+ if isinstance(call, dict) and isinstance(call.get("id"), str)
+ else call
+ for call in calls
+ ]
+ message = {**message, "tool_calls": calls}
+ if isinstance(call_id, str):
+ message = {
+ **message,
+ "tool_call_id": _MINTED_TOOL_CALL_ID_SUFFIX.sub("", call_id) or call_id,
+ }
+ out.append(message)
+ return out
+
+
def _build_external_messages(
messages: list,
supports_vision: bool,
@@ -41457,10 +41529,15 @@ async def chat_count_tokens(
# As generation sends them; off the loop since restoring may parse an envelope.
_count_dumped = await asyncio.to_thread(restore_replayed_tool_results, _count_dumped)
+ _count_kept = _strip_provider_synthetic_tool_history(
+ _drop_empty_assistant_sentinels(_count_dumped)
+ )
+ if not _takes_passthrough:
+ _count_kept = _live_tool_call_ids(_count_kept)
# Awaited: stripping parses the whole envelope, and this recount runs in the
# background on every turn, so it must not do that on the shared loop.
openai_messages = await _promote_mcp_history_images_async(
- _strip_provider_synthetic_tool_history(_drop_empty_assistant_sentinels(_count_dumped)),
+ _count_kept,
# Never as pixels: /apply-template prices a media marker, not the image.
# This drops the base64 the completion path would have promoted, which is
# the one part a count must not price as text.
@@ -41595,11 +41672,7 @@ async def chat_count_tokens(
payload.enable_thinking,
payload.reasoning_effort,
# Off the loop: first use probes the template.
- payload.preserve_thinking
- if _takes_passthrough
- else await asyncio.to_thread(
- _replay_preserve_thinking, llama_backend, payload, openai_messages
- ),
+ await asyncio.to_thread(_replay_preserve_thinking, llama_backend, payload, openai_messages),
)
# Whose tokenizer this is, in the shape /api/inference/status publishes: another tab's load
@@ -41834,18 +41907,13 @@ async def anthropic_count_tokens(
# templates thinking / reasoning_effort / preserve_thinking change the
# rendered prompt, so counting the load-time default would under- or
# over-count the request the client is about to send.
- _reasoning_args = _anthropic_reasoning_args(payload)
- _count_preserve = _reasoning_args["preserve_thinking"]
- _resolve_preserve = getattr(llama_backend, "_replay_preserve_thinking", None)
- if _resolve_preserve is not None and not _count_client_tools:
- # Off the loop: first use probes the template.
- _count_preserve = await asyncio.to_thread(
- _resolve_preserve,
- _count_preserve,
- openai_messages,
- _reasoning_args["enable_thinking"],
- _reasoning_args["reasoning_effort"],
- )
+ if _count_server_tools:
+ from core.inference.tool_loop_controller import restore_replayed_tool_results
+ openai_messages = await asyncio.to_thread(restore_replayed_tool_results, openai_messages)
+ # Off the loop: first use probes the template.
+ _reasoning_args = await _resolved_anthropic_reasoning_args_async(
+ llama_backend, payload, openai_messages
+ )
try:
count = await asyncio.to_thread(
lambda: llama_backend.count_chat_tokens(
@@ -41853,11 +41921,8 @@ async def anthropic_count_tokens(
None,
openai_tools,
strict = True,
- chat_template_kwargs = _reasoning_template_kwargs(
- llama_backend,
- _reasoning_args["enable_thinking"],
- _reasoning_args["reasoning_effort"],
- _count_preserve,
+ chat_template_kwargs = _anthropic_count_template_kwargs(
+ llama_backend, _reasoning_args
),
)
)
@@ -42507,6 +42572,10 @@ async def anthropic_messages(
else:
reservation.cancel()
+ _reasoning_args = await _resolved_anthropic_reasoning_args_async(
+ llama_backend, payload, openai_messages
+ )
+
# ── Client-side pass-through path ─────────────────────────
if client_tools or response_format is not None:
openai_tools = openai_client_tools if client_tools else []
@@ -42537,7 +42606,7 @@ async def anthropic_messages(
auto_heal_tool_calls = payload.auto_heal_tool_calls,
response_format = response_format,
parse_think = _think_parsing_expected(llama_backend, payload),
- **_anthropic_reasoning_args(payload),
+ **_reasoning_args,
)
)
return await _admitted_anthropic(
@@ -42564,7 +42633,7 @@ async def anthropic_messages(
cancel_event = cancel_event,
response_format = response_format,
parse_think = _think_parsing_expected(llama_backend, payload),
- **_anthropic_reasoning_args(payload),
+ **_reasoning_args,
)
)
@@ -42574,6 +42643,10 @@ async def anthropic_messages(
_think_prov: dict = {"wrapped": 0}
if server_tools:
+ from core.inference.tool_loop_controller import restore_replayed_tool_results
+
+ # As the tool loop does, so the opening usage count prices what it sends.
+ openai_messages = await asyncio.to_thread(restore_replayed_tool_results, openai_messages)
# Bypass Permissions suppresses confirm, so both flags together is fine.
if bool(getattr(payload, "confirm_tool_calls", False)) and not bool(
getattr(payload, "bypass_permissions", False)
@@ -42697,7 +42770,8 @@ async def anthropic_messages(
monitor_id,
llama_backend.context_length,
),
- **_anthropic_reasoning_args(payload),
+ replay_preserve_thinking = not _anthropic_history_has_thinking(payload),
+ **_reasoning_args,
)
if payload.stream:
@@ -42714,7 +42788,9 @@ async def anthropic_messages(
disable_parallel_tool_use = _disable_parallel,
parse_think = _think_parsing_expected(llama_backend, payload),
think_provenance = _think_prov,
- count_template_kwargs = _anthropic_count_template_kwargs(llama_backend, payload),
+ count_template_kwargs = _anthropic_count_template_kwargs(
+ llama_backend, _reasoning_args
+ ),
),
# Same server-side loop the chat route runs, up to 25 rounds on one lease.
tool_loop = True,
@@ -42754,7 +42830,8 @@ async def anthropic_messages(
monitor_id,
llama_backend.context_length,
),
- **_anthropic_reasoning_args(payload),
+ replay_preserve_thinking = not _anthropic_history_has_thinking(payload),
+ **_reasoning_args,
)
if payload.stream:
@@ -42769,7 +42846,9 @@ async def anthropic_messages(
openai_messages = openai_messages,
parse_think = _think_parsing_expected(llama_backend, payload),
think_provenance = _think_prov,
- count_template_kwargs = _anthropic_count_template_kwargs(llama_backend, payload),
+ count_template_kwargs = _anthropic_count_template_kwargs(
+ llama_backend, _reasoning_args
+ ),
)
)
return await _admitted_anthropic(
@@ -44813,11 +44892,14 @@ def _openai_messages_for_gguf_chat(
# Coalesce only on the GGUF chat path (strict Jinja template); the tool path
# reuses this via _set_or_prepend_system_message. Passthrough forwards verbatim.
+ # Ids lose their minted suffix only after the synthetic pass, which drops results by id.
messages = _coalesce_consecutive_user_turns(
- _strip_provider_synthetic_tool_history(
- _drop_empty_assistant_sentinels(
- restore_replayed_tool_results(
- [m.model_dump(exclude_none = True) for m in payload.messages]
+ _live_tool_call_ids(
+ _strip_provider_synthetic_tool_history(
+ _drop_empty_assistant_sentinels(
+ restore_replayed_tool_results(
+ [m.model_dump(exclude_none = True) for m in payload.messages]
+ )
)
)
)
@@ -44846,7 +44928,10 @@ async def _openai_messages_for_gguf_chat_async(
is_vision: bool,
promoted_out: "Optional[list]" = None,
) -> tuple[list[dict], bool]:
- if _request_has_image(payload):
+ # Restoring a tool result may parse its envelope or source map.
+ if _request_has_image(payload) or any(
+ getattr(message, "role", None) == "tool" for message in payload.messages
+ ):
return await asyncio.to_thread(
_openai_messages_for_gguf_chat, payload, is_vision, promoted_out
)
@@ -44945,7 +45030,7 @@ def _build_openai_passthrough_body(
llama_backend._request_reasoning_kwargs(
payload.enable_thinking,
payload.reasoning_effort,
- payload.preserve_thinking,
+ _replay_preserve_thinking(llama_backend, payload, messages),
)
if llama_backend is not None
else None
@@ -44988,7 +45073,7 @@ async def _build_openai_passthrough_body_async(
backend_ctx = None,
llama_backend = None,
) -> dict:
- if _request_has_image(payload):
+ if _request_has_image(payload) or _preserve_probe_pending(llama_backend):
return await asyncio.to_thread(
_build_openai_passthrough_body,
payload,
diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.py
index 0b5891b490d..01554635e48 100644
--- a/studio/backend/tests/test_anthropic_messages.py
+++ b/studio/backend/tests/test_anthropic_messages.py
@@ -431,12 +431,49 @@ def test_replayed_thinking_preserved_only_when_requested():
{"type": "text", "text": "the answer"},
],
},
+ {"role": "user", "content": "and then?"},
]
dropped = anthropic_messages_to_openai(messages)
- assert "reasoning_content" not in dropped[-1]
+ assert "reasoning_content" not in dropped[1]
kept = anthropic_messages_to_openai(messages, preserve_thinking = True)
- assert kept[-1]["reasoning_content"] == "prior trace"
- assert kept[-1]["content"] == "the answer"
+ assert kept[1]["reasoning_content"] == "prior trace"
+ assert kept[1]["content"] == "the answer"
+
+
+def test_the_current_turns_thinking_survives_without_preservation():
+ # Templates render reasoning after the last real query, so an agent's tool rounds keep theirs;
+ # dropping it made every round re-read the previous one's reasoning.
+ from core.inference.anthropic_compat import anthropic_messages_to_openai
+
+ def _round(trace, call_id):
+ return {
+ "role": "assistant",
+ "content": [
+ {"type": "thinking", "thinking": trace, "signature": ""},
+ {"type": "tool_use", "id": call_id, "name": "bash", "input": {"cmd": "ls"}},
+ ],
+ }
+
+ def _result(call_id):
+ return {
+ "role": "user",
+ "content": [{"type": "tool_result", "tool_use_id": call_id, "content": "ok"}],
+ }
+
+ messages = [
+ {"role": "user", "content": "first task"},
+ _round("old trace", "t0"),
+ _result("t0"),
+ {"role": "assistant", "content": [{"type": "text", "text": "done"}]},
+ {"role": "user", "content": "second task"},
+ _round("round one", "t1"),
+ _result("t1"),
+ _round("round two", "t2"),
+ _result("t2"),
+ ]
+ out = anthropic_messages_to_openai(messages)
+ traces = [m.get("reasoning_content") for m in out if m.get("role") == "assistant"]
+ assert traces == [None, None, "round one", "round two"]
def test_anthropic_emitter_only_leading_think_is_reasoning():
@@ -3215,6 +3252,46 @@ class TestAnthropicMessagesToolRouting:
assert captured["seed"] == 3407
+ @pytest.mark.parametrize("stream", [False, True])
+ def test_client_tool_passthrough_sends_the_resolved_preserve_thinking(
+ self, monkeypatch, stream
+ ):
+ # Claude Code brings its own tools, so its turns ride the passthrough: a thinking-off
+ # follow-up must replay the empty think blocks generation would, or every turn re-reads.
+ import routes.inference as inf_mod
+ from fastapi.responses import JSONResponse
+
+ _mock_backend(
+ monkeypatch,
+ _replay_preserve_thinking = lambda preserve, messages, enable, effort: (
+ True if enable is False and preserve is not True else preserve
+ ),
+ )
+ captured = {}
+
+ async def _passthrough(*args, **kwargs):
+ captured.update(kwargs)
+ return JSONResponse({"type": "message", "content": []})
+
+ helper = (
+ "_anthropic_passthrough_stream" if stream else "_anthropic_passthrough_non_streaming"
+ )
+ monkeypatch.setattr(inf_mod, helper, _passthrough)
+ payload = _basic_payload(
+ stream = stream,
+ messages = [
+ {"role": "user", "content": "hi"},
+ {"role": "assistant", "content": "hello"},
+ {"role": "user", "content": "again"},
+ ],
+ thinking = {"type": "disabled"},
+ tools = [{"name": "lookup", "input_schema": {"type": "object"}}],
+ )
+
+ _drive(anthropic_messages(payload, request = self._Request(), current_subject = "t"))
+
+ assert captured["preserve_thinking"] is True
+
_BUDGET_CASES = [
({"thinking": {"type": "enabled", "budget_tokens": 128}}, 128),
({"thinking": {"type": "adaptive", "budget_tokens": 128}}, 128),
@@ -3553,6 +3630,143 @@ class TestAnthropicMessagesToolRouting:
assert response.status_code == 200
assert [t["function"]["name"] for t in seen["tools"]] == expected
+ def test_replayed_thinking_keeps_the_callers_preserve_setting(self, monkeypatch):
+ # Conversion drops thinking it does not preserve, so the resolver must not read that
+ # history as reasoning-free and swap real thoughts for empty think blocks.
+ import routes.inference as inf_mod
+ from fastapi.responses import JSONResponse
+
+ _mock_backend(
+ monkeypatch,
+ _replay_preserve_thinking = lambda preserve, messages, enable, effort: True,
+ )
+ captured = {}
+
+ async def _passthrough(*args, **kwargs):
+ captured.update(kwargs)
+ return JSONResponse({"type": "message", "content": []})
+
+ monkeypatch.setattr(inf_mod, "_anthropic_passthrough_non_streaming", _passthrough)
+ payload = _basic_payload(
+ messages = [
+ {"role": "user", "content": "hi"},
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "thinking", "thinking": "Greet back.", "signature": "s"},
+ {"type": "text", "text": "hello"},
+ ],
+ },
+ {"role": "user", "content": "again"},
+ ],
+ thinking = {"type": "disabled"},
+ tools = [{"name": "lookup", "input_schema": {"type": "object"}}],
+ )
+
+ _drive(anthropic_messages(payload, request = self._Request(), current_subject = "t"))
+
+ assert captured["preserve_thinking"] is not True
+
+ @pytest.mark.parametrize(
+ ("extra", "expected_path"),
+ [({}, "plain"), ({"enable_tools": True, "permission_mode": "off"}, "tools")],
+ ids = ["plain", "server-tools"],
+ )
+ @pytest.mark.parametrize("replays_thinking", [True, False])
+ def test_generation_skips_the_replay_resolver_only_when_thinking_was_dropped(
+ self, monkeypatch, extra, expected_path, replays_thinking
+ ):
+ # The generator resolves from the converted history, where dropped thinking reads as none.
+ backend = _mock_backend(monkeypatch)
+ answer = [{"type": "text", "text": "hello"}]
+ if replays_thinking:
+ answer.insert(0, {"type": "thinking", "thinking": "Greet back.", "signature": "s"})
+ payload = _basic_payload(
+ messages = [
+ {"role": "user", "content": "hi"},
+ {"role": "assistant", "content": answer},
+ {"role": "user", "content": "again"},
+ ],
+ thinking = {"type": "disabled"},
+ **extra,
+ )
+
+ _drive(anthropic_messages(payload, request = self._Request(), current_subject = "t"))
+
+ [(path, kwargs)] = backend.calls
+ assert path == expected_path
+ assert kwargs["replay_preserve_thinking"] is (not replays_thinking)
+
+ def test_the_admission_recount_keeps_the_callers_preserve_when_thinking_was_dropped(self):
+ from routes.inference import _replay_preserve_thinking
+
+ class _Backend:
+ def _replay_preserve_thinking(self, preserve, messages, enable, effort):
+ return True
+
+ payload = _basic_payload(
+ messages = [
+ {"role": "user", "content": "hi"},
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "thinking", "thinking": "Greet back.", "signature": "s"},
+ {"type": "text", "text": "hello"},
+ ],
+ },
+ {"role": "user", "content": "again"},
+ ],
+ preserve_thinking = False,
+ )
+ assert _replay_preserve_thinking(_Backend(), payload, []) is False
+
+ def test_server_tool_count_prices_the_restored_failed_result(self, monkeypatch):
+ # The server-tool loop renders a replayed failure with the nudge the model saw, so the
+ # count has to price the same bytes.
+ from core.inference.tool_call_parser import TOOL_ERROR_NUDGE
+ from routes.inference import anthropic_count_tokens
+
+ seen = {}
+
+ def _count(messages, system, tools, **kwargs):
+ seen["messages"] = messages
+ return 2
+
+ _mock_backend(monkeypatch, count_chat_tokens = _count)
+ payload = _basic_payload(
+ enable_tools = True,
+ permission_mode = "off",
+ tools = [{"type": "web_search_20250305", "name": "web_search"}],
+ messages = [
+ {"role": "user", "content": "search"},
+ {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "tool_use",
+ "id": "t1",
+ "name": "web_search",
+ "input": {"query": "q"},
+ }
+ ],
+ },
+ {
+ "role": "user",
+ "content": [
+ {"type": "tool_result", "tool_use_id": "t1", "content": "Error: boom"}
+ ],
+ },
+ ],
+ )
+
+ response = _drive(
+ anthropic_count_tokens(payload, request = self._Request(), current_subject = "t")
+ )
+
+ assert response.status_code == 200
+ tool = next(m for m in seen["messages"] if m.get("role") == "tool")
+ assert tool["content"].endswith(TOOL_ERROR_NUDGE)
+
def test_count_tokens_prices_the_catalog_left_when_the_forced_tool_is_dropped(
self, monkeypatch
):
@@ -4480,7 +4694,7 @@ class TestAnthropicMessagesToolRouting:
def test_resumed_session_thinking_and_null_content_do_not_400():
# A resumed session replays assistant turns with `thinking` (and sometimes null)
- # content. Those must be accepted (thinking dropped by the converter), not 400ed.
+ # content. Those must be accepted, not 400ed.
from pydantic import ValidationError
req = AnthropicMessagesRequest(
@@ -4507,7 +4721,9 @@ def test_resumed_session_thinking_and_null_content_do_not_400():
openai = anthropic_messages_to_openai([m.model_dump() for m in req.messages])
assistant = next(m for m in openai if m["role"] == "assistant" and m.get("content"))
assert assistant["content"] == "the answer"
- assert "secret reasoning" not in json.dumps(openai) # thinking never forwarded
+ # The current turn's thinking rides as reasoning, never as visible content.
+ assert assistant["reasoning_content"] == "secret reasoning"
+ assert all("secret reasoning" not in (m.get("content") or "") for m in openai)
# A malformed KNOWN block still fails cleanly instead of being swallowed.
with pytest.raises(ValidationError):
diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.py
index f712194c8f9..646e438a2c2 100644
--- a/studio/backend/tests/test_llama_cpp_tool_loop.py
+++ b/studio/backend/tests/test_llama_cpp_tool_loop.py
@@ -7507,3 +7507,59 @@ def test_restored_replay_is_the_branch_the_tools_are_given(monkeypatch):
assert branches
replayed = [m for m in branches[0] if m.get("tool_call_id") == "call_old"]
assert len(replayed) == 1
+
+
+def test_budget_notice_folded_into_a_denied_result_is_reported_for_replay(monkeypatch):
+ # A denial is the round's last tool message too, so the notice can land on it: the client has
+ # to be told, or it replays the bare denial and the next turn misses the cache there.
+ from core.inference.tool_call_parser import BUDGET_EXHAUSTED_NUDGE
+
+ # One call runs and spends the budget; the batch's last call is declined.
+ blocks = (
+ '<tool_call>{"name":"python","arguments":{"code":"print(1)"}}</tool_call>'
+ '<tool_call>{"name":"python","arguments":{"code":"print(2)"}}</tool_call>'
+ )
+ streams = [[_sse({"content": blocks}), _done()], [_sse({"content": "Done."}), _done()]]
+ backend, _payloads = _backend_and_payloads(monkeypatch, streams)
+ calls = _record_tool_calls(monkeypatch, "OK")
+ decisions = iter(["allow", "deny"])
+ monkeypatch.setattr("core.inference.llama_cpp.new_approval_id", lambda: "approval-1")
+ monkeypatch.setattr("core.inference.llama_cpp.begin_tool_decision", lambda *_a, **_k: object())
+ monkeypatch.setattr(
+ "core.inference.llama_cpp.wait_tool_decision", lambda *_a, **_k: next(decisions)
+ )
+
+ events = list(
+ backend.generate_chat_completion_with_tools(
+ messages = [{"role": "user", "content": "run python"}],
+ tools = [{"type": "function", "function": {"name": "python"}}],
+ max_tool_iterations = 1,
+ confirm_tool_calls = True,
+ permission_mode = "ask",
+ session_id = "sess",
+ )
+ )
+
+ assert calls == [("python", {"code": "print(1)"})]
+ tool_end = [e for e in events if e.get("type") == "tool_end"][-1]
+ notes = [e for e in events if e.get("type") == "tool_model_note"]
+ assert notes == [
+ {
+ "type": "tool_model_note",
+ "tool_call_id": tool_end["tool_call_id"],
+ "suffix": f"\n\n{BUDGET_EXHAUSTED_NUDGE}",
+ }
+ ]
+
+
+def test_appended_notice_keeps_an_image_part_an_image():
+ from core.inference.tool_loop_controller import with_appended_text as _with_appended_text
+
+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}
+ out = _with_appended_text([{"type": "text", "text": "Over the cap."}, image], "\n\nStop.")
+ assert out == [
+ {"type": "text", "text": "Over the cap."},
+ image,
+ {"type": "text", "text": "\n\nStop."},
+ ]
+ assert _with_appended_text("Over the cap.", "\n\nStop.") == "Over the cap.\n\nStop."
diff --git a/studio/backend/tests/test_preserve_thinking_replay.py b/studio/backend/tests/test_preserve_thinking_replay.py
index d358cb6300e..94790f81200 100644
--- a/studio/backend/tests/test_preserve_thinking_replay.py
+++ b/studio/backend/tests/test_preserve_thinking_replay.py
@@ -49,7 +49,9 @@ def test_the_users_choice_stands_wherever_there_is_reasoning_or_no_proof(monkeyp
backend = _backend()
monkeypatch.setattr(LlamaCppBackend, "_empty_think_replays_with_preserve", lambda self: True)
assert backend._replay_preserve_thinking(False, _history("thought about it"), False) is False
- assert backend._replay_preserve_thinking(None, _history(), False) is None
+ # Omitted (every API client that never sends it) resolves the same way as an explicit off.
+ assert backend._replay_preserve_thinking(None, _history(), False) is True
+ assert backend._replay_preserve_thinking(None, _history("thought about it"), False) is None
assert backend._replay_preserve_thinking(True, _history(), False) is True
assert _backend(supports = False)._replay_preserve_thinking(False, _history(), False) is False
# Thinking on: its reasoning is real, so the caller's choice to drop it stands.
@@ -129,3 +131,164 @@ def test_probe_failure_keeps_the_setting_and_retries(monkeypatch):
backend = _backend()
assert backend._empty_think_replays_with_preserve() is False
assert getattr(backend, "_empty_think_preserve_probe", None) is None
+
+
+class _RouteBackend:
+ """What the passthrough body builder reads off the llama backend."""
+
+ is_vision = False
+ markup_profile = None
+
+ def _request_reasoning_kwargs(self, enable_thinking, reasoning_effort, preserve_thinking):
+ kwargs = {"enable_thinking": enable_thinking}
+ if preserve_thinking is not None:
+ kwargs["preserve_thinking"] = preserve_thinking
+ return kwargs
+
+ def _replay_preserve_thinking(
+ self,
+ preserve,
+ messages,
+ enable_thinking = None,
+ effort = None,
+ ):
+ answered = any(m.get("role") == "assistant" for m in messages)
+ return True if preserve is not True and enable_thinking is False and answered else preserve
+
+
+def test_client_tool_passthrough_replays_the_empty_think_blocks_too():
+ # Coding agents bring their own tools, so their turns ride the passthrough: it has to send the
+ # preserve_thinking generation would, or every follow-up re-reads the previous turn.
+ from models.inference import ChatCompletionRequest
+ from routes.inference import _build_openai_passthrough_body
+
+ payload = ChatCompletionRequest(
+ messages = [
+ {"role": "user", "content": "list files"},
+ {"role": "assistant", "content": "Done."},
+ {"role": "user", "content": "and again"},
+ ],
+ tools = [
+ {"type": "function", "function": {"name": "bash", "parameters": {"type": "object"}}}
+ ],
+ enable_thinking = False,
+ )
+ body = _build_openai_passthrough_body(payload, backend_ctx = 4096, llama_backend = _RouteBackend())
+ assert body["chat_template_kwargs"] == {"enable_thinking": False, "preserve_thinking": True}
+
+
+def test_gguf_history_carries_the_tool_call_ids_the_model_generated():
+ from models.inference import ChatCompletionRequest
+ from routes.inference import _openai_messages_for_gguf_chat
+
+ minted = "call_0:2f1c0a9e-4b7d-4c1e-9a3b-5d6e7f809a1b"
+ payload = ChatCompletionRequest(
+ messages = [
+ {"role": "user", "content": "run it"},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": minted,
+ "type": "function",
+ "function": {"name": "python", "arguments": "{}"},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": minted, "name": "python", "content": "ok"},
+ {"role": "user", "content": "next"},
+ ],
+ )
+ messages, _ = _openai_messages_for_gguf_chat(payload, is_vision = False)
+ assert messages[1]["tool_calls"][0]["id"] == "call_0"
+ assert messages[2]["tool_call_id"] == "call_0"
+
+
+def test_a_synthetic_provider_call_sharing_a_base_id_does_not_take_a_real_result_with_it():
+ # A Gemini code_execution card and a later GGUF call can both be call_0 under their minted
+ # suffix; the synthetic pass drops by id, so the suffix has to survive it.
+ from models.inference import ChatCompletionRequest
+ from routes.inference import _openai_messages_for_gguf_chat
+
+ synthetic = "call_0:11111111-2222-4333-8444-555555555555"
+ real = "call_0:66666666-7777-4888-9999-aaaaaaaaaaaa"
+ payload = ChatCompletionRequest(
+ messages = [
+ {"role": "user", "content": "compute"},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": synthetic,
+ "type": "function",
+ "function": {
+ "name": "code_execution",
+ "arguments": '{"_server_tool": true}',
+ },
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": synthetic, "name": "code_execution", "content": "4"},
+ {"role": "user", "content": "now run it locally"},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": real,
+ "type": "function",
+ "function": {"name": "python", "arguments": "{}"},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": real, "name": "python", "content": "ok"},
+ {"role": "user", "content": "next"},
+ ],
+ )
+ messages, _ = _openai_messages_for_gguf_chat(payload, is_vision = False)
+ tools = [m for m in messages if m.get("role") == "tool"]
+ assert [(m["tool_call_id"], m["content"]) for m in tools] == [("call_0", "ok")]
+
+
+def test_an_omitted_thinking_switch_follows_the_load_time_default(monkeypatch):
+ # Agents rarely send enable_thinking: llama-server renders the load-time default, so a model
+ # loaded with thinking off gets the same replay as an explicit off.
+ monkeypatch.setattr(LlamaCppBackend, "_empty_think_replays_with_preserve", lambda self: True)
+ backend = _backend()
+ backend._reasoning_default = False
+ assert backend._replay_preserve_thinking(None, _history(), None) is True
+ backend._reasoning_default = True
+ assert backend._replay_preserve_thinking(None, _history(), None) is None
+
+
+def test_a_template_the_server_refuses_to_render_is_probed_once(monkeypatch):
+ # Agents send many thinking-off turns; a 400 from /apply-template is an answer, not a blip.
+ calls: list = []
+
+ class _Refusing(_FakeClient):
+ def post(self, url, json):
+ self._calls.append(json)
+ return type("R", (), {"status_code": 400, "json": lambda self: {"error": "bad"}})()
+
+ monkeypatch.setattr(llama_cpp.httpx, "Client", lambda **kw: _Refusing(None, calls))
+ backend = _backend()
+ assert backend._empty_think_replays_with_preserve() is False
+ assert backend._empty_think_replays_with_preserve() is False
+ assert len(calls) == 3
+
+
+def test_a_busy_server_is_asked_again(monkeypatch):
+ calls: list = []
+
+ class _Busy(_FakeClient):
+ def post(self, url, json):
+ self._calls.append(json)
+ return type("R", (), {"status_code": 429, "json": lambda self: {"error": "busy"}})()
+
+ monkeypatch.setattr(llama_cpp.httpx, "Client", lambda **kw: _Busy(None, calls))
+ backend = _backend()
+ assert backend._empty_think_replays_with_preserve() is False
+ assert backend._empty_think_replays_with_preserve() is False
+ assert len(calls) == 6
diff --git a/studio/backend/tests/test_studio_tool_loop.py b/studio/backend/tests/test_studio_tool_loop.py
index 43cc5d32f6d..2753a50b87a 100644
--- a/studio/backend/tests/test_studio_tool_loop.py
+++ b/studio/backend/tests/test_studio_tool_loop.py
@@ -1811,3 +1811,21 @@ def test_preserved_thinking_before_a_tool_call_reaches_the_next_turn(executed, f
if message.get("role") == "assistant"
][-1]
assert assistant["reasoning_content"] == "look it up"
+
+
+def test_a_trailing_image_turn_takes_the_notice_as_a_text_part():
+ from core.inference.studio_tool_loop import _append_user_turn
+
+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}
+ conversation = [{"role": "user", "content": [{"type": "text", "text": "Screenshot:"}, image]}]
+ _append_user_turn(conversation, "Stop now.")
+ assert conversation == [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "Screenshot:"},
+ image,
+ {"type": "text", "text": "\n\nStop now."},
+ ],
+ }
+ ]
diff --git a/studio/backend/tests/test_tool_error_nudge_replay.py b/studio/backend/tests/test_tool_error_nudge_replay.py
index e68cce5c20e..5c79306014f 100644
--- a/studio/backend/tests/test_tool_error_nudge_replay.py
+++ b/studio/backend/tests/test_tool_error_nudge_replay.py
@@ -142,3 +142,20 @@ def test_replayed_retrieval_result_drops_the_source_map_the_model_never_saw():
# Any other tool keeps text that merely carries the marker.
other = {**replayed, "name": "terminal"}
assert restore_replayed_tool_results([other])[0]["content"] == raw
+
+
+def test_a_forced_retrieval_card_replays_as_its_own_exchange():
+ # Local and round-less: the client flushes it as one call/result pair, so a real tool round
+ # carrying a round_id right after it is not merged into the same assistant message.
+ from core.inference.tools import build_synthetic_search_exchange
+
+ built = build_synthetic_search_exchange(
+ tool_name = "search_conversation",
+ call_prefix = "recall_",
+ status_label = "Recalling",
+ query = "the code",
+ text = "It was 5150.",
+ sources = [],
+ )
+ cards = [e for e in built["events"] if e["type"] in ("tool_start", "tool_end")]
+ assert [c["provenance"] for c in cards] == [{"source": "local"}, {"source": "local"}]
diff --git a/studio/frontend/src/features/chat/prompt-storage/prompt-storage-dialog.tsx b/studio/frontend/src/features/chat/prompt-storage/prompt-storage-dialog.tsx
index efc9ac1484d..f2135828e54 100644
--- a/studio/frontend/src/features/chat/prompt-storage/prompt-storage-dialog.tsx
+++ b/studio/frontend/src/features/chat/prompt-storage/prompt-storage-dialog.tsx
@@ -418,7 +418,13 @@ function messageToOpenAI(msg: { role: unknown; content: unknown; attachments?: u
pushText("[image attachment]");
} else if (p.type === "tool-call") {
const callRoundId = codexLocalToolRoundId(p.provenance);
- if (toolCalls.length > 0 && startsNewCodexToolRound(roundId, callRoundId)) flush();
+ // A completed local call with no round (a forced recall) is its own exchange, as in the chat adapter.
+ const ownPair =
+ callRoundId === null &&
+ (p.provenance as { source?: unknown } | undefined)?.source === "local" &&
+ p.result !== undefined &&
+ p.result !== null;
+ if (toolCalls.length > 0 && (ownPair || startsNewCodexToolRound(roundId, callRoundId))) flush();
if (callRoundId !== null) roundId = callRoundId;
const id = typeof p.toolCallId === "string" ? p.toolCallId : `call_${callCount}`;
callCount++;
@@ -434,7 +440,7 @@ function messageToOpenAI(msg: { role: unknown; content: unknown; attachments?: u
const resultStr =
typeof modelText === "string" ? modelText : JSON.stringify(modelText);
toolResults.push({ role: "tool", tool_call_id: id, name, content: resultStr });
- if (callRoundId === null && (p.provenance as { source?: unknown } | undefined)?.source === "local") flush();
+ if (ownPair) flush();
}
}
}
diff --git a/studio/frontend/tests/local-reasoning-replay.test.ts b/studio/frontend/tests/local-reasoning-replay.test.ts
index 8081ae6b051..c70d5295b27 100644
--- a/studio/frontend/tests/local-reasoning-replay.test.ts
+++ b/studio/frontend/tests/local-reasoning-replay.test.ts
@@ -6,6 +6,7 @@ import test from "node:test";
import {
codexLocalToolRoundId,
+ localToolExchangeIndexes,
shouldReplayAssistantReasoning,
startsNewCodexToolRound,
} from "../src/features/chat/codex-reasoning.ts";
@@ -61,3 +62,26 @@ test("Codex local tool provenance preserves parallel calls and round boundaries"
assert.equal(startsNewCodexToolRound(1, 1), false);
assert.equal(startsNewCodexToolRound(1, codexLocalToolRoundId(secondRound)), true);
});
+
+test("a forced retrieval card stays its own exchange before a numbered round", () => {
+ const exchanges = (parts: { provenance?: unknown; result?: string }[]) =>
+ localToolExchangeIndexes(
+ parts,
+ (part) => codexLocalToolRoundId(part.provenance),
+ (part) =>
+ (part.provenance as { source?: unknown } | undefined)?.source === "local" &&
+ part.result !== undefined,
+ );
+ const realRound = { provenance: { source: "local", round_id: 0 }, result: "ok" };
+ // What the backend sends for a forced recall: local, no round_id.
+ assert.deepEqual(exchanges([{ provenance: { source: "local" }, result: "recall" }, realRound]), [0, 1]);
+ // Without provenance the two merge into one assistant message, which Llama 3 templates reject.
+ assert.deepEqual(exchanges([{ result: "recall" }, realRound]), [0, 0]);
+});
+
+test("the JSONL export also starts a new exchange before a forced recall card", async () => {
+ const { readSrc } = await import("./helpers/kit.ts");
+ const src = readSrc("features/chat/prompt-storage/prompt-storage-dialog.tsx");
+ // Same rule as the chat adapter: flush what is pending before a round-less local pair.
+ assert.match(src, /toolCalls\.length > 0 && \(ownPair \|\| startsNewCodexToolRound\(roundId, callRoundId\)\)\) flush\(\)/);
+});