Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: search earlier turns for the user's question, not the tool loop's own notices (#16204)

Debojyoti Banerjeef4ee09876755
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index db6ff584af1..eb6884310db 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -2176,8 +2176,10 @@ def _archive_and_recall(
branch_messages: Optional[list[dict]] = None,
cancel_check: Optional[Callable[[], bool]] = None,
background: bool = False,
+ query_messages: Optional[list[dict]] = None,
) -> dict:
- """Never raises: any failure falls back to the plain fitted messages. Recall runs at most once per request or refits stack recall blocks."""
+ """Never raises: any failure falls back to the plain fitted messages. Recall runs at most once per request or refits stack recall blocks.
+ ``query_messages`` is the client's transcript: a tool loop's live branch ends on its own notices, not the user's question."""
result = {
"conversation": conversation,
"events": [],
@@ -2238,6 +2240,9 @@ def _archive_and_recall(
break
attempt_k = max(1, attempt_k // 2)
+ recall_kwargs = {"branch_messages": branch_messages}
+ if query_messages is not None:
+ recall_kwargs["query_messages"] = query_messages
# Retries re-ask the same queries: encode each once and reuse lexical passes and branch checks.
with query_vector_memo(), conversation_archive.recall_session():
for index, k in enumerate(attempts):
@@ -2246,7 +2251,7 @@ def _archive_and_recall(
thread_id,
style = style,
top_k = k,
- branch_messages = branch_messages,
+ **recall_kwargs,
)
if not recall:
result["counts"] = counts
@@ -37856,6 +37861,7 @@ class LlamaCppBackend:
conversation,
_before_fit,
branch_messages = _extend_live_branch(_before_fit),
+ query_messages = _request_branch,
thread_id = thread_id,
# Forge a tool exchange only when search_conversation is advertised.
style = (
@@ -40365,6 +40371,8 @@ class LlamaCppBackend:
conversation,
_before_final_fit,
branch_messages = _extend_live_branch(_before_final_fit),
+ # The budget nudge and loop notices are user turns; search for the user's question.
+ query_messages = _request_branch,
thread_id = thread_id,
# Inline: under tool_choice "none" a forged tool role breaks strict templates.
style = "inline",
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 3482c4d53e6..1db58f9b4e3 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -2951,6 +2951,7 @@ class InferenceOrchestrator:
fitted,
conversation,
branch_messages = live_branch or conversation,
+ query_messages = request_branch,
thread_id = thread_id,
# A forged tool exchange is only safe when the request advertises the tool.
style = "tool" if recall_offered else "inline",
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index 3dd59407071..544522e1ceb 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -14050,8 +14050,10 @@ def build_conversation_recall(
style: str = "tool",
top_k: int | None = None,
branch_messages: list[dict] | None = None,
+ query_messages: list[dict] | None = None,
) -> dict | None:
- """Not gated on rag_scope, since compaction happens with or without document RAG; style=inline avoids forged tool_calls, which break strict templates."""
+ """Not gated on rag_scope, since compaction happens with or without document RAG; style=inline avoids forged tool_calls, which break strict templates.
+ ``query_messages`` (the client's transcript) supplies the query when the branch carries a loop's own user-role notices."""
if not thread_id:
return None
try:
@@ -14062,7 +14064,8 @@ def build_conversation_recall(
return None
# The branch's last user turn: the loop conversation may end with an internal nudge.
- query = _last_user_text(branch_messages or conversation) or _last_user_text(conversation)
+ _asked = query_messages or branch_messages or conversation
+ query = _last_user_text(_asked) or _last_user_text(conversation)
if not query:
return None
# A nudge ("continue") retrieves nothing, so also query the last real instruction. Not for
@@ -14073,7 +14076,7 @@ def build_conversation_recall(
from core.inference import instruction_pin
thin = instruction_pin.is_thin_query(query)
if thin:
- _behind = branch_messages or conversation
+ _behind = _asked
anchor = instruction_pin.last_substantive_instruction(_behind)
if not anchor:
# Use is_thin_query, not a length rule: short prompts can still name something.
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index de87baa8a2f..3c342630ec5 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -28276,8 +28276,8 @@ def _fit_external_context(
branch: Optional[list[dict]] = None,
tool_loop: bool = False,
) -> tuple[list[dict], Optional[dict], Optional[int]]:
- """``branch`` is the client's request when ``messages`` is a tool loop's conversation, so recall
- queries the user's turn rather than a controller re-prompt the loop appended."""
+ """``branch`` is the client's request when ``messages`` is a tool loop's conversation: recall
+ queries the user's turn, not a controller re-prompt, and still filters on the loop's turns."""
from core.inference.context_window import (
estimate_message_tokens_without_unpriced_media,
estimate_messages_tokens_conservative,
@@ -28362,7 +28362,9 @@ def _fit_external_context(
recall = _archive_and_recall(
fitted,
messages,
- branch_messages = branch or messages,
+ # The loop's own conversation filters, or tool results this turn evicted are unrecallable.
+ branch_messages = messages,
+ query_messages = branch,
thread_id = str(thread_id) if thread_id else None,
style = "inline",
recall_done = not truncation.get("fits", True),
diff --git a/studio/backend/tests/test_conversation_recall_injection.py b/studio/backend/tests/test_conversation_recall_injection.py
index 314e00e5bc1..b0e10418c35 100644
--- a/studio/backend/tests/test_conversation_recall_injection.py
+++ b/studio/backend/tests/test_conversation_recall_injection.py
@@ -1115,6 +1115,58 @@ def test_the_forced_recall_searches_for_the_USERS_question(archived, monkeypatch
assert seen["query"] == "what was the VULPINE code from earlier"
+def test_the_forced_recall_skips_notices_a_tool_loop_added_to_its_branch(archived, monkeypatch):
+ """A tool loop's live branch ends on its own user-role notices; query_messages names the
+ user's question so the final pass does not search for the budget nudge (#16187)."""
+ seen = {}
+
+ def fake_recall(thread_id, query, **kwargs):
+ seen["query"] = query
+ seen["branch_messages"] = kwargs.get("branch_messages")
+ return ("earlier turn", [{"id": "1"}])
+
+ monkeypatch.setattr(conversation_archive, "recall", fake_recall)
+ request = [{"role": "user", "content": "what was the VULPINE code from earlier"}]
+ live = request + [
+ {"role": "assistant", "content": "", "tool_calls": [{"id": "c1"}]},
+ {"role": "tool", "content": "result", "tool_call_id": "c1"},
+ {"role": "user", "content": "You have used all available tool calls. Answer now."},
+ ]
+
+ tools_mod.build_conversation_recall(
+ live, THREAD, style = "inline", branch_messages = live, query_messages = request
+ )
+
+ assert seen["query"] == "what was the VULPINE code from earlier"
+ assert seen["branch_messages"] is live
+
+
+def test_archive_and_recall_hands_query_messages_to_the_recall(archived, monkeypatch):
+ seen = {}
+
+ def build(_conversation, _thread_id, **kwargs):
+ seen.update(kwargs)
+ return None
+
+ monkeypatch.setattr(tools_mod, "build_conversation_recall", build)
+ request = [{"role": "user", "content": "the question"}]
+ live = request + [{"role": "user", "content": "a loop notice"}]
+
+ llama_cpp._archive_and_recall(
+ live,
+ live,
+ thread_id = THREAD,
+ style = "inline",
+ recall_done = False,
+ recall_budget_tokens = 100_000,
+ branch_messages = live,
+ query_messages = request,
+ )
+
+ assert seen["query_messages"] is request
+ assert seen["branch_messages"] is live
+
+
def test_a_model_initiated_search_is_filtered_to_the_request_branch(archived, monkeypatch):
"""A model-initiated search must be branch-filtered too, or a replaced reply returns through it."""
seen = {}
diff --git a/studio/backend/tests/test_external_conversation_archive.py b/studio/backend/tests/test_external_conversation_archive.py
index d51a72ed8b1..7a357734ad6 100644
--- a/studio/backend/tests/test_external_conversation_archive.py
+++ b/studio/backend/tests/test_external_conversation_archive.py
@@ -214,9 +214,10 @@ def test_a_tool_loop_refit_recalls_for_the_users_turn_not_a_loop_reprompt(
thread_id,
*,
branch_messages = None,
+ query_messages = None,
**kwargs,
):
- seen.append(tools_mod._last_user_text(branch_messages or conversation))
+ seen.append(tools_mod._last_user_text(query_messages or branch_messages or conversation))
return None
monkeypatch.setattr(tools_mod, "build_conversation_recall", spy)
@@ -270,3 +271,39 @@ def test_no_compaction_nudge_when_tool_calls_are_disabled(monkeypatch):
system = " ".join(str(m.get("content")) for m in sent["messages"] if m.get("role") == "system")
assert "call search_conversation" not in system
+
+
+def test_a_tool_result_this_turn_evicted_is_recalled_for_the_users_question(archive_ready):
+ """The recall filters on the loop's turns and queries the client's, so the model keeps what it
+ found this turn after a fit evicts it (#16187)."""
+ finding = "The ZEBRA-4471 valve opens at 37 bar"
+ request = _chat()[:3] + [
+ {"role": "user", "content": "What pressure does the ZEBRA-4471 valve open at?"}
+ ]
+ _persist(THREAD, request)
+ loop = list(request)
+ for index in range(6):
+ call_id = f"c{index}"
+ loop.append(
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": call_id,
+ "type": "function",
+ "function": {"name": "web_search", "arguments": f'{{"query": "v{index}"}}'},
+ }
+ ],
+ }
+ )
+ found = f"{finding}. " if index == 0 else ""
+ loop.append({"role": "tool", "tool_call_id": call_id, "content": found + _FILLER})
+ loop.append({"role": "user", "content": "You have used all available tool calls. Answer now."})
+
+ fitted, truncation, _ = ri._fit_external_context(
+ loop, _payload(THREAD), branch = request, tool_loop = True
+ )
+
+ assert truncation["recalled_chunks"] > 0
+ assert finding in str(fitted)
diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.py
index b58009d6e9b..63d9b8395b9 100644
--- a/studio/backend/tests/test_llama_cpp_tool_loop.py
+++ b/studio/backend/tests/test_llama_cpp_tool_loop.py
@@ -3643,6 +3643,64 @@ def test_media_compaction_recall_recount_uses_the_stripped_view(monkeypatch, wit
assert payloads[0]["messages"][-1] == audio_turn
+def test_recall_searches_the_users_question_not_the_loops_own_notices(monkeypatch):
+ """Repeated calls leave user-role notices at the end of the loop; the compacting final pass
+ must still search the archive for what the user asked (#16187)."""
+ from core.rag import conversation_archive
+
+ rounds = 4
+ payloads: list[dict] = []
+ streams = [
+ [
+ _tool_call_sse("web_search", {"query": "a"}, f"c{r}a", index = 0),
+ _tool_call_sse("web_search", {"query": "b"}, f"c{r}b", index = 1),
+ _finish("tool_calls"),
+ _done(),
+ ]
+ for r in range(rounds)
+ ] + [[_sse({"content": "answer"}), _finish("stop"), _done()]]
+ backend = _make_backend(monkeypatch, streams, payloads)
+ backend._effective_context_length = 4000
+ monkeypatch.setattr(
+ backend,
+ "count_chat_tokens",
+ lambda candidate, *_args, **_kwargs: sum(
+ len(str(message.get("content", ""))) for message in candidate
+ ),
+ )
+ _record_tool_calls(monkeypatch, "R" * 500)
+ monkeypatch.setattr(conversation_archive, "enabled", lambda: True)
+ monkeypatch.setattr(conversation_archive, "archive_turns", lambda *_args, **_kwargs: 1)
+ monkeypatch.setattr(conversation_archive, "note_branch", lambda *_args, **_kwargs: None)
+ queries: list[str] = []
+
+ def fake_recall(_thread_id, query, **_kwargs):
+ queries.append(query)
+ return None
+
+ monkeypatch.setattr(conversation_archive, "recall", fake_recall)
+ question = "what did the VULPINE search find"
+
+ events = _run_tool_loop(
+ backend,
+ [
+ {"role": "user", "content": "u" * 1200},
+ {"role": "assistant", "content": "a" * 1200},
+ {"role": "user", "content": question},
+ ],
+ [_web_search_tool()],
+ max_tool_iterations = rounds,
+ max_tokens = 200,
+ context_overflow = "truncate_oldest",
+ thread_id = "recall-query",
+ )
+
+ assert any(event.get("type") == "context_truncated" for event in events)
+ final = payloads[-1]["messages"]
+ assert final[-1]["role"] == "user" and question not in final[-1]["content"]
+ assert queries and set(queries) == {question}
+
+
def test_a_respawn_refit_that_misses_its_target_still_archives_and_reports(monkeypatch):
"""A rescued respawn refit archives its evictions and emits metadata."""
# Captured once: the second pass would otherwise wrap the first pass's spy.
diff --git a/studio/backend/tests/test_mlx_context_compaction.py b/studio/backend/tests/test_mlx_context_compaction.py
index acdcf38f3ac..f387492908c 100644
--- a/studio/backend/tests/test_mlx_context_compaction.py
+++ b/studio/backend/tests/test_mlx_context_compaction.py
@@ -229,6 +229,8 @@ def test_the_tool_loop_refits_every_turn_and_keeps_the_question_past_a_reprompt(
ran = [m for m in last_fit["branch_messages"] if m["role"] == "tool"][-3:]
assert [m["content"][:8] for m in ran] == ["result 1", "result 2", "result 3"]
assert [m for m in last_fit["branch_messages"] if m["role"] == "user"] == [old, QUESTION]
+ # The recall query comes from the client's turns, never from a re-prompt the loop added (#16187).
+ assert [m for m in last_fit["query_messages"] if m["role"] == "user"][-1] is QUESTION
assert len(truncated) == fits and all(event["fits"] for event in truncated)
assert bool(truncated[0].get("checkpoint_started")) is (policy == "checkpoint")
assert last_fit["thread_id"] == "thread-1"