Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: search earlier turns for the user's question, not the tool loop's own notices (#16204)
Debojyoti Banerjeef4ee09876755
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex db6ff584af1..eb6884310db 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -2176,8 +2176,10 @@ def _archive_and_recall(branch_messages: Optional[list[dict]] = None,cancel_check: Optional[Callable[[], bool]] = None,background: bool = False,+ query_messages: Optional[list[dict]] = None,) -> dict:- """Never raises: any failure falls back to the plain fitted messages. Recall runs at most once per request or refits stack recall blocks."""+ """Never raises: any failure falls back to the plain fitted messages. Recall runs at most once per request or refits stack recall blocks.+ ``query_messages`` is the client's transcript: a tool loop's live branch ends on its own notices, not the user's question."""result = {"conversation": conversation,"events": [],@@ -2238,6 +2240,9 @@ def _archive_and_recall(breakattempt_k = max(1, attempt_k // 2)+ recall_kwargs = {"branch_messages": branch_messages}+ if query_messages is not None:+ recall_kwargs["query_messages"] = query_messages# Retries re-ask the same queries: encode each once and reuse lexical passes and branch checks.with query_vector_memo(), conversation_archive.recall_session():for index, k in enumerate(attempts):@@ -2246,7 +2251,7 @@ def _archive_and_recall(thread_id,style = style,top_k = k,- branch_messages = branch_messages,+ **recall_kwargs,)if not recall:result["counts"] = counts@@ -37856,6 +37861,7 @@ class LlamaCppBackend:conversation,_before_fit,branch_messages = _extend_live_branch(_before_fit),+ query_messages = _request_branch,thread_id = thread_id,# Forge a tool exchange only when search_conversation is advertised.style = (@@ -40365,6 +40371,8 @@ class LlamaCppBackend:conversation,_before_final_fit,branch_messages = _extend_live_branch(_before_final_fit),+ # The budget nudge and loop notices are user turns; search for the user's question.+ query_messages = _request_branch,thread_id = thread_id,# Inline: under tool_choice "none" a forged tool role breaks strict templates.style = "inline",diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 3482c4d53e6..1db58f9b4e3 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -2951,6 +2951,7 @@ class InferenceOrchestrator:fitted,conversation,branch_messages = live_branch or conversation,+ query_messages = request_branch,thread_id = thread_id,# A forged tool exchange is only safe when the request advertises the tool.style = "tool" if recall_offered else "inline",diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex 3dd59407071..544522e1ceb 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -14050,8 +14050,10 @@ def build_conversation_recall(style: str = "tool",top_k: int | None = None,branch_messages: list[dict] | None = None,+ query_messages: list[dict] | None = None,) -> dict | None:- """Not gated on rag_scope, since compaction happens with or without document RAG; style=inline avoids forged tool_calls, which break strict templates."""+ """Not gated on rag_scope, since compaction happens with or without document RAG; style=inline avoids forged tool_calls, which break strict templates.+ ``query_messages`` (the client's transcript) supplies the query when the branch carries a loop's own user-role notices."""if not thread_id:return Nonetry:@@ -14062,7 +14064,8 @@ def build_conversation_recall(return None# The branch's last user turn: the loop conversation may end with an internal nudge.- query = _last_user_text(branch_messages or conversation) or _last_user_text(conversation)+ _asked = query_messages or branch_messages or conversation+ query = _last_user_text(_asked) or _last_user_text(conversation)if not query:return None# A nudge ("continue") retrieves nothing, so also query the last real instruction. Not for@@ -14073,7 +14076,7 @@ def build_conversation_recall(from core.inference import instruction_pinthin = instruction_pin.is_thin_query(query)if thin:- _behind = branch_messages or conversation+ _behind = _askedanchor = instruction_pin.last_substantive_instruction(_behind)if not anchor:# Use is_thin_query, not a length rule: short prompts can still name something.diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex de87baa8a2f..3c342630ec5 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -28276,8 +28276,8 @@ def _fit_external_context(branch: Optional[list[dict]] = None,tool_loop: bool = False,) -> tuple[list[dict], Optional[dict], Optional[int]]:- """``branch`` is the client's request when ``messages`` is a tool loop's conversation, so recall- queries the user's turn rather than a controller re-prompt the loop appended."""+ """``branch`` is the client's request when ``messages`` is a tool loop's conversation: recall+ queries the user's turn, not a controller re-prompt, and still filters on the loop's turns."""from core.inference.context_window import (estimate_message_tokens_without_unpriced_media,estimate_messages_tokens_conservative,@@ -28362,7 +28362,9 @@ def _fit_external_context(recall = _archive_and_recall(fitted,messages,- branch_messages = branch or messages,+ # The loop's own conversation filters, or tool results this turn evicted are unrecallable.+ branch_messages = messages,+ query_messages = branch,thread_id = str(thread_id) if thread_id else None,style = "inline",recall_done = not truncation.get("fits", True),diff --git a/studio/backend/tests/test_conversation_recall_injection.py b/studio/backend/tests/test_conversation_recall_injection.pyindex 314e00e5bc1..b0e10418c35 100644--- a/studio/backend/tests/test_conversation_recall_injection.py+++ b/studio/backend/tests/test_conversation_recall_injection.py@@ -1115,6 +1115,58 @@ def test_the_forced_recall_searches_for_the_USERS_question(archived, monkeypatchassert seen["query"] == "what was the VULPINE code from earlier"+def test_the_forced_recall_skips_notices_a_tool_loop_added_to_its_branch(archived, monkeypatch):+ """A tool loop's live branch ends on its own user-role notices; query_messages names the+ user's question so the final pass does not search for the budget nudge (#16187)."""+ seen = {}++ def fake_recall(thread_id, query, **kwargs):+ seen["query"] = query+ seen["branch_messages"] = kwargs.get("branch_messages")+ return ("earlier turn", [{"id": "1"}])++ monkeypatch.setattr(conversation_archive, "recall", fake_recall)+ request = [{"role": "user", "content": "what was the VULPINE code from earlier"}]+ live = request + [+ {"role": "assistant", "content": "", "tool_calls": [{"id": "c1"}]},+ {"role": "tool", "content": "result", "tool_call_id": "c1"},+ {"role": "user", "content": "You have used all available tool calls. Answer now."},+ ]++ tools_mod.build_conversation_recall(+ live, THREAD, style = "inline", branch_messages = live, query_messages = request+ )++ assert seen["query"] == "what was the VULPINE code from earlier"+ assert seen["branch_messages"] is live+++def test_archive_and_recall_hands_query_messages_to_the_recall(archived, monkeypatch):+ seen = {}++ def build(_conversation, _thread_id, **kwargs):+ seen.update(kwargs)+ return None++ monkeypatch.setattr(tools_mod, "build_conversation_recall", build)+ request = [{"role": "user", "content": "the question"}]+ live = request + [{"role": "user", "content": "a loop notice"}]++ llama_cpp._archive_and_recall(+ live,+ live,+ thread_id = THREAD,+ style = "inline",+ recall_done = False,+ recall_budget_tokens = 100_000,+ branch_messages = live,+ query_messages = request,+ )++ assert seen["query_messages"] is request+ assert seen["branch_messages"] is live++def test_a_model_initiated_search_is_filtered_to_the_request_branch(archived, monkeypatch):"""A model-initiated search must be branch-filtered too, or a replaced reply returns through it."""seen = {}diff --git a/studio/backend/tests/test_external_conversation_archive.py b/studio/backend/tests/test_external_conversation_archive.pyindex d51a72ed8b1..7a357734ad6 100644--- a/studio/backend/tests/test_external_conversation_archive.py+++ b/studio/backend/tests/test_external_conversation_archive.py@@ -214,9 +214,10 @@ def test_a_tool_loop_refit_recalls_for_the_users_turn_not_a_loop_reprompt(thread_id,*,branch_messages = None,+ query_messages = None,**kwargs,):- seen.append(tools_mod._last_user_text(branch_messages or conversation))+ seen.append(tools_mod._last_user_text(query_messages or branch_messages or conversation))return Nonemonkeypatch.setattr(tools_mod, "build_conversation_recall", spy)@@ -270,3 +271,39 @@ def test_no_compaction_nudge_when_tool_calls_are_disabled(monkeypatch):system = " ".join(str(m.get("content")) for m in sent["messages"] if m.get("role") == "system")assert "call search_conversation" not in system+++def test_a_tool_result_this_turn_evicted_is_recalled_for_the_users_question(archive_ready):+ """The recall filters on the loop's turns and queries the client's, so the model keeps what it+ found this turn after a fit evicts it (#16187)."""+ finding = "The ZEBRA-4471 valve opens at 37 bar"+ request = _chat()[:3] + [+ {"role": "user", "content": "What pressure does the ZEBRA-4471 valve open at?"}+ ]+ _persist(THREAD, request)+ loop = list(request)+ for index in range(6):+ call_id = f"c{index}"+ loop.append(+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": call_id,+ "type": "function",+ "function": {"name": "web_search", "arguments": f'{{"query": "v{index}"}}'},+ }+ ],+ }+ )+ found = f"{finding}. " if index == 0 else ""+ loop.append({"role": "tool", "tool_call_id": call_id, "content": found + _FILLER})+ loop.append({"role": "user", "content": "You have used all available tool calls. Answer now."})++ fitted, truncation, _ = ri._fit_external_context(+ loop, _payload(THREAD), branch = request, tool_loop = True+ )++ assert truncation["recalled_chunks"] > 0+ assert finding in str(fitted)diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.pyindex b58009d6e9b..63d9b8395b9 100644--- a/studio/backend/tests/test_llama_cpp_tool_loop.py+++ b/studio/backend/tests/test_llama_cpp_tool_loop.py@@ -3643,6 +3643,64 @@ def test_media_compaction_recall_recount_uses_the_stripped_view(monkeypatch, witassert payloads[0]["messages"][-1] == audio_turn+def test_recall_searches_the_users_question_not_the_loops_own_notices(monkeypatch):+ """Repeated calls leave user-role notices at the end of the loop; the compacting final pass+ must still search the archive for what the user asked (#16187)."""+ from core.rag import conversation_archive++ rounds = 4+ payloads: list[dict] = []+ streams = [+ [+ _tool_call_sse("web_search", {"query": "a"}, f"c{r}a", index = 0),+ _tool_call_sse("web_search", {"query": "b"}, f"c{r}b", index = 1),+ _finish("tool_calls"),+ _done(),+ ]+ for r in range(rounds)+ ] + [[_sse({"content": "answer"}), _finish("stop"), _done()]]+ backend = _make_backend(monkeypatch, streams, payloads)+ backend._effective_context_length = 4000+ monkeypatch.setattr(+ backend,+ "count_chat_tokens",+ lambda candidate, *_args, **_kwargs: sum(+ len(str(message.get("content", ""))) for message in candidate+ ),+ )+ _record_tool_calls(monkeypatch, "R" * 500)+ monkeypatch.setattr(conversation_archive, "enabled", lambda: True)+ monkeypatch.setattr(conversation_archive, "archive_turns", lambda *_args, **_kwargs: 1)+ monkeypatch.setattr(conversation_archive, "note_branch", lambda *_args, **_kwargs: None)+ queries: list[str] = []++ def fake_recall(_thread_id, query, **_kwargs):+ queries.append(query)+ return None++ monkeypatch.setattr(conversation_archive, "recall", fake_recall)+ question = "what did the VULPINE search find"++ events = _run_tool_loop(+ backend,+ [+ {"role": "user", "content": "u" * 1200},+ {"role": "assistant", "content": "a" * 1200},+ {"role": "user", "content": question},+ ],+ [_web_search_tool()],+ max_tool_iterations = rounds,+ max_tokens = 200,+ context_overflow = "truncate_oldest",+ thread_id = "recall-query",+ )++ assert any(event.get("type") == "context_truncated" for event in events)+ final = payloads[-1]["messages"]+ assert final[-1]["role"] == "user" and question not in final[-1]["content"]+ assert queries and set(queries) == {question}++def test_a_respawn_refit_that_misses_its_target_still_archives_and_reports(monkeypatch):"""A rescued respawn refit archives its evictions and emits metadata."""# Captured once: the second pass would otherwise wrap the first pass's spy.diff --git a/studio/backend/tests/test_mlx_context_compaction.py b/studio/backend/tests/test_mlx_context_compaction.pyindex acdcf38f3ac..f387492908c 100644--- a/studio/backend/tests/test_mlx_context_compaction.py+++ b/studio/backend/tests/test_mlx_context_compaction.py@@ -229,6 +229,8 @@ def test_the_tool_loop_refits_every_turn_and_keeps_the_question_past_a_reprompt(ran = [m for m in last_fit["branch_messages"] if m["role"] == "tool"][-3:]assert [m["content"][:8] for m in ran] == ["result 1", "result 2", "result 3"]assert [m for m in last_fit["branch_messages"] if m["role"] == "user"] == [old, QUESTION]+ # The recall query comes from the client's turns, never from a re-prompt the loop added (#16187).+ assert [m for m in last_fit["query_messages"] if m["role"] == "user"][-1] is QUESTIONassert len(truncated) == fits and all(event["fits"] for event in truncated)assert bool(truncated[0].get("checkpoint_started")) is (policy == "checkpoint")assert last_fit["thread_id"] == "thread-1"