Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep turns an external provider's compaction drops searchable (#14731)

Debojyoti Banerjee744ee65f717e
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 97ff35a0f0e..5f6817e7dce 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -30455,16 +30455,23 @@ def _fit_external_context(
payload,
*,
tools: Optional[list[dict]] = None,
+ branch: Optional[list[dict]] = None,
+ tool_loop: bool = False,
) -> tuple[list[dict], Optional[dict], Optional[int]]:
+ """``branch`` is the client's request when ``messages`` is a tool loop's conversation, so recall
+ queries the user's turn rather than a controller re-prompt the loop appended."""
from core.inference.context_window import (
estimate_message_tokens_without_unpriced_media,
estimate_messages_tokens_conservative,
messages_without_unpriced_media,
prompt_budget,
+ retrieval_budget,
)
from core.inference.llama_cpp import (
+ _archive_and_recall,
_boundary_metadata,
_compaction_fit_kwargs,
+ _conversation_recall_reserve,
_fit_with_instruction_pins,
_keeps_compaction_boundary,
_records_boundary,
@@ -30505,7 +30512,7 @@ def _fit_external_context(
policy = _request_context_policy(payload)
ratio = _request_compaction_headroom_ratio(payload)
thread_id = payload.thread_id
- # No archive to search here, so a checkpoint reset may not start and the fit stays rolling.
+ # Recall below is inline and rolling (every evicting fit), so a checkpoint reset may not start.
sticky, sticky_is_checkpoint = _sticky_compaction_state(
thread_id,
messages,
@@ -30520,13 +30527,37 @@ def _fit_external_context(
count_tokens = _count,
keeps_boundary = _keeps_compaction_boundary(thread_id),
recall_offered = False,
+ reserve_tokens = _conversation_recall_reserve(thread_id),
sticky_dropped = sticky,
sticky_is_checkpoint = sticky_is_checkpoint,
estimate_message = estimate_message_tokens_without_unpriced_media,
**_compaction_fit_kwargs(policy, ratio),
)
if truncation and _records_boundary(truncation):
+ # From the fit, not the recalled list: inline recall swaps the latest user dict and the
+ # anchor is matched by identity.
truncation = {**truncation, **_boundary_metadata(fitted, messages, ratio)}
+ if truncation:
+ # The catalogue is chosen before this fit, so the turn that first evicts has no
+ # search_conversation: recall inline, as the local plain path does, or a question about
+ # an evicted attachment on that turn gets "I have no record of it".
+ recall = _archive_and_recall(
+ fitted,
+ messages,
+ branch_messages = branch or messages,
+ thread_id = str(thread_id) if thread_id else None,
+ style = "inline",
+ recall_done = not truncation.get("fits", True),
+ recall_budget_tokens = retrieval_budget(
+ context_length,
+ max_tokens,
+ truncation.get("prompt_tokens_after") or 0,
+ reply_returns = tool_loop,
+ ),
+ count_tokens = _count,
+ )
+ fitted = recall["conversation"]
+ truncation = {**truncation, **recall["counts"]}
return fitted, truncation, max_tokens
@@ -31319,6 +31350,14 @@ async def _proxy_to_external_provider(
full_access = bool(payload.bypass_permissions),
full_access_only = True,
)
+ # The compaction nudge is not part of the general nudge: once the thread has an archive
+ # the catalogue carries search_conversation, and without this the model is never told
+ # that turns it cannot see exist, so it answers that it has no record of them. Not when
+ # no call can happen: the loop withdraws the catalogue and inline recall still runs.
+ if not _tool_calls_are_disabled(payload):
+ _external_nudge = _apply_compaction_nudge(
+ _external_nudge, external_studio_tools, payload = payload
+ )
_served_template = None
if (
provider_type == "llama_cpp"
@@ -31366,6 +31405,7 @@ async def _proxy_to_external_provider(
_external_context_fitter = None
_compaction_fallback = None
if payload.compaction_threshold and _rolling_context_policy(payload) is not None:
+ _request_branch = chat_messages
async def _external_context_fitter(messages):
nonlocal _external_truncation
@@ -31374,6 +31414,8 @@ async def _proxy_to_external_provider(
messages,
payload,
tools = _external_fit_tools,
+ branch = _request_branch,
+ tool_loop = run_studio_tool_loop and not _tool_calls_are_disabled(payload),
)
fitted = _date_a_fitted_system_turn(fitted, request, run_studio_tool_loop)
if truncation and truncation.get("dropped_messages"):
diff --git a/studio/backend/routes/managed_engine_chat.py b/studio/backend/routes/managed_engine_chat.py
index fccf299233e..37d4b5a2cd8 100644
--- a/studio/backend/routes/managed_engine_chat.py
+++ b/studio/backend/routes/managed_engine_chat.py
@@ -103,7 +103,13 @@ def _with_reasoning_content(line: str) -> str:
return "data: " + json.dumps(event) if changed else line
-def _message_fitter(payload, engine, public, tools):
+def _message_fitter(
+ payload,
+ engine,
+ public,
+ tools,
+ studio_loop = False,
+):
"""Auto-compact to the vLLM engine's window; the exact count refuses tools, so this estimates."""
from routes import inference as api
@@ -123,9 +129,21 @@ def _message_fitter(payload, engine, public, tools):
}
)
+ # The first fit sees the client's branch; later ones see the loop's conversation, which can end
+ # on a controller re-prompt that recall must not query for.
+ branch: list = []
+ tool_loop = studio_loop and not api._tool_calls_are_disabled(payload)
+
async def fit(messages):
+ if not branch:
+ branch.extend(messages)
fitted, truncation, max_tokens = await asyncio.to_thread(
- api._fit_external_context, messages, fit_payload, tools = tools
+ api._fit_external_context,
+ messages,
+ fit_payload,
+ tools = tools,
+ branch = branch,
+ tool_loop = tool_loop,
)
line = None
if truncation and truncation.get("dropped_messages"):
@@ -223,6 +241,8 @@ async def managed_tool_chat(
full_access = bool(payload.bypass_permissions),
)
nudge = await api._apply_rag_nudge(nudge, tools, rag_scope = payload.rag_scope)
+ if not api._tool_calls_are_disabled(payload):
+ nudge = api._apply_compaction_nudge(nudge, tools, payload = payload)
if nudge:
messages = api._append_to_system_message(messages, nudge)
try:
@@ -287,7 +307,9 @@ async def managed_tool_chat(
engine,
body,
preserve_thinking = payload.preserve_thinking,
- message_fitter = _message_fitter(payload, engine, public, tools or payload.tools),
+ message_fitter = _message_fitter(
+ payload, engine, public, tools or payload.tools, studio_loop = bool(tools)
+ ),
)
stripper = api.ServerToolCallStripper()
stream = None
diff --git a/studio/backend/tests/test_custom_provider_api_type.py b/studio/backend/tests/test_custom_provider_api_type.py
index f1d8471b628..f3a5cf70768 100644
--- a/studio/backend/tests/test_custom_provider_api_type.py
+++ b/studio/backend/tests/test_custom_provider_api_type.py
@@ -246,6 +246,7 @@ def test_non_stream_route_reports_local_context_truncation(monkeypatch, server_c
_payload,
*,
tools = None,
+ **_kwargs,
):
assert tools is None
return messages[-1:], truncation, 128
diff --git a/studio/backend/tests/test_external_auto_compaction.py b/studio/backend/tests/test_external_auto_compaction.py
index 9d1264dc559..4e2a6d6794b 100644
--- a/studio/backend/tests/test_external_auto_compaction.py
+++ b/studio/backend/tests/test_external_auto_compaction.py
@@ -659,6 +659,7 @@ def test_a_checkpoint_turn_the_fit_creates_still_carries_the_date(monkeypatch):
messages,
payload,
tools = None,
+ **_kwargs,
):
return [{"role": "system", "content": "Earlier: a summary."}, *messages], None, None
diff --git a/studio/backend/tests/test_external_conversation_archive.py b/studio/backend/tests/test_external_conversation_archive.py
new file mode 100644
index 00000000000..d51a72ed8b1
--- /dev/null
+++ b/studio/backend/tests/test_external_conversation_archive.py
@@ -0,0 +1,272 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Turns an external provider's fit evicts stay reachable through search_conversation."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+from starlette.requests import Request
+
+import routes.inference as ri
+from core.inference import tools as tools_mod
+from core.rag import conversation_archive
+from models.inference import ChatCompletionRequest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_external_auto_compaction import _proxy # noqa: E402
+
+THREAD = "thread-external-archive"
+_FILLER = "word " * 400
+_DOCUMENT = "[PDF: referto.pdf]\nReparto di cardiologia, telefono 055 794 3321."
+
+
+def _chat() -> list[dict]:
+ messages: list[dict] = [{"role": "system", "content": "Answer briefly."}]
+ messages.append({"role": "user", "content": f"Read this document.\n\n{_DOCUMENT}"})
+ messages.append({"role": "assistant", "content": "I have read the cardiology report."})
+ for index in range(10):
+ messages.append({"role": "user", "content": f"question {index} {_FILLER}"})
+ messages.append({"role": "assistant", "content": f"answer {index} {_FILLER}"})
+ messages.append({"role": "user", "content": "Was there a phone number on the report?"})
+ return messages
+
+
+def _persist(thread_id: str, messages: list[dict]) -> None:
+ from storage import studio_db
+ studio_db.upsert_chat_thread(
+ {
+ "id": thread_id,
+ "title": "t",
+ "modelType": "base",
+ "modelId": "a-model",
+ "createdAt": 1,
+ }
+ )
+ for index, message in enumerate(m for m in messages if m["role"] != "system"):
+ studio_db.upsert_chat_message(
+ {
+ "id": f"{thread_id}-{index}",
+ "threadId": thread_id,
+ "role": message["role"],
+ "content": [{"type": "text", "text": message["content"]}],
+ "createdAt": index + 2,
+ }
+ )
+
+
+def _payload(thread_id: str | None) -> ChatCompletionRequest:
+ return ChatCompletionRequest(
+ messages = [{"role": "user", "content": "latest"}],
+ provider_type = "openrouter",
+ model = "a-model",
+ thread_id = thread_id,
+ context_overflow = "truncate_oldest",
+ compaction_threshold = 3_000,
+ context_window = 4_000,
+ max_tokens = 512,
+ )
+
+
+def archive_ready(rag_home, rag_conn, stub_embeddings):
+ return rag_conn
+
+
+def test_an_external_fit_archives_the_turns_it_evicts(archive_ready):
+ messages = _chat()
+ _persist(THREAD, messages)
+
+ fitted, truncation, _ = ri._fit_external_context(messages, _payload(THREAD))
+
+ assert truncation and truncation["dropped_messages"] > 0
+ assert all(_DOCUMENT not in str(m.get("content")) for m in fitted[:-1])
+ assert conversation_archive.has_archive(THREAD)
+
+
+def test_the_evicting_fit_recalls_the_document_into_the_latest_turn(archive_ready):
+ # The catalogue is fixed before the fit, so on this turn the model has no search tool.
+ messages = _chat()
+ _persist(THREAD, messages)
+
+ fitted, truncation, _ = ri._fit_external_context(messages, _payload(THREAD))
+
+ assert truncation["recalled_chunks"] > 0
+ assert "055 794 3321" in str(fitted[-1]["content"])
+ assert "Was there a phone number on the report?" in str(fitted[-1]["content"])
+ # The rewritten latest turn must not hide where the boundary landed.
+ assert truncation["boundary_anchor"]
+
+
+def test_an_attached_document_from_an_evicted_turn_is_searchable(archive_ready):
+ messages = _chat()
+ _persist(THREAD, messages)
+
+ ri._fit_external_context(messages, _payload(THREAD))
+ result = tools_mod.execute_tool(
+ "search_conversation",
+ {"query": "cardiology report phone number"},
+ thread_id = THREAD,
+ timeout = None,
+ )
+
+ assert "055 794 3321" in result
+
+
+def test_a_fit_without_a_thread_archives_nothing(archive_ready):
+ fitted, truncation, _ = ri._fit_external_context(_chat(), _payload(None))
+
+ assert truncation and truncation["dropped_messages"] > 0
+ assert not conversation_archive.has_archive(THREAD)
+
+
+def test_a_chat_that_fits_archives_nothing(archive_ready):
+ messages = [
+ {"role": "user", "content": f"Read this document.\n\n{_DOCUMENT}"},
+ {"role": "assistant", "content": "Done."},
+ {"role": "user", "content": "Thanks."},
+ ]
+ _persist(THREAD, messages)
+
+ fitted, truncation, _ = ri._fit_external_context(messages, _payload(THREAD))
+
+ assert truncation is None
+ assert fitted is messages
+ assert not conversation_archive.has_archive(THREAD)
+
+
+def test_an_archive_failure_still_returns_the_fitted_chat(archive_ready, monkeypatch):
+ messages = _chat()
+ _persist(THREAD, messages)
+
+ def broken(*args, **kwargs):
+ raise RuntimeError("store unavailable")
+
+ monkeypatch.setattr(conversation_archive, "archive_turns", broken)
+
+ fitted, truncation, _ = ri._fit_external_context(messages, _payload(THREAD))
+
+ assert truncation and truncation["dropped_messages"] > 0
+ assert fitted[-1]["content"] == "Was there a phone number on the report?"
+ assert "recalled_chunks" not in truncation
+
+
+def test_an_external_tool_chat_with_an_archive_is_told_about_compaction(monkeypatch):
+ monkeypatch.setattr(ri, "_thread_has_conversation_archive", lambda thread_id: True)
+
+ async def connected(self) -> bool:
+ return False
+
+ # The tool loop polls for a disconnect and would stop before reaching the provider.
+ monkeypatch.setattr(Request, "is_disconnected", connected)
+ _, sent = _proxy(
+ "openrouter",
+ [{"role": "user", "content": "Was there a phone number on the report?"}],
+ thread_id = THREAD,
+ enable_tools = True,
+ enabled_tools = ["python"],
+ permission_mode = "off",
+ )
+
+ names = {tool["function"]["name"] for tool in sent["tools"]}
+ system = " ".join(str(m.get("content")) for m in sent["messages"] if m.get("role") == "system")
+ assert "search_conversation" in names
+ assert "call search_conversation before answering" in system
+
+
+def test_an_external_tool_chat_without_an_archive_gets_no_compaction_nudge(monkeypatch):
+ monkeypatch.setattr(ri, "_thread_has_conversation_archive", lambda thread_id: False)
+
+ async def connected(self) -> bool:
+ return False
+
+ monkeypatch.setattr(Request, "is_disconnected", connected)
+ _, sent = _proxy(
+ "openrouter",
+ [{"role": "user", "content": "hello"}],
+ thread_id = THREAD,
+ enable_tools = True,
+ enabled_tools = ["python"],
+ permission_mode = "off",
+ )
+
+ system = " ".join(str(m.get("content")) for m in sent["messages"] if m.get("role") == "system")
+ assert "search_conversation" not in system
+
+
+def test_a_tool_loop_refit_recalls_for_the_users_turn_not_a_loop_reprompt(
+ archive_ready, monkeypatch
+):
+ branch = _chat()
+ _persist(THREAD, branch)
+ loop = [
+ *branch,
+ {"role": "assistant", "content": "I will look it up."},
+ {"role": "user", "content": "You described a plan without acting. Call a tool now."},
+ ]
+ seen = []
+
+ def spy(
+ conversation,
+ thread_id,
+ *,
+ branch_messages = None,
+ **kwargs,
+ ):
+ seen.append(tools_mod._last_user_text(branch_messages or conversation))
+ return None
+
+ monkeypatch.setattr(tools_mod, "build_conversation_recall", spy)
+
+ ri._fit_external_context(loop, _payload(THREAD), branch = branch)
+
+ assert seen == ["Was there a phone number on the report?"]
+
+
+def test_caller_owned_tools_do_not_halve_the_recall_budget(archive_ready, monkeypatch):
+ from core.inference import context_window
+
+ messages = _chat()
+ _persist(THREAD, messages)
+ seen = []
+ real = context_window.retrieval_budget
+
+ def spy(
+ *args,
+ reply_returns = False,
+ **kwargs,
+ ):
+ seen.append(reply_returns)
+ return real(*args, reply_returns = reply_returns, **kwargs)
+
+ monkeypatch.setattr(context_window, "retrieval_budget", spy)
+ caller_tools = [{"type": "function", "function": {"name": "lookup", "parameters": {}}}]
+
+ ri._fit_external_context(messages, _payload(THREAD), tools = caller_tools)
+ ri._fit_external_context(messages, _payload(THREAD), tools = caller_tools, tool_loop = True)
+
+ assert seen == [False, True]
+
+
+def test_no_compaction_nudge_when_tool_calls_are_disabled(monkeypatch):
+ monkeypatch.setattr(ri, "_thread_has_conversation_archive", lambda thread_id: True)
+
+ async def connected(self) -> bool:
+ return False
+
+ monkeypatch.setattr(Request, "is_disconnected", connected)
+ _, sent = _proxy(
+ "openrouter",
+ [{"role": "user", "content": "Was there a phone number on the report?"}],
+ thread_id = THREAD,
+ enable_tools = True,
+ enabled_tools = ["python"],
+ permission_mode = "off",
+ tool_choice = "none",
+ )
+
+ system = " ".join(str(m.get("content")) for m in sent["messages"] if m.get("role") == "system")
+ assert "call search_conversation" not in system
diff --git a/studio/backend/tests/test_managed_engine_reasoning.py b/studio/backend/tests/test_managed_engine_reasoning.py
index b98aada6e65..b6afabbd2a8 100644
--- a/studio/backend/tests/test_managed_engine_reasoning.py
+++ b/studio/backend/tests/test_managed_engine_reasoning.py
@@ -348,6 +348,40 @@ def test_tool_loop_turns_are_fitted_to_the_engine_window():
)
+def test_a_managed_refit_recalls_with_the_first_branch_it_saw(monkeypatch):
+ import asyncio
+
+ from routes import inference as api
+ from routes.managed_engine_chat import _message_fitter
+
+ seen = []
+
+ def fake_fit(
+ messages,
+ payload,
+ *,
+ tools = None,
+ branch = None,
+ tool_loop = False,
+ ):
+ seen.append((list(branch), tool_loop))
+ return messages, None, 64
+
+ monkeypatch.setattr(api, "_fit_external_context", fake_fit)
+ first = [{"role": "user", "content": "Was there a phone number?"}]
+ later = [
+ *first,
+ {"role": "assistant", "content": "Let me check."},
+ {"role": "user", "content": "You described a plan without acting."},
+ ]
+ fit = _message_fitter(
+ _tool_payload(context_overflow = "truncate_oldest"), _Engine(), "m", None, studio_loop = True
+ )
+ asyncio.run(fit(first))
+ asyncio.run(fit(later))
+ assert seen == [(first, True), (first, True)]
+
+
def test_tool_loop_honors_the_server_default_overflow(monkeypatch):
from routes.managed_engine_chat import _message_fitter