Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep turns an external provider's compaction drops searchable (#14731)
Debojyoti Banerjee744ee65f717e
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 97ff35a0f0e..5f6817e7dce 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -30455,16 +30455,23 @@ def _fit_external_context(payload,*,tools: Optional[list[dict]] = None,+ branch: Optional[list[dict]] = None,+ tool_loop: bool = False,) -> tuple[list[dict], Optional[dict], Optional[int]]:+ """``branch`` is the client's request when ``messages`` is a tool loop's conversation, so recall+ queries the user's turn rather than a controller re-prompt the loop appended."""from core.inference.context_window import (estimate_message_tokens_without_unpriced_media,estimate_messages_tokens_conservative,messages_without_unpriced_media,prompt_budget,+ retrieval_budget,)from core.inference.llama_cpp import (+ _archive_and_recall,_boundary_metadata,_compaction_fit_kwargs,+ _conversation_recall_reserve,_fit_with_instruction_pins,_keeps_compaction_boundary,_records_boundary,@@ -30505,7 +30512,7 @@ def _fit_external_context(policy = _request_context_policy(payload)ratio = _request_compaction_headroom_ratio(payload)thread_id = payload.thread_id- # No archive to search here, so a checkpoint reset may not start and the fit stays rolling.+ # Recall below is inline and rolling (every evicting fit), so a checkpoint reset may not start.sticky, sticky_is_checkpoint = _sticky_compaction_state(thread_id,messages,@@ -30520,13 +30527,37 @@ def _fit_external_context(count_tokens = _count,keeps_boundary = _keeps_compaction_boundary(thread_id),recall_offered = False,+ reserve_tokens = _conversation_recall_reserve(thread_id),sticky_dropped = sticky,sticky_is_checkpoint = sticky_is_checkpoint,estimate_message = estimate_message_tokens_without_unpriced_media,**_compaction_fit_kwargs(policy, ratio),)if truncation and _records_boundary(truncation):+ # From the fit, not the recalled list: inline recall swaps the latest user dict and the+ # anchor is matched by identity.truncation = {**truncation, **_boundary_metadata(fitted, messages, ratio)}+ if truncation:+ # The catalogue is chosen before this fit, so the turn that first evicts has no+ # search_conversation: recall inline, as the local plain path does, or a question about+ # an evicted attachment on that turn gets "I have no record of it".+ recall = _archive_and_recall(+ fitted,+ messages,+ branch_messages = branch or messages,+ thread_id = str(thread_id) if thread_id else None,+ style = "inline",+ recall_done = not truncation.get("fits", True),+ recall_budget_tokens = retrieval_budget(+ context_length,+ max_tokens,+ truncation.get("prompt_tokens_after") or 0,+ reply_returns = tool_loop,+ ),+ count_tokens = _count,+ )+ fitted = recall["conversation"]+ truncation = {**truncation, **recall["counts"]}return fitted, truncation, max_tokens@@ -31319,6 +31350,14 @@ async def _proxy_to_external_provider(full_access = bool(payload.bypass_permissions),full_access_only = True,)+ # The compaction nudge is not part of the general nudge: once the thread has an archive+ # the catalogue carries search_conversation, and without this the model is never told+ # that turns it cannot see exist, so it answers that it has no record of them. Not when+ # no call can happen: the loop withdraws the catalogue and inline recall still runs.+ if not _tool_calls_are_disabled(payload):+ _external_nudge = _apply_compaction_nudge(+ _external_nudge, external_studio_tools, payload = payload+ )_served_template = Noneif (provider_type == "llama_cpp"@@ -31366,6 +31405,7 @@ async def _proxy_to_external_provider(_external_context_fitter = None_compaction_fallback = Noneif payload.compaction_threshold and _rolling_context_policy(payload) is not None:+ _request_branch = chat_messagesasync def _external_context_fitter(messages):nonlocal _external_truncation@@ -31374,6 +31414,8 @@ async def _proxy_to_external_provider(messages,payload,tools = _external_fit_tools,+ branch = _request_branch,+ tool_loop = run_studio_tool_loop and not _tool_calls_are_disabled(payload),)fitted = _date_a_fitted_system_turn(fitted, request, run_studio_tool_loop)if truncation and truncation.get("dropped_messages"):diff --git a/studio/backend/routes/managed_engine_chat.py b/studio/backend/routes/managed_engine_chat.pyindex fccf299233e..37d4b5a2cd8 100644--- a/studio/backend/routes/managed_engine_chat.py+++ b/studio/backend/routes/managed_engine_chat.py@@ -103,7 +103,13 @@ def _with_reasoning_content(line: str) -> str:return "data: " + json.dumps(event) if changed else line-def _message_fitter(payload, engine, public, tools):+def _message_fitter(+ payload,+ engine,+ public,+ tools,+ studio_loop = False,+):"""Auto-compact to the vLLM engine's window; the exact count refuses tools, so this estimates."""from routes import inference as api@@ -123,9 +129,21 @@ def _message_fitter(payload, engine, public, tools):})+ # The first fit sees the client's branch; later ones see the loop's conversation, which can end+ # on a controller re-prompt that recall must not query for.+ branch: list = []+ tool_loop = studio_loop and not api._tool_calls_are_disabled(payload)+async def fit(messages):+ if not branch:+ branch.extend(messages)fitted, truncation, max_tokens = await asyncio.to_thread(- api._fit_external_context, messages, fit_payload, tools = tools+ api._fit_external_context,+ messages,+ fit_payload,+ tools = tools,+ branch = branch,+ tool_loop = tool_loop,)line = Noneif truncation and truncation.get("dropped_messages"):@@ -223,6 +241,8 @@ async def managed_tool_chat(full_access = bool(payload.bypass_permissions),)nudge = await api._apply_rag_nudge(nudge, tools, rag_scope = payload.rag_scope)+ if not api._tool_calls_are_disabled(payload):+ nudge = api._apply_compaction_nudge(nudge, tools, payload = payload)if nudge:messages = api._append_to_system_message(messages, nudge)try:@@ -287,7 +307,9 @@ async def managed_tool_chat(engine,body,preserve_thinking = payload.preserve_thinking,- message_fitter = _message_fitter(payload, engine, public, tools or payload.tools),+ message_fitter = _message_fitter(+ payload, engine, public, tools or payload.tools, studio_loop = bool(tools)+ ),)stripper = api.ServerToolCallStripper()stream = Nonediff --git a/studio/backend/tests/test_custom_provider_api_type.py b/studio/backend/tests/test_custom_provider_api_type.pyindex f1d8471b628..f3a5cf70768 100644--- a/studio/backend/tests/test_custom_provider_api_type.py+++ b/studio/backend/tests/test_custom_provider_api_type.py@@ -246,6 +246,7 @@ def test_non_stream_route_reports_local_context_truncation(monkeypatch, server_c_payload,*,tools = None,+ **_kwargs,):assert tools is Nonereturn messages[-1:], truncation, 128diff --git a/studio/backend/tests/test_external_auto_compaction.py b/studio/backend/tests/test_external_auto_compaction.pyindex 9d1264dc559..4e2a6d6794b 100644--- a/studio/backend/tests/test_external_auto_compaction.py+++ b/studio/backend/tests/test_external_auto_compaction.py@@ -659,6 +659,7 @@ def test_a_checkpoint_turn_the_fit_creates_still_carries_the_date(monkeypatch):messages,payload,tools = None,+ **_kwargs,):return [{"role": "system", "content": "Earlier: a summary."}, *messages], None, Nonediff --git a/studio/backend/tests/test_external_conversation_archive.py b/studio/backend/tests/test_external_conversation_archive.pynew file mode 100644index 00000000000..d51a72ed8b1--- /dev/null+++ b/studio/backend/tests/test_external_conversation_archive.py@@ -0,0 +1,272 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Turns an external provider's fit evicts stay reachable through search_conversation."""++from __future__ import annotations++import sys+from pathlib import Path++import pytest+from starlette.requests import Request++import routes.inference as ri+from core.inference import tools as tools_mod+from core.rag import conversation_archive+from models.inference import ChatCompletionRequest++sys.path.insert(0, str(Path(__file__).resolve().parent))+from test_external_auto_compaction import _proxy # noqa: E402++THREAD = "thread-external-archive"+_FILLER = "word " * 400+_DOCUMENT = "[PDF: referto.pdf]\nReparto di cardiologia, telefono 055 794 3321."+++def _chat() -> list[dict]:+ messages: list[dict] = [{"role": "system", "content": "Answer briefly."}]+ messages.append({"role": "user", "content": f"Read this document.\n\n{_DOCUMENT}"})+ messages.append({"role": "assistant", "content": "I have read the cardiology report."})+ for index in range(10):+ messages.append({"role": "user", "content": f"question {index} {_FILLER}"})+ messages.append({"role": "assistant", "content": f"answer {index} {_FILLER}"})+ messages.append({"role": "user", "content": "Was there a phone number on the report?"})+ return messages+++def _persist(thread_id: str, messages: list[dict]) -> None:+ from storage import studio_db+ studio_db.upsert_chat_thread(+ {+ "id": thread_id,+ "title": "t",+ "modelType": "base",+ "modelId": "a-model",+ "createdAt": 1,+ }+ )+ for index, message in enumerate(m for m in messages if m["role"] != "system"):+ studio_db.upsert_chat_message(+ {+ "id": f"{thread_id}-{index}",+ "threadId": thread_id,+ "role": message["role"],+ "content": [{"type": "text", "text": message["content"]}],+ "createdAt": index + 2,+ }+ )+++def _payload(thread_id: str | None) -> ChatCompletionRequest:+ return ChatCompletionRequest(+ messages = [{"role": "user", "content": "latest"}],+ provider_type = "openrouter",+ model = "a-model",+ thread_id = thread_id,+ context_overflow = "truncate_oldest",+ compaction_threshold = 3_000,+ context_window = 4_000,+ max_tokens = 512,+ )+++def archive_ready(rag_home, rag_conn, stub_embeddings):+ return rag_conn+++def test_an_external_fit_archives_the_turns_it_evicts(archive_ready):+ messages = _chat()+ _persist(THREAD, messages)++ fitted, truncation, _ = ri._fit_external_context(messages, _payload(THREAD))++ assert truncation and truncation["dropped_messages"] > 0+ assert all(_DOCUMENT not in str(m.get("content")) for m in fitted[:-1])+ assert conversation_archive.has_archive(THREAD)+++def test_the_evicting_fit_recalls_the_document_into_the_latest_turn(archive_ready):+ # The catalogue is fixed before the fit, so on this turn the model has no search tool.+ messages = _chat()+ _persist(THREAD, messages)++ fitted, truncation, _ = ri._fit_external_context(messages, _payload(THREAD))++ assert truncation["recalled_chunks"] > 0+ assert "055 794 3321" in str(fitted[-1]["content"])+ assert "Was there a phone number on the report?" in str(fitted[-1]["content"])+ # The rewritten latest turn must not hide where the boundary landed.+ assert truncation["boundary_anchor"]+++def test_an_attached_document_from_an_evicted_turn_is_searchable(archive_ready):+ messages = _chat()+ _persist(THREAD, messages)++ ri._fit_external_context(messages, _payload(THREAD))+ result = tools_mod.execute_tool(+ "search_conversation",+ {"query": "cardiology report phone number"},+ thread_id = THREAD,+ timeout = None,+ )++ assert "055 794 3321" in result+++def test_a_fit_without_a_thread_archives_nothing(archive_ready):+ fitted, truncation, _ = ri._fit_external_context(_chat(), _payload(None))++ assert truncation and truncation["dropped_messages"] > 0+ assert not conversation_archive.has_archive(THREAD)+++def test_a_chat_that_fits_archives_nothing(archive_ready):+ messages = [+ {"role": "user", "content": f"Read this document.\n\n{_DOCUMENT}"},+ {"role": "assistant", "content": "Done."},+ {"role": "user", "content": "Thanks."},+ ]+ _persist(THREAD, messages)++ fitted, truncation, _ = ri._fit_external_context(messages, _payload(THREAD))++ assert truncation is None+ assert fitted is messages+ assert not conversation_archive.has_archive(THREAD)+++def test_an_archive_failure_still_returns_the_fitted_chat(archive_ready, monkeypatch):+ messages = _chat()+ _persist(THREAD, messages)++ def broken(*args, **kwargs):+ raise RuntimeError("store unavailable")++ monkeypatch.setattr(conversation_archive, "archive_turns", broken)++ fitted, truncation, _ = ri._fit_external_context(messages, _payload(THREAD))++ assert truncation and truncation["dropped_messages"] > 0+ assert fitted[-1]["content"] == "Was there a phone number on the report?"+ assert "recalled_chunks" not in truncation+++def test_an_external_tool_chat_with_an_archive_is_told_about_compaction(monkeypatch):+ monkeypatch.setattr(ri, "_thread_has_conversation_archive", lambda thread_id: True)++ async def connected(self) -> bool:+ return False++ # The tool loop polls for a disconnect and would stop before reaching the provider.+ monkeypatch.setattr(Request, "is_disconnected", connected)+ _, sent = _proxy(+ "openrouter",+ [{"role": "user", "content": "Was there a phone number on the report?"}],+ thread_id = THREAD,+ enable_tools = True,+ enabled_tools = ["python"],+ permission_mode = "off",+ )++ names = {tool["function"]["name"] for tool in sent["tools"]}+ system = " ".join(str(m.get("content")) for m in sent["messages"] if m.get("role") == "system")+ assert "search_conversation" in names+ assert "call search_conversation before answering" in system+++def test_an_external_tool_chat_without_an_archive_gets_no_compaction_nudge(monkeypatch):+ monkeypatch.setattr(ri, "_thread_has_conversation_archive", lambda thread_id: False)++ async def connected(self) -> bool:+ return False++ monkeypatch.setattr(Request, "is_disconnected", connected)+ _, sent = _proxy(+ "openrouter",+ [{"role": "user", "content": "hello"}],+ thread_id = THREAD,+ enable_tools = True,+ enabled_tools = ["python"],+ permission_mode = "off",+ )++ system = " ".join(str(m.get("content")) for m in sent["messages"] if m.get("role") == "system")+ assert "search_conversation" not in system+++def test_a_tool_loop_refit_recalls_for_the_users_turn_not_a_loop_reprompt(+ archive_ready, monkeypatch+):+ branch = _chat()+ _persist(THREAD, branch)+ loop = [+ *branch,+ {"role": "assistant", "content": "I will look it up."},+ {"role": "user", "content": "You described a plan without acting. Call a tool now."},+ ]+ seen = []++ def spy(+ conversation,+ thread_id,+ *,+ branch_messages = None,+ **kwargs,+ ):+ seen.append(tools_mod._last_user_text(branch_messages or conversation))+ return None++ monkeypatch.setattr(tools_mod, "build_conversation_recall", spy)++ ri._fit_external_context(loop, _payload(THREAD), branch = branch)++ assert seen == ["Was there a phone number on the report?"]+++def test_caller_owned_tools_do_not_halve_the_recall_budget(archive_ready, monkeypatch):+ from core.inference import context_window++ messages = _chat()+ _persist(THREAD, messages)+ seen = []+ real = context_window.retrieval_budget++ def spy(+ *args,+ reply_returns = False,+ **kwargs,+ ):+ seen.append(reply_returns)+ return real(*args, reply_returns = reply_returns, **kwargs)++ monkeypatch.setattr(context_window, "retrieval_budget", spy)+ caller_tools = [{"type": "function", "function": {"name": "lookup", "parameters": {}}}]++ ri._fit_external_context(messages, _payload(THREAD), tools = caller_tools)+ ri._fit_external_context(messages, _payload(THREAD), tools = caller_tools, tool_loop = True)++ assert seen == [False, True]+++def test_no_compaction_nudge_when_tool_calls_are_disabled(monkeypatch):+ monkeypatch.setattr(ri, "_thread_has_conversation_archive", lambda thread_id: True)++ async def connected(self) -> bool:+ return False++ monkeypatch.setattr(Request, "is_disconnected", connected)+ _, sent = _proxy(+ "openrouter",+ [{"role": "user", "content": "Was there a phone number on the report?"}],+ thread_id = THREAD,+ enable_tools = True,+ enabled_tools = ["python"],+ permission_mode = "off",+ tool_choice = "none",+ )++ system = " ".join(str(m.get("content")) for m in sent["messages"] if m.get("role") == "system")+ assert "call search_conversation" not in systemdiff --git a/studio/backend/tests/test_managed_engine_reasoning.py b/studio/backend/tests/test_managed_engine_reasoning.pyindex b98aada6e65..b6afabbd2a8 100644--- a/studio/backend/tests/test_managed_engine_reasoning.py+++ b/studio/backend/tests/test_managed_engine_reasoning.py@@ -348,6 +348,40 @@ def test_tool_loop_turns_are_fitted_to_the_engine_window():)+def test_a_managed_refit_recalls_with_the_first_branch_it_saw(monkeypatch):+ import asyncio++ from routes import inference as api+ from routes.managed_engine_chat import _message_fitter++ seen = []++ def fake_fit(+ messages,+ payload,+ *,+ tools = None,+ branch = None,+ tool_loop = False,+ ):+ seen.append((list(branch), tool_loop))+ return messages, None, 64++ monkeypatch.setattr(api, "_fit_external_context", fake_fit)+ first = [{"role": "user", "content": "Was there a phone number?"}]+ later = [+ *first,+ {"role": "assistant", "content": "Let me check."},+ {"role": "user", "content": "You described a plan without acting."},+ ]+ fit = _message_fitter(+ _tool_payload(context_overflow = "truncate_oldest"), _Engine(), "m", None, studio_loop = True+ )+ asyncio.run(fit(first))+ asyncio.run(fit(later))+ assert seen == [(first, True), (first, True)]++def test_tool_loop_honors_the_server_default_overflow(monkeypatch):from routes.managed_engine_chat import _message_fitter