Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: encode each recall query once per compaction and let archive widening widen (#15960)
Daniel Han8cfa93fcf2ef
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 3736f053a3c..659f1996ae1 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -2189,6 +2189,7 @@ def _archive_and_recall(try:from core.inference.tools import build_conversation_recallfrom core.rag import conversation_archive+ from core.rag.retrieval import query_vector_memoif not conversation_archive.enabled():return result@@ -2236,48 +2237,50 @@ def _archive_and_recall(breakattempt_k = max(1, attempt_k // 2)- for index, k in enumerate(attempts):- recall = build_conversation_recall(- conversation,- thread_id,- style = style,- top_k = k,- branch_messages = branch_messages,- )- if not recall:- result["counts"] = counts- return result+ # One encode per query across the retries and the widenings inside each recall.+ with query_vector_memo():+ for index, k in enumerate(attempts):+ recall = build_conversation_recall(+ conversation,+ thread_id,+ style = style,+ top_k = k,+ branch_messages = branch_messages,+ )+ if not recall:+ result["counts"] = counts+ return result- candidate = _inject_recall(conversation, recall, style)- if candidate is None:- result["counts"] = counts- return result+ candidate = _inject_recall(conversation, recall, style)+ if candidate is None:+ result["counts"] = counts+ return result- grew = None- if count_tokens is not None and recall_budget_tokens > 0:- try:- grew = count_tokens(candidate["conversation"]) - count_tokens(conversation)- except Exception:- grew = None- if grew is not None and grew > recall_budget_tokens:- logger.info(- "conversation_recall.over_budget grew=%s budget=%s top_k=%s",- grew,- recall_budget_tokens,- k,- )- if index + 1 < len(attempts):- continue- logger.info(- "conversation_recall.dropped_over_budget budget=%s", recall_budget_tokens- )- result["counts"] = counts- return result+ grew = None+ if count_tokens is not None and recall_budget_tokens > 0:+ try:+ grew = count_tokens(candidate["conversation"]) - count_tokens(conversation)+ except Exception:+ grew = None+ if grew is not None and grew > recall_budget_tokens:+ logger.info(+ "conversation_recall.over_budget grew=%s budget=%s top_k=%s",+ grew,+ recall_budget_tokens,+ k,+ )+ if index + 1 < len(attempts):+ continue+ logger.info(+ "conversation_recall.dropped_over_budget budget=%s", recall_budget_tokens+ )+ result["counts"] = counts+ return result- result["conversation"] = candidate["conversation"]- result["events"] = candidate["events"]- result["anchored"] = candidate["anchored"]- break+ result["conversation"] = candidate["conversation"]+ result["events"] = candidate["events"]+ result["anchored"] = candidate["anchored"]+ breakcounts["recalled_chunks"] = int(recall.get("sources") or 0)result["counts"] = countsdiff --git a/studio/backend/core/rag/conversation_archive.py b/studio/backend/core/rag/conversation_archive.pyindex 53945a7ac99..74b5d3f58f8 100644--- a/studio/backend/core/rag/conversation_archive.py+++ b/studio/backend/core/rag/conversation_archive.py@@ -2262,8 +2262,17 @@ def _candidates(conn, scope: str, query: str, model, fetch: int, thread_id: str)if len(hits) < fetch:try:seen = {hit.chunk_id for hit in hits}+ # Legs sized to the fetch: at the default 30 each, widening past 60 reads as an exhausted archive.+ legs = max(fetch, config.TOP_K_LEXICAL, config.TOP_K_DENSE)for hit in retrieval.retrieve_hybrid(- conn, scope, query, k = fetch, model_name = model, mode = "hybrid"+ conn,+ scope,+ query,+ k = fetch,+ model_name = model,+ mode = "hybrid",+ lexical_k = legs,+ dense_k = legs,):if hit.chunk_id not in seen:hits.append(hit)@@ -2290,6 +2299,26 @@ def recall(forced: bool = False,) -> Optional[tuple[str, list[dict]]]:"""Lexical first, since recall is mostly exact match; no relevance floor, as weak matches beat none."""+ with retrieval.query_vector_memo():+ return _recall(+ thread_id,+ query,+ top_k = top_k,+ branch_messages = branch_messages,+ extra_queries = extra_queries,+ forced = forced,+ )+++def _recall(+ thread_id: str,+ query: str,+ *,+ top_k: Optional[int] = None,+ branch_messages: Optional[list[dict]] = None,+ extra_queries: Optional[list[str]] = None,+ forced: bool = False,+) -> Optional[tuple[str, list[dict]]]:query = (query or "").strip()if not thread_id or not query or not enabled():return Nonediff --git a/studio/backend/core/rag/retrieval.py b/studio/backend/core/rag/retrieval.pyindex 9ba8361305c..088a15f96e9 100644--- a/studio/backend/core/rag/retrieval.py+++ b/studio/backend/core/rag/retrieval.py@@ -6,6 +6,8 @@ kept so callers can apply a similarity floor."""from __future__ import annotations+import contextlib+import contextvarsimport loggingimport sqlite3from dataclasses import dataclass@@ -14,6 +16,23 @@ from . import config, embeddings, reembed, storelogger = logging.getLogger(__name__)+_query_vectors: contextvars.ContextVar[dict | None] = contextvars.ContextVar(+ "rag_query_vectors", default = None+)+++def query_vector_memo():+ """Within the block, each (model, query) is encoded once; a nested entry shares the outer memo."""+ if _query_vectors.get() is not None:+ yield+ return+ token = _query_vectors.set({})+ try:+ yield+ finally:+ _query_vectors.reset(token)+@dataclassclass Hit:@@ -58,11 +77,18 @@ def retrieve_dense() -> list[Hit]:k = k or config.TOP_K_DENSEeffective = model_name or config.effective_embedding_model()- # Take identity from the encode, not the process: a concurrent ST failure may have swapped it.- vectors, identity = embeddings.encode_with_identity(- [query], model_name = effective, normalize = True- )- vec = vectors[0]+ memo = _query_vectors.get()+ key = (effective, query)+ if memo is not None and key in memo:+ vec, identity = memo[key]+ else:+ # Take identity from the encode, not the process: a concurrent ST failure may have swapped it.+ vectors, identity = embeddings.encode_with_identity(+ [query], model_name = effective, normalize = True+ )+ vec = vectors[0]+ if memo is not None:+ memo[key] = (vec, identity)_warn_once_on_untagged(conn)try:reembed.start_after_backend_change(identity)@@ -130,16 +156,21 @@ def retrieve_hybrid(model_name: str | None = None,mode: str = "hybrid",lexical_query: str | None = None,+ lexical_k: int | None = None,+ dense_k: int | None = None,) -> list[Hit]:- """lexical_query changes only the FTS5 leg; the dense leg always encodes the natural-language query."""+ """lexical_query changes only the FTS5 leg; the dense leg always encodes the natural-language query.+ lexical_k / dense_k size the legs (default TOP_K_LEXICAL / TOP_K_DENSE): fused output never exceeds their sum."""k = k if k is not None else config.TOP_K_HYBRIDk = int(k) # tool-call / scope top_k may arrive as a float; LIMIT + slice need intif mode == "lexical":return retrieve_lexical(conn, scope, query, k, match_query = lexical_query)if mode == "dense":return retrieve_dense(conn, scope, query, k, model_name = model_name)- lexical = retrieve_lexical(conn, scope, query, config.TOP_K_LEXICAL, match_query = lexical_query)- dense = retrieve_dense(conn, scope, query, config.TOP_K_DENSE, model_name = model_name)+ lexical = retrieve_lexical(+ conn, scope, query, lexical_k or config.TOP_K_LEXICAL, match_query = lexical_query+ )+ dense = retrieve_dense(conn, scope, query, dense_k or config.TOP_K_DENSE, model_name = model_name)return _rrf([lexical, dense], config.RRF_K, k)diff --git a/studio/backend/tests/test_conversation_archive.py b/studio/backend/tests/test_conversation_archive.pyindex 35f124f423a..bd57e5b33ae 100644--- a/studio/backend/tests/test_conversation_archive.py+++ b/studio/backend/tests/test_conversation_archive.py@@ -213,6 +213,7 @@ def test_recall_degrades_to_lexical_when_dense_retrieval_raises(monkeypatch, conmodel_name = None,mode = "hybrid",lexical_query = None,+ **_legs,):if mode != "lexical":raise RuntimeError("no embedding backend available")diff --git a/studio/backend/tests/test_rag_query_memo.py b/studio/backend/tests/test_rag_query_memo.pynew file mode 100644index 00000000000..faf0bf28d65--- /dev/null+++ b/studio/backend/tests/test_rag_query_memo.py@@ -0,0 +1,137 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Query vectors are encoded once per recall or compaction, and archive widening reaches past the hybrid legs."""++import math+import os+import sys++import pytest++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from core.inference import llama_cpp # noqa: E402+from core.rag import conversation_archive, embeddings, retrieval, store # noqa: E402++THREAD = "thread-memo"+++def _save(thread_id, messages):+ from storage import studio_db++ studio_db.upsert_chat_thread(+ {"id": thread_id, "title": "t", "modelType": "base", "modelId": "m", "createdAt": 1}+ )+ existing = len(studio_db.list_chat_messages(thread_id) or [])+ for index, message in enumerate(messages):+ studio_db.upsert_chat_message(+ {+ "id": f"{thread_id}-{existing + index}",+ "threadId": thread_id,+ "role": message["role"],+ "content": [{"type": "text", "text": message["content"]}],+ "createdAt": existing + index + 2,+ }+ )+++def _archive(turn):+ _save(THREAD, turn)+ conversation_archive.archive_turns(THREAD, turn)+++def _turn(question, answer):+ return [{"role": "user", "content": question}, {"role": "assistant", "content": answer}]+++def encodes(rag_home, rag_conn, stub_embeddings, monkeypatch):+ calls = []+ real = embeddings.encode++ def counting(texts, **kwargs):+ calls.append(list(texts))+ return real(texts, **kwargs)++ monkeypatch.setattr(embeddings, "encode", counting)+ return calls+++def test_retrieve_dense_encodes_once_inside_a_memo_and_again_outside(encodes, rag_conn):+ scope = store.conversation_archive_scope(THREAD)+ with retrieval.query_vector_memo():+ with retrieval.query_vector_memo():+ retrieval.retrieve_dense(rag_conn, scope, "pelicans", 5)+ retrieval.retrieve_dense(rag_conn, scope, "pelicans", 5)+ retrieval.retrieve_dense(rag_conn, scope, "pelicans", 5)+ assert encodes.count(["pelicans"]) == 2+++def test_a_compaction_encodes_its_query_once_across_budget_retries(encodes):+ _archive(_turn("write me a limerick about pelicans", "There once was a bird with a bill"))+ query = "what was that pelicans limerick"+ conversation = [{"role": "system", "content": "sys"}, {"role": "user", "content": query}]+ del encodes[:]++ result = llama_cpp._archive_and_recall(+ conversation,+ conversation,+ thread_id = THREAD,+ style = "inline",+ recall_done = False,+ recall_budget_tokens = 4000,+ # Every attempt overflows, so all of 4 -> 2 -> 1 run.+ count_tokens = lambda conv: 10**7 if "recalled" in repr(conv) else 0,+ )++ assert result["recalled"] is False+ assert encodes.count([query]) == 1+++def test_separate_recalls_do_not_share_vectors(encodes):+ _archive(_turn("write me a limerick about pelicans", "There once was a bird with a bill"))+ del encodes[:]+ conversation_archive.recall(THREAD, "pelicans limerick")+ conversation_archive.recall(THREAD, "pelicans limerick")+ assert encodes.count(["pelicans limerick"]) == 2+++def test_a_recall_lexical_fills_never_encodes(encodes):+ for i in range(20):+ _archive(_turn(f"question {i} about zx81tape", f"answer {i} zx81tape"))+ del encodes[:]+ found = conversation_archive.recall(THREAD, "zx81tape", top_k = 4)+ assert found is not None+ assert not any(texts == ["zx81tape"] for texts in encodes)+++def test_widening_reaches_a_live_turn_ranked_past_the_dense_leg(+ rag_home, rag_conn, stub_embeddings, monkeypatch+):+ """40 abandoned-branch turns outrank the live one by cosine; widening to 64 must reach it."""+ dim = stub_embeddings++ def vec(text):+ v = [0.0] * dim+ if "flamingo" in text:+ v[0] = 1.0+ elif "decoy" in text:+ v[0], v[1] = 1.0, 0.05 + 0.001 * (len(text) % 7)+ elif "zebra" in text:+ v[0], v[1] = 0.6, 0.8+ else:+ v[2] = 1.0+ n = math.sqrt(sum(x * x for x in v))+ return [x / n for x in v]++ monkeypatch.setattr(embeddings, "encode", lambda texts, **_k: [vec(t) for t in texts])+ for i in range(40):+ _archive(_turn(f"decoy question number {i}", f"decoy reply {i}"))+ live = _turn("tell me about zebra stripes", "zebra stripes deter flies")+ _archive(live)++ found = conversation_archive.recall(THREAD, "flamingo", top_k = 4, branch_messages = live)++ assert found is not None+ assert "zebra" in found[0]