Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: encode each recall query once per compaction and let archive widening widen (#15960)

Daniel Han8cfa93fcf2ef
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 3736f053a3c..659f1996ae1 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -2189,6 +2189,7 @@ def _archive_and_recall(
try:
from core.inference.tools import build_conversation_recall
from core.rag import conversation_archive
+ from core.rag.retrieval import query_vector_memo
if not conversation_archive.enabled():
return result
@@ -2236,48 +2237,50 @@ def _archive_and_recall(
break
attempt_k = max(1, attempt_k // 2)
- for index, k in enumerate(attempts):
- recall = build_conversation_recall(
- conversation,
- thread_id,
- style = style,
- top_k = k,
- branch_messages = branch_messages,
- )
- if not recall:
- result["counts"] = counts
- return result
+ # One encode per query across the retries and the widenings inside each recall.
+ with query_vector_memo():
+ for index, k in enumerate(attempts):
+ recall = build_conversation_recall(
+ conversation,
+ thread_id,
+ style = style,
+ top_k = k,
+ branch_messages = branch_messages,
+ )
+ if not recall:
+ result["counts"] = counts
+ return result
- candidate = _inject_recall(conversation, recall, style)
- if candidate is None:
- result["counts"] = counts
- return result
+ candidate = _inject_recall(conversation, recall, style)
+ if candidate is None:
+ result["counts"] = counts
+ return result
- grew = None
- if count_tokens is not None and recall_budget_tokens > 0:
- try:
- grew = count_tokens(candidate["conversation"]) - count_tokens(conversation)
- except Exception:
- grew = None
- if grew is not None and grew > recall_budget_tokens:
- logger.info(
- "conversation_recall.over_budget grew=%s budget=%s top_k=%s",
- grew,
- recall_budget_tokens,
- k,
- )
- if index + 1 < len(attempts):
- continue
- logger.info(
- "conversation_recall.dropped_over_budget budget=%s", recall_budget_tokens
- )
- result["counts"] = counts
- return result
+ grew = None
+ if count_tokens is not None and recall_budget_tokens > 0:
+ try:
+ grew = count_tokens(candidate["conversation"]) - count_tokens(conversation)
+ except Exception:
+ grew = None
+ if grew is not None and grew > recall_budget_tokens:
+ logger.info(
+ "conversation_recall.over_budget grew=%s budget=%s top_k=%s",
+ grew,
+ recall_budget_tokens,
+ k,
+ )
+ if index + 1 < len(attempts):
+ continue
+ logger.info(
+ "conversation_recall.dropped_over_budget budget=%s", recall_budget_tokens
+ )
+ result["counts"] = counts
+ return result
- result["conversation"] = candidate["conversation"]
- result["events"] = candidate["events"]
- result["anchored"] = candidate["anchored"]
- break
+ result["conversation"] = candidate["conversation"]
+ result["events"] = candidate["events"]
+ result["anchored"] = candidate["anchored"]
+ break
counts["recalled_chunks"] = int(recall.get("sources") or 0)
result["counts"] = counts
diff --git a/studio/backend/core/rag/conversation_archive.py b/studio/backend/core/rag/conversation_archive.py
index 53945a7ac99..74b5d3f58f8 100644
--- a/studio/backend/core/rag/conversation_archive.py
+++ b/studio/backend/core/rag/conversation_archive.py
@@ -2262,8 +2262,17 @@ def _candidates(conn, scope: str, query: str, model, fetch: int, thread_id: str)
if len(hits) < fetch:
try:
seen = {hit.chunk_id for hit in hits}
+ # Legs sized to the fetch: at the default 30 each, widening past 60 reads as an exhausted archive.
+ legs = max(fetch, config.TOP_K_LEXICAL, config.TOP_K_DENSE)
for hit in retrieval.retrieve_hybrid(
- conn, scope, query, k = fetch, model_name = model, mode = "hybrid"
+ conn,
+ scope,
+ query,
+ k = fetch,
+ model_name = model,
+ mode = "hybrid",
+ lexical_k = legs,
+ dense_k = legs,
):
if hit.chunk_id not in seen:
hits.append(hit)
@@ -2290,6 +2299,26 @@ def recall(
forced: bool = False,
) -> Optional[tuple[str, list[dict]]]:
"""Lexical first, since recall is mostly exact match; no relevance floor, as weak matches beat none."""
+ with retrieval.query_vector_memo():
+ return _recall(
+ thread_id,
+ query,
+ top_k = top_k,
+ branch_messages = branch_messages,
+ extra_queries = extra_queries,
+ forced = forced,
+ )
+
+
+def _recall(
+ thread_id: str,
+ query: str,
+ *,
+ top_k: Optional[int] = None,
+ branch_messages: Optional[list[dict]] = None,
+ extra_queries: Optional[list[str]] = None,
+ forced: bool = False,
+) -> Optional[tuple[str, list[dict]]]:
query = (query or "").strip()
if not thread_id or not query or not enabled():
return None
diff --git a/studio/backend/core/rag/retrieval.py b/studio/backend/core/rag/retrieval.py
index 9ba8361305c..088a15f96e9 100644
--- a/studio/backend/core/rag/retrieval.py
+++ b/studio/backend/core/rag/retrieval.py
@@ -6,6 +6,8 @@ kept so callers can apply a similarity floor."""
from __future__ import annotations
+import contextlib
+import contextvars
import logging
import sqlite3
from dataclasses import dataclass
@@ -14,6 +16,23 @@ from . import config, embeddings, reembed, store
logger = logging.getLogger(__name__)
+_query_vectors: contextvars.ContextVar[dict | None] = contextvars.ContextVar(
+ "rag_query_vectors", default = None
+)
+
+
+def query_vector_memo():
+ """Within the block, each (model, query) is encoded once; a nested entry shares the outer memo."""
+ if _query_vectors.get() is not None:
+ yield
+ return
+ token = _query_vectors.set({})
+ try:
+ yield
+ finally:
+ _query_vectors.reset(token)
+
@dataclass
class Hit:
@@ -58,11 +77,18 @@ def retrieve_dense(
) -> list[Hit]:
k = k or config.TOP_K_DENSE
effective = model_name or config.effective_embedding_model()
- # Take identity from the encode, not the process: a concurrent ST failure may have swapped it.
- vectors, identity = embeddings.encode_with_identity(
- [query], model_name = effective, normalize = True
- )
- vec = vectors[0]
+ memo = _query_vectors.get()
+ key = (effective, query)
+ if memo is not None and key in memo:
+ vec, identity = memo[key]
+ else:
+ # Take identity from the encode, not the process: a concurrent ST failure may have swapped it.
+ vectors, identity = embeddings.encode_with_identity(
+ [query], model_name = effective, normalize = True
+ )
+ vec = vectors[0]
+ if memo is not None:
+ memo[key] = (vec, identity)
_warn_once_on_untagged(conn)
try:
reembed.start_after_backend_change(identity)
@@ -130,16 +156,21 @@ def retrieve_hybrid(
model_name: str | None = None,
mode: str = "hybrid",
lexical_query: str | None = None,
+ lexical_k: int | None = None,
+ dense_k: int | None = None,
) -> list[Hit]:
- """lexical_query changes only the FTS5 leg; the dense leg always encodes the natural-language query."""
+ """lexical_query changes only the FTS5 leg; the dense leg always encodes the natural-language query.
+ lexical_k / dense_k size the legs (default TOP_K_LEXICAL / TOP_K_DENSE): fused output never exceeds their sum."""
k = k if k is not None else config.TOP_K_HYBRID
k = int(k) # tool-call / scope top_k may arrive as a float; LIMIT + slice need int
if mode == "lexical":
return retrieve_lexical(conn, scope, query, k, match_query = lexical_query)
if mode == "dense":
return retrieve_dense(conn, scope, query, k, model_name = model_name)
- lexical = retrieve_lexical(conn, scope, query, config.TOP_K_LEXICAL, match_query = lexical_query)
- dense = retrieve_dense(conn, scope, query, config.TOP_K_DENSE, model_name = model_name)
+ lexical = retrieve_lexical(
+ conn, scope, query, lexical_k or config.TOP_K_LEXICAL, match_query = lexical_query
+ )
+ dense = retrieve_dense(conn, scope, query, dense_k or config.TOP_K_DENSE, model_name = model_name)
return _rrf([lexical, dense], config.RRF_K, k)
diff --git a/studio/backend/tests/test_conversation_archive.py b/studio/backend/tests/test_conversation_archive.py
index 35f124f423a..bd57e5b33ae 100644
--- a/studio/backend/tests/test_conversation_archive.py
+++ b/studio/backend/tests/test_conversation_archive.py
@@ -213,6 +213,7 @@ def test_recall_degrades_to_lexical_when_dense_retrieval_raises(monkeypatch, con
model_name = None,
mode = "hybrid",
lexical_query = None,
+ **_legs,
):
if mode != "lexical":
raise RuntimeError("no embedding backend available")
diff --git a/studio/backend/tests/test_rag_query_memo.py b/studio/backend/tests/test_rag_query_memo.py
new file mode 100644
index 00000000000..faf0bf28d65
--- /dev/null
+++ b/studio/backend/tests/test_rag_query_memo.py
@@ -0,0 +1,137 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Query vectors are encoded once per recall or compaction, and archive widening reaches past the hybrid legs."""
+
+import math
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from core.inference import llama_cpp # noqa: E402
+from core.rag import conversation_archive, embeddings, retrieval, store # noqa: E402
+
+THREAD = "thread-memo"
+
+
+def _save(thread_id, messages):
+ from storage import studio_db
+
+ studio_db.upsert_chat_thread(
+ {"id": thread_id, "title": "t", "modelType": "base", "modelId": "m", "createdAt": 1}
+ )
+ existing = len(studio_db.list_chat_messages(thread_id) or [])
+ for index, message in enumerate(messages):
+ studio_db.upsert_chat_message(
+ {
+ "id": f"{thread_id}-{existing + index}",
+ "threadId": thread_id,
+ "role": message["role"],
+ "content": [{"type": "text", "text": message["content"]}],
+ "createdAt": existing + index + 2,
+ }
+ )
+
+
+def _archive(turn):
+ _save(THREAD, turn)
+ conversation_archive.archive_turns(THREAD, turn)
+
+
+def _turn(question, answer):
+ return [{"role": "user", "content": question}, {"role": "assistant", "content": answer}]
+
+
+def encodes(rag_home, rag_conn, stub_embeddings, monkeypatch):
+ calls = []
+ real = embeddings.encode
+
+ def counting(texts, **kwargs):
+ calls.append(list(texts))
+ return real(texts, **kwargs)
+
+ monkeypatch.setattr(embeddings, "encode", counting)
+ return calls
+
+
+def test_retrieve_dense_encodes_once_inside_a_memo_and_again_outside(encodes, rag_conn):
+ scope = store.conversation_archive_scope(THREAD)
+ with retrieval.query_vector_memo():
+ with retrieval.query_vector_memo():
+ retrieval.retrieve_dense(rag_conn, scope, "pelicans", 5)
+ retrieval.retrieve_dense(rag_conn, scope, "pelicans", 5)
+ retrieval.retrieve_dense(rag_conn, scope, "pelicans", 5)
+ assert encodes.count(["pelicans"]) == 2
+
+
+def test_a_compaction_encodes_its_query_once_across_budget_retries(encodes):
+ _archive(_turn("write me a limerick about pelicans", "There once was a bird with a bill"))
+ query = "what was that pelicans limerick"
+ conversation = [{"role": "system", "content": "sys"}, {"role": "user", "content": query}]
+ del encodes[:]
+
+ result = llama_cpp._archive_and_recall(
+ conversation,
+ conversation,
+ thread_id = THREAD,
+ style = "inline",
+ recall_done = False,
+ recall_budget_tokens = 4000,
+ # Every attempt overflows, so all of 4 -> 2 -> 1 run.
+ count_tokens = lambda conv: 10**7 if "recalled" in repr(conv) else 0,
+ )
+
+ assert result["recalled"] is False
+ assert encodes.count([query]) == 1
+
+
+def test_separate_recalls_do_not_share_vectors(encodes):
+ _archive(_turn("write me a limerick about pelicans", "There once was a bird with a bill"))
+ del encodes[:]
+ conversation_archive.recall(THREAD, "pelicans limerick")
+ conversation_archive.recall(THREAD, "pelicans limerick")
+ assert encodes.count(["pelicans limerick"]) == 2
+
+
+def test_a_recall_lexical_fills_never_encodes(encodes):
+ for i in range(20):
+ _archive(_turn(f"question {i} about zx81tape", f"answer {i} zx81tape"))
+ del encodes[:]
+ found = conversation_archive.recall(THREAD, "zx81tape", top_k = 4)
+ assert found is not None
+ assert not any(texts == ["zx81tape"] for texts in encodes)
+
+
+def test_widening_reaches_a_live_turn_ranked_past_the_dense_leg(
+ rag_home, rag_conn, stub_embeddings, monkeypatch
+):
+ """40 abandoned-branch turns outrank the live one by cosine; widening to 64 must reach it."""
+ dim = stub_embeddings
+
+ def vec(text):
+ v = [0.0] * dim
+ if "flamingo" in text:
+ v[0] = 1.0
+ elif "decoy" in text:
+ v[0], v[1] = 1.0, 0.05 + 0.001 * (len(text) % 7)
+ elif "zebra" in text:
+ v[0], v[1] = 0.6, 0.8
+ else:
+ v[2] = 1.0
+ n = math.sqrt(sum(x * x for x in v))
+ return [x / n for x in v]
+
+ monkeypatch.setattr(embeddings, "encode", lambda texts, **_k: [vec(t) for t in texts])
+ for i in range(40):
+ _archive(_turn(f"decoy question number {i}", f"decoy reply {i}"))
+ live = _turn("tell me about zebra stripes", "zebra stripes deter flies")
+ _archive(live)
+
+ found = conversation_archive.recall(THREAD, "flamingo", top_k = 4, branch_messages = live)
+
+ assert found is not None
+ assert "zebra" in found[0]