Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: reuse lexical passes and branch checks across a compaction and prune branch-run starts (#15972)

Daniel Han0d9b63e89eef
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 7d76e5c201d..bd8b5cb203e 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -2237,8 +2237,8 @@ def _archive_and_recall(
break
attempt_k = max(1, attempt_k // 2)
- # One encode per query across the retries and the widenings inside each recall.
- with query_vector_memo():
+ # Retries re-ask the same queries: encode each once and reuse lexical passes and branch checks.
+ with query_vector_memo(), conversation_archive.recall_session():
for index, k in enumerate(attempts):
recall = build_conversation_recall(
conversation,
diff --git a/studio/backend/core/rag/conversation_archive.py b/studio/backend/core/rag/conversation_archive.py
index 74b5d3f58f8..44dad7c4d56 100644
--- a/studio/backend/core/rag/conversation_archive.py
+++ b/studio/backend/core/rag/conversation_archive.py
@@ -6,9 +6,12 @@ never reset per compaction; entry points no-op on failure."""
from __future__ import annotations
+import bisect
+import contextvars
+import dataclasses
+import functools
import hashlib
import logging
-import bisect
import json
import os
import re
@@ -54,6 +57,97 @@ _BRANCH_FILTER_OVERFETCH = 4
# Long abandoned branches can fill any fixed window, so widen and re-ask, up to this bound: a chat request, not a crawl.
_BRANCH_FILTER_MAX_CANDIDATES = 256
+# Bumped after every archive write in this process; a recall session drops its memos when it moves.
+_write_generation = 0
+_write_generation_lock = threading.Lock()
+
+
+def _writes(fn):
+ @functools.wraps(fn)
+ def wrapper(*args, **kwargs):
+ global _write_generation
+ try:
+ return fn(*args, **kwargs)
+ finally:
+ with _write_generation_lock:
+ _write_generation += 1
+
+ return wrapper
+
+
+class _RecallSession:
+ """Lexical results, transcripts and branch decisions shared by the recalls of one compaction."""
+
+ __slots__ = ("generation", "lexical", "matching", "transcripts")
+
+ def __init__(self):
+ self.generation = _write_generation
+ self.lexical: dict = {}
+ self.matching: dict = {}
+ self.transcripts: dict = {}
+
+ def current(self) -> "_RecallSession":
+ if self.generation != _write_generation:
+ self.__init__()
+ return self
+
+
+_session: contextvars.ContextVar[Optional[_RecallSession]] = contextvars.ContextVar(
+ "rag_recall_session", default = None
+)
+
+
+@contextmanager
+def recall_session():
+ """Recalls inside share one session; a nested entry joins the outer one."""
+ if _session.get() is not None:
+ yield
+ return
+ token = _session.set(_RecallSession())
+ try:
+ yield
+ finally:
+ _session.reset(token)
+
+
+def _active_session() -> Optional[_RecallSession]:
+ session = _session.get()
+ return session.current() if session is not None else None
+
+
+class _TranscriptIndex:
+ """Which messages contain a probe, from one C-level scan of the joined transcript."""
+
+ __slots__ = ("texts", "joined", "offsets", "found")
+
+ def __init__(self, texts: list[str]):
+ self.texts = texts
+ self.joined = "\0".join(texts)
+ self.offsets = []
+ at = 0
+ for text in texts:
+ self.offsets.append(at)
+ at += len(text) + 1
+ self.found: dict = {}
+
+ def containing(self, probe: str) -> list[int]:
+ hit = self.found.get(probe)
+ if hit is not None:
+ return hit
+ if "\0" in probe:
+ hit = [i for i, text in enumerate(self.texts) if probe in text]
+ else:
+ hit = []
+ at = self.joined.find(probe)
+ while at >= 0:
+ index = bisect.bisect_right(self.offsets, at) - 1
+ hit.append(index)
+ if index + 1 >= len(self.offsets):
+ break
+ at = self.joined.find(probe, self.offsets[index + 1])
+ self.found[probe] = hit
+ return hit
+
def _text_of(content, *, include_tool_calls: bool = False) -> str:
"""Tool calls are flattened only when include_tool_calls is set, which the branch check needs."""
@@ -702,6 +796,7 @@ def archive_turns(
return _archive_turns(thread_id, evicted, live, branch, cancel_check)
+@_writes
def _archive_turns(
thread_id: str,
evicted: list[dict],
@@ -1706,6 +1801,7 @@ def _retire_surplus(
return False
+@_writes
def _restamp(
conn,
scope: str,
@@ -1767,6 +1863,7 @@ def _archived_under(
return _stale_document(conn, scope, digest, identity, occurrences = occurrences) is _ARCHIVED
+@_writes
def _widen_span(conn, scope: str, digest: str, span: int) -> None:
"""Only widens: a stored window is a maximum, so a longer span still matches the shorter turn."""
try:
@@ -2003,7 +2100,13 @@ def _rendered_message_count(rows) -> int:
return total
-def _document_on_live_branch(conn, document_id: str, transcript: list[str], cache: dict) -> bool:
+def _document_on_live_branch(
+ conn,
+ document_id: str,
+ transcript: list[str],
+ cache: dict,
+ index: Optional[_TranscriptIndex] = None,
+) -> bool:
"""Every chunk must still be on the branch: an edit to any part of a turn retires the whole copy."""
if document_id in cache:
return cache[document_id]
@@ -2021,7 +2124,7 @@ def _document_on_live_branch(conn, document_id: str, transcript: list[str], cach
# Never fail a recall on the strictness pass: fall back to what the candidate chunk itself said.
cache[document_id] = True
return True
- cache[document_id] = _document_matches_one_run(rows, transcript, message_count)
+ cache[document_id] = _document_matches_one_run(rows, transcript, message_count, index)
return cache[document_id]
@@ -2029,6 +2132,7 @@ def _document_matches_one_run(
rows,
transcript: Optional[list[str]],
message_count: Optional[int] = None,
+ index: Optional[_TranscriptIndex] = None,
) -> bool:
"""Every chunk must match within one run of adjacent messages, not be assembled from scattered parts."""
if not rows or not transcript:
@@ -2068,7 +2172,19 @@ def _document_matches_one_run(
return False
return partial_tail or cursor >= len(transcript[position])
- return any(_one_run_from(start) for start in range(len(transcript)))
+ # A run opens on the first message at or after its start that holds the first probe, and must open inside
+ # its window, so only starts within one window before such a message can succeed.
+ first = probe_lists[0][0][0]
+ if index is not None and index.texts is transcript:
+ holders = index.containing(first)
+ else:
+ holders = [i for i, text in enumerate(transcript) if first in text]
+ starts = []
+ previous = -1
+ for holder in holders:
+ starts.extend(range(max(previous + 1, holder - window + 1), holder + 1))
+ previous = holder
+ return any(_one_run_from(start) for start in starts)
def _order_key(ordinal, created_at, document_rowid, chunk_index) -> tuple:
@@ -2146,6 +2262,46 @@ def _lexical_pass(
*,
newest_first: bool = False,
oldest_first: bool = False,
+) -> list:
+ session = _active_session()
+ if session is None:
+ return _lexical_pass_uncached(
+ conn,
+ scope,
+ query,
+ model,
+ k,
+ expression,
+ newest_first = newest_first,
+ oldest_first = oldest_first,
+ )
+ key = (scope, query, k, expression, newest_first, oldest_first)
+ hits = session.lexical.get(key)
+ if hits is None:
+ hits = session.lexical[key] = _lexical_pass_uncached(
+ conn,
+ scope,
+ query,
+ model,
+ k,
+ expression,
+ newest_first = newest_first,
+ oldest_first = oldest_first,
+ )
+ # Callers re-rank and refill these lists; hand out copies.
+ return [dataclasses.replace(hit) for hit in hits]
+
+
+def _lexical_pass_uncached(
+ conn,
+ scope: str,
+ query: str,
+ model,
+ k: int,
+ expression,
+ *,
+ newest_first: bool = False,
+ oldest_first: bool = False,
) -> list:
if newest_first or oldest_first:
# Archive legs are always "lexical", so this is the same call with the tie-break reversed.
@@ -2181,6 +2337,19 @@ def _both_ends(oldest: list, newest: list) -> list:
return oldest + [hit for hit in newest if hit.chunk_id not in seen]
+def _lexical_matching_ids(conn, chunk_ids: list, expression) -> set:
+ session = _active_session()
+ if session is None:
+ return store.lexical_matching_ids(conn, chunk_ids, expression)
+ key = (tuple(chunk_ids), expression)
+ found = session.matching.get(key)
+ if found is None:
+ found = session.matching[key] = frozenset(
+ store.lexical_matching_ids(conn, chunk_ids, expression)
+ )
+ return set(found)
+
+
def _focused_lexical(conn, scope: str, query: str, model, fetch: int) -> list:
"""Identifiers filter candidates and content words rank them, as BM25 is flat on common identifiers."""
expressions = (
@@ -2235,9 +2404,7 @@ def _focused_lexical(conn, scope: str, query: str, model, fetch: int) -> list:
# Ask the index for eligibility: the strict pass is capped and arbitrarily ordered.
eligible = {hit.chunk_id for hit in strict}
try:
- eligible |= store.lexical_matching_ids(
- conn, [hit.chunk_id for hit in loose], expressions[0]
- )
+ eligible |= _lexical_matching_ids(conn, [hit.chunk_id for hit in loose], expressions[0])
except Exception:
# Exact membership is an improvement, not a dependency; on failure the strict rows alone stand.
logger.warning("conversation_archive.eligibility_probe_failed", exc_info = True)
@@ -2249,6 +2416,26 @@ def _focused_lexical(conn, scope: str, query: str, model, fetch: int) -> list:
return ranked
+def _recall_transcript(branch_messages, thread_id: str):
+ """(transcript, its index, branch decisions per document), shared across a session's recalls."""
+ session = _active_session()
+ if session is None:
+ transcript = branch_message_texts(branch_messages) or _live_transcript(thread_id)
+ return transcript, (_TranscriptIndex(transcript) if transcript else None), {}
+ # Keyed on message identity: the session holds the messages, so ids stay theirs.
+ key = (tuple(map(id, branch_messages or ())), thread_id)
+ entry = session.transcripts.get(key)
+ if entry is None:
+ transcript = branch_message_texts(branch_messages) or _live_transcript(thread_id)
+ entry = session.transcripts[key] = (
+ transcript,
+ _TranscriptIndex(transcript) if transcript else None,
+ {},
+ list(branch_messages or ()),
+ )
+ return entry[0], entry[1], entry[2]
+
+
def _candidates(conn, scope: str, query: str, model, fetch: int, thread_id: str) -> list:
"""Lexical runs twice with an identifier: requiring it, then over content words for order, so an empty filter is not read as an empty archive."""
hits: list = []
@@ -2299,7 +2486,7 @@ def recall(
forced: bool = False,
) -> Optional[tuple[str, list[dict]]]:
"""Lexical first, since recall is mostly exact match; no relevance floor, as weak matches beat none."""
- with retrieval.query_vector_memo():
+ with retrieval.query_vector_memo(), recall_session():
return _recall(
thread_id,
query,
@@ -2379,11 +2566,10 @@ def _recall(
conn = rag_db.get_connection()
model = config.effective_embedding_model()
# Request's branch first; stored rows are the whole DAG, a fallback that still beats no filtering.
- transcript = branch_message_texts(branch_messages) or _live_transcript(thread_id)
+ transcript, index, live_documents = _recall_transcript(branch_messages, thread_id)
fetch = limit * _BRANCH_FILTER_OVERFETCH
rows: dict = {}
hits: list = []
- live_documents: dict = {}
while True:
fetched = _candidates(conn, scope, query, model, fetch, thread_id)
if not fetched:
@@ -2410,7 +2596,7 @@ def _recall(
for hit in candidates
if hit.chunk_id in rows
and _document_on_live_branch(
- conn, rows[hit.chunk_id]["document_id"], transcript, live_documents
+ conn, rows[hit.chunk_id]["document_id"], transcript, live_documents, index
)
]
if len(hits) != len(candidates):
@@ -2459,6 +2645,7 @@ def _scope_select(scope: str, created_before: Optional[str]) -> tuple:
)
+@_writes
def _delete_scope_without_vec(
scope: str,
thread_id: str,
@@ -2515,6 +2702,7 @@ def delete_thread_documents(thread_id: str, *, created_before: Optional[str] = N
return len(removed)
+@_writes
def copy_thread_documents(
source_thread_id: str,
thread_id: str,
@@ -2583,6 +2771,7 @@ def thread_has_documents(thread_id: str) -> bool:
conn.close()
+@_writes
def _delete_scope(
scope: str,
thread_id: str,
diff --git a/studio/backend/tests/test_conversation_recall_session.py b/studio/backend/tests/test_conversation_recall_session.py
new file mode 100644
index 00000000000..508b68f7ed0
--- /dev/null
+++ b/studio/backend/tests/test_conversation_recall_session.py
@@ -0,0 +1,196 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import os
+import random
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from core.rag import conversation_archive, store # noqa: E402
+
+THREAD = "thread-session"
+
+
+def _save_thread(thread_id, messages):
+ from storage import studio_db
+
+ studio_db.upsert_chat_thread(
+ {"id": thread_id, "title": "t", "modelType": "base", "modelId": "m", "createdAt": 1}
+ )
+ existing = len(studio_db.list_chat_messages(thread_id) or [])
+ for index, message in enumerate(messages):
+ studio_db.upsert_chat_message(
+ {
+ "id": f"{thread_id}-{existing + index}",
+ "threadId": thread_id,
+ "role": message["role"],
+ "content": [{"type": "text", "text": message["content"]}],
+ "createdAt": existing + index + 2,
+ }
+ )
+
+
+def _archive(messages):
+ _save_thread(THREAD, messages)
+ return conversation_archive.archive_turns(THREAD, messages)
+
+
+def conn(rag_home, rag_conn, stub_embeddings):
+ return rag_conn
+
+
+class _AllStarts:
+ """An index that names every message, which is the unpruned search."""
+
+ def __init__(self, texts):
+ self.texts = texts
+
+ def containing(self, probe):
+ return list(range(len(self.texts)))
+
+
+def _random_case(rng):
+ words = ["alpha", "beta", "gamma", "delta", "run", "fix", "TICKET-7", "the", "cache"]
+ pool = [" ".join(rng.choice(words) for _ in range(rng.randint(1, 4))) for _ in range(6)]
+ messages = [
+ {"role": rng.choice(["user", "assistant"]), "content": rng.choice(pool)}
+ for _ in range(rng.randint(1, 6))
+ ]
+ transcript = [rng.choice(pool) for _ in range(rng.randint(1, 30))]
+ if rng.random() < 0.5:
+ # Plant the turn so a real match is possible, sometimes edited.
+ at = rng.randint(0, len(transcript))
+ planted = [m["content"] for m in messages]
+ if rng.random() < 0.3:
+ planted[0] = planted[0] + " edited"
+ transcript[at:at] = planted
+ lines = conversation_archive.render_turn(messages).splitlines()
+ if len(lines) > 1 and rng.random() < 0.6:
+ # Chunked like a long turn: consecutive slices that may overlap by a line.
+ cut = rng.randint(1, len(lines) - 1)
+ back = rng.randint(0, 1)
+ rows = [{"text": "\n".join(lines[:cut])}, {"text": "\n".join(lines[cut - back :])}]
+ else:
+ rows = [{"text": "\n".join(lines)}]
+ count = rng.choice([None, len(messages), len(messages) + 1])
+ return rows, transcript, count
+
+
+def test_pruned_branch_match_equals_the_exhaustive_search():
+ rng = random.Random(1234)
+ matched = 0
+ for _ in range(20000):
+ rows, transcript, count = _random_case(rng)
+ exhaustive = conversation_archive._document_matches_one_run(
+ rows, transcript, count, _AllStarts(transcript)
+ )
+ index = conversation_archive._TranscriptIndex(transcript)
+ assert (
+ conversation_archive._document_matches_one_run(rows, transcript, count, index)
+ == exhaustive
+ )
+ assert conversation_archive._document_matches_one_run(rows, transcript, count) == exhaustive
+ matched += exhaustive
+ assert matched > 200, "the random cases must exercise real matches"
+
+
+def test_transcript_index_finds_every_holder_once():
+ texts = ["abc abc", "", "xabc", "ab", "c abc\nabc"]
+ index = conversation_archive._TranscriptIndex(texts)
+ assert index.containing("abc") == [0, 2, 4]
+ assert index.containing("ab") == [0, 2, 3, 4]
+ assert index.containing("zzz") == []
+
+
+def _seed(n = 40):
+ turns = []
+ for i in range(n):
+ turns += [
+ {"role": "user", "content": f"how do we handle TICKET-{i % 5} in module m{i}"},
+ {"role": "assistant", "content": f"we retry the cache for m{i} twice"},
+ ]
+ _archive(turns)
+ return turns
+
+
+def _compaction(turns, query, session):
+ from core.inference import tools
+
+ branch = turns + [{"role": "user", "content": query}]
+
+ def attempts():
+ return [
+ tools.build_conversation_recall(
+ branch[-3:], THREAD, style = "inline", top_k = k, branch_messages = branch
+ )
+ for k in (4, 2, 1)
+ ]
+
+ if not session:
+ return attempts()
+ with conversation_archive.recall_session():
+ return attempts()
+
+
+ "query", ["TICKET-3 retry cache", "module m7", "continue", "nothing matches xyzzy"]
+)
+def test_a_session_returns_what_separate_recalls_return(conn, query):
+ turns = _seed()
+ assert _compaction(turns, query, session = True) == _compaction(turns, query, session = False)
+
+
+def test_retries_in_a_session_share_lexical_passes_and_branch_checks(conn, monkeypatch):
+ turns = _seed()
+ calls = {"fts": 0, "texts": 0, "match": 0}
+ real_fts, real_texts, real_match = (
+ store.search_lexical,
+ conversation_archive.branch_message_texts,
+ conversation_archive._document_matches_one_run,
+ )
+
+ def count(key, fn):
+ def wrapped(*a, **kw):
+ calls[key] += 1
+ return fn(*a, **kw)
+
+ return wrapped
+
+ monkeypatch.setattr(store, "search_lexical", count("fts", real_fts))
+ monkeypatch.setattr(conversation_archive, "branch_message_texts", count("texts", real_texts))
+ monkeypatch.setattr(
+ conversation_archive, "_document_matches_one_run", count("match", real_match)
+ )
+
+ _compaction(turns, "TICKET-3 retry cache", session = False)
+ separate = dict(calls)
+ calls.update(fts = 0, texts = 0, match = 0)
+ _compaction(turns, "TICKET-3 retry cache", session = True)
+
+ assert separate["texts"] == 3 and calls["texts"] == 1
+ assert calls["fts"] < separate["fts"]
+ assert calls["match"] < separate["match"]
+
+
+def test_an_archive_write_inside_a_session_is_seen_by_the_next_recall(conn, monkeypatch):
+ from core.rag import retrieval
+
+ # No hybrid top-up, so only the memoized focused lexical passes can find the new turn.
+ monkeypatch.setattr(retrieval, "retrieve_hybrid", lambda *a, **kw: [])
+ turns = _seed(6)
+ branch = list(turns)
+ with conversation_archive.recall_session():
+ before = conversation_archive.recall(THREAD, "zanzibar", branch_messages = branch)
+ assert not before or "zanzibar" not in before[0]
+ new = [
+ {"role": "user", "content": "what about zanzibar"},
+ {"role": "assistant", "content": "zanzibar ships friday"},
+ ]
+ _archive(new)
+ branch += new
+ found = conversation_archive.recall(THREAD, "zanzibar", branch_messages = branch)
+ assert found and "zanzibar" in found[0]