Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: reuse lexical passes and branch checks across a compaction and prune branch-run starts (#15972)
Daniel Han0d9b63e89eef
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 7d76e5c201d..bd8b5cb203e 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -2237,8 +2237,8 @@ def _archive_and_recall(breakattempt_k = max(1, attempt_k // 2)- # One encode per query across the retries and the widenings inside each recall.- with query_vector_memo():+ # Retries re-ask the same queries: encode each once and reuse lexical passes and branch checks.+ with query_vector_memo(), conversation_archive.recall_session():for index, k in enumerate(attempts):recall = build_conversation_recall(conversation,diff --git a/studio/backend/core/rag/conversation_archive.py b/studio/backend/core/rag/conversation_archive.pyindex 74b5d3f58f8..44dad7c4d56 100644--- a/studio/backend/core/rag/conversation_archive.py+++ b/studio/backend/core/rag/conversation_archive.py@@ -6,9 +6,12 @@ never reset per compaction; entry points no-op on failure."""from __future__ import annotations+import bisect+import contextvars+import dataclasses+import functoolsimport hashlibimport logging-import bisectimport jsonimport osimport re@@ -54,6 +57,97 @@ _BRANCH_FILTER_OVERFETCH = 4# Long abandoned branches can fill any fixed window, so widen and re-ask, up to this bound: a chat request, not a crawl._BRANCH_FILTER_MAX_CANDIDATES = 256+# Bumped after every archive write in this process; a recall session drops its memos when it moves.+_write_generation = 0+_write_generation_lock = threading.Lock()+++def _writes(fn):+ @functools.wraps(fn)+ def wrapper(*args, **kwargs):+ global _write_generation+ try:+ return fn(*args, **kwargs)+ finally:+ with _write_generation_lock:+ _write_generation += 1++ return wrapper+++class _RecallSession:+ """Lexical results, transcripts and branch decisions shared by the recalls of one compaction."""++ __slots__ = ("generation", "lexical", "matching", "transcripts")++ def __init__(self):+ self.generation = _write_generation+ self.lexical: dict = {}+ self.matching: dict = {}+ self.transcripts: dict = {}++ def current(self) -> "_RecallSession":+ if self.generation != _write_generation:+ self.__init__()+ return self+++_session: contextvars.ContextVar[Optional[_RecallSession]] = contextvars.ContextVar(+ "rag_recall_session", default = None+)+++@contextmanager+def recall_session():+ """Recalls inside share one session; a nested entry joins the outer one."""+ if _session.get() is not None:+ yield+ return+ token = _session.set(_RecallSession())+ try:+ yield+ finally:+ _session.reset(token)+++def _active_session() -> Optional[_RecallSession]:+ session = _session.get()+ return session.current() if session is not None else None+++class _TranscriptIndex:+ """Which messages contain a probe, from one C-level scan of the joined transcript."""++ __slots__ = ("texts", "joined", "offsets", "found")++ def __init__(self, texts: list[str]):+ self.texts = texts+ self.joined = "\0".join(texts)+ self.offsets = []+ at = 0+ for text in texts:+ self.offsets.append(at)+ at += len(text) + 1+ self.found: dict = {}++ def containing(self, probe: str) -> list[int]:+ hit = self.found.get(probe)+ if hit is not None:+ return hit+ if "\0" in probe:+ hit = [i for i, text in enumerate(self.texts) if probe in text]+ else:+ hit = []+ at = self.joined.find(probe)+ while at >= 0:+ index = bisect.bisect_right(self.offsets, at) - 1+ hit.append(index)+ if index + 1 >= len(self.offsets):+ break+ at = self.joined.find(probe, self.offsets[index + 1])+ self.found[probe] = hit+ return hit+def _text_of(content, *, include_tool_calls: bool = False) -> str:"""Tool calls are flattened only when include_tool_calls is set, which the branch check needs."""@@ -702,6 +796,7 @@ def archive_turns(return _archive_turns(thread_id, evicted, live, branch, cancel_check)+@_writesdef _archive_turns(thread_id: str,evicted: list[dict],@@ -1706,6 +1801,7 @@ def _retire_surplus(return False+@_writesdef _restamp(conn,scope: str,@@ -1767,6 +1863,7 @@ def _archived_under(return _stale_document(conn, scope, digest, identity, occurrences = occurrences) is _ARCHIVED+@_writesdef _widen_span(conn, scope: str, digest: str, span: int) -> None:"""Only widens: a stored window is a maximum, so a longer span still matches the shorter turn."""try:@@ -2003,7 +2100,13 @@ def _rendered_message_count(rows) -> int:return total-def _document_on_live_branch(conn, document_id: str, transcript: list[str], cache: dict) -> bool:+def _document_on_live_branch(+ conn,+ document_id: str,+ transcript: list[str],+ cache: dict,+ index: Optional[_TranscriptIndex] = None,+) -> bool:"""Every chunk must still be on the branch: an edit to any part of a turn retires the whole copy."""if document_id in cache:return cache[document_id]@@ -2021,7 +2124,7 @@ def _document_on_live_branch(conn, document_id: str, transcript: list[str], cach# Never fail a recall on the strictness pass: fall back to what the candidate chunk itself said.cache[document_id] = Truereturn True- cache[document_id] = _document_matches_one_run(rows, transcript, message_count)+ cache[document_id] = _document_matches_one_run(rows, transcript, message_count, index)return cache[document_id]@@ -2029,6 +2132,7 @@ def _document_matches_one_run(rows,transcript: Optional[list[str]],message_count: Optional[int] = None,+ index: Optional[_TranscriptIndex] = None,) -> bool:"""Every chunk must match within one run of adjacent messages, not be assembled from scattered parts."""if not rows or not transcript:@@ -2068,7 +2172,19 @@ def _document_matches_one_run(return Falsereturn partial_tail or cursor >= len(transcript[position])- return any(_one_run_from(start) for start in range(len(transcript)))+ # A run opens on the first message at or after its start that holds the first probe, and must open inside+ # its window, so only starts within one window before such a message can succeed.+ first = probe_lists[0][0][0]+ if index is not None and index.texts is transcript:+ holders = index.containing(first)+ else:+ holders = [i for i, text in enumerate(transcript) if first in text]+ starts = []+ previous = -1+ for holder in holders:+ starts.extend(range(max(previous + 1, holder - window + 1), holder + 1))+ previous = holder+ return any(_one_run_from(start) for start in starts)def _order_key(ordinal, created_at, document_rowid, chunk_index) -> tuple:@@ -2146,6 +2262,46 @@ def _lexical_pass(*,newest_first: bool = False,oldest_first: bool = False,+) -> list:+ session = _active_session()+ if session is None:+ return _lexical_pass_uncached(+ conn,+ scope,+ query,+ model,+ k,+ expression,+ newest_first = newest_first,+ oldest_first = oldest_first,+ )+ key = (scope, query, k, expression, newest_first, oldest_first)+ hits = session.lexical.get(key)+ if hits is None:+ hits = session.lexical[key] = _lexical_pass_uncached(+ conn,+ scope,+ query,+ model,+ k,+ expression,+ newest_first = newest_first,+ oldest_first = oldest_first,+ )+ # Callers re-rank and refill these lists; hand out copies.+ return [dataclasses.replace(hit) for hit in hits]+++def _lexical_pass_uncached(+ conn,+ scope: str,+ query: str,+ model,+ k: int,+ expression,+ *,+ newest_first: bool = False,+ oldest_first: bool = False,) -> list:if newest_first or oldest_first:# Archive legs are always "lexical", so this is the same call with the tie-break reversed.@@ -2181,6 +2337,19 @@ def _both_ends(oldest: list, newest: list) -> list:return oldest + [hit for hit in newest if hit.chunk_id not in seen]+def _lexical_matching_ids(conn, chunk_ids: list, expression) -> set:+ session = _active_session()+ if session is None:+ return store.lexical_matching_ids(conn, chunk_ids, expression)+ key = (tuple(chunk_ids), expression)+ found = session.matching.get(key)+ if found is None:+ found = session.matching[key] = frozenset(+ store.lexical_matching_ids(conn, chunk_ids, expression)+ )+ return set(found)++def _focused_lexical(conn, scope: str, query: str, model, fetch: int) -> list:"""Identifiers filter candidates and content words rank them, as BM25 is flat on common identifiers."""expressions = (@@ -2235,9 +2404,7 @@ def _focused_lexical(conn, scope: str, query: str, model, fetch: int) -> list:# Ask the index for eligibility: the strict pass is capped and arbitrarily ordered.eligible = {hit.chunk_id for hit in strict}try:- eligible |= store.lexical_matching_ids(- conn, [hit.chunk_id for hit in loose], expressions[0]- )+ eligible |= _lexical_matching_ids(conn, [hit.chunk_id for hit in loose], expressions[0])except Exception:# Exact membership is an improvement, not a dependency; on failure the strict rows alone stand.logger.warning("conversation_archive.eligibility_probe_failed", exc_info = True)@@ -2249,6 +2416,26 @@ def _focused_lexical(conn, scope: str, query: str, model, fetch: int) -> list:return ranked+def _recall_transcript(branch_messages, thread_id: str):+ """(transcript, its index, branch decisions per document), shared across a session's recalls."""+ session = _active_session()+ if session is None:+ transcript = branch_message_texts(branch_messages) or _live_transcript(thread_id)+ return transcript, (_TranscriptIndex(transcript) if transcript else None), {}+ # Keyed on message identity: the session holds the messages, so ids stay theirs.+ key = (tuple(map(id, branch_messages or ())), thread_id)+ entry = session.transcripts.get(key)+ if entry is None:+ transcript = branch_message_texts(branch_messages) or _live_transcript(thread_id)+ entry = session.transcripts[key] = (+ transcript,+ _TranscriptIndex(transcript) if transcript else None,+ {},+ list(branch_messages or ()),+ )+ return entry[0], entry[1], entry[2]++def _candidates(conn, scope: str, query: str, model, fetch: int, thread_id: str) -> list:"""Lexical runs twice with an identifier: requiring it, then over content words for order, so an empty filter is not read as an empty archive."""hits: list = []@@ -2299,7 +2486,7 @@ def recall(forced: bool = False,) -> Optional[tuple[str, list[dict]]]:"""Lexical first, since recall is mostly exact match; no relevance floor, as weak matches beat none."""- with retrieval.query_vector_memo():+ with retrieval.query_vector_memo(), recall_session():return _recall(thread_id,query,@@ -2379,11 +2566,10 @@ def _recall(conn = rag_db.get_connection()model = config.effective_embedding_model()# Request's branch first; stored rows are the whole DAG, a fallback that still beats no filtering.- transcript = branch_message_texts(branch_messages) or _live_transcript(thread_id)+ transcript, index, live_documents = _recall_transcript(branch_messages, thread_id)fetch = limit * _BRANCH_FILTER_OVERFETCHrows: dict = {}hits: list = []- live_documents: dict = {}while True:fetched = _candidates(conn, scope, query, model, fetch, thread_id)if not fetched:@@ -2410,7 +2596,7 @@ def _recall(for hit in candidatesif hit.chunk_id in rowsand _document_on_live_branch(- conn, rows[hit.chunk_id]["document_id"], transcript, live_documents+ conn, rows[hit.chunk_id]["document_id"], transcript, live_documents, index)]if len(hits) != len(candidates):@@ -2459,6 +2645,7 @@ def _scope_select(scope: str, created_before: Optional[str]) -> tuple:)+@_writesdef _delete_scope_without_vec(scope: str,thread_id: str,@@ -2515,6 +2702,7 @@ def delete_thread_documents(thread_id: str, *, created_before: Optional[str] = Nreturn len(removed)+@_writesdef copy_thread_documents(source_thread_id: str,thread_id: str,@@ -2583,6 +2771,7 @@ def thread_has_documents(thread_id: str) -> bool:conn.close()+@_writesdef _delete_scope(scope: str,thread_id: str,diff --git a/studio/backend/tests/test_conversation_recall_session.py b/studio/backend/tests/test_conversation_recall_session.pynew file mode 100644index 00000000000..508b68f7ed0--- /dev/null+++ b/studio/backend/tests/test_conversation_recall_session.py@@ -0,0 +1,196 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import os+import random+import sys++import pytest++sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))++from core.rag import conversation_archive, store # noqa: E402++THREAD = "thread-session"+++def _save_thread(thread_id, messages):+ from storage import studio_db++ studio_db.upsert_chat_thread(+ {"id": thread_id, "title": "t", "modelType": "base", "modelId": "m", "createdAt": 1}+ )+ existing = len(studio_db.list_chat_messages(thread_id) or [])+ for index, message in enumerate(messages):+ studio_db.upsert_chat_message(+ {+ "id": f"{thread_id}-{existing + index}",+ "threadId": thread_id,+ "role": message["role"],+ "content": [{"type": "text", "text": message["content"]}],+ "createdAt": existing + index + 2,+ }+ )+++def _archive(messages):+ _save_thread(THREAD, messages)+ return conversation_archive.archive_turns(THREAD, messages)+++def conn(rag_home, rag_conn, stub_embeddings):+ return rag_conn+++class _AllStarts:+ """An index that names every message, which is the unpruned search."""++ def __init__(self, texts):+ self.texts = texts++ def containing(self, probe):+ return list(range(len(self.texts)))+++def _random_case(rng):+ words = ["alpha", "beta", "gamma", "delta", "run", "fix", "TICKET-7", "the", "cache"]+ pool = [" ".join(rng.choice(words) for _ in range(rng.randint(1, 4))) for _ in range(6)]+ messages = [+ {"role": rng.choice(["user", "assistant"]), "content": rng.choice(pool)}+ for _ in range(rng.randint(1, 6))+ ]+ transcript = [rng.choice(pool) for _ in range(rng.randint(1, 30))]+ if rng.random() < 0.5:+ # Plant the turn so a real match is possible, sometimes edited.+ at = rng.randint(0, len(transcript))+ planted = [m["content"] for m in messages]+ if rng.random() < 0.3:+ planted[0] = planted[0] + " edited"+ transcript[at:at] = planted+ lines = conversation_archive.render_turn(messages).splitlines()+ if len(lines) > 1 and rng.random() < 0.6:+ # Chunked like a long turn: consecutive slices that may overlap by a line.+ cut = rng.randint(1, len(lines) - 1)+ back = rng.randint(0, 1)+ rows = [{"text": "\n".join(lines[:cut])}, {"text": "\n".join(lines[cut - back :])}]+ else:+ rows = [{"text": "\n".join(lines)}]+ count = rng.choice([None, len(messages), len(messages) + 1])+ return rows, transcript, count+++def test_pruned_branch_match_equals_the_exhaustive_search():+ rng = random.Random(1234)+ matched = 0+ for _ in range(20000):+ rows, transcript, count = _random_case(rng)+ exhaustive = conversation_archive._document_matches_one_run(+ rows, transcript, count, _AllStarts(transcript)+ )+ index = conversation_archive._TranscriptIndex(transcript)+ assert (+ conversation_archive._document_matches_one_run(rows, transcript, count, index)+ == exhaustive+ )+ assert conversation_archive._document_matches_one_run(rows, transcript, count) == exhaustive+ matched += exhaustive+ assert matched > 200, "the random cases must exercise real matches"+++def test_transcript_index_finds_every_holder_once():+ texts = ["abc abc", "", "xabc", "ab", "c abc\nabc"]+ index = conversation_archive._TranscriptIndex(texts)+ assert index.containing("abc") == [0, 2, 4]+ assert index.containing("ab") == [0, 2, 3, 4]+ assert index.containing("zzz") == []+++def _seed(n = 40):+ turns = []+ for i in range(n):+ turns += [+ {"role": "user", "content": f"how do we handle TICKET-{i % 5} in module m{i}"},+ {"role": "assistant", "content": f"we retry the cache for m{i} twice"},+ ]+ _archive(turns)+ return turns+++def _compaction(turns, query, session):+ from core.inference import tools++ branch = turns + [{"role": "user", "content": query}]++ def attempts():+ return [+ tools.build_conversation_recall(+ branch[-3:], THREAD, style = "inline", top_k = k, branch_messages = branch+ )+ for k in (4, 2, 1)+ ]++ if not session:+ return attempts()+ with conversation_archive.recall_session():+ return attempts()+++ "query", ["TICKET-3 retry cache", "module m7", "continue", "nothing matches xyzzy"]+)+def test_a_session_returns_what_separate_recalls_return(conn, query):+ turns = _seed()+ assert _compaction(turns, query, session = True) == _compaction(turns, query, session = False)+++def test_retries_in_a_session_share_lexical_passes_and_branch_checks(conn, monkeypatch):+ turns = _seed()+ calls = {"fts": 0, "texts": 0, "match": 0}+ real_fts, real_texts, real_match = (+ store.search_lexical,+ conversation_archive.branch_message_texts,+ conversation_archive._document_matches_one_run,+ )++ def count(key, fn):+ def wrapped(*a, **kw):+ calls[key] += 1+ return fn(*a, **kw)++ return wrapped++ monkeypatch.setattr(store, "search_lexical", count("fts", real_fts))+ monkeypatch.setattr(conversation_archive, "branch_message_texts", count("texts", real_texts))+ monkeypatch.setattr(+ conversation_archive, "_document_matches_one_run", count("match", real_match)+ )++ _compaction(turns, "TICKET-3 retry cache", session = False)+ separate = dict(calls)+ calls.update(fts = 0, texts = 0, match = 0)+ _compaction(turns, "TICKET-3 retry cache", session = True)++ assert separate["texts"] == 3 and calls["texts"] == 1+ assert calls["fts"] < separate["fts"]+ assert calls["match"] < separate["match"]+++def test_an_archive_write_inside_a_session_is_seen_by_the_next_recall(conn, monkeypatch):+ from core.rag import retrieval++ # No hybrid top-up, so only the memoized focused lexical passes can find the new turn.+ monkeypatch.setattr(retrieval, "retrieve_hybrid", lambda *a, **kw: [])+ turns = _seed(6)+ branch = list(turns)+ with conversation_archive.recall_session():+ before = conversation_archive.recall(THREAD, "zanzibar", branch_messages = branch)+ assert not before or "zanzibar" not in before[0]+ new = [+ {"role": "user", "content": "what about zanzibar"},+ {"role": "assistant", "content": "zanzibar ships friday"},+ ]+ _archive(new)+ branch += new+ found = conversation_archive.recall(THREAD, "zanzibar", branch_messages = branch)+ assert found and "zanzibar" in found[0]