Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: let small models read a long attached file when asked to summarize it (#12580)

Nilaydc8ede02175d
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index 8910f2f3421..cf21abeacbc 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -14697,6 +14697,22 @@ def rag_autoinject_reaches_retrieval(
return bool(enabled), whole_doc_requested
+def _thread_document_ids(thread_id) -> set | None:
+ """Ids of the thread's indexed attachments; None when the store cannot say."""
+ try:
+ from core.rag import store
+ from storage import rag_db
+
+ conn = rag_db.get_connection()
+ try:
+ docs = store.list_documents(conn, store.thread_scope(thread_id))
+ finally:
+ conn.close()
+ except Exception: # noqa: BLE001
+ return None
+ return {d["id"] for d in docs if d.get("status") == "completed" and d.get("num_chunks")}
+
+
def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> dict | None:
"""Pre-retrieve the latest user turn; if a hit clears the cosine floor return ``{"events": [...],
"messages": [...]}`` to splice into the loop, else ``None``. Toggle via ``rag_scope.autoinject``
@@ -14726,7 +14742,8 @@ def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> di
# Cap at the lean top_k, but honor a lower user setting.
lean_k = _autoinject_top_k()
sidebar_k = _opt_int(rag_scope.get("default_top_k"))
- top_k = min(sidebar_k, lean_k) if sidebar_k is not None else lean_k
+ # Zero or below is no limit to the search, which then returns its own default count.
+ top_k = min(sidebar_k, lean_k) if sidebar_k is not None and sidebar_k > 0 else lean_k
budget: int | None = None
# The window the budget was sized against, so `_text_token_cost` only trusts a GGUF actually serving this same
# window.
@@ -14800,19 +14817,34 @@ def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> di
found = search_for_autoinject(query = query, top_k = top_k, **scope)
return _trim(found[0], found[1], max_tokens) if found else None
+ thread_docs = _thread_document_ids(thread_id) if whole_doc_requested and text is None else set()
+
+ def retrieve_thread_unfloored(*, max_tokens = None):
+ # Lexical-only finds nothing for a generic request ("summarize this") whose words are not in the file, so
+ # this mandatory grounding retries with the dense leg. Chats with no attachment skip the query embedding.
+ if thread_docs is not None and not thread_docs:
+ return None
+ scope_kwargs = _scope_retrieval_kwargs(rag_scope)
+ found = retrieve(
+ max_tokens = max_tokens, scope_thread_id = thread_id, min_dense_score = None, **scope_kwargs
+ )
+ if not found and scope_kwargs["mode"] == "lexical":
+ found = retrieve(
+ max_tokens = max_tokens,
+ scope_thread_id = thread_id,
+ min_dense_score = None,
+ mode = "hybrid",
+ )
+ return found
+
# An oversized thread attachment is mandatory grounding: with auto-injection off, search it alone, without the
# optional-auto relevance floor, then add project context if the combination still fits. The budget binds on that
# path only: with auto-injection on this stays the single combined unbudgeted search, so a small context cannot
- # silently switch RAG off.
+ # silently switch RAG off, and the thread is searched again without the floor when none of it cleared.
if text is None and (enabled or whole_doc_requested):
try:
if whole_doc_requested and not enabled:
- found = retrieve(
- max_tokens = budget,
- scope_thread_id = thread_id,
- min_dense_score = None,
- **_scope_retrieval_kwargs(rag_scope),
- )
+ found = retrieve_thread_unfloored(max_tokens = budget)
project_id = rag_scope.get("project_id")
if found and project_id:
# Isolated like the whole-document companion above: an unavailable project index must not send the
@@ -14839,6 +14871,28 @@ def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> di
min_dense_score = floor,
**_scope_retrieval_kwargs(rag_scope),
)
+ # Project hits clearing the floor must not crowd out the attachment: without one of its passages,
+ # it goes first.
+ grounded = (
+ bool(found)
+ and thread_docs is not None
+ and any(s.get("documentId") in thread_docs for s in found[1])
+ )
+ if (
+ whole_doc_requested
+ and (not found or rag_scope.get("project_id"))
+ and not grounded
+ ):
+ thread_found = retrieve_thread_unfloored()
+ if thread_found and found:
+ cited = thread_docs or {s.get("documentId") for s in thread_found[1]}
+ if not any(s.get("documentId") in cited for s in found[1]):
+ # Still the lean top_k in total, so the unbudgeted path never doubles the injection.
+ n_proj = min(len(found[1]), top_k // 2)
+ merged = thread_found[1][: top_k - n_proj] + found[1][:n_proj]
+ found = (render_sources(merged), merged)
+ elif thread_found:
+ found = thread_found
except Exception as exc: # noqa: BLE001
logger.warning("RAG auto-inject retrieval failed: %s", exc)
return None
diff --git a/studio/backend/tests/test_rag_whole_document.py b/studio/backend/tests/test_rag_whole_document.py
index 60397b9bd60..32d3db004ad 100644
--- a/studio/backend/tests/test_rag_whole_document.py
+++ b/studio/backend/tests/test_rag_whole_document.py
@@ -8,6 +8,8 @@ No embedder is needed - the whole-doc path does no query embedding."""
import json
+import pytest
+
from core.rag import store, tool
from core.rag.chunking import Chunk
from core.inference import tools as inf_tools
@@ -301,6 +303,7 @@ def test_build_rag_autoinject_large_model_auto_falls_back_over_budget(rag_conn,
def test_build_rag_autoinject_fallback_is_thread_first_and_budgeted(rag_conn, monkeypatch):
monkeypatch.setattr(tool, "whole_document_context", lambda **kw: None)
+ monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: None)
calls = []
def fake_search(**kw):
@@ -434,6 +437,188 @@ def test_build_rag_autoinject_enabled_path_stays_unbudgeted(rag_conn, monkeypatc
assert calls[0]["min_dense_score"] == inf_tools._autoinject_floor()
+def test_build_rag_autoinject_on_still_grounds_over_budget_doc_below_floor(rag_conn, monkeypatch):
+ from core.rag import embeddings
+
+ _add_doc(
+ rag_conn,
+ store.thread_scope("t1"),
+ "d1",
+ "big.pdf",
+ "h1",
+ ["OVER_BUDGET_PASSAGE"],
+ tokens = [50_000],
+ )
+ monkeypatch.setattr(
+ embeddings,
+ "encode",
+ lambda texts, *, model_name = None, normalize = True: [[1.0, 0.0, 0.0, 0.0] for _ in texts],
+ )
+ monkeypatch.setattr(embeddings, "dim", lambda model_name = None: len(_VEC))
+
+ result = inf_tools.build_rag_autoinject(
+ _convo("Summarize this document"),
+ {"thread_id": "t1", "autoinject": True, "autoinject_min_score": 0.7},
+ )
+ assert result is not None
+ assert "OVER_BUDGET_PASSAGE" in _injected_text(result)
+
+
[email protected]("autoinject", [True, False])
+def test_build_rag_autoinject_lexical_mode_still_grounds_generic_request(
+ rag_conn, monkeypatch, autoinject
+):
+ from core.rag import embeddings
+
+ _add_doc(
+ rag_conn,
+ store.thread_scope("t1"),
+ "d1",
+ "big.pdf",
+ "h1",
+ ["OVER_BUDGET_PASSAGE"],
+ tokens = [50_000],
+ )
+ monkeypatch.setattr(
+ embeddings,
+ "encode",
+ lambda texts, *, model_name = None, normalize = True: [[1.0, 0.0, 0.0, 0.0] for _ in texts],
+ )
+ monkeypatch.setattr(embeddings, "dim", lambda model_name = None: len(_VEC))
+
+ result = inf_tools.build_rag_autoinject(
+ _convo("Summarize this document"),
+ {
+ "thread_id": "t1",
+ "autoinject": autoinject,
+ "autoinject_min_score": 0.7,
+ "mode": "lexical",
+ },
+ )
+ assert result is not None
+ assert "OVER_BUDGET_PASSAGE" in _injected_text(result)
+
+
+def test_build_rag_autoinject_on_grounds_attachment_beside_project_hits(rag_conn, monkeypatch):
+ from core.rag import embeddings
+
+ _add_doc(
+ rag_conn,
+ store.thread_scope("t1"),
+ "d1",
+ "big.pdf",
+ "h1",
+ ["OVER_BUDGET_PASSAGE"],
+ tokens = [50_000],
+ )
+ pscope = store.project_scope("p1")
+ store.create_document(
+ rag_conn, scope = pscope, filename = "notes.md", sha256 = "h2", document_id = "d2"
+ )
+ store.add_chunks(rag_conn, pscope, "d2", [_chunk("PROJECT_PASSAGE")], [[1.0, 0.0, 0.0, 0.0]])
+ store.set_document_status(rag_conn, "d2", "completed", num_chunks = 1)
+ monkeypatch.setattr(
+ embeddings,
+ "encode",
+ lambda texts, *, model_name = None, normalize = True: [[1.0, 0.0, 0.0, 0.0] for _ in texts],
+ )
+ monkeypatch.setattr(embeddings, "dim", lambda model_name = None: len(_VEC))
+
+ result = inf_tools.build_rag_autoinject(
+ _convo("Summarize this document"),
+ {"thread_id": "t1", "project_id": "p1", "autoinject": True, "autoinject_min_score": 0.7},
+ )
+ injected = _injected_text(result)
+ assert injected.index("OVER_BUDGET_PASSAGE") < injected.index("PROJECT_PASSAGE")
+
+
+def test_build_rag_autoinject_on_attachment_beside_project_keeps_lean_top_k(rag_conn, monkeypatch):
+ monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: None)
+
+ def fake_search(**kw):
+ if kw.get("min_dense_score") is None:
+ name, doc = "thread", "d1"
+ else:
+ name, doc = "project", "d2"
+ sources = [
+ {"citationId": i, "documentId": doc, "filename": f"{name}.txt", "text": f"{name}-{i}"}
+ for i in range(1, kw["top_k"] + 1)
+ ]
+ return tool.render_sources(sources), sources
+
+ monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
+ result = inf_tools.build_rag_autoinject(
+ _convo("Summarize this document"),
+ {"thread_id": "t1", "project_id": "p1", "autoinject": True},
+ )
+ injected = _injected_text(result)
+ assert injected.count("<chunk id=") == inf_tools._autoinject_top_k()
+ assert "thread-1" in injected and "project-1" in injected
+ assert injected.index("thread-1") < injected.index("project-1")
+
+
[email protected]("autoinject", [True, False])
+def test_build_rag_autoinject_skips_thread_fallback_without_attachment(
+ rag_conn, monkeypatch, autoinject
+):
+ calls = []
+ monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: calls.append(kw))
+ result = inf_tools.build_rag_autoinject(
+ _convo("Say hello in three words."), {"thread_id": "t1", "autoinject": autoinject}
+ )
+ assert result is None
+ assert not [c for c in calls if c.get("min_dense_score") is None]
+
+
+def test_build_rag_autoinject_on_skips_thread_search_when_attachment_already_hit(
+ rag_conn, monkeypatch
+):
+ monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: {"d1"})
+ calls = []
+
+ def fake_search(**kw):
+ calls.append(kw)
+ sources = [
+ {"citationId": 1, "documentId": "d1", "filename": "big.pdf", "text": "thread hit"}
+ ]
+ return tool.render_sources(sources), sources
+
+ monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
+ result = inf_tools.build_rag_autoinject(
+ _convo("Summarize this document"),
+ {"thread_id": "t1", "project_id": "p1", "autoinject": True},
+ )
+ assert "thread hit" in _injected_text(result)
+ assert len(calls) == 1
+
+
[email protected]("project_id", ["p1", None])
+def test_build_rag_autoinject_on_zero_top_k_still_grounds_attachment(
+ rag_conn, monkeypatch, project_id
+):
+ monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: {"d1"})
+
+ def fake_search(**kw):
+ if kw.get("min_dense_score") is not None and not kw.get("scope_project_id"):
+ return None
+ name, doc = ("thread", "d1") if kw.get("min_dense_score") is None else ("project", "d2")
+ # Zero is falsy to the search, which then returns its configured default count.
+ sources = [
+ {"citationId": i, "documentId": doc, "filename": f"{name}.txt", "text": f"{name}-{i}"}
+ for i in range(1, (kw["top_k"] or 10) + 1)
+ ]
+ return tool.render_sources(sources), sources
+
+ monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
+ result = inf_tools.build_rag_autoinject(
+ _convo("Summarize this document"),
+ {"thread_id": "t1", "project_id": project_id, "autoinject": True, "default_top_k": 0},
+ )
+ injected = _injected_text(result)
+ assert "thread-1" in injected
+ assert injected.count("<chunk id=") == inf_tools._autoinject_top_k()
+
+
def test_build_rag_autoinject_off_does_not_inject_project_alone(rag_conn, monkeypatch):
# The fallback exists to rescue the thread attachment. With auto-injection
# off and nothing found in the thread, project context is not a substitute: