Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: let small models read a long attached file when asked to summarize it (#12580)
Nilaydc8ede02175d
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex 8910f2f3421..cf21abeacbc 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -14697,6 +14697,22 @@ def rag_autoinject_reaches_retrieval(return bool(enabled), whole_doc_requested+def _thread_document_ids(thread_id) -> set | None:+ """Ids of the thread's indexed attachments; None when the store cannot say."""+ try:+ from core.rag import store+ from storage import rag_db++ conn = rag_db.get_connection()+ try:+ docs = store.list_documents(conn, store.thread_scope(thread_id))+ finally:+ conn.close()+ except Exception: # noqa: BLE001+ return None+ return {d["id"] for d in docs if d.get("status") == "completed" and d.get("num_chunks")}++def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> dict | None:"""Pre-retrieve the latest user turn; if a hit clears the cosine floor return ``{"events": [...],"messages": [...]}`` to splice into the loop, else ``None``. Toggle via ``rag_scope.autoinject``@@ -14726,7 +14742,8 @@ def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> di# Cap at the lean top_k, but honor a lower user setting.lean_k = _autoinject_top_k()sidebar_k = _opt_int(rag_scope.get("default_top_k"))- top_k = min(sidebar_k, lean_k) if sidebar_k is not None else lean_k+ # Zero or below is no limit to the search, which then returns its own default count.+ top_k = min(sidebar_k, lean_k) if sidebar_k is not None and sidebar_k > 0 else lean_kbudget: int | None = None# The window the budget was sized against, so `_text_token_cost` only trusts a GGUF actually serving this same# window.@@ -14800,19 +14817,34 @@ def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> difound = search_for_autoinject(query = query, top_k = top_k, **scope)return _trim(found[0], found[1], max_tokens) if found else None+ thread_docs = _thread_document_ids(thread_id) if whole_doc_requested and text is None else set()++ def retrieve_thread_unfloored(*, max_tokens = None):+ # Lexical-only finds nothing for a generic request ("summarize this") whose words are not in the file, so+ # this mandatory grounding retries with the dense leg. Chats with no attachment skip the query embedding.+ if thread_docs is not None and not thread_docs:+ return None+ scope_kwargs = _scope_retrieval_kwargs(rag_scope)+ found = retrieve(+ max_tokens = max_tokens, scope_thread_id = thread_id, min_dense_score = None, **scope_kwargs+ )+ if not found and scope_kwargs["mode"] == "lexical":+ found = retrieve(+ max_tokens = max_tokens,+ scope_thread_id = thread_id,+ min_dense_score = None,+ mode = "hybrid",+ )+ return found+# An oversized thread attachment is mandatory grounding: with auto-injection off, search it alone, without the# optional-auto relevance floor, then add project context if the combination still fits. The budget binds on that# path only: with auto-injection on this stays the single combined unbudgeted search, so a small context cannot- # silently switch RAG off.+ # silently switch RAG off, and the thread is searched again without the floor when none of it cleared.if text is None and (enabled or whole_doc_requested):try:if whole_doc_requested and not enabled:- found = retrieve(- max_tokens = budget,- scope_thread_id = thread_id,- min_dense_score = None,- **_scope_retrieval_kwargs(rag_scope),- )+ found = retrieve_thread_unfloored(max_tokens = budget)project_id = rag_scope.get("project_id")if found and project_id:# Isolated like the whole-document companion above: an unavailable project index must not send the@@ -14839,6 +14871,28 @@ def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> dimin_dense_score = floor,**_scope_retrieval_kwargs(rag_scope),)+ # Project hits clearing the floor must not crowd out the attachment: without one of its passages,+ # it goes first.+ grounded = (+ bool(found)+ and thread_docs is not None+ and any(s.get("documentId") in thread_docs for s in found[1])+ )+ if (+ whole_doc_requested+ and (not found or rag_scope.get("project_id"))+ and not grounded+ ):+ thread_found = retrieve_thread_unfloored()+ if thread_found and found:+ cited = thread_docs or {s.get("documentId") for s in thread_found[1]}+ if not any(s.get("documentId") in cited for s in found[1]):+ # Still the lean top_k in total, so the unbudgeted path never doubles the injection.+ n_proj = min(len(found[1]), top_k // 2)+ merged = thread_found[1][: top_k - n_proj] + found[1][:n_proj]+ found = (render_sources(merged), merged)+ elif thread_found:+ found = thread_foundexcept Exception as exc: # noqa: BLE001logger.warning("RAG auto-inject retrieval failed: %s", exc)return Nonediff --git a/studio/backend/tests/test_rag_whole_document.py b/studio/backend/tests/test_rag_whole_document.pyindex 60397b9bd60..32d3db004ad 100644--- a/studio/backend/tests/test_rag_whole_document.py+++ b/studio/backend/tests/test_rag_whole_document.py@@ -8,6 +8,8 @@ No embedder is needed - the whole-doc path does no query embedding."""import json+import pytest+from core.rag import store, toolfrom core.rag.chunking import Chunkfrom core.inference import tools as inf_tools@@ -301,6 +303,7 @@ def test_build_rag_autoinject_large_model_auto_falls_back_over_budget(rag_conn,def test_build_rag_autoinject_fallback_is_thread_first_and_budgeted(rag_conn, monkeypatch):monkeypatch.setattr(tool, "whole_document_context", lambda **kw: None)+ monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: None)calls = []def fake_search(**kw):@@ -434,6 +437,188 @@ def test_build_rag_autoinject_enabled_path_stays_unbudgeted(rag_conn, monkeypatcassert calls[0]["min_dense_score"] == inf_tools._autoinject_floor()+def test_build_rag_autoinject_on_still_grounds_over_budget_doc_below_floor(rag_conn, monkeypatch):+ from core.rag import embeddings++ _add_doc(+ rag_conn,+ store.thread_scope("t1"),+ "d1",+ "big.pdf",+ "h1",+ ["OVER_BUDGET_PASSAGE"],+ tokens = [50_000],+ )+ monkeypatch.setattr(+ embeddings,+ "encode",+ lambda texts, *, model_name = None, normalize = True: [[1.0, 0.0, 0.0, 0.0] for _ in texts],+ )+ monkeypatch.setattr(embeddings, "dim", lambda model_name = None: len(_VEC))++ result = inf_tools.build_rag_autoinject(+ _convo("Summarize this document"),+ {"thread_id": "t1", "autoinject": True, "autoinject_min_score": 0.7},+ )+ assert result is not None+ assert "OVER_BUDGET_PASSAGE" in _injected_text(result)++[email protected]("autoinject", [True, False])+def test_build_rag_autoinject_lexical_mode_still_grounds_generic_request(+ rag_conn, monkeypatch, autoinject+):+ from core.rag import embeddings++ _add_doc(+ rag_conn,+ store.thread_scope("t1"),+ "d1",+ "big.pdf",+ "h1",+ ["OVER_BUDGET_PASSAGE"],+ tokens = [50_000],+ )+ monkeypatch.setattr(+ embeddings,+ "encode",+ lambda texts, *, model_name = None, normalize = True: [[1.0, 0.0, 0.0, 0.0] for _ in texts],+ )+ monkeypatch.setattr(embeddings, "dim", lambda model_name = None: len(_VEC))++ result = inf_tools.build_rag_autoinject(+ _convo("Summarize this document"),+ {+ "thread_id": "t1",+ "autoinject": autoinject,+ "autoinject_min_score": 0.7,+ "mode": "lexical",+ },+ )+ assert result is not None+ assert "OVER_BUDGET_PASSAGE" in _injected_text(result)+++def test_build_rag_autoinject_on_grounds_attachment_beside_project_hits(rag_conn, monkeypatch):+ from core.rag import embeddings++ _add_doc(+ rag_conn,+ store.thread_scope("t1"),+ "d1",+ "big.pdf",+ "h1",+ ["OVER_BUDGET_PASSAGE"],+ tokens = [50_000],+ )+ pscope = store.project_scope("p1")+ store.create_document(+ rag_conn, scope = pscope, filename = "notes.md", sha256 = "h2", document_id = "d2"+ )+ store.add_chunks(rag_conn, pscope, "d2", [_chunk("PROJECT_PASSAGE")], [[1.0, 0.0, 0.0, 0.0]])+ store.set_document_status(rag_conn, "d2", "completed", num_chunks = 1)+ monkeypatch.setattr(+ embeddings,+ "encode",+ lambda texts, *, model_name = None, normalize = True: [[1.0, 0.0, 0.0, 0.0] for _ in texts],+ )+ monkeypatch.setattr(embeddings, "dim", lambda model_name = None: len(_VEC))++ result = inf_tools.build_rag_autoinject(+ _convo("Summarize this document"),+ {"thread_id": "t1", "project_id": "p1", "autoinject": True, "autoinject_min_score": 0.7},+ )+ injected = _injected_text(result)+ assert injected.index("OVER_BUDGET_PASSAGE") < injected.index("PROJECT_PASSAGE")+++def test_build_rag_autoinject_on_attachment_beside_project_keeps_lean_top_k(rag_conn, monkeypatch):+ monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: None)++ def fake_search(**kw):+ if kw.get("min_dense_score") is None:+ name, doc = "thread", "d1"+ else:+ name, doc = "project", "d2"+ sources = [+ {"citationId": i, "documentId": doc, "filename": f"{name}.txt", "text": f"{name}-{i}"}+ for i in range(1, kw["top_k"] + 1)+ ]+ return tool.render_sources(sources), sources++ monkeypatch.setattr(tool, "search_for_autoinject", fake_search)+ result = inf_tools.build_rag_autoinject(+ _convo("Summarize this document"),+ {"thread_id": "t1", "project_id": "p1", "autoinject": True},+ )+ injected = _injected_text(result)+ assert injected.count("<chunk id=") == inf_tools._autoinject_top_k()+ assert "thread-1" in injected and "project-1" in injected+ assert injected.index("thread-1") < injected.index("project-1")++[email protected]("autoinject", [True, False])+def test_build_rag_autoinject_skips_thread_fallback_without_attachment(+ rag_conn, monkeypatch, autoinject+):+ calls = []+ monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: calls.append(kw))+ result = inf_tools.build_rag_autoinject(+ _convo("Say hello in three words."), {"thread_id": "t1", "autoinject": autoinject}+ )+ assert result is None+ assert not [c for c in calls if c.get("min_dense_score") is None]+++def test_build_rag_autoinject_on_skips_thread_search_when_attachment_already_hit(+ rag_conn, monkeypatch+):+ monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: {"d1"})+ calls = []++ def fake_search(**kw):+ calls.append(kw)+ sources = [+ {"citationId": 1, "documentId": "d1", "filename": "big.pdf", "text": "thread hit"}+ ]+ return tool.render_sources(sources), sources++ monkeypatch.setattr(tool, "search_for_autoinject", fake_search)+ result = inf_tools.build_rag_autoinject(+ _convo("Summarize this document"),+ {"thread_id": "t1", "project_id": "p1", "autoinject": True},+ )+ assert "thread hit" in _injected_text(result)+ assert len(calls) == 1++[email protected]("project_id", ["p1", None])+def test_build_rag_autoinject_on_zero_top_k_still_grounds_attachment(+ rag_conn, monkeypatch, project_id+):+ monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: {"d1"})++ def fake_search(**kw):+ if kw.get("min_dense_score") is not None and not kw.get("scope_project_id"):+ return None+ name, doc = ("thread", "d1") if kw.get("min_dense_score") is None else ("project", "d2")+ # Zero is falsy to the search, which then returns its configured default count.+ sources = [+ {"citationId": i, "documentId": doc, "filename": f"{name}.txt", "text": f"{name}-{i}"}+ for i in range(1, (kw["top_k"] or 10) + 1)+ ]+ return tool.render_sources(sources), sources++ monkeypatch.setattr(tool, "search_for_autoinject", fake_search)+ result = inf_tools.build_rag_autoinject(+ _convo("Summarize this document"),+ {"thread_id": "t1", "project_id": project_id, "autoinject": True, "default_top_k": 0},+ )+ injected = _injected_text(result)+ assert "thread-1" in injected+ assert injected.count("<chunk id=") == inf_tools._autoinject_top_k()++def test_build_rag_autoinject_off_does_not_inject_project_alone(rag_conn, monkeypatch):# The fallback exists to rescue the thread attachment. With auto-injection# off and nothing found in the thread, project context is not a substitute: