Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: send each line of a file attached in Chat with files once (#13179)
Nilayfd6902036589
diff --git a/studio/backend/core/rag/chunking.py b/studio/backend/core/rag/chunking.pyindex 3200f87aa74..3e627d0c14a 100644--- a/studio/backend/core/rag/chunking.py+++ b/studio/backend/core/rag/chunking.py@@ -25,6 +25,7 @@ class Chunk:chunk_index: intpage_char_start: intpage_char_end: int+ whole_document_token_count: int | None = None@dataclass(frozen = True, slots = True)@@ -126,16 +127,24 @@ def chunk_pages(for piece in pieces:starts.append(cursor)cursor += len(piece.text)+ previous_end = 0for text, char_start, char_end in _merge(pieces, starts, max_tokens, overlap):+ token_count = count(text)+ overlap_chars = min(len(text), max(0, previous_end - char_start))+ unique_text = text[overlap_chars:].lstrip("\r\n")out.append(Chunk(text = text,- token_count = count(text),+ token_count = token_count,page_number = page.page_number,source_page_index = page_index,chunk_index = len(out),page_char_start = char_start,page_char_end = char_end,+ whole_document_token_count = (+ token_count if unique_text == text else count(unique_text)+ ),))+ previous_end = max(previous_end, char_end)return outdiff --git a/studio/backend/core/rag/store.py b/studio/backend/core/rag/store.pyindex 43b9c7632a6..8e7ecd48c5c 100644--- a/studio/backend/core/rag/store.py+++ b/studio/backend/core/rag/store.py@@ -491,8 +491,9 @@ def add_chunks(conn.execute("INSERT OR REPLACE INTO chunks(""id, document_id, scope, chunk_index, text, page_number, "- "source_page_index, token_count, kind, pdf_regions_json) "- "VALUES(?,?,?,?,?,?,?,?,?,?)",+ "source_page_index, page_char_start, page_char_end, token_count, "+ "whole_document_token_count, kind, pdf_regions_json) "+ "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)",(chunk_id,document_id,@@ -501,7 +502,10 @@ def add_chunks(chunk.text,chunk.page_number,chunk.source_page_index,+ getattr(chunk, "page_char_start", None),+ getattr(chunk, "page_char_end", None),chunk.token_count,+ getattr(chunk, "whole_document_token_count", None),getattr(chunk, "kind", "text"),regions_json,),@@ -556,9 +560,11 @@ def _copy_chunk_rows(conn.execute("INSERT INTO chunks(""id, document_id, scope, chunk_index, text, page_number, "- "source_page_index, token_count, kind, pdf_regions_json) "+ "source_page_index, page_char_start, page_char_end, token_count, "+ "whole_document_token_count, kind, pdf_regions_json) ""SELECT ? || ':' || chunk_index, ?, ?, chunk_index, text, page_number, "- "source_page_index, token_count, kind, pdf_regions_json "+ "source_page_index, page_char_start, page_char_end, token_count, "+ "whole_document_token_count, kind, pdf_regions_json ""FROM chunks WHERE document_id=?",(target_id, target_id, scope, source_id),)@@ -973,7 +979,9 @@ def all_chunks_for_scope(conn: sqlite3.Connection, scope) -> list[dict]:placeholders = ",".join("?" * len(scopes))rows = conn.execute(f"SELECT c.id, c.text, c.document_id, c.chunk_index, c.page_number, "- f"c.token_count, d.filename, d.created_at "+ f"c.source_page_index, c.page_char_start, c.page_char_end, c.token_count, "+ f"c.whole_document_token_count, "+ f"d.filename, d.created_at "f"FROM chunks c JOIN documents d ON d.id=c.document_id "f"WHERE c.scope IN ({placeholders}) AND d.status='completed' "f"AND NOT EXISTS "@@ -988,14 +996,17 @@ def all_chunks_for_scope(conn: sqlite3.Connection, scope) -> list[dict]:def scope_token_estimate(conn: sqlite3.Connection, scope) -> int:"""Upper-bound token total for a scope's completed chunks without hydrating text.- Mirrors ``all_chunks_for_scope`` + the ``tool._row_token_count`` fallback (stored- count, else length/4), so the whole-doc budget can be checked before loading text."""+ Mirrors ``all_chunks_for_scope`` + whole-document token budgeting (unique stored+ count, else full stored count, else length/4), so the budget can be checked before loading text."""scopes = _scopes(scope)if not scopes:return 0placeholders = ",".join("?" * len(scopes))row = conn.execute(- f"SELECT COALESCE(SUM(CASE WHEN c.token_count > 0 THEN c.token_count "+ f"SELECT COALESCE(SUM(CASE "+ f"WHEN c.whole_document_token_count IS NOT NULL "+ f"THEN MAX(0, c.whole_document_token_count) "+ f"WHEN c.token_count > 0 THEN c.token_count "f"ELSE MAX(1, length(COALESCE(c.text, '')) / 4) END), 0) AS total "f"FROM chunks c JOIN documents d ON d.id=c.document_id "f"WHERE c.scope IN ({placeholders}) AND d.status='completed' "diff --git a/studio/backend/core/rag/tool.py b/studio/backend/core/rag/tool.pyindex ac882579926..db57485db52 100644--- a/studio/backend/core/rag/tool.py+++ b/studio/backend/core/rag/tool.py@@ -216,6 +216,11 @@ def _row_token_count(row) -> int:return max(1, len(row["text"] or "") // 4)+def _whole_document_token_count(row) -> int:+ token_count = _row_value(row, "whole_document_token_count")+ return max(0, int(token_count)) if token_count is not None else _row_token_count(row)++def search_knowledge_base_with_sources(*,query: str,@@ -316,26 +321,25 @@ def search_for_autoinject(return (text, sources) if sources else None+def _drop_chunk_overlap(text: str, start: int | None, prev_end: int | None) -> str:+ if not isinstance(start, int) or not isinstance(prev_end, int) or start >= prev_end:+ return text+ return text[min(prev_end - start, len(text)) :].lstrip("\r\n")++def whole_document_context(*, scope_thread_id: str | None = None, max_tokens: int) -> tuple[str, list[dict]] | None:- """Render EVERY chunk of the THREAD's attached documents (in order) as the same- ``<chunk>`` blocks + citation source-map as retrieval, so the model reads the whole- file rather than top-K passages. Thread-attached files only: KB and project corpora- are search corpora, never whole-document, so this resolves the thread scope alone.- ``None`` (caller falls back to retrieval) when there is no thread scope, no completed- chunks, or the total exceeds ``max_tokens``."""+ """render ordered thread attachment chunks with citation blocks; KB and project corpora stay search-only; return None when absent or over max_tokens."""if not scope_thread_id:return None- # A non-positive budget means "never inject" (disable whole-doc via RAG_THREAD_WHOLE_DOC=0), not- # "inject the whole corpus unbounded".+ # a non-positive budget disables whole-document injection rather than removing the limitif max_tokens <= 0:return Nonescope = thread_scope(scope_thread_id)conn = rag_db.get_connection()try:- # Cheap SUM pre-check so an oversized attachment is rejected before the whole corpus is hydrated;- # all_chunks_for_scope runs only once it fits.+ # reject oversized attachments before all_chunks_for_scope hydrates the corpusif scope_token_estimate(conn, scope) > max_tokens:return Nonerows = all_chunks_for_scope(conn, scope)@@ -343,22 +347,30 @@ def whole_document_context(conn.close()if not rows:return None- total = sum(_row_token_count(r) for r in rows)+ total = sum(_whole_document_token_count(r) for r in rows)if total > max_tokens:return None- sources: list[dict] = [- {- "citationId": i,- "chunkId": r["id"],- "documentId": r["document_id"],- "filename": r["filename"] or "unknown",- "page": r["page_number"],- "text": r["text"] or "",- "score": None,- }- for i, r in enumerate(rows, 1)- ]+ sources: list[dict] = []+ prev_page, prev_end = None, None+ for i, r in enumerate(rows, 1):+ page = (r["document_id"], r["source_page_index"])+ text = r["text"] or ""+ trimmed = (+ _drop_chunk_overlap(text, r["page_char_start"], prev_end) if page == prev_page else text+ )+ sources.append(+ {+ "citationId": i,+ "chunkId": r["id"],+ "documentId": r["document_id"],+ "filename": r["filename"] or "unknown",+ "page": r["page_number"],+ "text": trimmed,+ "score": None,+ }+ )+ prev_page, prev_end = page, r["page_char_end"]rendered = render_sources(sources)if max(1, len(rendered) // 4) > max_tokens:return Nonediff --git a/studio/backend/storage/rag_db.py b/studio/backend/storage/rag_db.pyindex 5cf14d40735..1475dfdd177 100644--- a/studio/backend/storage/rag_db.py+++ b/studio/backend/storage/rag_db.py@@ -128,7 +128,10 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:text TEXT NOT NULL,page_number INTEGER,source_page_index INTEGER,+ page_char_start INTEGER,+ page_char_end INTEGER,token_count INTEGER,+ whole_document_token_count INTEGER,kind TEXT NOT NULL DEFAULT 'text',pdf_regions_json TEXT);@@ -252,6 +255,13 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:# recover the order within a compaction epoch.if "archive_ordinal" not in cols:conn.execute("ALTER TABLE documents ADD COLUMN archive_ordinal INTEGER")+ chunk_cols = {r[1] for r in conn.execute("PRAGMA table_info(chunks)").fetchall()}+ if "page_char_start" not in chunk_cols:+ conn.execute("ALTER TABLE chunks ADD COLUMN page_char_start INTEGER")+ if "page_char_end" not in chunk_cols:+ conn.execute("ALTER TABLE chunks ADD COLUMN page_char_end INTEGER")+ if "whole_document_token_count" not in chunk_cols:+ conn.execute("ALTER TABLE chunks ADD COLUMN whole_document_token_count INTEGER")# Partial, so it is empty until a chat is compacted and the MAX() that allocates the next ordinal is# an index probe rather than a scan.conn.execute(diff --git a/studio/backend/tests/test_rag_whole_document.py b/studio/backend/tests/test_rag_whole_document.pyindex 32d3db004ad..8570f201321 100644--- a/studio/backend/tests/test_rag_whole_document.py+++ b/studio/backend/tests/test_rag_whole_document.py@@ -1,18 +1,18 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""Whole-document context mode: a thread-attached file small enough to fit is-injected in full (every chunk, in order) instead of top-K retrieval. Covers the-new store query, the tool-level renderer, and the auto-inject wiring + fallback.-No embedder is needed - the whole-doc path does no query embedding."""+"""whole-document mode injects fitting thread files in full and in order without query embedding."""import json+import re+import sqlite3import pytest-from core.rag import store, tool+from core.rag import chunking, parsers, store, toolfrom core.rag.chunking import Chunkfrom core.inference import tools as inf_tools+from storage import rag_dbdef _shared_setup_1(fake_search, monkeypatch):@@ -49,16 +49,21 @@ def _chunk(text,index = 0,page = None,+ source_page = 0,tokens = None,+ start = 0,+ end = None,+ whole_tokens = None,):return Chunk(text = text,token_count = tokens if tokens is not None else len(text.split()),page_number = page,- source_page_index = 0,+ source_page_index = source_page,chunk_index = index,- page_char_start = 0,- page_char_end = len(text),+ page_char_start = start,+ page_char_end = len(text) if end is None else end,+ whole_document_token_count = whole_tokens,)@@ -74,15 +79,26 @@ def _add_doc(tokens = None,pages = None,):- chunks = [- _chunk(- t,- i,- page = (pages[i] if pages else None),- tokens = (tokens[i] if tokens else None),+ page_indices = {}+ cursors = {}+ chunks = []+ for i, text in enumerate(texts):+ page = pages[i] if pages else None+ source_page = page_indices.setdefault(page, len(page_indices))+ start = cursors.get(source_page, 0)+ end = start + len(text)+ chunks.append(+ _chunk(+ text,+ i,+ page = page,+ source_page = source_page,+ tokens = (tokens[i] if tokens else None),+ start = start,+ end = end,+ ))- for i, t in enumerate(texts)- ]+ cursors[source_page] = end + 1vectors = [list(_VEC) for _ in texts]store.create_document(conn, scope = scope, filename = filename, sha256 = sha, document_id = doc_id)store.add_chunks(conn, scope, doc_id, chunks, vectors)@@ -98,6 +114,19 @@ def _injected_text(result) -> str:# ── store.all_chunks_for_scope ───────────────────────────────────────+def test_schema_upgrade_adds_chunk_offsets():+ conn = sqlite3.connect(":memory:")+ conn.execute(+ "CREATE TABLE chunks(id TEXT PRIMARY KEY, document_id TEXT, scope TEXT, "+ "chunk_index INTEGER, text TEXT, page_number INTEGER, source_page_index INTEGER, "+ "token_count INTEGER, kind TEXT, pdf_regions_json TEXT)"+ )+ rag_db._ensure_schema(conn)+ columns = {row[1] for row in conn.execute("PRAGMA table_info(chunks)")}+ conn.close()+ assert {"page_char_start", "page_char_end", "whole_document_token_count"} <= columns++def test_all_chunks_for_scope_orders_by_document_then_index(rag_conn):scope = store.thread_scope("t1")_add_doc(rag_conn, scope, "d1", "first.pdf", "h1", ["a", "b", "c"])@@ -107,6 +136,34 @@ def test_all_chunks_for_scope_orders_by_document_then_index(rag_conn):assert rows[0]["filename"] == "first.pdf"assert rows[-1]["filename"] == "second.pdf"assert rows[0]["text"] == "a"+ assert (rows[0]["page_char_start"], rows[0]["page_char_end"]) == (0, 1)+++def test_copy_document_index_preserves_chunk_offsets(rag_conn):+ scope = store.thread_scope("t1")+ store.create_document(+ rag_conn, scope = scope, filename = "source.txt", sha256 = "h1", document_id = "src"+ )+ store.add_chunks(+ rag_conn,+ scope,+ "src",+ [_chunk("overlap", start = 4, end = 11, whole_tokens = 5)],+ [_VEC],+ )+ store.create_document(+ rag_conn, scope = scope, filename = "copy.txt", sha256 = "h2", document_id = "dst"+ )+ store.copy_document_index(rag_conn, store.get_document(rag_conn, "src"), "dst", scope)+ row = rag_conn.execute(+ "SELECT page_char_start, page_char_end, whole_document_token_count "+ "FROM chunks WHERE document_id='dst'"+ ).fetchone()+ assert (row["page_char_start"], row["page_char_end"], row["whole_document_token_count"]) == (+ 4,+ 11,+ 5,+ )def test_all_chunks_for_scope_excludes_non_completed(rag_conn):@@ -229,7 +286,130 @@ def test_whole_document_context_spans_multiple_docs(rag_conn):assert {s["filename"] for s in sources} == {"a.pdf", "b.pdf"}-# ── build_rag_autoinject wiring ──────────────────────────────────────+def test_whole_document_context_reads_each_overlapping_line_once(rag_conn):+ lines = [f"R{i:03d} | Travel | Vendor number {i} | ${i}.00" for i in range(1, 71)]+ pages = parsers.parse_text("\n".join(lines))+ chunks = chunking.chunk_pages(pages, max_tokens = 200, overlap = 32, count = lambda t: len(t.split()))+ assert len(chunks) > 1+ store.create_document(+ rag_conn, scope = store.thread_scope("t1"), filename = "exp.docx", sha256 = "h1", document_id = "d1"+ )+ store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])+ store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))+ text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)+ assert re.findall(r"R\d{3}", text) == [f"R{i:03d}" for i in range(1, 71)]+ assert [s["chunkId"] for s in sources] == [f"d1:{i}" for i in range(len(chunks))]+++def test_whole_document_context_keeps_text_repeated_on_the_next_page(rag_conn):+ _add_doc(+ rag_conn,+ store.thread_scope("t1"),+ "d1",+ "report.pdf",+ "h1",+ ["page one body\nACME Confidential", "ACME Confidential\npage two body"],+ pages = [1, 2],+ )+ text, _sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)+ assert text.count("ACME Confidential") == 2+++def test_whole_document_context_keeps_indentation_after_overlap(rag_conn):+ page = "paragraph\nshared line\n indented code"+ overlap = page.index("shared line")+ chunks = [+ _chunk(page[: page.index("\n indented code")], end = page.index("\n indented code")),+ _chunk(page[overlap:], 1, start = overlap, end = len(page)),+ ]+ store.create_document(+ rag_conn,+ scope = store.thread_scope("t1"),+ filename = "code.md",+ sha256 = "h1",+ document_id = "d1",+ )+ store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])+ store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))+ _text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)+ assert sources[1]["text"] == " indented code"+++def test_whole_document_context_keeps_matching_text_without_actual_overlap(rag_conn):+ page = "section\nTOTAL\nTOTAL\nnext"+ second_start = page.index("TOTAL", page.index("TOTAL") + 1)+ chunks = [+ _chunk(page[: second_start - 1], end = second_start - 1),+ _chunk(page[second_start:], 1, start = second_start, end = len(page)),+ ]+ store.create_document(+ rag_conn,+ scope = store.thread_scope("t1"),+ filename = "totals.txt",+ sha256 = "h1",+ document_id = "d1",+ )+ store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])+ store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))+ text, _sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)+ assert text.count("TOTAL") == 2+++def test_whole_document_context_removes_overlap_inside_unbroken_text(rag_conn):+ page = "0123456789" * 8+ chunks = chunking.chunk_pages(parsers.parse_text(page), max_tokens = 20, overlap = 5, count = len)+ assert len(chunks) > 1+ store.create_document(+ rag_conn,+ scope = store.thread_scope("t1"),+ filename = "data.txt",+ sha256 = "h1",+ document_id = "d1",+ )+ store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])+ store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))+ _text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)+ assert "".join(source["text"] for source in sources) == page+++def test_whole_document_context_budgets_deoverlapped_content(rag_conn):+ page = " ".join(f"word{i}" for i in range(30))+ chunks = chunking.chunk_pages(parsers.parse_text(page), max_tokens = 30, overlap = 10, count = len)+ previous_end = 0+ unique_tokens = 0+ for chunk in chunks:+ unique_start = max(previous_end, chunk.page_char_start)+ unique_tokens += len(chunk.text[unique_start - chunk.page_char_start :])+ previous_end = max(previous_end, chunk.page_char_end)+ assert sum(chunk.token_count for chunk in chunks) > unique_tokens+ assert sum(chunk.whole_document_token_count for chunk in chunks) == unique_tokens+ store.create_document(+ rag_conn,+ scope = store.thread_scope("t1"),+ filename = "near-limit.txt",+ sha256 = "h1",+ document_id = "d1",+ )+ store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])+ store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))+ assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = unique_tokens)+++def test_whole_document_context_keeps_legacy_chunks_without_offsets(rag_conn):+ _add_doc(+ rag_conn,+ store.thread_scope("t1"),+ "d1",+ "legacy.txt",+ "h1",+ ["section\nTOTAL", "TOTAL\nnext"],+ )+ rag_conn.execute(+ "UPDATE chunks SET page_char_start=NULL, page_char_end=NULL WHERE document_id='d1'"+ )+ rag_conn.commit()+ text, _sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)+ assert text.count("TOTAL") == 2def _convo(text = "summarize the whole document"):