Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: send each line of a file attached in Chat with files once (#13179)

Nilayfd6902036589
diff --git a/studio/backend/core/rag/chunking.py b/studio/backend/core/rag/chunking.py
index 3200f87aa74..3e627d0c14a 100644
--- a/studio/backend/core/rag/chunking.py
+++ b/studio/backend/core/rag/chunking.py
@@ -25,6 +25,7 @@ class Chunk:
chunk_index: int
page_char_start: int
page_char_end: int
+ whole_document_token_count: int | None = None
@dataclass(frozen = True, slots = True)
@@ -126,16 +127,24 @@ def chunk_pages(
for piece in pieces:
starts.append(cursor)
cursor += len(piece.text)
+ previous_end = 0
for text, char_start, char_end in _merge(pieces, starts, max_tokens, overlap):
+ token_count = count(text)
+ overlap_chars = min(len(text), max(0, previous_end - char_start))
+ unique_text = text[overlap_chars:].lstrip("\r\n")
out.append(
Chunk(
text = text,
- token_count = count(text),
+ token_count = token_count,
page_number = page.page_number,
source_page_index = page_index,
chunk_index = len(out),
page_char_start = char_start,
page_char_end = char_end,
+ whole_document_token_count = (
+ token_count if unique_text == text else count(unique_text)
+ ),
)
)
+ previous_end = max(previous_end, char_end)
return out
diff --git a/studio/backend/core/rag/store.py b/studio/backend/core/rag/store.py
index 43b9c7632a6..8e7ecd48c5c 100644
--- a/studio/backend/core/rag/store.py
+++ b/studio/backend/core/rag/store.py
@@ -491,8 +491,9 @@ def add_chunks(
conn.execute(
"INSERT OR REPLACE INTO chunks("
"id, document_id, scope, chunk_index, text, page_number, "
- "source_page_index, token_count, kind, pdf_regions_json) "
- "VALUES(?,?,?,?,?,?,?,?,?,?)",
+ "source_page_index, page_char_start, page_char_end, token_count, "
+ "whole_document_token_count, kind, pdf_regions_json) "
+ "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)",
(
chunk_id,
document_id,
@@ -501,7 +502,10 @@ def add_chunks(
chunk.text,
chunk.page_number,
chunk.source_page_index,
+ getattr(chunk, "page_char_start", None),
+ getattr(chunk, "page_char_end", None),
chunk.token_count,
+ getattr(chunk, "whole_document_token_count", None),
getattr(chunk, "kind", "text"),
regions_json,
),
@@ -556,9 +560,11 @@ def _copy_chunk_rows(
conn.execute(
"INSERT INTO chunks("
"id, document_id, scope, chunk_index, text, page_number, "
- "source_page_index, token_count, kind, pdf_regions_json) "
+ "source_page_index, page_char_start, page_char_end, token_count, "
+ "whole_document_token_count, kind, pdf_regions_json) "
"SELECT ? || ':' || chunk_index, ?, ?, chunk_index, text, page_number, "
- "source_page_index, token_count, kind, pdf_regions_json "
+ "source_page_index, page_char_start, page_char_end, token_count, "
+ "whole_document_token_count, kind, pdf_regions_json "
"FROM chunks WHERE document_id=?",
(target_id, target_id, scope, source_id),
)
@@ -973,7 +979,9 @@ def all_chunks_for_scope(conn: sqlite3.Connection, scope) -> list[dict]:
placeholders = ",".join("?" * len(scopes))
rows = conn.execute(
f"SELECT c.id, c.text, c.document_id, c.chunk_index, c.page_number, "
- f"c.token_count, d.filename, d.created_at "
+ f"c.source_page_index, c.page_char_start, c.page_char_end, c.token_count, "
+ f"c.whole_document_token_count, "
+ f"d.filename, d.created_at "
f"FROM chunks c JOIN documents d ON d.id=c.document_id "
f"WHERE c.scope IN ({placeholders}) AND d.status='completed' "
f"AND NOT EXISTS "
@@ -988,14 +996,17 @@ def all_chunks_for_scope(conn: sqlite3.Connection, scope) -> list[dict]:
def scope_token_estimate(conn: sqlite3.Connection, scope) -> int:
"""Upper-bound token total for a scope's completed chunks without hydrating text.
- Mirrors ``all_chunks_for_scope`` + the ``tool._row_token_count`` fallback (stored
- count, else length/4), so the whole-doc budget can be checked before loading text."""
+ Mirrors ``all_chunks_for_scope`` + whole-document token budgeting (unique stored
+ count, else full stored count, else length/4), so the budget can be checked before loading text."""
scopes = _scopes(scope)
if not scopes:
return 0
placeholders = ",".join("?" * len(scopes))
row = conn.execute(
- f"SELECT COALESCE(SUM(CASE WHEN c.token_count > 0 THEN c.token_count "
+ f"SELECT COALESCE(SUM(CASE "
+ f"WHEN c.whole_document_token_count IS NOT NULL "
+ f"THEN MAX(0, c.whole_document_token_count) "
+ f"WHEN c.token_count > 0 THEN c.token_count "
f"ELSE MAX(1, length(COALESCE(c.text, '')) / 4) END), 0) AS total "
f"FROM chunks c JOIN documents d ON d.id=c.document_id "
f"WHERE c.scope IN ({placeholders}) AND d.status='completed' "
diff --git a/studio/backend/core/rag/tool.py b/studio/backend/core/rag/tool.py
index ac882579926..db57485db52 100644
--- a/studio/backend/core/rag/tool.py
+++ b/studio/backend/core/rag/tool.py
@@ -216,6 +216,11 @@ def _row_token_count(row) -> int:
return max(1, len(row["text"] or "") // 4)
+def _whole_document_token_count(row) -> int:
+ token_count = _row_value(row, "whole_document_token_count")
+ return max(0, int(token_count)) if token_count is not None else _row_token_count(row)
+
+
def search_knowledge_base_with_sources(
*,
query: str,
@@ -316,26 +321,25 @@ def search_for_autoinject(
return (text, sources) if sources else None
+def _drop_chunk_overlap(text: str, start: int | None, prev_end: int | None) -> str:
+ if not isinstance(start, int) or not isinstance(prev_end, int) or start >= prev_end:
+ return text
+ return text[min(prev_end - start, len(text)) :].lstrip("\r\n")
+
+
def whole_document_context(
*, scope_thread_id: str | None = None, max_tokens: int
) -> tuple[str, list[dict]] | None:
- """Render EVERY chunk of the THREAD's attached documents (in order) as the same
- ``<chunk>`` blocks + citation source-map as retrieval, so the model reads the whole
- file rather than top-K passages. Thread-attached files only: KB and project corpora
- are search corpora, never whole-document, so this resolves the thread scope alone.
- ``None`` (caller falls back to retrieval) when there is no thread scope, no completed
- chunks, or the total exceeds ``max_tokens``."""
+ """render ordered thread attachment chunks with citation blocks; KB and project corpora stay search-only; return None when absent or over max_tokens."""
if not scope_thread_id:
return None
- # A non-positive budget means "never inject" (disable whole-doc via RAG_THREAD_WHOLE_DOC=0), not
- # "inject the whole corpus unbounded".
+ # a non-positive budget disables whole-document injection rather than removing the limit
if max_tokens <= 0:
return None
scope = thread_scope(scope_thread_id)
conn = rag_db.get_connection()
try:
- # Cheap SUM pre-check so an oversized attachment is rejected before the whole corpus is hydrated;
- # all_chunks_for_scope runs only once it fits.
+ # reject oversized attachments before all_chunks_for_scope hydrates the corpus
if scope_token_estimate(conn, scope) > max_tokens:
return None
rows = all_chunks_for_scope(conn, scope)
@@ -343,22 +347,30 @@ def whole_document_context(
conn.close()
if not rows:
return None
- total = sum(_row_token_count(r) for r in rows)
+ total = sum(_whole_document_token_count(r) for r in rows)
if total > max_tokens:
return None
- sources: list[dict] = [
- {
- "citationId": i,
- "chunkId": r["id"],
- "documentId": r["document_id"],
- "filename": r["filename"] or "unknown",
- "page": r["page_number"],
- "text": r["text"] or "",
- "score": None,
- }
- for i, r in enumerate(rows, 1)
- ]
+ sources: list[dict] = []
+ prev_page, prev_end = None, None
+ for i, r in enumerate(rows, 1):
+ page = (r["document_id"], r["source_page_index"])
+ text = r["text"] or ""
+ trimmed = (
+ _drop_chunk_overlap(text, r["page_char_start"], prev_end) if page == prev_page else text
+ )
+ sources.append(
+ {
+ "citationId": i,
+ "chunkId": r["id"],
+ "documentId": r["document_id"],
+ "filename": r["filename"] or "unknown",
+ "page": r["page_number"],
+ "text": trimmed,
+ "score": None,
+ }
+ )
+ prev_page, prev_end = page, r["page_char_end"]
rendered = render_sources(sources)
if max(1, len(rendered) // 4) > max_tokens:
return None
diff --git a/studio/backend/storage/rag_db.py b/studio/backend/storage/rag_db.py
index 5cf14d40735..1475dfdd177 100644
--- a/studio/backend/storage/rag_db.py
+++ b/studio/backend/storage/rag_db.py
@@ -128,7 +128,10 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:
text TEXT NOT NULL,
page_number INTEGER,
source_page_index INTEGER,
+ page_char_start INTEGER,
+ page_char_end INTEGER,
token_count INTEGER,
+ whole_document_token_count INTEGER,
kind TEXT NOT NULL DEFAULT 'text',
pdf_regions_json TEXT
);
@@ -252,6 +255,13 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:
# recover the order within a compaction epoch.
if "archive_ordinal" not in cols:
conn.execute("ALTER TABLE documents ADD COLUMN archive_ordinal INTEGER")
+ chunk_cols = {r[1] for r in conn.execute("PRAGMA table_info(chunks)").fetchall()}
+ if "page_char_start" not in chunk_cols:
+ conn.execute("ALTER TABLE chunks ADD COLUMN page_char_start INTEGER")
+ if "page_char_end" not in chunk_cols:
+ conn.execute("ALTER TABLE chunks ADD COLUMN page_char_end INTEGER")
+ if "whole_document_token_count" not in chunk_cols:
+ conn.execute("ALTER TABLE chunks ADD COLUMN whole_document_token_count INTEGER")
# Partial, so it is empty until a chat is compacted and the MAX() that allocates the next ordinal is
# an index probe rather than a scan.
conn.execute(
diff --git a/studio/backend/tests/test_rag_whole_document.py b/studio/backend/tests/test_rag_whole_document.py
index 32d3db004ad..8570f201321 100644
--- a/studio/backend/tests/test_rag_whole_document.py
+++ b/studio/backend/tests/test_rag_whole_document.py
@@ -1,18 +1,18 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Whole-document context mode: a thread-attached file small enough to fit is
-injected in full (every chunk, in order) instead of top-K retrieval. Covers the
-new store query, the tool-level renderer, and the auto-inject wiring + fallback.
-No embedder is needed - the whole-doc path does no query embedding."""
+"""whole-document mode injects fitting thread files in full and in order without query embedding."""
import json
+import re
+import sqlite3
import pytest
-from core.rag import store, tool
+from core.rag import chunking, parsers, store, tool
from core.rag.chunking import Chunk
from core.inference import tools as inf_tools
+from storage import rag_db
def _shared_setup_1(fake_search, monkeypatch):
@@ -49,16 +49,21 @@ def _chunk(
text,
index = 0,
page = None,
+ source_page = 0,
tokens = None,
+ start = 0,
+ end = None,
+ whole_tokens = None,
):
return Chunk(
text = text,
token_count = tokens if tokens is not None else len(text.split()),
page_number = page,
- source_page_index = 0,
+ source_page_index = source_page,
chunk_index = index,
- page_char_start = 0,
- page_char_end = len(text),
+ page_char_start = start,
+ page_char_end = len(text) if end is None else end,
+ whole_document_token_count = whole_tokens,
)
@@ -74,15 +79,26 @@ def _add_doc(
tokens = None,
pages = None,
):
- chunks = [
- _chunk(
- t,
- i,
- page = (pages[i] if pages else None),
- tokens = (tokens[i] if tokens else None),
+ page_indices = {}
+ cursors = {}
+ chunks = []
+ for i, text in enumerate(texts):
+ page = pages[i] if pages else None
+ source_page = page_indices.setdefault(page, len(page_indices))
+ start = cursors.get(source_page, 0)
+ end = start + len(text)
+ chunks.append(
+ _chunk(
+ text,
+ i,
+ page = page,
+ source_page = source_page,
+ tokens = (tokens[i] if tokens else None),
+ start = start,
+ end = end,
+ )
)
- for i, t in enumerate(texts)
- ]
+ cursors[source_page] = end + 1
vectors = [list(_VEC) for _ in texts]
store.create_document(conn, scope = scope, filename = filename, sha256 = sha, document_id = doc_id)
store.add_chunks(conn, scope, doc_id, chunks, vectors)
@@ -98,6 +114,19 @@ def _injected_text(result) -> str:
# ── store.all_chunks_for_scope ───────────────────────────────────────
+def test_schema_upgrade_adds_chunk_offsets():
+ conn = sqlite3.connect(":memory:")
+ conn.execute(
+ "CREATE TABLE chunks(id TEXT PRIMARY KEY, document_id TEXT, scope TEXT, "
+ "chunk_index INTEGER, text TEXT, page_number INTEGER, source_page_index INTEGER, "
+ "token_count INTEGER, kind TEXT, pdf_regions_json TEXT)"
+ )
+ rag_db._ensure_schema(conn)
+ columns = {row[1] for row in conn.execute("PRAGMA table_info(chunks)")}
+ conn.close()
+ assert {"page_char_start", "page_char_end", "whole_document_token_count"} <= columns
+
+
def test_all_chunks_for_scope_orders_by_document_then_index(rag_conn):
scope = store.thread_scope("t1")
_add_doc(rag_conn, scope, "d1", "first.pdf", "h1", ["a", "b", "c"])
@@ -107,6 +136,34 @@ def test_all_chunks_for_scope_orders_by_document_then_index(rag_conn):
assert rows[0]["filename"] == "first.pdf"
assert rows[-1]["filename"] == "second.pdf"
assert rows[0]["text"] == "a"
+ assert (rows[0]["page_char_start"], rows[0]["page_char_end"]) == (0, 1)
+
+
+def test_copy_document_index_preserves_chunk_offsets(rag_conn):
+ scope = store.thread_scope("t1")
+ store.create_document(
+ rag_conn, scope = scope, filename = "source.txt", sha256 = "h1", document_id = "src"
+ )
+ store.add_chunks(
+ rag_conn,
+ scope,
+ "src",
+ [_chunk("overlap", start = 4, end = 11, whole_tokens = 5)],
+ [_VEC],
+ )
+ store.create_document(
+ rag_conn, scope = scope, filename = "copy.txt", sha256 = "h2", document_id = "dst"
+ )
+ store.copy_document_index(rag_conn, store.get_document(rag_conn, "src"), "dst", scope)
+ row = rag_conn.execute(
+ "SELECT page_char_start, page_char_end, whole_document_token_count "
+ "FROM chunks WHERE document_id='dst'"
+ ).fetchone()
+ assert (row["page_char_start"], row["page_char_end"], row["whole_document_token_count"]) == (
+ 4,
+ 11,
+ 5,
+ )
def test_all_chunks_for_scope_excludes_non_completed(rag_conn):
@@ -229,7 +286,130 @@ def test_whole_document_context_spans_multiple_docs(rag_conn):
assert {s["filename"] for s in sources} == {"a.pdf", "b.pdf"}
-# ── build_rag_autoinject wiring ──────────────────────────────────────
+def test_whole_document_context_reads_each_overlapping_line_once(rag_conn):
+ lines = [f"R{i:03d} | Travel | Vendor number {i} | ${i}.00" for i in range(1, 71)]
+ pages = parsers.parse_text("\n".join(lines))
+ chunks = chunking.chunk_pages(pages, max_tokens = 200, overlap = 32, count = lambda t: len(t.split()))
+ assert len(chunks) > 1
+ store.create_document(
+ rag_conn, scope = store.thread_scope("t1"), filename = "exp.docx", sha256 = "h1", document_id = "d1"
+ )
+ store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])
+ store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))
+ text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
+ assert re.findall(r"R\d{3}", text) == [f"R{i:03d}" for i in range(1, 71)]
+ assert [s["chunkId"] for s in sources] == [f"d1:{i}" for i in range(len(chunks))]
+
+
+def test_whole_document_context_keeps_text_repeated_on_the_next_page(rag_conn):
+ _add_doc(
+ rag_conn,
+ store.thread_scope("t1"),
+ "d1",
+ "report.pdf",
+ "h1",
+ ["page one body\nACME Confidential", "ACME Confidential\npage two body"],
+ pages = [1, 2],
+ )
+ text, _sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
+ assert text.count("ACME Confidential") == 2
+
+
+def test_whole_document_context_keeps_indentation_after_overlap(rag_conn):
+ page = "paragraph\nshared line\n indented code"
+ overlap = page.index("shared line")
+ chunks = [
+ _chunk(page[: page.index("\n indented code")], end = page.index("\n indented code")),
+ _chunk(page[overlap:], 1, start = overlap, end = len(page)),
+ ]
+ store.create_document(
+ rag_conn,
+ scope = store.thread_scope("t1"),
+ filename = "code.md",
+ sha256 = "h1",
+ document_id = "d1",
+ )
+ store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])
+ store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))
+ _text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
+ assert sources[1]["text"] == " indented code"
+
+
+def test_whole_document_context_keeps_matching_text_without_actual_overlap(rag_conn):
+ page = "section\nTOTAL\nTOTAL\nnext"
+ second_start = page.index("TOTAL", page.index("TOTAL") + 1)
+ chunks = [
+ _chunk(page[: second_start - 1], end = second_start - 1),
+ _chunk(page[second_start:], 1, start = second_start, end = len(page)),
+ ]
+ store.create_document(
+ rag_conn,
+ scope = store.thread_scope("t1"),
+ filename = "totals.txt",
+ sha256 = "h1",
+ document_id = "d1",
+ )
+ store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])
+ store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))
+ text, _sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
+ assert text.count("TOTAL") == 2
+
+
+def test_whole_document_context_removes_overlap_inside_unbroken_text(rag_conn):
+ page = "0123456789" * 8
+ chunks = chunking.chunk_pages(parsers.parse_text(page), max_tokens = 20, overlap = 5, count = len)
+ assert len(chunks) > 1
+ store.create_document(
+ rag_conn,
+ scope = store.thread_scope("t1"),
+ filename = "data.txt",
+ sha256 = "h1",
+ document_id = "d1",
+ )
+ store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])
+ store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))
+ _text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
+ assert "".join(source["text"] for source in sources) == page
+
+
+def test_whole_document_context_budgets_deoverlapped_content(rag_conn):
+ page = " ".join(f"word{i}" for i in range(30))
+ chunks = chunking.chunk_pages(parsers.parse_text(page), max_tokens = 30, overlap = 10, count = len)
+ previous_end = 0
+ unique_tokens = 0
+ for chunk in chunks:
+ unique_start = max(previous_end, chunk.page_char_start)
+ unique_tokens += len(chunk.text[unique_start - chunk.page_char_start :])
+ previous_end = max(previous_end, chunk.page_char_end)
+ assert sum(chunk.token_count for chunk in chunks) > unique_tokens
+ assert sum(chunk.whole_document_token_count for chunk in chunks) == unique_tokens
+ store.create_document(
+ rag_conn,
+ scope = store.thread_scope("t1"),
+ filename = "near-limit.txt",
+ sha256 = "h1",
+ document_id = "d1",
+ )
+ store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])
+ store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))
+ assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = unique_tokens)
+
+
+def test_whole_document_context_keeps_legacy_chunks_without_offsets(rag_conn):
+ _add_doc(
+ rag_conn,
+ store.thread_scope("t1"),
+ "d1",
+ "legacy.txt",
+ "h1",
+ ["section\nTOTAL", "TOTAL\nnext"],
+ )
+ rag_conn.execute(
+ "UPDATE chunks SET page_char_start=NULL, page_char_end=NULL WHERE document_id='d1'"
+ )
+ rag_conn.commit()
+ text, _sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
+ assert text.count("TOTAL") == 2
def _convo(text = "summarize the whole document"):