Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""whole-document mode injects fitting thread files in full and in order without query embedding."""
import json
import re
import sqlite3
import pytest
from core.rag import chunking, parsers, store, tool
from core.rag.chunking import Chunk
from core.inference import tools as inf_tools
from storage import rag_db
def _shared_setup_1(fake_search, monkeypatch):
monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
injected = _injected_text(
inf_tools.build_rag_autoinject(
_convo(),
{
"thread_id": "t1",
"project_id": "p1",
"autoinject": False,
"context_length": 8192,
"response_headroom": 1024,
},
)
)
return injected
def _shared_setup_2(monkeypatch):
monkeypatch.setattr(
tool,
"search_for_autoinject",
lambda **kw: (_ for _ in ()).throw(AssertionError("retrieval should not run")),
)
# The whole-doc path never reads vectors; dimension must only be consistent per connection.
_VEC = [0.1, 0.2, 0.3, 0.4]
def _chunk(
text,
index = 0,
page = None,
source_page = 0,
tokens = None,
start = 0,
end = None,
whole_tokens = None,
):
return Chunk(
text = text,
token_count = tokens if tokens is not None else len(text.split()),
page_number = page,
source_page_index = source_page,
chunk_index = index,
page_char_start = start,
page_char_end = len(text) if end is None else end,
whole_document_token_count = whole_tokens,
)
def _add_doc(
conn,
scope,
doc_id,
filename,
sha,
texts,
*,
status = "completed",
tokens = None,
pages = None,
):
page_indices = {}
cursors = {}
chunks = []
for i, text in enumerate(texts):
page = pages[i] if pages else None
source_page = page_indices.setdefault(page, len(page_indices))
start = cursors.get(source_page, 0)
end = start + len(text)
chunks.append(
_chunk(
text,
i,
page = page,
source_page = source_page,
tokens = (tokens[i] if tokens else None),
start = start,
end = end,
)
)
cursors[source_page] = end + 1
vectors = [list(_VEC) for _ in texts]
store.create_document(conn, scope = scope, filename = filename, sha256 = sha, document_id = doc_id)
store.add_chunks(conn, scope, doc_id, chunks, vectors)
store.set_document_status(conn, doc_id, status, num_chunks = len(texts))
def _injected_text(result) -> str:
"""The text spliced into the conversation as the synthetic tool result."""
tool_msg = next(m for m in result["messages"] if m.get("role") == "tool")
return tool_msg["content"]
# ── store.all_chunks_for_scope ───────────────────────────────────────
def test_schema_upgrade_adds_chunk_offsets():
conn = sqlite3.connect(":memory:")
conn.execute(
"CREATE TABLE chunks(id TEXT PRIMARY KEY, document_id TEXT, scope TEXT, "
"chunk_index INTEGER, text TEXT, page_number INTEGER, source_page_index INTEGER, "
"token_count INTEGER, kind TEXT, pdf_regions_json TEXT)"
)
rag_db._ensure_schema(conn)
columns = {row[1] for row in conn.execute("PRAGMA table_info(chunks)")}
conn.close()
assert {"page_char_start", "page_char_end", "whole_document_token_count"} <= columns
def test_all_chunks_for_scope_orders_by_document_then_index(rag_conn):
scope = store.thread_scope("t1")
_add_doc(rag_conn, scope, "d1", "first.pdf", "h1", ["a", "b", "c"])
_add_doc(rag_conn, scope, "d2", "second.pdf", "h2", ["x", "y"])
rows = store.all_chunks_for_scope(rag_conn, scope)
assert [r["id"] for r in rows] == ["d1:0", "d1:1", "d1:2", "d2:0", "d2:1"]
assert rows[0]["filename"] == "first.pdf"
assert rows[-1]["filename"] == "second.pdf"
assert rows[0]["text"] == "a"
assert (rows[0]["page_char_start"], rows[0]["page_char_end"]) == (0, 1)
def test_copy_document_index_preserves_chunk_offsets(rag_conn):
scope = store.thread_scope("t1")
store.create_document(
rag_conn, scope = scope, filename = "source.txt", sha256 = "h1", document_id = "src"
)
store.add_chunks(
rag_conn,
scope,
"src",
[_chunk("overlap", start = 4, end = 11, whole_tokens = 5)],
[_VEC],
)
store.create_document(
rag_conn, scope = scope, filename = "copy.txt", sha256 = "h2", document_id = "dst"
)
store.copy_document_index(rag_conn, store.get_document(rag_conn, "src"), "dst", scope)
row = rag_conn.execute(
"SELECT page_char_start, page_char_end, whole_document_token_count "
"FROM chunks WHERE document_id='dst'"
).fetchone()
assert (row["page_char_start"], row["page_char_end"], row["whole_document_token_count"]) == (
4,
11,
5,
)
def test_all_chunks_for_scope_excludes_non_completed(rag_conn):
scope = store.thread_scope("t1")
_add_doc(rag_conn, scope, "done", "done.pdf", "h1", ["ready"])
_add_doc(rag_conn, scope, "pend", "pend.pdf", "h2", ["indexing"], status = "pending")
rows = store.all_chunks_for_scope(rag_conn, scope)
assert [r["id"] for r in rows] == ["done:0"]
def test_all_chunks_for_scope_empty_scope(rag_conn):
assert store.all_chunks_for_scope(rag_conn, store.thread_scope("nope")) == []
def test_all_chunks_for_scope_isolates_scopes(rag_conn):
_add_doc(rag_conn, store.thread_scope("t1"), "d1", "f", "h1", ["mine"])
_add_doc(rag_conn, store.thread_scope("t2"), "d2", "f", "h2", ["theirs"])
rows = store.all_chunks_for_scope(rag_conn, store.thread_scope("t1"))
assert [r["text"] for r in rows] == ["mine"]
def test_scope_token_estimate_sums_without_hydrating(rag_conn):
scope = store.thread_scope("t1")
_add_doc(rag_conn, scope, "d1", "a.pdf", "h1", ["alpha", "bravo"], tokens = [10, 20])
_add_doc(rag_conn, scope, "d2", "b.pdf", "h2", ["x" * 40], tokens = [0])
_add_doc(rag_conn, scope, "d3", "c.pdf", "h3", ["pending"], status = "pending", tokens = [99])
assert store.scope_token_estimate(rag_conn, scope) == 10 + 20 + 10
assert store.scope_token_estimate(rag_conn, store.thread_scope("none")) == 0
def test_scope_token_estimate_matches_row_sum(rag_conn):
from core.rag.tool import _row_token_count
scope = store.thread_scope("t1")
_add_doc(
rag_conn, scope, "d1", "a.pdf", "h1", ["a long-ish chunk body here", "tail"], tokens = [0, 5]
)
rows = store.all_chunks_for_scope(rag_conn, scope)
assert store.scope_token_estimate(rag_conn, scope) == sum(_row_token_count(r) for r in rows)
def test_whole_document_context_returns_full_text_and_sources(rag_conn):
scope = store.thread_scope("t1")
_add_doc(
rag_conn,
scope,
"d1",
"report.pdf",
"h1",
["chapter one body", "chapter two body"],
pages = [1, 2],
)
result = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
assert result is not None
text, sources = result
assert "chapter one body" in text
assert "chapter two body" in text
assert '<chunk id="1"' in text
assert '<chunk id="2"' in text
assert text.index("chapter one") < text.index("chapter two")
assert [s["citationId"] for s in sources] == [1, 2]
assert all(s["filename"] == "report.pdf" for s in sources)
assert all(s["score"] is None for s in sources)
assert [s["page"] for s in sources] == [1, 2]
assert [s["chunkId"] for s in sources] == ["d1:0", "d1:1"]
def test_whole_document_context_none_over_budget(rag_conn):
scope = store.thread_scope("t1")
_add_doc(rag_conn, scope, "d1", "big.pdf", "h1", ["huge"], tokens = [50_000])
assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000) is None
assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = 100_000) is not None
def test_whole_document_context_none_when_empty(rag_conn):
assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000) is None
def test_whole_document_context_non_positive_budget_returns_none(rag_conn):
# A non-positive budget disables whole-doc rather than injecting everything unbounded.
scope = store.thread_scope("t1")
_add_doc(rag_conn, scope, "d1", "a.pdf", "h1", ["tiny body"])
assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = 0) is None
assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = -5) is None
def test_whole_document_context_none_without_scope(rag_conn):
assert tool.whole_document_context(max_tokens = 6000) is None
def test_whole_document_context_null_token_count_enforces_budget(rag_conn):
big = "word " * 20_000
_add_doc(rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", [big], tokens = [None])
assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000) is None
assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = 1_000_000) is not None
def test_whole_document_context_spans_multiple_docs(rag_conn):
scope = store.thread_scope("t1")
_add_doc(rag_conn, scope, "d1", "a.pdf", "h1", ["alpha text"])
_add_doc(rag_conn, scope, "d2", "b.pdf", "h2", ["bravo text"])
text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
assert "alpha text" in text and "bravo text" in text
assert {s["filename"] for s in sources} == {"a.pdf", "b.pdf"}
def test_whole_document_context_reads_each_overlapping_line_once(rag_conn):
lines = [f"R{i:03d} | Travel | Vendor number {i} | ${i}.00" for i in range(1, 71)]
pages = parsers.parse_text("\n".join(lines))
chunks = chunking.chunk_pages(pages, max_tokens = 200, overlap = 32, count = lambda t: len(t.split()))
assert len(chunks) > 1
store.create_document(
rag_conn, scope = store.thread_scope("t1"), filename = "exp.docx", sha256 = "h1", document_id = "d1"
)
store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])
store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))
text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
assert re.findall(r"R\d{3}", text) == [f"R{i:03d}" for i in range(1, 71)]
assert [s["chunkId"] for s in sources] == [f"d1:{i}" for i in range(len(chunks))]
def test_whole_document_context_keeps_text_repeated_on_the_next_page(rag_conn):
_add_doc(
rag_conn,
store.thread_scope("t1"),
"d1",
"report.pdf",
"h1",
["page one body\nACME Confidential", "ACME Confidential\npage two body"],
pages = [1, 2],
)
text, _sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
assert text.count("ACME Confidential") == 2
def test_whole_document_context_keeps_indentation_after_overlap(rag_conn):
page = "paragraph\nshared line\n indented code"
overlap = page.index("shared line")
chunks = [
_chunk(page[: page.index("\n indented code")], end = page.index("\n indented code")),
_chunk(page[overlap:], 1, start = overlap, end = len(page)),
]
store.create_document(
rag_conn,
scope = store.thread_scope("t1"),
filename = "code.md",
sha256 = "h1",
document_id = "d1",
)
store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])
store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))
_text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
assert sources[1]["text"] == " indented code"
def test_whole_document_context_keeps_matching_text_without_actual_overlap(rag_conn):
page = "section\nTOTAL\nTOTAL\nnext"
second_start = page.index("TOTAL", page.index("TOTAL") + 1)
chunks = [
_chunk(page[: second_start - 1], end = second_start - 1),
_chunk(page[second_start:], 1, start = second_start, end = len(page)),
]
store.create_document(
rag_conn,
scope = store.thread_scope("t1"),
filename = "totals.txt",
sha256 = "h1",
document_id = "d1",
)
store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])
store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))
text, _sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
assert text.count("TOTAL") == 2
def test_whole_document_context_removes_overlap_inside_unbroken_text(rag_conn):
page = "0123456789" * 8
chunks = chunking.chunk_pages(parsers.parse_text(page), max_tokens = 20, overlap = 5, count = len)
assert len(chunks) > 1
store.create_document(
rag_conn,
scope = store.thread_scope("t1"),
filename = "data.txt",
sha256 = "h1",
document_id = "d1",
)
store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])
store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))
_text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
assert "".join(source["text"] for source in sources) == page
def test_whole_document_context_budgets_deoverlapped_content(rag_conn):
page = " ".join(f"word{i}" for i in range(30))
chunks = chunking.chunk_pages(parsers.parse_text(page), max_tokens = 30, overlap = 10, count = len)
previous_end = 0
unique_tokens = 0
for chunk in chunks:
unique_start = max(previous_end, chunk.page_char_start)
unique_tokens += len(chunk.text[unique_start - chunk.page_char_start :])
previous_end = max(previous_end, chunk.page_char_end)
assert sum(chunk.token_count for chunk in chunks) > unique_tokens
assert sum(chunk.whole_document_token_count for chunk in chunks) == unique_tokens
store.create_document(
rag_conn,
scope = store.thread_scope("t1"),
filename = "near-limit.txt",
sha256 = "h1",
document_id = "d1",
)
store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])
store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))
assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = unique_tokens)
def test_whole_document_context_keeps_legacy_chunks_without_offsets(rag_conn):
_add_doc(
rag_conn,
store.thread_scope("t1"),
"d1",
"legacy.txt",
"h1",
["section\nTOTAL", "TOTAL\nnext"],
)
rag_conn.execute(
"UPDATE chunks SET page_char_start=NULL, page_char_end=NULL WHERE document_id='d1'"
)
rag_conn.commit()
text, _sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
assert text.count("TOTAL") == 2
def _convo(text = "summarize the whole document"):
return [{"role": "user", "content": text}]
def test_build_rag_autoinject_uses_whole_doc(rag_conn):
scope = store.thread_scope("t1")
_add_doc(rag_conn, scope, "d1", "doc.pdf", "h1", ["whole alpha part", "whole bravo part"])
result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1"})
assert result is not None
injected = _injected_text(result)
assert "whole alpha part" in injected
assert "whole bravo part" in injected
assert inf_tools.RAG_SOURCES_SENTINEL not in injected
def test_build_rag_autoinject_whole_doc_runs_when_autoinject_false(rag_conn, monkeypatch):
_add_doc(rag_conn, store.thread_scope("t1"), "d1", "doc.pdf", "h1", ["entire file body"])
_shared_setup_2(monkeypatch)
result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1", "autoinject": False})
assert result is not None
assert "entire file body" in _injected_text(result)
def test_build_rag_autoinject_explicit_off_disables_whole_doc(rag_conn, monkeypatch):
_add_doc(rag_conn, store.thread_scope("t1"), "d1", "doc.pdf", "h1", ["small body"])
_shared_setup_2(monkeypatch)
assert (
inf_tools.build_rag_autoinject(
_convo(), {"thread_id": "t1", "autoinject": False, "whole_doc": False}
)
is None
)
def test_build_rag_autoinject_falls_back_over_budget(rag_conn, monkeypatch):
scope = store.thread_scope("t1")
_add_doc(rag_conn, scope, "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000])
sentinel = ("TOPK_FALLBACK_TEXT", [{"citationId": 1, "filename": "big.pdf", "text": "x"}])
monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: sentinel)
result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1"})
assert result is not None
assert _injected_text(result) == "TOPK_FALLBACK_TEXT"
def test_build_rag_autoinject_large_model_auto_falls_back_over_budget(rag_conn, monkeypatch):
# Auto resolves to autoinject=False over 9B; oversized attachments fall back to top-K.
scope = store.thread_scope("t1")
_add_doc(rag_conn, scope, "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000])
sentinel = (
"TOPK_LARGE_MODEL_FALLBACK",
[{"citationId": 1, "filename": "big.pdf", "text": "x"}],
)
monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: sentinel)
result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1", "autoinject": False})
assert result is not None
assert _injected_text(result) == "TOPK_LARGE_MODEL_FALLBACK"
def test_build_rag_autoinject_fallback_is_thread_first_and_budgeted(rag_conn, monkeypatch):
monkeypatch.setattr(tool, "whole_document_context", lambda **kw: None)
monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: None)
calls = []
def fake_search(**kw):
calls.append(kw)
if kw.get("scope_thread_id"):
if kw.get("scope_project_id") or kw.get("min_dense_score") is not None:
return None
sources = [
{"citationId": i, "filename": "thread.txt", "text": "x" * 1600}
for i in range(1, kw["top_k"] + 1)
]
else:
sources = [{"citationId": 1, "filename": "project.txt", "text": "project"}]
return tool.render_sources(sources), sources
monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
result = inf_tools.build_rag_autoinject(
_convo("summarize the attached document"),
{
"thread_id": "t1",
"project_id": "p1",
"autoinject": False,
"context_length": 3000,
"response_headroom": 1024,
},
)
injected = _injected_text(result)
assert "x" * 1600 in injected and "project" in injected
assert [(c.get("scope_thread_id"), c.get("scope_project_id")) for c in calls] == [
("t1", None),
(None, "p1"),
]
assert calls[0]["min_dense_score"] is None
assert calls[0]["top_k"] == 4
# Budget leaves room for one 1600-char passage plus the project hit, not all four.
assert injected.count("x" * 1600) == 1
def test_build_rag_autoinject_zero_budget_still_grounds(rag_conn, monkeypatch):
_add_doc(
rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000]
)
sentinel = ("TOPK_ZERO_BUDGET", [{"citationId": 1, "filename": "big.pdf", "text": "x"}])
monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: sentinel)
for scope in (
{"thread_id": "t1", "autoinject": True, "context_length": 1200},
{"thread_id": "t1", "autoinject": True, "context_length": 512},
{"thread_id": "t1", "autoinject": True, "context_length": 8192, "response_headroom": 8192},
{
"thread_id": "t1",
"autoinject": True,
"context_length": 2048,
"response_headroom": 999_999,
},
{"thread_id": "t1", "autoinject": True, "context_length": 2048, "default_top_k": 0},
):
result = inf_tools.build_rag_autoinject(_convo(), scope)
assert result is not None, scope
assert _injected_text(result) == "TOPK_ZERO_BUDGET", scope
def test_build_rag_autoinject_refuses_when_not_even_one_passage_fits(rag_conn, monkeypatch):
# The current turn may not be evicted, so when no passage fits, inject nothing rather than fail.
_add_doc(
rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000]
)
def fake_search(**kw):
sources = [
{"citationId": 0, "filename": "thread.txt", "text": "T" * 2000}
for _ in range(kw["top_k"] or 4)
]
return tool.render_sources(sources), sources
monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
scope = {"thread_id": "t1", "autoinject": False, "context_length": 1200}
assert inf_tools._whole_doc_budget(scope, _convo()) == 0
assert inf_tools.build_rag_autoinject(_convo(), scope) is None
assert (
inf_tools.build_rag_autoinject(
_convo(), {"thread_id": "t1", "autoinject": True, "context_length": 1200}
)
is not None
)
def test_build_rag_autoinject_enabled_path_stays_unbudgeted(rag_conn, monkeypatch):
_add_doc(
rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000]
)
calls = []
def fake_search(**kw):
calls.append(kw)
sources = [
{"citationId": i, "filename": "thread.txt", "text": "y" * 6000}
for i in range(1, (kw["top_k"] or 4) + 1)
]
return tool.render_sources(sources), sources
monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
long_thread = [
{"role": "user", "content": "x " * 6000},
{"role": "assistant", "content": "ok"},
{"role": "user", "content": "summarize the attached document"},
]
result = inf_tools.build_rag_autoinject(
long_thread, {"thread_id": "t1", "autoinject": True, "context_length": 8192}
)
assert result is not None
assert _injected_text(result).count("y" * 6000) == 4
assert len(calls) == 1
assert calls[0]["min_dense_score"] == inf_tools._autoinject_floor()
def test_build_rag_autoinject_on_still_grounds_over_budget_doc_below_floor(rag_conn, monkeypatch):
from core.rag import embeddings
_add_doc(
rag_conn,
store.thread_scope("t1"),
"d1",
"big.pdf",
"h1",
["OVER_BUDGET_PASSAGE"],
tokens = [50_000],
)
monkeypatch.setattr(
embeddings,
"encode",
lambda texts, *, model_name = None, normalize = True: [[1.0, 0.0, 0.0, 0.0] for _ in texts],
)
monkeypatch.setattr(embeddings, "dim", lambda model_name = None: len(_VEC))
result = inf_tools.build_rag_autoinject(
_convo("Summarize this document"),
{"thread_id": "t1", "autoinject": True, "autoinject_min_score": 0.7},
)
assert result is not None
assert "OVER_BUDGET_PASSAGE" in _injected_text(result)
@pytest.mark.parametrize("autoinject", [True, False])
def test_build_rag_autoinject_lexical_mode_still_grounds_generic_request(
rag_conn, monkeypatch, autoinject
):
from core.rag import embeddings
_add_doc(
rag_conn,
store.thread_scope("t1"),
"d1",
"big.pdf",
"h1",
["OVER_BUDGET_PASSAGE"],
tokens = [50_000],
)
monkeypatch.setattr(
embeddings,
"encode",
lambda texts, *, model_name = None, normalize = True: [[1.0, 0.0, 0.0, 0.0] for _ in texts],
)
monkeypatch.setattr(embeddings, "dim", lambda model_name = None: len(_VEC))
result = inf_tools.build_rag_autoinject(
_convo("Summarize this document"),
{
"thread_id": "t1",
"autoinject": autoinject,
"autoinject_min_score": 0.7,
"mode": "lexical",
},
)
assert result is not None
assert "OVER_BUDGET_PASSAGE" in _injected_text(result)
def test_build_rag_autoinject_on_grounds_attachment_beside_project_hits(rag_conn, monkeypatch):
from core.rag import embeddings
_add_doc(
rag_conn,
store.thread_scope("t1"),
"d1",
"big.pdf",
"h1",
["OVER_BUDGET_PASSAGE"],
tokens = [50_000],
)
pscope = store.project_scope("p1")
store.create_document(
rag_conn, scope = pscope, filename = "notes.md", sha256 = "h2", document_id = "d2"
)
store.add_chunks(rag_conn, pscope, "d2", [_chunk("PROJECT_PASSAGE")], [[1.0, 0.0, 0.0, 0.0]])
store.set_document_status(rag_conn, "d2", "completed", num_chunks = 1)
monkeypatch.setattr(
embeddings,
"encode",
lambda texts, *, model_name = None, normalize = True: [[1.0, 0.0, 0.0, 0.0] for _ in texts],
)
monkeypatch.setattr(embeddings, "dim", lambda model_name = None: len(_VEC))
result = inf_tools.build_rag_autoinject(
_convo("Summarize this document"),
{"thread_id": "t1", "project_id": "p1", "autoinject": True, "autoinject_min_score": 0.7},
)
injected = _injected_text(result)
assert injected.index("OVER_BUDGET_PASSAGE") < injected.index("PROJECT_PASSAGE")
def test_build_rag_autoinject_on_attachment_beside_project_keeps_lean_top_k(rag_conn, monkeypatch):
monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: None)
def fake_search(**kw):
if kw.get("min_dense_score") is None:
name, doc = "thread", "d1"
else:
name, doc = "project", "d2"
sources = [
{"citationId": i, "documentId": doc, "filename": f"{name}.txt", "text": f"{name}-{i}"}
for i in range(1, kw["top_k"] + 1)
]
return tool.render_sources(sources), sources
monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
result = inf_tools.build_rag_autoinject(
_convo("Summarize this document"),
{"thread_id": "t1", "project_id": "p1", "autoinject": True},
)
injected = _injected_text(result)
assert injected.count("<chunk id=") == inf_tools._autoinject_top_k()
assert "thread-1" in injected and "project-1" in injected
assert injected.index("thread-1") < injected.index("project-1")
@pytest.mark.parametrize("autoinject", [True, False])
def test_build_rag_autoinject_skips_thread_fallback_without_attachment(
rag_conn, monkeypatch, autoinject
):
calls = []
monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: calls.append(kw))
result = inf_tools.build_rag_autoinject(
_convo("Say hello in three words."), {"thread_id": "t1", "autoinject": autoinject}
)
assert result is None
assert not [c for c in calls if c.get("min_dense_score") is None]
def test_build_rag_autoinject_on_skips_thread_search_when_attachment_already_hit(
rag_conn, monkeypatch
):
monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: {"d1"})
calls = []
def fake_search(**kw):
calls.append(kw)
sources = [
{"citationId": 1, "documentId": "d1", "filename": "big.pdf", "text": "thread hit"}
]
return tool.render_sources(sources), sources
monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
result = inf_tools.build_rag_autoinject(
_convo("Summarize this document"),
{"thread_id": "t1", "project_id": "p1", "autoinject": True},
)
assert "thread hit" in _injected_text(result)
assert len(calls) == 1
@pytest.mark.parametrize("project_id", ["p1", None])
def test_build_rag_autoinject_on_zero_top_k_still_grounds_attachment(
rag_conn, monkeypatch, project_id
):
monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: {"d1"})
def fake_search(**kw):
if kw.get("min_dense_score") is not None and not kw.get("scope_project_id"):
return None
name, doc = ("thread", "d1") if kw.get("min_dense_score") is None else ("project", "d2")
# Zero is falsy to the search, which then returns its configured default count.
sources = [
{"citationId": i, "documentId": doc, "filename": f"{name}.txt", "text": f"{name}-{i}"}
for i in range(1, (kw["top_k"] or 10) + 1)
]
return tool.render_sources(sources), sources
monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
result = inf_tools.build_rag_autoinject(
_convo("Summarize this document"),
{"thread_id": "t1", "project_id": project_id, "autoinject": True, "default_top_k": 0},
)
injected = _injected_text(result)
assert "thread-1" in injected
assert injected.count("<chunk id=") == inf_tools._autoinject_top_k()
def test_build_rag_autoinject_off_does_not_inject_project_alone(rag_conn, monkeypatch):
_add_doc(
rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000]
)
def fake_search(**kw):
if kw.get("scope_thread_id"):
return None
sources = [{"citationId": 1, "filename": "project.txt", "text": "project"}]
return tool.render_sources(sources), sources
monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
assert (
inf_tools.build_rag_autoinject(
_convo(), {"thread_id": "t1", "project_id": "p1", "autoinject": False}
)
is None
)
def test_build_rag_autoinject_keeps_thread_hits_when_project_retrieval_raises(
rag_conn, monkeypatch
):
_add_doc(
rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000]
)
def fake_search(**kw):
if kw.get("scope_project_id"):
raise RuntimeError("project index unavailable")
sources = [{"citationId": 1, "filename": "thread.txt", "text": "THREAD_GROUNDING"}]
return tool.render_sources(sources), sources
monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
result = inf_tools.build_rag_autoinject(
_convo(), {"thread_id": "t1", "project_id": "p1", "autoinject": False}
)
assert result is not None
assert "THREAD_GROUNDING" in _injected_text(result)
def test_build_rag_autoinject_keeps_the_project_hits_that_fit(rag_conn, monkeypatch):
_add_doc(
rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000]
)
def fake_search(**kw):
if kw.get("scope_project_id"):
sources = [
{"citationId": 0, "filename": "project.txt", "text": "P" * 2000} for _ in range(4)
]
else:
sources = [{"citationId": 0, "filename": "thread.txt", "text": "T" * 8000}]
return tool.render_sources(sources), sources
injected = _shared_setup_1(fake_search, monkeypatch)
assert "T" * 8000 in injected
assert 1 <= injected.count("P" * 2000) < 4
def test_build_rag_autoinject_whole_doc_merge_is_priced_in_tokens(rag_conn, monkeypatch):
# A CJK character is about one token, not four.
_add_doc(rag_conn, store.thread_scope("t1"), "d1", "doc.pdf", "h1", ["THREAD_DOC"])
def fake_search(**kw):
sources = [
{"citationId": 0, "filename": "project.txt", "text": "漢" * 2000} for _ in range(4)
]
return tool.render_sources(sources), sources
injected = _shared_setup_1(fake_search, monkeypatch)
assert "THREAD_DOC" in injected
assert 1 <= injected.count("漢" * 2000) < 4
def test_build_rag_autoinject_whole_doc_merge_never_truncates_the_document(rag_conn, monkeypatch):
body = ["SECTION%02d " % i + "word " * 398 for i in range(8)]
_add_doc(
rag_conn,
store.thread_scope("t1"),
"d1",
"report.pdf",
"h1",
body,
tokens = [500] * len(body),
)
def fake_search(**kw):
sources = [{"citationId": 0, "filename": "notes.txt", "text": "PROJECT_HIT " + "x" * 500}]
return tool.render_sources(sources), sources
injected = _shared_setup_1(fake_search, monkeypatch)
assert [i for i in range(len(body)) if "SECTION%02d" % i in injected] == list(range(len(body)))
assert "PROJECT_HIT" not in injected
def test_build_rag_autoinject_budget_charges_multibyte_text_more(rag_conn, monkeypatch):
_add_doc(
rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000]
)
def make_search(body):
def fake_search(**kw):
sources = [{"citationId": 0, "filename": "thread.txt", "text": body} for _ in range(4)]
return tool.render_sources(sources), sources
return fake_search
scope = {
"thread_id": "t1",
"autoinject": False,
"context_length": 4096,
"response_headroom": 1024,
}
def _chunks(body):
monkeypatch.setattr(tool, "search_for_autoinject", make_search(body))
result = inf_tools.build_rag_autoinject(_convo(), scope)
return _injected_text(result).count('<chunk id="') if result else 0
ascii_chunks = _chunks("a" * 2000)
cjk_chunks = _chunks("漢" * 2000)
assert ascii_chunks >= 1
assert cjk_chunks < ascii_chunks
def test_build_rag_autoinject_context_budget_falls_back(rag_conn, monkeypatch):
_add_doc(
rag_conn, store.thread_scope("t1"), "d1", "small.pdf", "h1", ["fits global"], tokens = [900]
)
sentinel = ("TOPK_CONTEXT_FALLBACK", [{"citationId": 1, "filename": "small.pdf", "text": "x"}])
monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: sentinel)
result = inf_tools.build_rag_autoinject(
_convo(), {"thread_id": "t1", "context_length": 1200, "whole_doc": True}
)
assert result is not None
assert _injected_text(result) == "TOPK_CONTEXT_FALLBACK"
def test_whole_doc_budget_reserves_image_parts(monkeypatch):
from core.rag import config
monkeypatch.setattr(config, "WHOLE_DOC_MAX_TOKENS", 10_000)
scope = {"context_length": 7000, "response_headroom": 1000}
text_only = [{"role": "user", "content": [{"type": "text", "text": "summarize"}]}]
with_image = [
{
"role": "user",
"content": [
{"type": "text", "text": "summarize"},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}},
],
}
]
assert (
inf_tools._whole_doc_budget(scope, text_only)
- inf_tools._whole_doc_budget(scope, with_image)
== inf_tools._IMAGE_PART_TOKEN_ESTIMATE
)
def test_build_rag_autoinject_server_kill_switch_blocks_whole_doc(rag_conn, monkeypatch):
from core.rag import config
monkeypatch.setattr(config, "THREAD_WHOLE_DOC", False)
_add_doc(rag_conn, store.thread_scope("t1"), "d1", "doc.pdf", "h1", ["small body"])
_shared_setup_2(monkeypatch)
assert (
inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1", "autoinject": False}) is None
)
def test_whole_document_context_budgets_rendered_wrappers(rag_conn):
# Wrapper overhead counts: budget the rendered prompt, not stored text.
texts = ["x" for _ in range(120)]
_add_doc(
rag_conn,
store.thread_scope("t1"),
"d1",
"many-pages.pdf",
"h1",
texts,
tokens = [1 for _ in texts],
)
assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = 500) is None
def test_build_rag_autoinject_whole_doc_disabled_via_override(rag_conn, monkeypatch):
scope = store.thread_scope("t1")
_add_doc(rag_conn, scope, "d1", "doc.pdf", "h1", ["small body"])
sentinel = ("TOPK_TEXT", [{"citationId": 1, "filename": "doc.pdf", "text": "x"}])
monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: sentinel)
result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1", "whole_doc": False})
assert result is not None
assert _injected_text(result) == "TOPK_TEXT"
def test_build_rag_autoinject_kb_scope_never_whole_doc(rag_conn, monkeypatch):
kb_scope = store.kb_scope("K1")
_add_doc(rag_conn, kb_scope, "d1", "kb.pdf", "h1", ["kb body one", "kb body two"])
sentinel = ("KB_RETRIEVAL_TEXT", [{"citationId": 1, "filename": "kb.pdf", "text": "x"}])
monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: sentinel)
result = inf_tools.build_rag_autoinject(_convo(), {"kb_id": "K1"})
assert result is not None
assert _injected_text(result) == "KB_RETRIEVAL_TEXT"
def test_whole_document_context_thread_scope_only(rag_conn):
_add_doc(rag_conn, store.thread_scope("t1"), "td", "thread.txt", "h1", ["thread attachment"])
_add_doc(rag_conn, store.project_scope("p1"), "pd", "project.txt", "h2", ["project corpus"])
text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
assert "thread attachment" in text
assert "project corpus" not in text
assert {s["filename"] for s in sources} == {"thread.txt"}
def test_build_rag_autoinject_appends_project_retrieval(rag_conn, monkeypatch):
_add_doc(
rag_conn,
store.thread_scope("t1"),
"td",
"thread.txt",
"h1",
["thread chunk one", "thread chunk two"],
)
proj = (
"PROJ",
[
{
"citationId": 1,
"chunkId": "pj:0",
"documentId": "pj",
"filename": "project.txt",
"page": None,
"text": "project passage zeta",
"score": 0.91,
}
],
)
captured = {}
def fake_search(**kw):
captured.update(kw)
return proj
monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1", "project_id": "p1"})
injected = _injected_text(result)
assert "thread chunk one" in injected
assert "thread chunk two" in injected
assert "project passage zeta" in injected
assert captured.get("scope_project_id") == "p1"
assert captured.get("scope_thread_id") is None
assert captured.get("scope_kb_id") is None
assert '<chunk id="1"' in injected
assert '<chunk id="2"' in injected
assert '<chunk id="3"' in injected
def test_build_rag_autoinject_skips_project_companion_over_budget(rag_conn, monkeypatch):
_add_doc(rag_conn, store.thread_scope("t1"), "td", "thread.txt", "h1", ["thread body"])
project_text = "project overflow " * 2000
proj = (
"PROJ",
[
{
"citationId": 1,
"chunkId": "pj:0",
"documentId": "pj",
"filename": "project.txt",
"page": None,
"text": project_text,
"score": 0.91,
}
],
)
monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: proj)
result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1", "project_id": "p1"})
injected = _injected_text(result)
assert "thread body" in injected
assert "project overflow" not in injected
def test_build_rag_autoinject_thread_whole_doc_ignores_project_size(rag_conn, monkeypatch):
monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: None)
_add_doc(rag_conn, store.thread_scope("t1"), "td", "thread.txt", "h1", ["small thread file"])
_add_doc(
rag_conn, store.project_scope("p1"), "pd", "project.txt", "h2", ["big"], tokens = [50_000]
)
result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1", "project_id": "p1"})
assert "small thread file" in _injected_text(result)
def test_build_rag_autoinject_kb_defers_to_retrieval(rag_conn, monkeypatch):
_add_doc(rag_conn, store.thread_scope("t1"), "td", "thread.txt", "h1", ["thread attachment"])
sentinel = ("KB_RETRIEVAL", [{"citationId": 1, "filename": "kb.pdf", "text": "x"}])
monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: sentinel)
result = inf_tools.build_rag_autoinject(_convo(), {"kb_id": "K1", "thread_id": "t1"})
assert _injected_text(result) == "KB_RETRIEVAL"
def test_build_rag_autoinject_no_scope_returns_none(rag_conn):
assert inf_tools.build_rag_autoinject(_convo(), None) is None
assert inf_tools.build_rag_autoinject(_convo(), {}) is None
def test_build_rag_autoinject_args_carry_user_query(rag_conn):
scope = store.thread_scope("t1")
_add_doc(rag_conn, scope, "d1", "doc.pdf", "h1", ["small body"])
result = inf_tools.build_rag_autoinject(_convo("what is in here"), {"thread_id": "t1"})
assistant_msg = next(m for m in result["messages"] if m.get("role") == "assistant")
args = json.loads(assistant_msg["tool_calls"][0]["function"]["arguments"])
assert args["query"] == "what is in here"
def test_real_ingestion_feeds_whole_document(rag_conn, stub_embeddings, tmp_path):
"""Real ingestion feeds whole-document injection the full text, not just retrieved chunks."""
from core.rag import ingestion
scope = store.thread_scope("t1")
body = (
"# Quarterly Report\n\n"
+ ("Revenue rose across every region this period. " * 40)
+ "\n\nThe unique closing marker is xyzzy-sentinel for the final page. " * 40
)
src = tmp_path / "report.md"
src.write_text(body, encoding = "utf-8")
document_id = store.create_document(
rag_conn,
scope = scope,
filename = "report.md",
sha256 = "sha-e2e",
thread_id = "t1",
status = "pending",
stored_path = str(src),
)
job_id = ingestion._new_job(rag_conn, document_id, scope)
ingestion._run(job_id, document_id, scope, str(src), None)
doc = store.get_document(rag_conn, document_id)
assert doc["status"] == "completed"
assert doc["num_chunks"] >= 2
result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1"})
assert result is not None
injected = _injected_text(result)
assert "Revenue rose" in injected
assert "xyzzy-sentinel" in injected
assert injected.count("<chunk id=") == doc["num_chunks"]