Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

41.7 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""whole-document mode injects fitting thread files in full and in order without query embedding."""

import json
import re
import sqlite3

import pytest

from core.rag import chunking, parsers, store, tool
from core.rag.chunking import Chunk
from core.inference import tools as inf_tools
from storage import rag_db


def _shared_setup_1(fake_search, monkeypatch):
    monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
    injected = _injected_text(
        inf_tools.build_rag_autoinject(
            _convo(),
            {
                "thread_id": "t1",
                "project_id": "p1",
                "autoinject": False,
                "context_length": 8192,
                "response_headroom": 1024,
            },
        )
    )
    return injected


def _shared_setup_2(monkeypatch):
    monkeypatch.setattr(
        tool,
        "search_for_autoinject",
        lambda **kw: (_ for _ in ()).throw(AssertionError("retrieval should not run")),
    )


# The whole-doc path never reads vectors; dimension must only be consistent per connection.
_VEC = [0.1, 0.2, 0.3, 0.4]


def _chunk(
    text,
    index = 0,
    page = None,
    source_page = 0,
    tokens = None,
    start = 0,
    end = None,
    whole_tokens = None,
):
    return Chunk(
        text = text,
        token_count = tokens if tokens is not None else len(text.split()),
        page_number = page,
        source_page_index = source_page,
        chunk_index = index,
        page_char_start = start,
        page_char_end = len(text) if end is None else end,
        whole_document_token_count = whole_tokens,
    )


def _add_doc(
    conn,
    scope,
    doc_id,
    filename,
    sha,
    texts,
    *,
    status = "completed",
    tokens = None,
    pages = None,
):
    page_indices = {}
    cursors = {}
    chunks = []
    for i, text in enumerate(texts):
        page = pages[i] if pages else None
        source_page = page_indices.setdefault(page, len(page_indices))
        start = cursors.get(source_page, 0)
        end = start + len(text)
        chunks.append(
            _chunk(
                text,
                i,
                page = page,
                source_page = source_page,
                tokens = (tokens[i] if tokens else None),
                start = start,
                end = end,
            )
        )
        cursors[source_page] = end + 1
    vectors = [list(_VEC) for _ in texts]
    store.create_document(conn, scope = scope, filename = filename, sha256 = sha, document_id = doc_id)
    store.add_chunks(conn, scope, doc_id, chunks, vectors)
    store.set_document_status(conn, doc_id, status, num_chunks = len(texts))


def _injected_text(result) -> str:
    """The text spliced into the conversation as the synthetic tool result."""
    tool_msg = next(m for m in result["messages"] if m.get("role") == "tool")
    return tool_msg["content"]


# ── store.all_chunks_for_scope ───────────────────────────────────────


def test_schema_upgrade_adds_chunk_offsets():
    conn = sqlite3.connect(":memory:")
    conn.execute(
        "CREATE TABLE chunks(id TEXT PRIMARY KEY, document_id TEXT, scope TEXT, "
        "chunk_index INTEGER, text TEXT, page_number INTEGER, source_page_index INTEGER, "
        "token_count INTEGER, kind TEXT, pdf_regions_json TEXT)"
    )
    rag_db._ensure_schema(conn)
    columns = {row[1] for row in conn.execute("PRAGMA table_info(chunks)")}
    conn.close()
    assert {"page_char_start", "page_char_end", "whole_document_token_count"} <= columns


def test_all_chunks_for_scope_orders_by_document_then_index(rag_conn):
    scope = store.thread_scope("t1")
    _add_doc(rag_conn, scope, "d1", "first.pdf", "h1", ["a", "b", "c"])
    _add_doc(rag_conn, scope, "d2", "second.pdf", "h2", ["x", "y"])
    rows = store.all_chunks_for_scope(rag_conn, scope)
    assert [r["id"] for r in rows] == ["d1:0", "d1:1", "d1:2", "d2:0", "d2:1"]
    assert rows[0]["filename"] == "first.pdf"
    assert rows[-1]["filename"] == "second.pdf"
    assert rows[0]["text"] == "a"
    assert (rows[0]["page_char_start"], rows[0]["page_char_end"]) == (0, 1)


def test_copy_document_index_preserves_chunk_offsets(rag_conn):
    scope = store.thread_scope("t1")
    store.create_document(
        rag_conn, scope = scope, filename = "source.txt", sha256 = "h1", document_id = "src"
    )
    store.add_chunks(
        rag_conn,
        scope,
        "src",
        [_chunk("overlap", start = 4, end = 11, whole_tokens = 5)],
        [_VEC],
    )
    store.create_document(
        rag_conn, scope = scope, filename = "copy.txt", sha256 = "h2", document_id = "dst"
    )
    store.copy_document_index(rag_conn, store.get_document(rag_conn, "src"), "dst", scope)
    row = rag_conn.execute(
        "SELECT page_char_start, page_char_end, whole_document_token_count "
        "FROM chunks WHERE document_id='dst'"
    ).fetchone()
    assert (row["page_char_start"], row["page_char_end"], row["whole_document_token_count"]) == (
        4,
        11,
        5,
    )


def test_all_chunks_for_scope_excludes_non_completed(rag_conn):
    scope = store.thread_scope("t1")
    _add_doc(rag_conn, scope, "done", "done.pdf", "h1", ["ready"])
    _add_doc(rag_conn, scope, "pend", "pend.pdf", "h2", ["indexing"], status = "pending")
    rows = store.all_chunks_for_scope(rag_conn, scope)
    assert [r["id"] for r in rows] == ["done:0"]


def test_all_chunks_for_scope_empty_scope(rag_conn):
    assert store.all_chunks_for_scope(rag_conn, store.thread_scope("nope")) == []


def test_all_chunks_for_scope_isolates_scopes(rag_conn):
    _add_doc(rag_conn, store.thread_scope("t1"), "d1", "f", "h1", ["mine"])
    _add_doc(rag_conn, store.thread_scope("t2"), "d2", "f", "h2", ["theirs"])
    rows = store.all_chunks_for_scope(rag_conn, store.thread_scope("t1"))
    assert [r["text"] for r in rows] == ["mine"]


def test_scope_token_estimate_sums_without_hydrating(rag_conn):
    scope = store.thread_scope("t1")
    _add_doc(rag_conn, scope, "d1", "a.pdf", "h1", ["alpha", "bravo"], tokens = [10, 20])
    _add_doc(rag_conn, scope, "d2", "b.pdf", "h2", ["x" * 40], tokens = [0])
    _add_doc(rag_conn, scope, "d3", "c.pdf", "h3", ["pending"], status = "pending", tokens = [99])
    assert store.scope_token_estimate(rag_conn, scope) == 10 + 20 + 10
    assert store.scope_token_estimate(rag_conn, store.thread_scope("none")) == 0


def test_scope_token_estimate_matches_row_sum(rag_conn):
    from core.rag.tool import _row_token_count

    scope = store.thread_scope("t1")
    _add_doc(
        rag_conn, scope, "d1", "a.pdf", "h1", ["a long-ish chunk body here", "tail"], tokens = [0, 5]
    )
    rows = store.all_chunks_for_scope(rag_conn, scope)
    assert store.scope_token_estimate(rag_conn, scope) == sum(_row_token_count(r) for r in rows)


def test_whole_document_context_returns_full_text_and_sources(rag_conn):
    scope = store.thread_scope("t1")
    _add_doc(
        rag_conn,
        scope,
        "d1",
        "report.pdf",
        "h1",
        ["chapter one body", "chapter two body"],
        pages = [1, 2],
    )
    result = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
    assert result is not None
    text, sources = result
    assert "chapter one body" in text
    assert "chapter two body" in text
    assert '<chunk id="1"' in text
    assert '<chunk id="2"' in text
    assert text.index("chapter one") < text.index("chapter two")
    assert [s["citationId"] for s in sources] == [1, 2]
    assert all(s["filename"] == "report.pdf" for s in sources)
    assert all(s["score"] is None for s in sources)
    assert [s["page"] for s in sources] == [1, 2]
    assert [s["chunkId"] for s in sources] == ["d1:0", "d1:1"]


def test_whole_document_context_none_over_budget(rag_conn):
    scope = store.thread_scope("t1")
    _add_doc(rag_conn, scope, "d1", "big.pdf", "h1", ["huge"], tokens = [50_000])
    assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000) is None
    assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = 100_000) is not None


def test_whole_document_context_none_when_empty(rag_conn):
    assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000) is None


def test_whole_document_context_non_positive_budget_returns_none(rag_conn):
    # A non-positive budget disables whole-doc rather than injecting everything unbounded.
    scope = store.thread_scope("t1")
    _add_doc(rag_conn, scope, "d1", "a.pdf", "h1", ["tiny body"])
    assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = 0) is None
    assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = -5) is None


def test_whole_document_context_none_without_scope(rag_conn):
    assert tool.whole_document_context(max_tokens = 6000) is None


def test_whole_document_context_null_token_count_enforces_budget(rag_conn):
    big = "word " * 20_000
    _add_doc(rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", [big], tokens = [None])
    assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000) is None
    assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = 1_000_000) is not None


def test_whole_document_context_spans_multiple_docs(rag_conn):
    scope = store.thread_scope("t1")
    _add_doc(rag_conn, scope, "d1", "a.pdf", "h1", ["alpha text"])
    _add_doc(rag_conn, scope, "d2", "b.pdf", "h2", ["bravo text"])
    text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
    assert "alpha text" in text and "bravo text" in text
    assert {s["filename"] for s in sources} == {"a.pdf", "b.pdf"}


def test_whole_document_context_reads_each_overlapping_line_once(rag_conn):
    lines = [f"R{i:03d} | Travel | Vendor number {i} | ${i}.00" for i in range(1, 71)]
    pages = parsers.parse_text("\n".join(lines))
    chunks = chunking.chunk_pages(pages, max_tokens = 200, overlap = 32, count = lambda t: len(t.split()))
    assert len(chunks) > 1
    store.create_document(
        rag_conn, scope = store.thread_scope("t1"), filename = "exp.docx", sha256 = "h1", document_id = "d1"
    )
    store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])
    store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))
    text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
    assert re.findall(r"R\d{3}", text) == [f"R{i:03d}" for i in range(1, 71)]
    assert [s["chunkId"] for s in sources] == [f"d1:{i}" for i in range(len(chunks))]


def test_whole_document_context_keeps_text_repeated_on_the_next_page(rag_conn):
    _add_doc(
        rag_conn,
        store.thread_scope("t1"),
        "d1",
        "report.pdf",
        "h1",
        ["page one body\nACME Confidential", "ACME Confidential\npage two body"],
        pages = [1, 2],
    )
    text, _sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
    assert text.count("ACME Confidential") == 2


def test_whole_document_context_keeps_indentation_after_overlap(rag_conn):
    page = "paragraph\nshared line\n    indented code"
    overlap = page.index("shared line")
    chunks = [
        _chunk(page[: page.index("\n    indented code")], end = page.index("\n    indented code")),
        _chunk(page[overlap:], 1, start = overlap, end = len(page)),
    ]
    store.create_document(
        rag_conn,
        scope = store.thread_scope("t1"),
        filename = "code.md",
        sha256 = "h1",
        document_id = "d1",
    )
    store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])
    store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))
    _text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
    assert sources[1]["text"] == "    indented code"


def test_whole_document_context_keeps_matching_text_without_actual_overlap(rag_conn):
    page = "section\nTOTAL\nTOTAL\nnext"
    second_start = page.index("TOTAL", page.index("TOTAL") + 1)
    chunks = [
        _chunk(page[: second_start - 1], end = second_start - 1),
        _chunk(page[second_start:], 1, start = second_start, end = len(page)),
    ]
    store.create_document(
        rag_conn,
        scope = store.thread_scope("t1"),
        filename = "totals.txt",
        sha256 = "h1",
        document_id = "d1",
    )
    store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])
    store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))
    text, _sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
    assert text.count("TOTAL") == 2


def test_whole_document_context_removes_overlap_inside_unbroken_text(rag_conn):
    page = "0123456789" * 8
    chunks = chunking.chunk_pages(parsers.parse_text(page), max_tokens = 20, overlap = 5, count = len)
    assert len(chunks) > 1
    store.create_document(
        rag_conn,
        scope = store.thread_scope("t1"),
        filename = "data.txt",
        sha256 = "h1",
        document_id = "d1",
    )
    store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])
    store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))
    _text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
    assert "".join(source["text"] for source in sources) == page


def test_whole_document_context_budgets_deoverlapped_content(rag_conn):
    page = " ".join(f"word{i}" for i in range(30))
    chunks = chunking.chunk_pages(parsers.parse_text(page), max_tokens = 30, overlap = 10, count = len)
    previous_end = 0
    unique_tokens = 0
    for chunk in chunks:
        unique_start = max(previous_end, chunk.page_char_start)
        unique_tokens += len(chunk.text[unique_start - chunk.page_char_start :])
        previous_end = max(previous_end, chunk.page_char_end)
    assert sum(chunk.token_count for chunk in chunks) > unique_tokens
    assert sum(chunk.whole_document_token_count for chunk in chunks) == unique_tokens
    store.create_document(
        rag_conn,
        scope = store.thread_scope("t1"),
        filename = "near-limit.txt",
        sha256 = "h1",
        document_id = "d1",
    )
    store.add_chunks(rag_conn, store.thread_scope("t1"), "d1", chunks, [list(_VEC) for _ in chunks])
    store.set_document_status(rag_conn, "d1", "completed", num_chunks = len(chunks))
    assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = unique_tokens)


def test_whole_document_context_keeps_legacy_chunks_without_offsets(rag_conn):
    _add_doc(
        rag_conn,
        store.thread_scope("t1"),
        "d1",
        "legacy.txt",
        "h1",
        ["section\nTOTAL", "TOTAL\nnext"],
    )
    rag_conn.execute(
        "UPDATE chunks SET page_char_start=NULL, page_char_end=NULL WHERE document_id='d1'"
    )
    rag_conn.commit()
    text, _sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
    assert text.count("TOTAL") == 2


def _convo(text = "summarize the whole document"):
    return [{"role": "user", "content": text}]


def test_build_rag_autoinject_uses_whole_doc(rag_conn):
    scope = store.thread_scope("t1")
    _add_doc(rag_conn, scope, "d1", "doc.pdf", "h1", ["whole alpha part", "whole bravo part"])
    result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1"})
    assert result is not None
    injected = _injected_text(result)
    assert "whole alpha part" in injected
    assert "whole bravo part" in injected
    assert inf_tools.RAG_SOURCES_SENTINEL not in injected


def test_build_rag_autoinject_whole_doc_runs_when_autoinject_false(rag_conn, monkeypatch):
    _add_doc(rag_conn, store.thread_scope("t1"), "d1", "doc.pdf", "h1", ["entire file body"])
    _shared_setup_2(monkeypatch)
    result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1", "autoinject": False})
    assert result is not None
    assert "entire file body" in _injected_text(result)


def test_build_rag_autoinject_explicit_off_disables_whole_doc(rag_conn, monkeypatch):
    _add_doc(rag_conn, store.thread_scope("t1"), "d1", "doc.pdf", "h1", ["small body"])
    _shared_setup_2(monkeypatch)
    assert (
        inf_tools.build_rag_autoinject(
            _convo(), {"thread_id": "t1", "autoinject": False, "whole_doc": False}
        )
        is None
    )


def test_build_rag_autoinject_falls_back_over_budget(rag_conn, monkeypatch):
    scope = store.thread_scope("t1")
    _add_doc(rag_conn, scope, "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000])

    sentinel = ("TOPK_FALLBACK_TEXT", [{"citationId": 1, "filename": "big.pdf", "text": "x"}])
    monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: sentinel)

    result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1"})
    assert result is not None
    assert _injected_text(result) == "TOPK_FALLBACK_TEXT"


def test_build_rag_autoinject_large_model_auto_falls_back_over_budget(rag_conn, monkeypatch):
    # Auto resolves to autoinject=False over 9B; oversized attachments fall back to top-K.
    scope = store.thread_scope("t1")
    _add_doc(rag_conn, scope, "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000])

    sentinel = (
        "TOPK_LARGE_MODEL_FALLBACK",
        [{"citationId": 1, "filename": "big.pdf", "text": "x"}],
    )
    monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: sentinel)

    result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1", "autoinject": False})
    assert result is not None
    assert _injected_text(result) == "TOPK_LARGE_MODEL_FALLBACK"


def test_build_rag_autoinject_fallback_is_thread_first_and_budgeted(rag_conn, monkeypatch):
    monkeypatch.setattr(tool, "whole_document_context", lambda **kw: None)
    monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: None)
    calls = []

    def fake_search(**kw):
        calls.append(kw)
        if kw.get("scope_thread_id"):
            if kw.get("scope_project_id") or kw.get("min_dense_score") is not None:
                return None
            sources = [
                {"citationId": i, "filename": "thread.txt", "text": "x" * 1600}
                for i in range(1, kw["top_k"] + 1)
            ]
        else:
            sources = [{"citationId": 1, "filename": "project.txt", "text": "project"}]
        return tool.render_sources(sources), sources

    monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
    result = inf_tools.build_rag_autoinject(
        _convo("summarize the attached document"),
        {
            "thread_id": "t1",
            "project_id": "p1",
            "autoinject": False,
            "context_length": 3000,
            "response_headroom": 1024,
        },
    )

    injected = _injected_text(result)
    assert "x" * 1600 in injected and "project" in injected
    assert [(c.get("scope_thread_id"), c.get("scope_project_id")) for c in calls] == [
        ("t1", None),
        (None, "p1"),
    ]
    assert calls[0]["min_dense_score"] is None
    assert calls[0]["top_k"] == 4
    # Budget leaves room for one 1600-char passage plus the project hit, not all four.
    assert injected.count("x" * 1600) == 1


def test_build_rag_autoinject_zero_budget_still_grounds(rag_conn, monkeypatch):
    _add_doc(
        rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000]
    )
    sentinel = ("TOPK_ZERO_BUDGET", [{"citationId": 1, "filename": "big.pdf", "text": "x"}])
    monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: sentinel)

    for scope in (
        {"thread_id": "t1", "autoinject": True, "context_length": 1200},
        {"thread_id": "t1", "autoinject": True, "context_length": 512},
        {"thread_id": "t1", "autoinject": True, "context_length": 8192, "response_headroom": 8192},
        {
            "thread_id": "t1",
            "autoinject": True,
            "context_length": 2048,
            "response_headroom": 999_999,
        },
        {"thread_id": "t1", "autoinject": True, "context_length": 2048, "default_top_k": 0},
    ):
        result = inf_tools.build_rag_autoinject(_convo(), scope)
        assert result is not None, scope
        assert _injected_text(result) == "TOPK_ZERO_BUDGET", scope


def test_build_rag_autoinject_refuses_when_not_even_one_passage_fits(rag_conn, monkeypatch):
    # The current turn may not be evicted, so when no passage fits, inject nothing rather than fail.
    _add_doc(
        rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000]
    )

    def fake_search(**kw):
        sources = [
            {"citationId": 0, "filename": "thread.txt", "text": "T" * 2000}
            for _ in range(kw["top_k"] or 4)
        ]
        return tool.render_sources(sources), sources

    monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
    scope = {"thread_id": "t1", "autoinject": False, "context_length": 1200}
    assert inf_tools._whole_doc_budget(scope, _convo()) == 0
    assert inf_tools.build_rag_autoinject(_convo(), scope) is None
    assert (
        inf_tools.build_rag_autoinject(
            _convo(), {"thread_id": "t1", "autoinject": True, "context_length": 1200}
        )
        is not None
    )


def test_build_rag_autoinject_enabled_path_stays_unbudgeted(rag_conn, monkeypatch):
    _add_doc(
        rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000]
    )
    calls = []

    def fake_search(**kw):
        calls.append(kw)
        sources = [
            {"citationId": i, "filename": "thread.txt", "text": "y" * 6000}
            for i in range(1, (kw["top_k"] or 4) + 1)
        ]
        return tool.render_sources(sources), sources

    monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
    long_thread = [
        {"role": "user", "content": "x " * 6000},
        {"role": "assistant", "content": "ok"},
        {"role": "user", "content": "summarize the attached document"},
    ]
    result = inf_tools.build_rag_autoinject(
        long_thread, {"thread_id": "t1", "autoinject": True, "context_length": 8192}
    )
    assert result is not None
    assert _injected_text(result).count("y" * 6000) == 4
    assert len(calls) == 1
    assert calls[0]["min_dense_score"] == inf_tools._autoinject_floor()


def test_build_rag_autoinject_on_still_grounds_over_budget_doc_below_floor(rag_conn, monkeypatch):
    from core.rag import embeddings

    _add_doc(
        rag_conn,
        store.thread_scope("t1"),
        "d1",
        "big.pdf",
        "h1",
        ["OVER_BUDGET_PASSAGE"],
        tokens = [50_000],
    )
    monkeypatch.setattr(
        embeddings,
        "encode",
        lambda texts, *, model_name = None, normalize = True: [[1.0, 0.0, 0.0, 0.0] for _ in texts],
    )
    monkeypatch.setattr(embeddings, "dim", lambda model_name = None: len(_VEC))

    result = inf_tools.build_rag_autoinject(
        _convo("Summarize this document"),
        {"thread_id": "t1", "autoinject": True, "autoinject_min_score": 0.7},
    )
    assert result is not None
    assert "OVER_BUDGET_PASSAGE" in _injected_text(result)


@pytest.mark.parametrize("autoinject", [True, False])
def test_build_rag_autoinject_lexical_mode_still_grounds_generic_request(
    rag_conn, monkeypatch, autoinject
):
    from core.rag import embeddings

    _add_doc(
        rag_conn,
        store.thread_scope("t1"),
        "d1",
        "big.pdf",
        "h1",
        ["OVER_BUDGET_PASSAGE"],
        tokens = [50_000],
    )
    monkeypatch.setattr(
        embeddings,
        "encode",
        lambda texts, *, model_name = None, normalize = True: [[1.0, 0.0, 0.0, 0.0] for _ in texts],
    )
    monkeypatch.setattr(embeddings, "dim", lambda model_name = None: len(_VEC))

    result = inf_tools.build_rag_autoinject(
        _convo("Summarize this document"),
        {
            "thread_id": "t1",
            "autoinject": autoinject,
            "autoinject_min_score": 0.7,
            "mode": "lexical",
        },
    )
    assert result is not None
    assert "OVER_BUDGET_PASSAGE" in _injected_text(result)


def test_build_rag_autoinject_on_grounds_attachment_beside_project_hits(rag_conn, monkeypatch):
    from core.rag import embeddings

    _add_doc(
        rag_conn,
        store.thread_scope("t1"),
        "d1",
        "big.pdf",
        "h1",
        ["OVER_BUDGET_PASSAGE"],
        tokens = [50_000],
    )
    pscope = store.project_scope("p1")
    store.create_document(
        rag_conn, scope = pscope, filename = "notes.md", sha256 = "h2", document_id = "d2"
    )
    store.add_chunks(rag_conn, pscope, "d2", [_chunk("PROJECT_PASSAGE")], [[1.0, 0.0, 0.0, 0.0]])
    store.set_document_status(rag_conn, "d2", "completed", num_chunks = 1)
    monkeypatch.setattr(
        embeddings,
        "encode",
        lambda texts, *, model_name = None, normalize = True: [[1.0, 0.0, 0.0, 0.0] for _ in texts],
    )
    monkeypatch.setattr(embeddings, "dim", lambda model_name = None: len(_VEC))

    result = inf_tools.build_rag_autoinject(
        _convo("Summarize this document"),
        {"thread_id": "t1", "project_id": "p1", "autoinject": True, "autoinject_min_score": 0.7},
    )
    injected = _injected_text(result)
    assert injected.index("OVER_BUDGET_PASSAGE") < injected.index("PROJECT_PASSAGE")


def test_build_rag_autoinject_on_attachment_beside_project_keeps_lean_top_k(rag_conn, monkeypatch):
    monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: None)

    def fake_search(**kw):
        if kw.get("min_dense_score") is None:
            name, doc = "thread", "d1"
        else:
            name, doc = "project", "d2"
        sources = [
            {"citationId": i, "documentId": doc, "filename": f"{name}.txt", "text": f"{name}-{i}"}
            for i in range(1, kw["top_k"] + 1)
        ]
        return tool.render_sources(sources), sources

    monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
    result = inf_tools.build_rag_autoinject(
        _convo("Summarize this document"),
        {"thread_id": "t1", "project_id": "p1", "autoinject": True},
    )
    injected = _injected_text(result)
    assert injected.count("<chunk id=") == inf_tools._autoinject_top_k()
    assert "thread-1" in injected and "project-1" in injected
    assert injected.index("thread-1") < injected.index("project-1")


@pytest.mark.parametrize("autoinject", [True, False])
def test_build_rag_autoinject_skips_thread_fallback_without_attachment(
    rag_conn, monkeypatch, autoinject
):
    calls = []
    monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: calls.append(kw))
    result = inf_tools.build_rag_autoinject(
        _convo("Say hello in three words."), {"thread_id": "t1", "autoinject": autoinject}
    )
    assert result is None
    assert not [c for c in calls if c.get("min_dense_score") is None]


def test_build_rag_autoinject_on_skips_thread_search_when_attachment_already_hit(
    rag_conn, monkeypatch
):
    monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: {"d1"})
    calls = []

    def fake_search(**kw):
        calls.append(kw)
        sources = [
            {"citationId": 1, "documentId": "d1", "filename": "big.pdf", "text": "thread hit"}
        ]
        return tool.render_sources(sources), sources

    monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
    result = inf_tools.build_rag_autoinject(
        _convo("Summarize this document"),
        {"thread_id": "t1", "project_id": "p1", "autoinject": True},
    )
    assert "thread hit" in _injected_text(result)
    assert len(calls) == 1


@pytest.mark.parametrize("project_id", ["p1", None])
def test_build_rag_autoinject_on_zero_top_k_still_grounds_attachment(
    rag_conn, monkeypatch, project_id
):
    monkeypatch.setattr(inf_tools, "_thread_document_ids", lambda thread_id: {"d1"})

    def fake_search(**kw):
        if kw.get("min_dense_score") is not None and not kw.get("scope_project_id"):
            return None
        name, doc = ("thread", "d1") if kw.get("min_dense_score") is None else ("project", "d2")
        # Zero is falsy to the search, which then returns its configured default count.
        sources = [
            {"citationId": i, "documentId": doc, "filename": f"{name}.txt", "text": f"{name}-{i}"}
            for i in range(1, (kw["top_k"] or 10) + 1)
        ]
        return tool.render_sources(sources), sources

    monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
    result = inf_tools.build_rag_autoinject(
        _convo("Summarize this document"),
        {"thread_id": "t1", "project_id": project_id, "autoinject": True, "default_top_k": 0},
    )
    injected = _injected_text(result)
    assert "thread-1" in injected
    assert injected.count("<chunk id=") == inf_tools._autoinject_top_k()


def test_build_rag_autoinject_off_does_not_inject_project_alone(rag_conn, monkeypatch):
    _add_doc(
        rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000]
    )

    def fake_search(**kw):
        if kw.get("scope_thread_id"):
            return None
        sources = [{"citationId": 1, "filename": "project.txt", "text": "project"}]
        return tool.render_sources(sources), sources

    monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
    assert (
        inf_tools.build_rag_autoinject(
            _convo(), {"thread_id": "t1", "project_id": "p1", "autoinject": False}
        )
        is None
    )


def test_build_rag_autoinject_keeps_thread_hits_when_project_retrieval_raises(
    rag_conn, monkeypatch
):
    _add_doc(
        rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000]
    )

    def fake_search(**kw):
        if kw.get("scope_project_id"):
            raise RuntimeError("project index unavailable")
        sources = [{"citationId": 1, "filename": "thread.txt", "text": "THREAD_GROUNDING"}]
        return tool.render_sources(sources), sources

    monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
    result = inf_tools.build_rag_autoinject(
        _convo(), {"thread_id": "t1", "project_id": "p1", "autoinject": False}
    )
    assert result is not None
    assert "THREAD_GROUNDING" in _injected_text(result)


def test_build_rag_autoinject_keeps_the_project_hits_that_fit(rag_conn, monkeypatch):
    _add_doc(
        rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000]
    )

    def fake_search(**kw):
        if kw.get("scope_project_id"):
            sources = [
                {"citationId": 0, "filename": "project.txt", "text": "P" * 2000} for _ in range(4)
            ]
        else:
            sources = [{"citationId": 0, "filename": "thread.txt", "text": "T" * 8000}]
        return tool.render_sources(sources), sources

    injected = _shared_setup_1(fake_search, monkeypatch)
    assert "T" * 8000 in injected
    assert 1 <= injected.count("P" * 2000) < 4


def test_build_rag_autoinject_whole_doc_merge_is_priced_in_tokens(rag_conn, monkeypatch):
    # A CJK character is about one token, not four.
    _add_doc(rag_conn, store.thread_scope("t1"), "d1", "doc.pdf", "h1", ["THREAD_DOC"])

    def fake_search(**kw):
        sources = [
            {"citationId": 0, "filename": "project.txt", "text": "漢" * 2000} for _ in range(4)
        ]
        return tool.render_sources(sources), sources

    injected = _shared_setup_1(fake_search, monkeypatch)

    assert "THREAD_DOC" in injected
    assert 1 <= injected.count("漢" * 2000) < 4


def test_build_rag_autoinject_whole_doc_merge_never_truncates_the_document(rag_conn, monkeypatch):
    body = ["SECTION%02d " % i + "word " * 398 for i in range(8)]
    _add_doc(
        rag_conn,
        store.thread_scope("t1"),
        "d1",
        "report.pdf",
        "h1",
        body,
        tokens = [500] * len(body),
    )

    def fake_search(**kw):
        sources = [{"citationId": 0, "filename": "notes.txt", "text": "PROJECT_HIT " + "x" * 500}]
        return tool.render_sources(sources), sources

    injected = _shared_setup_1(fake_search, monkeypatch)

    assert [i for i in range(len(body)) if "SECTION%02d" % i in injected] == list(range(len(body)))
    assert "PROJECT_HIT" not in injected


def test_build_rag_autoinject_budget_charges_multibyte_text_more(rag_conn, monkeypatch):
    _add_doc(
        rag_conn, store.thread_scope("t1"), "d1", "big.pdf", "h1", ["overflow"], tokens = [50_000]
    )

    def make_search(body):
        def fake_search(**kw):
            sources = [{"citationId": 0, "filename": "thread.txt", "text": body} for _ in range(4)]
            return tool.render_sources(sources), sources

        return fake_search

    scope = {
        "thread_id": "t1",
        "autoinject": False,
        "context_length": 4096,
        "response_headroom": 1024,
    }

    def _chunks(body):
        monkeypatch.setattr(tool, "search_for_autoinject", make_search(body))
        result = inf_tools.build_rag_autoinject(_convo(), scope)
        return _injected_text(result).count('<chunk id="') if result else 0

    ascii_chunks = _chunks("a" * 2000)
    cjk_chunks = _chunks("漢" * 2000)
    assert ascii_chunks >= 1
    assert cjk_chunks < ascii_chunks


def test_build_rag_autoinject_context_budget_falls_back(rag_conn, monkeypatch):
    _add_doc(
        rag_conn, store.thread_scope("t1"), "d1", "small.pdf", "h1", ["fits global"], tokens = [900]
    )
    sentinel = ("TOPK_CONTEXT_FALLBACK", [{"citationId": 1, "filename": "small.pdf", "text": "x"}])
    monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: sentinel)
    result = inf_tools.build_rag_autoinject(
        _convo(), {"thread_id": "t1", "context_length": 1200, "whole_doc": True}
    )
    assert result is not None
    assert _injected_text(result) == "TOPK_CONTEXT_FALLBACK"


def test_whole_doc_budget_reserves_image_parts(monkeypatch):
    from core.rag import config

    monkeypatch.setattr(config, "WHOLE_DOC_MAX_TOKENS", 10_000)
    scope = {"context_length": 7000, "response_headroom": 1000}
    text_only = [{"role": "user", "content": [{"type": "text", "text": "summarize"}]}]
    with_image = [
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "summarize"},
                {"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}},
            ],
        }
    ]

    assert (
        inf_tools._whole_doc_budget(scope, text_only)
        - inf_tools._whole_doc_budget(scope, with_image)
        == inf_tools._IMAGE_PART_TOKEN_ESTIMATE
    )


def test_build_rag_autoinject_server_kill_switch_blocks_whole_doc(rag_conn, monkeypatch):
    from core.rag import config

    monkeypatch.setattr(config, "THREAD_WHOLE_DOC", False)
    _add_doc(rag_conn, store.thread_scope("t1"), "d1", "doc.pdf", "h1", ["small body"])
    _shared_setup_2(monkeypatch)
    assert (
        inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1", "autoinject": False}) is None
    )


def test_whole_document_context_budgets_rendered_wrappers(rag_conn):
    # Wrapper overhead counts: budget the rendered prompt, not stored text.
    texts = ["x" for _ in range(120)]
    _add_doc(
        rag_conn,
        store.thread_scope("t1"),
        "d1",
        "many-pages.pdf",
        "h1",
        texts,
        tokens = [1 for _ in texts],
    )
    assert tool.whole_document_context(scope_thread_id = "t1", max_tokens = 500) is None


def test_build_rag_autoinject_whole_doc_disabled_via_override(rag_conn, monkeypatch):
    scope = store.thread_scope("t1")
    _add_doc(rag_conn, scope, "d1", "doc.pdf", "h1", ["small body"])

    sentinel = ("TOPK_TEXT", [{"citationId": 1, "filename": "doc.pdf", "text": "x"}])
    monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: sentinel)

    result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1", "whole_doc": False})
    assert result is not None
    assert _injected_text(result) == "TOPK_TEXT"


def test_build_rag_autoinject_kb_scope_never_whole_doc(rag_conn, monkeypatch):
    kb_scope = store.kb_scope("K1")
    _add_doc(rag_conn, kb_scope, "d1", "kb.pdf", "h1", ["kb body one", "kb body two"])

    sentinel = ("KB_RETRIEVAL_TEXT", [{"citationId": 1, "filename": "kb.pdf", "text": "x"}])
    monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: sentinel)

    result = inf_tools.build_rag_autoinject(_convo(), {"kb_id": "K1"})
    assert result is not None
    assert _injected_text(result) == "KB_RETRIEVAL_TEXT"


def test_whole_document_context_thread_scope_only(rag_conn):
    _add_doc(rag_conn, store.thread_scope("t1"), "td", "thread.txt", "h1", ["thread attachment"])
    _add_doc(rag_conn, store.project_scope("p1"), "pd", "project.txt", "h2", ["project corpus"])
    text, sources = tool.whole_document_context(scope_thread_id = "t1", max_tokens = 6000)
    assert "thread attachment" in text
    assert "project corpus" not in text
    assert {s["filename"] for s in sources} == {"thread.txt"}


def test_build_rag_autoinject_appends_project_retrieval(rag_conn, monkeypatch):
    _add_doc(
        rag_conn,
        store.thread_scope("t1"),
        "td",
        "thread.txt",
        "h1",
        ["thread chunk one", "thread chunk two"],
    )
    proj = (
        "PROJ",
        [
            {
                "citationId": 1,
                "chunkId": "pj:0",
                "documentId": "pj",
                "filename": "project.txt",
                "page": None,
                "text": "project passage zeta",
                "score": 0.91,
            }
        ],
    )
    captured = {}

    def fake_search(**kw):
        captured.update(kw)
        return proj

    monkeypatch.setattr(tool, "search_for_autoinject", fake_search)
    result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1", "project_id": "p1"})
    injected = _injected_text(result)
    assert "thread chunk one" in injected
    assert "thread chunk two" in injected
    assert "project passage zeta" in injected
    assert captured.get("scope_project_id") == "p1"
    assert captured.get("scope_thread_id") is None
    assert captured.get("scope_kb_id") is None
    assert '<chunk id="1"' in injected
    assert '<chunk id="2"' in injected
    assert '<chunk id="3"' in injected


def test_build_rag_autoinject_skips_project_companion_over_budget(rag_conn, monkeypatch):
    _add_doc(rag_conn, store.thread_scope("t1"), "td", "thread.txt", "h1", ["thread body"])
    project_text = "project overflow " * 2000
    proj = (
        "PROJ",
        [
            {
                "citationId": 1,
                "chunkId": "pj:0",
                "documentId": "pj",
                "filename": "project.txt",
                "page": None,
                "text": project_text,
                "score": 0.91,
            }
        ],
    )

    monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: proj)
    result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1", "project_id": "p1"})
    injected = _injected_text(result)
    assert "thread body" in injected
    assert "project overflow" not in injected


def test_build_rag_autoinject_thread_whole_doc_ignores_project_size(rag_conn, monkeypatch):
    monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: None)
    _add_doc(rag_conn, store.thread_scope("t1"), "td", "thread.txt", "h1", ["small thread file"])
    _add_doc(
        rag_conn, store.project_scope("p1"), "pd", "project.txt", "h2", ["big"], tokens = [50_000]
    )
    result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1", "project_id": "p1"})
    assert "small thread file" in _injected_text(result)


def test_build_rag_autoinject_kb_defers_to_retrieval(rag_conn, monkeypatch):
    _add_doc(rag_conn, store.thread_scope("t1"), "td", "thread.txt", "h1", ["thread attachment"])
    sentinel = ("KB_RETRIEVAL", [{"citationId": 1, "filename": "kb.pdf", "text": "x"}])
    monkeypatch.setattr(tool, "search_for_autoinject", lambda **kw: sentinel)
    result = inf_tools.build_rag_autoinject(_convo(), {"kb_id": "K1", "thread_id": "t1"})
    assert _injected_text(result) == "KB_RETRIEVAL"


def test_build_rag_autoinject_no_scope_returns_none(rag_conn):
    assert inf_tools.build_rag_autoinject(_convo(), None) is None
    assert inf_tools.build_rag_autoinject(_convo(), {}) is None


def test_build_rag_autoinject_args_carry_user_query(rag_conn):
    scope = store.thread_scope("t1")
    _add_doc(rag_conn, scope, "d1", "doc.pdf", "h1", ["small body"])
    result = inf_tools.build_rag_autoinject(_convo("what is in here"), {"thread_id": "t1"})
    assistant_msg = next(m for m in result["messages"] if m.get("role") == "assistant")
    args = json.loads(assistant_msg["tool_calls"][0]["function"]["arguments"])
    assert args["query"] == "what is in here"


def test_real_ingestion_feeds_whole_document(rag_conn, stub_embeddings, tmp_path):
    """Real ingestion feeds whole-document injection the full text, not just retrieved chunks."""
    from core.rag import ingestion

    scope = store.thread_scope("t1")
    body = (
        "# Quarterly Report\n\n"
        + ("Revenue rose across every region this period. " * 40)
        + "\n\nThe unique closing marker is xyzzy-sentinel for the final page. " * 40
    )
    src = tmp_path / "report.md"
    src.write_text(body, encoding = "utf-8")

    document_id = store.create_document(
        rag_conn,
        scope = scope,
        filename = "report.md",
        sha256 = "sha-e2e",
        thread_id = "t1",
        status = "pending",
        stored_path = str(src),
    )
    job_id = ingestion._new_job(rag_conn, document_id, scope)
    ingestion._run(job_id, document_id, scope, str(src), None)

    doc = store.get_document(rag_conn, document_id)
    assert doc["status"] == "completed"
    assert doc["num_chunks"] >= 2

    result = inf_tools.build_rag_autoinject(_convo(), {"thread_id": "t1"})
    assert result is not None
    injected = _injected_text(result)
    assert "Revenue rose" in injected
    assert "xyzzy-sentinel" in injected
    assert injected.count("<chunk id=") == doc["num_chunks"]