Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

17.4 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Retrieval + tool tests: RRF fusion, min-score floor, scope, source-map."""

import math
import threading
import time

import pytest

from core.rag import config, retrieval, store, tool
from core.rag.chunking import Chunk


def _shared_setup_1(monkeypatch):
    from core.inference import tools
    from storage import rag_db

    _rag_is_available(monkeypatch, rag_db)
    return tools


VOCAB = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel"]


def _embed(text):
    v = [float(text.lower().count(w)) for w in VOCAB]
    n = math.sqrt(sum(x * x for x in v)) or 1.0
    return [x / n for x in v]


@pytest.fixture
def bow_embeddings(monkeypatch):
    """Bag-of-words embedder matching the vectors stored in the db."""
    from core.rag import embeddings

    monkeypatch.setattr(
        embeddings,
        "encode",
        lambda texts, *, model_name = None, normalize = True: [_embed(t) for t in texts],
    )
    monkeypatch.setattr(embeddings, "dim", lambda model_name = None: len(VOCAB))


def _chunk(
    text,
    index = 0,
    page = None,
):
    return Chunk(
        text = text,
        token_count = len(text.split()),
        page_number = page,
        source_page_index = 0,
        chunk_index = index,
        page_char_start = 0,
        page_char_end = len(text),
    )


def _add_doc(
    conn,
    scope,
    doc_id,
    filename,
    sha,
    text,
    page = None,
):
    store.create_document(conn, scope = scope, filename = filename, sha256 = sha, document_id = doc_id)
    store.add_chunks(conn, scope, doc_id, [_chunk(text, 0, page)], [_embed(text)])


def test_rrf_ranks_doc_in_both_lists_first():
    lexical = [
        retrieval.Hit("a", 1.0, lexical_score = 1.0),
        retrieval.Hit("b", 0.5, lexical_score = 0.5),
    ]
    dense = [
        retrieval.Hit("a", 0.9, dense_score = 0.9),
        retrieval.Hit("c", 0.8, dense_score = 0.8),
    ]
    fused = retrieval._rrf([lexical, dense], rrf_k = 60, top_k = 10)
    assert fused[0].chunk_id == "a"
    assert fused[0].lexical_score == 1.0 and fused[0].dense_score == 0.9


def test_retrieve_hybrid_returns_relevant_chunk(rag_conn, bow_embeddings):
    _add_doc(rag_conn, "kb_a", "d1", "f1", "h1", "alpha bravo charlie")
    _add_doc(rag_conn, "kb_a", "d2", "f2", "h2", "golf hotel delta")
    hits = retrieval.retrieve_hybrid(rag_conn, "kb_a", "alpha bravo", k = 5)
    assert hits[0].chunk_id == "d1:0"


def test_retrieve_dense_round_trips(rag_conn, bow_embeddings):
    _add_doc(rag_conn, "kb_a", "d1", "f", "h1", "alpha alpha")
    _add_doc(rag_conn, "kb_a", "d2", "f", "h2", "hotel golf")
    hits = retrieval.retrieve_dense(rag_conn, "kb_a", "alpha", 5)
    assert hits[0].chunk_id == "d1:0"
    assert hits[0].dense_score is not None and hits[0].dense_score > 0.99


def test_filter_min_score_gates_dense_hits():
    hits = [
        retrieval.Hit("a", 1.0, dense_score = 0.9),
        retrieval.Hit("b", 0.5, dense_score = 0.2),
        retrieval.Hit("c", 0.4, lexical_score = 0.4),
    ]
    out = retrieval.filter_min_score(hits, 0.5)
    ids = {h.chunk_id for h in out}
    assert ids == {"a", "c"}
    assert retrieval.filter_min_score(hits, 0.0) == hits


def test_tool_kb_scope_wins_over_thread(rag_conn, bow_embeddings, monkeypatch):
    seen = {}

    def fake(conn, scope, q, **k):
        seen["scope"] = scope
        return []

    monkeypatch.setattr(retrieval, "retrieve_hybrid", fake)
    tool.search_knowledge_base(query = "q", scope_kb_id = "K", scope_thread_id = "T")
    assert seen["scope"] == "kb_K"


def test_tool_empty_query_errors(rag_home):
    assert tool.search_knowledge_base(query = "   ").startswith("Error")


def test_tool_missing_scope_message(rag_home):
    out = tool.search_knowledge_base(query = "hello")
    assert "No documents" in out


def test_tool_formats_chunks_and_sources(rag_conn, bow_embeddings, monkeypatch):
    _add_doc(rag_conn, "kb_a", "d1", "paper.pdf", "h1", "body text here", page = 3)
    monkeypatch.setattr(
        retrieval,
        "retrieve_hybrid",
        lambda conn, scope, q, **k: [retrieval.Hit("d1:0", 1.0)],
    )
    text, sources = tool.search_knowledge_base_with_sources(query = "q", scope_kb_id = "a")
    assert '<chunk id="1" source="paper.pdf" page="3">' in text
    assert "body text here" in text
    assert sources == [
        {
            "citationId": 1,
            "chunkId": "d1:0",
            "documentId": "d1",
            "filename": "paper.pdf",
            "page": 3,
            "text": "body text here",
            "score": 1.0,
        }
    ]


def test_tool_kb_scope_retrieves_from_db(rag_conn, bow_embeddings):
    _add_doc(rag_conn, "kb_K", "d1", "kb.pdf", "h1", "alpha bravo charlie", page = 1)
    text, sources = tool.search_knowledge_base_with_sources(query = "alpha bravo", scope_kb_id = "K")
    assert "No matching chunks" not in text
    assert sources and sources[0]["chunkId"] == "d1:0"
    assert sources[0]["filename"] == "kb.pdf"
    other, other_sources = tool.search_knowledge_base_with_sources(
        query = "alpha bravo", scope_kb_id = "OTHER"
    )
    assert other_sources == [] and "No matching chunks" in other


def test_dispatcher_appends_sources_sentinel(rag_conn, bow_embeddings, monkeypatch):
    import json

    from core.inference import tools

    _add_doc(rag_conn, "kb_a", "d1", "paper.pdf", "h1", "body text here", page = 3)
    monkeypatch.setattr(
        retrieval,
        "retrieve_hybrid",
        lambda conn, scope, q, **k: [retrieval.Hit("d1:0", 1.0)],
    )
    out = tools._search_knowledge_base({"query": "q"}, {"kb_id": "a"})
    assert tools.RAG_SOURCES_SENTINEL in out
    model_text, _, payload = out.partition(tools.RAG_SOURCES_SENTINEL)
    assert "__RAG_SOURCES__" not in model_text
    assert '<chunk id="1"' in model_text
    sources = json.loads(payload)
    assert sources[0]["documentId"] == "d1"
    assert sources[0]["chunkId"] == "d1:0"
    assert sources[0]["page"] == 3


def test_dispatcher_no_sentinel_when_no_hits(rag_home, monkeypatch):
    from core.inference import tools

    monkeypatch.setattr(retrieval, "retrieve_hybrid", lambda conn, scope, q, **k: [])
    out = tools._search_knowledge_base({"query": "hello"}, {"kb_id": "missing"})
    assert tools.RAG_SOURCES_SENTINEL not in out


def test_knowledge_search_honors_cancellation_and_timeout(monkeypatch):
    from core.inference import tools

    started = threading.Event()
    release = threading.Event()
    calls = 0

    def stalled_search(arguments, rag_scope):
        nonlocal calls
        calls += 1
        started.set()
        release.wait()
        return "late"

    monkeypatch.setattr(tools, "_search_knowledge_base", stalled_search)
    cancel = threading.Event()

    def cancel_after_start():
        started.wait()
        cancel.set()

    threading.Thread(target = cancel_after_start, daemon = True).start()
    began = time.monotonic()
    try:
        cancelled = tools.execute_tool(
            "search_knowledge_base",
            {"query": "q"},
            cancel_event = cancel,
            timeout = 30,
            rag_scope = {"kb_id": "a"},
        )
        assert "cancelled" in cancelled.lower()
        assert time.monotonic() - began < 1

        started.clear()
        timed_out = tools.execute_tool(
            "search_knowledge_base",
            {"query": "q"},
            timeout = 0,
            rag_scope = {"kb_id": "a"},
        )
        assert "timed out" in timed_out.lower()
        assert calls == 1
    finally:
        release.set()
        assert tools._RAG_SEARCH_SLOT.acquire(timeout = 1)
        tools._RAG_SEARCH_SLOT.release()


def test_timed_out_search_keeps_slot_until_worker_exits(monkeypatch):
    # A timed-out search still owns the sole RAG slot until its worker finishes.
    from core.inference import tools

    started = threading.Event()
    release = threading.Event()

    def stalled_search(arguments, rag_scope):
        started.set()
        release.wait()
        return "late"

    monkeypatch.setattr(tools, "_search_knowledge_base", stalled_search)
    try:
        timed_out = tools._search_knowledge_base_with_budget(
            {"query": "q"}, {"kb_id": "a"}, timeout = 1
        )
        assert "timed out" in timed_out.lower()
        assert started.is_set()
        assert not tools._RAG_SEARCH_SLOT.acquire(timeout = 0.2)
        release.set()
        assert tools._RAG_SEARCH_SLOT.acquire(timeout = 2)
        tools._RAG_SEARCH_SLOT.release()
    finally:
        release.set()


def test_search_for_autoinject_gates_on_dense_score(rag_conn, bow_embeddings, monkeypatch):
    _add_doc(rag_conn, "kb_a", "d1", "paper.pdf", "h1", "body text here", page = 3)

    def _hits(score, **kw):
        return lambda conn, scope, q, **k: [retrieval.Hit("d1:0", 1.0, **{kw["key"]: score})]

    monkeypatch.setattr(retrieval, "retrieve_hybrid", _hits(0.8, key = "dense_score"))
    found = tool.search_for_autoinject(query = "q", scope_kb_id = "a", min_dense_score = 0.55)
    assert found is not None
    text, sources = found
    assert '<chunk id="1"' in text and sources[0]["chunkId"] == "d1:0"

    monkeypatch.setattr(retrieval, "retrieve_hybrid", _hits(0.30, key = "dense_score"))
    assert tool.search_for_autoinject(query = "q", scope_kb_id = "a", min_dense_score = 0.55) is None

    monkeypatch.setattr(retrieval, "retrieve_hybrid", _hits(1.0, key = "lexical_score"))
    assert tool.search_for_autoinject(query = "q", scope_kb_id = "a", min_dense_score = 0.55) is None


def test_search_for_autoinject_bm25_gates_on_dense_probe(rag_conn, bow_embeddings, monkeypatch):
    # BM25 hits carry no cosine, so the gate uses a dense 1-NN probe.
    _add_doc(rag_conn, "kb_a", "d1", "paper.pdf", "h1", "body text here", page = 3)
    monkeypatch.setattr(
        retrieval,
        "retrieve_hybrid",
        lambda conn, scope, q, **k: [retrieval.Hit("d1:0", 1.0, lexical_score = 2.5)],
    )

    monkeypatch.setattr(
        retrieval,
        "retrieve_dense",
        lambda conn, scope, q, k = None, **kw: [retrieval.Hit("d1:0", 0.82, dense_score = 0.82)],
    )
    found = tool.search_for_autoinject(
        query = "q", scope_kb_id = "a", mode = "lexical", min_dense_score = 0.70
    )
    assert found is not None and found[1][0]["chunkId"] == "d1:0"

    monkeypatch.setattr(
        retrieval,
        "retrieve_dense",
        lambda conn, scope, q, k = None, **kw: [retrieval.Hit("d1:0", 0.40, dense_score = 0.40)],
    )
    assert (
        tool.search_for_autoinject(query = "q", scope_kb_id = "a", mode = "lexical", min_dense_score = 0.70)
        is None
    )


def _rag_is_available(monkeypatch, rag_db) -> None:
    monkeypatch.setattr(rag_db, "RAG_AVAILABLE", True, raising = False)
    monkeypatch.setattr(rag_db, "rag_available", lambda: True, raising = False)


def test_search_for_autoinject_empty_query_or_scope(rag_home):
    assert tool.search_for_autoinject(query = "  ", scope_kb_id = "a") is None
    assert tool.search_for_autoinject(query = "hello") is None


def test_build_rag_autoinject_emits_pipeline(monkeypatch):
    tools = _shared_setup_1(monkeypatch)
    monkeypatch.setattr(
        tool,
        "search_for_autoinject",
        lambda **k: (
            '<chunk id="1" source="d.pdf">hi</chunk>',
            [{"citationId": 1, "filename": "d.pdf"}],
        ),
    )
    conv = [{"role": "user", "content": "When was DeepSeek V4 released?"}]
    out = tools.build_rag_autoinject(conv, {"thread_id": "t1"})
    assert out is not None
    kinds = [e["type"] for e in out["events"]]
    assert "tool_start" in kinds and "tool_end" in kinds
    te = next(e for e in out["events"] if e["type"] == "tool_end")
    assert te["tool_name"] == "search_knowledge_base"
    assert tools.RAG_SOURCES_SENTINEL in te["result"]
    assert out["messages"][0]["tool_calls"][0]["function"]["name"] == "search_knowledge_base"
    assert "__RAG_SOURCES__" not in out["messages"][1]["content"]


def test_build_rag_autoinject_skips_without_hit(monkeypatch):
    tools = _shared_setup_1(monkeypatch)
    monkeypatch.setattr(tool, "search_for_autoinject", lambda **k: None)
    assert (
        tools.build_rag_autoinject([{"role": "user", "content": "hi"}], {"thread_id": "t1"}) is None
    )


def test_build_rag_autoinject_enabled_by_default(monkeypatch):
    from core.inference import tools
    from storage import rag_db

    monkeypatch.delenv("RAG_AUTOINJECT", raising = False)
    monkeypatch.delenv("RAG_AUTOINJECT_MIN_SCORE", raising = False)
    _rag_is_available(monkeypatch, rag_db)
    seen: dict = {}

    def fake(**k):
        seen.update(k)
        return ("x", [{"citationId": 1}])

    monkeypatch.setattr(tool, "search_for_autoinject", fake)
    out = tools.build_rag_autoinject([{"role": "user", "content": "hi"}], {"thread_id": "t1"})
    assert out is not None
    assert seen["min_dense_score"] == 0.70


def test_build_rag_autoinject_caps_top_k(monkeypatch):
    from core.inference import tools
    from storage import rag_db

    monkeypatch.setenv("RAG_AUTOINJECT", "1")
    monkeypatch.setenv("RAG_AUTOINJECT_TOP_K", "4")
    _rag_is_available(monkeypatch, rag_db)
    seen: dict = {}

    def fake(**k):
        seen.update(k)
        return ("x", [{"citationId": 1}])

    monkeypatch.setattr(tool, "search_for_autoinject", fake)
    conv = [{"role": "user", "content": "q"}]
    tools.build_rag_autoinject(conv, {"thread_id": "t1"})
    assert seen["top_k"] == 4
    tools.build_rag_autoinject(conv, {"thread_id": "t1", "default_top_k": 2})
    assert seen["top_k"] == 2


def test_build_rag_autoinject_disabled_by_env(monkeypatch):
    from core.inference import tools

    monkeypatch.setenv("RAG_AUTOINJECT", "0")
    assert (
        tools.build_rag_autoinject([{"role": "user", "content": "hi"}], {"thread_id": "t1"}) is None
    )
    monkeypatch.delenv("RAG_AUTOINJECT", raising = False)
    assert tools.build_rag_autoinject([{"role": "user", "content": "hi"}], None) is None


def test_retrieve_hybrid_mode_selects_backend(monkeypatch):
    calls: list = []
    monkeypatch.setattr(
        retrieval,
        "retrieve_lexical",
        lambda c, s, q, k = None, *, match_query = None: calls.append(("lex", k)) or [],
    )
    monkeypatch.setattr(
        retrieval,
        "retrieve_dense",
        lambda c, s, q, k = None, *, model_name = None: calls.append(("dense", k)) or [],
    )
    monkeypatch.setattr(
        retrieval,
        "_rrf",
        lambda rankings, rrf_k, top_k: calls.append(("rrf", rrf_k, top_k)) or [],
    )

    calls.clear()
    retrieval.retrieve_hybrid(None, "kb_a", "q", k = 5, mode = "lexical")
    assert [c[0] for c in calls] == ["lex"]

    calls.clear()
    retrieval.retrieve_hybrid(None, "kb_a", "q", k = 5, mode = "dense")
    assert [c[0] for c in calls] == ["dense"]

    calls.clear()
    retrieval.retrieve_hybrid(None, "kb_a", "q", k = 5, mode = "hybrid")
    assert ("lex", config.TOP_K_LEXICAL) in calls
    assert ("dense", config.TOP_K_DENSE) in calls
    rrf = next(c for c in calls if c[0] == "rrf")
    assert rrf[1] == config.RRF_K and rrf[2] == 5


def test_scope_overrides_reach_retrieval(monkeypatch):
    tools = _shared_setup_1(monkeypatch)
    seen: dict = {}

    def fake_search(**kw):
        seen.update(kw)
        return ("text", [])

    monkeypatch.setattr(tool, "search_knowledge_base_with_sources", fake_search)
    tools._search_knowledge_base(
        {"query": "q"},
        {"kb_id": "a", "mode": "dense", "default_top_k": 11},
    )
    assert seen["mode"] == "dense"
    assert seen["top_k"] == 11
    seen.clear()
    tools._search_knowledge_base({"query": "q"}, {"kb_id": "a", "mode": "bogus"})
    assert seen["mode"] == "hybrid"


def test_model_top_k_is_clamped_in_every_mode(rag_conn, bow_embeddings, monkeypatch):
    tools = _shared_setup_1(monkeypatch)
    for i in range(60):
        _add_doc(rag_conn, "kb_a", f"d{i}", f"f{i}.txt", f"h{i}", f"alpha note {i}")

    def returned(top_k, mode):
        out = tools._search_knowledge_base(
            {"query": "alpha", "top_k": top_k}, {"kb_id": "a", "mode": mode, "default_top_k": 6}
        )
        return out.partition(tools.RAG_SOURCES_SENTINEL)[0].count("<chunk id=")

    for mode in ("hybrid", "lexical", "dense"):
        assert returned(3, mode) == 3, mode
        # Not a count, so the sidebar's setting applies.
        assert returned(-1, mode) == 6, mode
        assert returned(-5, mode) == 6, mode
    assert returned(2000, "lexical") == tools._MAX_KNOWLEDGE_SEARCH_TOP_K
    assert returned(2000, "dense") == tools._MAX_KNOWLEDGE_SEARCH_TOP_K


def test_build_rag_autoinject_scope_overrides_env(monkeypatch):
    tools = _shared_setup_1(monkeypatch)
    seen: dict = {}

    def fake_autoinject(**k):
        seen.update(k)
        return ('<chunk id="1" source="d.pdf">hi</chunk>', [{"citationId": 1}])

    monkeypatch.setattr(tool, "search_for_autoinject", fake_autoinject)
    conv = [{"role": "user", "content": "q"}]

    monkeypatch.setenv("RAG_AUTOINJECT", "0")
    out = tools.build_rag_autoinject(
        conv,
        {
            "thread_id": "t1",
            "autoinject": True,
            "autoinject_min_score": 0.8,
            "mode": "dense",
        },
    )
    assert out is not None
    assert seen["min_dense_score"] == 0.8
    assert seen["mode"] == "dense"

    # autoinject=False alone is also large-model Auto and must still allow thread-doc grounding.
    monkeypatch.setenv("RAG_AUTOINJECT", "1")
    assert (
        tools.build_rag_autoinject(
            conv,
            {"thread_id": "t1", "autoinject": False, "whole_doc": False},
        )
        is None
    )