Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

6.7 KB
Raw Blame
"""Tests for benchmark API functions."""

from unittest.mock import patch

import pytest


@pytest.fixture(autouse=True)
def _pin_data_directory(tmp_path, monkeypatch):
    """Keep comparison output out of the real user data directory.

    compare_configurations resolves its output root inside LDR_DATA_DIR and
    replaces any other location with ``benchmark_comparisons`` there.
    Pinning the data directory to this test's tmp_path keeps that root per
    test.
    """
    monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))


class TestGetAvailableBenchmarks:
    def test_returns_list(self):
        from local_deep_research.api.benchmark_functions import (
            get_available_benchmarks,
        )

        result = get_available_benchmarks()
        assert isinstance(result, list)

    def test_contains_simpleqa(self):
        from local_deep_research.api.benchmark_functions import (
            get_available_benchmarks,
        )

        result = get_available_benchmarks()
        ids = [b["id"] for b in result]
        assert "simpleqa" in ids

    def test_contains_browsecomp(self):
        from local_deep_research.api.benchmark_functions import (
            get_available_benchmarks,
        )

        result = get_available_benchmarks()
        ids = [b["id"] for b in result]
        assert "browsecomp" in ids

    def test_contains_xbench(self):
        from local_deep_research.api.benchmark_functions import (
            get_available_benchmarks,
        )

        result = get_available_benchmarks()
        ids = [b["id"] for b in result]
        assert "xbench_deepsearch" in ids

    def test_benchmarks_have_required_fields(self):
        from local_deep_research.api.benchmark_functions import (
            get_available_benchmarks,
        )

        result = get_available_benchmarks()
        for benchmark in result:
            assert "id" in benchmark
            assert "name" in benchmark
            assert "description" in benchmark
            assert "recommended_examples" in benchmark


class TestEvaluateSimpleqa:
    def test_calls_run_benchmark(self):
        from local_deep_research.api.benchmark_functions import (
            evaluate_simpleqa,
        )

        with patch(
            "local_deep_research.api.benchmark_functions.run_simpleqa_benchmark"
        ) as mock:
            mock.return_value = {"status": "complete"}
            result = evaluate_simpleqa(num_examples=10)
            mock.assert_called_once()
            assert result["status"] == "complete"

    def test_passes_search_config(self):
        from local_deep_research.api.benchmark_functions import (
            evaluate_simpleqa,
        )

        with patch(
            "local_deep_research.api.benchmark_functions.run_simpleqa_benchmark"
        ) as mock:
            mock.return_value = {}
            evaluate_simpleqa(search_iterations=5, questions_per_iteration=7)
            call_args = mock.call_args
            assert call_args.kwargs["search_config"]["iterations"] == 5
            assert (
                call_args.kwargs["search_config"]["questions_per_iteration"]
                == 7
            )

    def test_passes_evaluation_config(self):
        from local_deep_research.api.benchmark_functions import (
            evaluate_simpleqa,
        )

        with patch(
            "local_deep_research.api.benchmark_functions.run_simpleqa_benchmark"
        ) as mock:
            mock.return_value = {}
            evaluate_simpleqa(
                evaluation_model="gpt-4", evaluation_provider="openai"
            )
            call_args = mock.call_args
            assert (
                call_args.kwargs["evaluation_config"]["model_name"] == "gpt-4"
            )
            assert call_args.kwargs["evaluation_config"]["provider"] == "openai"


class TestEvaluateBrowsecomp:
    def test_calls_run_benchmark(self):
        from local_deep_research.api.benchmark_functions import (
            evaluate_browsecomp,
        )

        with patch(
            "local_deep_research.api.benchmark_functions.run_browsecomp_benchmark"
        ) as mock:
            mock.return_value = {"status": "complete"}
            result = evaluate_browsecomp(num_examples=10)
            mock.assert_called_once()
            assert result["status"] == "complete"


class TestEvaluateXbenchDeepsearch:
    def test_calls_run_benchmark(self):
        from local_deep_research.api.benchmark_functions import (
            evaluate_xbench_deepsearch,
        )

        with patch(
            "local_deep_research.api.benchmark_functions.run_xbench_deepsearch_benchmark"
        ) as mock:
            mock.return_value = {"status": "complete"}
            result = evaluate_xbench_deepsearch(num_examples=10)
            mock.assert_called_once()
            assert result["status"] == "complete"


class TestCompareConfigurations:
    def test_returns_dict(self):
        from local_deep_research.api.benchmark_functions import (
            compare_configurations,
        )

        with patch(
            "local_deep_research.api.benchmark_functions.run_benchmark"
        ) as mock_run:
            with patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ):
                mock_run.return_value = {"metrics": {"accuracy": 0.8}}
                result = compare_configurations(num_examples=5)
                assert isinstance(result, dict)
                assert "status" in result

    def test_uses_default_configurations(self):
        from local_deep_research.api.benchmark_functions import (
            compare_configurations,
        )

        with patch(
            "local_deep_research.api.benchmark_functions.run_benchmark"
        ) as mock_run:
            with patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ):
                mock_run.return_value = {"metrics": {}}
                result = compare_configurations(num_examples=5)
                assert result["configurations_tested"] == 3

    def test_custom_configurations(self):
        from local_deep_research.api.benchmark_functions import (
            compare_configurations,
        )

        custom_configs = [
            {"name": "Custom", "search_tool": "wikipedia", "iterations": 2}
        ]
        with patch(
            "local_deep_research.api.benchmark_functions.run_benchmark"
        ) as mock_run:
            with patch(
                "local_deep_research.security.file_write_verifier.write_file_verified"
            ):
                mock_run.return_value = {"metrics": {}}
                result = compare_configurations(
                    num_examples=5, configurations=custom_configs
                )
                assert result["configurations_tested"] == 1