~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Tests for benchmark API functions."""
from unittest.mock import patch
import pytest
@pytest.fixture(autouse=True)
def _pin_data_directory(tmp_path, monkeypatch):
"""Keep comparison output out of the real user data directory.
compare_configurations resolves its output root inside LDR_DATA_DIR and
replaces any other location with ``benchmark_comparisons`` there.
Pinning the data directory to this test's tmp_path keeps that root per
test.
"""
monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
class TestGetAvailableBenchmarks:
def test_returns_list(self):
from local_deep_research.api.benchmark_functions import (
get_available_benchmarks,
)
result = get_available_benchmarks()
assert isinstance(result, list)
def test_contains_simpleqa(self):
from local_deep_research.api.benchmark_functions import (
get_available_benchmarks,
)
result = get_available_benchmarks()
ids = [b["id"] for b in result]
assert "simpleqa" in ids
def test_contains_browsecomp(self):
from local_deep_research.api.benchmark_functions import (
get_available_benchmarks,
)
result = get_available_benchmarks()
ids = [b["id"] for b in result]
assert "browsecomp" in ids
def test_contains_xbench(self):
from local_deep_research.api.benchmark_functions import (
get_available_benchmarks,
)
result = get_available_benchmarks()
ids = [b["id"] for b in result]
assert "xbench_deepsearch" in ids
def test_benchmarks_have_required_fields(self):
from local_deep_research.api.benchmark_functions import (
get_available_benchmarks,
)
result = get_available_benchmarks()
for benchmark in result:
assert "id" in benchmark
assert "name" in benchmark
assert "description" in benchmark
assert "recommended_examples" in benchmark
class TestEvaluateSimpleqa:
def test_calls_run_benchmark(self):
from local_deep_research.api.benchmark_functions import (
evaluate_simpleqa,
)
with patch(
"local_deep_research.api.benchmark_functions.run_simpleqa_benchmark"
) as mock:
mock.return_value = {"status": "complete"}
result = evaluate_simpleqa(num_examples=10)
mock.assert_called_once()
assert result["status"] == "complete"
def test_passes_search_config(self):
from local_deep_research.api.benchmark_functions import (
evaluate_simpleqa,
)
with patch(
"local_deep_research.api.benchmark_functions.run_simpleqa_benchmark"
) as mock:
mock.return_value = {}
evaluate_simpleqa(search_iterations=5, questions_per_iteration=7)
call_args = mock.call_args
assert call_args.kwargs["search_config"]["iterations"] == 5
assert (
call_args.kwargs["search_config"]["questions_per_iteration"]
== 7
)
def test_passes_evaluation_config(self):
from local_deep_research.api.benchmark_functions import (
evaluate_simpleqa,
)
with patch(
"local_deep_research.api.benchmark_functions.run_simpleqa_benchmark"
) as mock:
mock.return_value = {}
evaluate_simpleqa(
evaluation_model="gpt-4", evaluation_provider="openai"
)
call_args = mock.call_args
assert (
call_args.kwargs["evaluation_config"]["model_name"] == "gpt-4"
)
assert call_args.kwargs["evaluation_config"]["provider"] == "openai"
class TestEvaluateBrowsecomp:
def test_calls_run_benchmark(self):
from local_deep_research.api.benchmark_functions import (
evaluate_browsecomp,
)
with patch(
"local_deep_research.api.benchmark_functions.run_browsecomp_benchmark"
) as mock:
mock.return_value = {"status": "complete"}
result = evaluate_browsecomp(num_examples=10)
mock.assert_called_once()
assert result["status"] == "complete"
class TestEvaluateXbenchDeepsearch:
def test_calls_run_benchmark(self):
from local_deep_research.api.benchmark_functions import (
evaluate_xbench_deepsearch,
)
with patch(
"local_deep_research.api.benchmark_functions.run_xbench_deepsearch_benchmark"
) as mock:
mock.return_value = {"status": "complete"}
result = evaluate_xbench_deepsearch(num_examples=10)
mock.assert_called_once()
assert result["status"] == "complete"
class TestCompareConfigurations:
def test_returns_dict(self):
from local_deep_research.api.benchmark_functions import (
compare_configurations,
)
with patch(
"local_deep_research.api.benchmark_functions.run_benchmark"
) as mock_run:
with patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
):
mock_run.return_value = {"metrics": {"accuracy": 0.8}}
result = compare_configurations(num_examples=5)
assert isinstance(result, dict)
assert "status" in result
def test_uses_default_configurations(self):
from local_deep_research.api.benchmark_functions import (
compare_configurations,
)
with patch(
"local_deep_research.api.benchmark_functions.run_benchmark"
) as mock_run:
with patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
):
mock_run.return_value = {"metrics": {}}
result = compare_configurations(num_examples=5)
assert result["configurations_tested"] == 3
def test_custom_configurations(self):
from local_deep_research.api.benchmark_functions import (
compare_configurations,
)
custom_configs = [
{"name": "Custom", "search_tool": "wikipedia", "iterations": 2}
]
with patch(
"local_deep_research.api.benchmark_functions.run_benchmark"
) as mock_run:
with patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
):
mock_run.return_value = {"metrics": {}}
result = compare_configurations(
num_examples=5, configurations=custom_configs
)
assert result["configurations_tested"] == 1