~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Extended tests for benchmarks/benchmark_functions.py.
Covers get_available_benchmarks() and compare_configurations().
"""
import tempfile
from copy import deepcopy
from unittest.mock import patch
import pytest
from local_deep_research.benchmarks.benchmark_functions import (
compare_configurations,
get_available_benchmarks,
)
@pytest.fixture(autouse=True)
def _pin_data_directory(tmp_path, monkeypatch):
"""Keep comparison output out of the real user data directory.
compare_configurations resolves its output root inside LDR_DATA_DIR and
replaces any other location with ``benchmark_comparisons`` there.
Pinning the data directory to this test's tmp_path keeps that root per
test.
"""
monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
# ── get_available_benchmarks ───────────────────────────────────────
class TestGetAvailableBenchmarks:
"""Tests for get_available_benchmarks."""
def test_returns_list(self):
"""Return type is a list."""
result = get_available_benchmarks()
assert isinstance(result, list)
assert len(result) > 0
def test_has_required_fields(self):
"""Each benchmark dict has id, name, description, recommended_examples."""
for b in get_available_benchmarks():
assert "id" in b
assert "name" in b
assert "description" in b
assert "recommended_examples" in b
assert isinstance(b["recommended_examples"], int)
def test_includes_all_types(self):
"""All three benchmark types are present."""
ids = {b["id"] for b in get_available_benchmarks()}
assert "simpleqa" in ids
assert "browsecomp" in ids
assert "xbench_deepsearch" in ids
# ── compare_configurations ─────────────────────────────────────────
class TestCompareConfigurations:
"""Tests for compare_configurations."""
def _mock_benchmark_result(self, config_name="test"):
return {
"metrics": {"accuracy": 0.85, "average_processing_time": 1.5},
"total_examples": 20,
}
def test_default_configurations(self):
"""Default 3 configurations are used when none provided."""
with (
patch(
"local_deep_research.benchmarks.benchmark_functions.run_benchmark"
) as mock_run,
patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
),
):
mock_run.return_value = self._mock_benchmark_result()
with tempfile.TemporaryDirectory() as tmpdir:
result = compare_configurations(
num_examples=5, output_dir=tmpdir
)
# 3 default configs → 3 calls
assert mock_run.call_count == 3
assert result["status"] == "complete"
assert result["configurations_tested"] == 3
def test_custom_configurations(self):
"""Custom configurations list is used."""
custom_configs = [
{
"name": "Custom A",
"search_tool": "brave",
"iterations": 2,
"questions_per_iteration": 4,
},
{
"name": "Custom B",
"search_tool": "duckduckgo",
"iterations": 1,
"questions_per_iteration": 2,
},
]
with (
patch(
"local_deep_research.benchmarks.benchmark_functions.run_benchmark"
) as mock_run,
patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
),
):
mock_run.return_value = self._mock_benchmark_result()
with tempfile.TemporaryDirectory() as tmpdir:
result = compare_configurations(
configurations=deepcopy(custom_configs),
num_examples=5,
output_dir=tmpdir,
)
assert mock_run.call_count == 2
assert result["configurations_tested"] == 2
def test_pops_name_from_config(self):
"""config.pop('name') mutates the caller's dicts (documenting the bug)."""
configs = [
{
"name": "Foo",
"search_tool": "searxng",
"iterations": 1,
"questions_per_iteration": 3,
},
]
with (
patch(
"local_deep_research.benchmarks.benchmark_functions.run_benchmark"
) as mock_run,
patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
),
):
mock_run.return_value = self._mock_benchmark_result()
with tempfile.TemporaryDirectory() as tmpdir:
compare_configurations(
configurations=configs,
num_examples=5,
output_dir=tmpdir,
)
# The original dict has been mutated: "name" was popped
assert "name" not in configs[0]
def test_generates_report(self):
"""Comparison report is written via write_file_verified."""
with (
patch(
"local_deep_research.benchmarks.benchmark_functions.run_benchmark"
) as mock_run,
patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
) as mock_write,
):
mock_run.return_value = self._mock_benchmark_result()
with tempfile.TemporaryDirectory() as tmpdir:
result = compare_configurations(
num_examples=5, output_dir=tmpdir
)
mock_write.assert_called_once()
# Report path is returned
assert "report_path" in result
def test_handles_benchmark_error(self):
"""Error in run_benchmark for one config propagates (no swallowing)."""
configs = [
{
"name": "Good",
"search_tool": "searxng",
"iterations": 1,
"questions_per_iteration": 3,
},
{
"name": "Bad",
"search_tool": "searxng",
"iterations": 1,
"questions_per_iteration": 3,
},
]
call_count = 0
def side_effect(**kwargs):
nonlocal call_count
call_count += 1
if call_count == 2:
raise RuntimeError("benchmark failed")
return self._mock_benchmark_result()
with (
patch(
"local_deep_research.benchmarks.benchmark_functions.run_benchmark",
side_effect=side_effect,
),
patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
),
):
with tempfile.TemporaryDirectory() as tmpdir:
with pytest.raises(RuntimeError, match="benchmark failed"):
compare_configurations(
configurations=configs,
num_examples=5,
output_dir=tmpdir,
)