~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for uncovered code paths in api/benchmark_functions.py.
Targets:
- evaluate_simpleqa: evaluation_config with model only, provider only
- evaluate_browsecomp: evaluation_config branches
- evaluate_xbench_deepsearch: evaluation_config branches
- compare_configurations: default configs, report generation, metrics formatting
"""
from unittest.mock import patch
import pytest
MODULE = "local_deep_research.api.benchmark_functions"
@pytest.fixture(autouse=True)
def _pin_data_directory(tmp_path, monkeypatch):
"""Keep comparison output out of the real user data directory.
compare_configurations resolves its output root inside LDR_DATA_DIR and
replaces any other location with ``benchmark_comparisons`` there.
Pinning the data directory to this test's tmp_path keeps that root per
test.
"""
monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
class TestEvaluateSimpleqa:
@patch(f"{MODULE}.run_simpleqa_benchmark")
def test_evaluation_config_with_model_only(self, mock_run):
"""evaluation_config is set when only model is provided."""
from local_deep_research.api.benchmark_functions import (
evaluate_simpleqa,
)
mock_run.return_value = {"status": "complete"}
evaluate_simpleqa(
num_examples=5,
evaluation_model="gpt-4",
)
call_kwargs = mock_run.call_args[1]
assert call_kwargs["evaluation_config"] == {"model_name": "gpt-4"}
@patch(f"{MODULE}.run_simpleqa_benchmark")
def test_evaluation_config_with_provider_only(self, mock_run):
"""evaluation_config is set when only provider is provided."""
from local_deep_research.api.benchmark_functions import (
evaluate_simpleqa,
)
mock_run.return_value = {"status": "complete"}
evaluate_simpleqa(
num_examples=5,
evaluation_provider="openai",
)
call_kwargs = mock_run.call_args[1]
assert call_kwargs["evaluation_config"] == {"provider": "openai"}
@patch(f"{MODULE}.run_simpleqa_benchmark")
def test_evaluation_config_with_both(self, mock_run):
"""evaluation_config includes both model and provider."""
from local_deep_research.api.benchmark_functions import (
evaluate_simpleqa,
)
mock_run.return_value = {"status": "complete"}
evaluate_simpleqa(
num_examples=5,
evaluation_model="gpt-4",
evaluation_provider="openai",
)
call_kwargs = mock_run.call_args[1]
assert call_kwargs["evaluation_config"]["model_name"] == "gpt-4"
assert call_kwargs["evaluation_config"]["provider"] == "openai"
@patch(f"{MODULE}.run_simpleqa_benchmark")
def test_no_evaluation_config(self, mock_run):
"""evaluation_config is None when neither model nor provider."""
from local_deep_research.api.benchmark_functions import (
evaluate_simpleqa,
)
mock_run.return_value = {"status": "complete"}
evaluate_simpleqa(num_examples=5)
call_kwargs = mock_run.call_args[1]
assert call_kwargs["evaluation_config"] is None
class TestEvaluateBrowsecomp:
@patch(f"{MODULE}.run_browsecomp_benchmark")
def test_evaluation_config_with_model(self, mock_run):
"""evaluation_config with model for browsecomp."""
from local_deep_research.api.benchmark_functions import (
evaluate_browsecomp,
)
mock_run.return_value = {"status": "complete"}
evaluate_browsecomp(
num_examples=5,
evaluation_model="claude-3",
)
call_kwargs = mock_run.call_args[1]
assert call_kwargs["evaluation_config"]["model_name"] == "claude-3"
@patch(f"{MODULE}.run_browsecomp_benchmark")
def test_evaluation_config_with_provider(self, mock_run):
"""evaluation_config with provider for browsecomp."""
from local_deep_research.api.benchmark_functions import (
evaluate_browsecomp,
)
mock_run.return_value = {"status": "complete"}
evaluate_browsecomp(
num_examples=5,
evaluation_provider="anthropic",
)
call_kwargs = mock_run.call_args[1]
assert call_kwargs["evaluation_config"]["provider"] == "anthropic"
class TestEvaluateXbenchDeepsearch:
@patch(f"{MODULE}.run_xbench_deepsearch_benchmark")
def test_evaluation_config_with_model(self, mock_run):
"""evaluation_config with model for xbench."""
from local_deep_research.api.benchmark_functions import (
evaluate_xbench_deepsearch,
)
mock_run.return_value = {"status": "complete"}
evaluate_xbench_deepsearch(
num_examples=5,
evaluation_model="gpt-4o",
)
call_kwargs = mock_run.call_args[1]
assert call_kwargs["evaluation_config"]["model_name"] == "gpt-4o"
@patch(f"{MODULE}.run_xbench_deepsearch_benchmark")
def test_evaluation_config_with_provider(self, mock_run):
"""evaluation_config with provider for xbench."""
from local_deep_research.api.benchmark_functions import (
evaluate_xbench_deepsearch,
)
mock_run.return_value = {"status": "complete"}
evaluate_xbench_deepsearch(
num_examples=5,
evaluation_provider="openai",
)
call_kwargs = mock_run.call_args[1]
assert call_kwargs["evaluation_config"]["provider"] == "openai"
class TestCompareConfigurations:
@patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
)
@patch(f"{MODULE}.run_benchmark")
def test_default_configurations(self, mock_run, mock_write):
"""Uses default configurations when none provided."""
from local_deep_research.api.benchmark_functions import (
compare_configurations,
)
mock_run.return_value = {
"metrics": {"accuracy": 0.85, "average_processing_time": 1.5},
"total_examples": 20,
}
result = compare_configurations(
dataset_type="simpleqa",
num_examples=5,
output_dir="/tmp/test_bench",
)
assert result["status"] == "complete"
assert result["configurations_tested"] == 3 # 3 default configs
assert mock_run.call_count == 3
@patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
)
@patch(f"{MODULE}.run_benchmark")
def test_custom_configurations(self, mock_run, mock_write):
"""Passes custom configurations correctly."""
from local_deep_research.api.benchmark_functions import (
compare_configurations,
)
mock_run.return_value = {
"metrics": {"accuracy": 0.9, "average_processing_time": 2.0},
"total_examples": 10,
}
configs = [
{
"name": "Fast",
"search_tool": "wikipedia",
"iterations": 1,
"questions_per_iteration": 2,
},
]
result = compare_configurations(
configurations=configs,
num_examples=10,
output_dir="/tmp/test_bench",
)
assert result["configurations_tested"] == 1
@patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
)
@patch(f"{MODULE}.run_benchmark")
def test_report_written(self, mock_run, mock_write):
"""Comparison report is written to file."""
from local_deep_research.api.benchmark_functions import (
compare_configurations,
)
mock_run.return_value = {
"metrics": {"accuracy": 0.85, "average_processing_time": 1.5},
"total_examples": 20,
}
result = compare_configurations(
num_examples=5,
output_dir="/tmp/test_bench",
)
mock_write.assert_called_once()
# Report content should contain markdown table
report_content = mock_write.call_args[0][1]
assert "Configuration" in report_content
assert "Accuracy" in report_content
assert result["report_path"].endswith(".md")
@patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
)
@patch(f"{MODULE}.run_benchmark")
def test_extra_config_items_passed(self, mock_run, mock_write):
"""Extra config items beyond standard ones are passed through."""
from local_deep_research.api.benchmark_functions import (
compare_configurations,
)
mock_run.return_value = {
"metrics": {"accuracy": 0.5},
"total_examples": 5,
}
configs = [
{
"name": "Custom",
"search_tool": "searxng",
"iterations": 1,
"questions_per_iteration": 1,
"custom_param": "value",
},
]
compare_configurations(
configurations=configs,
num_examples=5,
output_dir="/tmp/test_bench",
)
call_kwargs = mock_run.call_args[1]
assert call_kwargs["search_config"]["custom_param"] == "value"