~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for benchmarks/benchmark_functions.py
Tests cover:
- evaluate_simpleqa function
- evaluate_browsecomp function
- evaluate_xbench function
- Configuration handling
"""
from unittest.mock import patch
import tempfile
class TestEvaluateSimpleqa:
"""Tests for the evaluate_simpleqa function."""
def test_evaluate_simpleqa_default_params(self):
"""evaluate_simpleqa works with default parameters."""
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
with patch(
"local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
) as mock_run:
mock_run.return_value = {"metrics": {}, "results": []}
with patch(
"local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
) as mock_settings:
mock_settings.return_value = None
with tempfile.TemporaryDirectory() as tmpdir:
result = evaluate_simpleqa(
num_examples=1,
output_dir=tmpdir,
)
mock_run.assert_called_once()
assert isinstance(result, dict)
def test_evaluate_simpleqa_custom_search_config(self):
"""evaluate_simpleqa passes custom search config."""
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
with patch(
"local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
) as mock_run:
mock_run.return_value = {"metrics": {}, "results": []}
with patch(
"local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
) as mock_settings:
mock_settings.return_value = None
with tempfile.TemporaryDirectory() as tmpdir:
evaluate_simpleqa(
num_examples=1,
search_iterations=5,
questions_per_iteration=2,
search_tool="wikipedia",
output_dir=tmpdir,
)
# Check that search config was passed correctly
call_kwargs = mock_run.call_args[1]
assert (
call_kwargs.get("search_config", {}).get("iterations")
== 5
)
def test_evaluate_simpleqa_with_evaluation_model(self):
"""evaluate_simpleqa accepts evaluation model config."""
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
with patch(
"local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
) as mock_run:
mock_run.return_value = {"metrics": {}, "results": []}
with patch(
"local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
) as mock_settings:
mock_settings.return_value = None
with tempfile.TemporaryDirectory() as tmpdir:
result = evaluate_simpleqa(
num_examples=1,
evaluation_model="gpt-4",
evaluation_provider="openai",
output_dir=tmpdir,
)
assert isinstance(result, dict)
def test_evaluate_simpleqa_human_evaluation(self):
"""evaluate_simpleqa accepts human_evaluation flag."""
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
with patch(
"local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
) as mock_run:
mock_run.return_value = {"metrics": {}, "results": []}
with patch(
"local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
) as mock_settings:
mock_settings.return_value = None
with tempfile.TemporaryDirectory() as tmpdir:
evaluate_simpleqa(
num_examples=1,
human_evaluation=True,
output_dir=tmpdir,
)
call_kwargs = mock_run.call_args[1]
assert call_kwargs.get("human_evaluation") is True
class TestEvaluateBrowsecomp:
"""Tests for the evaluate_browsecomp function."""
def test_evaluate_browsecomp_default_params(self):
"""evaluate_browsecomp works with default parameters."""
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_browsecomp,
)
with patch(
"local_deep_research.benchmarks.benchmark_functions.run_browsecomp_benchmark"
) as mock_run:
mock_run.return_value = {"metrics": {}, "results": []}
with patch(
"local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
) as mock_settings:
mock_settings.return_value = None
with tempfile.TemporaryDirectory() as tmpdir:
result = evaluate_browsecomp(
num_examples=1,
output_dir=tmpdir,
)
mock_run.assert_called_once()
assert isinstance(result, dict)
def test_evaluate_browsecomp_custom_strategy(self):
"""evaluate_browsecomp accepts custom search strategy."""
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_browsecomp,
)
with patch(
"local_deep_research.benchmarks.benchmark_functions.run_browsecomp_benchmark"
) as mock_run:
mock_run.return_value = {"metrics": {}, "results": []}
with patch(
"local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
) as mock_settings:
mock_settings.return_value = None
with tempfile.TemporaryDirectory() as tmpdir:
evaluate_browsecomp(
num_examples=1,
search_strategy="iterdrag",
output_dir=tmpdir,
)
call_kwargs = mock_run.call_args[1]
search_config = call_kwargs.get("search_config", {})
assert search_config.get("search_strategy") == "iterdrag"
class TestEvaluateXbenchDeepsearch:
"""Tests for the evaluate_xbench_deepsearch function."""
def test_evaluate_xbench_deepsearch_default_params(self):
"""evaluate_xbench_deepsearch works with default parameters."""
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_xbench_deepsearch,
)
with patch(
"local_deep_research.benchmarks.benchmark_functions.run_xbench_deepsearch_benchmark"
) as mock_run:
mock_run.return_value = {"metrics": {}, "results": []}
with patch(
"local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
) as mock_settings:
mock_settings.return_value = None
with tempfile.TemporaryDirectory() as tmpdir:
result = evaluate_xbench_deepsearch(
num_examples=1,
output_dir=tmpdir,
)
mock_run.assert_called_once()
assert isinstance(result, dict)
class TestSettingsIntegration:
"""Tests for settings integration in benchmark functions."""
def test_uses_settings_for_model(self):
"""Benchmark functions use settings for model configuration."""
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
with patch(
"local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
) as mock_run:
mock_run.return_value = {"metrics": {}, "results": []}
with patch(
"local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
) as mock_settings:
# Return values for different setting keys
def get_setting(key, *args, **kwargs):
settings = {
"llm.model": "custom-model",
"llm.provider": "custom-provider",
"llm.openai_endpoint.url": "http://custom-endpoint",
}
return settings.get(key)
mock_settings.side_effect = get_setting
with tempfile.TemporaryDirectory() as tmpdir:
evaluate_simpleqa(
num_examples=1,
output_dir=tmpdir,
)
# Should have used the settings
call_kwargs = mock_run.call_args[1]
search_config = call_kwargs.get("search_config", {})
assert search_config.get("model_name") == "custom-model"
class TestBenchmarkOutputDir:
"""Tests for output directory handling."""
def test_output_dir_default(self):
"""Default output directory is benchmark_results."""
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
with patch(
"local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
) as mock_run:
mock_run.return_value = {"metrics": {}, "results": []}
with patch(
"local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
) as mock_settings:
mock_settings.return_value = None
evaluate_simpleqa(num_examples=1)
call_kwargs = mock_run.call_args[1]
assert "output_dir" in call_kwargs
assert call_kwargs["output_dir"] == "benchmark_results"
def test_output_dir_custom(self):
"""Custom output directory is used."""
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
with patch(
"local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
) as mock_run:
mock_run.return_value = {"metrics": {}, "results": []}
with patch(
"local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
) as mock_settings:
mock_settings.return_value = None
with tempfile.TemporaryDirectory() as tmpdir:
evaluate_simpleqa(
num_examples=1,
output_dir=tmpdir,
)
call_kwargs = mock_run.call_args[1]
assert call_kwargs["output_dir"] == tmpdir
class TestSearchToolConfiguration:
"""Tests for search tool configuration."""
def test_default_search_tool_is_searxng(self):
"""Default search tool is searxng."""
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
with patch(
"local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
) as mock_run:
mock_run.return_value = {"metrics": {}, "results": []}
with patch(
"local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
) as mock_settings:
mock_settings.return_value = None
with tempfile.TemporaryDirectory() as tmpdir:
evaluate_simpleqa(
num_examples=1,
output_dir=tmpdir,
)
call_kwargs = mock_run.call_args[1]
search_config = call_kwargs.get("search_config", {})
assert search_config.get("search_tool") == "searxng"
def test_custom_search_tool(self):
"""Custom search tool is used."""
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
with patch(
"local_deep_research.benchmarks.benchmark_functions.run_simpleqa_benchmark"
) as mock_run:
mock_run.return_value = {"metrics": {}, "results": []}
with patch(
"local_deep_research.benchmarks.benchmark_functions.get_setting_from_snapshot"
) as mock_settings:
mock_settings.return_value = None
with tempfile.TemporaryDirectory() as tmpdir:
evaluate_simpleqa(
num_examples=1,
search_tool="wikipedia",
output_dir=tmpdir,
)
call_kwargs = mock_run.call_args[1]
search_config = call_kwargs.get("search_config", {})
assert search_config.get("search_tool") == "wikipedia"