~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for benchmarks/runners.py
Tests cover:
- format_query function
- run_benchmark function
- process_example function
- Result saving and loading
"""
from unittest.mock import Mock, patch
from pathlib import Path
import tempfile
import pytest
@pytest.fixture(autouse=True)
def _pin_data_directory(tmp_path, monkeypatch):
"""Keep benchmark output out of the real user data directory.
run_benchmark resolves its output directory inside LDR_DATA_DIR and replaces
any other location with a default directory there. Pinning the data
directory to this test's tmp_path keeps that default per test.
"""
monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
class TestFormatQuery:
"""Tests for the format_query function."""
def test_format_query_simpleqa(self):
"""SimpleQA returns question unchanged."""
from local_deep_research.benchmarks.runners import format_query
question = "What is the capital of France?"
result = format_query(question, "simpleqa")
assert result == question
def test_format_query_browsecomp(self):
"""BrowseComp formats with template."""
from local_deep_research.benchmarks.runners import format_query
question = "What is the capital of France?"
result = format_query(question, "browsecomp")
# Should contain the question
assert question in result
# Should be longer than the question (template added)
assert len(result) > len(question)
def test_format_query_default(self):
"""Default format returns question unchanged."""
from local_deep_research.benchmarks.runners import format_query
question = "What is the capital of France?"
result = format_query(question, "unknown_type")
assert result == question
def test_format_query_case_insensitive(self):
"""Dataset type is case insensitive."""
from local_deep_research.benchmarks.runners import format_query
question = "What is the capital of France?"
result1 = format_query(question, "BROWSECOMP")
result2 = format_query(question, "BrowseComp")
result3 = format_query(question, "browsecomp")
assert result1 == result2 == result3
class TestRunBenchmark:
"""Tests for the run_benchmark function."""
def test_run_benchmark_creates_output_dir(self, monkeypatch):
"""run_benchmark creates output directory."""
from local_deep_research.benchmarks.runners import run_benchmark
with tempfile.TemporaryDirectory() as tmpdir:
# run_benchmark output_dir is now confined to the LDR data directory,
# so point the data dir at tmpdir to keep this out-of-tree output
# location honoured rather than clamped to the default.
monkeypatch.setenv("LDR_DATA_DIR", tmpdir)
output_dir = Path(tmpdir) / "new_dir"
# Mock the dataset loading and search
with patch(
"local_deep_research.benchmarks.runners.DatasetRegistry"
) as mock_registry:
mock_dataset = Mock()
mock_dataset.load.return_value = []
mock_registry.create_dataset.return_value = mock_dataset
try:
run_benchmark(
dataset_type="simpleqa",
num_examples=0,
output_dir=str(output_dir),
run_evaluation=False,
)
except Exception:
pass # May fail for other reasons, but dir should be created
# Directory should be created
assert output_dir.exists()
def test_run_benchmark_default_search_config(self):
"""run_benchmark uses default search config when not provided."""
from local_deep_research.benchmarks.runners import run_benchmark
with tempfile.TemporaryDirectory() as tmpdir:
with patch(
"local_deep_research.benchmarks.runners.DatasetRegistry"
) as mock_registry:
mock_dataset = Mock()
mock_dataset.load.return_value = []
mock_registry.create_dataset.return_value = mock_dataset
with patch(
"local_deep_research.benchmarks.runners.generate_report"
) as mock_report:
mock_report.return_value = "Test report"
result = run_benchmark(
dataset_type="simpleqa",
num_examples=0,
output_dir=tmpdir,
run_evaluation=False,
)
# Should return a result dict
assert isinstance(result, dict)
def test_run_benchmark_custom_search_config(self):
"""run_benchmark uses custom search config when provided."""
from local_deep_research.benchmarks.runners import run_benchmark
with tempfile.TemporaryDirectory() as tmpdir:
with patch(
"local_deep_research.benchmarks.runners.DatasetRegistry"
) as mock_registry:
mock_dataset = Mock()
mock_dataset.load.return_value = []
mock_registry.create_dataset.return_value = mock_dataset
with patch(
"local_deep_research.benchmarks.runners.generate_report"
) as mock_report:
mock_report.return_value = "Test report"
custom_config = {
"iterations": 5,
"questions_per_iteration": 2,
"search_tool": "wikipedia",
}
result = run_benchmark(
dataset_type="simpleqa",
num_examples=0,
output_dir=tmpdir,
search_config=custom_config,
run_evaluation=False,
)
assert isinstance(result, dict)
def test_run_benchmark_with_progress_callback(self):
"""run_benchmark calls progress callback."""
from local_deep_research.benchmarks.runners import run_benchmark
callback = Mock()
with tempfile.TemporaryDirectory() as tmpdir:
with patch(
"local_deep_research.benchmarks.runners.DatasetRegistry"
) as mock_registry:
mock_dataset = Mock()
mock_dataset.load.return_value = [
{"problem": "Q1", "answer": "A1"}
]
mock_registry.create_dataset.return_value = mock_dataset
# Return a class that will fail isinstance check gracefully
mock_registry.get_dataset_class.return_value = type(
"FakeDataset", (), {}
)
with patch(
"local_deep_research.benchmarks.runners.quick_summary"
) as mock_summary:
mock_summary.return_value = {"content": "Answer"}
with patch(
"local_deep_research.benchmarks.runners.grade_results"
) as mock_grade:
mock_grade.return_value = []
with patch(
"local_deep_research.benchmarks.runners.generate_report"
) as mock_report:
mock_report.return_value = "Report"
run_benchmark(
dataset_type="simpleqa",
num_examples=1,
output_dir=tmpdir,
progress_callback=callback,
run_evaluation=False,
)
# Callback should be called
assert callback.call_count >= 1
class TestDatasetRegistry:
"""Tests for DatasetRegistry interaction."""
def test_dataset_registry_get_available_datasets(self):
"""DatasetRegistry returns available datasets."""
from local_deep_research.benchmarks.datasets.base import (
DatasetRegistry,
)
# Check that registry has registered datasets
registered = DatasetRegistry.get_available_datasets()
assert isinstance(registered, list)
def test_dataset_registry_create_dataset_method_exists(self):
"""DatasetRegistry has create_dataset method."""
from local_deep_research.benchmarks.datasets.base import (
DatasetRegistry,
)
assert hasattr(DatasetRegistry, "create_dataset")
assert callable(DatasetRegistry.create_dataset)
def test_dataset_registry_load_dataset_method_exists(self):
"""DatasetRegistry has load_dataset method."""
from local_deep_research.benchmarks.datasets.base import (
DatasetRegistry,
)
assert hasattr(DatasetRegistry, "load_dataset")
assert callable(DatasetRegistry.load_dataset)
class TestResultsSaving:
"""Tests for results saving functionality."""
def test_results_saved_as_json(self):
"""Results are saved as JSON files."""
from local_deep_research.benchmarks.runners import run_benchmark
with tempfile.TemporaryDirectory() as tmpdir:
with patch(
"local_deep_research.benchmarks.runners.DatasetRegistry"
) as mock_registry:
mock_dataset = Mock()
mock_dataset.load.return_value = []
mock_registry.create_dataset.return_value = mock_dataset
with patch(
"local_deep_research.benchmarks.runners.generate_report"
) as mock_report:
mock_report.return_value = "Test report"
run_benchmark(
dataset_type="simpleqa",
num_examples=0,
output_dir=tmpdir,
run_evaluation=False,
)
# Check for JSON files in output directory
files = list(Path(tmpdir).iterdir())
json_files = [f for f in files if f.suffix == ".json"]
# May or may not have files depending on results
assert isinstance(json_files, list)
class TestBrowseCompSpecificBehavior:
"""Tests for BrowseComp-specific benchmark behavior."""
def test_browsecomp_uses_template(self):
"""BrowseComp benchmark uses the template."""
from local_deep_research.benchmarks.runners import format_query
from local_deep_research.benchmarks.templates import (
BROWSECOMP_QUERY_TEMPLATE,
)
question = "Test question"
result = format_query(question, "browsecomp")
# Result should be the template with question substituted
expected = BROWSECOMP_QUERY_TEMPLATE.format(question=question)
assert result == expected
class TestEvaluationConfig:
"""Tests for evaluation configuration."""
def test_run_benchmark_with_evaluation_config(self):
"""run_benchmark accepts evaluation config."""
from local_deep_research.benchmarks.runners import run_benchmark
with tempfile.TemporaryDirectory() as tmpdir:
with patch(
"local_deep_research.benchmarks.runners.DatasetRegistry"
) as mock_registry:
mock_dataset = Mock()
mock_dataset.load.return_value = []
mock_registry.create_dataset.return_value = mock_dataset
with patch(
"local_deep_research.benchmarks.runners.generate_report"
) as mock_report:
mock_report.return_value = "Test report"
eval_config = {
"model_name": "gpt-4",
"temperature": 0,
}
result = run_benchmark(
dataset_type="simpleqa",
num_examples=0,
output_dir=tmpdir,
run_evaluation=False,
evaluation_config=eval_config,
)
assert isinstance(result, dict)
def test_run_benchmark_human_evaluation_flag(self):
"""run_benchmark accepts human_evaluation flag."""
from local_deep_research.benchmarks.runners import run_benchmark
with tempfile.TemporaryDirectory() as tmpdir:
with patch(
"local_deep_research.benchmarks.runners.DatasetRegistry"
) as mock_registry:
mock_dataset = Mock()
mock_dataset.load.return_value = []
mock_registry.create_dataset.return_value = mock_dataset
with patch(
"local_deep_research.benchmarks.runners.generate_report"
) as mock_report:
mock_report.return_value = "Test report"
result = run_benchmark(
dataset_type="simpleqa",
num_examples=0,
output_dir=tmpdir,
run_evaluation=False,
human_evaluation=True,
)
assert isinstance(result, dict)