~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for SimpleQAEvaluator class.
Tests the SimpleQA benchmark evaluator implementation.
"""
import tempfile
from pathlib import Path
from unittest.mock import patch
from local_deep_research.benchmarks.evaluators.simpleqa import SimpleQAEvaluator
from local_deep_research.benchmarks.evaluators.base import (
BaseBenchmarkEvaluator,
)
class TestSimpleQAEvaluatorInit:
"""Test initialization of SimpleQAEvaluator."""
def test_init_sets_name(self):
"""Test that initialization sets the benchmark name to 'simpleqa'."""
evaluator = SimpleQAEvaluator()
assert evaluator.name == "simpleqa"
def test_inherits_from_base(self):
"""Test that SimpleQAEvaluator inherits from BaseBenchmarkEvaluator."""
evaluator = SimpleQAEvaluator()
assert isinstance(evaluator, BaseBenchmarkEvaluator)
def test_get_name_returns_simpleqa(self):
"""Test that get_name returns 'simpleqa'."""
evaluator = SimpleQAEvaluator()
assert evaluator.get_name() == "simpleqa"
class TestSimpleQAEvaluateWithRunner:
"""Test evaluate method with legacy runner."""
@patch(
"local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
)
def test_evaluate_calls_runner_when_not_direct(self, mock_runner):
"""Test that evaluate calls run_simpleqa_benchmark when use_direct_dataset=False."""
mock_runner.return_value = {
"metrics": {"accuracy": 0.75},
"report_path": "/tmp/report.md",
}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = SimpleQAEvaluator()
evaluator.evaluate(
system_config={"key": "value"},
num_examples=10,
output_dir=tmpdir,
use_direct_dataset=False,
)
mock_runner.assert_called_once()
call_kwargs = mock_runner.call_args[1]
assert call_kwargs["num_examples"] == 10
assert call_kwargs["run_evaluation"] is True
@patch(
"local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
)
def test_evaluate_returns_accuracy_from_runner(self, mock_runner):
"""Test that evaluate returns accuracy from runner results."""
mock_runner.return_value = {
"metrics": {"accuracy": 0.85},
"report_path": "/tmp/report.md",
}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = SimpleQAEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
use_direct_dataset=False,
)
assert result["accuracy"] == 0.85
assert result["quality_score"] == 0.85
@patch(
"local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
)
def test_evaluate_returns_benchmark_type(self, mock_runner):
"""Test that evaluate returns correct benchmark_type."""
mock_runner.return_value = {
"metrics": {"accuracy": 0.5},
"report_path": "/tmp/report.md",
}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = SimpleQAEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
use_direct_dataset=False,
)
assert result["benchmark_type"] == "simpleqa"
@patch(
"local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
)
def test_evaluate_includes_raw_results(self, mock_runner):
"""Test that evaluate includes raw_results from runner."""
raw_data = {
"metrics": {"accuracy": 0.6},
"report_path": "/tmp/report.md",
"extra_data": "test",
}
mock_runner.return_value = raw_data
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = SimpleQAEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
use_direct_dataset=False,
)
assert result["raw_results"] == raw_data
@patch(
"local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
)
def test_evaluate_passes_search_config(self, mock_runner):
"""Test that evaluate passes search_config to runner."""
mock_runner.return_value = {
"metrics": {"accuracy": 0.5},
"report_path": None,
}
config = {"iterations": 5, "search_tool": "google"}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = SimpleQAEvaluator()
evaluator.evaluate(
system_config=config,
num_examples=10,
output_dir=tmpdir,
use_direct_dataset=False,
)
call_kwargs = mock_runner.call_args[1]
assert call_kwargs["search_config"] == config
class TestSimpleQAEvaluateWithDirectDataset:
"""Test evaluate method with direct dataset class."""
@patch.object(SimpleQAEvaluator, "_run_with_dataset_class")
def test_evaluate_uses_direct_dataset_by_default(self, mock_method):
"""Test that evaluate uses direct dataset by default."""
mock_method.return_value = {
"status": "complete",
"metrics": {"accuracy": 0.8},
"accuracy": 0.8,
}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = SimpleQAEvaluator()
evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
)
mock_method.assert_called_once()
@patch.object(SimpleQAEvaluator, "_run_with_dataset_class")
def test_evaluate_passes_params_to_direct_method(self, mock_method):
"""Test that evaluate passes correct params to direct method."""
mock_method.return_value = {
"status": "complete",
"metrics": {"accuracy": 0.7},
"accuracy": 0.7,
}
config = {"seed": 123, "iterations": 3}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = SimpleQAEvaluator()
evaluator.evaluate(
system_config=config,
num_examples=15,
output_dir=tmpdir,
use_direct_dataset=True,
)
call_kwargs = mock_method.call_args[1]
assert call_kwargs["system_config"] == config
assert call_kwargs["num_examples"] == 15
class TestSimpleQAEvaluateErrors:
"""Test error handling in evaluate method."""
@patch(
"local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
)
def test_evaluate_handles_runner_exception(self, mock_runner):
"""Test that evaluate handles exceptions from runner."""
mock_runner.side_effect = RuntimeError("Benchmark failed")
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = SimpleQAEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
use_direct_dataset=False,
)
assert result["benchmark_type"] == "simpleqa"
assert result["quality_score"] == 0.0
assert result["accuracy"] == 0.0
assert "error" in result
assert "Benchmark failed" in result["error"]
@patch(
"local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
)
def test_evaluate_handles_missing_metrics(self, mock_runner):
"""Test that evaluate handles missing metrics in results."""
mock_runner.return_value = {} # No metrics key
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = SimpleQAEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
use_direct_dataset=False,
)
assert result["accuracy"] == 0.0
assert result["quality_score"] == 0.0
@patch(
"local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
)
def test_evaluate_handles_missing_accuracy(self, mock_runner):
"""Test that evaluate handles missing accuracy in metrics."""
mock_runner.return_value = {"metrics": {}} # No accuracy key
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = SimpleQAEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
use_direct_dataset=False,
)
assert result["accuracy"] == 0.0
assert result["quality_score"] == 0.0
class TestSimpleQACreateSubdirectory:
"""Test subdirectory creation."""
@patch(
"local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
)
def test_evaluate_creates_subdirectory(self, mock_runner):
"""Test that evaluate creates benchmark-specific subdirectory."""
mock_runner.return_value = {
"metrics": {"accuracy": 0.5},
"report_path": None,
}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = SimpleQAEvaluator()
evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
use_direct_dataset=False,
)
# Check that subdirectory was created
expected_subdir = Path(tmpdir) / "simpleqa"
assert expected_subdir.exists()
class TestSimpleQAQualityScore:
"""Test quality_score mapping."""
@patch(
"local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
)
def test_quality_score_equals_accuracy(self, mock_runner):
"""Test that quality_score is mapped directly from accuracy."""
mock_runner.return_value = {
"metrics": {"accuracy": 0.923},
"report_path": None,
}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = SimpleQAEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
use_direct_dataset=False,
)
assert result["quality_score"] == result["accuracy"]
assert result["quality_score"] == 0.923
@patch(
"local_deep_research.benchmarks.evaluators.simpleqa.run_simpleqa_benchmark"
)
def test_quality_score_zero_on_error(self, mock_runner):
"""Test that quality_score is 0 on error."""
mock_runner.side_effect = Exception("Test error")
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = SimpleQAEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
use_direct_dataset=False,
)
assert result["quality_score"] == 0.0