~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for BrowseCompEvaluator class.
Tests the BrowseComp benchmark evaluator implementation.
"""
import tempfile
from pathlib import Path
from unittest.mock import patch
from local_deep_research.benchmarks.evaluators.browsecomp import (
BrowseCompEvaluator,
)
from local_deep_research.benchmarks.evaluators.base import (
BaseBenchmarkEvaluator,
)
class TestBrowseCompEvaluatorInit:
"""Test initialization of BrowseCompEvaluator."""
def test_init_sets_name(self):
"""Test that initialization sets the benchmark name to 'browsecomp'."""
evaluator = BrowseCompEvaluator()
assert evaluator.name == "browsecomp"
def test_inherits_from_base(self):
"""Test that BrowseCompEvaluator inherits from BaseBenchmarkEvaluator."""
evaluator = BrowseCompEvaluator()
assert isinstance(evaluator, BaseBenchmarkEvaluator)
def test_get_name_returns_browsecomp(self):
"""Test that get_name returns 'browsecomp'."""
evaluator = BrowseCompEvaluator()
assert evaluator.get_name() == "browsecomp"
class TestBrowseCompEvaluate:
"""Test evaluate method."""
@patch(
"local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
)
def test_evaluate_calls_runner(self, mock_runner):
"""Test that evaluate calls run_browsecomp_benchmark."""
mock_runner.return_value = {
"metrics": {"accuracy": 0.75},
"report_path": "/tmp/report.md",
}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = BrowseCompEvaluator()
evaluator.evaluate(
system_config={"key": "value"},
num_examples=10,
output_dir=tmpdir,
)
mock_runner.assert_called_once()
call_kwargs = mock_runner.call_args[1]
assert call_kwargs["num_examples"] == 10
assert call_kwargs["run_evaluation"] is True
@patch(
"local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
)
def test_evaluate_returns_accuracy(self, mock_runner):
"""Test that evaluate returns accuracy from results."""
mock_runner.return_value = {
"metrics": {"accuracy": 0.85},
"report_path": "/tmp/report.md",
}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = BrowseCompEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
)
assert result["accuracy"] == 0.85
assert result["quality_score"] == 0.85
@patch(
"local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
)
def test_evaluate_returns_benchmark_type(self, mock_runner):
"""Test that evaluate returns correct benchmark_type."""
mock_runner.return_value = {
"metrics": {"accuracy": 0.5},
"report_path": "/tmp/report.md",
}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = BrowseCompEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
)
assert result["benchmark_type"] == "browsecomp"
@patch(
"local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
)
def test_evaluate_includes_raw_results(self, mock_runner):
"""Test that evaluate includes raw_results from runner."""
raw_data = {
"metrics": {"accuracy": 0.6},
"report_path": "/tmp/report.md",
"extra_data": "test",
}
mock_runner.return_value = raw_data
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = BrowseCompEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
)
assert result["raw_results"] == raw_data
@patch(
"local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
)
def test_evaluate_includes_report_path(self, mock_runner):
"""Test that evaluate includes report_path from results."""
mock_runner.return_value = {
"metrics": {"accuracy": 0.5},
"report_path": "/output/browsecomp/report.md",
}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = BrowseCompEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
)
assert result["report_path"] == "/output/browsecomp/report.md"
@patch(
"local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
)
def test_evaluate_creates_subdirectory(self, mock_runner):
"""Test that evaluate creates benchmark-specific subdirectory."""
mock_runner.return_value = {
"metrics": {"accuracy": 0.5},
"report_path": None,
}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = BrowseCompEvaluator()
evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
)
# Check that subdirectory was created
expected_subdir = Path(tmpdir) / "browsecomp"
assert expected_subdir.exists()
@patch(
"local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
)
def test_evaluate_passes_search_config(self, mock_runner):
"""Test that evaluate passes search_config to runner."""
mock_runner.return_value = {
"metrics": {"accuracy": 0.5},
"report_path": None,
}
config = {"iterations": 5, "search_tool": "google"}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = BrowseCompEvaluator()
evaluator.evaluate(
system_config=config,
num_examples=10,
output_dir=tmpdir,
)
call_kwargs = mock_runner.call_args[1]
assert call_kwargs["search_config"] == config
class TestBrowseCompEvaluateErrors:
"""Test error handling in evaluate method."""
@patch(
"local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
)
def test_evaluate_handles_runner_exception(self, mock_runner):
"""Test that evaluate handles exceptions from runner."""
mock_runner.side_effect = RuntimeError("Benchmark failed")
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = BrowseCompEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
)
assert result["benchmark_type"] == "browsecomp"
assert result["quality_score"] == 0.0
assert result["accuracy"] == 0.0
assert "error" in result
assert "Benchmark failed" in result["error"]
@patch(
"local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
)
def test_evaluate_handles_missing_metrics(self, mock_runner):
"""Test that evaluate handles missing metrics in results."""
mock_runner.return_value = {} # No metrics key
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = BrowseCompEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
)
assert result["accuracy"] == 0.0
assert result["quality_score"] == 0.0
@patch(
"local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
)
def test_evaluate_handles_missing_accuracy(self, mock_runner):
"""Test that evaluate handles missing accuracy in metrics."""
mock_runner.return_value = {"metrics": {}} # No accuracy key
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = BrowseCompEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
)
assert result["accuracy"] == 0.0
assert result["quality_score"] == 0.0
class TestBrowseCompQualityScore:
"""Test quality_score mapping."""
@patch(
"local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
)
def test_quality_score_equals_accuracy(self, mock_runner):
"""Test that quality_score is mapped directly from accuracy."""
mock_runner.return_value = {
"metrics": {"accuracy": 0.923},
"report_path": None,
}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = BrowseCompEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
)
assert result["quality_score"] == result["accuracy"]
assert result["quality_score"] == 0.923
@patch(
"local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
)
def test_quality_score_zero_on_zero_accuracy(self, mock_runner):
"""Test that quality_score is 0 when accuracy is 0."""
mock_runner.return_value = {
"metrics": {"accuracy": 0.0},
"report_path": None,
}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = BrowseCompEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
)
assert result["quality_score"] == 0.0
@patch(
"local_deep_research.benchmarks.evaluators.browsecomp.run_browsecomp_benchmark"
)
def test_quality_score_one_on_perfect_accuracy(self, mock_runner):
"""Test that quality_score is 1.0 when accuracy is 1.0."""
mock_runner.return_value = {
"metrics": {"accuracy": 1.0},
"report_path": None,
}
with tempfile.TemporaryDirectory() as tmpdir:
evaluator = BrowseCompEvaluator()
result = evaluator.evaluate(
system_config={},
num_examples=5,
output_dir=tmpdir,
)
assert result["quality_score"] == 1.0