~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Extra coverage tests for benchmarks/benchmark_functions.py.
Targets the 36 missing lines not covered by test_benchmark_functions_coverage.py:
- compare_configurations() with default configurations
- compare_configurations() with custom configurations
- compare_configurations() with various dataset_type values
- get_available_benchmarks() return structure
- evaluate_xbench_deepsearch() with non-openai_endpoint provider
- evaluate_browsecomp() with non-openai_endpoint provider
- compare_configurations() report content and path
"""
from unittest.mock import patch
import pytest
MODULE = "local_deep_research.benchmarks.benchmark_functions"
@pytest.fixture(autouse=True)
def _pin_data_directory(tmp_path, monkeypatch):
"""Keep comparison output out of the real user data directory.
compare_configurations resolves its output root inside LDR_DATA_DIR and
replaces any other location with ``benchmark_comparisons`` there.
Pinning the data directory to this test's tmp_path keeps that root per
test.
"""
monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
def _no_settings(key, *args, **kwargs):
return None
# ---------------------------------------------------------------------------
# get_available_benchmarks
# ---------------------------------------------------------------------------
class TestGetAvailableBenchmarks:
def test_returns_three_benchmarks(self):
from local_deep_research.benchmarks.benchmark_functions import (
get_available_benchmarks,
)
benchmarks = get_available_benchmarks()
assert len(benchmarks) == 3
def test_each_benchmark_has_required_fields(self):
from local_deep_research.benchmarks.benchmark_functions import (
get_available_benchmarks,
)
benchmarks = get_available_benchmarks()
for b in benchmarks:
assert "id" in b
assert "name" in b
assert "description" in b
def test_simpleqa_is_present(self):
from local_deep_research.benchmarks.benchmark_functions import (
get_available_benchmarks,
)
ids = [b["id"] for b in get_available_benchmarks()]
assert "simpleqa" in ids
def test_xbench_deepsearch_is_present(self):
from local_deep_research.benchmarks.benchmark_functions import (
get_available_benchmarks,
)
ids = [b["id"] for b in get_available_benchmarks()]
assert "xbench_deepsearch" in ids
# ---------------------------------------------------------------------------
# compare_configurations – default configs used when none provided
# ---------------------------------------------------------------------------
class TestCompareConfigurationsDefaults:
def test_default_configurations_used_when_none_provided(self):
with (
patch(f"{MODULE}.run_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
),
):
mock_run.return_value = {
"metrics": {"accuracy": 0.5, "average_processing_time": 1.0},
"total_examples": 10,
"configuration_name": "test",
"search_config": {},
}
from local_deep_research.benchmarks.benchmark_functions import (
compare_configurations,
)
result = compare_configurations(
dataset_type="simpleqa",
num_examples=1,
)
# Default has 3 configurations
assert result["configurations_tested"] == 3
def test_status_is_complete(self):
with (
patch(f"{MODULE}.run_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
),
):
mock_run.return_value = {
"metrics": {"accuracy": 0.5, "average_processing_time": 1.0},
"total_examples": 10,
"configuration_name": "test",
"search_config": {},
}
from local_deep_research.benchmarks.benchmark_functions import (
compare_configurations,
)
result = compare_configurations(num_examples=1)
assert result["status"] == "complete"
# ---------------------------------------------------------------------------
# compare_configurations – custom configurations
# ---------------------------------------------------------------------------
class TestCompareConfigurationsCustom:
def test_custom_configs_run_for_each(self):
with (
patch(f"{MODULE}.run_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
),
):
mock_run.return_value = {
"metrics": {"accuracy": 0.6, "average_processing_time": 2.0},
"total_examples": 5,
"configuration_name": "custom",
"search_config": {},
}
from local_deep_research.benchmarks.benchmark_functions import (
compare_configurations,
)
cfgs = [
{
"name": "Cfg A",
"search_tool": "searxng",
"iterations": 1,
"questions_per_iteration": 2,
},
{
"name": "Cfg B",
"search_tool": "wikipedia",
"iterations": 2,
"questions_per_iteration": 3,
},
]
result = compare_configurations(configurations=cfgs, num_examples=1)
assert mock_run.call_count == 2
assert result["configurations_tested"] == 2
def test_report_path_in_result(self):
with (
patch(f"{MODULE}.run_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
),
):
mock_run.return_value = {
"metrics": {"accuracy": 0.7, "average_processing_time": 1.5},
"total_examples": 3,
"configuration_name": "rep",
"search_config": {},
}
from local_deep_research.benchmarks.benchmark_functions import (
compare_configurations,
)
result = compare_configurations(
configurations=[
{
"name": "Only",
"search_tool": "searxng",
"iterations": 1,
"questions_per_iteration": 2,
}
],
num_examples=1,
)
assert "report_path" in result
def test_dataset_type_reflected_in_result(self):
with (
patch(f"{MODULE}.run_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
),
):
mock_run.return_value = {
"metrics": {"accuracy": 0.8, "average_processing_time": 1.0},
"total_examples": 2,
"configuration_name": "t",
"search_config": {},
}
from local_deep_research.benchmarks.benchmark_functions import (
compare_configurations,
)
result = compare_configurations(
dataset_type="browsecomp",
configurations=[
{
"name": "X",
"search_tool": "searxng",
"iterations": 1,
"questions_per_iteration": 2,
}
],
num_examples=1,
)
assert result["dataset_type"] == "browsecomp"
# ---------------------------------------------------------------------------
# evaluate_browsecomp – non-openai_endpoint provider
# ---------------------------------------------------------------------------
class TestEvaluateBrowsecompNonOpenaiEndpoint:
def test_non_openai_endpoint_provider_has_no_url(self):
with (
patch(f"{MODULE}.run_browsecomp_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_browsecomp,
)
evaluate_browsecomp(
num_examples=1,
evaluation_model="my-model",
evaluation_provider="ollama",
endpoint_url="http://ignored.com",
)
ec = mock_run.call_args[1]["evaluation_config"]
assert "openai_endpoint_url" not in ec
def test_evaluation_config_none_when_no_eval_params(self):
with (
patch(f"{MODULE}.run_browsecomp_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_browsecomp,
)
evaluate_browsecomp(num_examples=1)
ec = mock_run.call_args[1]["evaluation_config"]
assert ec is None
# ---------------------------------------------------------------------------
# evaluate_xbench_deepsearch – non-openai_endpoint provider
# ---------------------------------------------------------------------------
class TestEvaluateXbenchNonOpenaiEndpoint:
def test_non_openai_endpoint_provider_has_no_url(self):
with (
patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_xbench_deepsearch,
)
evaluate_xbench_deepsearch(
num_examples=1,
evaluation_model="m",
evaluation_provider="anthropic",
endpoint_url="http://ignored.com",
)
ec = mock_run.call_args[1]["evaluation_config"]
assert "openai_endpoint_url" not in ec
def test_full_settings_override_xbench_search_config(self):
def full_settings(key, *args, **kwargs):
return {
"llm.model": "env-model",
"llm.provider": "env-provider",
"llm.openai_endpoint.url": "http://env-xb.com",
}.get(key)
with (
patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=full_settings
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_xbench_deepsearch,
)
evaluate_xbench_deepsearch(num_examples=1)
sc = mock_run.call_args[1]["search_config"]
assert sc["model_name"] == "env-model"
assert sc["provider"] == "env-provider"
assert sc["openai_endpoint_url"] == "http://env-xb.com"
def test_human_evaluation_passed_through(self):
with (
patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_xbench_deepsearch,
)
evaluate_xbench_deepsearch(num_examples=1, human_evaluation=True)
assert mock_run.call_args[1]["human_evaluation"] is True