~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Coverage tests for benchmarks/benchmark_functions.py.
Focuses on logic paths not covered by existing test files:
- search_model / search_provider / endpoint_url propagation into search_config
- openai_endpoint evaluation provider with url param vs env fallback
- Settings snapshot overriding explicit params
- temperature=0 in evaluation_config
- evaluate_xbench_deepsearch default iterations=4
"""
from unittest.mock import patch
MODULE = "local_deep_research.benchmarks.benchmark_functions"
def _no_settings(key, *args, **kwargs):
return None
def _settings_with_url(key, *args, **kwargs):
if key == "llm.openai_endpoint.url":
return "http://env-endpoint.example.com"
return None
def _full_settings(key, *args, **kwargs):
return {
"llm.model": "env-model",
"llm.provider": "env-provider",
"llm.openai_endpoint.url": "http://env-url.example.com",
}.get(key)
class TestSimpleqaSearchModelConfig:
def test_search_model_added_to_config(self):
with (
patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
evaluate_simpleqa(num_examples=1, search_model="my-model")
sc = mock_run.call_args[1]["search_config"]
assert sc["model_name"] == "my-model"
def test_search_provider_added_to_config(self):
with (
patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
evaluate_simpleqa(num_examples=1, search_provider="ollama")
sc = mock_run.call_args[1]["search_config"]
assert sc["provider"] == "ollama"
def test_endpoint_url_added_to_config(self):
with (
patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
evaluate_simpleqa(
num_examples=1, endpoint_url="http://my-endpoint.com"
)
sc = mock_run.call_args[1]["search_config"]
assert sc["openai_endpoint_url"] == "http://my-endpoint.com"
class TestSimpleqaOpenaiEndpointEvaluation:
def test_openai_endpoint_uses_explicit_url(self):
with (
patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
evaluate_simpleqa(
num_examples=1,
evaluation_model="gpt-4",
evaluation_provider="openai_endpoint",
endpoint_url="http://explicit.example.com",
)
ec = mock_run.call_args[1]["evaluation_config"]
assert ec["temperature"] == 0
assert ec["provider"] == "openai_endpoint"
assert ec["openai_endpoint_url"] == "http://explicit.example.com"
def test_openai_endpoint_falls_back_to_env_url(self):
with (
patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot",
side_effect=_settings_with_url,
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
evaluate_simpleqa(
num_examples=1,
evaluation_model="gpt-4",
evaluation_provider="openai_endpoint",
)
ec = mock_run.call_args[1]["evaluation_config"]
assert (
ec["openai_endpoint_url"] == "http://env-endpoint.example.com"
)
def test_non_openai_endpoint_provider_no_url(self):
with (
patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot",
side_effect=_settings_with_url,
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
evaluate_simpleqa(
num_examples=1,
evaluation_model="gpt-4",
evaluation_provider="openai",
endpoint_url="http://explicit.example.com",
)
ec = mock_run.call_args[1]["evaluation_config"]
assert "openai_endpoint_url" not in ec
class TestSimpleqaSettingsOverride:
def test_settings_override_explicit_params(self):
with (
patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot",
side_effect=_full_settings,
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
evaluate_simpleqa(
num_examples=1,
search_model="param-model",
search_provider="param-provider",
endpoint_url="http://param-url.com",
)
sc = mock_run.call_args[1]["search_config"]
assert sc["model_name"] == "env-model"
assert sc["provider"] == "env-provider"
assert sc["openai_endpoint_url"] == "http://env-url.example.com"
class TestSimpleqaEvaluationTemperature:
def test_temperature_zero_with_model_only(self):
with (
patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
evaluate_simpleqa(num_examples=1, evaluation_model="gpt-4")
ec = mock_run.call_args[1]["evaluation_config"]
assert ec["temperature"] == 0
assert ec["model_name"] == "gpt-4"
assert "provider" not in ec
def test_no_evaluation_config_when_no_eval_params(self):
with (
patch(f"{MODULE}.run_simpleqa_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
evaluate_simpleqa(num_examples=1)
ec = mock_run.call_args[1]["evaluation_config"]
assert ec is None
class TestBrowsecompSearchModelConfig:
def test_search_model_and_provider_in_config(self):
with (
patch(f"{MODULE}.run_browsecomp_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_browsecomp,
)
evaluate_browsecomp(
num_examples=1,
search_model="bc-model",
search_provider="bc-provider",
endpoint_url="http://bc-url.com",
)
sc = mock_run.call_args[1]["search_config"]
assert sc["model_name"] == "bc-model"
assert sc["provider"] == "bc-provider"
assert sc["openai_endpoint_url"] == "http://bc-url.com"
def test_evaluation_openai_endpoint_with_explicit_url(self):
with (
patch(f"{MODULE}.run_browsecomp_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_browsecomp,
)
evaluate_browsecomp(
num_examples=1,
evaluation_model="eval-m",
evaluation_provider="openai_endpoint",
endpoint_url="http://bc-eval-url.com",
)
ec = mock_run.call_args[1]["evaluation_config"]
assert ec["temperature"] == 0
assert ec["openai_endpoint_url"] == "http://bc-eval-url.com"
def test_evaluation_openai_endpoint_env_fallback(self):
with (
patch(f"{MODULE}.run_browsecomp_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot",
side_effect=_settings_with_url,
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_browsecomp,
)
evaluate_browsecomp(
num_examples=1,
evaluation_provider="openai_endpoint",
)
ec = mock_run.call_args[1]["evaluation_config"]
assert (
ec["openai_endpoint_url"] == "http://env-endpoint.example.com"
)
class TestXbenchDeepsearchDefaults:
def test_default_iterations_is_4(self):
with (
patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_xbench_deepsearch,
)
evaluate_xbench_deepsearch(num_examples=1)
sc = mock_run.call_args[1]["search_config"]
assert sc["iterations"] == 4
def test_search_model_and_endpoint_in_config(self):
with (
patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot", side_effect=_no_settings
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_xbench_deepsearch,
)
evaluate_xbench_deepsearch(
num_examples=1,
search_model="xb-model",
search_provider="xb-prov",
endpoint_url="http://xb-url.com",
)
sc = mock_run.call_args[1]["search_config"]
assert sc["model_name"] == "xb-model"
assert sc["provider"] == "xb-prov"
assert sc["openai_endpoint_url"] == "http://xb-url.com"
def test_evaluation_openai_endpoint_env_fallback(self):
with (
patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot",
side_effect=_settings_with_url,
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_xbench_deepsearch,
)
evaluate_xbench_deepsearch(
num_examples=1,
evaluation_model="xb-eval",
evaluation_provider="openai_endpoint",
)
ec = mock_run.call_args[1]["evaluation_config"]
assert ec["temperature"] == 0
assert (
ec["openai_endpoint_url"] == "http://env-endpoint.example.com"
)
def test_settings_override_search_config(self):
with (
patch(f"{MODULE}.run_xbench_deepsearch_benchmark") as mock_run,
patch(
f"{MODULE}.get_setting_from_snapshot",
side_effect=_full_settings,
),
):
mock_run.return_value = {}
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_xbench_deepsearch,
)
evaluate_xbench_deepsearch(
num_examples=1,
search_model="param-m",
endpoint_url="http://param-url.com",
)
sc = mock_run.call_args[1]["search_config"]
assert sc["model_name"] == "env-model"
assert sc["provider"] == "env-provider"
assert sc["openai_endpoint_url"] == "http://env-url.example.com"