~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""The benchmark grader only reuses the research endpoint key on the same origin.
The grader defaults to ``openai_endpoint`` at ``benchmark.evaluation.endpoint_url``
(OpenRouter). Its key comes from ``benchmark.evaluation.api_key``; the
research key ``llm.openai_endpoint.api_key`` is reused only when the grader
endpoint has the same origin as ``llm.openai_endpoint.url``.
The decision tests patch ``graders.get_llm`` and inspect what it receives,
so no client is built and no network is touched. The integration test runs
the real ``get_llm`` with the langchain client class patched, on loopback
URLs only.
"""
from unittest.mock import MagicMock, patch
import pytest
from langchain_core.language_models.fake_chat_models import FakeListChatModel
from local_deep_research.benchmarks import graders
from local_deep_research.config.llm_config import LLMNotConfiguredError
from local_deep_research.config.openai_endpoint_binding import (
resolve_openai_endpoint_override_key,
)
from local_deep_research.config.thread_settings import settings_context
from local_deep_research.settings.manager import SnapshotSettingsContext
RESEARCH_KEY = "sk-research-gateway-key"
GRADER_KEY = "sk-grader-key"
GATEWAY_URL = "https://llm-gateway.corp.example/v1"
OPENROUTER_URL = "https://openrouter.ai/api/v1"
def _snapshot(research_url=GATEWAY_URL, grader_key=None, grader_url=None):
snap = {
"llm.openai_endpoint.url": {"value": research_url},
"llm.openai_endpoint.api_key": {
"value": RESEARCH_KEY,
"ui_element": "password",
},
}
if grader_url is not None:
snap["benchmark.evaluation.endpoint_url"] = {"value": grader_url}
if grader_key is not None:
snap["benchmark.evaluation.api_key"] = {
"value": grader_key,
"ui_element": "password",
}
return snap
class TestGraderKeyResolution:
def test_default_grader_on_another_origin_does_not_get_research_key(self):
with patch.object(graders, "get_llm") as mock_get_llm:
with pytest.raises(LLMNotConfiguredError) as exc:
graders.get_evaluation_llm(settings_snapshot=_snapshot())
mock_get_llm.assert_not_called()
message = str(exc.value)
assert "benchmark.evaluation.api_key" in message
assert RESEARCH_KEY not in message
def test_same_origin_grader_reuses_research_key(self):
with patch.object(graders, "get_llm") as mock_get_llm:
mock_get_llm.return_value = MagicMock()
graders.get_evaluation_llm(
# Different path and host case, same origin.
custom_config={
"openai_endpoint_url": "https://LLM-Gateway.corp.example/v1/"
},
settings_snapshot=_snapshot(),
)
kwargs = mock_get_llm.call_args.kwargs
assert kwargs["openai_endpoint_api_key"] == RESEARCH_KEY
def test_grader_key_setting_is_used_for_another_origin(self):
with patch.object(graders, "get_llm") as mock_get_llm:
mock_get_llm.return_value = MagicMock()
graders.get_evaluation_llm(
settings_snapshot=_snapshot(grader_key=GRADER_KEY)
)
kwargs = mock_get_llm.call_args.kwargs
assert kwargs["openai_endpoint_url"] == OPENROUTER_URL
assert kwargs["openai_endpoint_api_key"] == GRADER_KEY
assert RESEARCH_KEY not in repr(
{k: v for k, v in kwargs.items() if k != "settings_snapshot"}
)
def test_grader_key_setting_wins_on_the_same_origin(self):
with patch.object(graders, "get_llm") as mock_get_llm:
mock_get_llm.return_value = MagicMock()
graders.get_evaluation_llm(
settings_snapshot=_snapshot(
research_url=OPENROUTER_URL, grader_key=GRADER_KEY
)
)
assert mock_get_llm.call_args.kwargs["openai_endpoint_api_key"] == (
GRADER_KEY
)
def test_custom_config_api_key_is_forwarded_as_endpoint_key(self):
with patch.object(graders, "get_llm") as mock_get_llm:
mock_get_llm.return_value = MagicMock()
graders.get_evaluation_llm(
custom_config={"api_key": GRADER_KEY},
settings_snapshot=_snapshot(),
)
kwargs = mock_get_llm.call_args.kwargs
assert "api_key" not in kwargs
assert kwargs["openai_endpoint_api_key"] == GRADER_KEY
def test_no_research_key_and_no_grader_key_proceeds_without_a_key(self):
snapshot = _snapshot()
snapshot["llm.openai_endpoint.api_key"] = {"value": ""}
with patch.object(graders, "get_llm") as mock_get_llm:
mock_get_llm.return_value = MagicMock()
graders.get_evaluation_llm(settings_snapshot=snapshot)
assert mock_get_llm.call_args.kwargs["openai_endpoint_api_key"] == ""
def test_non_endpoint_grader_is_unaffected(self):
with patch.object(graders, "get_llm") as mock_get_llm:
mock_get_llm.return_value = MagicMock()
graders.get_evaluation_llm(
custom_config={"provider": "openai", "model_name": "m"},
settings_snapshot=_snapshot(),
)
assert "openai_endpoint_api_key" not in mock_get_llm.call_args.kwargs
class TestGraderWithAnEnvironmentResearchKey:
"""Grader With An Environment Research Key."""
ENV_KEY = "sk-operator-env-key"
@pytest.fixture(autouse=True)
def _env(self, monkeypatch):
monkeypatch.setenv("LDR_LLM_OPENAI_ENDPOINT_API_KEY", self.ENV_KEY)
monkeypatch.delenv("LDR_LLM_OPENAI_ENDPOINT_URL", raising=False)
monkeypatch.delenv("LDR_BENCHMARK_EVALUATION_API_KEY", raising=False)
def test_default_grader_does_not_get_the_env_key(self):
with patch.object(graders, "get_llm") as mock_get_llm:
with pytest.raises(LLMNotConfiguredError) as exc:
graders.get_evaluation_llm(settings_snapshot=_snapshot())
mock_get_llm.assert_not_called()
assert self.ENV_KEY not in str(exc.value)
def test_default_grader_with_an_empty_saved_url_raises(self):
"""An empty saved URL means api.openai.com, not the OpenRouter grader."""
with patch.object(graders, "get_llm") as mock_get_llm:
with pytest.raises(LLMNotConfiguredError):
graders.get_evaluation_llm(
settings_snapshot=_snapshot(research_url="")
)
mock_get_llm.assert_not_called()
def test_grader_on_the_saved_url_gets_the_env_key(self):
with patch.object(graders, "get_llm") as mock_get_llm:
mock_get_llm.return_value = MagicMock()
graders.get_evaluation_llm(
settings_snapshot=_snapshot(grader_url=GATEWAY_URL)
)
kwargs = mock_get_llm.call_args.kwargs
assert kwargs["openai_endpoint_api_key"] == self.ENV_KEY
class TestGraderKeyOnTheThreadContextPath:
"""The documented Python-API shape: ``settings_snapshot=None``, thread context.
``evaluate_simpleqa`` / ``compare_configurations`` / ``SimpleQAEvaluator``
reach ``resolve_grader_endpoint_api_key`` with ``settings_snapshot=None``
while a thread settings context is bound. The key resolves through that
context (``get_setting_from_snapshot``'s fallback), so the key's owner —
the configured ``llm.openai_endpoint.url`` — must resolve through the
SAME source. Otherwise the key is withheld from its own endpoint
(falsifying the error message's remedy: "point the grader at the
research endpoint (llm.openai_endpoint.url)") and, worse, the provider
default origin is treated as the owner and DOES receive it.
"""
@pytest.fixture(autouse=True)
def _ctx(self, monkeypatch):
monkeypatch.delenv("LDR_LLM_OPENAI_ENDPOINT_URL", raising=False)
monkeypatch.delenv("LDR_LLM_OPENAI_ENDPOINT_API_KEY", raising=False)
ctx = SnapshotSettingsContext(
{
"llm.openai_endpoint.url": {
"value": GATEWAY_URL,
"ui_element": "text",
},
"llm.openai_endpoint.api_key": {
"value": RESEARCH_KEY,
"ui_element": "password",
},
"benchmark.evaluation.endpoint_url": {
"value": GATEWAY_URL,
"ui_element": "text",
},
}
)
with settings_context(ctx):
yield
def test_grader_on_the_research_endpoint_gets_the_research_key(self):
with patch.object(graders, "get_llm") as mock_get_llm:
mock_get_llm.return_value = MagicMock()
graders.get_evaluation_llm(settings_snapshot=None)
kwargs = mock_get_llm.call_args.kwargs
assert kwargs["openai_endpoint_url"] == GATEWAY_URL
assert kwargs["openai_endpoint_api_key"] == RESEARCH_KEY
def test_grader_config_pointed_at_the_research_endpoint_gets_the_key(self):
"""The remedy in GRADER_KEY_ERROR_MESSAGE, followed literally."""
with patch.object(graders, "get_llm") as mock_get_llm:
mock_get_llm.return_value = MagicMock()
graders.get_evaluation_llm(
custom_config={"openai_endpoint_url": GATEWAY_URL},
settings_snapshot=None,
)
assert mock_get_llm.call_args.kwargs["openai_endpoint_api_key"] == (
RESEARCH_KEY
)
def test_key_is_withheld_from_foreign_and_default_origins(self):
"""A foreign origin — including the provider default — gets no key.
The provider default (https://api.openai.com/v1) is not the key's
owner just because it is where the provider goes unconfigured: a
default is reached only when it is the saved URL. Here the saved
URL is the gateway, so the default must be refused like any other
foreign origin.
"""
for grader_url in (
"https://api.openai.com/v1",
OPENROUTER_URL,
):
assert resolve_openai_endpoint_override_key(grader_url, None) == (
"",
True,
)
with patch.object(graders, "get_llm") as mock_get_llm:
with pytest.raises(LLMNotConfiguredError) as exc:
graders.get_evaluation_llm(
custom_config={"openai_endpoint_url": grader_url},
settings_snapshot=None,
)
mock_get_llm.assert_not_called()
assert str(exc.value) == graders.GRADER_KEY_ERROR_MESSAGE
assert RESEARCH_KEY not in str(exc.value)
class TestGraderKeyBoundToGraderEndpoint:
"""benchmark.evaluation.api_key only goes to benchmark.evaluation.endpoint_url."""
JUDGE_URL = "https://judge.example/v1"
def test_grader_key_is_not_sent_to_another_config_url(self):
with patch.object(graders, "get_llm") as mock_get_llm:
with pytest.raises(LLMNotConfiguredError):
graders.get_evaluation_llm(
custom_config={
"openai_endpoint_url": "https://elsewhere.example/v1"
},
settings_snapshot=_snapshot(
grader_key=GRADER_KEY, grader_url=self.JUDGE_URL
),
)
mock_get_llm.assert_not_called()
@pytest.mark.parametrize(
"custom_config",
[
{"provider": "openai_endpoint"}, # request-supplied config
{"openai_endpoint_url": None},
None,
],
ids=["request-config", "explicit-none-url", "no-config"],
)
def test_config_without_url_uses_the_configured_grader_endpoint(
self, custom_config
):
with patch.object(graders, "get_llm") as mock_get_llm:
mock_get_llm.return_value = MagicMock()
graders.get_evaluation_llm(
custom_config=custom_config,
settings_snapshot=_snapshot(
grader_key=GRADER_KEY, grader_url=self.JUDGE_URL
),
)
kwargs = mock_get_llm.call_args.kwargs
assert kwargs["openai_endpoint_url"] == self.JUDGE_URL
assert kwargs["openai_endpoint_api_key"] == GRADER_KEY
def test_preflight_raises_the_fixed_message_without_building_a_client(
self,
):
with patch.object(graders, "get_llm") as mock_get_llm:
with pytest.raises(LLMNotConfiguredError) as exc:
graders.check_grader_endpoint_key(
{"provider": "openai_endpoint"}, _snapshot()
)
mock_get_llm.assert_not_called()
assert str(exc.value) == graders.GRADER_KEY_ERROR_MESSAGE
assert "different origin (scheme, host or port)" in str(exc.value)
def test_preflight_passes_when_the_grader_key_matches(self):
graders.check_grader_endpoint_key(
None, _snapshot(grader_key=GRADER_KEY)
)
class TestGraderKeyReachesClient:
"""End to end through the real get_llm, loopback URLs only."""
@pytest.fixture(autouse=True)
def _providers(self):
from local_deep_research.llm.providers import discover_providers
discover_providers(force_refresh=True)
def _build(self, snapshot, grader_url):
snapshot = {
"llm.supports_max_tokens": True,
"llm.max_tokens": 4096,
"rate_limiting.llm_enabled": False,
"search.tool": "searxng",
**snapshot,
}
with patch(
"local_deep_research.llm.providers.openai_base.ChatOpenAI",
return_value=FakeListChatModel(responses=["ok"]),
) as mock_chat:
graders.get_evaluation_llm(
custom_config={
"model_name": "judge-model",
"openai_endpoint_url": grader_url,
},
settings_snapshot=snapshot,
)
return mock_chat.call_args.kwargs
def test_grader_key_reaches_client_for_another_origin(self):
kwargs = self._build(
_snapshot(
research_url="http://localhost:9000/v1",
grader_key=GRADER_KEY,
grader_url="http://localhost:8000/v1",
),
"http://localhost:8000/v1",
)
assert kwargs["base_url"] == "http://localhost:8000/v1"
assert kwargs["api_key"] == GRADER_KEY
def test_research_key_reaches_client_for_the_same_origin(self):
kwargs = self._build(
_snapshot(research_url="http://localhost:9000/v1"),
"http://localhost:9000/other/",
)
assert kwargs["api_key"] == RESEARCH_KEY
class TestKeylessLocalGrader:
def test_local_grader_without_a_key_runs_keyless(self):
"""A local grader (e.g. Ollama) needs no key; the research key on
another origin is withheld but grading is not refused."""
with patch.object(graders, "get_llm") as mock_get_llm:
mock_get_llm.return_value = MagicMock()
graders.get_evaluation_llm(
custom_config={
"openai_endpoint_url": "http://localhost:11434/v1"
},
settings_snapshot=_snapshot(),
)
kwargs = mock_get_llm.call_args.kwargs
assert kwargs["openai_endpoint_api_key"] == ""
assert RESEARCH_KEY not in repr(
{k: v for k, v in kwargs.items() if k != "settings_snapshot"}
)
@pytest.mark.parametrize(
"url", ["http://192.168.1.20:8000/v1", "http://gpu-box.local:8000/v1"]
)
def test_private_address_grader_runs_keyless(self, url):
with patch.object(graders, "get_llm") as mock_get_llm:
mock_get_llm.return_value = MagicMock()
graders.get_evaluation_llm(
custom_config={"openai_endpoint_url": url},
settings_snapshot=_snapshot(),
)
assert mock_get_llm.call_args.kwargs["openai_endpoint_api_key"] == ""
def test_error_message_explains_keyless_graders(self):
assert "localhost" in graders.GRADER_KEY_ERROR_MESSAGE
assert "placeholder" in graders.GRADER_KEY_ERROR_MESSAGE
class TestRunnerPrecheck:
@pytest.fixture(autouse=True)
def _pin_data_directory(self, tmp_path, monkeypatch):
"""Keep benchmark output out of the real user data directory.
run_benchmark resolves its output directory inside LDR_DATA_DIR and
creates it before loading the dataset.
"""
monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
def test_run_benchmark_refuses_before_loading_the_dataset(self, tmp_path):
from local_deep_research.benchmarks import runners
with (
patch.object(runners, "DatasetRegistry") as registry,
patch.object(runners, "load_dataset") as legacy_loader,
patch.object(runners, "quick_summary") as research,
):
with pytest.raises(LLMNotConfiguredError):
runners.run_benchmark(
"simpleqa",
num_examples=1,
settings_snapshot=_snapshot(),
output_dir=str(tmp_path),
)
registry.create_dataset.assert_not_called()
legacy_loader.assert_not_called()
research.assert_not_called()
def test_run_benchmark_without_evaluation_skips_the_check(self, tmp_path):
from local_deep_research.benchmarks import runners
with (
patch.object(runners, "check_grader_endpoint_key") as check,
patch.object(
runners.DatasetRegistry,
"create_dataset",
side_effect=RuntimeError("stop"),
),
patch.object(
runners, "load_dataset", side_effect=RuntimeError("stop")
),
):
with pytest.raises(RuntimeError, match="stop"):
runners.run_benchmark(
"simpleqa",
num_examples=1,
run_evaluation=False,
output_dir=str(tmp_path),
)
check.assert_not_called()
def test_simpleqa_evaluator_checks_before_research(self, tmp_path):
from local_deep_research.benchmarks.evaluators import simpleqa
with (
patch.object(
graders,
"check_grader_endpoint_key",
side_effect=LLMNotConfiguredError("no key"),
),
patch.object(simpleqa, "DatasetRegistry") as registry,
):
with pytest.raises(LLMNotConfiguredError):
simpleqa.SimpleQAEvaluator()._run_with_dataset_class(
system_config={}, num_examples=1, output_dir=str(tmp_path)
)
registry.create_dataset.assert_not_called()
class TestBenchmarkFunctionsGraderKey:
@pytest.mark.parametrize(
("fn", "runner"),
[
("evaluate_simpleqa", "run_simpleqa_benchmark"),
("evaluate_browsecomp", "run_browsecomp_benchmark"),
("evaluate_xbench_deepsearch", "run_xbench_deepsearch_benchmark"),
],
)
def test_evaluation_api_key_reaches_the_grader_config(self, fn, runner):
from local_deep_research.benchmarks import benchmark_functions
with (
patch.object(benchmark_functions, runner) as run,
patch.object(
benchmark_functions,
"get_setting_from_snapshot",
return_value=None,
),
):
run.return_value = {}
getattr(benchmark_functions, fn)(
num_examples=1, evaluation_api_key=GRADER_KEY
)
assert run.call_args.kwargs["evaluation_config"]["api_key"] == (
GRADER_KEY
)
@pytest.mark.parametrize(
"provider", ['"openai_endpoint"', " 'OpenAI_Endpoint' ", "OPENAI_ENDPOINT"]
)
def test_quoted_provider_grades_against_the_grader_endpoint(provider):
"""get_llm strips quotes from the provider; the grader must normalise it
the same way, or a quoted provider skips the grader-endpoint fill and
get_llm grades against the built-in OpenRouter URL."""
grader_url = "http://localhost:11434/v1"
with patch.object(graders, "get_llm") as mock_get_llm:
mock_get_llm.return_value = MagicMock()
graders.get_evaluation_llm(
custom_config={"provider": provider},
settings_snapshot=_snapshot(grader_url=grader_url),
)
kwargs = mock_get_llm.call_args.kwargs
assert kwargs["openai_endpoint_url"] == grader_url
assert kwargs["provider"] == "openai_endpoint"
assert RESEARCH_KEY not in repr(
{k: v for k, v in kwargs.items() if k != "settings_snapshot"}
)
@pytest.mark.parametrize("provider", ['""', "''", "", " ", None])
def test_provider_that_normalises_to_nothing_uses_the_grader_endpoint(
provider,
):
"""An empty or quotes-only grader provider means the same as an unset
one: the openai_endpoint default at benchmark.evaluation.endpoint_url.
It must not keep the built-in OpenRouter URL or fall back to the
research provider."""
grader_url = "http://localhost:11434/v1"
with patch.object(graders, "get_llm") as mock_get_llm:
mock_get_llm.return_value = MagicMock()
graders.get_evaluation_llm(
custom_config={"provider": provider},
settings_snapshot=_snapshot(grader_url=grader_url),
)
kwargs = mock_get_llm.call_args.kwargs
assert kwargs["provider"] == "openai_endpoint"
assert kwargs["openai_endpoint_url"] == grader_url