Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

21.4 KB
Raw Blame
"""The benchmark grader only reuses the research endpoint key on the same origin.

The grader defaults to ``openai_endpoint`` at ``benchmark.evaluation.endpoint_url``
(OpenRouter). Its key comes from ``benchmark.evaluation.api_key``; the
research key ``llm.openai_endpoint.api_key`` is reused only when the grader
endpoint has the same origin as ``llm.openai_endpoint.url``.

The decision tests patch ``graders.get_llm`` and inspect what it receives,
so no client is built and no network is touched. The integration test runs
the real ``get_llm`` with the langchain client class patched, on loopback
URLs only.
"""

from unittest.mock import MagicMock, patch

import pytest
from langchain_core.language_models.fake_chat_models import FakeListChatModel

from local_deep_research.benchmarks import graders
from local_deep_research.config.llm_config import LLMNotConfiguredError
from local_deep_research.config.openai_endpoint_binding import (
    resolve_openai_endpoint_override_key,
)
from local_deep_research.config.thread_settings import settings_context
from local_deep_research.settings.manager import SnapshotSettingsContext

RESEARCH_KEY = "sk-research-gateway-key"
GRADER_KEY = "sk-grader-key"
GATEWAY_URL = "https://llm-gateway.corp.example/v1"
OPENROUTER_URL = "https://openrouter.ai/api/v1"


def _snapshot(research_url=GATEWAY_URL, grader_key=None, grader_url=None):
    snap = {
        "llm.openai_endpoint.url": {"value": research_url},
        "llm.openai_endpoint.api_key": {
            "value": RESEARCH_KEY,
            "ui_element": "password",
        },
    }
    if grader_url is not None:
        snap["benchmark.evaluation.endpoint_url"] = {"value": grader_url}
    if grader_key is not None:
        snap["benchmark.evaluation.api_key"] = {
            "value": grader_key,
            "ui_element": "password",
        }
    return snap


class TestGraderKeyResolution:
    def test_default_grader_on_another_origin_does_not_get_research_key(self):
        with patch.object(graders, "get_llm") as mock_get_llm:
            with pytest.raises(LLMNotConfiguredError) as exc:
                graders.get_evaluation_llm(settings_snapshot=_snapshot())

        mock_get_llm.assert_not_called()
        message = str(exc.value)
        assert "benchmark.evaluation.api_key" in message
        assert RESEARCH_KEY not in message

    def test_same_origin_grader_reuses_research_key(self):
        with patch.object(graders, "get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            graders.get_evaluation_llm(
                # Different path and host case, same origin.
                custom_config={
                    "openai_endpoint_url": "https://LLM-Gateway.corp.example/v1/"
                },
                settings_snapshot=_snapshot(),
            )

        kwargs = mock_get_llm.call_args.kwargs
        assert kwargs["openai_endpoint_api_key"] == RESEARCH_KEY

    def test_grader_key_setting_is_used_for_another_origin(self):
        with patch.object(graders, "get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            graders.get_evaluation_llm(
                settings_snapshot=_snapshot(grader_key=GRADER_KEY)
            )

        kwargs = mock_get_llm.call_args.kwargs
        assert kwargs["openai_endpoint_url"] == OPENROUTER_URL
        assert kwargs["openai_endpoint_api_key"] == GRADER_KEY
        assert RESEARCH_KEY not in repr(
            {k: v for k, v in kwargs.items() if k != "settings_snapshot"}
        )

    def test_grader_key_setting_wins_on_the_same_origin(self):
        with patch.object(graders, "get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            graders.get_evaluation_llm(
                settings_snapshot=_snapshot(
                    research_url=OPENROUTER_URL, grader_key=GRADER_KEY
                )
            )

        assert mock_get_llm.call_args.kwargs["openai_endpoint_api_key"] == (
            GRADER_KEY
        )

    def test_custom_config_api_key_is_forwarded_as_endpoint_key(self):
        with patch.object(graders, "get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            graders.get_evaluation_llm(
                custom_config={"api_key": GRADER_KEY},
                settings_snapshot=_snapshot(),
            )

        kwargs = mock_get_llm.call_args.kwargs
        assert "api_key" not in kwargs
        assert kwargs["openai_endpoint_api_key"] == GRADER_KEY

    def test_no_research_key_and_no_grader_key_proceeds_without_a_key(self):
        snapshot = _snapshot()
        snapshot["llm.openai_endpoint.api_key"] = {"value": ""}
        with patch.object(graders, "get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            graders.get_evaluation_llm(settings_snapshot=snapshot)

        assert mock_get_llm.call_args.kwargs["openai_endpoint_api_key"] == ""

    def test_non_endpoint_grader_is_unaffected(self):
        with patch.object(graders, "get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            graders.get_evaluation_llm(
                custom_config={"provider": "openai", "model_name": "m"},
                settings_snapshot=_snapshot(),
            )

        assert "openai_endpoint_api_key" not in mock_get_llm.call_args.kwargs


class TestGraderWithAnEnvironmentResearchKey:
    """Grader With An Environment Research Key."""

    ENV_KEY = "sk-operator-env-key"

    @pytest.fixture(autouse=True)
    def _env(self, monkeypatch):
        monkeypatch.setenv("LDR_LLM_OPENAI_ENDPOINT_API_KEY", self.ENV_KEY)
        monkeypatch.delenv("LDR_LLM_OPENAI_ENDPOINT_URL", raising=False)
        monkeypatch.delenv("LDR_BENCHMARK_EVALUATION_API_KEY", raising=False)

    def test_default_grader_does_not_get_the_env_key(self):
        with patch.object(graders, "get_llm") as mock_get_llm:
            with pytest.raises(LLMNotConfiguredError) as exc:
                graders.get_evaluation_llm(settings_snapshot=_snapshot())

        mock_get_llm.assert_not_called()
        assert self.ENV_KEY not in str(exc.value)

    def test_default_grader_with_an_empty_saved_url_raises(self):
        """An empty saved URL means api.openai.com, not the OpenRouter grader."""
        with patch.object(graders, "get_llm") as mock_get_llm:
            with pytest.raises(LLMNotConfiguredError):
                graders.get_evaluation_llm(
                    settings_snapshot=_snapshot(research_url="")
                )

        mock_get_llm.assert_not_called()

    def test_grader_on_the_saved_url_gets_the_env_key(self):
        with patch.object(graders, "get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            graders.get_evaluation_llm(
                settings_snapshot=_snapshot(grader_url=GATEWAY_URL)
            )

        kwargs = mock_get_llm.call_args.kwargs
        assert kwargs["openai_endpoint_api_key"] == self.ENV_KEY


class TestGraderKeyOnTheThreadContextPath:
    """The documented Python-API shape: ``settings_snapshot=None``, thread context.

    ``evaluate_simpleqa`` / ``compare_configurations`` / ``SimpleQAEvaluator``
    reach ``resolve_grader_endpoint_api_key`` with ``settings_snapshot=None``
    while a thread settings context is bound. The key resolves through that
    context (``get_setting_from_snapshot``'s fallback), so the key's owner —
    the configured ``llm.openai_endpoint.url`` — must resolve through the
    SAME source. Otherwise the key is withheld from its own endpoint
    (falsifying the error message's remedy: "point the grader at the
    research endpoint (llm.openai_endpoint.url)") and, worse, the provider
    default origin is treated as the owner and DOES receive it.
    """

    @pytest.fixture(autouse=True)
    def _ctx(self, monkeypatch):
        monkeypatch.delenv("LDR_LLM_OPENAI_ENDPOINT_URL", raising=False)
        monkeypatch.delenv("LDR_LLM_OPENAI_ENDPOINT_API_KEY", raising=False)
        ctx = SnapshotSettingsContext(
            {
                "llm.openai_endpoint.url": {
                    "value": GATEWAY_URL,
                    "ui_element": "text",
                },
                "llm.openai_endpoint.api_key": {
                    "value": RESEARCH_KEY,
                    "ui_element": "password",
                },
                "benchmark.evaluation.endpoint_url": {
                    "value": GATEWAY_URL,
                    "ui_element": "text",
                },
            }
        )
        with settings_context(ctx):
            yield

    def test_grader_on_the_research_endpoint_gets_the_research_key(self):
        with patch.object(graders, "get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            graders.get_evaluation_llm(settings_snapshot=None)

        kwargs = mock_get_llm.call_args.kwargs
        assert kwargs["openai_endpoint_url"] == GATEWAY_URL
        assert kwargs["openai_endpoint_api_key"] == RESEARCH_KEY

    def test_grader_config_pointed_at_the_research_endpoint_gets_the_key(self):
        """The remedy in GRADER_KEY_ERROR_MESSAGE, followed literally."""
        with patch.object(graders, "get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            graders.get_evaluation_llm(
                custom_config={"openai_endpoint_url": GATEWAY_URL},
                settings_snapshot=None,
            )

        assert mock_get_llm.call_args.kwargs["openai_endpoint_api_key"] == (
            RESEARCH_KEY
        )

    def test_key_is_withheld_from_foreign_and_default_origins(self):
        """A foreign origin — including the provider default — gets no key.

        The provider default (https://api.openai.com/v1) is not the key's
        owner just because it is where the provider goes unconfigured: a
        default is reached only when it is the saved URL. Here the saved
        URL is the gateway, so the default must be refused like any other
        foreign origin.
        """
        for grader_url in (
            "https://api.openai.com/v1",
            OPENROUTER_URL,
        ):
            assert resolve_openai_endpoint_override_key(grader_url, None) == (
                "",
                True,
            )
            with patch.object(graders, "get_llm") as mock_get_llm:
                with pytest.raises(LLMNotConfiguredError) as exc:
                    graders.get_evaluation_llm(
                        custom_config={"openai_endpoint_url": grader_url},
                        settings_snapshot=None,
                    )
            mock_get_llm.assert_not_called()
            assert str(exc.value) == graders.GRADER_KEY_ERROR_MESSAGE
            assert RESEARCH_KEY not in str(exc.value)


class TestGraderKeyBoundToGraderEndpoint:
    """benchmark.evaluation.api_key only goes to benchmark.evaluation.endpoint_url."""

    JUDGE_URL = "https://judge.example/v1"

    def test_grader_key_is_not_sent_to_another_config_url(self):
        with patch.object(graders, "get_llm") as mock_get_llm:
            with pytest.raises(LLMNotConfiguredError):
                graders.get_evaluation_llm(
                    custom_config={
                        "openai_endpoint_url": "https://elsewhere.example/v1"
                    },
                    settings_snapshot=_snapshot(
                        grader_key=GRADER_KEY, grader_url=self.JUDGE_URL
                    ),
                )
        mock_get_llm.assert_not_called()

    @pytest.mark.parametrize(
        "custom_config",
        [
            {"provider": "openai_endpoint"},  # request-supplied config
            {"openai_endpoint_url": None},
            None,
        ],
        ids=["request-config", "explicit-none-url", "no-config"],
    )
    def test_config_without_url_uses_the_configured_grader_endpoint(
        self, custom_config
    ):
        with patch.object(graders, "get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            graders.get_evaluation_llm(
                custom_config=custom_config,
                settings_snapshot=_snapshot(
                    grader_key=GRADER_KEY, grader_url=self.JUDGE_URL
                ),
            )

        kwargs = mock_get_llm.call_args.kwargs
        assert kwargs["openai_endpoint_url"] == self.JUDGE_URL
        assert kwargs["openai_endpoint_api_key"] == GRADER_KEY

    def test_preflight_raises_the_fixed_message_without_building_a_client(
        self,
    ):
        with patch.object(graders, "get_llm") as mock_get_llm:
            with pytest.raises(LLMNotConfiguredError) as exc:
                graders.check_grader_endpoint_key(
                    {"provider": "openai_endpoint"}, _snapshot()
                )
        mock_get_llm.assert_not_called()
        assert str(exc.value) == graders.GRADER_KEY_ERROR_MESSAGE
        assert "different origin (scheme, host or port)" in str(exc.value)

    def test_preflight_passes_when_the_grader_key_matches(self):
        graders.check_grader_endpoint_key(
            None, _snapshot(grader_key=GRADER_KEY)
        )


class TestGraderKeyReachesClient:
    """End to end through the real get_llm, loopback URLs only."""

    @pytest.fixture(autouse=True)
    def _providers(self):
        from local_deep_research.llm.providers import discover_providers

        discover_providers(force_refresh=True)

    def _build(self, snapshot, grader_url):
        snapshot = {
            "llm.supports_max_tokens": True,
            "llm.max_tokens": 4096,
            "rate_limiting.llm_enabled": False,
            "search.tool": "searxng",
            **snapshot,
        }
        with patch(
            "local_deep_research.llm.providers.openai_base.ChatOpenAI",
            return_value=FakeListChatModel(responses=["ok"]),
        ) as mock_chat:
            graders.get_evaluation_llm(
                custom_config={
                    "model_name": "judge-model",
                    "openai_endpoint_url": grader_url,
                },
                settings_snapshot=snapshot,
            )
        return mock_chat.call_args.kwargs

    def test_grader_key_reaches_client_for_another_origin(self):
        kwargs = self._build(
            _snapshot(
                research_url="http://localhost:9000/v1",
                grader_key=GRADER_KEY,
                grader_url="http://localhost:8000/v1",
            ),
            "http://localhost:8000/v1",
        )
        assert kwargs["base_url"] == "http://localhost:8000/v1"
        assert kwargs["api_key"] == GRADER_KEY

    def test_research_key_reaches_client_for_the_same_origin(self):
        kwargs = self._build(
            _snapshot(research_url="http://localhost:9000/v1"),
            "http://localhost:9000/other/",
        )
        assert kwargs["api_key"] == RESEARCH_KEY


class TestKeylessLocalGrader:
    def test_local_grader_without_a_key_runs_keyless(self):
        """A local grader (e.g. Ollama) needs no key; the research key on
        another origin is withheld but grading is not refused."""
        with patch.object(graders, "get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            graders.get_evaluation_llm(
                custom_config={
                    "openai_endpoint_url": "http://localhost:11434/v1"
                },
                settings_snapshot=_snapshot(),
            )

        kwargs = mock_get_llm.call_args.kwargs
        assert kwargs["openai_endpoint_api_key"] == ""
        assert RESEARCH_KEY not in repr(
            {k: v for k, v in kwargs.items() if k != "settings_snapshot"}
        )

    @pytest.mark.parametrize(
        "url", ["http://192.168.1.20:8000/v1", "http://gpu-box.local:8000/v1"]
    )
    def test_private_address_grader_runs_keyless(self, url):
        with patch.object(graders, "get_llm") as mock_get_llm:
            mock_get_llm.return_value = MagicMock()
            graders.get_evaluation_llm(
                custom_config={"openai_endpoint_url": url},
                settings_snapshot=_snapshot(),
            )
        assert mock_get_llm.call_args.kwargs["openai_endpoint_api_key"] == ""

    def test_error_message_explains_keyless_graders(self):
        assert "localhost" in graders.GRADER_KEY_ERROR_MESSAGE
        assert "placeholder" in graders.GRADER_KEY_ERROR_MESSAGE


class TestRunnerPrecheck:
    @pytest.fixture(autouse=True)
    def _pin_data_directory(self, tmp_path, monkeypatch):
        """Keep benchmark output out of the real user data directory.

        run_benchmark resolves its output directory inside LDR_DATA_DIR and
        creates it before loading the dataset.
        """
        monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))

    def test_run_benchmark_refuses_before_loading_the_dataset(self, tmp_path):
        from local_deep_research.benchmarks import runners

        with (
            patch.object(runners, "DatasetRegistry") as registry,
            patch.object(runners, "load_dataset") as legacy_loader,
            patch.object(runners, "quick_summary") as research,
        ):
            with pytest.raises(LLMNotConfiguredError):
                runners.run_benchmark(
                    "simpleqa",
                    num_examples=1,
                    settings_snapshot=_snapshot(),
                    output_dir=str(tmp_path),
                )
        registry.create_dataset.assert_not_called()
        legacy_loader.assert_not_called()
        research.assert_not_called()

    def test_run_benchmark_without_evaluation_skips_the_check(self, tmp_path):
        from local_deep_research.benchmarks import runners

        with (
            patch.object(runners, "check_grader_endpoint_key") as check,
            patch.object(
                runners.DatasetRegistry,
                "create_dataset",
                side_effect=RuntimeError("stop"),
            ),
            patch.object(
                runners, "load_dataset", side_effect=RuntimeError("stop")
            ),
        ):
            with pytest.raises(RuntimeError, match="stop"):
                runners.run_benchmark(
                    "simpleqa",
                    num_examples=1,
                    run_evaluation=False,
                    output_dir=str(tmp_path),
                )
        check.assert_not_called()

    def test_simpleqa_evaluator_checks_before_research(self, tmp_path):
        from local_deep_research.benchmarks.evaluators import simpleqa

        with (
            patch.object(
                graders,
                "check_grader_endpoint_key",
                side_effect=LLMNotConfiguredError("no key"),
            ),
            patch.object(simpleqa, "DatasetRegistry") as registry,
        ):
            with pytest.raises(LLMNotConfiguredError):
                simpleqa.SimpleQAEvaluator()._run_with_dataset_class(
                    system_config={}, num_examples=1, output_dir=str(tmp_path)
                )
        registry.create_dataset.assert_not_called()


class TestBenchmarkFunctionsGraderKey:
    @pytest.mark.parametrize(
        ("fn", "runner"),
        [
            ("evaluate_simpleqa", "run_simpleqa_benchmark"),
            ("evaluate_browsecomp", "run_browsecomp_benchmark"),
            ("evaluate_xbench_deepsearch", "run_xbench_deepsearch_benchmark"),
        ],
    )
    def test_evaluation_api_key_reaches_the_grader_config(self, fn, runner):
        from local_deep_research.benchmarks import benchmark_functions

        with (
            patch.object(benchmark_functions, runner) as run,
            patch.object(
                benchmark_functions,
                "get_setting_from_snapshot",
                return_value=None,
            ),
        ):
            run.return_value = {}
            getattr(benchmark_functions, fn)(
                num_examples=1, evaluation_api_key=GRADER_KEY
            )
        assert run.call_args.kwargs["evaluation_config"]["api_key"] == (
            GRADER_KEY
        )


@pytest.mark.parametrize(
    "provider", ['"openai_endpoint"', " 'OpenAI_Endpoint' ", "OPENAI_ENDPOINT"]
)
def test_quoted_provider_grades_against_the_grader_endpoint(provider):
    """get_llm strips quotes from the provider; the grader must normalise it
    the same way, or a quoted provider skips the grader-endpoint fill and
    get_llm grades against the built-in OpenRouter URL."""
    grader_url = "http://localhost:11434/v1"
    with patch.object(graders, "get_llm") as mock_get_llm:
        mock_get_llm.return_value = MagicMock()
        graders.get_evaluation_llm(
            custom_config={"provider": provider},
            settings_snapshot=_snapshot(grader_url=grader_url),
        )

    kwargs = mock_get_llm.call_args.kwargs
    assert kwargs["openai_endpoint_url"] == grader_url
    assert kwargs["provider"] == "openai_endpoint"
    assert RESEARCH_KEY not in repr(
        {k: v for k, v in kwargs.items() if k != "settings_snapshot"}
    )


@pytest.mark.parametrize("provider", ['""', "''", "", "   ", None])
def test_provider_that_normalises_to_nothing_uses_the_grader_endpoint(
    provider,
):
    """An empty or quotes-only grader provider means the same as an unset
    one: the openai_endpoint default at benchmark.evaluation.endpoint_url.
    It must not keep the built-in OpenRouter URL or fall back to the
    research provider."""
    grader_url = "http://localhost:11434/v1"
    with patch.object(graders, "get_llm") as mock_get_llm:
        mock_get_llm.return_value = MagicMock()
        graders.get_evaluation_llm(
            custom_config={"provider": provider},
            settings_snapshot=_snapshot(grader_url=grader_url),
        )

    kwargs = mock_get_llm.call_args.kwargs
    assert kwargs["provider"] == "openai_endpoint"
    assert kwargs["openai_endpoint_url"] == grader_url