Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

8.1 KB
Raw Blame
"""
Path separation for the benchmark graders.

``grade_single_result`` / ``grade_results`` are called from plain threads
(the benchmark daemon, the CLI). They must reach the grader LLM through
``invoke`` and must never create an event loop: langchain caches one async
HTTP client per process and that client stays bound to the loop that first
used it, so a throwaway ``asyncio.run`` loop leaves a closed-loop
keep-alive connection behind and the provider SDK silently re-sends the
request on the next call — every graded item billed twice.

``grade_single_result_async`` / ``grade_results_async`` are the mirror
image: they use ``ainvoke``.
"""

import asyncio
import json
from unittest.mock import AsyncMock, MagicMock, Mock, patch

import pytest

from local_deep_research.benchmarks.graders import (
    grade_results,
    grade_results_async,
    grade_single_result,
    grade_single_result_async,
)

RESULT = {
    "id": "1",
    "problem": "What is 2+2?",
    "correct_answer": "4",
    "response": "4",
}

SYNC_GRADE = "Extracted Answer: 4\nReasoning: sync\nCorrect: yes"
ASYNC_GRADE = "Extracted Answer: 4\nReasoning: async\nCorrect: yes"


def _dual_llm() -> MagicMock:
    """A grader LLM exposing both interfaces so either could be chosen."""
    llm = MagicMock()
    del llm.chat_messages  # take the plain-prompt sub-branch
    llm.invoke.return_value = Mock(content=SYNC_GRADE)
    llm.ainvoke = AsyncMock(return_value=Mock(content=ASYNC_GRADE))
    return llm


class _EventLoopCreated(AssertionError):
    """Raised when the code under test tries to start an event loop."""


def _no_event_loop():
    """Patchers that make any attempt to start an event loop fail loudly."""
    boom = Mock(
        side_effect=_EventLoopCreated(
            "the synchronous path must not create an event loop"
        )
    )
    return (
        patch("asyncio.run", boom),
        patch("asyncio.new_event_loop", boom),
    )


def _write_results(tmp_path):
    results_file = tmp_path / "results.jsonl"
    results_file.write_text(json.dumps(RESULT) + "\n", encoding="utf-8")
    return str(results_file), str(tmp_path / "graded.jsonl")


class TestGradeSingleResultSyncPath:
    """grade_single_result() stays on invoke and starts no event loop."""

    def test_uses_invoke_and_not_ainvoke(self):
        llm = _dual_llm()

        with patch(
            "local_deep_research.benchmarks.graders.get_evaluation_llm",
            return_value=llm,
        ):
            graded = grade_single_result(RESULT)

        assert graded["is_correct"] is True
        assert graded["reasoning"] == "sync"
        llm.invoke.assert_called_once()
        llm.ainvoke.assert_not_called()

    def test_starts_no_event_loop(self):
        # spec without "ainvoke": the sync path may not even look for it.
        llm = Mock(spec=["invoke", "close"])
        llm.invoke.return_value = Mock(content=SYNC_GRADE)

        run_patch, new_loop_patch = _no_event_loop()
        with (
            patch(
                "local_deep_research.benchmarks.graders.get_evaluation_llm",
                return_value=llm,
            ),
            run_patch,
            new_loop_patch,
        ):
            graded = grade_single_result(RESULT)

        assert graded["is_correct"] is True
        assert "grading_error" not in graded

    def test_chat_messages_sub_branch_without_ainvoke(self):
        """A sync-only chat model still takes the HumanMessage sub-branch."""
        from langchain_core.messages.human import HumanMessage

        llm = Mock(spec=["invoke", "chat_messages"])
        llm.invoke.return_value = Mock(content=SYNC_GRADE)

        with patch(
            "local_deep_research.benchmarks.graders.get_evaluation_llm",
            return_value=llm,
        ):
            graded = grade_single_result(RESULT)

        assert graded["is_correct"] is True
        (messages,), _ = llm.invoke.call_args
        assert isinstance(messages[0], HumanMessage)

    def test_event_loop_guard_is_armed(self):
        """The guard used above really does fire on asyncio.run()."""

        async def _noop():
            return None

        coro = _noop()
        run_patch, _ = _no_event_loop()
        try:
            with run_patch, pytest.raises(_EventLoopCreated):
                asyncio.run(coro)
        finally:
            coro.close()


class TestGradeSingleResultAsyncPath:
    """grade_single_result_async() stays on ainvoke."""

    def test_uses_ainvoke_and_not_invoke(self):
        llm = _dual_llm()

        with patch(
            "local_deep_research.benchmarks.graders.get_evaluation_llm",
            return_value=llm,
        ):
            graded = asyncio.run(grade_single_result_async(RESULT))

        assert graded["reasoning"] == "async"
        llm.ainvoke.assert_awaited_once()
        llm.invoke.assert_not_called()


class TestGradeResultsSyncPath:
    """grade_results() stays on invoke and starts no event loop."""

    def test_uses_invoke_and_not_ainvoke(self, tmp_path):
        llm = _dual_llm()
        results_file, output_file = _write_results(tmp_path)

        with patch(
            "local_deep_research.benchmarks.graders.get_evaluation_llm",
            return_value=llm,
        ):
            graded = grade_results(results_file, output_file)

        assert [g["reasoning"] for g in graded] == ["sync"]
        llm.invoke.assert_called_once()
        llm.ainvoke.assert_not_called()

    def test_starts_no_event_loop(self, tmp_path):
        llm = Mock(spec=["invoke", "close"])
        llm.invoke.return_value = Mock(content=SYNC_GRADE)
        results_file, output_file = _write_results(tmp_path)

        run_patch, new_loop_patch = _no_event_loop()
        with (
            patch(
                "local_deep_research.benchmarks.graders.get_evaluation_llm",
                return_value=llm,
            ),
            run_patch,
            new_loop_patch,
        ):
            graded = grade_results(results_file, output_file)

        assert len(graded) == 1
        assert "grading_error" not in graded[0]


class TestGradeResultsAsyncPath:
    """grade_results_async() stays on ainvoke."""

    def test_uses_ainvoke_and_not_invoke(self, tmp_path):
        llm = _dual_llm()
        results_file, output_file = _write_results(tmp_path)

        with patch(
            "local_deep_research.benchmarks.graders.get_evaluation_llm",
            return_value=llm,
        ):
            graded = asyncio.run(grade_results_async(results_file, output_file))

        assert [g["reasoning"] for g in graded] == ["async"]
        llm.ainvoke.assert_awaited_once()
        llm.invoke.assert_not_called()


class TestGradingErrorMarkerUnchanged:
    """grade_single_result keeps its documented failure payload."""

    def test_failure_payload(self):
        llm = Mock(spec=["invoke", "close"])
        llm.invoke.side_effect = RuntimeError("provider down")

        with patch(
            "local_deep_research.benchmarks.graders.get_evaluation_llm",
            return_value=llm,
        ):
            graded = grade_single_result(RESULT)

        assert graded["is_correct"] is False
        assert graded["grading_error"] == "provider down"
        assert graded["graded_confidence"] == "0"


@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("chat_messages", [False, True])
def test_structured_grade_uses_text_blocks(asynchronous, chat_messages):
    llm = _dual_llm()
    if chat_messages:
        llm.chat_messages = []
    content = [
        {"type": "thinking", "thinking": "Correct: no"},
        {"type": "text", "text": "Extracted Answer: 4\nReasoning: matches\n"},
        {"type": "text", "text": "Correct: yes"},
    ]
    llm.invoke.return_value = Mock(content=content)
    llm.ainvoke.return_value = Mock(content=content)
    with patch(
        "local_deep_research.benchmarks.graders.get_evaluation_llm",
        return_value=llm,
    ):
        graded = (
            asyncio.run(grade_single_result_async(RESULT))
            if asynchronous
            else grade_single_result(RESULT)
        )
    assert graded["is_correct"] is True
    assert graded["extracted_by_grader"] == "4"
    assert graded["reasoning"] == "matches"
    assert "grading_error" not in graded
    assert "Correct: no" not in graded["grader_response"]