~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Path separation for the benchmark graders.
``grade_single_result`` / ``grade_results`` are called from plain threads
(the benchmark daemon, the CLI). They must reach the grader LLM through
``invoke`` and must never create an event loop: langchain caches one async
HTTP client per process and that client stays bound to the loop that first
used it, so a throwaway ``asyncio.run`` loop leaves a closed-loop
keep-alive connection behind and the provider SDK silently re-sends the
request on the next call — every graded item billed twice.
``grade_single_result_async`` / ``grade_results_async`` are the mirror
image: they use ``ainvoke``.
"""
import asyncio
import json
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import pytest
from local_deep_research.benchmarks.graders import (
grade_results,
grade_results_async,
grade_single_result,
grade_single_result_async,
)
RESULT = {
"id": "1",
"problem": "What is 2+2?",
"correct_answer": "4",
"response": "4",
}
SYNC_GRADE = "Extracted Answer: 4\nReasoning: sync\nCorrect: yes"
ASYNC_GRADE = "Extracted Answer: 4\nReasoning: async\nCorrect: yes"
def _dual_llm() -> MagicMock:
"""A grader LLM exposing both interfaces so either could be chosen."""
llm = MagicMock()
del llm.chat_messages # take the plain-prompt sub-branch
llm.invoke.return_value = Mock(content=SYNC_GRADE)
llm.ainvoke = AsyncMock(return_value=Mock(content=ASYNC_GRADE))
return llm
class _EventLoopCreated(AssertionError):
"""Raised when the code under test tries to start an event loop."""
def _no_event_loop():
"""Patchers that make any attempt to start an event loop fail loudly."""
boom = Mock(
side_effect=_EventLoopCreated(
"the synchronous path must not create an event loop"
)
)
return (
patch("asyncio.run", boom),
patch("asyncio.new_event_loop", boom),
)
def _write_results(tmp_path):
results_file = tmp_path / "results.jsonl"
results_file.write_text(json.dumps(RESULT) + "\n", encoding="utf-8")
return str(results_file), str(tmp_path / "graded.jsonl")
class TestGradeSingleResultSyncPath:
"""grade_single_result() stays on invoke and starts no event loop."""
def test_uses_invoke_and_not_ainvoke(self):
llm = _dual_llm()
with patch(
"local_deep_research.benchmarks.graders.get_evaluation_llm",
return_value=llm,
):
graded = grade_single_result(RESULT)
assert graded["is_correct"] is True
assert graded["reasoning"] == "sync"
llm.invoke.assert_called_once()
llm.ainvoke.assert_not_called()
def test_starts_no_event_loop(self):
# spec without "ainvoke": the sync path may not even look for it.
llm = Mock(spec=["invoke", "close"])
llm.invoke.return_value = Mock(content=SYNC_GRADE)
run_patch, new_loop_patch = _no_event_loop()
with (
patch(
"local_deep_research.benchmarks.graders.get_evaluation_llm",
return_value=llm,
),
run_patch,
new_loop_patch,
):
graded = grade_single_result(RESULT)
assert graded["is_correct"] is True
assert "grading_error" not in graded
def test_chat_messages_sub_branch_without_ainvoke(self):
"""A sync-only chat model still takes the HumanMessage sub-branch."""
from langchain_core.messages.human import HumanMessage
llm = Mock(spec=["invoke", "chat_messages"])
llm.invoke.return_value = Mock(content=SYNC_GRADE)
with patch(
"local_deep_research.benchmarks.graders.get_evaluation_llm",
return_value=llm,
):
graded = grade_single_result(RESULT)
assert graded["is_correct"] is True
(messages,), _ = llm.invoke.call_args
assert isinstance(messages[0], HumanMessage)
def test_event_loop_guard_is_armed(self):
"""The guard used above really does fire on asyncio.run()."""
async def _noop():
return None
coro = _noop()
run_patch, _ = _no_event_loop()
try:
with run_patch, pytest.raises(_EventLoopCreated):
asyncio.run(coro)
finally:
coro.close()
class TestGradeSingleResultAsyncPath:
"""grade_single_result_async() stays on ainvoke."""
def test_uses_ainvoke_and_not_invoke(self):
llm = _dual_llm()
with patch(
"local_deep_research.benchmarks.graders.get_evaluation_llm",
return_value=llm,
):
graded = asyncio.run(grade_single_result_async(RESULT))
assert graded["reasoning"] == "async"
llm.ainvoke.assert_awaited_once()
llm.invoke.assert_not_called()
class TestGradeResultsSyncPath:
"""grade_results() stays on invoke and starts no event loop."""
def test_uses_invoke_and_not_ainvoke(self, tmp_path):
llm = _dual_llm()
results_file, output_file = _write_results(tmp_path)
with patch(
"local_deep_research.benchmarks.graders.get_evaluation_llm",
return_value=llm,
):
graded = grade_results(results_file, output_file)
assert [g["reasoning"] for g in graded] == ["sync"]
llm.invoke.assert_called_once()
llm.ainvoke.assert_not_called()
def test_starts_no_event_loop(self, tmp_path):
llm = Mock(spec=["invoke", "close"])
llm.invoke.return_value = Mock(content=SYNC_GRADE)
results_file, output_file = _write_results(tmp_path)
run_patch, new_loop_patch = _no_event_loop()
with (
patch(
"local_deep_research.benchmarks.graders.get_evaluation_llm",
return_value=llm,
),
run_patch,
new_loop_patch,
):
graded = grade_results(results_file, output_file)
assert len(graded) == 1
assert "grading_error" not in graded[0]
class TestGradeResultsAsyncPath:
"""grade_results_async() stays on ainvoke."""
def test_uses_ainvoke_and_not_invoke(self, tmp_path):
llm = _dual_llm()
results_file, output_file = _write_results(tmp_path)
with patch(
"local_deep_research.benchmarks.graders.get_evaluation_llm",
return_value=llm,
):
graded = asyncio.run(grade_results_async(results_file, output_file))
assert [g["reasoning"] for g in graded] == ["async"]
llm.ainvoke.assert_awaited_once()
llm.invoke.assert_not_called()
class TestGradingErrorMarkerUnchanged:
"""grade_single_result keeps its documented failure payload."""
def test_failure_payload(self):
llm = Mock(spec=["invoke", "close"])
llm.invoke.side_effect = RuntimeError("provider down")
with patch(
"local_deep_research.benchmarks.graders.get_evaluation_llm",
return_value=llm,
):
graded = grade_single_result(RESULT)
assert graded["is_correct"] is False
assert graded["grading_error"] == "provider down"
assert graded["graded_confidence"] == "0"
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("chat_messages", [False, True])
def test_structured_grade_uses_text_blocks(asynchronous, chat_messages):
llm = _dual_llm()
if chat_messages:
llm.chat_messages = []
content = [
{"type": "thinking", "thinking": "Correct: no"},
{"type": "text", "text": "Extracted Answer: 4\nReasoning: matches\n"},
{"type": "text", "text": "Correct: yes"},
]
llm.invoke.return_value = Mock(content=content)
llm.ainvoke.return_value = Mock(content=content)
with patch(
"local_deep_research.benchmarks.graders.get_evaluation_llm",
return_value=llm,
):
graded = (
asyncio.run(grade_single_result_async(RESULT))
if asynchronous
else grade_single_result(RESULT)
)
assert graded["is_correct"] is True
assert graded["extracted_by_grader"] == "4"
assert graded["reasoning"] == "matches"
assert "grading_error" not in graded
assert "Correct: no" not in graded["grader_response"]