~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Coverage tests for benchmarks/graders.py.
Targets the 37 missing lines:
- get_evaluation_llm() with/without custom_config, with/without settings_snapshot
- get_evaluation_llm() openai_endpoint branch with api_key from snapshot
- extract_answer_from_response() for browsecomp and simpleqa
- grade_single_result() simpleqa success path (invoke without chat_messages)
- grade_single_result() browsecomp success path
- grade_single_result() grading error path
- grade_results() basic flow with progress_callback
- human_evaluation() non-interactive mode
"""
import json
from unittest.mock import Mock, patch
MODULE = "local_deep_research.benchmarks.graders"
# ---------------------------------------------------------------------------
# get_evaluation_llm
# ---------------------------------------------------------------------------
class TestGetEvaluationLlm:
@patch(f"{MODULE}.get_llm")
def test_default_config_uses_claude_sonnet(self, mock_get_llm):
mock_get_llm.return_value = Mock()
from local_deep_research.benchmarks.graders import get_evaluation_llm
get_evaluation_llm()
call_kwargs = mock_get_llm.call_args[1]
assert call_kwargs["model_name"] == "anthropic/claude-3.7-sonnet"
assert call_kwargs["provider"] == "openai_endpoint"
assert call_kwargs["temperature"] == 0
@patch(f"{MODULE}.get_llm")
def test_custom_config_overrides_default(self, mock_get_llm):
mock_get_llm.return_value = Mock()
from local_deep_research.benchmarks.graders import get_evaluation_llm
get_evaluation_llm(
custom_config={"model_name": "gpt-4", "provider": "openai"}
)
call_kwargs = mock_get_llm.call_args[1]
assert call_kwargs["model_name"] == "gpt-4"
assert call_kwargs["provider"] == "openai"
@patch(f"{MODULE}.get_llm")
def test_settings_snapshot_dict_api_key_suppresses_warning(
self, mock_get_llm
):
mock_get_llm.return_value = Mock()
from local_deep_research.benchmarks.graders import get_evaluation_llm
snapshot = {
"llm.openai_endpoint.api_key": {"value": "sk-test-key"},
"llm.openai_endpoint.url": "https://openrouter.ai/api/v1",
}
with patch(f"{MODULE}.logger") as mock_logger:
get_evaluation_llm(settings_snapshot=snapshot)
# With a valid api_key from snapshot, no "no API key found" warning
no_key_warnings = [
c
for c in mock_logger.warning.call_args_list
if "no api key" in str(c).lower()
]
assert len(no_key_warnings) == 0
@patch(f"{MODULE}.get_llm")
def test_settings_snapshot_string_api_key_suppresses_warning(
self, mock_get_llm
):
mock_get_llm.return_value = Mock()
from local_deep_research.benchmarks.graders import get_evaluation_llm
snapshot = {
"llm.openai_endpoint.api_key": "sk-direct-key",
"llm.openai_endpoint.url": "https://openrouter.ai/api/v1",
}
with patch(f"{MODULE}.logger") as mock_logger:
get_evaluation_llm(settings_snapshot=snapshot)
no_key_warnings = [
c
for c in mock_logger.warning.call_args_list
if "no api key" in str(c).lower()
]
assert len(no_key_warnings) == 0
@patch(f"{MODULE}.get_llm")
def test_no_settings_snapshot_logs_warning(self, mock_get_llm):
mock_get_llm.return_value = Mock()
from local_deep_research.benchmarks.graders import get_evaluation_llm
with patch(f"{MODULE}.logger") as mock_logger:
get_evaluation_llm()
# Warning about missing settings snapshot (provider is openai_endpoint by default)
warning_calls = [str(c) for c in mock_logger.warning.call_args_list]
assert any(
"settings snapshot" in w.lower() or "api key" in w.lower()
for w in warning_calls
)
# ---------------------------------------------------------------------------
# extract_answer_from_response
# ---------------------------------------------------------------------------
class TestExtractAnswerFromResponse:
def test_browsecomp_extracts_exact_answer(self):
from local_deep_research.benchmarks.graders import (
extract_answer_from_response,
)
response = "Exact Answer: Paris\nConfidence: 95%"
result = extract_answer_from_response(
response, dataset_type="browsecomp"
)
assert result["extracted_answer"] == "Paris"
assert result["confidence"] == "95"
def test_browsecomp_no_match_returns_none(self):
from local_deep_research.benchmarks.graders import (
extract_answer_from_response,
)
response = "I don't know"
result = extract_answer_from_response(
response, dataset_type="browsecomp"
)
assert result["extracted_answer"] == "None"
def test_simpleqa_returns_full_response(self):
from local_deep_research.benchmarks.graders import (
extract_answer_from_response,
)
response = "The answer is 42."
result = extract_answer_from_response(response, dataset_type="simpleqa")
assert result["extracted_answer"] == "The answer is 42."
assert result["confidence"] == "100"
def test_citations_removed_from_response(self):
from local_deep_research.benchmarks.graders import (
extract_answer_from_response,
)
response = "Paris[1] is the capital[2]."
result = extract_answer_from_response(response, dataset_type="simpleqa")
assert "[1]" not in result["extracted_answer"]
assert "[2]" not in result["extracted_answer"]
# ---------------------------------------------------------------------------
# grade_single_result – simpleqa success
# ---------------------------------------------------------------------------
class TestGradeSingleResultSimpleqa:
@patch(f"{MODULE}.get_evaluation_llm")
@patch("local_deep_research.utilities.resource_utils.safe_close")
def test_simpleqa_correct_answer(self, mock_safe_close, mock_get_llm):
mock_llm = Mock(spec=[]) # no chat_messages attr, has invoke
mock_llm.invoke = Mock(
return_value="Extracted Answer: Paris\nReasoning: Correct.\nCorrect: yes"
)
mock_get_llm.return_value = mock_llm
from local_deep_research.benchmarks.graders import grade_single_result
result = grade_single_result(
result_data={
"problem": "Capital of France?",
"correct_answer": "Paris",
"response": "Paris is the capital of France.",
},
dataset_type="simpleqa",
)
assert result["is_correct"] is True
@patch(f"{MODULE}.get_evaluation_llm")
@patch("local_deep_research.utilities.resource_utils.safe_close")
def test_simpleqa_incorrect_answer(self, mock_safe_close, mock_get_llm):
mock_llm = Mock(spec=[])
mock_llm.invoke = Mock(
return_value="Extracted Answer: London\nReasoning: Wrong city.\nCorrect: no"
)
mock_get_llm.return_value = mock_llm
from local_deep_research.benchmarks.graders import grade_single_result
result = grade_single_result(
result_data={
"problem": "Capital of France?",
"correct_answer": "Paris",
"response": "London is the capital.",
},
dataset_type="simpleqa",
)
assert result["is_correct"] is False
@patch(f"{MODULE}.get_evaluation_llm")
@patch("local_deep_research.utilities.resource_utils.safe_close")
def test_simpleqa_llm_with_content_attribute(
self, mock_safe_close, mock_get_llm
):
response_obj = Mock()
response_obj.content = (
"Extracted Answer: Tokyo\nReasoning: Correct.\nCorrect: yes"
)
mock_llm = Mock(spec=[])
mock_llm.invoke = Mock(return_value=response_obj)
mock_get_llm.return_value = mock_llm
from local_deep_research.benchmarks.graders import grade_single_result
result = grade_single_result(
result_data={
"problem": "Capital of Japan?",
"correct_answer": "Tokyo",
"response": "Tokyo.",
},
dataset_type="simpleqa",
)
assert result["is_correct"] is True
# ---------------------------------------------------------------------------
# grade_single_result – browsecomp success
# ---------------------------------------------------------------------------
class TestGradeSingleResultBrowsecomp:
@patch(f"{MODULE}.get_evaluation_llm")
@patch("local_deep_research.utilities.resource_utils.safe_close")
def test_browsecomp_correct_answer(self, mock_safe_close, mock_get_llm):
mock_llm = Mock(spec=[])
mock_llm.invoke = Mock(
return_value=(
"extracted_final_answer: Eiffel Tower\n"
"reasoning: The answer matches.\n"
"correct: yes\n"
"confidence: 90"
)
)
mock_get_llm.return_value = mock_llm
from local_deep_research.benchmarks.graders import grade_single_result
result = grade_single_result(
result_data={
"problem": "Famous Paris landmark?",
"correct_answer": "Eiffel Tower",
"response": "The Eiffel Tower is in Paris.",
},
dataset_type="browsecomp",
)
assert result["is_correct"] is True
assert result["extracted_by_grader"] == "Eiffel Tower"
@patch(f"{MODULE}.get_evaluation_llm")
@patch("local_deep_research.utilities.resource_utils.safe_close")
def test_browsecomp_incorrect_returns_false(
self, mock_safe_close, mock_get_llm
):
mock_llm = Mock(spec=[])
mock_llm.invoke = Mock(
return_value=(
"extracted_final_answer: None\n"
"reasoning: No match.\n"
"correct: no\n"
"confidence: 50"
)
)
mock_get_llm.return_value = mock_llm
from local_deep_research.benchmarks.graders import grade_single_result
result = grade_single_result(
result_data={
"problem": "Q?",
"correct_answer": "A",
"response": "wrong",
},
dataset_type="browsecomp",
)
assert result["is_correct"] is False
# ---------------------------------------------------------------------------
# grade_single_result – error path
# ---------------------------------------------------------------------------
class TestGradeSingleResultError:
@patch(f"{MODULE}.get_evaluation_llm")
@patch("local_deep_research.utilities.resource_utils.safe_close")
def test_llm_error_returns_error_dict(self, mock_safe_close, mock_get_llm):
mock_llm = Mock(spec=[])
mock_llm.invoke = Mock(side_effect=RuntimeError("LLM crashed"))
mock_get_llm.return_value = mock_llm
from local_deep_research.benchmarks.graders import grade_single_result
result = grade_single_result(
result_data={
"problem": "Q?",
"correct_answer": "A",
"response": "R",
}
)
assert result["is_correct"] is False
assert "grading_error" in result
assert "LLM crashed" in result["grading_error"]
# ---------------------------------------------------------------------------
# grade_results – basic flow
# ---------------------------------------------------------------------------
class TestGradeResults:
@patch(f"{MODULE}.get_evaluation_llm")
@patch("local_deep_research.utilities.resource_utils.safe_close")
def test_grade_results_processes_all_lines(
self, mock_safe_close, mock_get_llm, tmp_path
):
mock_llm = Mock(spec=[])
mock_llm.invoke = Mock(
return_value="Extracted Answer: A\nReasoning: ok.\nCorrect: yes"
)
mock_get_llm.return_value = mock_llm
results_file = tmp_path / "results.jsonl"
output_file = tmp_path / "graded.jsonl"
data = [
{"problem": "Q1", "correct_answer": "A1", "response": "A1"},
{"problem": "Q2", "correct_answer": "A2", "response": "A2"},
]
results_file.write_text("\n".join(json.dumps(d) for d in data))
from local_deep_research.benchmarks.graders import grade_results
graded = grade_results(
results_file=str(results_file),
output_file=str(output_file),
dataset_type="simpleqa",
)
assert len(graded) == 2
@patch(f"{MODULE}.get_evaluation_llm")
@patch("local_deep_research.utilities.resource_utils.safe_close")
def test_grade_results_with_progress_callback(
self, mock_safe_close, mock_get_llm, tmp_path
):
mock_llm = Mock(spec=[])
mock_llm.invoke = Mock(
return_value="Extracted Answer: A\nReasoning: ok.\nCorrect: no"
)
mock_get_llm.return_value = mock_llm
results_file = tmp_path / "results.jsonl"
output_file = tmp_path / "graded.jsonl"
data = [{"problem": "Q1", "correct_answer": "A1", "response": "wrong"}]
results_file.write_text(json.dumps(data[0]))
callback = Mock()
from local_deep_research.benchmarks.graders import grade_results
grade_results(
results_file=str(results_file),
output_file=str(output_file),
dataset_type="simpleqa",
progress_callback=callback,
)
assert callback.call_count >= 1
# ---------------------------------------------------------------------------
# human_evaluation – non-interactive mode
# ---------------------------------------------------------------------------
class TestHumanEvaluation:
def test_non_interactive_returns_false_for_each(self, tmp_path):
results_file = tmp_path / "results.jsonl"
output_file = tmp_path / "human_graded.jsonl"
data = [
{"problem": "Q1", "correct_answer": "A1", "response": "R1"},
{"problem": "Q2", "correct_answer": "A2", "response": "R2"},
]
results_file.write_text("\n".join(json.dumps(d) for d in data))
from local_deep_research.benchmarks.graders import human_evaluation
graded = human_evaluation(
results_file=str(results_file),
output_file=str(output_file),
interactive=False,
)
assert len(graded) == 2
# Non-interactive mode always marks as incorrect
for item in graded:
assert item["is_correct"] is False
assert item["human_evaluation"] is True