~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Coverage tests for benchmarks/datasets/simpleqa.py.
Targets the 12 missing lines:
- get_dataset_info()
- get_default_dataset_path()
- process_example() – normal, missing problem, missing answer, missing correct_answer
- get_question()
- get_answer()
"""
from unittest.mock import patch
MODULE = "local_deep_research.benchmarks.datasets.simpleqa"
def _make_dataset(**kwargs):
from local_deep_research.benchmarks.datasets.simpleqa import SimpleQADataset
return SimpleQADataset(**kwargs)
# ---------------------------------------------------------------------------
# Dataset metadata
# ---------------------------------------------------------------------------
class TestSimpleQADatasetInfo:
def test_get_dataset_info_has_simpleqa_id(self):
from local_deep_research.benchmarks.datasets.simpleqa import (
SimpleQADataset,
)
info = SimpleQADataset.get_dataset_info()
assert info["id"] == "simpleqa"
def test_get_dataset_info_has_all_required_keys(self):
from local_deep_research.benchmarks.datasets.simpleqa import (
SimpleQADataset,
)
info = SimpleQADataset.get_dataset_info()
assert "name" in info
assert "description" in info
assert "url" in info
def test_get_default_dataset_path_is_url(self):
from local_deep_research.benchmarks.datasets.simpleqa import (
SimpleQADataset,
)
path = SimpleQADataset.get_default_dataset_path()
assert path.startswith("https://")
assert "simple" in path.lower()
def test_dataset_info_url_matches_default_path(self):
from local_deep_research.benchmarks.datasets.simpleqa import (
SimpleQADataset,
)
info = SimpleQADataset.get_dataset_info()
assert info["url"] == SimpleQADataset.get_default_dataset_path()
def test_default_path_ends_with_csv(self):
from local_deep_research.benchmarks.datasets.simpleqa import (
SimpleQADataset,
)
path = SimpleQADataset.get_default_dataset_path()
assert path.endswith(".csv")
# ---------------------------------------------------------------------------
# process_example – normal case
# ---------------------------------------------------------------------------
class TestSimpleQAProcessExampleNormal:
def test_process_example_preserves_problem_and_answer(self):
ds = _make_dataset()
example = {"problem": "What is 2+2?", "answer": "4"}
result = ds.process_example(example)
assert result["problem"] == "What is 2+2?"
assert result["answer"] == "4"
def test_process_example_adds_correct_answer(self):
ds = _make_dataset()
example = {"problem": "Q", "answer": "A"}
result = ds.process_example(example)
assert result["correct_answer"] == "A"
def test_process_example_preserves_existing_correct_answer(self):
ds = _make_dataset()
example = {"problem": "Q", "answer": "A", "correct_answer": "CA"}
result = ds.process_example(example)
assert result["correct_answer"] == "CA"
def test_process_example_does_not_mutate_original(self):
ds = _make_dataset()
original = {"problem": "Q", "answer": "A"}
ds.process_example(original)
assert "correct_answer" not in original
# ---------------------------------------------------------------------------
# process_example – missing fields
# ---------------------------------------------------------------------------
class TestSimpleQAProcessExampleMissingFields:
def test_missing_problem_added_as_empty_string(self):
ds = _make_dataset()
result = ds.process_example({"answer": "A"})
assert result["problem"] == ""
def test_missing_problem_logs_warning(self):
ds = _make_dataset()
with patch(f"{MODULE}.logger") as mock_logger:
ds.process_example({"answer": "A"})
mock_logger.warning.assert_called_once()
def test_missing_answer_added_as_empty_string(self):
ds = _make_dataset()
result = ds.process_example({"problem": "P"})
assert result["answer"] == ""
def test_missing_answer_logs_warning(self):
ds = _make_dataset()
with patch(f"{MODULE}.logger") as mock_logger:
ds.process_example({"problem": "P"})
mock_logger.warning.assert_called_once()
# ---------------------------------------------------------------------------
# get_question / get_answer
# ---------------------------------------------------------------------------
class TestSimpleQAGetQuestionAndAnswer:
def test_get_question_returns_problem(self):
ds = _make_dataset()
example = {"problem": "Capital of Germany?", "answer": "Berlin"}
assert ds.get_question(example) == "Capital of Germany?"
def test_get_question_returns_empty_when_missing(self):
ds = _make_dataset()
assert ds.get_question({}) == ""
def test_get_answer_returns_answer_field(self):
ds = _make_dataset()
example = {"answer": "Berlin"}
assert ds.get_answer(example) == "Berlin"
def test_get_answer_returns_empty_when_missing(self):
ds = _make_dataset()
assert ds.get_answer({}) == ""