~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Coverage tests for benchmarks/datasets/custom_dataset_template.py.
Targets all 25 missing lines (0% coverage):
- get_dataset_info()
- get_default_dataset_path()
- process_example() – normal, missing problem, missing answer, missing correct_answer
- get_question()
- get_answer() – correct_answer present, fallback to answer, empty
"""
from unittest.mock import patch
MODULE = "local_deep_research.benchmarks.datasets.custom_dataset_template"
def _make_dataset(**kwargs):
from local_deep_research.benchmarks.datasets.custom_dataset_template import (
CustomDataset,
)
return CustomDataset(**kwargs)
# ---------------------------------------------------------------------------
# Dataset metadata
# ---------------------------------------------------------------------------
class TestCustomDatasetInfo:
def test_get_dataset_info_returns_required_keys(self):
from local_deep_research.benchmarks.datasets.custom_dataset_template import (
CustomDataset,
)
info = CustomDataset.get_dataset_info()
assert "id" in info
assert "name" in info
assert "description" in info
assert "url" in info
def test_get_dataset_info_id_is_custom(self):
from local_deep_research.benchmarks.datasets.custom_dataset_template import (
CustomDataset,
)
info = CustomDataset.get_dataset_info()
assert info["id"] == "custom"
def test_get_default_dataset_path_is_string(self):
from local_deep_research.benchmarks.datasets.custom_dataset_template import (
CustomDataset,
)
path = CustomDataset.get_default_dataset_path()
assert isinstance(path, str)
assert len(path) > 0
def test_dataset_info_url_matches_default_path(self):
from local_deep_research.benchmarks.datasets.custom_dataset_template import (
CustomDataset,
)
info = CustomDataset.get_dataset_info()
assert info["url"] == CustomDataset.get_default_dataset_path()
# ---------------------------------------------------------------------------
# process_example – normal case
# ---------------------------------------------------------------------------
class TestProcessExampleNormal:
def test_process_example_preserves_problem_and_answer(self):
ds = _make_dataset()
example = {"problem": "What is 2+2?", "answer": "4"}
result = ds.process_example(example)
assert result["problem"] == "What is 2+2?"
assert result["answer"] == "4"
def test_process_example_adds_correct_answer_from_answer(self):
ds = _make_dataset()
example = {"problem": "Q", "answer": "A"}
result = ds.process_example(example)
assert result["correct_answer"] == "A"
def test_process_example_preserves_existing_correct_answer(self):
ds = _make_dataset()
example = {"problem": "Q", "answer": "A", "correct_answer": "CA"}
result = ds.process_example(example)
assert result["correct_answer"] == "CA"
def test_process_example_does_not_modify_original(self):
ds = _make_dataset()
original = {"problem": "Q", "answer": "A"}
ds.process_example(original)
assert "correct_answer" not in original
# ---------------------------------------------------------------------------
# process_example – missing fields
# ---------------------------------------------------------------------------
class TestProcessExampleMissingFields:
def test_missing_problem_field_added_as_empty_string(self):
ds = _make_dataset()
example = {"answer": "some_answer"}
result = ds.process_example(example)
assert result["problem"] == ""
def test_missing_problem_logs_warning(self):
ds = _make_dataset()
with patch(f"{MODULE}.logger") as mock_logger:
ds.process_example({"answer": "A"})
mock_logger.warning.assert_called_once()
assert "problem" in mock_logger.warning.call_args[0][0].lower()
def test_missing_answer_field_added_as_empty_string(self):
ds = _make_dataset()
example = {"problem": "some_problem"}
result = ds.process_example(example)
assert result["answer"] == ""
def test_missing_answer_logs_warning(self):
ds = _make_dataset()
with patch(f"{MODULE}.logger") as mock_logger:
ds.process_example({"problem": "P"})
mock_logger.warning.assert_called_once()
assert "answer" in mock_logger.warning.call_args[0][0].lower()
def test_missing_both_problem_and_answer(self):
ds = _make_dataset()
result = ds.process_example({})
assert result["problem"] == ""
assert result["answer"] == ""
assert result["correct_answer"] == ""
def test_extra_fields_preserved(self):
ds = _make_dataset()
example = {
"problem": "Q",
"answer": "A",
"category": "math",
"difficulty": "easy",
}
result = ds.process_example(example)
assert result["category"] == "math"
assert result["difficulty"] == "easy"
# ---------------------------------------------------------------------------
# get_question
# ---------------------------------------------------------------------------
class TestGetQuestion:
def test_get_question_returns_problem_field(self):
ds = _make_dataset()
example = {"problem": "What is the capital of France?"}
assert ds.get_question(example) == "What is the capital of France?"
def test_get_question_returns_empty_when_missing(self):
ds = _make_dataset()
assert ds.get_question({}) == ""
def test_get_question_returns_empty_string_not_none(self):
ds = _make_dataset()
result = ds.get_question({"other": "field"})
assert result == ""
# ---------------------------------------------------------------------------
# get_answer
# ---------------------------------------------------------------------------
class TestGetAnswer:
def test_get_answer_prefers_correct_answer(self):
ds = _make_dataset()
example = {"answer": "raw", "correct_answer": "preferred"}
assert ds.get_answer(example) == "preferred"
def test_get_answer_falls_back_to_answer(self):
ds = _make_dataset()
example = {"answer": "fallback"}
assert ds.get_answer(example) == "fallback"
def test_get_answer_returns_empty_when_neither_present(self):
ds = _make_dataset()
assert ds.get_answer({}) == ""