~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
High-value pure logic tests for benchmarks/runners.py.
Focuses on format_query() and other pure logic elements.
"""
from local_deep_research.benchmarks.runners import format_query
from local_deep_research.benchmarks.templates import BROWSECOMP_QUERY_TEMPLATE
class TestFormatQuerySimpleQA:
"""Tests for format_query with simpleqa dataset type."""
def test_simpleqa_returns_question_unchanged(self):
question = "What is the capital of France?"
result = format_query(question, "simpleqa")
assert result == question
def test_simpleqa_preserves_whitespace(self):
question = " leading and trailing spaces "
result = format_query(question, "simpleqa")
assert result == question
def test_simpleqa_empty_string(self):
result = format_query("", "simpleqa")
assert result == ""
def test_simpleqa_multiline_question(self):
question = "Line one\nLine two\nLine three"
result = format_query(question, "simpleqa")
assert result == question
def test_simpleqa_is_default_dataset_type(self):
question = "What year was Python created?"
result = format_query(question)
assert result == question
class TestFormatQueryBrowseComp:
"""Tests for format_query with browsecomp dataset type."""
def test_browsecomp_applies_template(self):
question = "Find the author of this article."
result = format_query(question, "browsecomp")
expected = BROWSECOMP_QUERY_TEMPLATE.format(question=question)
assert result == expected
def test_browsecomp_case_insensitive_lowercase(self):
question = "Some question"
result = format_query(question, "browsecomp")
expected = BROWSECOMP_QUERY_TEMPLATE.format(question=question)
assert result == expected
def test_browsecomp_case_insensitive_uppercase(self):
question = "Some question"
result = format_query(question, "BROWSECOMP")
expected = BROWSECOMP_QUERY_TEMPLATE.format(question=question)
assert result == expected
def test_browsecomp_case_insensitive_mixed(self):
question = "Some question"
result = format_query(question, "BrowseComp")
expected = BROWSECOMP_QUERY_TEMPLATE.format(question=question)
assert result == expected
def test_browsecomp_case_insensitive_random_casing(self):
question = "Some question"
result = format_query(question, "bRoWsEcOmP")
expected = BROWSECOMP_QUERY_TEMPLATE.format(question=question)
assert result == expected
def test_browsecomp_empty_question(self):
result = format_query("", "browsecomp")
expected = BROWSECOMP_QUERY_TEMPLATE.format(question="")
assert result == expected
def test_browsecomp_result_contains_question_text(self):
question = "Who wrote the first web browser?"
result = format_query(question, "browsecomp")
assert question in result
def test_browsecomp_result_contains_format_instructions(self):
question = "Test question"
result = format_query(question, "browsecomp")
assert "Exact Answer:" in result
assert "Confidence:" in result
assert "Explanation:" in result
def test_browsecomp_result_differs_from_raw_question(self):
question = "A simple question?"
result = format_query(question, "browsecomp")
assert result != question
assert len(result) > len(question)
class TestFormatQueryUnknownDatasetTypes:
"""Tests for format_query with unknown/other dataset types."""
def test_unknown_type_returns_question_unchanged(self):
question = "What is 2+2?"
result = format_query(question, "unknown_dataset")
assert result == question
def test_xbench_deepsearch_returns_question_unchanged(self):
question = "Complex research question"
result = format_query(question, "xbench_deepsearch")
assert result == question
def test_empty_dataset_type_returns_question_unchanged(self):
"""Empty string dataset type is not 'browsecomp', so returns unchanged."""
question = "Some question"
result = format_query(question, "")
assert result == question
def test_browsecomp_prefix_does_not_match(self):
"""Only exact match (case-insensitive) should trigger template."""
question = "Some question"
result = format_query(question, "browsecomp_v2")
assert result == question
def test_browsecomp_with_extra_spaces_does_not_match(self):
question = "Some question"
result = format_query(question, " browsecomp ")
assert result == question
class TestFormatQuerySpecialCharacters:
"""Tests for format_query with special characters in questions."""
def test_question_with_curly_braces(self):
"""Curly braces in question should not break template formatting."""
question = "What is {x} in the equation?"
# For simpleqa, returned as-is
result = format_query(question, "simpleqa")
assert result == question
def test_browsecomp_question_with_unicode(self):
question = "Qu'est-ce que c'est? Aeoeue"
result = format_query(question, "browsecomp")
expected = BROWSECOMP_QUERY_TEMPLATE.format(question=question)
assert result == expected
def test_question_with_newlines_browsecomp(self):
question = "First part\nSecond part"
result = format_query(question, "browsecomp")
assert "First part\nSecond part" in result
assert "Exact Answer:" in result