~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for benchmark evaluation templates.
This module tests the prompt templates used for evaluating model outputs
against reference answers.
"""
from local_deep_research.benchmarks.templates import (
BROWSECOMP_GRADER_TEMPLATE,
BROWSECOMP_QUERY_TEMPLATE,
SIMPLEQA_GRADER_TEMPLATE,
)
class TestSimpleQAGraderTemplate:
"""Tests for the SimpleQA grader template."""
def test_contains_question_placeholder(self):
"""Template must contain the question placeholder."""
assert "{question}" in SIMPLEQA_GRADER_TEMPLATE
def test_contains_correct_answer_placeholder(self):
"""Template must contain the correct_answer placeholder."""
assert "{correct_answer}" in SIMPLEQA_GRADER_TEMPLATE
def test_contains_response_placeholder(self):
"""Template must contain the response placeholder."""
assert "{response}" in SIMPLEQA_GRADER_TEMPLATE
def test_can_format_with_all_placeholders(self):
"""Template can be formatted with all required placeholders."""
formatted = SIMPLEQA_GRADER_TEMPLATE.format(
question="What is the capital of France?",
correct_answer="Paris",
response="The capital of France is Paris.",
)
assert "What is the capital of France?" in formatted
assert "Paris" in formatted
assert "The capital of France is Paris." in formatted
# Ensure no unformatted placeholders remain
assert "{question}" not in formatted
assert "{correct_answer}" not in formatted
assert "{response}" not in formatted
def test_contains_grading_instructions(self):
"""Template should contain instructions for grading."""
assert "Correct:" in SIMPLEQA_GRADER_TEMPLATE
assert "yes/no" in SIMPLEQA_GRADER_TEMPLATE
class TestBrowseCompGraderTemplate:
"""Tests for the BrowseComp grader template."""
def test_contains_question_placeholder(self):
"""Template must contain the question placeholder."""
assert "{question}" in BROWSECOMP_GRADER_TEMPLATE
def test_contains_correct_answer_placeholder(self):
"""Template must contain the correct_answer placeholder."""
assert "{correct_answer}" in BROWSECOMP_GRADER_TEMPLATE
def test_contains_response_placeholder(self):
"""Template must contain the response placeholder."""
assert "{response}" in BROWSECOMP_GRADER_TEMPLATE
def test_can_format_with_all_placeholders(self):
"""Template can be formatted with all required placeholders."""
formatted = BROWSECOMP_GRADER_TEMPLATE.format(
question="Who wrote Romeo and Juliet?",
correct_answer="William Shakespeare",
response="William Shakespeare wrote Romeo and Juliet.",
)
assert "Who wrote Romeo and Juliet?" in formatted
assert "William Shakespeare" in formatted
# Ensure no unformatted placeholders remain
assert "{question}" not in formatted
assert "{correct_answer}" not in formatted
assert "{response}" not in formatted
def test_contains_grading_fields(self):
"""Template should contain expected grading fields."""
assert "extracted_final_answer:" in BROWSECOMP_GRADER_TEMPLATE
assert "reasoning:" in BROWSECOMP_GRADER_TEMPLATE
assert "correct:" in BROWSECOMP_GRADER_TEMPLATE
assert "confidence:" in BROWSECOMP_GRADER_TEMPLATE
class TestBrowseCompQueryTemplate:
"""Tests for the BrowseComp query template."""
def test_contains_question_placeholder(self):
"""Template must contain the question placeholder."""
assert "{question}" in BROWSECOMP_QUERY_TEMPLATE
def test_can_format_with_question(self):
"""Template can be formatted with a question."""
formatted = BROWSECOMP_QUERY_TEMPLATE.format(
question="What year was the Eiffel Tower built?"
)
assert "What year was the Eiffel Tower built?" in formatted
assert "{question}" not in formatted
def test_contains_expected_response_format(self):
"""Template should specify the expected response format."""
assert "Explanation:" in BROWSECOMP_QUERY_TEMPLATE
assert "Exact Answer:" in BROWSECOMP_QUERY_TEMPLATE
assert "Confidence:" in BROWSECOMP_QUERY_TEMPLATE
def test_uses_escaped_braces_for_format_instructions(self):
"""Template uses escaped braces for format instructions that should remain literal."""
# The template uses {{ and }} for literal braces in format instructions
# After formatting, these should become { and }
formatted = BROWSECOMP_QUERY_TEMPLATE.format(question="Test question")
assert "{your explanation" in formatted
assert "{your succinct" in formatted
assert "{your confidence" in formatted