~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
SimpleQA benchmark evaluator.
This module provides a benchmark evaluator implementation for the SimpleQA
benchmark, which tests simple question-answering capabilities.
"""
import json
from loguru import logger
from pathlib import Path
import time
from typing import Any, Dict
from ..datasets.base import DatasetRegistry
from ..metrics import calculate_metrics, generate_report
from ..runners import run_simpleqa_benchmark # Keep for backward compatibility
from .base import BaseBenchmarkEvaluator
class SimpleQAEvaluator(BaseBenchmarkEvaluator):
"""
Evaluator for the SimpleQA benchmark.
This evaluator runs the SimpleQA benchmark, which tests a system's ability
to accurately answer straightforward factual questions.
"""
def __init__(self):
"""Initialize the SimpleQA evaluator."""
super().__init__("simpleqa")
def evaluate(
self,
system_config: Dict[str, Any],
num_examples: int,
output_dir: str,
use_direct_dataset: bool = True,
) -> Dict[str, Any]:
"""
Run SimpleQA benchmark and return metrics.
Args:
system_config: Search and LLM configuration parameters
num_examples: Number of benchmark examples to run
output_dir: Directory to save evaluation results
use_direct_dataset: Whether to use dataset classes directly (recommended)
or fall back to runner functions
Returns:
Dictionary with metrics including quality_score based on accuracy
"""
# Create benchmark-specific directory
benchmark_dir = self._create_subdirectory(output_dir)
# Log benchmark execution
logger.info(f"Running SimpleQA benchmark with {num_examples} examples")
try:
if use_direct_dataset:
# Use dataset classes directly (new approach)
results = self._run_with_dataset_class(
system_config=system_config,
num_examples=num_examples,
output_dir=benchmark_dir,
)
else:
# Fall back to legacy runner function
results = run_simpleqa_benchmark(
num_examples=num_examples,
output_dir=benchmark_dir,
search_config=system_config,
run_evaluation=True,
)
# Extract metrics
metrics = results.get("metrics", {})
accuracy = metrics.get("accuracy", 0.0)
# Return evaluation results with quality score
return {
"benchmark_type": self.name,
"accuracy": accuracy,
"quality_score": accuracy, # Map accuracy directly to quality score
"raw_results": results,
"report_path": results.get("report_path"),
}
except Exception as e:
logger.exception("Error in SimpleQA evaluation")
# Return error information
return {
"benchmark_type": self.name,
"error": str(e),
"quality_score": 0.0,
"accuracy": 0.0,
}
def _run_with_dataset_class(
self,
system_config: Dict[str, Any],
num_examples: int,
output_dir: str,
) -> Dict[str, Any]:
"""
Run SimpleQA benchmark using dataset classes directly.
This implementation directly uses the dataset classes rather than
going through the runner functions, allowing for more flexibility
and better integration with the object-oriented architecture.
Args:
system_config: Search and LLM configuration parameters
num_examples: Number of benchmark examples to run
output_dir: Directory to save evaluation results. The files
are written there directly; unlike ``run_benchmark``, this
path does not resolve the directory below the data
directory.
Returns:
Dictionary with benchmark results
"""
# Refuse before any research when the grader would have no API key
# for its endpoint (grading runs only after every example).
from ..graders import check_grader_endpoint_key
check_grader_endpoint_key()
# Create a dataset instance using the registry
try:
dataset_instance = DatasetRegistry.create_dataset(
dataset_id="simpleqa",
num_examples=num_examples,
seed=system_config.get("seed", 42),
)
# Load dataset examples
examples = dataset_instance.load()
logger.info(f"Loaded {len(examples)} SimpleQA examples")
# Set up output files
timestamp = time.strftime("%Y%m%d_%H%M%S")
results_file = str(
Path(output_dir) / f"simpleqa_{timestamp}_results.jsonl"
)
evaluation_file = str(
Path(output_dir) / f"simpleqa_{timestamp}_evaluation.jsonl"
)
report_file = str(
Path(output_dir) / f"simpleqa_{timestamp}_report.md"
)
# Process each example
results = []
for i, example in enumerate(examples):
# Extract question and answer using dataset methods
question = dataset_instance.get_question(example)
correct_answer = dataset_instance.get_answer(example)
logger.info(
f"Processing {i + 1}/{len(examples)}: {question[:50]}..."
)
try:
# Format query based on dataset type
formatted_query = question # Simple format for SimpleQA
# Time the search
start_time = time.time()
# Create search config from system_config
search_params = {
"iterations": system_config.get("iterations", 3),
"questions_per_iteration": system_config.get(
"questions_per_iteration", 3
),
"search_tool": system_config.get(
"search_tool", "searxng"
),
# Note: search_strategy is stored in the config but not passed to quick_summary
# as it's not supported by the underlying API
}
# Get response from LDR
from local_deep_research.api import quick_summary
search_result = quick_summary(
query=formatted_query,
iterations=search_params.get("iterations"),
questions_per_iteration=search_params.get(
"questions_per_iteration"
),
search_tool=search_params.get("search_tool"),
)
end_time = time.time()
processing_time = end_time - start_time
# Extract response
response = search_result.get("summary", "")
# Extract structured answer
from ..graders import extract_answer_from_response
extracted = extract_answer_from_response(
response, "simpleqa"
)
# Format result
result = {
"id": example.get("id", f"example_{i}"),
"problem": question,
"correct_answer": correct_answer,
"response": response,
"extracted_answer": extracted["extracted_answer"],
"confidence": extracted["confidence"],
"processing_time": processing_time,
"sources": search_result.get("sources", []),
"search_config": search_params,
}
# Add to results list
results.append(result)
# Write result to file
# nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python SimpleQA output root with a fixed prefix and generated timestamp
with open(results_file, "a", encoding="utf-8") as f:
f.write(json.dumps(result) + "\n")
except Exception as e:
logger.exception(f"Error processing example {i + 1}")
# Create error result
error_result = {
"id": example.get("id", f"example_{i}"),
"problem": question,
"correct_answer": correct_answer,
"error": str(e),
"processing_time": 0,
}
# Add to results list
results.append(error_result)
# Write error result to file
# nosemgrep: semgrep.rules.path-traversal-risk, reason: Same generated SimpleQA results file used by the success path above
with open(results_file, "a", encoding="utf-8") as f:
f.write(json.dumps(error_result) + "\n")
# Grade results
from ..graders import grade_results
grade_results(
results_file=results_file,
output_file=evaluation_file,
dataset_type="simpleqa",
)
# Calculate metrics
metrics = calculate_metrics(evaluation_file)
# Generate report
dataset_name = "SimpleQA"
report_path = generate_report(
metrics=metrics,
results_file=evaluation_file,
output_file=report_file,
dataset_name=dataset_name,
config_info={
"Dataset": "SimpleQA",
"Examples": len(examples),
"Iterations": search_params.get("iterations", 3),
"Questions per iteration": search_params.get(
"questions_per_iteration", 3
),
"Search tool": search_params.get("search_tool", "searxng"),
"Search strategy": search_params.get(
"search_strategy", "source_based"
),
},
)
# Return results
return {
"status": "complete",
"dataset_type": "simpleqa",
"results_path": results_file,
"evaluation_path": evaluation_file,
"report_path": report_path,
"metrics": metrics,
"total_examples": len(examples),
"accuracy": metrics.get("accuracy", 0),
}
except Exception as e:
logger.exception("Error in direct dataset evaluation")
return {
"status": "error",
"dataset_type": "simpleqa",
"error": str(e),
}