~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
API functions for benchmarking.
This module provides functions for running benchmarks programmatically.
"""
from pathlib import Path
from typing import Any, Dict, List, Optional
from loguru import logger
from ..config.thread_settings import get_setting_from_snapshot
from .graders import normalize_grader_provider
from ..benchmarks import (
calculate_metrics,
generate_report,
run_benchmark,
run_browsecomp_benchmark,
run_simpleqa_benchmark,
run_xbench_deepsearch_benchmark,
)
from .runners import resolve_output_dir
def evaluate_simpleqa(
num_examples: int = 100,
search_iterations: int = 3,
questions_per_iteration: int = 3,
search_tool: str = "searxng",
human_evaluation: bool = False,
evaluation_model: Optional[str] = None,
evaluation_provider: Optional[str] = None,
output_dir: str = "benchmark_results",
search_model: Optional[str] = None,
search_provider: Optional[str] = None,
endpoint_url: Optional[str] = None,
search_strategy: str = "source_based",
evaluation_api_key: Optional[str] = None,
) -> Dict[str, Any]:
"""
Run SimpleQA benchmark evaluation.
Args:
num_examples: Number of examples to evaluate
search_iterations: Number of search iterations per query
questions_per_iteration: Number of questions per iteration
search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
human_evaluation: Whether to use human evaluation
evaluation_model: Optional custom model for evaluation
evaluation_provider: Optional custom provider for evaluation
output_dir: Directory to save results
search_model: Optional model to use for the search system
search_provider: Optional provider to use for the search system
endpoint_url: Optional endpoint URL for OpenRouter or other API services
search_strategy: Search strategy to use (default: 'source_based')
evaluation_api_key: Optional API key for the grader's endpoint. It is
only used for grading, with the grader endpoint of this run,
and only when the evaluation provider is ``openai_endpoint``;
other providers ignore it.
Returns:
Dictionary with benchmark results
"""
logger.info(f"Starting SimpleQA benchmark with {num_examples} examples")
# Set up search configuration
search_config = {
"iterations": search_iterations,
"questions_per_iteration": questions_per_iteration,
"search_tool": search_tool,
"search_strategy": search_strategy,
}
# Add model configurations if provided
if search_model:
search_config["model_name"] = search_model
if search_provider:
search_config["provider"] = search_provider
if endpoint_url:
search_config["openai_endpoint_url"] = endpoint_url
# Check settings for additional configuration
if env_model := get_setting_from_snapshot("llm.model"):
search_config["model_name"] = env_model
if env_provider := get_setting_from_snapshot("llm.provider"):
search_config["provider"] = env_provider
if env_url := get_setting_from_snapshot("llm.openai_endpoint.url"):
search_config["openai_endpoint_url"] = env_url
# Set up evaluation configuration if needed
evaluation_config = None
if evaluation_model or evaluation_provider:
evaluation_config = {
"temperature": 0 # Always use zero temperature for evaluation
}
if evaluation_model:
evaluation_config["model_name"] = evaluation_model
if evaluation_provider:
evaluation_provider = normalize_grader_provider(evaluation_provider)
evaluation_config["provider"] = evaluation_provider
# Add endpoint URL if using openai_endpoint
if evaluation_provider == "openai_endpoint" and endpoint_url:
evaluation_config["openai_endpoint_url"] = endpoint_url
elif evaluation_provider == "openai_endpoint" and env_url:
evaluation_config["openai_endpoint_url"] = env_url
if evaluation_api_key:
evaluation_config = dict(evaluation_config or {"temperature": 0})
evaluation_config["api_key"] = evaluation_api_key
# Run the benchmark
return run_simpleqa_benchmark(
num_examples=num_examples,
output_dir=output_dir,
search_config=search_config,
evaluation_config=evaluation_config,
human_evaluation=human_evaluation,
)
def evaluate_browsecomp(
num_examples: int = 100,
search_iterations: int = 3,
questions_per_iteration: int = 3,
search_tool: str = "searxng",
human_evaluation: bool = False,
evaluation_model: Optional[str] = None,
evaluation_provider: Optional[str] = None,
output_dir: str = "benchmark_results",
search_model: Optional[str] = None,
search_provider: Optional[str] = None,
endpoint_url: Optional[str] = None,
search_strategy: str = "source_based",
evaluation_api_key: Optional[str] = None,
) -> Dict[str, Any]:
"""
Run BrowseComp benchmark evaluation.
Args:
num_examples: Number of examples to evaluate
search_iterations: Number of search iterations per query
questions_per_iteration: Number of questions per iteration
search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
human_evaluation: Whether to use human evaluation
evaluation_model: Optional custom model for evaluation
evaluation_provider: Optional custom provider for evaluation
output_dir: Directory to save results
search_model: Optional model to use for the search system
search_provider: Optional provider to use for the search system
endpoint_url: Optional endpoint URL for OpenRouter or other API services
search_strategy: Search strategy to use (default: 'source_based')
evaluation_api_key: Optional API key for the grader's endpoint. It is
only used for grading, with the grader endpoint of this run,
and only when the evaluation provider is ``openai_endpoint``;
other providers ignore it.
Returns:
Dictionary with benchmark results
"""
logger.info(f"Starting BrowseComp benchmark with {num_examples} examples")
# Set up search configuration
search_config = {
"iterations": search_iterations,
"questions_per_iteration": questions_per_iteration,
"search_tool": search_tool,
"search_strategy": search_strategy,
}
# Add model configurations if provided
if search_model:
search_config["model_name"] = search_model
if search_provider:
search_config["provider"] = search_provider
if endpoint_url:
search_config["openai_endpoint_url"] = endpoint_url
# Check settings for additional configuration
if env_model := get_setting_from_snapshot("llm.model"):
search_config["model_name"] = env_model
if env_provider := get_setting_from_snapshot("llm.provider"):
search_config["provider"] = env_provider
if env_url := get_setting_from_snapshot("llm.openai_endpoint.url"):
search_config["openai_endpoint_url"] = env_url
# Set up evaluation configuration if needed
evaluation_config = None
if evaluation_model or evaluation_provider:
evaluation_config = {
"temperature": 0 # Always use zero temperature for evaluation
}
if evaluation_model:
evaluation_config["model_name"] = evaluation_model
if evaluation_provider:
evaluation_provider = normalize_grader_provider(evaluation_provider)
evaluation_config["provider"] = evaluation_provider
# Add endpoint URL if using openai_endpoint
if evaluation_provider == "openai_endpoint" and endpoint_url:
evaluation_config["openai_endpoint_url"] = endpoint_url
elif evaluation_provider == "openai_endpoint" and env_url:
evaluation_config["openai_endpoint_url"] = env_url
if evaluation_api_key:
evaluation_config = dict(evaluation_config or {"temperature": 0})
evaluation_config["api_key"] = evaluation_api_key
# Run the benchmark
return run_browsecomp_benchmark(
num_examples=num_examples,
output_dir=output_dir,
search_config=search_config,
evaluation_config=evaluation_config,
human_evaluation=human_evaluation,
)
def evaluate_xbench_deepsearch(
num_examples: int = 100,
search_iterations: int = 4,
questions_per_iteration: int = 3,
search_tool: str = "searxng",
human_evaluation: bool = False,
evaluation_model: Optional[str] = None,
evaluation_provider: Optional[str] = None,
output_dir: str = "benchmark_results",
search_model: Optional[str] = None,
search_provider: Optional[str] = None,
endpoint_url: Optional[str] = None,
search_strategy: str = "source_based",
evaluation_api_key: Optional[str] = None,
) -> Dict[str, Any]:
"""
Run xbench-DeepSearch benchmark evaluation.
Args:
num_examples: Number of examples to evaluate (default 100 - full dataset)
search_iterations: Number of search iterations per query
questions_per_iteration: Number of questions per iteration
search_tool: Search engine to use
human_evaluation: Whether to use human evaluation
evaluation_model: Optional custom model for evaluation
evaluation_provider: Optional custom provider for evaluation
output_dir: Directory to save results
search_model: Optional model to use for the search system
search_provider: Optional provider to use for the search system
endpoint_url: Optional endpoint URL for API services
search_strategy: Search strategy to use
evaluation_api_key: Optional API key for the grader's endpoint. It is
only used for grading, with the grader endpoint of this run,
and only when the evaluation provider is ``openai_endpoint``;
other providers ignore it.
Returns:
Dictionary with benchmark results
"""
logger.info(
f"Starting xbench-DeepSearch benchmark with {num_examples} examples"
)
# Set up search configuration
search_config = {
"iterations": search_iterations,
"questions_per_iteration": questions_per_iteration,
"search_tool": search_tool,
"search_strategy": search_strategy,
}
# Add model configurations if provided
if search_model:
search_config["model_name"] = search_model
if search_provider:
search_config["provider"] = search_provider
if endpoint_url:
search_config["openai_endpoint_url"] = endpoint_url
# Check settings for additional configuration
if env_model := get_setting_from_snapshot("llm.model"):
search_config["model_name"] = env_model
if env_provider := get_setting_from_snapshot("llm.provider"):
search_config["provider"] = env_provider
if env_url := get_setting_from_snapshot("llm.openai_endpoint.url"):
search_config["openai_endpoint_url"] = env_url
# Set up evaluation configuration if needed
evaluation_config = None
if evaluation_model or evaluation_provider:
evaluation_config = {
"temperature": 0 # Always use zero temperature for evaluation
}
if evaluation_model:
evaluation_config["model_name"] = evaluation_model
if evaluation_provider:
evaluation_provider = normalize_grader_provider(evaluation_provider)
evaluation_config["provider"] = evaluation_provider
# Add endpoint URL if using openai_endpoint
if evaluation_provider == "openai_endpoint" and endpoint_url:
evaluation_config["openai_endpoint_url"] = endpoint_url
elif evaluation_provider == "openai_endpoint" and env_url:
evaluation_config["openai_endpoint_url"] = env_url
if evaluation_api_key:
evaluation_config = dict(evaluation_config or {"temperature": 0})
evaluation_config["api_key"] = evaluation_api_key
# Run the benchmark
return run_xbench_deepsearch_benchmark(
num_examples=num_examples,
output_dir=output_dir,
search_config=search_config,
evaluation_config=evaluation_config,
human_evaluation=human_evaluation,
)
def get_available_benchmarks() -> List[Dict[str, str]]:
"""
Get information about available benchmarks.
Returns:
List of dictionaries with benchmark information
"""
return [
{
"id": "simpleqa",
"name": "SimpleQA",
"description": "Benchmark for factual question answering",
"recommended_examples": 100,
},
{
"id": "browsecomp",
"name": "BrowseComp",
"description": "Benchmark for web browsing comprehension",
"recommended_examples": 100,
},
{
"id": "xbench_deepsearch",
"name": "xbench-DeepSearch",
"description": "Deep research and search capability evaluation",
"recommended_examples": 100,
},
]
def compare_configurations(
dataset_type: str = "simpleqa",
num_examples: int = 20,
configurations: List[Dict[str, Any]] = None,
output_dir: str = "benchmark_comparisons",
) -> Dict[str, Any]:
"""
Compare multiple search configurations on the same benchmark.
Args:
dataset_type: Type of dataset to use
num_examples: Number of examples to evaluate
configurations: List of search configurations to compare
output_dir: Directory to save results, resolved below the data
directory like ``run_benchmark``'s: a value outside it (or the
data directory itself) is replaced by ``benchmark_comparisons``
there, and ``BenchmarkOutputDirectoryError`` is raised if that
is not usable either.
Returns:
Dictionary with comparison results
"""
if not configurations:
# Default configurations to compare
configurations = [
{
"name": "Base Config",
"search_tool": "searxng",
"iterations": 1,
"questions_per_iteration": 3,
},
{
"name": "More Iterations",
"search_tool": "searxng",
"iterations": 3,
"questions_per_iteration": 3,
},
{
"name": "More Questions",
"search_tool": "searxng",
"iterations": 1,
"questions_per_iteration": 5,
},
]
# Resolve once, so the per-configuration results and the comparison
# report share one root below the data directory, also when the
# requested root is replaced. The exception is a configuration name
# that leads out of that root (for example an absolute name or ".."
# components): run_benchmark then resolves that subdirectory on its
# own, so those results land elsewhere below the data directory (in
# benchmark_results when the path leaves the data directory).
output_dir = str(
resolve_output_dir(output_dir, fallback="benchmark_comparisons")
)
# nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python benchmark API intentionally accepts an output directory; the web runner does not call this function
Path(output_dir).mkdir(parents=True, exist_ok=True)
# Run benchmarks for each configuration
results = []
for config in configurations:
config_name = config.pop("name", f"Config-{len(results)}")
logger.info(f"Running benchmark with configuration: {config_name}")
search_config = {
"iterations": config.pop("iterations", 1),
"questions_per_iteration": config.pop("questions_per_iteration", 3),
"search_tool": config.pop("search_tool", "searxng"),
}
# Add any remaining config items
search_config.update(config)
# Run benchmark with this configuration
benchmark_result = run_benchmark(
dataset_type=dataset_type,
num_examples=num_examples,
output_dir=str(Path(output_dir) / config_name.replace(" ", "_")),
search_config=search_config,
run_evaluation=True,
)
# Add configuration name to results
benchmark_result["configuration_name"] = config_name
benchmark_result["search_config"] = search_config
results.append(benchmark_result)
# Generate comparison report
import time
from ..security.file_write_verifier import write_file_verified
timestamp = time.strftime("%Y%m%d_%H%M%S")
report_file = str(
Path(output_dir) / f"comparison_{dataset_type}_{timestamp}.md"
)
# Build report content
content = f"# Configuration Comparison - {dataset_type.capitalize()}\n\n"
# Write summary table
content += "## Summary\n\n"
content += "| Configuration | Accuracy | Avg. Time | Examples |\n"
content += "|---------------|----------|-----------|----------|\n"
for result in results:
accuracy = result.get("metrics", {}).get("accuracy", 0)
avg_time = result.get("metrics", {}).get("average_processing_time", 0)
examples = result.get("total_examples", 0)
content += f"| {result['configuration_name']} | {accuracy:.3f} | {avg_time:.2f}s | {examples} |\n"
content += "\n## Configuration Details\n\n"
for result in results:
content += f"### {result['configuration_name']}\n\n"
config = result.get("search_config", {})
content += "```\n"
for key, value in config.items():
content += f"{key}: {value}\n"
content += "```\n\n"
write_file_verified(
report_file,
content,
"benchmark.allow_file_output",
context="benchmark comparison report",
)
logger.info(f"Comparison report saved to {report_file}")
return {
"status": "complete",
"dataset_type": dataset_type,
"configurations_tested": len(configurations),
"report_path": report_file,
"results": results,
}
# Export the API functions
__all__ = [
"calculate_metrics",
"compare_configurations",
"evaluate_browsecomp",
"evaluate_simpleqa",
"generate_report",
"get_available_benchmarks",
"run_benchmark", # For advanced users
]