~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
API functions for benchmarking.
This module provides functions for running benchmarks programmatically.
"""
from loguru import logger
from pathlib import Path
from typing import Any
from ..benchmarks import (
calculate_metrics,
generate_report,
run_benchmark,
run_browsecomp_benchmark,
run_simpleqa_benchmark,
run_xbench_deepsearch_benchmark,
)
from ..benchmarks.runners import resolve_output_dir
def evaluate_simpleqa(
num_examples: int = 100,
search_iterations: int = 3,
questions_per_iteration: int = 3,
search_tool: str = "searxng",
human_evaluation: bool = False,
evaluation_model: str | None = None,
evaluation_provider: str | None = None,
output_dir: str = "benchmark_results",
) -> dict[str, Any]:
"""
Run SimpleQA benchmark evaluation.
Args:
num_examples: Number of examples to evaluate
search_iterations: Number of search iterations per query
questions_per_iteration: Number of questions per iteration
search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
human_evaluation: Whether to use human evaluation
evaluation_model: Optional custom model for evaluation
evaluation_provider: Optional custom provider for evaluation
output_dir: Directory to save results
Returns:
Dictionary with benchmark results
"""
logger.info(f"Starting SimpleQA benchmark with {num_examples} examples")
# Set up search configuration
search_config = {
"iterations": search_iterations,
"questions_per_iteration": questions_per_iteration,
"search_tool": search_tool,
}
# Set up evaluation configuration if needed
evaluation_config = None
if evaluation_model or evaluation_provider:
evaluation_config = {}
if evaluation_model:
evaluation_config["model_name"] = evaluation_model
if evaluation_provider:
evaluation_config["provider"] = evaluation_provider
# Run the benchmark
return run_simpleqa_benchmark(
num_examples=num_examples,
output_dir=output_dir,
search_config=search_config,
evaluation_config=evaluation_config,
human_evaluation=human_evaluation,
)
def evaluate_browsecomp(
num_examples: int = 100,
search_iterations: int = 3,
questions_per_iteration: int = 3,
search_tool: str = "searxng",
human_evaluation: bool = False,
evaluation_model: str | None = None,
evaluation_provider: str | None = None,
output_dir: str = "benchmark_results",
) -> dict[str, Any]:
"""
Run BrowseComp benchmark evaluation.
Args:
num_examples: Number of examples to evaluate
search_iterations: Number of search iterations per query
questions_per_iteration: Number of questions per iteration
search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
human_evaluation: Whether to use human evaluation
evaluation_model: Optional custom model for evaluation
evaluation_provider: Optional custom provider for evaluation
output_dir: Directory to save results
Returns:
Dictionary with benchmark results
"""
logger.info(f"Starting BrowseComp benchmark with {num_examples} examples")
# Set up search configuration
search_config = {
"iterations": search_iterations,
"questions_per_iteration": questions_per_iteration,
"search_tool": search_tool,
}
# Set up evaluation configuration if needed
evaluation_config = None
if evaluation_model or evaluation_provider:
evaluation_config = {}
if evaluation_model:
evaluation_config["model_name"] = evaluation_model
if evaluation_provider:
evaluation_config["provider"] = evaluation_provider
# Run the benchmark
return run_browsecomp_benchmark(
num_examples=num_examples,
output_dir=output_dir,
search_config=search_config,
evaluation_config=evaluation_config,
human_evaluation=human_evaluation,
)
def evaluate_xbench_deepsearch(
num_examples: int = 100,
search_iterations: int = 4,
questions_per_iteration: int = 3,
search_tool: str = "searxng",
human_evaluation: bool = False,
evaluation_model: str | None = None,
evaluation_provider: str | None = None,
output_dir: str = "benchmark_results",
) -> dict[str, Any]:
"""
Run xbench-DeepSearch benchmark evaluation.
Args:
num_examples: Number of examples to evaluate (default 100 - full dataset)
search_iterations: Number of search iterations per query
questions_per_iteration: Number of questions per iteration
search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
human_evaluation: Whether to use human evaluation
evaluation_model: Optional custom model for evaluation
evaluation_provider: Optional custom provider for evaluation
output_dir: Directory to save results
Returns:
Dictionary with benchmark results
"""
logger.info(
f"Starting xbench-DeepSearch benchmark with {num_examples} examples"
)
# Set up search configuration
search_config = {
"iterations": search_iterations,
"questions_per_iteration": questions_per_iteration,
"search_tool": search_tool,
}
# Set up evaluation configuration if needed
evaluation_config = None
if evaluation_model or evaluation_provider:
evaluation_config = {}
if evaluation_model:
evaluation_config["model_name"] = evaluation_model
if evaluation_provider:
evaluation_config["provider"] = evaluation_provider
# Run the benchmark
return run_xbench_deepsearch_benchmark(
num_examples=num_examples,
output_dir=output_dir,
search_config=search_config,
evaluation_config=evaluation_config,
human_evaluation=human_evaluation,
)
def get_available_benchmarks() -> list[dict[str, str]]:
"""
Get information about available benchmarks.
Returns:
List of dictionaries with benchmark information
"""
return [
{
"id": "simpleqa",
"name": "SimpleQA",
"description": "Benchmark for factual question answering",
"recommended_examples": 100,
},
{
"id": "browsecomp",
"name": "BrowseComp",
"description": "Benchmark for web browsing comprehension",
"recommended_examples": 100,
},
{
"id": "xbench_deepsearch",
"name": "xbench-DeepSearch",
"description": "Benchmark for deep search and investigation queries",
"recommended_examples": 100,
},
]
def compare_configurations(
dataset_type: str = "simpleqa",
num_examples: int = 20,
configurations: list[dict[str, Any]] = None,
output_dir: str = "benchmark_comparisons",
) -> dict[str, Any]:
"""
Compare multiple search configurations on the same benchmark.
Args:
dataset_type: Type of dataset to use
num_examples: Number of examples to evaluate
configurations: List of search configurations to compare
output_dir: Directory to save results, resolved below the data
directory like ``run_benchmark``'s: a value outside it (or the
data directory itself) is replaced by ``benchmark_comparisons``
there, and ``BenchmarkOutputDirectoryError`` is raised if that
is not usable either.
Returns:
Dictionary with comparison results
"""
if not configurations:
# Default configurations to compare
configurations = [
{
"name": "Base Config",
"search_tool": "searxng",
"iterations": 1,
"questions_per_iteration": 3,
},
{
"name": "More Iterations",
"search_tool": "searxng",
"iterations": 3,
"questions_per_iteration": 3,
},
{
"name": "More Questions",
"search_tool": "searxng",
"iterations": 1,
"questions_per_iteration": 5,
},
]
# Resolve once, so the per-configuration results and the comparison
# report share one root below the data directory, also when the
# requested root is replaced. The exception is a configuration name
# that leads out of that root (for example an absolute name or ".."
# components): run_benchmark then resolves that subdirectory on its
# own, so those results land elsewhere below the data directory (in
# benchmark_results when the path leaves the data directory).
output_dir = str(
resolve_output_dir(output_dir, fallback="benchmark_comparisons")
)
import os
os.makedirs(output_dir, exist_ok=True)
# Run benchmarks for each configuration
results = []
for config in configurations:
config_name = config.pop("name", f"Config-{len(results)}")
logger.info(f"Running benchmark with configuration: {config_name}")
search_config = {
"iterations": config.pop("iterations", 1),
"questions_per_iteration": config.pop("questions_per_iteration", 3),
"search_tool": config.pop("search_tool", "searxng"),
}
# Add any remaining config items
search_config.update(config)
# Run benchmark with this configuration
benchmark_result = run_benchmark(
dataset_type=dataset_type,
num_examples=num_examples,
output_dir=str(Path(output_dir) / config_name.replace(" ", "_")),
search_config=search_config,
run_evaluation=True,
)
# Add configuration name to results
benchmark_result["configuration_name"] = config_name
benchmark_result["search_config"] = search_config
results.append(benchmark_result)
# Generate comparison report
import time
from ..security.file_write_verifier import write_file_verified
timestamp = time.strftime("%Y%m%d_%H%M%S")
report_file = str(
Path(output_dir) / f"comparison_{dataset_type}_{timestamp}.md"
)
# Build report content
content = f"# Configuration Comparison - {dataset_type.capitalize()}\n\n"
# Write summary table
content += "## Summary\n\n"
content += "| Configuration | Accuracy | Avg. Time | Examples |\n"
content += "|---------------|----------|-----------|----------|\n"
for result in results:
accuracy = result.get("metrics", {}).get("accuracy", 0)
avg_time = result.get("metrics", {}).get("average_processing_time", 0)
examples = result.get("total_examples", 0)
content += f"| {result['configuration_name']} | {accuracy:.3f} | {avg_time:.2f}s | {examples} |\n"
content += "\n## Configuration Details\n\n"
for result in results:
content += f"### {result['configuration_name']}\n\n"
config = result.get("search_config", {})
content += "```\n"
for key, value in config.items():
content += f"{key}: {value}\n"
content += "```\n\n"
write_file_verified(
report_file,
content,
"benchmark.allow_file_output",
context="benchmark comparison report",
)
logger.info(f"Comparison report saved to {report_file}")
return {
"status": "complete",
"dataset_type": dataset_type,
"configurations_tested": len(configurations),
"report_path": report_file,
"results": results,
}
# Export the API functions
__all__ = [
"calculate_metrics",
"compare_configurations",
"evaluate_browsecomp",
"evaluate_simpleqa",
"evaluate_xbench_deepsearch",
"generate_report",
"get_available_benchmarks",
"run_benchmark", # For advanced users
]