Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

18.3 KB
Raw Blame
"""
API functions for benchmarking.

This module provides functions for running benchmarks programmatically.
"""

from pathlib import Path
from typing import Any, Dict, List, Optional

from loguru import logger

from ..config.thread_settings import get_setting_from_snapshot
from .graders import normalize_grader_provider

from ..benchmarks import (
    calculate_metrics,
    generate_report,
    run_benchmark,
    run_browsecomp_benchmark,
    run_simpleqa_benchmark,
    run_xbench_deepsearch_benchmark,
)
from .runners import resolve_output_dir


def evaluate_simpleqa(
    num_examples: int = 100,
    search_iterations: int = 3,
    questions_per_iteration: int = 3,
    search_tool: str = "searxng",
    human_evaluation: bool = False,
    evaluation_model: Optional[str] = None,
    evaluation_provider: Optional[str] = None,
    output_dir: str = "benchmark_results",
    search_model: Optional[str] = None,
    search_provider: Optional[str] = None,
    endpoint_url: Optional[str] = None,
    search_strategy: str = "source_based",
    evaluation_api_key: Optional[str] = None,
) -> Dict[str, Any]:
    """
    Run SimpleQA benchmark evaluation.

    Args:
        num_examples: Number of examples to evaluate
        search_iterations: Number of search iterations per query
        questions_per_iteration: Number of questions per iteration
        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
        human_evaluation: Whether to use human evaluation
        evaluation_model: Optional custom model for evaluation
        evaluation_provider: Optional custom provider for evaluation
        output_dir: Directory to save results
        search_model: Optional model to use for the search system
        search_provider: Optional provider to use for the search system
        endpoint_url: Optional endpoint URL for OpenRouter or other API services
        search_strategy: Search strategy to use (default: 'source_based')
        evaluation_api_key: Optional API key for the grader's endpoint. It is
            only used for grading, with the grader endpoint of this run,
            and only when the evaluation provider is ``openai_endpoint``;
            other providers ignore it.

    Returns:
        Dictionary with benchmark results
    """
    logger.info(f"Starting SimpleQA benchmark with {num_examples} examples")

    # Set up search configuration
    search_config = {
        "iterations": search_iterations,
        "questions_per_iteration": questions_per_iteration,
        "search_tool": search_tool,
        "search_strategy": search_strategy,
    }

    # Add model configurations if provided
    if search_model:
        search_config["model_name"] = search_model
    if search_provider:
        search_config["provider"] = search_provider
    if endpoint_url:
        search_config["openai_endpoint_url"] = endpoint_url

    # Check settings for additional configuration
    if env_model := get_setting_from_snapshot("llm.model"):
        search_config["model_name"] = env_model
    if env_provider := get_setting_from_snapshot("llm.provider"):
        search_config["provider"] = env_provider
    if env_url := get_setting_from_snapshot("llm.openai_endpoint.url"):
        search_config["openai_endpoint_url"] = env_url

    # Set up evaluation configuration if needed
    evaluation_config = None
    if evaluation_model or evaluation_provider:
        evaluation_config = {
            "temperature": 0  # Always use zero temperature for evaluation
        }
        if evaluation_model:
            evaluation_config["model_name"] = evaluation_model
        if evaluation_provider:
            evaluation_provider = normalize_grader_provider(evaluation_provider)
            evaluation_config["provider"] = evaluation_provider
            # Add endpoint URL if using openai_endpoint
            if evaluation_provider == "openai_endpoint" and endpoint_url:
                evaluation_config["openai_endpoint_url"] = endpoint_url
            elif evaluation_provider == "openai_endpoint" and env_url:
                evaluation_config["openai_endpoint_url"] = env_url
    if evaluation_api_key:
        evaluation_config = dict(evaluation_config or {"temperature": 0})
        evaluation_config["api_key"] = evaluation_api_key

    # Run the benchmark
    return run_simpleqa_benchmark(
        num_examples=num_examples,
        output_dir=output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=human_evaluation,
    )


def evaluate_browsecomp(
    num_examples: int = 100,
    search_iterations: int = 3,
    questions_per_iteration: int = 3,
    search_tool: str = "searxng",
    human_evaluation: bool = False,
    evaluation_model: Optional[str] = None,
    evaluation_provider: Optional[str] = None,
    output_dir: str = "benchmark_results",
    search_model: Optional[str] = None,
    search_provider: Optional[str] = None,
    endpoint_url: Optional[str] = None,
    search_strategy: str = "source_based",
    evaluation_api_key: Optional[str] = None,
) -> Dict[str, Any]:
    """
    Run BrowseComp benchmark evaluation.

    Args:
        num_examples: Number of examples to evaluate
        search_iterations: Number of search iterations per query
        questions_per_iteration: Number of questions per iteration
        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
        human_evaluation: Whether to use human evaluation
        evaluation_model: Optional custom model for evaluation
        evaluation_provider: Optional custom provider for evaluation
        output_dir: Directory to save results
        search_model: Optional model to use for the search system
        search_provider: Optional provider to use for the search system
        endpoint_url: Optional endpoint URL for OpenRouter or other API services
        search_strategy: Search strategy to use (default: 'source_based')
        evaluation_api_key: Optional API key for the grader's endpoint. It is
            only used for grading, with the grader endpoint of this run,
            and only when the evaluation provider is ``openai_endpoint``;
            other providers ignore it.

    Returns:
        Dictionary with benchmark results
    """
    logger.info(f"Starting BrowseComp benchmark with {num_examples} examples")

    # Set up search configuration
    search_config = {
        "iterations": search_iterations,
        "questions_per_iteration": questions_per_iteration,
        "search_tool": search_tool,
        "search_strategy": search_strategy,
    }

    # Add model configurations if provided
    if search_model:
        search_config["model_name"] = search_model
    if search_provider:
        search_config["provider"] = search_provider
    if endpoint_url:
        search_config["openai_endpoint_url"] = endpoint_url

    # Check settings for additional configuration
    if env_model := get_setting_from_snapshot("llm.model"):
        search_config["model_name"] = env_model
    if env_provider := get_setting_from_snapshot("llm.provider"):
        search_config["provider"] = env_provider
    if env_url := get_setting_from_snapshot("llm.openai_endpoint.url"):
        search_config["openai_endpoint_url"] = env_url

    # Set up evaluation configuration if needed
    evaluation_config = None
    if evaluation_model or evaluation_provider:
        evaluation_config = {
            "temperature": 0  # Always use zero temperature for evaluation
        }
        if evaluation_model:
            evaluation_config["model_name"] = evaluation_model
        if evaluation_provider:
            evaluation_provider = normalize_grader_provider(evaluation_provider)
            evaluation_config["provider"] = evaluation_provider
            # Add endpoint URL if using openai_endpoint
            if evaluation_provider == "openai_endpoint" and endpoint_url:
                evaluation_config["openai_endpoint_url"] = endpoint_url
            elif evaluation_provider == "openai_endpoint" and env_url:
                evaluation_config["openai_endpoint_url"] = env_url
    if evaluation_api_key:
        evaluation_config = dict(evaluation_config or {"temperature": 0})
        evaluation_config["api_key"] = evaluation_api_key

    # Run the benchmark
    return run_browsecomp_benchmark(
        num_examples=num_examples,
        output_dir=output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=human_evaluation,
    )


def evaluate_xbench_deepsearch(
    num_examples: int = 100,
    search_iterations: int = 4,
    questions_per_iteration: int = 3,
    search_tool: str = "searxng",
    human_evaluation: bool = False,
    evaluation_model: Optional[str] = None,
    evaluation_provider: Optional[str] = None,
    output_dir: str = "benchmark_results",
    search_model: Optional[str] = None,
    search_provider: Optional[str] = None,
    endpoint_url: Optional[str] = None,
    search_strategy: str = "source_based",
    evaluation_api_key: Optional[str] = None,
) -> Dict[str, Any]:
    """
    Run xbench-DeepSearch benchmark evaluation.

    Args:
        num_examples: Number of examples to evaluate (default 100 - full dataset)
        search_iterations: Number of search iterations per query
        questions_per_iteration: Number of questions per iteration
        search_tool: Search engine to use
        human_evaluation: Whether to use human evaluation
        evaluation_model: Optional custom model for evaluation
        evaluation_provider: Optional custom provider for evaluation
        output_dir: Directory to save results
        search_model: Optional model to use for the search system
        search_provider: Optional provider to use for the search system
        endpoint_url: Optional endpoint URL for API services
        search_strategy: Search strategy to use
        evaluation_api_key: Optional API key for the grader's endpoint. It is
            only used for grading, with the grader endpoint of this run,
            and only when the evaluation provider is ``openai_endpoint``;
            other providers ignore it.

    Returns:
        Dictionary with benchmark results
    """
    logger.info(
        f"Starting xbench-DeepSearch benchmark with {num_examples} examples"
    )

    # Set up search configuration
    search_config = {
        "iterations": search_iterations,
        "questions_per_iteration": questions_per_iteration,
        "search_tool": search_tool,
        "search_strategy": search_strategy,
    }

    # Add model configurations if provided
    if search_model:
        search_config["model_name"] = search_model
    if search_provider:
        search_config["provider"] = search_provider
    if endpoint_url:
        search_config["openai_endpoint_url"] = endpoint_url

    # Check settings for additional configuration
    if env_model := get_setting_from_snapshot("llm.model"):
        search_config["model_name"] = env_model
    if env_provider := get_setting_from_snapshot("llm.provider"):
        search_config["provider"] = env_provider
    if env_url := get_setting_from_snapshot("llm.openai_endpoint.url"):
        search_config["openai_endpoint_url"] = env_url

    # Set up evaluation configuration if needed
    evaluation_config = None
    if evaluation_model or evaluation_provider:
        evaluation_config = {
            "temperature": 0  # Always use zero temperature for evaluation
        }
        if evaluation_model:
            evaluation_config["model_name"] = evaluation_model
        if evaluation_provider:
            evaluation_provider = normalize_grader_provider(evaluation_provider)
            evaluation_config["provider"] = evaluation_provider
            # Add endpoint URL if using openai_endpoint
            if evaluation_provider == "openai_endpoint" and endpoint_url:
                evaluation_config["openai_endpoint_url"] = endpoint_url
            elif evaluation_provider == "openai_endpoint" and env_url:
                evaluation_config["openai_endpoint_url"] = env_url
    if evaluation_api_key:
        evaluation_config = dict(evaluation_config or {"temperature": 0})
        evaluation_config["api_key"] = evaluation_api_key

    # Run the benchmark
    return run_xbench_deepsearch_benchmark(
        num_examples=num_examples,
        output_dir=output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=human_evaluation,
    )


def get_available_benchmarks() -> List[Dict[str, str]]:
    """
    Get information about available benchmarks.

    Returns:
        List of dictionaries with benchmark information
    """
    return [
        {
            "id": "simpleqa",
            "name": "SimpleQA",
            "description": "Benchmark for factual question answering",
            "recommended_examples": 100,
        },
        {
            "id": "browsecomp",
            "name": "BrowseComp",
            "description": "Benchmark for web browsing comprehension",
            "recommended_examples": 100,
        },
        {
            "id": "xbench_deepsearch",
            "name": "xbench-DeepSearch",
            "description": "Deep research and search capability evaluation",
            "recommended_examples": 100,
        },
    ]


def compare_configurations(
    dataset_type: str = "simpleqa",
    num_examples: int = 20,
    configurations: List[Dict[str, Any]] = None,
    output_dir: str = "benchmark_comparisons",
) -> Dict[str, Any]:
    """
    Compare multiple search configurations on the same benchmark.

    Args:
        dataset_type: Type of dataset to use
        num_examples: Number of examples to evaluate
        configurations: List of search configurations to compare
        output_dir: Directory to save results, resolved below the data
            directory like ``run_benchmark``'s: a value outside it (or the
            data directory itself) is replaced by ``benchmark_comparisons``
            there, and ``BenchmarkOutputDirectoryError`` is raised if that
            is not usable either.

    Returns:
        Dictionary with comparison results
    """
    if not configurations:
        # Default configurations to compare
        configurations = [
            {
                "name": "Base Config",
                "search_tool": "searxng",
                "iterations": 1,
                "questions_per_iteration": 3,
            },
            {
                "name": "More Iterations",
                "search_tool": "searxng",
                "iterations": 3,
                "questions_per_iteration": 3,
            },
            {
                "name": "More Questions",
                "search_tool": "searxng",
                "iterations": 1,
                "questions_per_iteration": 5,
            },
        ]

    # Resolve once, so the per-configuration results and the comparison
    # report share one root below the data directory, also when the
    # requested root is replaced. The exception is a configuration name
    # that leads out of that root (for example an absolute name or ".."
    # components): run_benchmark then resolves that subdirectory on its
    # own, so those results land elsewhere below the data directory (in
    # benchmark_results when the path leaves the data directory).
    output_dir = str(
        resolve_output_dir(output_dir, fallback="benchmark_comparisons")
    )
    # nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python benchmark API intentionally accepts an output directory; the web runner does not call this function
    Path(output_dir).mkdir(parents=True, exist_ok=True)

    # Run benchmarks for each configuration
    results = []
    for config in configurations:
        config_name = config.pop("name", f"Config-{len(results)}")

        logger.info(f"Running benchmark with configuration: {config_name}")

        search_config = {
            "iterations": config.pop("iterations", 1),
            "questions_per_iteration": config.pop("questions_per_iteration", 3),
            "search_tool": config.pop("search_tool", "searxng"),
        }

        # Add any remaining config items
        search_config.update(config)

        # Run benchmark with this configuration
        benchmark_result = run_benchmark(
            dataset_type=dataset_type,
            num_examples=num_examples,
            output_dir=str(Path(output_dir) / config_name.replace(" ", "_")),
            search_config=search_config,
            run_evaluation=True,
        )

        # Add configuration name to results
        benchmark_result["configuration_name"] = config_name
        benchmark_result["search_config"] = search_config

        results.append(benchmark_result)

    # Generate comparison report
    import time

    from ..security.file_write_verifier import write_file_verified

    timestamp = time.strftime("%Y%m%d_%H%M%S")
    report_file = str(
        Path(output_dir) / f"comparison_{dataset_type}_{timestamp}.md"
    )

    # Build report content
    content = f"# Configuration Comparison - {dataset_type.capitalize()}\n\n"

    # Write summary table
    content += "## Summary\n\n"
    content += "| Configuration | Accuracy | Avg. Time | Examples |\n"
    content += "|---------------|----------|-----------|----------|\n"

    for result in results:
        accuracy = result.get("metrics", {}).get("accuracy", 0)
        avg_time = result.get("metrics", {}).get("average_processing_time", 0)
        examples = result.get("total_examples", 0)

        content += f"| {result['configuration_name']} | {accuracy:.3f} | {avg_time:.2f}s | {examples} |\n"

    content += "\n## Configuration Details\n\n"

    for result in results:
        content += f"### {result['configuration_name']}\n\n"

        config = result.get("search_config", {})
        content += "```\n"
        for key, value in config.items():
            content += f"{key}: {value}\n"
        content += "```\n\n"

    write_file_verified(
        report_file,
        content,
        "benchmark.allow_file_output",
        context="benchmark comparison report",
    )

    logger.info(f"Comparison report saved to {report_file}")

    return {
        "status": "complete",
        "dataset_type": dataset_type,
        "configurations_tested": len(configurations),
        "report_path": report_file,
        "results": results,
    }


# Export the API functions
__all__ = [
    "calculate_metrics",
    "compare_configurations",
    "evaluate_browsecomp",
    "evaluate_simpleqa",
    "generate_report",
    "get_available_benchmarks",
    "run_benchmark",  # For advanced users
]