Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

11.8 KB
Raw Blame
"""
API functions for benchmarking.

This module provides functions for running benchmarks programmatically.
"""

from loguru import logger
from pathlib import Path
from typing import Any

from ..benchmarks import (
    calculate_metrics,
    generate_report,
    run_benchmark,
    run_browsecomp_benchmark,
    run_simpleqa_benchmark,
    run_xbench_deepsearch_benchmark,
)
from ..benchmarks.runners import resolve_output_dir


def evaluate_simpleqa(
    num_examples: int = 100,
    search_iterations: int = 3,
    questions_per_iteration: int = 3,
    search_tool: str = "searxng",
    human_evaluation: bool = False,
    evaluation_model: str | None = None,
    evaluation_provider: str | None = None,
    output_dir: str = "benchmark_results",
) -> dict[str, Any]:
    """
    Run SimpleQA benchmark evaluation.

    Args:
        num_examples: Number of examples to evaluate
        search_iterations: Number of search iterations per query
        questions_per_iteration: Number of questions per iteration
        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
        human_evaluation: Whether to use human evaluation
        evaluation_model: Optional custom model for evaluation
        evaluation_provider: Optional custom provider for evaluation
        output_dir: Directory to save results

    Returns:
        Dictionary with benchmark results
    """
    logger.info(f"Starting SimpleQA benchmark with {num_examples} examples")

    # Set up search configuration
    search_config = {
        "iterations": search_iterations,
        "questions_per_iteration": questions_per_iteration,
        "search_tool": search_tool,
    }

    # Set up evaluation configuration if needed
    evaluation_config = None
    if evaluation_model or evaluation_provider:
        evaluation_config = {}
        if evaluation_model:
            evaluation_config["model_name"] = evaluation_model
        if evaluation_provider:
            evaluation_config["provider"] = evaluation_provider

    # Run the benchmark
    return run_simpleqa_benchmark(
        num_examples=num_examples,
        output_dir=output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=human_evaluation,
    )


def evaluate_browsecomp(
    num_examples: int = 100,
    search_iterations: int = 3,
    questions_per_iteration: int = 3,
    search_tool: str = "searxng",
    human_evaluation: bool = False,
    evaluation_model: str | None = None,
    evaluation_provider: str | None = None,
    output_dir: str = "benchmark_results",
) -> dict[str, Any]:
    """
    Run BrowseComp benchmark evaluation.

    Args:
        num_examples: Number of examples to evaluate
        search_iterations: Number of search iterations per query
        questions_per_iteration: Number of questions per iteration
        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
        human_evaluation: Whether to use human evaluation
        evaluation_model: Optional custom model for evaluation
        evaluation_provider: Optional custom provider for evaluation
        output_dir: Directory to save results

    Returns:
        Dictionary with benchmark results
    """
    logger.info(f"Starting BrowseComp benchmark with {num_examples} examples")

    # Set up search configuration
    search_config = {
        "iterations": search_iterations,
        "questions_per_iteration": questions_per_iteration,
        "search_tool": search_tool,
    }

    # Set up evaluation configuration if needed
    evaluation_config = None
    if evaluation_model or evaluation_provider:
        evaluation_config = {}
        if evaluation_model:
            evaluation_config["model_name"] = evaluation_model
        if evaluation_provider:
            evaluation_config["provider"] = evaluation_provider

    # Run the benchmark
    return run_browsecomp_benchmark(
        num_examples=num_examples,
        output_dir=output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=human_evaluation,
    )


def evaluate_xbench_deepsearch(
    num_examples: int = 100,
    search_iterations: int = 4,
    questions_per_iteration: int = 3,
    search_tool: str = "searxng",
    human_evaluation: bool = False,
    evaluation_model: str | None = None,
    evaluation_provider: str | None = None,
    output_dir: str = "benchmark_results",
) -> dict[str, Any]:
    """
    Run xbench-DeepSearch benchmark evaluation.

    Args:
        num_examples: Number of examples to evaluate (default 100 - full dataset)
        search_iterations: Number of search iterations per query
        questions_per_iteration: Number of questions per iteration
        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
        human_evaluation: Whether to use human evaluation
        evaluation_model: Optional custom model for evaluation
        evaluation_provider: Optional custom provider for evaluation
        output_dir: Directory to save results

    Returns:
        Dictionary with benchmark results
    """
    logger.info(
        f"Starting xbench-DeepSearch benchmark with {num_examples} examples"
    )

    # Set up search configuration
    search_config = {
        "iterations": search_iterations,
        "questions_per_iteration": questions_per_iteration,
        "search_tool": search_tool,
    }

    # Set up evaluation configuration if needed
    evaluation_config = None
    if evaluation_model or evaluation_provider:
        evaluation_config = {}
        if evaluation_model:
            evaluation_config["model_name"] = evaluation_model
        if evaluation_provider:
            evaluation_config["provider"] = evaluation_provider

    # Run the benchmark
    return run_xbench_deepsearch_benchmark(
        num_examples=num_examples,
        output_dir=output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=human_evaluation,
    )


def get_available_benchmarks() -> list[dict[str, str]]:
    """
    Get information about available benchmarks.

    Returns:
        List of dictionaries with benchmark information
    """
    return [
        {
            "id": "simpleqa",
            "name": "SimpleQA",
            "description": "Benchmark for factual question answering",
            "recommended_examples": 100,
        },
        {
            "id": "browsecomp",
            "name": "BrowseComp",
            "description": "Benchmark for web browsing comprehension",
            "recommended_examples": 100,
        },
        {
            "id": "xbench_deepsearch",
            "name": "xbench-DeepSearch",
            "description": "Benchmark for deep search and investigation queries",
            "recommended_examples": 100,
        },
    ]


def compare_configurations(
    dataset_type: str = "simpleqa",
    num_examples: int = 20,
    configurations: list[dict[str, Any]] = None,
    output_dir: str = "benchmark_comparisons",
) -> dict[str, Any]:
    """
    Compare multiple search configurations on the same benchmark.

    Args:
        dataset_type: Type of dataset to use
        num_examples: Number of examples to evaluate
        configurations: List of search configurations to compare
        output_dir: Directory to save results, resolved below the data
            directory like ``run_benchmark``'s: a value outside it (or the
            data directory itself) is replaced by ``benchmark_comparisons``
            there, and ``BenchmarkOutputDirectoryError`` is raised if that
            is not usable either.

    Returns:
        Dictionary with comparison results
    """
    if not configurations:
        # Default configurations to compare
        configurations = [
            {
                "name": "Base Config",
                "search_tool": "searxng",
                "iterations": 1,
                "questions_per_iteration": 3,
            },
            {
                "name": "More Iterations",
                "search_tool": "searxng",
                "iterations": 3,
                "questions_per_iteration": 3,
            },
            {
                "name": "More Questions",
                "search_tool": "searxng",
                "iterations": 1,
                "questions_per_iteration": 5,
            },
        ]

    # Resolve once, so the per-configuration results and the comparison
    # report share one root below the data directory, also when the
    # requested root is replaced. The exception is a configuration name
    # that leads out of that root (for example an absolute name or ".."
    # components): run_benchmark then resolves that subdirectory on its
    # own, so those results land elsewhere below the data directory (in
    # benchmark_results when the path leaves the data directory).
    output_dir = str(
        resolve_output_dir(output_dir, fallback="benchmark_comparisons")
    )
    import os

    os.makedirs(output_dir, exist_ok=True)

    # Run benchmarks for each configuration
    results = []
    for config in configurations:
        config_name = config.pop("name", f"Config-{len(results)}")

        logger.info(f"Running benchmark with configuration: {config_name}")

        search_config = {
            "iterations": config.pop("iterations", 1),
            "questions_per_iteration": config.pop("questions_per_iteration", 3),
            "search_tool": config.pop("search_tool", "searxng"),
        }

        # Add any remaining config items
        search_config.update(config)

        # Run benchmark with this configuration
        benchmark_result = run_benchmark(
            dataset_type=dataset_type,
            num_examples=num_examples,
            output_dir=str(Path(output_dir) / config_name.replace(" ", "_")),
            search_config=search_config,
            run_evaluation=True,
        )

        # Add configuration name to results
        benchmark_result["configuration_name"] = config_name
        benchmark_result["search_config"] = search_config

        results.append(benchmark_result)

    # Generate comparison report
    import time
    from ..security.file_write_verifier import write_file_verified

    timestamp = time.strftime("%Y%m%d_%H%M%S")
    report_file = str(
        Path(output_dir) / f"comparison_{dataset_type}_{timestamp}.md"
    )

    # Build report content
    content = f"# Configuration Comparison - {dataset_type.capitalize()}\n\n"

    # Write summary table
    content += "## Summary\n\n"
    content += "| Configuration | Accuracy | Avg. Time | Examples |\n"
    content += "|---------------|----------|-----------|----------|\n"

    for result in results:
        accuracy = result.get("metrics", {}).get("accuracy", 0)
        avg_time = result.get("metrics", {}).get("average_processing_time", 0)
        examples = result.get("total_examples", 0)

        content += f"| {result['configuration_name']} | {accuracy:.3f} | {avg_time:.2f}s | {examples} |\n"

    content += "\n## Configuration Details\n\n"

    for result in results:
        content += f"### {result['configuration_name']}\n\n"

        config = result.get("search_config", {})
        content += "```\n"
        for key, value in config.items():
            content += f"{key}: {value}\n"
        content += "```\n\n"

    write_file_verified(
        report_file,
        content,
        "benchmark.allow_file_output",
        context="benchmark comparison report",
    )

    logger.info(f"Comparison report saved to {report_file}")

    return {
        "status": "complete",
        "dataset_type": dataset_type,
        "configurations_tested": len(configurations),
        "report_path": report_file,
        "results": results,
    }


# Export the API functions
__all__ = [
    "calculate_metrics",
    "compare_configurations",
    "evaluate_browsecomp",
    "evaluate_simpleqa",
    "evaluate_xbench_deepsearch",
    "generate_report",
    "get_available_benchmarks",
    "run_benchmark",  # For advanced users
]