Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

21.6 KB
Raw Blame
"""
Benchmark runners for Local Deep Research.

This module provides the main functions for running benchmarks using LDR.
"""

import json
from loguru import logger
import os
from pathlib import Path
import time
from typing import Any, Callable, Dict, Optional

from ..api import quick_summary
from .datasets import DEFAULT_DATASET_URLS, load_dataset
from .datasets.base import DatasetRegistry
from .graders import (
    check_grader_endpoint_key,
    extract_answer_from_response,
    grade_results,
)
from .metrics import calculate_metrics, generate_report
from .templates import BROWSECOMP_QUERY_TEMPLATE


def format_query(question: str, dataset_type: str = "simpleqa") -> str:
    """
    Format query based on dataset type.

    Args:
        question: Original question
        dataset_type: Type of dataset

    Returns:
        Formatted query for LDR
    """
    if dataset_type.lower() == "browsecomp":
        # BrowseComp requires specific formatting
        return BROWSECOMP_QUERY_TEMPLATE.format(question=question)

    # Simple format for SimpleQA
    return question


class BenchmarkOutputDirectoryError(ValueError):
    """No benchmark output directory below the data directory is usable."""


def _below_data_directory(value: str, data_dir: Path) -> Optional[Path]:
    """Resolve ``value`` and return it if it is strictly below ``data_dir``.

    Returns None for a value that resolves to ``data_dir`` itself, outside
    it, or cannot be resolved.
    """
    from ..security.path_validator import resolve_allow_missing

    try:
        path = Path(os.path.expandvars(str(value))).expanduser()
        if not path.is_absolute():
            path = data_dir / path
        resolved = resolve_allow_missing(path)
    except (OSError, RuntimeError, ValueError):
        return None
    if resolved != data_dir and resolved.is_relative_to(data_dir):
        return resolved
    return None


def resolve_output_dir(output_dir: Optional[str], *, fallback: str) -> Path:
    """Return the benchmark output directory, resolved below the data dir.

    ``$VAR`` and ``~`` are expanded, and a relative value is taken relative
    to the data directory (``LDR_DATA_DIR`` or the platform default), not
    the working directory. ``None``, ``""`` and ``"."`` select ``fallback``.
    Any other value that resolves outside the data directory, to the data
    directory itself, or cannot be resolved, is replaced by ``fallback``
    (relative to the data directory) and a warning is logged.

    Raises:
        BenchmarkOutputDirectoryError: ``fallback`` is not usable either, for
            example because ``<data dir>/<fallback>`` is a symlink to
            another disk. Benchmark files are never written into the data
            directory root.
        ValueError, OSError: ``LDR_DATA_DIR`` is invalid or the data
            directory cannot be resolved.

    Symlinks are followed with ``resolve_allow_missing`` before the
    comparison, so a link that leaves the data directory, a symlink loop and
    a chain whose resolved form outgrows PATH_MAX are not accepted. The
    check is a snapshot: it does not protect against a concurrent change to
    the filesystem.
    """
    # Lazy imports: path_validator imports config.paths at module level.
    from ..config.paths import get_data_directory
    from ..security.path_validator import resolve_allow_missing

    data_dir = resolve_allow_missing(get_data_directory())
    if output_dir is not None and Path(str(output_dir)) != Path("."):
        resolved = _below_data_directory(output_dir, data_dir)
        if resolved is not None:
            return resolved
        logger.warning(
            "Benchmark output directory is not below the data directory or "
            f"cannot be resolved; using the default {fallback!r} there"
        )
    resolved = _below_data_directory(fallback, data_dir)
    if resolved is not None:
        return resolved
    raise BenchmarkOutputDirectoryError(
        "No benchmark output directory below the data directory is usable: "
        f"neither the requested directory nor the default {fallback!r} "
        "resolves to a directory inside it. Choose an output directory "
        "inside the data directory, set LDR_DATA_DIR to a data directory "
        "that contains the location you want, or remove the symlink that "
        f"points {fallback!r} outside the data directory."
    )


def run_benchmark(
    dataset_type: str,
    dataset_path: Optional[str] = None,
    num_examples: Optional[int] = None,
    output_dir: str = "benchmark_results",
    run_evaluation: bool = True,
    evaluation_config: Optional[Dict[str, Any]] = None,
    search_config: Optional[Dict[str, Any]] = None,
    human_evaluation: bool = False,
    progress_callback: Optional[Callable[[str, int, Dict], None]] = None,
    seed: int = 42,
    settings_snapshot: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
    """
    Run a benchmark on the specified dataset.

    Args:
        dataset_type: Type of dataset ("simpleqa" or "browsecomp")
        dataset_path: Optional custom dataset path
        num_examples: Number of examples to use
        output_dir: Directory to save results. It is resolved below the
            data directory by :func:`resolve_output_dir`: a relative value
            is relative to the data directory, and a value outside it (or
            the data directory itself) is replaced by ``benchmark_results``
            there. Raises ``BenchmarkOutputDirectoryError`` before any
            work when ``benchmark_results`` is not usable either.
        run_evaluation: Whether to evaluate results
        evaluation_config: Custom LLM config for evaluation
        search_config: Custom search parameters
        human_evaluation: Whether to use human evaluation
        progress_callback: Optional callback for progress updates
        seed: Random seed for reproducibility

    Returns:
        Dictionary with benchmark results and metrics. The returned file
        paths are inside the resolved output directory.
    """
    # Refuse up front when the grader would have no API key for its
    # endpoint, before any research budget is spent. Raises
    # LLMNotConfiguredError naming benchmark.evaluation.api_key.
    if run_evaluation and not human_evaluation:
        check_grader_endpoint_key(evaluation_config, settings_snapshot)

    output_dir = str(
        resolve_output_dir(output_dir, fallback="benchmark_results")
    )

    # Ensure output directory exists
    os.makedirs(output_dir, exist_ok=True)

    # Default search configuration
    if not search_config:
        search_config = {
            "iterations": 3,
            "questions_per_iteration": 3,
            "search_tool": "searxng",
        }

    # Load dataset using the class-based approach
    try:
        # Create the dataset instance from registry
        dataset_instance = DatasetRegistry.create_dataset(
            dataset_id=dataset_type.lower(),
            dataset_path=dataset_path,
            num_examples=num_examples,
            seed=seed,
        )
        # Load the examples
        dataset = dataset_instance.load()

        logger.info(
            f"Loaded {len(dataset)} examples using dataset class {type(dataset_instance).__name__}"
        )
    except Exception:
        # Fallback to legacy function if there's any issue
        logger.warning(
            "Error using dataset class. Falling back to legacy function."
        )
        # This is the local benchmark registry factory, not a Hugging Face download API.
        dataset = load_dataset(  # nosec B615
            dataset_type=dataset_type,
            dataset_path=dataset_path,
            num_examples=num_examples,
            seed=seed,
        )

    # Set up output files
    timestamp = time.strftime("%Y%m%d_%H%M%S")
    results_file = str(
        Path(output_dir) / f"{dataset_type}_{timestamp}_results.jsonl"
    )
    evaluation_file = str(
        Path(output_dir) / f"{dataset_type}_{timestamp}_evaluation.jsonl"
    )
    report_file = str(
        Path(output_dir) / f"{dataset_type}_{timestamp}_report.md"
    )

    # Make sure output files don't exist
    for file in [results_file, evaluation_file, report_file]:
        file_path = Path(file)
        if file_path.exists():
            # nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python benchmark output paths are selected by the invoking program; web benchmarks use the separate database runner
            file_path.unlink()

    # Progress tracking
    total_examples = len(dataset)

    if progress_callback:
        progress_callback(
            "Starting benchmark",
            0,
            {
                "status": "started",
                "dataset_type": dataset_type,
                "total_examples": total_examples,
            },
        )

    # Process each example
    results = []

    for i, example in enumerate(dataset):
        # Extract question and answer in a way that uses the dataset class when available
        if "dataset_instance" in locals() and isinstance(
            dataset_instance,
            DatasetRegistry.get_dataset_class(dataset_type.lower()),
        ):
            # Use the dataset class methods to extract question and answer
            question = dataset_instance.get_question(example)
            correct_answer = dataset_instance.get_answer(example)
            logger.debug(
                "Using dataset class methods to extract question and answer"
            )
        else:
            # Fallback to the legacy approach
            if dataset_type.lower() == "simpleqa":
                question = example.get("problem", "")
                correct_answer = example.get("answer", "")
            else:  # browsecomp
                question = example.get("problem", "")
                # For BrowseComp, the answer should be in "correct_answer" after decryption
                correct_answer = example.get("correct_answer", "")
                if not correct_answer and "answer" in example:
                    # Fallback to "answer" field if "correct_answer" is not available
                    correct_answer = example.get("answer", "")

        # Update progress
        if progress_callback:
            progress_callback(
                f"Processing example {i + 1}/{total_examples}",
                int(i / total_examples * 50),
                {
                    "status": "processing",
                    "current": i + 1,
                    "total": total_examples,
                    "question": (
                        question[:50] + "..."
                        if len(question) > 50
                        else question
                    ),
                },
            )

        logger.info(f"Processing {i + 1}/{total_examples}: {question[:50]}...")

        try:
            # Format query based on dataset type
            formatted_query = format_query(question, dataset_type)

            # Time the search
            start_time = time.time()

            # Get response from LDR. Pass settings_snapshot through so
            # the egress policy PEP fires on the benchmark's inner
            # search-engine / LLM construction calls. Without it, the
            # PEP's ``if settings_snapshot is not None`` guard skips and
            # benchmarks silently bypass the user's saved scope.
            search_result = quick_summary(
                query=formatted_query,
                iterations=search_config.get("iterations", 3),
                questions_per_iteration=search_config.get(
                    "questions_per_iteration", 3
                ),
                search_tool=search_config.get("search_tool", "searxng"),
                settings_snapshot=settings_snapshot,
            )

            end_time = time.time()
            processing_time = end_time - start_time

            # Extract response and search info
            response = search_result.get("summary", "")

            # Extract structured information
            extracted = extract_answer_from_response(response, dataset_type)

            # Format result
            result = {
                "id": example.get("id", f"example_{i}"),
                "problem": question,
                "correct_answer": correct_answer,
                "response": response,
                "extracted_answer": extracted["extracted_answer"],
                "confidence": extracted["confidence"],
                "processing_time": processing_time,
                "sources": search_result.get("sources", []),
                "search_config": search_config,
            }

            # Add to results list
            results.append(result)

            # Write result to file
            # nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python benchmark result file generated under its caller-selected output root
            with open(results_file, "a", encoding="utf-8") as f:
                f.write(json.dumps(result) + "\n")

            # Update progress
            if progress_callback:
                progress_callback(
                    f"Completed example {i + 1}/{total_examples}",
                    int((i + 0.5) / total_examples * 50),
                    {
                        "status": "completed_example",
                        "current": i + 1,
                        "total": total_examples,
                        "result": result,
                    },
                )

        except Exception as e:
            logger.exception(f"Error processing example {i + 1}")

            # Create error result
            error_result = {
                "id": example.get("id", f"example_{i}"),
                "problem": question,
                "correct_answer": correct_answer,
                "error": str(e),
                "processing_time": (
                    time.time() - start_time if "start_time" in locals() else 0
                ),
            }

            # Add to results list
            results.append(error_result)

            # Write error result to file
            # nosemgrep: semgrep.rules.path-traversal-risk, reason: Same generated benchmark result file used by the success path above
            with open(results_file, "a", encoding="utf-8") as f:
                f.write(json.dumps(error_result) + "\n")

            # Update progress
            if progress_callback:
                progress_callback(
                    f"Error processing example {i + 1}/{total_examples}",
                    int((i + 0.5) / total_examples * 50),
                    {
                        "status": "error",
                        "current": i + 1,
                        "total": total_examples,
                        "error": str(e),
                        "result": error_result,
                    },
                )

    logger.info(f"Completed processing {total_examples} examples")

    # Run evaluation if requested
    if run_evaluation:
        if progress_callback:
            progress_callback(
                "Starting evaluation",
                50,
                {"status": "evaluating", "results_file": results_file},
            )

        if human_evaluation:
            from .graders import human_evaluation as evaluate

            logger.info("Running human evaluation...")
            evaluation_results = evaluate(
                results_file=results_file,
                output_file=evaluation_file,
                interactive=True,
            )
        else:
            logger.info("Running automated evaluation...")
            try:
                evaluation_results = grade_results(
                    results_file=results_file,
                    output_file=evaluation_file,
                    dataset_type=dataset_type,
                    evaluation_config=evaluation_config,
                    settings_snapshot=settings_snapshot,
                    progress_callback=lambda current, total, meta: (
                        progress_callback(
                            f"Evaluating {current + 1}/{total}",
                            50 + int((current + 0.5) / total * 40),
                            {**meta, "status": "evaluating"},
                        )
                        if progress_callback
                        else None
                    ),
                )
            except Exception as e:
                logger.exception("Automated evaluation failed")

                if progress_callback:
                    progress_callback(
                        "Automated evaluation failed. Falling back to human evaluation.",
                        60,
                        {"status": "evaluation_fallback", "error": str(e)},
                    )

                # Ask if user wants to fall back to human evaluation
                fallback_to_human = False
                print("\nAutomated evaluation failed with error:", str(e))
                response = input(
                    "Do you want to fall back to human evaluation? (y/n): "
                )
                fallback_to_human = response.strip().lower() == "y"

                if fallback_to_human:
                    logger.info("Falling back to human evaluation...")
                    from .graders import human_evaluation as evaluate

                    evaluation_results = evaluate(
                        results_file=results_file,
                        output_file=evaluation_file,
                        interactive=True,
                    )
                else:
                    from ..security.file_write_verifier import (
                        write_file_verified,
                    )

                    logger.info("Skipping evaluation due to error.")
                    # Create an empty evaluation file to prevent issues
                    write_file_verified(
                        evaluation_file,
                        "",
                        "benchmark.allow_file_output",
                        context="empty evaluation placeholder",
                    )

                    return {
                        "status": "evaluation_error",
                        "dataset_type": dataset_type,
                        "results_path": results_file,
                        "evaluation_error": str(e),
                        "total_examples": total_examples,
                    }

        # Calculate metrics
        if progress_callback:
            progress_callback(
                "Calculating metrics", 90, {"status": "calculating_metrics"}
            )

        metrics = calculate_metrics(evaluation_file)

        # Generate report
        if progress_callback:
            progress_callback(
                "Generating report", 95, {"status": "generating_report"}
            )

        dataset_name = dataset_type.capitalize()
        report_path = generate_report(
            metrics=metrics,
            results_file=evaluation_file,
            output_file=report_file,
            dataset_name=dataset_name,
            config_info={
                "Dataset": dataset_path
                or DEFAULT_DATASET_URLS.get(dataset_type, "Unknown"),
                "Examples": total_examples,
                "Iterations": search_config.get("iterations", 3),
                "Questions per iteration": search_config.get(
                    "questions_per_iteration", 3
                ),
                "Search tool": search_config.get("search_tool", "searxng"),
                "Evaluation method": "Human"
                if human_evaluation
                else "Automated",
            },
        )

        # Mark as complete
        if progress_callback:
            progress_callback(
                "Benchmark complete",
                100,
                {
                    "status": "complete",
                    "metrics": metrics,
                    "report_path": report_path,
                },
            )

        return {
            "status": "complete",
            "dataset_type": dataset_type,
            "results_path": results_file,
            "evaluation_path": evaluation_file,
            "report_path": report_path,
            "metrics": metrics,
            "total_examples": total_examples,
            "accuracy": metrics.get("accuracy", 0),
        }

    # No evaluation, just return results
    if progress_callback:
        progress_callback(
            "Benchmark complete (no evaluation)",
            100,
            {"status": "complete_no_eval", "results_path": results_file},
        )

    return {
        "status": "complete_no_eval",
        "dataset_type": dataset_type,
        "results_path": results_file,
        "total_examples": total_examples,
    }


def run_simpleqa_benchmark(num_examples: int = 100, **kwargs) -> Dict[str, Any]:
    """
    Run SimpleQA benchmark with default settings.

    Args:
        num_examples: Number of examples to process
        **kwargs: Additional arguments to pass to run_benchmark

    Returns:
        Dictionary with benchmark results
    """
    return run_benchmark(
        dataset_type="simpleqa", num_examples=num_examples, **kwargs
    )


def run_browsecomp_benchmark(
    num_examples: int = 100, **kwargs
) -> Dict[str, Any]:
    """
    Run BrowseComp benchmark with default settings.

    Args:
        num_examples: Number of examples to process
        **kwargs: Additional arguments to pass to run_benchmark

    Returns:
        Dictionary with benchmark results
    """
    return run_benchmark(
        dataset_type="browsecomp", num_examples=num_examples, **kwargs
    )


def run_xbench_deepsearch_benchmark(
    num_examples: int = 100, **kwargs
) -> Dict[str, Any]:
    """
    Run xbench-DeepSearch benchmark with default settings.

    Args:
        num_examples: Number of examples to process
        **kwargs: Additional arguments to pass to run_benchmark

    Returns:
        Dictionary with benchmark results
    """
    return run_benchmark(
        dataset_type="xbench_deepsearch", num_examples=num_examples, **kwargs
    )