Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

2.6 KB
Raw Blame
"""
Base class for benchmark evaluators.

This module defines the abstract base class that all benchmark evaluators
must implement, establishing a common interface for different benchmark types.
"""

from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any, Dict


class BaseBenchmarkEvaluator(ABC):
    """
    Abstract base class for benchmark evaluators.

    All benchmark evaluator implementations must inherit from this class and
    implement the evaluate method to run their specific benchmark type.
    """

    def __init__(self, name: str):
        """
        Initialize benchmark evaluator with a name.

        Args:
            name: Unique identifier for this benchmark type
        """
        self.name = name

    def get_name(self) -> str:
        """
        Get the benchmark name.

        Returns:
            The benchmark identifier
        """
        return self.name

    @abstractmethod
    def evaluate(
        self,
        system_config: Dict[str, Any],
        num_examples: int,
        output_dir: str,
    ) -> Dict[str, Any]:
        """
        Run benchmark evaluation with given system configuration.

        Args:
            system_config: Configuration parameters for the system under test
            num_examples: Number of benchmark examples to evaluate
            output_dir: Directory to save evaluation results

        Returns:
            Dictionary with evaluation metrics including quality_score (0-1)
        """
        pass

    def _create_subdirectory(self, output_dir: str) -> str:
        """
        Create a benchmark-specific subdirectory for output.

        The subdirectory is created where the caller says (a relative
        path is relative to the working directory); it is not resolved
        below the data directory. Files that ``run_benchmark`` writes for
        it (the BrowseComp evaluator and the SimpleQA evaluator with
        ``use_direct_dataset=False``) follow ``runners.resolve_output_dir``
        instead, so unless ``output_dir`` is an absolute path below the
        data directory they land elsewhere below the data directory and
        this subdirectory stays empty.

        Args:
            output_dir: Parent directory for output

        Returns:
            Path to the benchmark-specific directory
        """
        benchmark_dir = Path(output_dir) / self.name
        # nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python benchmark output root and evaluator name; no request path is accepted by this helper
        benchmark_dir.mkdir(parents=True, exist_ok=True)
        return str(benchmark_dir)