~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Base class for benchmark evaluators.
This module defines the abstract base class that all benchmark evaluators
must implement, establishing a common interface for different benchmark types.
"""
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any, Dict
class BaseBenchmarkEvaluator(ABC):
"""
Abstract base class for benchmark evaluators.
All benchmark evaluator implementations must inherit from this class and
implement the evaluate method to run their specific benchmark type.
"""
def __init__(self, name: str):
"""
Initialize benchmark evaluator with a name.
Args:
name: Unique identifier for this benchmark type
"""
self.name = name
def get_name(self) -> str:
"""
Get the benchmark name.
Returns:
The benchmark identifier
"""
return self.name
@abstractmethod
def evaluate(
self,
system_config: Dict[str, Any],
num_examples: int,
output_dir: str,
) -> Dict[str, Any]:
"""
Run benchmark evaluation with given system configuration.
Args:
system_config: Configuration parameters for the system under test
num_examples: Number of benchmark examples to evaluate
output_dir: Directory to save evaluation results
Returns:
Dictionary with evaluation metrics including quality_score (0-1)
"""
pass
def _create_subdirectory(self, output_dir: str) -> str:
"""
Create a benchmark-specific subdirectory for output.
The subdirectory is created where the caller says (a relative
path is relative to the working directory); it is not resolved
below the data directory. Files that ``run_benchmark`` writes for
it (the BrowseComp evaluator and the SimpleQA evaluator with
``use_direct_dataset=False``) follow ``runners.resolve_output_dir``
instead, so unless ``output_dir`` is an absolute path below the
data directory they land elsewhere below the data directory and
this subdirectory stays empty.
Args:
output_dir: Parent directory for output
Returns:
Path to the benchmark-specific directory
"""
benchmark_dir = Path(output_dir) / self.name
# nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python benchmark output root and evaluator name; no request path is accepted by this helper
benchmark_dir.mkdir(parents=True, exist_ok=True)
return str(benchmark_dir)