Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

1.7 KB
Raw Blame
"""
Dataset handling for benchmarks.

This module provides functions and classes for loading and processing
benchmark datasets in a maintainable, extensible way.
"""

from .base import DatasetRegistry
from .browsecomp import BrowseCompDataset
from .simpleqa import SimpleQADataset
from .xbench_deepsearch import XBenchDeepSearchDataset

# Register datasets
DatasetRegistry.register(SimpleQADataset)
DatasetRegistry.register(BrowseCompDataset)
DatasetRegistry.register(XBenchDeepSearchDataset)

# Add convenience functions mirroring the old interface
DEFAULT_DATASET_URLS = {
    "simpleqa": SimpleQADataset.get_default_dataset_path(),
    "browsecomp": BrowseCompDataset.get_default_dataset_path(),
    "xbench_deepsearch": XBenchDeepSearchDataset.get_default_dataset_path(),
}


def get_available_datasets():
    """Get information about all registered datasets."""
    return DatasetRegistry.get_available_datasets()


def load_dataset(
    dataset_type,
    dataset_path=None,
    num_examples=None,
    seed=42,
):
    """Load a dataset by name.

    This is a backwards-compatible function that uses the new dataset classes
    under the hood. It exists to maintain compatibility with existing code.

    Args:
        dataset_type: ID of a registered dataset ('simpleqa', 'browsecomp',
            'xbench_deepsearch')
        dataset_path: Path or URL to dataset (if None, uses default URL)
        num_examples: Optional number of examples to sample
        seed: Random seed for sampling

    Returns:
        List of processed examples
    """
    return DatasetRegistry.load_dataset(
        dataset_id=dataset_type.lower(),
        dataset_path=dataset_path,
        num_examples=num_examples,
        seed=seed,
    )