~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
SimpleQA dataset implementation.
This module provides a class for the SimpleQA benchmark dataset.
"""
from typing import Any, Dict
from loguru import logger
from .base import BenchmarkDataset
class SimpleQADataset(BenchmarkDataset):
"""SimpleQA benchmark dataset.
This class handles loading and processing the SimpleQA dataset, which
contains straightforward question-answering pairs.
"""
@classmethod
def get_dataset_info(cls) -> Dict[str, str]:
"""Get basic information about the dataset."""
return {
"id": "simpleqa",
"name": "SimpleQA",
"description": "Simple question-answering evaluation dataset",
"url": cls.get_default_dataset_path(),
}
@classmethod
def get_default_dataset_path(cls) -> str:
"""Get the default URL for the dataset."""
return "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv"
def process_example(self, example: Dict[str, Any]) -> Dict[str, Any]:
"""Process a single example from the dataset.
SimpleQA examples are already in plaintext format, so this just
ensures that the necessary fields are present.
Args:
example: Raw example from the dataset.
Returns:
Processed example ready for use.
"""
# Make a copy to avoid modifying the original
processed = dict(example)
# Ensure problem field exists
if "problem" not in processed:
logger.warning("SimpleQA example missing 'problem' field")
processed["problem"] = ""
# Ensure answer field exists
if "answer" not in processed:
logger.warning("SimpleQA example missing 'answer' field")
processed["answer"] = ""
# Add correct_answer field if not present
if "correct_answer" not in processed:
processed["correct_answer"] = processed["answer"]
return processed
def get_question(self, example: Dict[str, Any]) -> str:
"""Extract the question from an example."""
return example.get("problem", "")
def get_answer(self, example: Dict[str, Any]) -> str:
"""Extract the answer from an example."""
return example.get("answer", "")