Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

11.2 KB
Raw Blame
"""
SimpleQA benchmark evaluator.

This module provides a benchmark evaluator implementation for the SimpleQA
benchmark, which tests simple question-answering capabilities.
"""

import json
from loguru import logger
from pathlib import Path
import time
from typing import Any, Dict

from ..datasets.base import DatasetRegistry
from ..metrics import calculate_metrics, generate_report
from ..runners import run_simpleqa_benchmark  # Keep for backward compatibility
from .base import BaseBenchmarkEvaluator


class SimpleQAEvaluator(BaseBenchmarkEvaluator):
    """
    Evaluator for the SimpleQA benchmark.

    This evaluator runs the SimpleQA benchmark, which tests a system's ability
    to accurately answer straightforward factual questions.
    """

    def __init__(self):
        """Initialize the SimpleQA evaluator."""
        super().__init__("simpleqa")

    def evaluate(
        self,
        system_config: Dict[str, Any],
        num_examples: int,
        output_dir: str,
        use_direct_dataset: bool = True,
    ) -> Dict[str, Any]:
        """
        Run SimpleQA benchmark and return metrics.

        Args:
            system_config: Search and LLM configuration parameters
            num_examples: Number of benchmark examples to run
            output_dir: Directory to save evaluation results
            use_direct_dataset: Whether to use dataset classes directly (recommended)
                                or fall back to runner functions

        Returns:
            Dictionary with metrics including quality_score based on accuracy
        """
        # Create benchmark-specific directory
        benchmark_dir = self._create_subdirectory(output_dir)

        # Log benchmark execution
        logger.info(f"Running SimpleQA benchmark with {num_examples} examples")

        try:
            if use_direct_dataset:
                # Use dataset classes directly (new approach)
                results = self._run_with_dataset_class(
                    system_config=system_config,
                    num_examples=num_examples,
                    output_dir=benchmark_dir,
                )
            else:
                # Fall back to legacy runner function
                results = run_simpleqa_benchmark(
                    num_examples=num_examples,
                    output_dir=benchmark_dir,
                    search_config=system_config,
                    run_evaluation=True,
                )

            # Extract metrics
            metrics = results.get("metrics", {})
            accuracy = metrics.get("accuracy", 0.0)

            # Return evaluation results with quality score
            return {
                "benchmark_type": self.name,
                "accuracy": accuracy,
                "quality_score": accuracy,  # Map accuracy directly to quality score
                "raw_results": results,
                "report_path": results.get("report_path"),
            }

        except Exception as e:
            logger.exception("Error in SimpleQA evaluation")

            # Return error information
            return {
                "benchmark_type": self.name,
                "error": str(e),
                "quality_score": 0.0,
                "accuracy": 0.0,
            }

    def _run_with_dataset_class(
        self,
        system_config: Dict[str, Any],
        num_examples: int,
        output_dir: str,
    ) -> Dict[str, Any]:
        """
        Run SimpleQA benchmark using dataset classes directly.

        This implementation directly uses the dataset classes rather than
        going through the runner functions, allowing for more flexibility
        and better integration with the object-oriented architecture.

        Args:
            system_config: Search and LLM configuration parameters
            num_examples: Number of benchmark examples to run
            output_dir: Directory to save evaluation results. The files
                are written there directly; unlike ``run_benchmark``, this
                path does not resolve the directory below the data
                directory.

        Returns:
            Dictionary with benchmark results
        """
        # Refuse before any research when the grader would have no API key
        # for its endpoint (grading runs only after every example).
        from ..graders import check_grader_endpoint_key

        check_grader_endpoint_key()

        # Create a dataset instance using the registry
        try:
            dataset_instance = DatasetRegistry.create_dataset(
                dataset_id="simpleqa",
                num_examples=num_examples,
                seed=system_config.get("seed", 42),
            )

            # Load dataset examples
            examples = dataset_instance.load()
            logger.info(f"Loaded {len(examples)} SimpleQA examples")

            # Set up output files
            timestamp = time.strftime("%Y%m%d_%H%M%S")
            results_file = str(
                Path(output_dir) / f"simpleqa_{timestamp}_results.jsonl"
            )
            evaluation_file = str(
                Path(output_dir) / f"simpleqa_{timestamp}_evaluation.jsonl"
            )
            report_file = str(
                Path(output_dir) / f"simpleqa_{timestamp}_report.md"
            )

            # Process each example
            results = []

            for i, example in enumerate(examples):
                # Extract question and answer using dataset methods
                question = dataset_instance.get_question(example)
                correct_answer = dataset_instance.get_answer(example)

                logger.info(
                    f"Processing {i + 1}/{len(examples)}: {question[:50]}..."
                )

                try:
                    # Format query based on dataset type
                    formatted_query = question  # Simple format for SimpleQA

                    # Time the search
                    start_time = time.time()

                    # Create search config from system_config
                    search_params = {
                        "iterations": system_config.get("iterations", 3),
                        "questions_per_iteration": system_config.get(
                            "questions_per_iteration", 3
                        ),
                        "search_tool": system_config.get(
                            "search_tool", "searxng"
                        ),
                        # Note: search_strategy is stored in the config but not passed to quick_summary
                        # as it's not supported by the underlying API
                    }

                    # Get response from LDR
                    from local_deep_research.api import quick_summary

                    search_result = quick_summary(
                        query=formatted_query,
                        iterations=search_params.get("iterations"),
                        questions_per_iteration=search_params.get(
                            "questions_per_iteration"
                        ),
                        search_tool=search_params.get("search_tool"),
                    )

                    end_time = time.time()
                    processing_time = end_time - start_time

                    # Extract response
                    response = search_result.get("summary", "")

                    # Extract structured answer
                    from ..graders import extract_answer_from_response

                    extracted = extract_answer_from_response(
                        response, "simpleqa"
                    )

                    # Format result
                    result = {
                        "id": example.get("id", f"example_{i}"),
                        "problem": question,
                        "correct_answer": correct_answer,
                        "response": response,
                        "extracted_answer": extracted["extracted_answer"],
                        "confidence": extracted["confidence"],
                        "processing_time": processing_time,
                        "sources": search_result.get("sources", []),
                        "search_config": search_params,
                    }

                    # Add to results list
                    results.append(result)

                    # Write result to file
                    # nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python SimpleQA output root with a fixed prefix and generated timestamp
                    with open(results_file, "a", encoding="utf-8") as f:
                        f.write(json.dumps(result) + "\n")

                except Exception as e:
                    logger.exception(f"Error processing example {i + 1}")

                    # Create error result
                    error_result = {
                        "id": example.get("id", f"example_{i}"),
                        "problem": question,
                        "correct_answer": correct_answer,
                        "error": str(e),
                        "processing_time": 0,
                    }

                    # Add to results list
                    results.append(error_result)

                    # Write error result to file
                    # nosemgrep: semgrep.rules.path-traversal-risk, reason: Same generated SimpleQA results file used by the success path above
                    with open(results_file, "a", encoding="utf-8") as f:
                        f.write(json.dumps(error_result) + "\n")

            # Grade results
            from ..graders import grade_results

            grade_results(
                results_file=results_file,
                output_file=evaluation_file,
                dataset_type="simpleqa",
            )

            # Calculate metrics
            metrics = calculate_metrics(evaluation_file)

            # Generate report
            dataset_name = "SimpleQA"
            report_path = generate_report(
                metrics=metrics,
                results_file=evaluation_file,
                output_file=report_file,
                dataset_name=dataset_name,
                config_info={
                    "Dataset": "SimpleQA",
                    "Examples": len(examples),
                    "Iterations": search_params.get("iterations", 3),
                    "Questions per iteration": search_params.get(
                        "questions_per_iteration", 3
                    ),
                    "Search tool": search_params.get("search_tool", "searxng"),
                    "Search strategy": search_params.get(
                        "search_strategy", "source_based"
                    ),
                },
            )

            # Return results
            return {
                "status": "complete",
                "dataset_type": "simpleqa",
                "results_path": results_file,
                "evaluation_path": evaluation_file,
                "report_path": report_path,
                "metrics": metrics,
                "total_examples": len(examples),
                "accuracy": metrics.get("accuracy", 0),
            }

        except Exception as e:
            logger.exception("Error in direct dataset evaluation")
            return {
                "status": "error",
                "dataset_type": "simpleqa",
                "error": str(e),
            }