Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

5.6 KB
Raw Blame
"""
Tests for BaseBenchmarkEvaluator class.

Tests the abstract base class functionality and interface contract.
"""

import tempfile
from pathlib import Path
from typing import Any, Dict

import pytest

from local_deep_research.benchmarks.evaluators.base import (
    BaseBenchmarkEvaluator,
)


class ConcreteBenchmarkEvaluator(BaseBenchmarkEvaluator):
    """Concrete implementation for testing the abstract base class."""

    def __init__(self, name: str = "test_benchmark"):
        super().__init__(name)
        self.evaluate_called = False

    def evaluate(
        self,
        system_config: Dict[str, Any],
        num_examples: int,
        output_dir: str,
    ) -> Dict[str, Any]:
        """Concrete implementation of evaluate."""
        self.evaluate_called = True
        return {
            "benchmark_type": self.name,
            "quality_score": 0.5,
            "num_examples": num_examples,
        }


class TestBaseBenchmarkEvaluatorInit:
    """Test initialization of BaseBenchmarkEvaluator."""

    def test_init_with_name(self):
        """Test initialization with a benchmark name."""
        evaluator = ConcreteBenchmarkEvaluator("my_benchmark")
        assert evaluator.name == "my_benchmark"

    def test_init_with_default_name(self):
        """Test initialization with default name."""
        evaluator = ConcreteBenchmarkEvaluator()
        assert evaluator.name == "test_benchmark"

    def test_init_with_empty_name(self):
        """Test initialization with empty name."""
        evaluator = ConcreteBenchmarkEvaluator("")
        assert evaluator.name == ""


class TestGetName:
    """Test get_name method."""

    def test_get_name_returns_name(self):
        """Test that get_name returns the benchmark name."""
        evaluator = ConcreteBenchmarkEvaluator("simpleqa")
        assert evaluator.get_name() == "simpleqa"

    def test_get_name_matches_attribute(self):
        """Test that get_name returns same value as name attribute."""
        evaluator = ConcreteBenchmarkEvaluator("browsecomp")
        assert evaluator.get_name() == evaluator.name


class TestCreateSubdirectory:
    """Test _create_subdirectory method."""

    def test_create_subdirectory_creates_directory(self):
        """Test that subdirectory is created."""
        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = ConcreteBenchmarkEvaluator("test_bench")
            result = evaluator._create_subdirectory(tmpdir)

            expected_path = Path(tmpdir) / "test_bench"
            assert Path(result).exists()
            assert Path(result) == expected_path

    def test_create_subdirectory_returns_string_path(self):
        """Test that subdirectory method returns string path."""
        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = ConcreteBenchmarkEvaluator("my_test")
            result = evaluator._create_subdirectory(tmpdir)

            assert isinstance(result, str)

    def test_create_subdirectory_with_nested_parent(self):
        """Test creating subdirectory in nested parent."""
        with tempfile.TemporaryDirectory() as tmpdir:
            nested_dir = str(Path(tmpdir) / "level1" / "level2")
            evaluator = ConcreteBenchmarkEvaluator("nested_bench")

            result = evaluator._create_subdirectory(nested_dir)

            assert Path(result).exists()
            assert Path(result).name == "nested_bench"

    def test_create_subdirectory_idempotent(self):
        """Test that calling _create_subdirectory multiple times is safe."""
        with tempfile.TemporaryDirectory() as tmpdir:
            evaluator = ConcreteBenchmarkEvaluator("repeat_bench")

            result1 = evaluator._create_subdirectory(tmpdir)
            result2 = evaluator._create_subdirectory(tmpdir)

            assert result1 == result2
            assert Path(result1).exists()


class TestEvaluateAbstract:
    """Test evaluate method interface."""

    def test_evaluate_is_callable(self):
        """Test that evaluate can be called."""
        evaluator = ConcreteBenchmarkEvaluator("test")
        evaluator.evaluate(
            system_config={"key": "value"},
            num_examples=10,
            output_dir="/tmp",
        )
        assert evaluator.evaluate_called

    def test_evaluate_returns_dict(self):
        """Test that evaluate returns a dictionary."""
        evaluator = ConcreteBenchmarkEvaluator("test")
        result = evaluator.evaluate(
            system_config={},
            num_examples=5,
            output_dir="/tmp",
        )
        assert isinstance(result, dict)

    def test_evaluate_includes_benchmark_type(self):
        """Test that evaluate result includes benchmark_type."""
        evaluator = ConcreteBenchmarkEvaluator("my_bench")
        result = evaluator.evaluate(
            system_config={},
            num_examples=5,
            output_dir="/tmp",
        )
        assert result["benchmark_type"] == "my_bench"

    def test_evaluate_includes_quality_score(self):
        """Test that evaluate result includes quality_score."""
        evaluator = ConcreteBenchmarkEvaluator("test")
        result = evaluator.evaluate(
            system_config={},
            num_examples=5,
            output_dir="/tmp",
        )
        assert "quality_score" in result
        assert 0 <= result["quality_score"] <= 1


class TestAbstractMethodEnforcement:
    """Test that abstract method is properly enforced."""

    def test_cannot_instantiate_base_class(self):
        """Test that BaseBenchmarkEvaluator cannot be instantiated directly."""
        with pytest.raises(TypeError):
            BaseBenchmarkEvaluator("test")