Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

10.7 KB
Raw Blame
"""Benchmark output directories stay below the LDR data directory.

``resolve_output_dir`` (benchmarks/runners.py) resolves the output root of
``run_benchmark``, of both ``compare_configurations`` helpers
(``api.benchmark_functions`` and ``benchmarks.benchmark_functions``) and of
``evaluate_benchmark_quality``'s temporary directory. A relative value is
taken relative to the data directory. A value outside it, the data directory
itself, or one that cannot be resolved is replaced by a default directory
below it; if that default is not usable either, the resolver raises rather
than writing into the data directory root.
``tests/security/test_settings_path_escape.py`` drives the real runner with
an outside directory (the #7417 tripwire).

Library storage paths follow a different, operator-managed policy and are
not covered here. Every hostile destination stays beneath ``tmp_path``.
"""

from pathlib import Path
from unittest.mock import patch

import pytest

from local_deep_research.benchmarks.runners import (
    BenchmarkOutputDirectoryError,
    resolve_output_dir,
)

from tests.security.path_max_fixtures import build_path_max_escape


@pytest.fixture
def data_dir(tmp_path, monkeypatch):
    path = tmp_path / "ldr-data"
    path.mkdir()
    monkeypatch.setenv("LDR_DATA_DIR", str(path))
    return path.resolve()


@pytest.fixture
def outside(tmp_path):
    path = tmp_path / "outside"
    path.mkdir()
    (path / "CANARY").write_text("do not alter")
    return path


def _files_below(directory: Path) -> dict:
    return {
        path.relative_to(directory): path.read_bytes()
        for path in directory.rglob("*")
        if path.is_file()
    }


# ---------------------------------------------------------------------------
# resolve_output_dir
# ---------------------------------------------------------------------------


def test_relative_output_dir_resolves_under_the_data_directory(
    data_dir, tmp_path, monkeypatch
):
    workspace = tmp_path / "workspace"
    workspace.mkdir()
    monkeypatch.chdir(workspace)

    resolved = resolve_output_dir("nested/results", fallback="default")

    assert resolved == data_dir / "nested" / "results"


def test_absolute_output_dir_inside_the_data_directory_is_kept(data_dir):
    requested = data_dir / "kept" / "results"

    assert resolve_output_dir(str(requested), fallback="default") == requested


@pytest.mark.parametrize("kind", ["absolute", "relative", "expandvars"])
def test_outside_output_dir_is_replaced_by_the_fallback(
    data_dir, outside, monkeypatch, kind
):
    monkeypatch.setenv("LDR_TEST_OUTSIDE", str(outside))
    requested = {
        "absolute": str(outside / "results"),
        "relative": "../outside/results",
        "expandvars": "$LDR_TEST_OUTSIDE/results",
    }[kind]

    resolved = resolve_output_dir(requested, fallback="benchmark_results")

    assert resolved == data_dir / "benchmark_results"


def test_symlink_out_of_the_data_directory_is_replaced(data_dir, outside):
    link = data_dir / "link"
    try:
        link.symlink_to(outside, target_is_directory=True)
    except (OSError, NotImplementedError):
        pytest.skip("Cannot create symlinks in this environment")

    resolved = resolve_output_dir("link/results", fallback="benchmark_results")

    assert resolved == data_dir / "benchmark_results"


def test_path_max_symlink_chain_out_of_the_data_directory_is_replaced(
    data_dir, outside
):
    """A bare non-strict resolve() reports this escape as contained."""
    escape = build_path_max_escape(data_dir, outside)

    resolved = resolve_output_dir(
        f"{escape}/results", fallback="benchmark_results"
    )

    assert resolved == data_dir / "benchmark_results"


@pytest.mark.parametrize("unset", [None, "", ".", "./"])
def test_unset_output_dir_selects_the_fallback(data_dir, unset):
    assert (
        resolve_output_dir(unset, fallback="benchmark_results")
        == data_dir / "benchmark_results"
    )


@pytest.mark.parametrize("kind", ["absolute", "dotdot"])
def test_data_directory_root_is_replaced_by_the_fallback(data_dir, kind):
    requested = str(data_dir) if kind == "absolute" else "sub/.."

    assert (
        resolve_output_dir(requested, fallback="benchmark_results")
        == data_dir / "benchmark_results"
    )


def test_malformed_output_dir_is_replaced_by_the_fallback(data_dir):
    assert (
        resolve_output_dir("bad\x00path", fallback="benchmark_results")
        == data_dir / "benchmark_results"
    )


def test_unusable_fallback_raises_instead_of_using_the_data_root(
    data_dir, outside
):
    with pytest.raises(BenchmarkOutputDirectoryError, match="LDR_DATA_DIR"):
        resolve_output_dir(str(outside), fallback="bad\x00fallback")


def test_symlinked_default_fails_closed(data_dir, outside):
    """An operator symlinked the default directory to another disk."""
    link = data_dir / "benchmark_results"
    try:
        link.symlink_to(outside, target_is_directory=True)
    except (OSError, NotImplementedError):
        pytest.skip("Cannot create symlinks in this environment")

    for requested in (str(link), "benchmark_results", None):
        with pytest.raises(
            BenchmarkOutputDirectoryError, match="remove the symlink"
        ):
            resolve_output_dir(requested, fallback="benchmark_results")


def test_unavailable_home_expansion_uses_the_fallback(data_dir, monkeypatch):
    original = Path.expanduser

    def expanduser(path):
        if str(path) == "~unavailable":
            raise RuntimeError("home unavailable")
        return original(path)

    monkeypatch.setattr(Path, "expanduser", expanduser)

    assert (
        resolve_output_dir("~unavailable", fallback="benchmark_results")
        == data_dir / "benchmark_results"
    )


def test_invalid_operator_data_root_is_not_hidden():
    with patch(
        "local_deep_research.config.paths.get_data_directory",
        side_effect=ValueError("invalid operator root"),
    ):
        with pytest.raises(ValueError, match="invalid operator root"):
            resolve_output_dir("results", fallback="benchmark_results")


# ---------------------------------------------------------------------------
# Writers
# ---------------------------------------------------------------------------


def test_run_benchmark_writes_a_relative_output_dir_under_the_data_directory(
    data_dir, tmp_path, monkeypatch
):
    from local_deep_research.benchmarks import runners

    workspace = tmp_path / "workspace"
    workspace.mkdir()
    monkeypatch.chdir(workspace)

    with (
        patch.object(
            runners.DatasetRegistry, "create_dataset", side_effect=RuntimeError
        ),
        patch.object(
            runners,
            "load_dataset",
            return_value=[{"id": "1", "problem": "q", "answer": "a"}],
        ),
        patch.object(
            runners,
            "quick_summary",
            return_value={"summary": "a", "sources": []},
        ),
    ):
        result = runners.run_benchmark(
            "simpleqa", output_dir="nested/results", run_evaluation=False
        )

    results = Path(result["results_path"])
    assert results.parent == data_dir / "nested" / "results"
    assert results.is_file()
    assert list(workspace.iterdir()) == []


def test_run_benchmark_fails_closed_without_writing(
    data_dir, outside, monkeypatch
):
    """No result file lands in the data root or behind the symlink."""
    from local_deep_research.benchmarks import runners

    link = data_dir / "benchmark_results"
    try:
        link.symlink_to(outside, target_is_directory=True)
    except (OSError, NotImplementedError):
        pytest.skip("Cannot create symlinks in this environment")
    quick_summary = []
    monkeypatch.setattr(
        runners, "quick_summary", lambda **kwargs: quick_summary.append(1)
    )
    before_outside = _files_below(outside)

    with pytest.raises(BenchmarkOutputDirectoryError):
        runners.run_benchmark(
            "simpleqa", output_dir=str(link), run_evaluation=False
        )

    assert quick_summary == []
    assert sorted(path.name for path in data_dir.iterdir()) == [
        "benchmark_results"
    ]
    assert _files_below(outside) == before_outside


@pytest.mark.parametrize(
    "module_name",
    [
        "local_deep_research.api.benchmark_functions",
        "local_deep_research.benchmarks.benchmark_functions",
    ],
)
@pytest.mark.parametrize("output_kind", ["absolute", "relative"])
def test_comparison_keeps_results_and_report_under_one_root(
    data_dir, tmp_path, module_name, output_kind
):
    import importlib

    module = importlib.import_module(module_name)
    outside = tmp_path / "outside-comparison"
    requested = str(outside) if output_kind == "absolute" else "comparisons"
    expected = data_dir / (
        "benchmark_comparisons" if output_kind == "absolute" else "comparisons"
    )
    output_paths = []

    def run_benchmark(**kwargs):
        output_paths.append(Path(kwargs["output_dir"]))
        return {"metrics": {}, "total_examples": 1}

    with (
        patch.object(module, "run_benchmark", side_effect=run_benchmark),
        patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as write,
    ):
        result = module.compare_configurations(
            output_dir=requested,
            configurations=[{"name": "First Setup"}, {"name": "Second Setup"}],
        )

    assert output_paths == [expected / "First_Setup", expected / "Second_Setup"]
    report = Path(result["report_path"])
    assert report.parent == expected
    write.assert_called_once()
    assert write.call_args.args[0] == str(report)
    assert expected.is_dir()
    assert not outside.exists()


def test_quality_evaluation_keeps_its_temporary_results_contained(data_dir):
    """The runner keeps the temporary directory, so the cleanup removes it.

    A system temporary directory is outside the data directory, so the
    runner would write to its fallback instead and the results would
    outlive the cleanup.
    """
    from local_deep_research.benchmarks.metrics import calculation

    used = []

    def run_simpleqa_benchmark(**kwargs):
        directory = resolve_output_dir(
            kwargs["output_dir"], fallback="benchmark_results"
        )
        directory.mkdir(parents=True, exist_ok=True)
        (directory / "results.jsonl").write_text("{}\n")
        used.append(directory)
        return {"metrics": {"accuracy": 1.0}}

    with patch(
        "local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
        side_effect=run_simpleqa_benchmark,
    ):
        result = calculation.evaluate_benchmark_quality({}, output_dir=None)

    assert result["accuracy"] == 1.0
    assert len(used) == 1
    assert used[0].parent == data_dir / "benchmark_results"
    assert not used[0].exists()
    assert list((data_dir / "benchmark_results").iterdir()) == []