~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Benchmark output directories stay below the LDR data directory.
``resolve_output_dir`` (benchmarks/runners.py) resolves the output root of
``run_benchmark``, of both ``compare_configurations`` helpers
(``api.benchmark_functions`` and ``benchmarks.benchmark_functions``) and of
``evaluate_benchmark_quality``'s temporary directory. A relative value is
taken relative to the data directory. A value outside it, the data directory
itself, or one that cannot be resolved is replaced by a default directory
below it; if that default is not usable either, the resolver raises rather
than writing into the data directory root.
``tests/security/test_settings_path_escape.py`` drives the real runner with
an outside directory (the #7417 tripwire).
Library storage paths follow a different, operator-managed policy and are
not covered here. Every hostile destination stays beneath ``tmp_path``.
"""
from pathlib import Path
from unittest.mock import patch
import pytest
from local_deep_research.benchmarks.runners import (
BenchmarkOutputDirectoryError,
resolve_output_dir,
)
from tests.security.path_max_fixtures import build_path_max_escape
@pytest.fixture
def data_dir(tmp_path, monkeypatch):
path = tmp_path / "ldr-data"
path.mkdir()
monkeypatch.setenv("LDR_DATA_DIR", str(path))
return path.resolve()
@pytest.fixture
def outside(tmp_path):
path = tmp_path / "outside"
path.mkdir()
(path / "CANARY").write_text("do not alter")
return path
def _files_below(directory: Path) -> dict:
return {
path.relative_to(directory): path.read_bytes()
for path in directory.rglob("*")
if path.is_file()
}
# ---------------------------------------------------------------------------
# resolve_output_dir
# ---------------------------------------------------------------------------
def test_relative_output_dir_resolves_under_the_data_directory(
data_dir, tmp_path, monkeypatch
):
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.chdir(workspace)
resolved = resolve_output_dir("nested/results", fallback="default")
assert resolved == data_dir / "nested" / "results"
def test_absolute_output_dir_inside_the_data_directory_is_kept(data_dir):
requested = data_dir / "kept" / "results"
assert resolve_output_dir(str(requested), fallback="default") == requested
@pytest.mark.parametrize("kind", ["absolute", "relative", "expandvars"])
def test_outside_output_dir_is_replaced_by_the_fallback(
data_dir, outside, monkeypatch, kind
):
monkeypatch.setenv("LDR_TEST_OUTSIDE", str(outside))
requested = {
"absolute": str(outside / "results"),
"relative": "../outside/results",
"expandvars": "$LDR_TEST_OUTSIDE/results",
}[kind]
resolved = resolve_output_dir(requested, fallback="benchmark_results")
assert resolved == data_dir / "benchmark_results"
def test_symlink_out_of_the_data_directory_is_replaced(data_dir, outside):
link = data_dir / "link"
try:
link.symlink_to(outside, target_is_directory=True)
except (OSError, NotImplementedError):
pytest.skip("Cannot create symlinks in this environment")
resolved = resolve_output_dir("link/results", fallback="benchmark_results")
assert resolved == data_dir / "benchmark_results"
def test_path_max_symlink_chain_out_of_the_data_directory_is_replaced(
data_dir, outside
):
"""A bare non-strict resolve() reports this escape as contained."""
escape = build_path_max_escape(data_dir, outside)
resolved = resolve_output_dir(
f"{escape}/results", fallback="benchmark_results"
)
assert resolved == data_dir / "benchmark_results"
@pytest.mark.parametrize("unset", [None, "", ".", "./"])
def test_unset_output_dir_selects_the_fallback(data_dir, unset):
assert (
resolve_output_dir(unset, fallback="benchmark_results")
== data_dir / "benchmark_results"
)
@pytest.mark.parametrize("kind", ["absolute", "dotdot"])
def test_data_directory_root_is_replaced_by_the_fallback(data_dir, kind):
requested = str(data_dir) if kind == "absolute" else "sub/.."
assert (
resolve_output_dir(requested, fallback="benchmark_results")
== data_dir / "benchmark_results"
)
def test_malformed_output_dir_is_replaced_by_the_fallback(data_dir):
assert (
resolve_output_dir("bad\x00path", fallback="benchmark_results")
== data_dir / "benchmark_results"
)
def test_unusable_fallback_raises_instead_of_using_the_data_root(
data_dir, outside
):
with pytest.raises(BenchmarkOutputDirectoryError, match="LDR_DATA_DIR"):
resolve_output_dir(str(outside), fallback="bad\x00fallback")
def test_symlinked_default_fails_closed(data_dir, outside):
"""An operator symlinked the default directory to another disk."""
link = data_dir / "benchmark_results"
try:
link.symlink_to(outside, target_is_directory=True)
except (OSError, NotImplementedError):
pytest.skip("Cannot create symlinks in this environment")
for requested in (str(link), "benchmark_results", None):
with pytest.raises(
BenchmarkOutputDirectoryError, match="remove the symlink"
):
resolve_output_dir(requested, fallback="benchmark_results")
def test_unavailable_home_expansion_uses_the_fallback(data_dir, monkeypatch):
original = Path.expanduser
def expanduser(path):
if str(path) == "~unavailable":
raise RuntimeError("home unavailable")
return original(path)
monkeypatch.setattr(Path, "expanduser", expanduser)
assert (
resolve_output_dir("~unavailable", fallback="benchmark_results")
== data_dir / "benchmark_results"
)
def test_invalid_operator_data_root_is_not_hidden():
with patch(
"local_deep_research.config.paths.get_data_directory",
side_effect=ValueError("invalid operator root"),
):
with pytest.raises(ValueError, match="invalid operator root"):
resolve_output_dir("results", fallback="benchmark_results")
# ---------------------------------------------------------------------------
# Writers
# ---------------------------------------------------------------------------
def test_run_benchmark_writes_a_relative_output_dir_under_the_data_directory(
data_dir, tmp_path, monkeypatch
):
from local_deep_research.benchmarks import runners
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.chdir(workspace)
with (
patch.object(
runners.DatasetRegistry, "create_dataset", side_effect=RuntimeError
),
patch.object(
runners,
"load_dataset",
return_value=[{"id": "1", "problem": "q", "answer": "a"}],
),
patch.object(
runners,
"quick_summary",
return_value={"summary": "a", "sources": []},
),
):
result = runners.run_benchmark(
"simpleqa", output_dir="nested/results", run_evaluation=False
)
results = Path(result["results_path"])
assert results.parent == data_dir / "nested" / "results"
assert results.is_file()
assert list(workspace.iterdir()) == []
def test_run_benchmark_fails_closed_without_writing(
data_dir, outside, monkeypatch
):
"""No result file lands in the data root or behind the symlink."""
from local_deep_research.benchmarks import runners
link = data_dir / "benchmark_results"
try:
link.symlink_to(outside, target_is_directory=True)
except (OSError, NotImplementedError):
pytest.skip("Cannot create symlinks in this environment")
quick_summary = []
monkeypatch.setattr(
runners, "quick_summary", lambda **kwargs: quick_summary.append(1)
)
before_outside = _files_below(outside)
with pytest.raises(BenchmarkOutputDirectoryError):
runners.run_benchmark(
"simpleqa", output_dir=str(link), run_evaluation=False
)
assert quick_summary == []
assert sorted(path.name for path in data_dir.iterdir()) == [
"benchmark_results"
]
assert _files_below(outside) == before_outside
@pytest.mark.parametrize(
"module_name",
[
"local_deep_research.api.benchmark_functions",
"local_deep_research.benchmarks.benchmark_functions",
],
)
@pytest.mark.parametrize("output_kind", ["absolute", "relative"])
def test_comparison_keeps_results_and_report_under_one_root(
data_dir, tmp_path, module_name, output_kind
):
import importlib
module = importlib.import_module(module_name)
outside = tmp_path / "outside-comparison"
requested = str(outside) if output_kind == "absolute" else "comparisons"
expected = data_dir / (
"benchmark_comparisons" if output_kind == "absolute" else "comparisons"
)
output_paths = []
def run_benchmark(**kwargs):
output_paths.append(Path(kwargs["output_dir"]))
return {"metrics": {}, "total_examples": 1}
with (
patch.object(module, "run_benchmark", side_effect=run_benchmark),
patch(
"local_deep_research.security.file_write_verifier.write_file_verified"
) as write,
):
result = module.compare_configurations(
output_dir=requested,
configurations=[{"name": "First Setup"}, {"name": "Second Setup"}],
)
assert output_paths == [expected / "First_Setup", expected / "Second_Setup"]
report = Path(result["report_path"])
assert report.parent == expected
write.assert_called_once()
assert write.call_args.args[0] == str(report)
assert expected.is_dir()
assert not outside.exists()
def test_quality_evaluation_keeps_its_temporary_results_contained(data_dir):
"""The runner keeps the temporary directory, so the cleanup removes it.
A system temporary directory is outside the data directory, so the
runner would write to its fallback instead and the results would
outlive the cleanup.
"""
from local_deep_research.benchmarks.metrics import calculation
used = []
def run_simpleqa_benchmark(**kwargs):
directory = resolve_output_dir(
kwargs["output_dir"], fallback="benchmark_results"
)
directory.mkdir(parents=True, exist_ok=True)
(directory / "results.jsonl").write_text("{}\n")
used.append(directory)
return {"metrics": {"accuracy": 1.0}}
with patch(
"local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
side_effect=run_simpleqa_benchmark,
):
result = calculation.evaluate_benchmark_quality({}, output_dir=None)
assert result["accuracy"] == 1.0
assert len(used) == 1
assert used[0].parent == data_dir / "benchmark_results"
assert not used[0].exists()
assert list((data_dir / "benchmark_results").iterdir()) == []