Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

13.2 KB
Raw Blame
import json
import os
import shutil
import tempfile
import time
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock

import pytest


def _result_line() -> str:
    return (
        json.dumps(
            {
                "id": "1",
                "problem": "question",
                "correct_answer": "answer",
                "response": "answer",
            }
        )
        + "\n"
    )


def _allow_benchmark_writes(monkeypatch) -> None:
    monkeypatch.setattr(
        "local_deep_research.config.search_config.get_setting_from_snapshot",
        lambda *_args, **_kwargs: True,
    )


def _grader() -> MagicMock:
    grader = MagicMock()
    grader.invoke.return_value = SimpleNamespace(
        content="Extracted Answer: answer\nReasoning: ok\nCorrect: yes"
    )
    return grader


def test_automated_grader_replaces_only_its_output_file(tmp_path):
    from local_deep_research.benchmarks.graders import _grade_results_inner

    results = tmp_path / "results.jsonl"
    output = tmp_path / "results.json"
    sibling = tmp_path / "results-backup.json"
    nested = tmp_path / "nested" / "results.json"
    results.write_text(_result_line())
    output.write_text("stale")
    sibling.write_text("canary")
    nested.parent.mkdir()
    nested.write_text("nested canary")

    graded = _grade_results_inner(
        _grader(), str(results), str(output), "simpleqa", None
    )

    assert len(graded) == 1
    assert [
        json.loads(line) for line in output.read_text().splitlines()
    ] == graded
    assert graded[0]["is_correct"] is True
    assert sibling.read_text() == "canary"
    assert nested.read_text() == "nested canary"


def test_automated_grader_unlinks_symlink_without_deleting_target(tmp_path):
    from local_deep_research.benchmarks.graders import _grade_results_inner

    results = tmp_path / "results.jsonl"
    output = tmp_path / "results.json"
    outside = tmp_path / "outside.json"
    results.write_text(_result_line())
    outside.write_text("outside canary")
    try:
        output.symlink_to(outside)
    except (OSError, NotImplementedError):
        pytest.skip("Cannot create symlinks in this environment")

    _grade_results_inner(_grader(), str(results), str(output), "simpleqa", None)

    assert outside.read_text() == "outside canary"
    assert output.is_file() and not output.is_symlink()


def test_human_grader_replaces_only_its_output_file(tmp_path):
    from local_deep_research.benchmarks.graders import human_evaluation

    results = tmp_path / "results.jsonl"
    output = tmp_path / "results.json"
    sibling = tmp_path / "results-backup.json"
    results.write_text(_result_line())
    output.write_text("stale")
    sibling.write_text("canary")

    graded = human_evaluation(str(results), str(output), interactive=False)

    assert len(graded) == 1
    assert [
        json.loads(line) for line in output.read_text().splitlines()
    ] == graded
    assert graded[0]["human_evaluation"] is True
    assert sibling.read_text() == "canary"


def test_runner_cleanup_only_unlinks_its_timestamped_outputs(
    tmp_path, monkeypatch
):
    from local_deep_research.benchmarks import runners

    # run_benchmark keeps its output directory below the data directory and
    # never writes into the data directory root itself.
    monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
    out = tmp_path / "benchmark-output"
    out.mkdir()
    timestamp = "20000101_000000"
    monkeypatch.setattr(
        runners,
        "time",
        SimpleNamespace(strftime=lambda _format: timestamp),
    )
    dataset = MagicMock()
    dataset.load.return_value = []
    monkeypatch.setattr(
        runners.DatasetRegistry, "create_dataset", lambda **_kwargs: dataset
    )
    for suffix in ("results.jsonl", "evaluation.jsonl", "report.md"):
        (out / f"simpleqa_{timestamp}_{suffix}").write_text("stale")
    sibling = out / f"simpleqa_{timestamp}_results-backup.jsonl"
    nested = out / "nested" / f"simpleqa_{timestamp}_results.jsonl"
    sibling.write_text("canary")
    nested.parent.mkdir()
    nested.write_text("nested canary")

    runners.run_benchmark("simpleqa", output_dir=str(out), run_evaluation=False)

    for suffix in ("results.jsonl", "evaluation.jsonl", "report.md"):
        assert not (out / f"simpleqa_{timestamp}_{suffix}").exists()
    assert sibling.read_text() == "canary"
    assert nested.read_text() == "nested canary"


def test_simpleqa_writer_keeps_adjacent_canary(tmp_path, monkeypatch):
    from local_deep_research.benchmarks.evaluators import simpleqa

    timestamp = "20000101_000000"
    monkeypatch.setattr(
        simpleqa,
        "time",
        SimpleNamespace(strftime=lambda _format: timestamp, time=time.time),
    )
    dataset = MagicMock()
    dataset.load.return_value = [{"id": "1"}]
    dataset.get_question.return_value = "question"
    dataset.get_answer.return_value = "answer"
    monkeypatch.setattr(
        simpleqa.DatasetRegistry, "create_dataset", lambda **_kwargs: dataset
    )
    monkeypatch.setattr(
        "local_deep_research.api.quick_summary",
        lambda **_kwargs: {"summary": "answer", "sources": []},
    )
    monkeypatch.setattr(
        "local_deep_research.benchmarks.graders.grade_results",
        lambda **_kwargs: [],
    )
    monkeypatch.setattr(simpleqa, "calculate_metrics", lambda _path: {})
    monkeypatch.setattr(simpleqa, "generate_report", lambda **_kwargs: "report")
    canary = tmp_path / f"simpleqa_{timestamp}_results-backup.jsonl"
    canary.write_text("canary")

    result = simpleqa.SimpleQAEvaluator()._run_with_dataset_class(
        {}, 1, str(tmp_path)
    )

    assert Path(result["results_path"]).is_file()
    assert canary.read_text() == "canary"


def test_optimizer_writes_only_named_result_files(tmp_path, monkeypatch):
    from local_deep_research.benchmarks.optimization.optuna_optimizer import (
        OptunaOptimizer,
    )

    _allow_benchmark_writes(monkeypatch)
    optimizer = OptunaOptimizer(
        "query", output_dir=str(tmp_path), study_name="study"
    )
    optimizer.trials_history = [{"score": 0.5}]
    optimizer.study = SimpleNamespace(
        best_params={"iterations": 1}, best_value=0.5, trials=[1]
    )
    canary = tmp_path / "study_history-backup.json"
    canary.write_text("canary")

    optimizer._save_results()

    assert (tmp_path / "study_history.json").is_file()
    assert (tmp_path / "study_best_params.json").is_file()
    assert (tmp_path / "study_study.pkl").is_file()
    assert canary.read_text() == "canary"


def test_comparison_results_writer_keeps_adjacent_canary(tmp_path, monkeypatch):
    from local_deep_research.benchmarks.comparison.results import (
        Benchmark_results,
    )

    _allow_benchmark_writes(monkeypatch)
    output = tmp_path / "results.json"
    canary = tmp_path / "results-backup.json"
    canary.write_text("canary")

    results = Benchmark_results(str(output))
    results.add_result("model", "hardware", 1, 1, 1, 1, 0, "v", "today")

    assert json.loads(output.read_text())[0]["model"] == "model"
    assert canary.read_text() == "canary"


def test_reporting_writer_keeps_adjacent_canary(tmp_path, monkeypatch):
    from local_deep_research.benchmarks.metrics.reporting import generate_report

    _allow_benchmark_writes(monkeypatch)
    results = tmp_path / "results.jsonl"
    output = tmp_path / "report.md"
    canary = tmp_path / "report-backup.md"
    results.write_text(_result_line())
    canary.write_text("canary")

    generate_report({}, str(results), str(output))

    assert output.is_file()
    assert canary.read_text() == "canary"


def test_comparison_api_writer_keeps_adjacent_canary(tmp_path, monkeypatch):
    from local_deep_research.api import benchmark_functions

    # compare_configurations keeps its output root below the data directory
    # and never writes into the data directory root itself.
    monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
    out = tmp_path / "comparison-output"
    out.mkdir()
    _allow_benchmark_writes(monkeypatch)
    monkeypatch.setattr(
        benchmark_functions,
        "run_benchmark",
        lambda **_kwargs: {"metrics": {}, "total_examples": 0},
    )
    canary = out / "comparison_simpleqa_backup.md"
    nested = out / "nested" / "comparison_simpleqa.md"
    canary.write_text("canary")
    nested.parent.mkdir()
    nested.write_text("nested canary")

    result = benchmark_functions.compare_configurations(
        configurations=[{"name": "only"}], output_dir=str(out)
    )

    assert Path(result["report_path"]).is_file()
    assert Path(result["report_path"]).parent == out.resolve()
    assert canary.read_text() == "canary"
    assert nested.read_text() == "nested canary"


def test_quality_evaluation_removes_real_mkdtemp_directory(
    tmp_path, monkeypatch
):
    from local_deep_research.benchmarks.metrics import calculation

    # The temporary directory is created inside the default
    # benchmark_results directory below the data directory.
    monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
    temp_dir = tmp_path / "ldr_benchmark_created"
    temp_dir.mkdir()
    monkeypatch.setattr(
        calculation,
        "tempfile",
        SimpleNamespace(mkdtemp=lambda prefix, dir: str(temp_dir)),
    )
    monkeypatch.setattr(
        "local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
        lambda **_kwargs: {"metrics": {"accuracy": 1.0}},
    )

    assert calculation.evaluate_benchmark_quality({})["accuracy"] == 1.0
    assert not temp_dir.exists()


def test_quality_evaluation_rmtree_targets_only_mkdtemp_dir_on_error(
    tmp_path, monkeypatch
):
    from local_deep_research.benchmarks.metrics import calculation

    # The temporary directory is created inside the default
    # benchmark_results directory below the data directory.
    monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
    temp_dir = tmp_path / "ldr_benchmark_created"
    temp_dir.mkdir()
    captured = []
    monkeypatch.setattr(
        calculation,
        "tempfile",
        SimpleNamespace(mkdtemp=lambda prefix, dir: str(temp_dir)),
    )
    monkeypatch.setattr(
        "local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
        lambda **_kwargs: (_ for _ in ()).throw(
            RuntimeError("benchmark failure")
        ),
    )
    monkeypatch.setattr(shutil, "rmtree", lambda path: captured.append(path))

    assert calculation.evaluate_benchmark_quality({})["accuracy"] == 0.0
    assert captured == [str(temp_dir)]


def test_sqlcipher_probe_removes_real_temp_file_on_success(
    monkeypatch, tmp_path
):
    from local_deep_research.database import encrypted_db

    # The probe spools through tempfile.NamedTemporaryFile
    # (encrypted_db.py:302-305); point tempfile at tmp_path so the test
    # never creates a file in the real system temp directory.
    monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
    paths = []
    original_unlink = os.unlink
    monkeypatch.setattr(
        encrypted_db, "apply_cipher_defaults_before_key", lambda _: None
    )
    monkeypatch.setattr(encrypted_db, "set_sqlcipher_key", lambda *_: None)
    monkeypatch.setattr(
        encrypted_db, "apply_sqlcipher_pragmas", lambda *_args, **_kwargs: None
    )
    monkeypatch.setattr(
        encrypted_db, "apply_performance_pragmas", lambda _: None
    )
    monkeypatch.setattr(encrypted_db, "get_sqlcipher_version", lambda _: "4.0")

    def new_cursor():
        return SimpleNamespace(execute=lambda _sql: None, close=lambda: None)

    connection = SimpleNamespace(
        cursor=new_cursor,
        execute=lambda _sql: SimpleNamespace(fetchone=lambda: (1,)),
        close=lambda: None,
    )
    monkeypatch.setattr(
        encrypted_db,
        "get_sqlcipher_module",
        lambda: SimpleNamespace(
            dbapi2=SimpleNamespace(connect=lambda _path: connection)
        ),
    )

    def observe_unlink(path):
        paths.append(Path(path))
        original_unlink(path)

    monkeypatch.setattr(os, "unlink", observe_unlink)

    manager = encrypted_db.DatabaseManager.__new__(encrypted_db.DatabaseManager)
    assert manager._check_encryption_available()
    assert len(paths) == 1
    assert paths[0].parent == tmp_path
    assert not paths[0].exists()


def test_sqlcipher_probe_removes_temp_file_when_probe_fails(
    monkeypatch, tmp_path
):
    from local_deep_research.database import encrypted_db

    # The probe spools through tempfile.NamedTemporaryFile
    # (encrypted_db.py:302-305); point tempfile at tmp_path so the test
    # never creates a file in the real system temp directory.
    monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
    paths = []
    original_unlink = os.unlink
    monkeypatch.setattr(
        encrypted_db,
        "get_sqlcipher_module",
        lambda: (_ for _ in ()).throw(RuntimeError("probe failed")),
    )
    monkeypatch.setattr(encrypted_db, "get_env_setting", lambda *_args: False)

    def observe_unlink(path):
        paths.append(Path(path))
        original_unlink(path)

    monkeypatch.setattr(os, "unlink", observe_unlink)

    manager = encrypted_db.DatabaseManager.__new__(encrypted_db.DatabaseManager)
    with pytest.raises(RuntimeError, match="SQLCipher not available"):
        manager._check_encryption_available()

    assert len(paths) == 1
    assert paths[0].parent == tmp_path
    assert not paths[0].exists()