~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
import json
import os
import shutil
import tempfile
import time
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
def _result_line() -> str:
return (
json.dumps(
{
"id": "1",
"problem": "question",
"correct_answer": "answer",
"response": "answer",
}
)
+ "\n"
)
def _allow_benchmark_writes(monkeypatch) -> None:
monkeypatch.setattr(
"local_deep_research.config.search_config.get_setting_from_snapshot",
lambda *_args, **_kwargs: True,
)
def _grader() -> MagicMock:
grader = MagicMock()
grader.invoke.return_value = SimpleNamespace(
content="Extracted Answer: answer\nReasoning: ok\nCorrect: yes"
)
return grader
def test_automated_grader_replaces_only_its_output_file(tmp_path):
from local_deep_research.benchmarks.graders import _grade_results_inner
results = tmp_path / "results.jsonl"
output = tmp_path / "results.json"
sibling = tmp_path / "results-backup.json"
nested = tmp_path / "nested" / "results.json"
results.write_text(_result_line())
output.write_text("stale")
sibling.write_text("canary")
nested.parent.mkdir()
nested.write_text("nested canary")
graded = _grade_results_inner(
_grader(), str(results), str(output), "simpleqa", None
)
assert len(graded) == 1
assert [
json.loads(line) for line in output.read_text().splitlines()
] == graded
assert graded[0]["is_correct"] is True
assert sibling.read_text() == "canary"
assert nested.read_text() == "nested canary"
def test_automated_grader_unlinks_symlink_without_deleting_target(tmp_path):
from local_deep_research.benchmarks.graders import _grade_results_inner
results = tmp_path / "results.jsonl"
output = tmp_path / "results.json"
outside = tmp_path / "outside.json"
results.write_text(_result_line())
outside.write_text("outside canary")
try:
output.symlink_to(outside)
except (OSError, NotImplementedError):
pytest.skip("Cannot create symlinks in this environment")
_grade_results_inner(_grader(), str(results), str(output), "simpleqa", None)
assert outside.read_text() == "outside canary"
assert output.is_file() and not output.is_symlink()
def test_human_grader_replaces_only_its_output_file(tmp_path):
from local_deep_research.benchmarks.graders import human_evaluation
results = tmp_path / "results.jsonl"
output = tmp_path / "results.json"
sibling = tmp_path / "results-backup.json"
results.write_text(_result_line())
output.write_text("stale")
sibling.write_text("canary")
graded = human_evaluation(str(results), str(output), interactive=False)
assert len(graded) == 1
assert [
json.loads(line) for line in output.read_text().splitlines()
] == graded
assert graded[0]["human_evaluation"] is True
assert sibling.read_text() == "canary"
def test_runner_cleanup_only_unlinks_its_timestamped_outputs(
tmp_path, monkeypatch
):
from local_deep_research.benchmarks import runners
# run_benchmark keeps its output directory below the data directory and
# never writes into the data directory root itself.
monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
out = tmp_path / "benchmark-output"
out.mkdir()
timestamp = "20000101_000000"
monkeypatch.setattr(
runners,
"time",
SimpleNamespace(strftime=lambda _format: timestamp),
)
dataset = MagicMock()
dataset.load.return_value = []
monkeypatch.setattr(
runners.DatasetRegistry, "create_dataset", lambda **_kwargs: dataset
)
for suffix in ("results.jsonl", "evaluation.jsonl", "report.md"):
(out / f"simpleqa_{timestamp}_{suffix}").write_text("stale")
sibling = out / f"simpleqa_{timestamp}_results-backup.jsonl"
nested = out / "nested" / f"simpleqa_{timestamp}_results.jsonl"
sibling.write_text("canary")
nested.parent.mkdir()
nested.write_text("nested canary")
runners.run_benchmark("simpleqa", output_dir=str(out), run_evaluation=False)
for suffix in ("results.jsonl", "evaluation.jsonl", "report.md"):
assert not (out / f"simpleqa_{timestamp}_{suffix}").exists()
assert sibling.read_text() == "canary"
assert nested.read_text() == "nested canary"
def test_simpleqa_writer_keeps_adjacent_canary(tmp_path, monkeypatch):
from local_deep_research.benchmarks.evaluators import simpleqa
timestamp = "20000101_000000"
monkeypatch.setattr(
simpleqa,
"time",
SimpleNamespace(strftime=lambda _format: timestamp, time=time.time),
)
dataset = MagicMock()
dataset.load.return_value = [{"id": "1"}]
dataset.get_question.return_value = "question"
dataset.get_answer.return_value = "answer"
monkeypatch.setattr(
simpleqa.DatasetRegistry, "create_dataset", lambda **_kwargs: dataset
)
monkeypatch.setattr(
"local_deep_research.api.quick_summary",
lambda **_kwargs: {"summary": "answer", "sources": []},
)
monkeypatch.setattr(
"local_deep_research.benchmarks.graders.grade_results",
lambda **_kwargs: [],
)
monkeypatch.setattr(simpleqa, "calculate_metrics", lambda _path: {})
monkeypatch.setattr(simpleqa, "generate_report", lambda **_kwargs: "report")
canary = tmp_path / f"simpleqa_{timestamp}_results-backup.jsonl"
canary.write_text("canary")
result = simpleqa.SimpleQAEvaluator()._run_with_dataset_class(
{}, 1, str(tmp_path)
)
assert Path(result["results_path"]).is_file()
assert canary.read_text() == "canary"
def test_optimizer_writes_only_named_result_files(tmp_path, monkeypatch):
from local_deep_research.benchmarks.optimization.optuna_optimizer import (
OptunaOptimizer,
)
_allow_benchmark_writes(monkeypatch)
optimizer = OptunaOptimizer(
"query", output_dir=str(tmp_path), study_name="study"
)
optimizer.trials_history = [{"score": 0.5}]
optimizer.study = SimpleNamespace(
best_params={"iterations": 1}, best_value=0.5, trials=[1]
)
canary = tmp_path / "study_history-backup.json"
canary.write_text("canary")
optimizer._save_results()
assert (tmp_path / "study_history.json").is_file()
assert (tmp_path / "study_best_params.json").is_file()
assert (tmp_path / "study_study.pkl").is_file()
assert canary.read_text() == "canary"
def test_comparison_results_writer_keeps_adjacent_canary(tmp_path, monkeypatch):
from local_deep_research.benchmarks.comparison.results import (
Benchmark_results,
)
_allow_benchmark_writes(monkeypatch)
output = tmp_path / "results.json"
canary = tmp_path / "results-backup.json"
canary.write_text("canary")
results = Benchmark_results(str(output))
results.add_result("model", "hardware", 1, 1, 1, 1, 0, "v", "today")
assert json.loads(output.read_text())[0]["model"] == "model"
assert canary.read_text() == "canary"
def test_reporting_writer_keeps_adjacent_canary(tmp_path, monkeypatch):
from local_deep_research.benchmarks.metrics.reporting import generate_report
_allow_benchmark_writes(monkeypatch)
results = tmp_path / "results.jsonl"
output = tmp_path / "report.md"
canary = tmp_path / "report-backup.md"
results.write_text(_result_line())
canary.write_text("canary")
generate_report({}, str(results), str(output))
assert output.is_file()
assert canary.read_text() == "canary"
def test_comparison_api_writer_keeps_adjacent_canary(tmp_path, monkeypatch):
from local_deep_research.api import benchmark_functions
# compare_configurations keeps its output root below the data directory
# and never writes into the data directory root itself.
monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
out = tmp_path / "comparison-output"
out.mkdir()
_allow_benchmark_writes(monkeypatch)
monkeypatch.setattr(
benchmark_functions,
"run_benchmark",
lambda **_kwargs: {"metrics": {}, "total_examples": 0},
)
canary = out / "comparison_simpleqa_backup.md"
nested = out / "nested" / "comparison_simpleqa.md"
canary.write_text("canary")
nested.parent.mkdir()
nested.write_text("nested canary")
result = benchmark_functions.compare_configurations(
configurations=[{"name": "only"}], output_dir=str(out)
)
assert Path(result["report_path"]).is_file()
assert Path(result["report_path"]).parent == out.resolve()
assert canary.read_text() == "canary"
assert nested.read_text() == "nested canary"
def test_quality_evaluation_removes_real_mkdtemp_directory(
tmp_path, monkeypatch
):
from local_deep_research.benchmarks.metrics import calculation
# The temporary directory is created inside the default
# benchmark_results directory below the data directory.
monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
temp_dir = tmp_path / "ldr_benchmark_created"
temp_dir.mkdir()
monkeypatch.setattr(
calculation,
"tempfile",
SimpleNamespace(mkdtemp=lambda prefix, dir: str(temp_dir)),
)
monkeypatch.setattr(
"local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
lambda **_kwargs: {"metrics": {"accuracy": 1.0}},
)
assert calculation.evaluate_benchmark_quality({})["accuracy"] == 1.0
assert not temp_dir.exists()
def test_quality_evaluation_rmtree_targets_only_mkdtemp_dir_on_error(
tmp_path, monkeypatch
):
from local_deep_research.benchmarks.metrics import calculation
# The temporary directory is created inside the default
# benchmark_results directory below the data directory.
monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
temp_dir = tmp_path / "ldr_benchmark_created"
temp_dir.mkdir()
captured = []
monkeypatch.setattr(
calculation,
"tempfile",
SimpleNamespace(mkdtemp=lambda prefix, dir: str(temp_dir)),
)
monkeypatch.setattr(
"local_deep_research.benchmarks.runners.run_simpleqa_benchmark",
lambda **_kwargs: (_ for _ in ()).throw(
RuntimeError("benchmark failure")
),
)
monkeypatch.setattr(shutil, "rmtree", lambda path: captured.append(path))
assert calculation.evaluate_benchmark_quality({})["accuracy"] == 0.0
assert captured == [str(temp_dir)]
def test_sqlcipher_probe_removes_real_temp_file_on_success(
monkeypatch, tmp_path
):
from local_deep_research.database import encrypted_db
# The probe spools through tempfile.NamedTemporaryFile
# (encrypted_db.py:302-305); point tempfile at tmp_path so the test
# never creates a file in the real system temp directory.
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
paths = []
original_unlink = os.unlink
monkeypatch.setattr(
encrypted_db, "apply_cipher_defaults_before_key", lambda _: None
)
monkeypatch.setattr(encrypted_db, "set_sqlcipher_key", lambda *_: None)
monkeypatch.setattr(
encrypted_db, "apply_sqlcipher_pragmas", lambda *_args, **_kwargs: None
)
monkeypatch.setattr(
encrypted_db, "apply_performance_pragmas", lambda _: None
)
monkeypatch.setattr(encrypted_db, "get_sqlcipher_version", lambda _: "4.0")
def new_cursor():
return SimpleNamespace(execute=lambda _sql: None, close=lambda: None)
connection = SimpleNamespace(
cursor=new_cursor,
execute=lambda _sql: SimpleNamespace(fetchone=lambda: (1,)),
close=lambda: None,
)
monkeypatch.setattr(
encrypted_db,
"get_sqlcipher_module",
lambda: SimpleNamespace(
dbapi2=SimpleNamespace(connect=lambda _path: connection)
),
)
def observe_unlink(path):
paths.append(Path(path))
original_unlink(path)
monkeypatch.setattr(os, "unlink", observe_unlink)
manager = encrypted_db.DatabaseManager.__new__(encrypted_db.DatabaseManager)
assert manager._check_encryption_available()
assert len(paths) == 1
assert paths[0].parent == tmp_path
assert not paths[0].exists()
def test_sqlcipher_probe_removes_temp_file_when_probe_fails(
monkeypatch, tmp_path
):
from local_deep_research.database import encrypted_db
# The probe spools through tempfile.NamedTemporaryFile
# (encrypted_db.py:302-305); point tempfile at tmp_path so the test
# never creates a file in the real system temp directory.
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
paths = []
original_unlink = os.unlink
monkeypatch.setattr(
encrypted_db,
"get_sqlcipher_module",
lambda: (_ for _ in ()).throw(RuntimeError("probe failed")),
)
monkeypatch.setattr(encrypted_db, "get_env_setting", lambda *_args: False)
def observe_unlink(path):
paths.append(Path(path))
original_unlink(path)
monkeypatch.setattr(os, "unlink", observe_unlink)
manager = encrypted_db.DatabaseManager.__new__(encrypted_db.DatabaseManager)
with pytest.raises(RuntimeError, match="SQLCipher not available"):
manager._check_encryption_available()
assert len(paths) == 1
assert paths[0].parent == tmp_path
assert not paths[0].exists()