~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""GREEN disk-write guardrail battery: pins CURRENT secure write/delete
behavior so any regression turns the suite red.
Surfaces covered (complements tests/vector_stores/*, tests/security/
test_file_write_verifier.py, and tests/web/routes/test_research_routes_config.py
— those drive mocked settings or the full client; this file uses real
tmp_path filesystems and unmocked write paths wherever possible):
1. FAISS atomic persist (tempfile.mkstemp + os.replace, 0700 dir, stale-.tmp sweep)
2. Legacy docstore migration symlink/hostile-path guards
3. Verified JSON writes scrubbing sensitive keys (unmocked happy path)
4. /api/save_raw_config: blocked-key refusal + verified-write gate composition
5. FileReportStorage research_id containment + delete scoping
6. Backup prune deleting only direct-child ldr_backup_*.db(.tmp) matches
Offline, tmp_path-only, no LLM/network. If symlinks cannot be created the
affected tests skip rather than pass vacuously.
"""
import json
import os
import sys
import tempfile
import time
from pathlib import Path
import numpy as np
import pytest
# --------------------------------------------------------------------------
# Surface 1: FAISS atomic persist
# (src/local_deep_research/vector_stores/implementations/faiss_store.py)
# --------------------------------------------------------------------------
def _make_store():
from local_deep_research.vector_stores.implementations.faiss_store import (
FaissVectorStore,
)
store = FaissVectorStore.create(
dimension=2,
index_type="flat",
metric="l2",
normalize=False,
)
store.add([1], np.array([[1.0, 2.0]], dtype="float32"))
return store
@pytest.mark.skipif(
sys.platform == "win32", reason="chmod is not supported on Windows"
)
def test_faiss_persist_stays_in_cache_root_and_leaves_no_tmp(tmp_path):
target = tmp_path / "cache" / "index.faiss"
_make_store().persist(target)
assert target.is_file()
assert target.resolve().is_relative_to((tmp_path / "cache").resolve())
assert (target.parent.stat().st_mode & 0o777) == 0o700
# The index is the ONLY file anywhere under the cache root — persist
# must not scatter bytes to any other location.
assert [p for p in tmp_path.rglob("*") if p.is_file()] == [target]
assert list(target.parent.glob(f"{target.name}.*.tmp")) == []
def test_faiss_persist_sweeps_stale_tmp_but_spares_fresh_tmp(tmp_path):
from local_deep_research.vector_stores.implementations.faiss_store import (
_STALE_TMP_AGE_SECONDS,
)
target = tmp_path / "cache" / "index.faiss"
target.parent.mkdir()
# Sibling temps with the REAL mkstemp prefix/suffix the write itself
# uses, so this fails if the sweep glob and the write ever drift apart.
stale_fd, stale_name = tempfile.mkstemp(
dir=target.parent, prefix=f"{target.name}.", suffix=".tmp"
)
fresh_fd, fresh_name = tempfile.mkstemp(
dir=target.parent, prefix=f"{target.name}.", suffix=".tmp"
)
os.close(stale_fd)
os.close(fresh_fd)
stale = Path(stale_name)
fresh = Path(fresh_name)
old_time = time.time() - _STALE_TMP_AGE_SECONDS - 1
os.utime(stale, (old_time, old_time))
_make_store().persist(target)
assert not stale.exists() # orphaned by a hard-killed prior write -> swept
assert fresh.exists() # concurrent worker's in-flight temp -> spared
def test_faiss_persist_replace_failure_keeps_original_index(
tmp_path, monkeypatch
):
import local_deep_research.vector_stores.implementations.faiss_store as module
target = tmp_path / "cache" / "index.faiss"
original = _make_store()
original.persist(target)
original_bytes = target.read_bytes()
def fail_replace(_source, _destination):
raise OSError("simulated rename failure")
monkeypatch.setattr(module.os, "replace", fail_replace)
with pytest.raises(OSError, match="simulated rename failure"):
_make_store().persist(target)
# os.replace-only persistence: the durable file is either the old bytes
# or the new bytes, never a partial write, and no temp litter remains.
assert target.read_bytes() == original_bytes
assert list(target.parent.glob(f"{target.name}.*.tmp")) == []
# --------------------------------------------------------------------------
# Surface 2: legacy cleanup symlink / hostile-path guards
# (src/local_deep_research/vector_stores/legacy_cleanup.py)
# --------------------------------------------------------------------------
def test_legacy_cleanup_refuses_symlinked_root(tmp_path, monkeypatch):
import local_deep_research.vector_stores.legacy_cleanup as cleanup
outside = tmp_path / "outside"
outside.mkdir()
foreign = outside / "foreign.pkl"
foreign.write_bytes(b"outside plaintext")
root = tmp_path / "rag_indices"
try:
root.symlink_to(outside, target_is_directory=True)
except (OSError, NotImplementedError):
pytest.skip("Cannot create symlinks in this environment")
monkeypatch.setattr(cleanup, "_rag_cache_root", lambda: root)
result = cleanup.migrate_legacy_docstores()
assert result["scan_errors"] == 1
assert result["deleted"] == 0
assert foreign.exists() # nothing outside the intended tree was touched
def test_legacy_cleanup_flags_symlinked_pkl_without_touching_target(
tmp_path, monkeypatch
):
import local_deep_research.vector_stores.legacy_cleanup as cleanup
root = tmp_path / "rag_indices"
root.mkdir()
outside = tmp_path / "outside.pkl"
outside.write_bytes(b"outside plaintext")
linked_pkl = root / "linked.pkl"
try:
linked_pkl.symlink_to(outside)
except (OSError, NotImplementedError):
pytest.skip("Cannot create symlinks in this environment")
monkeypatch.setattr(cleanup, "_rag_cache_root", lambda: root)
result = cleanup.migrate_legacy_docstores()
# A symlinked .pkl must be flagged (incomplete migration), never
# unlinked-and-declared-clean while the real plaintext target survives.
assert result["scan_errors"] == 1
assert result["deleted"] == 0
assert linked_pkl.is_symlink()
assert outside.read_bytes() == b"outside plaintext"
def test_legacy_cleanup_deletes_real_pkl_inside_cache_root(
tmp_path, monkeypatch
):
import local_deep_research.vector_stores.legacy_cleanup as cleanup
root = tmp_path / "rag_indices"
root.mkdir()
real_pkl = root / "legacy.pkl"
real_pkl.write_bytes(b"invalid legacy pickle")
monkeypatch.setattr(cleanup, "_rag_cache_root", lambda: root)
result = cleanup.migrate_legacy_docstores()
# Only REAL in-root .pkl files are migrated/deleted (unreadable ones
# still deleted for plaintext removal, falling back to reindex).
assert result["found"] == 1
assert result["deleted"] == 1
assert result["reindex_fallback"] == 1
assert not real_pkl.exists()
# --------------------------------------------------------------------------
# Surface 3: verified JSON writes scrub sensitive keys (unmocked happy path)
# (src/local_deep_research/security/file_write_verifier.py)
# --------------------------------------------------------------------------
def test_write_json_verified_scrubs_nested_secrets_to_real_file(tmp_path):
from local_deep_research.security.file_write_verifier import (
write_json_verified,
)
target = tmp_path / "verified.json"
secret = "nested-secret-value"
# No mocks: a real settings snapshot gates a real file write.
write_json_verified(
target,
{
"name": "safe",
"nested": {"api_key": secret},
"items": [{"password": secret}],
},
"system.allow_config_write",
settings_snapshot={"system.allow_config_write": True},
)
written_text = target.read_text()
assert secret not in written_text
assert json.loads(written_text) == {
"name": "safe",
"nested": {"api_key": "[REDACTED]"},
"items": [{"password": "[REDACTED]"}],
}
# --------------------------------------------------------------------------
# Surface 4: /api/save_raw_config verified-write composition
# (src/local_deep_research/web/routers/research.py)
#
# This surface used to be exercised here by calling the Flask route handler
# directly through a bare ``app.test_request_context()`` — the FastAPI
# migration (#3299) deleted ``web.routes.research_routes`` entirely, so
# that harness no longer imports. Rather than re-invent a second harness
# in this file, the coverage lives in
# ``tests/security/test_save_raw_config_blocked_keys_fastapi.py``, which
# drives the real FastAPI route end to end (auth + CSRF + the real
# ``TestClient``) and is a strict superset of what stood here: every
# blocked-key pattern, the open/closed write gate, and the real verified
# write landing on disk unmocked.
# --------------------------------------------------------------------------
# --------------------------------------------------------------------------
# Surface 5: report path containment + delete scoping
# (src/local_deep_research/storage/file.py)
# --------------------------------------------------------------------------
@pytest.mark.parametrize(
"research_id", ["../escape", "/tmp/escape", "bad\x00id"]
)
def test_report_path_rejects_hostile_research_ids(tmp_path, research_id):
from local_deep_research.storage.file import FileReportStorage
storage = FileReportStorage(tmp_path / "outputs")
with pytest.raises(ValueError):
storage._get_report_path(research_id)
def test_delete_report_cannot_unlink_outside_outputs_root(tmp_path):
from local_deep_research.storage.file import FileReportStorage
outputs = tmp_path / "outputs"
outside = tmp_path / "escape.md"
outside.write_text("canary")
storage = FileReportStorage(outputs)
assert storage.delete_report("../escape") is False
assert outside.read_text() == "canary"
def test_delete_report_only_removes_files_under_outputs_root(tmp_path):
from local_deep_research.storage.file import FileReportStorage
outputs = tmp_path / "outputs"
storage = FileReportStorage(outputs)
report = outputs / "report.md"
metadata = outputs / "report_metadata.json"
canary = tmp_path / "canary.md"
report.write_text("report")
metadata.write_text("{}")
canary.write_text("outside")
assert storage.delete_report("report") is True
assert not report.exists()
assert not metadata.exists()
assert canary.read_text() == "outside"
# --------------------------------------------------------------------------
# Surface 6: backup prune safety
# (src/local_deep_research/database/backup/backup_service.py)
# --------------------------------------------------------------------------
@pytest.fixture
def backup_service(tmp_path, monkeypatch):
# Resolve every BackupService path inside tmp_path via the real
# path config (no attribute overrides, nothing outside tmp_path).
monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
from local_deep_research.database.backup.backup_service import BackupService
return BackupService("test-user", "test-password", max_backups=1)
def test_backup_prune_only_removes_matching_direct_child_backups(
backup_service,
):
backup_dir = backup_service.backup_dir
newest = backup_dir / "ldr_backup_new.db"
old = backup_dir / "ldr_backup_old.db"
keep = backup_dir / "keep.txt"
odd_name = backup_dir / "ldr_backup_old.db.bak"
nested = backup_dir / "nested" / "ldr_backup_nested.db"
newest.write_bytes(b"new")
old.write_bytes(b"old")
keep.write_text("keep")
odd_name.write_text("keep")
nested.parent.mkdir()
nested.write_bytes(b"nested")
old_time = time.time() - 10
os.utime(old, (old_time, old_time))
assert backup_service._cleanup_old_backups() == 1
assert newest.exists() # within max count -> kept
assert not old.exists() # exceeds max count -> pruned
assert keep.exists() # non-matching name -> untouched
assert odd_name.exists() # weird name not matching the glob -> untouched
assert nested.exists() # nested dir (glob does not recurse) -> untouched
def test_backup_prune_only_removes_stale_matching_tmp_files(backup_service):
backup_dir = backup_service.backup_dir
stale_tmp = backup_dir / "ldr_backup_old.db.tmp"
fresh_tmp = backup_dir / "ldr_backup_fresh.db.tmp"
nonmatching_tmp = backup_dir / "keep.db.tmp"
nested_tmp = backup_dir / "nested" / "ldr_backup_nested.db.tmp"
for path in (stale_tmp, fresh_tmp, nonmatching_tmp):
path.write_bytes(b"tmp")
nested_tmp.parent.mkdir()
nested_tmp.write_bytes(b"tmp")
old_time = time.time() - 3601
os.utime(stale_tmp, (old_time, old_time))
# Stale .tmp sweeps are not counted as backup deletions.
assert backup_service._cleanup_old_backups() == 0
assert not stale_tmp.exists() # older than the 1h cutoff -> swept
assert fresh_tmp.exists() # in-flight temp -> spared
assert nonmatching_tmp.exists() # wrong prefix -> spared
assert nested_tmp.exists() # nested (non-child) -> spared
def test_backup_prune_spares_matching_symlink(backup_service, tmp_path):
backup_dir = backup_service.backup_dir
real = backup_dir / "ldr_backup_keep.db"
real.write_bytes(b"real")
outside = tmp_path / "outside.db"
outside.write_bytes(b"canary")
link = backup_dir / "ldr_backup_link.db"
try:
link.symlink_to(outside)
except (OSError, NotImplementedError):
pytest.skip("Cannot create symlinks in this environment")
assert backup_service._cleanup_old_backups() == 0
assert link.is_symlink() # a symlink match is never unlinked
assert outside.read_bytes() == b"canary"
assert real.exists()