~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Crash leftovers are cosmetic; sweep them only during a locked cold open."""
import multiprocessing
import os
from pathlib import Path
from uuid import uuid4
import pytest
from sqlalchemy import text
from local_deep_research.database import rekey_lock
from local_deep_research.database.encrypted_db import DatabaseManager
def _crash_writer(db, kind, ready, release):
path = Path(db)
def pause_replace(source, destination):
ready.send(str(source))
release.wait(30)
raise RuntimeError("parent did not kill writer")
rekey_lock.os.replace = pause_replace
with rekey_lock.acquire_rekey_lock(path):
rekey_lock._write_epoch_token(
path.with_name(f"{path.name}.rekey.{kind}"), str(uuid4()), path
)
def _orphan(db, kind):
ctx = multiprocessing.get_context("spawn")
receiver, sender = ctx.Pipe(duplex=False)
release = ctx.Event()
child = ctx.Process(
target=_crash_writer, args=(str(db), kind, sender, release)
)
child.start()
sender.close()
try:
assert receiver.poll(20), "writer never reached replace"
path = Path(receiver.recv())
assert path.is_file()
assert path.read_text() # real flushed production token
child.kill()
child.join(10)
assert not child.is_alive()
assert child.exitcode != 0
assert path.is_file(), "kill must bypass writer's finally unlink"
return path
finally:
if child.is_alive():
child.kill()
child.join(10)
receiver.close()
child.close()
@pytest.fixture
def native_db(tmp_path, monkeypatch):
monkeypatch.setenv("LDR_DB_KDF_ITERATIONS", "1000")
monkeypatch.setenv("LDR_ALLOW_UNENCRYPTED", "false")
manager = DatabaseManager()
assert manager.has_encryption
manager.data_dir = tmp_path
# Phase two concerns vector stores, not the main encrypted database.
monkeypatch.setattr(manager, "_run_phase2_rekey", lambda *args: None)
engine = manager.create_user_database("alice", "TestPass1!")
with engine.begin() as conn:
assert conn.exec_driver_sql("PRAGMA cipher_version").scalar()
conn.execute(text("CREATE TABLE cleanup_probe (value TEXT)"))
conn.execute(text("INSERT INTO cleanup_probe VALUES ('preserved')"))
manager.close_user_database("alice")
yield manager, manager._get_user_db_path("alice")
manager.close_all_databases()
def _open_preserved(manager):
engine = manager.open_user_database("alice", "TestPass1!")
assert engine is not None
with engine.connect() as conn:
assert (
conn.execute(text("SELECT value FROM cleanup_probe")).scalar()
== "preserved"
)
return engine
def test_cold_open_removes_real_crash_orphans(native_db):
manager, db = native_db
leftovers = [_orphan(db, kind) for kind in ("epoch", "pending")]
for path in leftovers:
assert path.name.startswith(f".{db.name}.rekey.")
os.utime(path, (1, 1))
hardlink = db.with_name(f".{db.name}.rekey.epoch.hardlink")
hardlink.hardlink_to(db)
symlink = db.with_name(f".{db.name}.rekey.pending.symlink0")
symlink.symlink_to(db)
os.utime(hardlink, (1, 1))
_open_preserved(manager)
assert all(not path.exists() for path in leftovers), (
"stale crash temps survived cold open"
)
assert hardlink.stat().st_ino == db.stat().st_ino
assert symlink.is_symlink()
manager.close_user_database("alice")
_open_preserved(manager)
def _temp(db, suffix="epoch.abcdefgh", mtime=1):
path = db.with_name(f".{db.name}.rekey.{suffix}")
path.write_text("cosmetic", encoding="ascii")
os.utime(path, (mtime, mtime))
return path
@pytest.mark.parametrize("kind", ["epoch", "pending"])
def test_literal_names_age_and_special_file_preservation(
tmp_path, monkeypatch, kind
):
db = tmp_path / ("user[ab].db" if os.name == "nt" else "user[ab]*?.db")
now = 10000
monkeypatch.setattr(rekey_lock.time, "time", lambda: now)
prefix = f".{db.name}.rekey.{kind}."
stale = _temp(db, f"{kind}.stale000", now - 3600 - 1)
keep = [
_temp(db, f"{kind}.boundary", now - 3600),
_temp(db, f"{kind}.fresh000", now - 3599),
_temp(db, f"{kind}.future00", now + 1),
_temp(db, f"{kind}.", 1),
_temp(
tmp_path / ("usera.db" if os.name == "nt" else "useraa.db"),
f"{kind}.other000",
),
_temp(tmp_path / f"{db.name}.other", f"{kind}.other000"),
]
# The database and every durable/non-temp metadata name are outside scope.
for suffix in (
"",
"-wal",
"-shm",
".salt",
"-journal",
".backup",
".rekey.epoch",
".rekey.pending",
".rekey.lock",
):
path = db.with_name(db.name + suffix)
path.write_text("preserve", encoding="ascii")
os.utime(path, (1, 1))
keep.append(path)
target = tmp_path / "target"
target.write_text("target", encoding="ascii")
symlink_target = tmp_path / "symlink-target"
symlink_target.write_text("symlink target", encoding="ascii")
os.utime(symlink_target, (1, 1))
symlink = tmp_path / (prefix + "symlink0")
symlink.symlink_to(symlink_target)
if os.utime in os.supports_follow_symlinks:
os.utime(symlink, (1, 1), follow_symlinks=False)
hardlink = tmp_path / (prefix + "hardlink")
hardlink.hardlink_to(target)
os.utime(target, (1, 1))
directory = tmp_path / (prefix + "director")
directory.mkdir()
fifo = tmp_path / (prefix + "fifo0000")
if hasattr(os, "mkfifo"):
os.mkfifo(fifo)
os.utime(fifo, (1, 1))
os.utime(directory, (1, 1))
nested_other = _temp(tmp_path / f"{db.name}.rekey.{kind}.otherdb0")
invalid_suffix = _temp(db, f"{kind}.INVALID!")
backup = _temp(db, f"{kind}.abcdefgh.backup")
with rekey_lock.acquire_rekey_lock(db):
rekey_lock.cleanup_stale_rekey_temps(db)
assert not stale.exists()
assert nested_other.exists()
assert invalid_suffix.exists()
assert backup.exists()
assert all(path.exists() for path in keep)
assert symlink.is_symlink()
assert hardlink.stat().st_nlink == 2
assert directory.is_dir()
if hasattr(os, "mkfifo"):
assert fifo.exists()
assert target.read_text() == "target"
assert symlink_target.read_text() == "symlink target"
assert all(path.read_text() == "preserve" for path in keep[6:])
def _hold_lock(db, ready, release):
with rekey_lock.acquire_rekey_lock(Path(db)):
ready.send("acquired")
if not release.wait(30):
raise RuntimeError("parent did not release holder")
def test_cold_open_excluded_by_foreign_lock_then_cleans(native_db):
manager, db = native_db
stale = _temp(db)
ctx = multiprocessing.get_context("spawn")
receiver, sender = ctx.Pipe(duplex=False)
release = ctx.Event()
child = ctx.Process(target=_hold_lock, args=(str(db), sender, release))
child.start()
sender.close()
try:
assert receiver.poll(20)
assert receiver.recv() == "acquired"
with pytest.raises(rekey_lock.RekeyLockContentionError):
manager.open_user_database("alice", "TestPass1!")
assert stale.exists()
release.set()
child.join(10)
assert not child.is_alive()
assert child.exitcode == 0
_open_preserved(manager)
assert not stale.exists()
manager.close_user_database("alice")
_open_preserved(manager)
finally:
release.set()
if child.is_alive():
child.kill()
child.join(10)
receiver.close()
child.close()
def test_cache_and_probes_never_scan_cold_open_holds_sidecar(
native_db, monkeypatch
):
manager, db = native_db
engine = _open_preserved(manager)
stale = _temp(db)
scans = []
real_iterdir = Path.iterdir
def observe(directory):
if directory == db.parent:
scans.append(directory)
# A separate process must actually contend during the scan.
ctx = multiprocessing.get_context("spawn")
receiver, sender = ctx.Pipe(duplex=False)
child = ctx.Process(target=_try_lock, args=(str(db), sender))
child.start()
sender.close()
try:
assert receiver.poll(20)
outcome = receiver.recv()
child.join(10)
assert not child.is_alive()
assert child.exitcode == 0
scans.append(outcome)
finally:
if child.is_alive():
child.kill()
child.join(10)
receiver.close()
child.close()
return real_iterdir(directory)
monkeypatch.setattr(Path, "iterdir", observe)
assert manager.open_user_database("alice", "TestPass1!") is engine
assert manager.is_user_connected("alice")
assert manager._password_matches_cached("alice", "TestPass1!")
assert scans == []
assert stale.exists()
manager.close_user_database("alice")
_open_preserved(manager)
# Assertions outside the cleanup's best-effort exception handlers.
assert scans == [db.parent, "contended"]
assert not stale.exists()
def _try_lock(db, ready):
try:
with rekey_lock.acquire_rekey_lock(Path(db)):
result = "unexpected acquisition"
except rekey_lock.RekeyLockContentionError:
result = "contended"
ready.send(result)
@pytest.mark.parametrize("operation", ["list", "lstat", "unlink", "disappear"])
def test_cosmetic_io_failure_still_opens_encrypted_data(
native_db, monkeypatch, operation
):
manager, db = native_db
stale = _temp(db)
reached = []
real_iterdir, real_lstat, real_unlink = (
Path.iterdir,
Path.lstat,
Path.unlink,
)
def failing_list(path):
if path == db.parent:
reached.append("list")
raise PermissionError("injected list denial")
return real_iterdir(path)
def failing_lstat(path):
if path == stale:
reached.append("lstat")
raise PermissionError("injected lstat denial")
return real_lstat(path)
def failing_unlink(path, *args, **kwargs):
if path == stale:
reached.append(operation)
if operation == "disappear":
real_unlink(path)
raise FileNotFoundError("entry disappeared")
raise PermissionError("injected unlink denial")
return real_unlink(path, *args, **kwargs)
with monkeypatch.context() as patch:
if operation == "list":
patch.setattr(Path, "iterdir", failing_list)
elif operation == "lstat":
patch.setattr(Path, "lstat", failing_lstat)
else:
patch.setattr(Path, "unlink", failing_unlink)
_open_preserved(manager)
assert reached == [operation]
assert stale.exists() == (operation != "disappear")
manager.close_user_database("alice")
_open_preserved(manager)
assert not stale.exists()
def test_sidecar_operational_failure_still_propagates(native_db, monkeypatch):
manager, db = native_db
stale = _temp(db)
reached = []
real_open = rekey_lock.os.open
def denied(path, *args, **kwargs):
if Path(path) == db.with_name(f"{db.name}.rekey.lock"):
reached.append(path)
raise PermissionError(13, "injected sidecar denial")
return real_open(path, *args, **kwargs)
with monkeypatch.context() as patch:
patch.setattr(rekey_lock.os, "open", denied)
with pytest.raises(rekey_lock.RekeySidecarIOError) as caught:
manager.open_user_database("alice", "TestPass1!")
assert reached
assert caught.value.operation == rekey_lock.RekeySidecarOperation.OPEN
assert stale.exists()
_open_preserved(manager)
assert not stale.exists()
@pytest.mark.nonroot
def test_real_unlink_permission_denial_is_cosmetic(native_db, monkeypatch):
if os.name == "nt" or os.geteuid() == 0:
pytest.skip("requires POSIX permission enforcement")
manager, db = native_db
stale = _temp(db)
real_unlink = Path.unlink
denials = []
def protected_unlink(path, *args, **kwargs):
if path != stale:
return real_unlink(path, *args, **kwargs)
original_mode = db.parent.stat().st_mode & 0o777
db.parent.chmod(0o500)
try:
try:
return real_unlink(path, *args, **kwargs)
except PermissionError as error:
denials.append(error.errno)
raise
finally:
db.parent.chmod(original_mode)
with monkeypatch.context() as patch:
patch.setattr(Path, "unlink", protected_unlink)
_open_preserved(manager)
assert denials == [13]
assert stale.exists()
manager.close_user_database("alice")
_open_preserved(manager)
assert not stale.exists()