~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
from __future__ import annotations
import threading
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from pathlib import Path
from typing import Literal, assert_never
from unittest.mock import MagicMock, patch
import pytest
from local_deep_research.database import rekey_lock as rekey_lock_module
from local_deep_research.database.encrypted_db import DatabaseManager
from local_deep_research.database.rekey_lock import (
RekeyEpochState,
acquire_rekey_lock,
)
def _new_manager(data_dir: Path) -> DatabaseManager:
with patch.object(
DatabaseManager, "_check_encryption_available", return_value=True
):
manager = DatabaseManager()
manager.data_dir = data_dir
return manager
@pytest.fixture
def manager(tmp_path: Path) -> DatabaseManager:
return _new_manager(tmp_path)
def _existing_db(manager: DatabaseManager, username: str) -> Path:
db_path = manager._get_user_db_path(username)
db_path.touch()
return db_path
def _initialize_epoch(db_path: Path) -> str:
with acquire_rekey_lock(db_path):
return rekey_lock_module.read_or_initialize_rekey_epoch(db_path)
type _AccessorName = Literal[
"is_user_connected",
"_password_matches_cached",
"open_user_database",
"get_session",
"check_database_integrity",
"create_thread_safe_session_for_metrics",
]
def _invoke_cache_accessor(
manager: DatabaseManager,
accessor: _AccessorName,
password: str,
) -> MagicMock | bool | None:
match accessor:
case "is_user_connected":
return manager.is_user_connected("alice")
case "_password_matches_cached":
return manager._password_matches_cached("alice", password)
case "open_user_database":
return manager.open_user_database("alice", password)
case "get_session":
return manager.get_session("alice")
case "check_database_integrity":
return manager.check_database_integrity("alice")
case "create_thread_safe_session_for_metrics":
return manager.create_thread_safe_session_for_metrics(
"alice", password
)
case unreachable:
assert_never(unreachable)
def _configure_integrity_engine(engine: MagicMock) -> None:
connection = engine.connect.return_value.__enter__.return_value
quick_check = MagicMock()
quick_check.fetchone.return_value = ("ok",)
wal_checkpoint = MagicMock()
wal_checkpoint.fetchone.return_value = (0, 0, 0)
cipher_check = MagicMock()
cipher_check.__iter__.return_value = iter(())
connection.execute.side_effect = [quick_check, wal_checkpoint, cipher_check]
@pytest.mark.parametrize(
"accessor",
[
"is_user_connected",
"_password_matches_cached",
"open_user_database",
"get_session",
"check_database_integrity",
"create_thread_safe_session_for_metrics",
],
)
def test_cache_accessor_retries_e1_snapshot_after_e2_publication(
manager: DatabaseManager,
accessor: _AccessorName,
) -> None:
# Given every accessor has snapshotted E1 before acquiring its sidecar
db_path = _existing_db(manager, "alice")
e1_epoch = _initialize_epoch(db_path)
e1_engine = MagicMock()
e2_engine = MagicMock()
_configure_integrity_engine(e2_engine)
manager._cache_connection(
"alice",
manager._make_connection_cache_entry(
e1_engine, "old-password", e1_epoch
),
)
snapshot_ready = threading.Event()
continue_claim = threading.Event()
real_acquire = acquire_rekey_lock
first_acquire = True
bound_engines: list[MagicMock] = []
expected_session = MagicMock()
@contextmanager
def pause_first_sidecar(path: Path, **kwargs: object):
nonlocal first_acquire
if first_acquire:
first_acquire = False
snapshot_ready.set()
assert continue_claim.wait(timeout=5)
with real_acquire(path, **kwargs):
yield
def make_session_factory(*, bind):
bound_engines.append(bind)
return MagicMock(return_value=expected_session)
with (
patch(
"local_deep_research.database.encrypted_db.acquire_rekey_lock",
side_effect=pause_first_sidecar,
),
patch(
"local_deep_research.database.encrypted_db.sessionmaker",
side_effect=make_session_factory,
),
ThreadPoolExecutor(max_workers=1) as executor,
):
accessor_result = executor.submit(
_invoke_cache_accessor, manager, accessor, "new-password"
)
assert snapshot_ready.wait(timeout=5)
# When E2 becomes durable and replaces all three local cache identities
with real_acquire(db_path):
e2_epoch = rekey_lock_module.advance_rekey_epoch(db_path)
manager._cache_connection(
"alice",
manager._make_connection_cache_entry(
e2_engine, "new-password", e2_epoch
),
)
continue_claim.set()
result = accessor_result.result(timeout=5)
# Then the accessor retries E2 without detaching or using either wrong owner
assert manager.connections["alice"] is e2_engine
e1_engine.dispose.assert_not_called()
e2_engine.dispose.assert_not_called()
e1_engine.connect.assert_not_called()
match accessor:
case "is_user_connected" | "_password_matches_cached":
assert result is True
case "open_user_database":
assert result is e2_engine
case "get_session" | "create_thread_safe_session_for_metrics":
assert result is expected_session
assert bound_engines == [e2_engine]
case "check_database_integrity":
assert result is True
e2_engine.connect.assert_called_once_with()
case unreachable:
assert_never(unreachable)
def test_e1_snapshot_detaches_only_e1_after_durable_e2(
manager: DatabaseManager,
) -> None:
# Given an accessor paused after snapshotting the only local E1 generation
db_path = _existing_db(manager, "alice")
e1_epoch = _initialize_epoch(db_path)
e1_engine = MagicMock()
manager._cache_connection(
"alice",
manager._make_connection_cache_entry(e1_engine, "password", e1_epoch),
)
snapshot_ready = threading.Event()
continue_claim = threading.Event()
real_acquire = acquire_rekey_lock
@contextmanager
def pause_sidecar(path: Path, **kwargs: object):
snapshot_ready.set()
assert continue_claim.wait(timeout=5)
with real_acquire(path, **kwargs):
yield
with (
patch(
"local_deep_research.database.encrypted_db.acquire_rekey_lock",
side_effect=pause_sidecar,
),
ThreadPoolExecutor(max_workers=1) as executor,
):
connected = executor.submit(manager.is_user_connected, "alice")
assert snapshot_ready.wait(timeout=5)
# When only the durable generation advances to E2
with real_acquire(db_path):
rekey_lock_module.advance_rekey_epoch(db_path)
continue_claim.set()
result = connected.result(timeout=5)
# Then the exact E1 snapshot is detached and disposed once without checkpoint
assert result is False
e1_engine.dispose.assert_called_once_with()
e1_engine.connect.assert_not_called()
assert "alice" not in manager.connections
def test_verifier_identity_change_retries_exact_generation(
manager: DatabaseManager,
) -> None:
# Given a password check paused with an old verifier identity snapshot
db_path = _existing_db(manager, "alice")
epoch = _initialize_epoch(db_path)
engine = MagicMock()
manager._cache_connection(
"alice",
manager._make_connection_cache_entry(engine, "old-password", epoch),
)
snapshot_ready = threading.Event()
continue_claim = threading.Event()
real_acquire = acquire_rekey_lock
@contextmanager
def pause_first_sidecar(path: Path, **kwargs: object):
snapshot_ready.set()
assert continue_claim.wait(timeout=5)
with real_acquire(path, **kwargs):
yield
with (
patch(
"local_deep_research.database.encrypted_db.acquire_rekey_lock",
side_effect=pause_first_sidecar,
),
ThreadPoolExecutor(max_workers=1) as executor,
):
password_matches = executor.submit(
manager._password_matches_cached, "alice", "new-password"
)
assert snapshot_ready.wait(timeout=5)
# When only the verifier object is replaced before the exact claim
replacement_verifier = manager._make_verifier("new-password")
with manager._connections_lock:
manager._password_verifiers["alice"] = replacement_verifier
continue_claim.set()
result = password_matches.result(timeout=5)
# Then identity change retries and validates only the replacement verifier
assert result is True
assert manager._password_verifiers["alice"] is replacement_verifier
engine.dispose.assert_not_called()
@pytest.mark.parametrize(
"accessor",
[
"is_user_connected",
"_password_matches_cached",
"open_user_database",
"get_session",
"check_database_integrity",
"create_thread_safe_session_for_metrics",
],
)
def test_pending_generation_fences_snapshotted_cache(
manager: DatabaseManager,
accessor: _AccessorName,
) -> None:
# Given every accessor is paused after snapshotting a stable generation
db_path = _existing_db(manager, "alice")
epoch = _initialize_epoch(db_path)
engine = MagicMock()
replacement_engine = MagicMock()
manager._cache_connection(
"alice",
manager._make_connection_cache_entry(engine, "password", epoch),
)
snapshot_ready = threading.Event()
continue_claim = threading.Event()
real_acquire = acquire_rekey_lock
first_acquire = True
expected_session = MagicMock()
bound_engines: list[MagicMock] = []
@contextmanager
def pause_first_sidecar(path: Path, **kwargs: object):
nonlocal first_acquire
if first_acquire:
first_acquire = False
snapshot_ready.set()
assert continue_claim.wait(timeout=5)
with real_acquire(path, **kwargs):
yield
def recover_pending(
path: Path, _password: str, state: RekeyEpochState
) -> bool:
pending_epoch = state.pending_epoch
assert pending_epoch is not None
rekey_lock_module.commit_rekey_epoch(path, pending_epoch)
rekey_lock_module.clear_pending_rekey_epoch(path, pending_epoch)
return True
def make_session_factory(*, bind):
bound_engines.append(bind)
return MagicMock(return_value=expected_session)
with (
patch(
"local_deep_research.database.encrypted_db.acquire_rekey_lock",
side_effect=pause_first_sidecar,
),
patch.object(
manager, "_recover_pending_rekey", side_effect=recover_pending
) as recover,
patch.object(
manager,
"_open_user_database_cold",
return_value=replacement_engine,
),
patch.object(manager, "_run_phase2_rekey"),
patch(
"local_deep_research.database.encrypted_db.sessionmaker",
side_effect=make_session_factory,
),
ThreadPoolExecutor(max_workers=1) as executor,
):
accessor_result = executor.submit(
_invoke_cache_accessor, manager, accessor, "new-password"
)
assert snapshot_ready.wait(timeout=5)
# When a pending generation becomes durable before the exact claim
with real_acquire(db_path):
rekey_lock_module.begin_rekey_epoch(db_path)
continue_claim.set()
result = accessor_result.result(timeout=5)
# Then no accessor uses the fenced engine and credentialed paths recover cold
engine.dispose.assert_called_once_with()
engine.connect.assert_not_called()
match accessor:
case "is_user_connected" | "_password_matches_cached":
assert result is False
recover.assert_not_called()
assert "alice" not in manager.connections
case "get_session":
assert result is None
recover.assert_not_called()
assert "alice" not in manager.connections
case "check_database_integrity":
assert result is False
recover.assert_not_called()
assert "alice" not in manager.connections
case "open_user_database":
assert result is replacement_engine
recover.assert_called_once()
assert manager.connections["alice"] is replacement_engine
case "create_thread_safe_session_for_metrics":
assert result is expected_session
assert bound_engines == [replacement_engine]
recover.assert_called_once()
assert manager.connections["alice"] is replacement_engine
case unreachable:
assert_never(unreachable)
def test_cache_accessor_waits_for_live_sidecar_before_claim(
manager: DatabaseManager,
) -> None:
# Given a current cache generation and another owner holding its sidecar
db_path = _existing_db(manager, "alice")
epoch = _initialize_epoch(db_path)
engine = MagicMock()
manager._cache_connection(
"alice",
manager._make_connection_cache_entry(engine, "password", epoch),
)
sidecar_attempted = threading.Event()
sidecar_released = threading.Event()
real_acquire = acquire_rekey_lock
@contextmanager
def observe_sidecar_attempt(path: Path, **kwargs: object):
sidecar_attempted.set()
with real_acquire(path, **kwargs):
yield
real_claim = manager._claim_connection_generation
def claim_after_release(generation, state):
assert sidecar_released.is_set()
return real_claim(generation, state)
with (
patch(
"local_deep_research.database.encrypted_db.acquire_rekey_lock",
side_effect=observe_sidecar_attempt,
),
patch.object(
manager,
"_claim_connection_generation",
side_effect=claim_after_release,
) as claim,
ThreadPoolExecutor(max_workers=1) as executor,
):
with real_acquire(db_path):
connected = executor.submit(manager.is_user_connected, "alice")
assert sidecar_attempted.wait(timeout=5)
sidecar_released.set()
assert connected.result(timeout=5) is True
assert sidecar_released.is_set()
claim.assert_called_once()
def test_two_managers_evict_cross_worker_stale_cache(tmp_path: Path) -> None:
# Given two worker-local managers sharing one database directory
first = _new_manager(tmp_path)
second = _new_manager(tmp_path)
db_path = _existing_db(first, "alice")
original_epoch = _initialize_epoch(db_path)
stale_engine = MagicMock()
entry = first._make_connection_cache_entry(
stale_engine, "old-password", original_epoch
)
first._cache_connection("alice", entry)
with acquire_rekey_lock(second._get_user_db_path("alice")):
rekey_lock_module.advance_rekey_epoch(db_path)
with patch.object(first, "_checkpoint_wal"):
# When the first worker next consults its process-local cache
connected = first.is_user_connected("alice")
# Then the durable epoch fences and disposes that stale engine
assert connected is False
assert rekey_lock_module.read_rekey_epoch(db_path) != original_epoch
stale_engine.dispose.assert_called_once_with()
assert "alice" not in first.connections
assert "alice" not in first._password_verifiers
assert "alice" not in first._connection_epochs
def test_stale_eviction_does_not_checkpoint_while_rekey_is_pending(
tmp_path: Path,
) -> None:
# Given one worker observes durable pending metadata from another worker
stale_manager = _new_manager(tmp_path)
db_path = _existing_db(stale_manager, "alice")
original_epoch = _initialize_epoch(db_path)
stale_engine = MagicMock()
stale_manager._cache_connection(
"alice",
stale_manager._make_connection_cache_entry(
stale_engine, "old-password", original_epoch
),
)
with acquire_rekey_lock(db_path):
rekey_lock_module.begin_rekey_epoch(db_path)
with (
patch.object(
stale_manager,
"_checkpoint_wal",
wraps=stale_manager._checkpoint_wal,
) as checkpoint,
):
# When the stale worker consults its cache during the rekey window
connected = stale_manager.is_user_connected("alice")
# Then eviction only disposes: checkpointing would open the old-key engine
assert connected is False
checkpoint.assert_not_called()
stale_engine.connect.assert_not_called()
stale_engine.dispose.assert_called_once_with()
@pytest.mark.parametrize(
"case",
[
("is_user_connected", ("alice",), False),
("get_session", ("alice",), None),
("check_database_integrity", ("alice",), False),
("_password_matches_cached", ("alice", "old-password"), False),
],
)
def test_passwordless_cache_accessors_fence_stale_epoch(
manager: DatabaseManager,
case: tuple[str, tuple[str, ...], bool | None],
) -> None:
# Given a cache entry published under an older durable epoch
method_name, arguments, expected = case
db_path = _existing_db(manager, "alice")
old_epoch = _initialize_epoch(db_path)
stale_engine = MagicMock()
entry = manager._make_connection_cache_entry(
stale_engine, "old-password", old_epoch
)
manager._cache_connection("alice", entry)
with acquire_rekey_lock(db_path):
current_epoch = rekey_lock_module.advance_rekey_epoch(db_path)
with manager._connections_lock:
assert not manager._cached_engine_trusted(
"alice", "old-password", current_epoch
)
with patch.object(manager, "_checkpoint_wal"):
# When a passwordless or verifier-only accessor consults the cache
result = getattr(manager, method_name)(*arguments)
# Then it fails closed and evicts all parts of the stale cache entry
assert result == expected
stale_engine.dispose.assert_called_once_with()
assert "alice" not in manager.connections
assert "alice" not in manager._password_verifiers
assert "alice" not in manager._connection_epochs
def test_metrics_session_fences_stale_epoch_before_reopen(
manager: DatabaseManager,
) -> None:
# Given a stale metrics cache entry and a replacement engine
db_path = _existing_db(manager, "alice")
old_epoch = _initialize_epoch(db_path)
stale_engine = MagicMock()
replacement_engine = MagicMock()
entry = manager._make_connection_cache_entry(
stale_engine, "password", old_epoch
)
manager._cache_connection("alice", entry)
with acquire_rekey_lock(db_path):
rekey_lock_module.advance_rekey_epoch(db_path)
expected_session = MagicMock()
with (
patch.object(manager, "_checkpoint_wal"),
patch.object(
manager, "open_user_database", return_value=replacement_engine
) as reopen,
patch(
"local_deep_research.database.encrypted_db.sessionmaker",
return_value=lambda: expected_session,
),
):
# When a metrics worker requests a session
session = manager.create_thread_safe_session_for_metrics(
"alice", "password"
)
# Then it evicts the stale engine and reopens through the fenced path
assert session is expected_session
stale_engine.dispose.assert_called_once_with()
reopen.assert_called_once_with("alice", "password")
def test_open_evicts_stale_cache_before_cold_publication(
manager: DatabaseManager,
) -> None:
# Given a cached engine whose published epoch is no longer current
db_path = _existing_db(manager, "alice")
stale_epoch = _initialize_epoch(db_path)
stale_engine = MagicMock()
replacement_engine = MagicMock()
entry = manager._make_connection_cache_entry(
stale_engine, "password", stale_epoch
)
manager._cache_connection("alice", entry)
with acquire_rekey_lock(db_path):
current_epoch = rekey_lock_module.advance_rekey_epoch(db_path)
with (
patch.object(manager, "_checkpoint_wal"),
patch.object(
manager,
"_open_user_database_cold",
return_value=replacement_engine,
) as cold_open,
patch.object(manager, "_run_phase2_rekey"),
):
# When the worker opens with the valid password
opened = manager.open_user_database("alice", "password")
# Then both cache checks reject the stale generation before publication
assert opened is replacement_engine
cold_open.assert_called_once()
stale_engine.dispose.assert_called_once_with()
assert manager.connections["alice"] is replacement_engine
assert manager._connection_epochs["alice"] == current_epoch
def test_cold_open_waits_for_epoch_advance_and_rekey(
tmp_path: Path,
) -> None:
# Given cold open reaches a sidecar owned by a generation publisher
opener_manager = _new_manager(tmp_path)
db_path = _existing_db(opener_manager, "alice")
_initialize_epoch(db_path)
replacement_engine = MagicMock()
sidecar_attempted = threading.Event()
sidecar_released = threading.Event()
cold_started = threading.Event()
real_acquire = acquire_rekey_lock
@contextmanager
def observe_sidecar_attempt(path: Path, **kwargs: object):
sidecar_attempted.set()
with real_acquire(path, **kwargs):
yield
def controlled_cold_open(
_username: str,
_password: str,
_open_start: float,
prederived_hex_key: str | None = None,
) -> MagicMock:
assert sidecar_released.is_set()
cold_started.set()
return replacement_engine
with (
patch(
"local_deep_research.database.encrypted_db.acquire_rekey_lock",
side_effect=observe_sidecar_attempt,
),
patch.object(
opener_manager,
"_open_user_database_cold",
side_effect=controlled_cold_open,
),
patch.object(opener_manager, "_run_phase2_rekey"),
ThreadPoolExecutor(max_workers=1) as executor,
):
with real_acquire(db_path):
cold_open = executor.submit(
opener_manager.open_user_database, "alice", "new-password"
)
assert sidecar_attempted.wait(timeout=5)
current_epoch = rekey_lock_module.advance_rekey_epoch(db_path)
sidecar_released.set()
# Then physical cold-open begins only after durable publication and release
assert cold_open.result(timeout=5) is replacement_engine
assert cold_started.is_set()
assert opener_manager._connection_epochs["alice"] == current_epoch
def test_close_user_database_disposes_outside_connections_lock(
manager: DatabaseManager,
) -> None:
# Given a current-generation cached engine and a non-reentrant cache lock
db_path = _existing_db(manager, "alice")
epoch = _initialize_epoch(db_path)
engine = MagicMock()
manager._cache_connection(
"alice",
manager._make_connection_cache_entry(engine, "password", epoch),
)
cache_lock = threading.Lock()
manager._connections_lock = cache_lock
def assert_cache_lock_released(*_args) -> None:
assert cache_lock.acquire(blocking=False)
cache_lock.release()
engine.dispose.side_effect = assert_cache_lock_released
with patch.object(
manager, "_checkpoint_wal", side_effect=assert_cache_lock_released
) as checkpoint:
# When the user database is closed
manager.close_user_database("alice")
# Then detachment is atomic but checkpoint and dispose run after unlocking
checkpoint.assert_called_once_with(engine, "for alice")
engine.dispose.assert_called_once_with()
assert "alice" not in manager.connections
assert "alice" not in manager._password_verifiers
assert "alice" not in manager._connection_epochs