Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

24.1 KB
Raw Blame
from __future__ import annotations

import threading
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from pathlib import Path
from typing import Literal, assert_never
from unittest.mock import MagicMock, patch

import pytest

from local_deep_research.database import rekey_lock as rekey_lock_module
from local_deep_research.database.encrypted_db import DatabaseManager
from local_deep_research.database.rekey_lock import (
    RekeyEpochState,
    acquire_rekey_lock,
)


def _new_manager(data_dir: Path) -> DatabaseManager:
    with patch.object(
        DatabaseManager, "_check_encryption_available", return_value=True
    ):
        manager = DatabaseManager()
    manager.data_dir = data_dir
    return manager


@pytest.fixture
def manager(tmp_path: Path) -> DatabaseManager:
    return _new_manager(tmp_path)


def _existing_db(manager: DatabaseManager, username: str) -> Path:
    db_path = manager._get_user_db_path(username)
    db_path.touch()
    return db_path


def _initialize_epoch(db_path: Path) -> str:
    with acquire_rekey_lock(db_path):
        return rekey_lock_module.read_or_initialize_rekey_epoch(db_path)


type _AccessorName = Literal[
    "is_user_connected",
    "_password_matches_cached",
    "open_user_database",
    "get_session",
    "check_database_integrity",
    "create_thread_safe_session_for_metrics",
]


def _invoke_cache_accessor(
    manager: DatabaseManager,
    accessor: _AccessorName,
    password: str,
) -> MagicMock | bool | None:
    match accessor:
        case "is_user_connected":
            return manager.is_user_connected("alice")
        case "_password_matches_cached":
            return manager._password_matches_cached("alice", password)
        case "open_user_database":
            return manager.open_user_database("alice", password)
        case "get_session":
            return manager.get_session("alice")
        case "check_database_integrity":
            return manager.check_database_integrity("alice")
        case "create_thread_safe_session_for_metrics":
            return manager.create_thread_safe_session_for_metrics(
                "alice", password
            )
        case unreachable:
            assert_never(unreachable)


def _configure_integrity_engine(engine: MagicMock) -> None:
    connection = engine.connect.return_value.__enter__.return_value
    quick_check = MagicMock()
    quick_check.fetchone.return_value = ("ok",)
    wal_checkpoint = MagicMock()
    wal_checkpoint.fetchone.return_value = (0, 0, 0)
    cipher_check = MagicMock()
    cipher_check.__iter__.return_value = iter(())
    connection.execute.side_effect = [quick_check, wal_checkpoint, cipher_check]


@pytest.mark.parametrize(
    "accessor",
    [
        "is_user_connected",
        "_password_matches_cached",
        "open_user_database",
        "get_session",
        "check_database_integrity",
        "create_thread_safe_session_for_metrics",
    ],
)
def test_cache_accessor_retries_e1_snapshot_after_e2_publication(
    manager: DatabaseManager,
    accessor: _AccessorName,
) -> None:
    # Given every accessor has snapshotted E1 before acquiring its sidecar
    db_path = _existing_db(manager, "alice")
    e1_epoch = _initialize_epoch(db_path)
    e1_engine = MagicMock()
    e2_engine = MagicMock()
    _configure_integrity_engine(e2_engine)
    manager._cache_connection(
        "alice",
        manager._make_connection_cache_entry(
            e1_engine, "old-password", e1_epoch
        ),
    )
    snapshot_ready = threading.Event()
    continue_claim = threading.Event()
    real_acquire = acquire_rekey_lock
    first_acquire = True
    bound_engines: list[MagicMock] = []
    expected_session = MagicMock()

    @contextmanager
    def pause_first_sidecar(path: Path, **kwargs: object):
        nonlocal first_acquire
        if first_acquire:
            first_acquire = False
            snapshot_ready.set()
            assert continue_claim.wait(timeout=5)
        with real_acquire(path, **kwargs):
            yield

    def make_session_factory(*, bind):
        bound_engines.append(bind)
        return MagicMock(return_value=expected_session)

    with (
        patch(
            "local_deep_research.database.encrypted_db.acquire_rekey_lock",
            side_effect=pause_first_sidecar,
        ),
        patch(
            "local_deep_research.database.encrypted_db.sessionmaker",
            side_effect=make_session_factory,
        ),
        ThreadPoolExecutor(max_workers=1) as executor,
    ):
        accessor_result = executor.submit(
            _invoke_cache_accessor, manager, accessor, "new-password"
        )
        assert snapshot_ready.wait(timeout=5)

        # When E2 becomes durable and replaces all three local cache identities
        with real_acquire(db_path):
            e2_epoch = rekey_lock_module.advance_rekey_epoch(db_path)
            manager._cache_connection(
                "alice",
                manager._make_connection_cache_entry(
                    e2_engine, "new-password", e2_epoch
                ),
            )
        continue_claim.set()
        result = accessor_result.result(timeout=5)

    # Then the accessor retries E2 without detaching or using either wrong owner
    assert manager.connections["alice"] is e2_engine
    e1_engine.dispose.assert_not_called()
    e2_engine.dispose.assert_not_called()
    e1_engine.connect.assert_not_called()
    match accessor:
        case "is_user_connected" | "_password_matches_cached":
            assert result is True
        case "open_user_database":
            assert result is e2_engine
        case "get_session" | "create_thread_safe_session_for_metrics":
            assert result is expected_session
            assert bound_engines == [e2_engine]
        case "check_database_integrity":
            assert result is True
            e2_engine.connect.assert_called_once_with()
        case unreachable:
            assert_never(unreachable)


def test_e1_snapshot_detaches_only_e1_after_durable_e2(
    manager: DatabaseManager,
) -> None:
    # Given an accessor paused after snapshotting the only local E1 generation
    db_path = _existing_db(manager, "alice")
    e1_epoch = _initialize_epoch(db_path)
    e1_engine = MagicMock()
    manager._cache_connection(
        "alice",
        manager._make_connection_cache_entry(e1_engine, "password", e1_epoch),
    )
    snapshot_ready = threading.Event()
    continue_claim = threading.Event()
    real_acquire = acquire_rekey_lock

    @contextmanager
    def pause_sidecar(path: Path, **kwargs: object):
        snapshot_ready.set()
        assert continue_claim.wait(timeout=5)
        with real_acquire(path, **kwargs):
            yield

    with (
        patch(
            "local_deep_research.database.encrypted_db.acquire_rekey_lock",
            side_effect=pause_sidecar,
        ),
        ThreadPoolExecutor(max_workers=1) as executor,
    ):
        connected = executor.submit(manager.is_user_connected, "alice")
        assert snapshot_ready.wait(timeout=5)

        # When only the durable generation advances to E2
        with real_acquire(db_path):
            rekey_lock_module.advance_rekey_epoch(db_path)
        continue_claim.set()
        result = connected.result(timeout=5)

    # Then the exact E1 snapshot is detached and disposed once without checkpoint
    assert result is False
    e1_engine.dispose.assert_called_once_with()
    e1_engine.connect.assert_not_called()
    assert "alice" not in manager.connections


def test_verifier_identity_change_retries_exact_generation(
    manager: DatabaseManager,
) -> None:
    # Given a password check paused with an old verifier identity snapshot
    db_path = _existing_db(manager, "alice")
    epoch = _initialize_epoch(db_path)
    engine = MagicMock()
    manager._cache_connection(
        "alice",
        manager._make_connection_cache_entry(engine, "old-password", epoch),
    )
    snapshot_ready = threading.Event()
    continue_claim = threading.Event()
    real_acquire = acquire_rekey_lock

    @contextmanager
    def pause_first_sidecar(path: Path, **kwargs: object):
        snapshot_ready.set()
        assert continue_claim.wait(timeout=5)
        with real_acquire(path, **kwargs):
            yield

    with (
        patch(
            "local_deep_research.database.encrypted_db.acquire_rekey_lock",
            side_effect=pause_first_sidecar,
        ),
        ThreadPoolExecutor(max_workers=1) as executor,
    ):
        password_matches = executor.submit(
            manager._password_matches_cached, "alice", "new-password"
        )
        assert snapshot_ready.wait(timeout=5)

        # When only the verifier object is replaced before the exact claim
        replacement_verifier = manager._make_verifier("new-password")
        with manager._connections_lock:
            manager._password_verifiers["alice"] = replacement_verifier
        continue_claim.set()
        result = password_matches.result(timeout=5)

    # Then identity change retries and validates only the replacement verifier
    assert result is True
    assert manager._password_verifiers["alice"] is replacement_verifier
    engine.dispose.assert_not_called()


@pytest.mark.parametrize(
    "accessor",
    [
        "is_user_connected",
        "_password_matches_cached",
        "open_user_database",
        "get_session",
        "check_database_integrity",
        "create_thread_safe_session_for_metrics",
    ],
)
def test_pending_generation_fences_snapshotted_cache(
    manager: DatabaseManager,
    accessor: _AccessorName,
) -> None:
    # Given every accessor is paused after snapshotting a stable generation
    db_path = _existing_db(manager, "alice")
    epoch = _initialize_epoch(db_path)
    engine = MagicMock()
    replacement_engine = MagicMock()
    manager._cache_connection(
        "alice",
        manager._make_connection_cache_entry(engine, "password", epoch),
    )
    snapshot_ready = threading.Event()
    continue_claim = threading.Event()
    real_acquire = acquire_rekey_lock
    first_acquire = True
    expected_session = MagicMock()
    bound_engines: list[MagicMock] = []

    @contextmanager
    def pause_first_sidecar(path: Path, **kwargs: object):
        nonlocal first_acquire
        if first_acquire:
            first_acquire = False
            snapshot_ready.set()
            assert continue_claim.wait(timeout=5)
        with real_acquire(path, **kwargs):
            yield

    def recover_pending(
        path: Path, _password: str, state: RekeyEpochState
    ) -> bool:
        pending_epoch = state.pending_epoch
        assert pending_epoch is not None
        rekey_lock_module.commit_rekey_epoch(path, pending_epoch)
        rekey_lock_module.clear_pending_rekey_epoch(path, pending_epoch)
        return True

    def make_session_factory(*, bind):
        bound_engines.append(bind)
        return MagicMock(return_value=expected_session)

    with (
        patch(
            "local_deep_research.database.encrypted_db.acquire_rekey_lock",
            side_effect=pause_first_sidecar,
        ),
        patch.object(
            manager, "_recover_pending_rekey", side_effect=recover_pending
        ) as recover,
        patch.object(
            manager,
            "_open_user_database_cold",
            return_value=replacement_engine,
        ),
        patch.object(manager, "_run_phase2_rekey"),
        patch(
            "local_deep_research.database.encrypted_db.sessionmaker",
            side_effect=make_session_factory,
        ),
        ThreadPoolExecutor(max_workers=1) as executor,
    ):
        accessor_result = executor.submit(
            _invoke_cache_accessor, manager, accessor, "new-password"
        )
        assert snapshot_ready.wait(timeout=5)

        # When a pending generation becomes durable before the exact claim
        with real_acquire(db_path):
            rekey_lock_module.begin_rekey_epoch(db_path)
        continue_claim.set()
        result = accessor_result.result(timeout=5)

    # Then no accessor uses the fenced engine and credentialed paths recover cold
    engine.dispose.assert_called_once_with()
    engine.connect.assert_not_called()
    match accessor:
        case "is_user_connected" | "_password_matches_cached":
            assert result is False
            recover.assert_not_called()
            assert "alice" not in manager.connections
        case "get_session":
            assert result is None
            recover.assert_not_called()
            assert "alice" not in manager.connections
        case "check_database_integrity":
            assert result is False
            recover.assert_not_called()
            assert "alice" not in manager.connections
        case "open_user_database":
            assert result is replacement_engine
            recover.assert_called_once()
            assert manager.connections["alice"] is replacement_engine
        case "create_thread_safe_session_for_metrics":
            assert result is expected_session
            assert bound_engines == [replacement_engine]
            recover.assert_called_once()
            assert manager.connections["alice"] is replacement_engine
        case unreachable:
            assert_never(unreachable)


def test_cache_accessor_waits_for_live_sidecar_before_claim(
    manager: DatabaseManager,
) -> None:
    # Given a current cache generation and another owner holding its sidecar
    db_path = _existing_db(manager, "alice")
    epoch = _initialize_epoch(db_path)
    engine = MagicMock()
    manager._cache_connection(
        "alice",
        manager._make_connection_cache_entry(engine, "password", epoch),
    )
    sidecar_attempted = threading.Event()
    sidecar_released = threading.Event()
    real_acquire = acquire_rekey_lock

    @contextmanager
    def observe_sidecar_attempt(path: Path, **kwargs: object):
        sidecar_attempted.set()
        with real_acquire(path, **kwargs):
            yield

    real_claim = manager._claim_connection_generation

    def claim_after_release(generation, state):
        assert sidecar_released.is_set()
        return real_claim(generation, state)

    with (
        patch(
            "local_deep_research.database.encrypted_db.acquire_rekey_lock",
            side_effect=observe_sidecar_attempt,
        ),
        patch.object(
            manager,
            "_claim_connection_generation",
            side_effect=claim_after_release,
        ) as claim,
        ThreadPoolExecutor(max_workers=1) as executor,
    ):
        with real_acquire(db_path):
            connected = executor.submit(manager.is_user_connected, "alice")
            assert sidecar_attempted.wait(timeout=5)
            sidecar_released.set()
        assert connected.result(timeout=5) is True

    assert sidecar_released.is_set()
    claim.assert_called_once()


def test_two_managers_evict_cross_worker_stale_cache(tmp_path: Path) -> None:
    # Given two worker-local managers sharing one database directory
    first = _new_manager(tmp_path)
    second = _new_manager(tmp_path)
    db_path = _existing_db(first, "alice")
    original_epoch = _initialize_epoch(db_path)
    stale_engine = MagicMock()
    entry = first._make_connection_cache_entry(
        stale_engine, "old-password", original_epoch
    )
    first._cache_connection("alice", entry)

    with acquire_rekey_lock(second._get_user_db_path("alice")):
        rekey_lock_module.advance_rekey_epoch(db_path)
    with patch.object(first, "_checkpoint_wal"):
        # When the first worker next consults its process-local cache
        connected = first.is_user_connected("alice")

    # Then the durable epoch fences and disposes that stale engine
    assert connected is False
    assert rekey_lock_module.read_rekey_epoch(db_path) != original_epoch
    stale_engine.dispose.assert_called_once_with()
    assert "alice" not in first.connections
    assert "alice" not in first._password_verifiers
    assert "alice" not in first._connection_epochs


def test_stale_eviction_does_not_checkpoint_while_rekey_is_pending(
    tmp_path: Path,
) -> None:
    # Given one worker observes durable pending metadata from another worker
    stale_manager = _new_manager(tmp_path)
    db_path = _existing_db(stale_manager, "alice")
    original_epoch = _initialize_epoch(db_path)
    stale_engine = MagicMock()
    stale_manager._cache_connection(
        "alice",
        stale_manager._make_connection_cache_entry(
            stale_engine, "old-password", original_epoch
        ),
    )
    with acquire_rekey_lock(db_path):
        rekey_lock_module.begin_rekey_epoch(db_path)

    with (
        patch.object(
            stale_manager,
            "_checkpoint_wal",
            wraps=stale_manager._checkpoint_wal,
        ) as checkpoint,
    ):
        # When the stale worker consults its cache during the rekey window
        connected = stale_manager.is_user_connected("alice")

    # Then eviction only disposes: checkpointing would open the old-key engine
    assert connected is False
    checkpoint.assert_not_called()
    stale_engine.connect.assert_not_called()
    stale_engine.dispose.assert_called_once_with()


@pytest.mark.parametrize(
    "case",
    [
        ("is_user_connected", ("alice",), False),
        ("get_session", ("alice",), None),
        ("check_database_integrity", ("alice",), False),
        ("_password_matches_cached", ("alice", "old-password"), False),
    ],
)
def test_passwordless_cache_accessors_fence_stale_epoch(
    manager: DatabaseManager,
    case: tuple[str, tuple[str, ...], bool | None],
) -> None:
    # Given a cache entry published under an older durable epoch
    method_name, arguments, expected = case
    db_path = _existing_db(manager, "alice")
    old_epoch = _initialize_epoch(db_path)
    stale_engine = MagicMock()
    entry = manager._make_connection_cache_entry(
        stale_engine, "old-password", old_epoch
    )
    manager._cache_connection("alice", entry)
    with acquire_rekey_lock(db_path):
        current_epoch = rekey_lock_module.advance_rekey_epoch(db_path)

    with manager._connections_lock:
        assert not manager._cached_engine_trusted(
            "alice", "old-password", current_epoch
        )

    with patch.object(manager, "_checkpoint_wal"):
        # When a passwordless or verifier-only accessor consults the cache
        result = getattr(manager, method_name)(*arguments)

    # Then it fails closed and evicts all parts of the stale cache entry
    assert result == expected
    stale_engine.dispose.assert_called_once_with()
    assert "alice" not in manager.connections
    assert "alice" not in manager._password_verifiers
    assert "alice" not in manager._connection_epochs


def test_metrics_session_fences_stale_epoch_before_reopen(
    manager: DatabaseManager,
) -> None:
    # Given a stale metrics cache entry and a replacement engine
    db_path = _existing_db(manager, "alice")
    old_epoch = _initialize_epoch(db_path)
    stale_engine = MagicMock()
    replacement_engine = MagicMock()
    entry = manager._make_connection_cache_entry(
        stale_engine, "password", old_epoch
    )
    manager._cache_connection("alice", entry)
    with acquire_rekey_lock(db_path):
        rekey_lock_module.advance_rekey_epoch(db_path)
    expected_session = MagicMock()

    with (
        patch.object(manager, "_checkpoint_wal"),
        patch.object(
            manager, "open_user_database", return_value=replacement_engine
        ) as reopen,
        patch(
            "local_deep_research.database.encrypted_db.sessionmaker",
            return_value=lambda: expected_session,
        ),
    ):
        # When a metrics worker requests a session
        session = manager.create_thread_safe_session_for_metrics(
            "alice", "password"
        )

    # Then it evicts the stale engine and reopens through the fenced path
    assert session is expected_session
    stale_engine.dispose.assert_called_once_with()
    reopen.assert_called_once_with("alice", "password")


def test_open_evicts_stale_cache_before_cold_publication(
    manager: DatabaseManager,
) -> None:
    # Given a cached engine whose published epoch is no longer current
    db_path = _existing_db(manager, "alice")
    stale_epoch = _initialize_epoch(db_path)
    stale_engine = MagicMock()
    replacement_engine = MagicMock()
    entry = manager._make_connection_cache_entry(
        stale_engine, "password", stale_epoch
    )
    manager._cache_connection("alice", entry)
    with acquire_rekey_lock(db_path):
        current_epoch = rekey_lock_module.advance_rekey_epoch(db_path)

    with (
        patch.object(manager, "_checkpoint_wal"),
        patch.object(
            manager,
            "_open_user_database_cold",
            return_value=replacement_engine,
        ) as cold_open,
        patch.object(manager, "_run_phase2_rekey"),
    ):
        # When the worker opens with the valid password
        opened = manager.open_user_database("alice", "password")

    # Then both cache checks reject the stale generation before publication
    assert opened is replacement_engine
    cold_open.assert_called_once()
    stale_engine.dispose.assert_called_once_with()
    assert manager.connections["alice"] is replacement_engine
    assert manager._connection_epochs["alice"] == current_epoch


def test_cold_open_waits_for_epoch_advance_and_rekey(
    tmp_path: Path,
) -> None:
    # Given cold open reaches a sidecar owned by a generation publisher
    opener_manager = _new_manager(tmp_path)
    db_path = _existing_db(opener_manager, "alice")
    _initialize_epoch(db_path)
    replacement_engine = MagicMock()
    sidecar_attempted = threading.Event()
    sidecar_released = threading.Event()
    cold_started = threading.Event()
    real_acquire = acquire_rekey_lock

    @contextmanager
    def observe_sidecar_attempt(path: Path, **kwargs: object):
        sidecar_attempted.set()
        with real_acquire(path, **kwargs):
            yield

    def controlled_cold_open(
        _username: str,
        _password: str,
        _open_start: float,
        prederived_hex_key: str | None = None,
    ) -> MagicMock:
        assert sidecar_released.is_set()
        cold_started.set()
        return replacement_engine

    with (
        patch(
            "local_deep_research.database.encrypted_db.acquire_rekey_lock",
            side_effect=observe_sidecar_attempt,
        ),
        patch.object(
            opener_manager,
            "_open_user_database_cold",
            side_effect=controlled_cold_open,
        ),
        patch.object(opener_manager, "_run_phase2_rekey"),
        ThreadPoolExecutor(max_workers=1) as executor,
    ):
        with real_acquire(db_path):
            cold_open = executor.submit(
                opener_manager.open_user_database, "alice", "new-password"
            )
            assert sidecar_attempted.wait(timeout=5)
            current_epoch = rekey_lock_module.advance_rekey_epoch(db_path)
            sidecar_released.set()

        # Then physical cold-open begins only after durable publication and release
        assert cold_open.result(timeout=5) is replacement_engine
        assert cold_started.is_set()
        assert opener_manager._connection_epochs["alice"] == current_epoch


def test_close_user_database_disposes_outside_connections_lock(
    manager: DatabaseManager,
) -> None:
    # Given a current-generation cached engine and a non-reentrant cache lock
    db_path = _existing_db(manager, "alice")
    epoch = _initialize_epoch(db_path)
    engine = MagicMock()
    manager._cache_connection(
        "alice",
        manager._make_connection_cache_entry(engine, "password", epoch),
    )
    cache_lock = threading.Lock()
    manager._connections_lock = cache_lock

    def assert_cache_lock_released(*_args) -> None:
        assert cache_lock.acquire(blocking=False)
        cache_lock.release()

    engine.dispose.side_effect = assert_cache_lock_released

    with patch.object(
        manager, "_checkpoint_wal", side_effect=assert_cache_lock_released
    ) as checkpoint:
        # When the user database is closed
        manager.close_user_database("alice")

    # Then detachment is atomic but checkpoint and dispose run after unlocking
    checkpoint.assert_called_once_with(engine, "for alice")
    engine.dispose.assert_called_once_with()
    assert "alice" not in manager.connections
    assert "alice" not in manager._password_verifiers
    assert "alice" not in manager._connection_epochs