Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

26.8 KB
Raw Blame
"""Contract tests for the encrypted-database *manager* surface.

Scope: engine/pool lifecycle around ``DatabaseManager.close_user_database``,
the ``data_dir`` property, the manager's error-swallowing paths, and the
backup service's coupling to the source database (salt binding, absence of a
restore path, chmod failure handling).

Deliberately NOT covered here (audited elsewhere): the KDF floor, credential
lifetime, and rekey. Deliberately NOT duplicated: DB/dir file modes
(``tests/database/test_db_file_permissions.py``), backup file/dir modes
(``tests/database/backup/test_backup_service.py``), and the getter's
live-data-root resolution (``tests/database/test_encrypted_db_live_data_dir.py``).

Every test builds its OWN ``DatabaseManager`` and its own plain-SQLite
engines. The global ``db_manager`` singleton is never mutated, and
``_check_encryption_available`` is patched everywhere so no test in this file
ever runs a SQLCipher key derivation.
"""

import contextlib
import os
import shutil
import sqlite3
import stat
import threading
from pathlib import Path
from unittest.mock import patch

import pytest
from loguru import logger
from sqlalchemy import create_engine, text
from sqlalchemy.pool import QueuePool

MODULE = "local_deep_research.database.encrypted_db"


# ---------------------------------------------------------------------------
# helpers
# ---------------------------------------------------------------------------


@contextlib.contextmanager
def _manager(root_getter, *, has_encryption=True):
    """Yield a fresh DatabaseManager whose default data root is ``root_getter``.

    ``root_getter`` is a zero-arg callable so a test can move the live data
    root underneath the manager. No SQLCipher probe runs.
    """
    from local_deep_research.database.encrypted_db import DatabaseManager

    with (
        patch(f"{MODULE}.get_data_directory", side_effect=root_getter),
        patch.object(
            DatabaseManager,
            "_check_encryption_available",
            return_value=has_encryption,
        ),
    ):
        yield DatabaseManager()


def _register(mgr, username, engine, password="pw", epoch="epoch"):
    """Publish ``engine`` for ``username`` exactly as a verified open would.

    Epoch-fenced cache: the engine, its verifier, and the durable epoch
    publish together. Tests that exercise the claim paths mock
    ``read_rekey_epoch_state`` to ``RekeyEpochState(epoch, None)`` via
    ``_trust_epoch``; tests of pure teardown semantics do not need it.
    """
    mgr._cache_connection(
        username, mgr._make_connection_cache_entry(engine, password, epoch)
    )


@contextlib.contextmanager
def _captured_logs():
    """Collect (level, message, has-traceback) for every record in the block."""
    records = []

    def sink(message):
        record = message.record
        records.append(
            {
                "level": record["level"].name,
                "message": record["message"],
                "exception": record["exception"] is not None,
            }
        )

    # ``local_deep_research/__init__.py`` calls
    # ``logger.disable("local_deep_research")`` at import; re-enable it for the
    # duration of the block exactly as the ``loguru_caplog`` fixture does, and
    # put it back afterwards so no other test's logging behaviour changes.
    sink_id = logger.add(sink, level="DEBUG")
    logger.enable("local_deep_research")
    try:
        yield records
    finally:
        logger.disable("local_deep_research")
        logger.remove(sink_id)


class _FakeEngine:
    """Minimal Engine stand-in for the failure/blocking paths."""

    def __init__(self, *, dispose_error=None, dispose_gate=None):
        self.dispose_error = dispose_error
        self.dispose_gate = dispose_gate
        self.dispose_calls = 0

    def connect(self):
        # Makes _checkpoint_wal take its except branch without any real I/O.
        raise RuntimeError("checkpoint connection refused")

    def dispose(self):
        self.dispose_calls += 1
        if self.dispose_gate is not None:
            entered, release = self.dispose_gate
            entered.set()
            release.wait(10.0)
        if self.dispose_error is not None:
            raise self.dispose_error


# ===========================================================================
# 1. Pool behaviour around close_user_database
# ===========================================================================


@contextlib.contextmanager
def _trust_epoch(epoch="epoch"):
    """Pretend the durable epoch state matches the fabricated cache.

    Production caches are only published by paths that initialize the
    durable epoch files first, so a cached engine always has matching
    on-disk state. Contract tests that fabricate a cache with no files
    use this to stand in for that durable state; the real-file fencing
    behaviour is covered by tests/database/test_rekey_cache_fencing.py.
    """
    from local_deep_research.database.rekey_lock import RekeyEpochState

    with patch(
        "local_deep_research.database.encrypted_db.read_rekey_epoch_state",
        return_value=RekeyEpochState(epoch, None),
    ):
        yield


def test_checked_out_session_outlives_close_user_database(tmp_path):
    """A Session handed out before the close keeps a LIVE connection after it.

    ``close_user_database`` calls ``Engine.dispose()``, which closes only the
    connections that are checked IN. Anything already checked out -- e.g. a
    Session obtained from ``get_session()`` and used once -- keeps its DBAPI
    connection to the decrypted database and remains fully usable.

    So logout/eviction does NOT sever in-flight database access: the answer to
    "can a connection outlive the close" is yes, by design of dispose(), and
    the manager does nothing to shorten that window.
    """
    db_file = tmp_path / "alice.db"
    engine = create_engine(f"sqlite:///{db_file}", poolclass=QueuePool)
    try:
        with _manager(lambda: tmp_path / "root") as mgr:
            _register(mgr, "alice", engine)

            with _trust_epoch():
                session = mgr.get_session("alice")
            assert session is not None
            assert session.execute(text("SELECT 1")).scalar() == 1
            raw = session.connection().connection.dbapi_connection

            with _trust_epoch():
                mgr.close_user_database("alice")

            # Manager-visible state says the user is gone...
            assert mgr.is_user_connected("alice") is False
            assert "alice" not in mgr.connections
            assert "alice" not in mgr._password_verifiers

            # ...but the checked-out connection is untouched and still reads.
            assert session.execute(text("SELECT 1")).scalar() == 1
            assert raw.execute("SELECT 1").fetchone() == (1,)

            session.close()
    finally:
        engine.dispose()


def test_close_user_database_closes_pooled_idle_connections(tmp_path):
    """The checked-IN half of the pool is disposed cleanly.

    Counterpart to the test above: a connection returned to the pool before
    the close really is closed by it, so ``close_user_database`` is not
    leaking idle SQLCipher file handles on the normal path.
    """
    db_file = tmp_path / "alice.db"
    engine = create_engine(f"sqlite:///{db_file}", poolclass=QueuePool)
    try:
        with _manager(lambda: tmp_path / "root") as mgr:
            _register(mgr, "alice", engine)

            conn = engine.connect()
            raw = conn.connection.dbapi_connection
            conn.close()  # back to the pool, still open at the DBAPI level
            assert raw.execute("SELECT 1").fetchone() == (1,)

            mgr.close_user_database("alice")

            with pytest.raises(sqlite3.ProgrammingError):
                raw.execute("SELECT 1")
    finally:
        engine.dispose()


def test_manager_holds_no_session_registry_to_reclaim(tmp_path):
    """Manager-side counterpart to the LRU-eviction defect (#5778).

    The per-thread ``LRUCache(maxsize=10)`` in ``utilities/db_utils.py`` drops
    live Sessions without closing them. The manager cannot compensate: it
    hands Sessions out from ``get_session()`` and keeps NO reference to them,
    so nothing at this layer can close an evicted Session, and
    ``get_memory_usage()`` reports ``active_sessions: 0`` no matter how many
    are outstanding. Both the accounting and the reclamation are missing.
    """
    db_file = tmp_path / "alice.db"
    engine = create_engine(f"sqlite:///{db_file}", poolclass=QueuePool)
    try:
        with _manager(lambda: tmp_path / "root") as mgr:
            _register(mgr, "alice", engine)

            with _trust_epoch():
                sessions = [mgr.get_session("alice") for _ in range(2)]
            for session in sessions:
                session.execute(text("SELECT 1"))

            # Two connections are really checked out of the pool...
            assert engine.pool.checkedout() == 2
            # ...and the manager's own accounting cannot see them.
            usage = mgr.get_memory_usage()
            assert usage["active_connections"] == 1
            assert usage["active_sessions"] == 0

            with _trust_epoch():
                mgr.close_user_database("alice")

            # Closing the user reclaims neither Session.
            for session in sessions:
                assert session.execute(text("SELECT 1")).scalar() == 1
                session.close()
    finally:
        engine.dispose()


def test_close_user_database_holds_global_lock_across_dispose(tmp_path):
    """``dispose()`` excludes THIS user's rekey lifecycle, not other users.

    Contract updated with the epoch-fenced teardown: the WAL checkpoint and
    dispose run under the per-user rekey lifecycle lock (and the on-disk
    sidecar), so a concurrent change_password for the SAME user cannot tear
    the dispose -- while ``_connections_lock`` is held only for the cache
    mutation, so a slow dispose no longer stalls unrelated users'
    ``get_session`` / ``is_user_connected`` / cached-open fast path.
    """
    entered = threading.Event()
    release = threading.Event()
    engine = _FakeEngine(dispose_gate=(entered, release))
    probe_done = threading.Event()
    rekey_gate_probe = {}

    with _manager(lambda: tmp_path / "root") as mgr:
        _register(mgr, "alice", engine)
        _register(mgr, "bob", _FakeEngine())

        def probe_rekey_gate():
            # From the OUTSIDE: while alice's dispose is mid-flight, her
            # rekey lifecycle lock must be held (a change_password for
            # alice cannot start), but bob's must be free. Only release
            # what was actually acquired.
            acquired = mgr._get_rekey_lock("alice").acquire(blocking=False)
            rekey_gate_probe["alice_blocked"] = not acquired
            if acquired:
                mgr._get_rekey_lock("alice").release()
            probe_done.set()

        closer = threading.Thread(
            target=mgr.close_user_database, args=("alice",)
        )
        probe = threading.Thread(target=probe_rekey_gate)
        closer.start()
        try:
            assert entered.wait(10.0), "dispose never ran"
            probe.start()
            assert probe_done.wait(10.0), "probe never ran"
            assert rekey_gate_probe["alice_blocked"], (
                "alice's rekey lifecycle lock was free while her dispose "
                "was mid-flight -- a concurrent change_password could tear "
                "the teardown"
            )
            assert mgr._get_rekey_lock("bob").acquire(blocking=False), (
                "bob's rekey lifecycle lock was held during alice's dispose "
                "-- the exclusion is no longer per-user"
            )
            mgr._get_rekey_lock("bob").release()
        finally:
            release.set()
            closer.join(10.0)
        probe.join(10.0)
        assert engine.dispose_calls == 1


# ===========================================================================
# 2. Error paths: what close_user_database throws away
# ===========================================================================


def test_dispose_failure_is_swallowed_without_any_diagnostic(tmp_path):
    """A failed dispose is swallowed, de-registered, and named by type only.

    Contract updated with the epoch-fenced teardown: the engine is still
    de-registered even when its dispose raises (the pool is replaced on the
    next open, and the dispose failure is surfaced at WARNING with exactly
    the exception TYPE NAME -- enough for an operator to chase the FD leak,
    never enough to leak a credential-bearing message or traceback, per the
    repo's sensitive-logging convention).
    """
    engine = _FakeEngine(dispose_error=RuntimeError("SENTINEL_DISPOSE_DETAIL"))

    with _manager(lambda: tmp_path / "root") as mgr:
        _register(mgr, "alice", engine)

        with _captured_logs() as records:
            mgr.close_user_database("alice")  # must not raise

    assert engine.dispose_calls == 1
    # De-registered even though the dispose failed -- the engine is now
    # unreachable, so its pooled connections can never be released.
    assert "alice" not in mgr.connections
    assert "alice" not in mgr._password_verifiers
    assert mgr.is_user_connected("alice") is False

    dispose_logs = [
        r
        for r in records
        if "Failed to dispose engine for alice" in r["message"]
    ]
    assert len(dispose_logs) == 1
    entry = dispose_logs[0]
    assert entry["level"] == "WARNING"
    # The type name is present; the detail and traceback are not:
    assert entry["exception"] is False, "a traceback is attached after all"
    assert "SENTINEL_DISPOSE_DETAIL" not in entry["message"]
    assert "RuntimeError" in entry["message"]
    # ...and the sensitive detail is not recovered anywhere else either.
    assert not [r for r in records if "SENTINEL_DISPOSE_DETAIL" in r["message"]]


def test_checkpoint_wal_failure_is_invisible_at_default_log_level(tmp_path):
    """A failed pre-dispose WAL checkpoint is logged at DEBUG only.

    ``_checkpoint_wal`` catches everything and logs at ``debug``. A checkpoint
    that could not run (or reported ``busy``) means pending WAL content was
    not flushed before the engine went away, yet at production log levels the
    close reports unqualified success.
    """
    engine = _FakeEngine()

    with _manager(lambda: tmp_path / "root") as mgr:
        _register(mgr, "alice", engine)
        with _captured_logs() as records, _trust_epoch():
            mgr.close_user_database("alice")

    checkpoint_logs = [
        r for r in records if "WAL checkpoint failed" in r["message"]
    ]
    assert checkpoint_logs, "checkpoint failure was not logged at all"
    assert all(r["level"] == "DEBUG" for r in checkpoint_logs)

    # Nothing at WARNING or above marks the close as degraded...
    assert not [
        r for r in records if r["level"] in {"WARNING", "ERROR", "CRITICAL"}
    ]
    # ...and the only INFO-or-above line reports plain success.
    closed = [
        r for r in records if "Closed database for user alice" in r["message"]
    ]
    assert len(closed) == 1
    assert closed[0]["level"] == "INFO"


def test_init_lock_is_retained_after_close(tmp_path):
    """The per-user init lock deliberately survives BOTH close paths.

    Documented invariant: dropping it would let a later open create a SECOND
    lock for the same user, so two cold-opens could migrate one file at once.
    Neither ``close_user_database`` nor ``close_all_databases`` clears it.
    """
    with _manager(lambda: tmp_path / "root") as mgr:
        lock = mgr._get_init_lock("alice")
        _register(mgr, "alice", _FakeEngine())

        mgr.close_user_database("alice")
        assert mgr._init_locks.get("alice") is lock

        mgr.close_all_databases()
        assert mgr._init_locks.get("alice") is lock


# ===========================================================================
# 3. The data_dir property
# ===========================================================================


def test_data_dir_round_trip_permanently_pins_the_override(tmp_path):
    """``original = mgr.data_dir; ...; mgr.data_dir = original`` does NOT restore.

    The getter RESOLVES a default (``_data_dir_override is None``); the setter
    WRITES an override. So the widespread save/restore fixture idiom converts
    a manager that tracks the live data root into one pinned to whatever the
    root happened to be when the fixture ran -- for the rest of the process.
    Against the global ``db_manager`` singleton under xdist that pins the
    whole worker.
    """
    roots = {"current": tmp_path / "first"}

    with _manager(lambda: roots["current"]) as mgr:
        assert mgr._data_dir_override is None  # dynamic to start with

        original = mgr.data_dir  # the "save" half of the idiom
        mgr.data_dir = original  # the "restore" half

        # The restore installed an override that was not there before.
        assert mgr._data_dir_override == original

        roots["current"] = tmp_path / "second"
        assert mgr.data_dir == original
        assert mgr.data_dir != tmp_path / "second" / "encrypted_databases"

    # A manager that never went through the round trip still follows the root.
    roots["current"] = tmp_path / "third"
    with _manager(lambda: roots["current"]) as fresh:
        assert fresh.data_dir == tmp_path / "third" / "encrypted_databases"


def test_data_dir_setter_offers_no_way_back_to_dynamic_resolution(tmp_path):
    """Only the private attribute can undo the pin; the property cannot.

    ``mgr.data_dir = None`` raises inside ``Path(value)`` before any state is
    written, so a fixture has no supported way to hand the manager back its
    default-tracking behaviour.
    """
    roots = {"current": tmp_path / "first"}

    with _manager(lambda: roots["current"]) as mgr:
        pinned = mgr.data_dir
        mgr.data_dir = pinned

        with pytest.raises(TypeError):
            mgr.data_dir = None
        assert mgr._data_dir_override == pinned  # unchanged by the failure

        roots["current"] = tmp_path / "second"
        assert mgr.data_dir == pinned

        mgr._data_dir_override = None  # the only real restore
        assert mgr.data_dir == tmp_path / "second" / "encrypted_databases"


def test_reading_data_dir_is_a_filesystem_mutation(tmp_path):
    """The getter creates and hardens the directory as a side effect.

    ``data_dir`` is a property read, but ``_ensure_dir`` mkdirs and chmods
    0o700 under ``_connections_lock``. Any code that merely *inspects*
    ``mgr.data_dir`` materialises the directory.
    """
    root = tmp_path / "root"
    target = root / "encrypted_databases"

    with _manager(lambda: root) as mgr:
        assert not target.exists()
        resolved = mgr.data_dir
        assert resolved == target
        assert target.is_dir()
        assert stat.S_IMODE(target.stat().st_mode) == 0o700


def test_data_dir_is_never_recreated_or_rehardened_after_first_access(tmp_path):
    """``_initialized_data_dirs`` memoises per path, so the manager never repairs.

    Once a path has been ensured, later accesses are a pure dict lookup. If
    the directory is subsequently removed or loosened -- a volume remount, an
    operator chmod, a tmpdir teardown -- the manager keeps handing out the
    path without recreating or re-hardening it, and ``_get_user_db_path``
    resolves under a parent that no longer exists.
    """
    root = tmp_path / "root"

    with _manager(lambda: root) as mgr:
        target = mgr.data_dir
        assert stat.S_IMODE(target.stat().st_mode) == 0o700

        os.chmod(target, 0o755)  # noqa: S103 - simulating an external loosening
        assert mgr.data_dir == target
        assert stat.S_IMODE(target.stat().st_mode) == 0o755, (
            "directory was re-hardened on access (contract changed)"
        )

        shutil.rmtree(target)
        assert mgr.data_dir == target
        assert not target.exists(), "directory was recreated (contract changed)"
        assert not mgr._get_user_db_path("alice").parent.exists()


def test_data_dir_override_does_not_redirect_the_backup_service(tmp_path):
    """The manager's override is manager-local; BackupService resolves its own root.

    ``BackupService.__init__`` calls ``get_encrypted_database_path()`` (i.e.
    ``get_data_directory()/encrypted_databases``) directly. It never consults
    ``db_manager.data_dir``. So a manager pinned to a different directory --
    the fixture idiom above -- makes the pre-migration backup in
    ``_open_user_database_cold`` point at a DIFFERENT file than the one being
    migrated (or at no file at all, in which case the backup silently fails
    and the migration proceeds unprotected).
    """
    from local_deep_research.database.backup.backup_service import (
        BackupService,
    )

    default_root = tmp_path / "default"
    override = tmp_path / "override" / "encrypted_databases"

    with _manager(lambda: default_root) as mgr:
        mgr.data_dir = override
        manager_db_path = mgr._get_user_db_path("alice")

    with patch(
        "local_deep_research.config.paths.get_data_directory",
        return_value=default_root,
    ):
        service = BackupService(username="alice", password="pw")

    assert manager_db_path.parent == override
    assert service.db_path.parent == default_root / "encrypted_databases"
    assert service.db_path != manager_db_path
    # Same user, same filename -- only the root diverges, which is exactly why
    # the divergence is easy to miss.
    assert service.db_path.name == manager_db_path.name


# ===========================================================================
# 4. Backup: salt binding, absence of restore, chmod handling
# ===========================================================================


def test_verify_backup_keys_off_its_own_salt(tmp_path):
    """Verification must not conceal a backup's missing recovery inputs."""
    import local_deep_research.database.backup.backup_service as bs

    db_dir = tmp_path / "encrypted_databases"
    db_dir.mkdir(parents=True)
    backup_dir = tmp_path / "backups"
    backup_dir.mkdir()
    backup_file = backup_dir / "ldr_backup_20250101_000000.db"
    backup_file.write_bytes(b"pretend-encrypted-backup")

    captured = {}

    def fake_set_key(cursor, password, db_path=None, **kwargs):
        captured["db_path"] = db_path

    class _Cursor:
        def execute(self, sql, *args):
            return self

        def fetchone(self):
            return ("ok",)

        def close(self):
            pass

    class _Conn:
        def cursor(self):
            return _Cursor()

        def close(self):
            pass

    with (
        patch.object(bs, "get_encrypted_database_path", return_value=db_dir),
        patch.object(
            bs, "get_user_database_filename", return_value="ldr_user_x.db"
        ),
        patch.object(bs, "get_user_backup_directory", return_value=backup_dir),
    ):
        service = bs.BackupService(username="alice", password="pw")

    with (
        patch(
            "local_deep_research.database.sqlcipher_compat.get_sqlcipher_module"
        ) as mod,
        patch.object(bs, "set_sqlcipher_key", side_effect=fake_set_key),
        patch.object(bs, "apply_sqlcipher_pragmas"),
        patch.object(bs, "verify_sqlcipher_connection", return_value=True),
    ):
        mod.return_value.connect.return_value = _Conn()
        assert service._verify_backup(backup_file) is True

    assert captured["db_path"] == backup_file
    assert captured["db_path"] != service.db_path


def test_backup_service_exposes_no_restore_path():
    """Guard for the SAFETY INVARIANT in ``create_user_database``.

    That function deletes an orphaned ``.salt`` on the strength of "nothing
    restores or imports a .db from a salt". Backups are write-only today: the
    service creates, lists, prunes and refreshes them, and never writes back
    to ``self.db_path``. If a restore/import that stages a ``.salt`` ahead of
    its ``.db`` is ever added, that deletion branch must be re-evaluated
    before this assertion is relaxed.
    """
    import local_deep_research.database.backup.backup_service as bs
    import local_deep_research.database.backup as backup_pkg

    offenders = [
        name
        for name in dir(bs.BackupService)
        if not name.startswith("__")
        and ("restore" in name.lower() or "import" in name.lower())
    ]
    assert not offenders, (
        f"BackupService gained {offenders}; re-read the SAFETY INVARIANT in "
        "encrypted_db.create_user_database before allowing this"
    )
    assert not [
        name
        for name in getattr(backup_pkg, "__all__", [])
        if "restore" in name.lower()
    ]


def test_backup_chmod_failure_aborts_the_whole_backup(tmp_path):
    """The backup chmod is NOT best-effort, unlike the database chmod.

    ``_create_backup_impl`` calls bare ``os.chmod(temp_path, 0o600)``. On a
    filesystem where chmod raises (the Docker bind-mount / FUSE case that
    ``_best_effort_chmod`` exists to tolerate -- see
    ``tests/database/test_db_file_permissions.py::
    test_db_creation_survives_chmod_failure``, where DB creation survives),
    the whole backup fails and the temp file is deleted. Such a deployment
    can create user databases but can never take a pre-migration backup.
    """
    import local_deep_research.database.backup.backup_service as bs

    db_dir = tmp_path / "encrypted_databases"
    db_dir.mkdir(parents=True)
    source_db = db_dir / "ldr_user_x.db"
    source_db.write_bytes(b"x" * 4096)
    backup_dir = tmp_path / "backups"
    backup_dir.mkdir()

    class _Cursor:
        def execute(self, sql, *args):
            if sql.startswith("ATTACH DATABASE '"):
                Path(sql.split("'", 2)[1]).write_bytes(b"exported-backup")
            return self

        def close(self):
            pass

    class _Conn:
        def cursor(self):
            return _Cursor()

        def close(self):
            pass

    real_chmod = os.chmod

    def picky_chmod(path, mode, *args, **kwargs):
        if str(path).endswith(".tmp"):
            raise OSError("chmod is not supported on this filesystem")
        return real_chmod(path, mode, *args, **kwargs)

    with (
        patch.object(bs, "get_encrypted_database_path", return_value=db_dir),
        patch.object(
            bs, "get_user_database_filename", return_value="ldr_user_x.db"
        ),
        patch.object(bs, "get_user_backup_directory", return_value=backup_dir),
    ):
        service = bs.BackupService(username="alice", password="pw")

    with (
        patch.object(bs, "create_sqlcipher_connection", return_value=_Conn()),
        patch.object(bs, "get_key_from_password", return_value=b"\xab" * 32),
        patch.object(bs.BackupService, "_verify_backup", return_value=True),
        patch.object(bs.os, "chmod", side_effect=picky_chmod),
    ):
        result = service._create_backup_impl()

    assert result.success is False
    assert "chmod" in result.error
    # Nothing survives: neither the temp file nor a finalized backup.
    assert list(backup_dir.iterdir()) == []