~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Contract tests for the encrypted-database *manager* surface.
Scope: engine/pool lifecycle around ``DatabaseManager.close_user_database``,
the ``data_dir`` property, the manager's error-swallowing paths, and the
backup service's coupling to the source database (salt binding, absence of a
restore path, chmod failure handling).
Deliberately NOT covered here (audited elsewhere): the KDF floor, credential
lifetime, and rekey. Deliberately NOT duplicated: DB/dir file modes
(``tests/database/test_db_file_permissions.py``), backup file/dir modes
(``tests/database/backup/test_backup_service.py``), and the getter's
live-data-root resolution (``tests/database/test_encrypted_db_live_data_dir.py``).
Every test builds its OWN ``DatabaseManager`` and its own plain-SQLite
engines. The global ``db_manager`` singleton is never mutated, and
``_check_encryption_available`` is patched everywhere so no test in this file
ever runs a SQLCipher key derivation.
"""
import contextlib
import os
import shutil
import sqlite3
import stat
import threading
from pathlib import Path
from unittest.mock import patch
import pytest
from loguru import logger
from sqlalchemy import create_engine, text
from sqlalchemy.pool import QueuePool
MODULE = "local_deep_research.database.encrypted_db"
# ---------------------------------------------------------------------------
# helpers
# ---------------------------------------------------------------------------
@contextlib.contextmanager
def _manager(root_getter, *, has_encryption=True):
"""Yield a fresh DatabaseManager whose default data root is ``root_getter``.
``root_getter`` is a zero-arg callable so a test can move the live data
root underneath the manager. No SQLCipher probe runs.
"""
from local_deep_research.database.encrypted_db import DatabaseManager
with (
patch(f"{MODULE}.get_data_directory", side_effect=root_getter),
patch.object(
DatabaseManager,
"_check_encryption_available",
return_value=has_encryption,
),
):
yield DatabaseManager()
def _register(mgr, username, engine, password="pw", epoch="epoch"):
"""Publish ``engine`` for ``username`` exactly as a verified open would.
Epoch-fenced cache: the engine, its verifier, and the durable epoch
publish together. Tests that exercise the claim paths mock
``read_rekey_epoch_state`` to ``RekeyEpochState(epoch, None)`` via
``_trust_epoch``; tests of pure teardown semantics do not need it.
"""
mgr._cache_connection(
username, mgr._make_connection_cache_entry(engine, password, epoch)
)
@contextlib.contextmanager
def _captured_logs():
"""Collect (level, message, has-traceback) for every record in the block."""
records = []
def sink(message):
record = message.record
records.append(
{
"level": record["level"].name,
"message": record["message"],
"exception": record["exception"] is not None,
}
)
# ``local_deep_research/__init__.py`` calls
# ``logger.disable("local_deep_research")`` at import; re-enable it for the
# duration of the block exactly as the ``loguru_caplog`` fixture does, and
# put it back afterwards so no other test's logging behaviour changes.
sink_id = logger.add(sink, level="DEBUG")
logger.enable("local_deep_research")
try:
yield records
finally:
logger.disable("local_deep_research")
logger.remove(sink_id)
class _FakeEngine:
"""Minimal Engine stand-in for the failure/blocking paths."""
def __init__(self, *, dispose_error=None, dispose_gate=None):
self.dispose_error = dispose_error
self.dispose_gate = dispose_gate
self.dispose_calls = 0
def connect(self):
# Makes _checkpoint_wal take its except branch without any real I/O.
raise RuntimeError("checkpoint connection refused")
def dispose(self):
self.dispose_calls += 1
if self.dispose_gate is not None:
entered, release = self.dispose_gate
entered.set()
release.wait(10.0)
if self.dispose_error is not None:
raise self.dispose_error
# ===========================================================================
# 1. Pool behaviour around close_user_database
# ===========================================================================
@contextlib.contextmanager
def _trust_epoch(epoch="epoch"):
"""Pretend the durable epoch state matches the fabricated cache.
Production caches are only published by paths that initialize the
durable epoch files first, so a cached engine always has matching
on-disk state. Contract tests that fabricate a cache with no files
use this to stand in for that durable state; the real-file fencing
behaviour is covered by tests/database/test_rekey_cache_fencing.py.
"""
from local_deep_research.database.rekey_lock import RekeyEpochState
with patch(
"local_deep_research.database.encrypted_db.read_rekey_epoch_state",
return_value=RekeyEpochState(epoch, None),
):
yield
def test_checked_out_session_outlives_close_user_database(tmp_path):
"""A Session handed out before the close keeps a LIVE connection after it.
``close_user_database`` calls ``Engine.dispose()``, which closes only the
connections that are checked IN. Anything already checked out -- e.g. a
Session obtained from ``get_session()`` and used once -- keeps its DBAPI
connection to the decrypted database and remains fully usable.
So logout/eviction does NOT sever in-flight database access: the answer to
"can a connection outlive the close" is yes, by design of dispose(), and
the manager does nothing to shorten that window.
"""
db_file = tmp_path / "alice.db"
engine = create_engine(f"sqlite:///{db_file}", poolclass=QueuePool)
try:
with _manager(lambda: tmp_path / "root") as mgr:
_register(mgr, "alice", engine)
with _trust_epoch():
session = mgr.get_session("alice")
assert session is not None
assert session.execute(text("SELECT 1")).scalar() == 1
raw = session.connection().connection.dbapi_connection
with _trust_epoch():
mgr.close_user_database("alice")
# Manager-visible state says the user is gone...
assert mgr.is_user_connected("alice") is False
assert "alice" not in mgr.connections
assert "alice" not in mgr._password_verifiers
# ...but the checked-out connection is untouched and still reads.
assert session.execute(text("SELECT 1")).scalar() == 1
assert raw.execute("SELECT 1").fetchone() == (1,)
session.close()
finally:
engine.dispose()
def test_close_user_database_closes_pooled_idle_connections(tmp_path):
"""The checked-IN half of the pool is disposed cleanly.
Counterpart to the test above: a connection returned to the pool before
the close really is closed by it, so ``close_user_database`` is not
leaking idle SQLCipher file handles on the normal path.
"""
db_file = tmp_path / "alice.db"
engine = create_engine(f"sqlite:///{db_file}", poolclass=QueuePool)
try:
with _manager(lambda: tmp_path / "root") as mgr:
_register(mgr, "alice", engine)
conn = engine.connect()
raw = conn.connection.dbapi_connection
conn.close() # back to the pool, still open at the DBAPI level
assert raw.execute("SELECT 1").fetchone() == (1,)
mgr.close_user_database("alice")
with pytest.raises(sqlite3.ProgrammingError):
raw.execute("SELECT 1")
finally:
engine.dispose()
def test_manager_holds_no_session_registry_to_reclaim(tmp_path):
"""Manager-side counterpart to the LRU-eviction defect (#5778).
The per-thread ``LRUCache(maxsize=10)`` in ``utilities/db_utils.py`` drops
live Sessions without closing them. The manager cannot compensate: it
hands Sessions out from ``get_session()`` and keeps NO reference to them,
so nothing at this layer can close an evicted Session, and
``get_memory_usage()`` reports ``active_sessions: 0`` no matter how many
are outstanding. Both the accounting and the reclamation are missing.
"""
db_file = tmp_path / "alice.db"
engine = create_engine(f"sqlite:///{db_file}", poolclass=QueuePool)
try:
with _manager(lambda: tmp_path / "root") as mgr:
_register(mgr, "alice", engine)
with _trust_epoch():
sessions = [mgr.get_session("alice") for _ in range(2)]
for session in sessions:
session.execute(text("SELECT 1"))
# Two connections are really checked out of the pool...
assert engine.pool.checkedout() == 2
# ...and the manager's own accounting cannot see them.
usage = mgr.get_memory_usage()
assert usage["active_connections"] == 1
assert usage["active_sessions"] == 0
with _trust_epoch():
mgr.close_user_database("alice")
# Closing the user reclaims neither Session.
for session in sessions:
assert session.execute(text("SELECT 1")).scalar() == 1
session.close()
finally:
engine.dispose()
def test_close_user_database_holds_global_lock_across_dispose(tmp_path):
"""``dispose()`` excludes THIS user's rekey lifecycle, not other users.
Contract updated with the epoch-fenced teardown: the WAL checkpoint and
dispose run under the per-user rekey lifecycle lock (and the on-disk
sidecar), so a concurrent change_password for the SAME user cannot tear
the dispose -- while ``_connections_lock`` is held only for the cache
mutation, so a slow dispose no longer stalls unrelated users'
``get_session`` / ``is_user_connected`` / cached-open fast path.
"""
entered = threading.Event()
release = threading.Event()
engine = _FakeEngine(dispose_gate=(entered, release))
probe_done = threading.Event()
rekey_gate_probe = {}
with _manager(lambda: tmp_path / "root") as mgr:
_register(mgr, "alice", engine)
_register(mgr, "bob", _FakeEngine())
def probe_rekey_gate():
# From the OUTSIDE: while alice's dispose is mid-flight, her
# rekey lifecycle lock must be held (a change_password for
# alice cannot start), but bob's must be free. Only release
# what was actually acquired.
acquired = mgr._get_rekey_lock("alice").acquire(blocking=False)
rekey_gate_probe["alice_blocked"] = not acquired
if acquired:
mgr._get_rekey_lock("alice").release()
probe_done.set()
closer = threading.Thread(
target=mgr.close_user_database, args=("alice",)
)
probe = threading.Thread(target=probe_rekey_gate)
closer.start()
try:
assert entered.wait(10.0), "dispose never ran"
probe.start()
assert probe_done.wait(10.0), "probe never ran"
assert rekey_gate_probe["alice_blocked"], (
"alice's rekey lifecycle lock was free while her dispose "
"was mid-flight -- a concurrent change_password could tear "
"the teardown"
)
assert mgr._get_rekey_lock("bob").acquire(blocking=False), (
"bob's rekey lifecycle lock was held during alice's dispose "
"-- the exclusion is no longer per-user"
)
mgr._get_rekey_lock("bob").release()
finally:
release.set()
closer.join(10.0)
probe.join(10.0)
assert engine.dispose_calls == 1
# ===========================================================================
# 2. Error paths: what close_user_database throws away
# ===========================================================================
def test_dispose_failure_is_swallowed_without_any_diagnostic(tmp_path):
"""A failed dispose is swallowed, de-registered, and named by type only.
Contract updated with the epoch-fenced teardown: the engine is still
de-registered even when its dispose raises (the pool is replaced on the
next open, and the dispose failure is surfaced at WARNING with exactly
the exception TYPE NAME -- enough for an operator to chase the FD leak,
never enough to leak a credential-bearing message or traceback, per the
repo's sensitive-logging convention).
"""
engine = _FakeEngine(dispose_error=RuntimeError("SENTINEL_DISPOSE_DETAIL"))
with _manager(lambda: tmp_path / "root") as mgr:
_register(mgr, "alice", engine)
with _captured_logs() as records:
mgr.close_user_database("alice") # must not raise
assert engine.dispose_calls == 1
# De-registered even though the dispose failed -- the engine is now
# unreachable, so its pooled connections can never be released.
assert "alice" not in mgr.connections
assert "alice" not in mgr._password_verifiers
assert mgr.is_user_connected("alice") is False
dispose_logs = [
r
for r in records
if "Failed to dispose engine for alice" in r["message"]
]
assert len(dispose_logs) == 1
entry = dispose_logs[0]
assert entry["level"] == "WARNING"
# The type name is present; the detail and traceback are not:
assert entry["exception"] is False, "a traceback is attached after all"
assert "SENTINEL_DISPOSE_DETAIL" not in entry["message"]
assert "RuntimeError" in entry["message"]
# ...and the sensitive detail is not recovered anywhere else either.
assert not [r for r in records if "SENTINEL_DISPOSE_DETAIL" in r["message"]]
def test_checkpoint_wal_failure_is_invisible_at_default_log_level(tmp_path):
"""A failed pre-dispose WAL checkpoint is logged at DEBUG only.
``_checkpoint_wal`` catches everything and logs at ``debug``. A checkpoint
that could not run (or reported ``busy``) means pending WAL content was
not flushed before the engine went away, yet at production log levels the
close reports unqualified success.
"""
engine = _FakeEngine()
with _manager(lambda: tmp_path / "root") as mgr:
_register(mgr, "alice", engine)
with _captured_logs() as records, _trust_epoch():
mgr.close_user_database("alice")
checkpoint_logs = [
r for r in records if "WAL checkpoint failed" in r["message"]
]
assert checkpoint_logs, "checkpoint failure was not logged at all"
assert all(r["level"] == "DEBUG" for r in checkpoint_logs)
# Nothing at WARNING or above marks the close as degraded...
assert not [
r for r in records if r["level"] in {"WARNING", "ERROR", "CRITICAL"}
]
# ...and the only INFO-or-above line reports plain success.
closed = [
r for r in records if "Closed database for user alice" in r["message"]
]
assert len(closed) == 1
assert closed[0]["level"] == "INFO"
def test_init_lock_is_retained_after_close(tmp_path):
"""The per-user init lock deliberately survives BOTH close paths.
Documented invariant: dropping it would let a later open create a SECOND
lock for the same user, so two cold-opens could migrate one file at once.
Neither ``close_user_database`` nor ``close_all_databases`` clears it.
"""
with _manager(lambda: tmp_path / "root") as mgr:
lock = mgr._get_init_lock("alice")
_register(mgr, "alice", _FakeEngine())
mgr.close_user_database("alice")
assert mgr._init_locks.get("alice") is lock
mgr.close_all_databases()
assert mgr._init_locks.get("alice") is lock
# ===========================================================================
# 3. The data_dir property
# ===========================================================================
def test_data_dir_round_trip_permanently_pins_the_override(tmp_path):
"""``original = mgr.data_dir; ...; mgr.data_dir = original`` does NOT restore.
The getter RESOLVES a default (``_data_dir_override is None``); the setter
WRITES an override. So the widespread save/restore fixture idiom converts
a manager that tracks the live data root into one pinned to whatever the
root happened to be when the fixture ran -- for the rest of the process.
Against the global ``db_manager`` singleton under xdist that pins the
whole worker.
"""
roots = {"current": tmp_path / "first"}
with _manager(lambda: roots["current"]) as mgr:
assert mgr._data_dir_override is None # dynamic to start with
original = mgr.data_dir # the "save" half of the idiom
mgr.data_dir = original # the "restore" half
# The restore installed an override that was not there before.
assert mgr._data_dir_override == original
roots["current"] = tmp_path / "second"
assert mgr.data_dir == original
assert mgr.data_dir != tmp_path / "second" / "encrypted_databases"
# A manager that never went through the round trip still follows the root.
roots["current"] = tmp_path / "third"
with _manager(lambda: roots["current"]) as fresh:
assert fresh.data_dir == tmp_path / "third" / "encrypted_databases"
def test_data_dir_setter_offers_no_way_back_to_dynamic_resolution(tmp_path):
"""Only the private attribute can undo the pin; the property cannot.
``mgr.data_dir = None`` raises inside ``Path(value)`` before any state is
written, so a fixture has no supported way to hand the manager back its
default-tracking behaviour.
"""
roots = {"current": tmp_path / "first"}
with _manager(lambda: roots["current"]) as mgr:
pinned = mgr.data_dir
mgr.data_dir = pinned
with pytest.raises(TypeError):
mgr.data_dir = None
assert mgr._data_dir_override == pinned # unchanged by the failure
roots["current"] = tmp_path / "second"
assert mgr.data_dir == pinned
mgr._data_dir_override = None # the only real restore
assert mgr.data_dir == tmp_path / "second" / "encrypted_databases"
def test_reading_data_dir_is_a_filesystem_mutation(tmp_path):
"""The getter creates and hardens the directory as a side effect.
``data_dir`` is a property read, but ``_ensure_dir`` mkdirs and chmods
0o700 under ``_connections_lock``. Any code that merely *inspects*
``mgr.data_dir`` materialises the directory.
"""
root = tmp_path / "root"
target = root / "encrypted_databases"
with _manager(lambda: root) as mgr:
assert not target.exists()
resolved = mgr.data_dir
assert resolved == target
assert target.is_dir()
assert stat.S_IMODE(target.stat().st_mode) == 0o700
def test_data_dir_is_never_recreated_or_rehardened_after_first_access(tmp_path):
"""``_initialized_data_dirs`` memoises per path, so the manager never repairs.
Once a path has been ensured, later accesses are a pure dict lookup. If
the directory is subsequently removed or loosened -- a volume remount, an
operator chmod, a tmpdir teardown -- the manager keeps handing out the
path without recreating or re-hardening it, and ``_get_user_db_path``
resolves under a parent that no longer exists.
"""
root = tmp_path / "root"
with _manager(lambda: root) as mgr:
target = mgr.data_dir
assert stat.S_IMODE(target.stat().st_mode) == 0o700
os.chmod(target, 0o755) # noqa: S103 - simulating an external loosening
assert mgr.data_dir == target
assert stat.S_IMODE(target.stat().st_mode) == 0o755, (
"directory was re-hardened on access (contract changed)"
)
shutil.rmtree(target)
assert mgr.data_dir == target
assert not target.exists(), "directory was recreated (contract changed)"
assert not mgr._get_user_db_path("alice").parent.exists()
def test_data_dir_override_does_not_redirect_the_backup_service(tmp_path):
"""The manager's override is manager-local; BackupService resolves its own root.
``BackupService.__init__`` calls ``get_encrypted_database_path()`` (i.e.
``get_data_directory()/encrypted_databases``) directly. It never consults
``db_manager.data_dir``. So a manager pinned to a different directory --
the fixture idiom above -- makes the pre-migration backup in
``_open_user_database_cold`` point at a DIFFERENT file than the one being
migrated (or at no file at all, in which case the backup silently fails
and the migration proceeds unprotected).
"""
from local_deep_research.database.backup.backup_service import (
BackupService,
)
default_root = tmp_path / "default"
override = tmp_path / "override" / "encrypted_databases"
with _manager(lambda: default_root) as mgr:
mgr.data_dir = override
manager_db_path = mgr._get_user_db_path("alice")
with patch(
"local_deep_research.config.paths.get_data_directory",
return_value=default_root,
):
service = BackupService(username="alice", password="pw")
assert manager_db_path.parent == override
assert service.db_path.parent == default_root / "encrypted_databases"
assert service.db_path != manager_db_path
# Same user, same filename -- only the root diverges, which is exactly why
# the divergence is easy to miss.
assert service.db_path.name == manager_db_path.name
# ===========================================================================
# 4. Backup: salt binding, absence of restore, chmod handling
# ===========================================================================
def test_verify_backup_keys_off_its_own_salt(tmp_path):
"""Verification must not conceal a backup's missing recovery inputs."""
import local_deep_research.database.backup.backup_service as bs
db_dir = tmp_path / "encrypted_databases"
db_dir.mkdir(parents=True)
backup_dir = tmp_path / "backups"
backup_dir.mkdir()
backup_file = backup_dir / "ldr_backup_20250101_000000.db"
backup_file.write_bytes(b"pretend-encrypted-backup")
captured = {}
def fake_set_key(cursor, password, db_path=None, **kwargs):
captured["db_path"] = db_path
class _Cursor:
def execute(self, sql, *args):
return self
def fetchone(self):
return ("ok",)
def close(self):
pass
class _Conn:
def cursor(self):
return _Cursor()
def close(self):
pass
with (
patch.object(bs, "get_encrypted_database_path", return_value=db_dir),
patch.object(
bs, "get_user_database_filename", return_value="ldr_user_x.db"
),
patch.object(bs, "get_user_backup_directory", return_value=backup_dir),
):
service = bs.BackupService(username="alice", password="pw")
with (
patch(
"local_deep_research.database.sqlcipher_compat.get_sqlcipher_module"
) as mod,
patch.object(bs, "set_sqlcipher_key", side_effect=fake_set_key),
patch.object(bs, "apply_sqlcipher_pragmas"),
patch.object(bs, "verify_sqlcipher_connection", return_value=True),
):
mod.return_value.connect.return_value = _Conn()
assert service._verify_backup(backup_file) is True
assert captured["db_path"] == backup_file
assert captured["db_path"] != service.db_path
def test_backup_service_exposes_no_restore_path():
"""Guard for the SAFETY INVARIANT in ``create_user_database``.
That function deletes an orphaned ``.salt`` on the strength of "nothing
restores or imports a .db from a salt". Backups are write-only today: the
service creates, lists, prunes and refreshes them, and never writes back
to ``self.db_path``. If a restore/import that stages a ``.salt`` ahead of
its ``.db`` is ever added, that deletion branch must be re-evaluated
before this assertion is relaxed.
"""
import local_deep_research.database.backup.backup_service as bs
import local_deep_research.database.backup as backup_pkg
offenders = [
name
for name in dir(bs.BackupService)
if not name.startswith("__")
and ("restore" in name.lower() or "import" in name.lower())
]
assert not offenders, (
f"BackupService gained {offenders}; re-read the SAFETY INVARIANT in "
"encrypted_db.create_user_database before allowing this"
)
assert not [
name
for name in getattr(backup_pkg, "__all__", [])
if "restore" in name.lower()
]
def test_backup_chmod_failure_aborts_the_whole_backup(tmp_path):
"""The backup chmod is NOT best-effort, unlike the database chmod.
``_create_backup_impl`` calls bare ``os.chmod(temp_path, 0o600)``. On a
filesystem where chmod raises (the Docker bind-mount / FUSE case that
``_best_effort_chmod`` exists to tolerate -- see
``tests/database/test_db_file_permissions.py::
test_db_creation_survives_chmod_failure``, where DB creation survives),
the whole backup fails and the temp file is deleted. Such a deployment
can create user databases but can never take a pre-migration backup.
"""
import local_deep_research.database.backup.backup_service as bs
db_dir = tmp_path / "encrypted_databases"
db_dir.mkdir(parents=True)
source_db = db_dir / "ldr_user_x.db"
source_db.write_bytes(b"x" * 4096)
backup_dir = tmp_path / "backups"
backup_dir.mkdir()
class _Cursor:
def execute(self, sql, *args):
if sql.startswith("ATTACH DATABASE '"):
Path(sql.split("'", 2)[1]).write_bytes(b"exported-backup")
return self
def close(self):
pass
class _Conn:
def cursor(self):
return _Cursor()
def close(self):
pass
real_chmod = os.chmod
def picky_chmod(path, mode, *args, **kwargs):
if str(path).endswith(".tmp"):
raise OSError("chmod is not supported on this filesystem")
return real_chmod(path, mode, *args, **kwargs)
with (
patch.object(bs, "get_encrypted_database_path", return_value=db_dir),
patch.object(
bs, "get_user_database_filename", return_value="ldr_user_x.db"
),
patch.object(bs, "get_user_backup_directory", return_value=backup_dir),
):
service = bs.BackupService(username="alice", password="pw")
with (
patch.object(bs, "create_sqlcipher_connection", return_value=_Conn()),
patch.object(bs, "get_key_from_password", return_value=b"\xab" * 32),
patch.object(bs.BackupService, "_verify_backup", return_value=True),
patch.object(bs.os, "chmod", side_effect=picky_chmod),
):
result = service._create_backup_impl()
assert result.success is False
assert "chmod" in result.error
# Nothing survives: neither the temp file nor a finalized backup.
assert list(backup_dir.iterdir()) == []