~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""A sidecar fault on the cache probe fails closed and keeps the engine.
The request fails closed, but the cached engine is kept: evicting on a
transient fault (an EMFILE spike) would dispose every user's engine.
The same holds when the sidecar opens but the epoch read then fails;
only an epoch that was read and does not vouch detaches the engine.
Every cache accessor claims the cached generation under the rekey
sidecar. When the sidecar cannot be opened (fd exhaustion, EROFS,
EACCES, a vanished data directory), the lock module raises the typed
``RekeySidecarIOError`` -- not an ``OSError`` -- so the probe's
untrusted-state branch never saw it and every caller had to catch it
itself (``get_session``, the log sink and the metrics session did not).
``is_user_connected`` and ``_password_matches_cached`` now read it as
"not connected". The fault is injected at the real ``os.open`` of the
sidecar file.
Three callers must not read such a fault as a plain miss.
``probe_user_connection`` reports it as UNAVAILABLE, so request
authentication can answer a retryable 503 instead of a 401 that logs the
user out. ``open_user_database`` raises it before deriving a key: its cold
path needs the same sidecar, so falling through only added a PBKDF2
derivation to every attempt for as long as the fault lasted. The metrics
session falls through to that raise and lets it out too. Its callers
decide what it means: the metrics writer reports its own "no session"
failure, and ``get_user_db_session`` raises
``DatabaseSessionUnavailableError``, which request handlers answer with a
retryable 503. Folding it into a None made it the same
``DatabaseSessionError`` a wrong password produces, and the home page
answered that by logging the user out. ``get_session`` raises it too:
``get_settings_manager`` reads its None as "this user has no database"
and answers every setting from the defaults file, so during the fault a
stored ``strict`` egress scope read as the default.
Releasing the sidecar after a claim can fail as well. That raises for
every caller when the claim's body finished cleanly, even when the probe
runs inside a caller's except block, and is added as a note to the body's
own exception when the body raised.
"""
from __future__ import annotations
import errno
import os
import stat
import threading
import uuid
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from contextvars import copy_context
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from local_deep_research.database import encrypted_db as encrypted_db_module
from local_deep_research.database import rekey_lock as rekey_lock_module
from local_deep_research.database import (
session_context as session_context_module,
)
from local_deep_research.database import (
thread_local_session as thread_local_session_module,
)
from local_deep_research.database import thread_metrics as thread_metrics_module
from local_deep_research.database.encrypted_db import (
DatabaseManager,
UserConnectionStatus,
)
from local_deep_research.database.rekey_lock import (
RekeyEpochUnreadableError,
RekeyLockContentionError,
RekeySidecarIOError,
RekeySidecarOperation,
acquire_rekey_lock,
)
from local_deep_research.database.sqlcipher_utils import get_salt_file_path
from local_deep_research.utilities import db_utils, log_utils
def _new_manager(data_dir: Path) -> DatabaseManager:
with patch.object(
DatabaseManager, "_check_encryption_available", return_value=True
):
manager = DatabaseManager()
manager.data_dir = data_dir
return manager
@pytest.mark.parametrize(
"accessor", ["is_user_connected", "get_session", "_password_matches_cached"]
)
def test_sidecar_open_fault_fails_closed_without_evicting(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, accessor: str
) -> None:
manager = _new_manager(tmp_path)
db_path = manager._get_user_db_path("alice")
db_path.touch()
with acquire_rekey_lock(db_path):
epoch = rekey_lock_module.read_or_initialize_rekey_epoch(db_path)
engine = MagicMock()
manager._cache_connection(
"alice", manager._make_connection_cache_entry(engine, "password", epoch)
)
real_open = rekey_lock_module.os.open
def emfile_on_sidecar(path, flags, *args, **kwargs):
if str(path).endswith(".rekey.lock"):
raise OSError(errno.EMFILE, "Too many open files")
return real_open(path, flags, *args, **kwargs)
monkeypatch.setattr(rekey_lock_module.os, "open", emfile_on_sidecar)
if accessor == "is_user_connected":
result = manager.is_user_connected("alice")
assert result is False
elif accessor == "get_session":
# Raised, not None: get_settings_manager reads None as "this user
# has no database" and answers every setting from the defaults
# file (see the get_settings_manager test below).
with pytest.raises(RekeySidecarIOError) as raised:
manager.get_session("alice")
assert raised.value.operation is RekeySidecarOperation.OPEN
assert raised.value.errno == errno.EMFILE
else:
assert manager._password_matches_cached("alice", "password") is False
# This request failed closed, but the healthy engine was NOT evicted:
# an fd-exhaustion spike must not dispose every user's engine at once.
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
# Once the fault clears, the next probe re-takes the sidecar, re-checks
# the epoch and hands the same engine out again.
monkeypatch.setattr(rekey_lock_module.os, "open", real_open)
assert manager.is_user_connected("alice") is True
def test_engine_kept_through_a_fault_is_still_fenced_by_a_later_rekey(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# The engine survives the faulted probe only because every later
# hand-out re-validates it: a rekey committed during the fault must
# detach it on the first successful probe.
manager = _new_manager(tmp_path)
db_path = manager._get_user_db_path("alice")
db_path.touch()
with acquire_rekey_lock(db_path):
epoch = rekey_lock_module.read_or_initialize_rekey_epoch(db_path)
engine = MagicMock()
manager._cache_connection(
"alice", manager._make_connection_cache_entry(engine, "password", epoch)
)
real_open = rekey_lock_module.os.open
def emfile_on_sidecar(path, flags, *args, **kwargs):
if str(path).endswith(".rekey.lock"):
raise OSError(errno.EMFILE, "Too many open files")
return real_open(path, flags, *args, **kwargs)
monkeypatch.setattr(rekey_lock_module.os, "open", emfile_on_sidecar)
assert manager.is_user_connected("alice") is False
# Kept through the fault (the precondition this test is about).
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
monkeypatch.setattr(rekey_lock_module.os, "open", real_open)
# Another worker commits a rekey while the engine sits in the cache.
with acquire_rekey_lock(db_path):
rekey_lock_module.advance_rekey_epoch(db_path)
assert manager.is_user_connected("alice") is False
assert "alice" not in manager.connections
engine.dispose.assert_called_once_with()
def _fail_epoch_reads(monkeypatch: pytest.MonkeyPatch, error_number: int):
# The epoch reader opens each token file with os.open (no-follow,
# non-blocking); fail that open for the token files only. Their names
# end in the suffix, while mkstemp's temporaries carry a random tail,
# so a publication is unaffected. Returns the call that clears it.
real_open = rekey_lock_module.os.open
def failing_open(path, flags, *args, **kwargs):
if os.fspath(path).endswith((".rekey.epoch", ".rekey.pending")):
raise OSError(error_number, "injected epoch read failure")
return real_open(path, flags, *args, **kwargs)
monkeypatch.setattr(rekey_lock_module.os, "open", failing_open)
return lambda: monkeypatch.setattr(rekey_lock_module.os, "open", real_open)
@pytest.mark.parametrize("error_number", [errno.EMFILE, errno.EIO])
def test_epoch_read_fault_after_the_sidecar_opens_keeps_the_engine(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, error_number: int
) -> None:
# The sidecar opens, then the epoch read fails (EMFILE right after the
# sidecar took the last fd). Nothing was read, so this must behave like
# a sidecar fault: the request sees "not connected" (get_session, for
# which that would mean "no database", gets the typed error), and the
# engine stays.
manager = _new_manager(tmp_path)
db_path = manager._get_user_db_path("alice")
db_path.touch()
with acquire_rekey_lock(db_path):
epoch = rekey_lock_module.read_or_initialize_rekey_epoch(db_path)
engine = MagicMock()
manager._cache_connection(
"alice", manager._make_connection_cache_entry(engine, "password", epoch)
)
clear_fault = _fail_epoch_reads(monkeypatch, error_number)
assert manager.is_user_connected("alice") is False
with pytest.raises(RekeyEpochUnreadableError) as raised:
manager.get_session("alice")
assert raised.value.errno == error_number
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
clear_fault()
assert manager.is_user_connected("alice") is True
def test_epoch_advanced_during_an_epoch_read_fault_still_detaches(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
manager = _new_manager(tmp_path)
db_path = manager._get_user_db_path("alice")
db_path.touch()
with acquire_rekey_lock(db_path):
epoch = rekey_lock_module.read_or_initialize_rekey_epoch(db_path)
engine = MagicMock()
manager._cache_connection(
"alice", manager._make_connection_cache_entry(engine, "password", epoch)
)
clear_fault = _fail_epoch_reads(monkeypatch, errno.EMFILE)
assert manager.is_user_connected("alice") is False
assert manager.connections["alice"] is engine
clear_fault()
with acquire_rekey_lock(db_path):
rekey_lock_module.advance_rekey_epoch(db_path)
assert manager.is_user_connected("alice") is False
assert "alice" not in manager.connections
engine.dispose.assert_called_once_with()
def test_malformed_epoch_on_the_probe_still_detaches(tmp_path: Path) -> None:
# Only "could not read" keeps the engine. An epoch that WAS read and
# cannot vouch for the generation (a malformed token) still detaches
# and disposes it.
manager = _new_manager(tmp_path)
db_path = manager._get_user_db_path("alice")
db_path.touch()
with acquire_rekey_lock(db_path):
epoch = rekey_lock_module.read_or_initialize_rekey_epoch(db_path)
engine = MagicMock()
manager._cache_connection(
"alice", manager._make_connection_cache_entry(engine, "password", epoch)
)
db_path.with_name(f"{db_path.name}.rekey.epoch").write_text(
"NOT-A-CANONICAL-UUID", encoding="ascii"
)
assert manager.is_user_connected("alice") is False
assert "alice" not in manager.connections
engine.dispose.assert_called_once_with()
def _cache_current_engine(
manager: DatabaseManager,
password: str = "password",
*,
username: str = "alice",
) -> tuple[Path, MagicMock]:
db_path = manager._get_user_db_path(username)
db_path.touch()
with acquire_rekey_lock(db_path):
epoch = rekey_lock_module.read_or_initialize_rekey_epoch(db_path)
engine = MagicMock()
manager._cache_connection(
username, manager._make_connection_cache_entry(engine, password, epoch)
)
return db_path, engine
def _fail_sidecar_opens(monkeypatch: pytest.MonkeyPatch, db_path: Path):
# EMFILE for this database's sidecar only: os.open is process-wide, and
# any other thread's files must keep opening normally.
sidecar = os.fspath(db_path.with_name(f"{db_path.name}.rekey.lock"))
real_open = rekey_lock_module.os.open
def emfile_on_sidecar(path, flags, *args, **kwargs):
if os.fspath(path) == sidecar:
raise OSError(errno.EMFILE, "Too many open files")
return real_open(path, flags, *args, **kwargs)
monkeypatch.setattr(rekey_lock_module.os, "open", emfile_on_sidecar)
return real_open
def _watch_key_derivation(
monkeypatch: pytest.MonkeyPatch, watched: Path
) -> list[Path]:
# Records each PBKDF2 derivation a cold open of THIS database starts and
# answers it with a well-formed key, so code that does derive carries on
# and the test can report that it did. Every other database passes
# straight through, so another thread in the process cannot skew it.
real_derive = encrypted_db_module.get_key_from_password
derived: list[Path] = []
def derive(password, db_path=None):
if db_path is not None and Path(db_path) == watched:
derived.append(Path(db_path))
return bytes(64)
return real_derive(password, db_path=db_path)
monkeypatch.setattr(encrypted_db_module, "get_key_from_password", derive)
return derived
def test_probe_user_connection_tells_a_sidecar_fault_from_a_miss(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
manager = _new_manager(tmp_path)
assert (
manager.probe_user_connection("alice")
is UserConnectionStatus.NOT_CONNECTED
)
db_path, engine = _cache_current_engine(manager)
assert (
manager.probe_user_connection("alice") is UserConnectionStatus.CONNECTED
)
real_open = _fail_sidecar_opens(monkeypatch, db_path)
# is_user_connected cannot tell the fault from a miss; the probe can.
assert manager.is_user_connected("alice") is False
assert (
manager.probe_user_connection("alice")
is UserConnectionStatus.UNAVAILABLE
)
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
monkeypatch.setattr(rekey_lock_module.os, "open", real_open)
assert (
manager.probe_user_connection("alice") is UserConnectionStatus.CONNECTED
)
# A stale engine is a miss, not a fault: detached and disposed.
with acquire_rekey_lock(db_path):
rekey_lock_module.advance_rekey_epoch(db_path)
assert (
manager.probe_user_connection("alice")
is UserConnectionStatus.NOT_CONNECTED
)
assert "alice" not in manager.connections
engine.dispose.assert_called_once_with()
@pytest.mark.parametrize("error_number", [errno.EMFILE, errno.EIO])
def test_probe_user_connection_reports_an_epoch_read_fault_as_unavailable(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, error_number: int
) -> None:
manager = _new_manager(tmp_path)
_, engine = _cache_current_engine(manager)
clear_fault = _fail_epoch_reads(monkeypatch, error_number)
assert (
manager.probe_user_connection("alice")
is UserConnectionStatus.UNAVAILABLE
)
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
clear_fault()
assert (
manager.probe_user_connection("alice") is UserConnectionStatus.CONNECTED
)
def test_probe_user_connection_reports_a_malformed_epoch_as_a_miss(
tmp_path: Path,
) -> None:
# An epoch that was read and does not vouch is a verdict, not a fault.
manager = _new_manager(tmp_path)
db_path, engine = _cache_current_engine(manager)
db_path.with_name(f"{db_path.name}.rekey.epoch").write_text(
"NOT-A-CANONICAL-UUID", encoding="ascii"
)
assert (
manager.probe_user_connection("alice")
is UserConnectionStatus.NOT_CONNECTED
)
assert "alice" not in manager.connections
engine.dispose.assert_called_once_with()
@pytest.mark.skipif(os.name == "nt", reason="injects a POSIX flock unlock")
def test_probe_user_connection_reports_a_release_fault_without_raising(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# The claim succeeds but releasing the sidecar fails. is_user_connected
# lets that RekeySidecarIOError(UNLOCK) escape; the probe never raises.
manager = _new_manager(tmp_path)
_, engine = _cache_current_engine(manager)
def failing_unlock(fd: int) -> None:
raise OSError(errno.EIO, "injected unlock failure")
monkeypatch.setattr(
rekey_lock_module, "_release_posix_lock", failing_unlock
)
with pytest.raises(RekeySidecarIOError) as raised:
manager.is_user_connected("alice")
assert raised.value.operation is RekeySidecarOperation.UNLOCK
assert (
manager.probe_user_connection("alice")
is UserConnectionStatus.UNAVAILABLE
)
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
class _CallerHandledError(Exception):
"""An unrelated exception a caller is handling when it probes."""
def _fail_posix_unlock(monkeypatch: pytest.MonkeyPatch) -> None:
def failing_unlock(fd: int) -> None:
raise OSError(errno.EIO, "injected unlock failure")
monkeypatch.setattr(
rekey_lock_module, "_release_posix_lock", failing_unlock
)
@pytest.mark.skipif(os.name == "nt", reason="injects a POSIX flock unlock")
def test_a_release_fault_is_reported_when_probed_inside_an_except_block(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# A probe can run inside a caller's error handler: render_template
# opens a database session for its egress-scope read, and routes render
# their error pages from except branches. The lock's cleanup used to
# take the exception that CALLER was handling for the claim's own
# failure (sys.exception()), attach the unlock failure to it as a note
# and report a clean release: CONNECTED, where the probe outside an
# except block answers UNAVAILABLE.
manager = _new_manager(tmp_path)
_, engine = _cache_current_engine(manager)
_fail_posix_unlock(monkeypatch)
handled = _CallerHandledError("the caller's own error")
try:
raise handled
except _CallerHandledError:
status = manager.probe_user_connection("alice")
with pytest.raises(RekeySidecarIOError) as raised:
manager.is_user_connected("alice")
assert status is UserConnectionStatus.UNAVAILABLE
assert raised.value.operation is RekeySidecarOperation.UNLOCK
assert not getattr(handled, "__notes__", None)
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
@pytest.mark.skipif(os.name == "nt", reason="injects a POSIX flock unlock")
def test_an_error_inside_the_claim_keeps_a_release_fault_as_a_note(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# The claim enters and leaves the sidecar lock by hand, so that it can
# tell a failed acquire from a failure in its own body. It hands the
# body's exception to the lock the way a with statement does, so the
# unlock failure becomes a note on that exception instead of replacing
# it with a RekeySidecarIOError.
manager = _new_manager(tmp_path)
_, engine = _cache_current_engine(manager)
_fail_posix_unlock(monkeypatch)
class _ClaimBodyError(Exception):
pass
def broken_sessionmaker(**_kwargs):
raise _ClaimBodyError("the claim's own failure")
monkeypatch.setattr(
encrypted_db_module, "sessionmaker", broken_sessionmaker
)
with pytest.raises(_ClaimBodyError) as raised:
manager.get_session("alice")
notes = getattr(raised.value, "__notes__", [])
assert len(notes) == 1
assert "unlock" in notes[0]
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
@pytest.mark.parametrize("fault", ["sidecar-open", "epoch-read"])
def test_get_settings_manager_does_not_answer_a_fault_with_defaults(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fault: str
) -> None:
# get_settings_manager without a session reaches the user's cached
# engine through db_manager.get_session, and answers that method's None
# ("this user has no database") from the defaults file. A sidecar or
# epoch fault used to come back as that None, so every stored setting
# read as its default for as long as the fault lasted: a "strict" or
# "private_only" egress scope as "adaptive", a custom library
# storage_path as the default directory. It must reach the caller.
manager = _new_manager(tmp_path)
username = f"settings_{uuid.uuid4().hex[:8]}"
db_path, engine = _cache_current_engine(manager, username=username)
monkeypatch.setattr(db_utils, "db_manager", manager)
expected: type[Exception]
if fault == "sidecar-open":
_fail_sidecar_opens(monkeypatch, db_path)
expected = RekeySidecarIOError
else:
_fail_epoch_reads(monkeypatch, errno.EIO)
expected = RekeyEpochUnreadableError
with pytest.raises(expected):
db_utils.get_settings_manager(username=username)
assert manager.connections[username] is engine
engine.dispose.assert_not_called()
# The defaults fallback itself is unchanged for a user with no engine.
no_engine = db_utils.get_settings_manager(
username=f"no_engine_{uuid.uuid4().hex[:8]}"
)
assert no_engine.db_session is None
def test_open_user_database_fails_fast_on_a_sidecar_fault(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# With an engine cached, the fault is raised before any key is derived.
# Falling through to the cold path paid a PBKDF2 derivation only to
# fail on the same sidecar, on every request, socket handshake and
# background session for as long as the fault lasted.
manager = _new_manager(tmp_path)
db_path, engine = _cache_current_engine(manager)
derived = _watch_key_derivation(monkeypatch, db_path)
real_open = _fail_sidecar_opens(monkeypatch, db_path)
for password in ("password", "wrong-password"):
# Never a credential verdict (None): the login route answers this
# typed error with a 503 that is not counted toward lockout.
with pytest.raises(RekeySidecarIOError) as raised:
manager.open_user_database("alice", password)
assert raised.value.operation is RekeySidecarOperation.OPEN
assert raised.value.errno == errno.EMFILE
assert derived == []
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
# Once the fault clears, the kept engine is handed out again.
monkeypatch.setattr(rekey_lock_module.os, "open", real_open)
assert manager.open_user_database("alice", "password") is engine
assert derived == []
def test_metrics_session_raises_the_typed_fault_and_the_writer_reports_none(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# create_thread_safe_session_for_metrics falls through to
# open_user_database, which raises the typed fault before deriving a
# key. The metrics session lets it out rather than answering None: a
# None reached get_user_db_session as the DatabaseSessionError a wrong
# password produces (see the get_user_db_session test below).
manager = _new_manager(tmp_path)
db_path, engine = _cache_current_engine(manager)
derived = _watch_key_derivation(monkeypatch, db_path)
_fail_sidecar_opens(monkeypatch, db_path)
with pytest.raises(RekeySidecarIOError) as raised:
manager.create_thread_safe_session_for_metrics("alice", "password")
assert raised.value.operation is RekeySidecarOperation.OPEN
assert raised.value.errno == errno.EMFILE
assert derived == []
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
# The metrics writer, whose callers do not know the typed error,
# reports its own "no session" failure instead (thread_metrics used to
# re-raise whatever the metrics session raised).
monkeypatch.setattr(thread_metrics_module, "db_manager", manager)
writer = thread_metrics_module.ThreadSafeMetricsWriter()
writer.set_user_password("alice", "password")
with pytest.raises(ValueError, match="Failed to create session") as failed:
with writer.get_session("alice"):
pytest.fail("a session was handed out during the fault")
# Raised after the fault was handled, so the fault (whose frames hold
# the password) is not chained onto it.
assert failed.value.__context__ is None
assert derived == []
def _fail_sidecar_opens_after(
monkeypatch: pytest.MonkeyPatch, db_path: Path, healthy_opens: int
) -> list[int]:
# An fd-exhaustion spike that starts part-way through a request: the
# first ``healthy_opens`` opens of this database's sidecar succeed,
# every later one fails with EMFILE. Other files are untouched.
sidecar = os.fspath(db_path.with_name(f"{db_path.name}.rekey.lock"))
real_open = rekey_lock_module.os.open
failed: list[int] = []
opened = 0
def flaky_sidecar(path, flags, *args, **kwargs):
nonlocal opened
if os.fspath(path) == sidecar:
opened += 1
if opened > healthy_opens:
failed.append(opened)
raise OSError(errno.EMFILE, "Too many open files")
return real_open(path, flags, *args, **kwargs)
monkeypatch.setattr(rekey_lock_module.os, "open", flaky_sidecar)
return failed
@pytest.mark.parametrize(
"healthy_opens",
[0, 1],
ids=["open-claim-faults", "metrics-claim-faults-after-open-succeeded"],
)
def test_get_user_db_session_reports_a_sidecar_fault_as_unavailable(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, healthy_opens: int
) -> None:
# On a thread without a session, get_user_db_session claims the cached
# engine twice: in open_user_database, then in
# create_thread_safe_session_for_metrics. A fluctuating fd-exhaustion
# spike can fail either claim, the second one even after the first
# (and require_auth's probe before it) succeeded. Each used to come
# back as None, i.e. the plain DatabaseSessionError a wrong password
# produces, which "/" answered by clearing the session. It must be the
# typed subclass that request handlers answer with a retryable 503.
manager = _new_manager(tmp_path)
db_path, engine = _cache_current_engine(manager)
derived = _watch_key_derivation(monkeypatch, db_path)
sessions = thread_local_session_module.ThreadLocalSessionManager()
monkeypatch.setattr(thread_local_session_module, "db_manager", manager)
monkeypatch.setattr(
thread_local_session_module, "thread_session_manager", sessions
)
monkeypatch.setattr(
session_context_module, "thread_session_manager", sessions
)
failed = _fail_sidecar_opens_after(monkeypatch, db_path, healthy_opens)
with pytest.raises(session_context_module.DatabaseSessionError) as raised:
with session_context_module.get_user_db_session("alice", "password"):
pytest.fail("a session was handed out during the fault")
assert failed, "the fault was never hit: measuring the wrong branch"
# Compared by name so a tree without the subclass reports what it
# raised instead (the plain error), rather than an AttributeError.
assert type(raised.value).__name__ == "DatabaseSessionUnavailableError", (
"a sidecar fault must not read like a failed open: got "
f"{type(raised.value).__name__}({raised.value})"
)
assert isinstance(
raised.value, session_context_module.DatabaseSessionUnavailableError
)
# Same scrub-safe fields the login route logs for this fault class:
# type name, sidecar operation and errno (EMFILE), and no path material.
# The sidecar open is the first fault; the second claim's fault
# surfaces from its own operation, so only the shape is pinned there.
assert raised.value.reason.startswith("RekeySidecarIOError (operation=")
assert f"errno={errno.EMFILE})" in raised.value.reason
assert str(db_path) not in raised.value.reason
assert db_path.name not in raised.value.reason
# Neither chained nor kept as context: the frames the fault came
# through hold the plaintext password.
assert raised.value.__cause__ is None
assert raised.value.__context__ is None
# Failed before any key derivation, kept the engine, cached nothing.
assert derived == []
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
assert sessions.get_current_session() is None
assert sessions._thread_credentials == {}
# A sidecar held past the claim's bound. The holder never releases on its
# own: another process's rekey or cold open of a large database, a stopped
# process, a hung mount. Every claim used to wait for it with a blocking
# flock, so every request for the user and the shared log-drain thread
# (which persists every user's log entries) hung for as long as it held it.
# The bound is shortened here; each wait below is bounded by the test too,
# so a regression fails instead of hanging the job.
_BOUND = 0.2
_JOIN_SECONDS = 10.0
@contextmanager
def _sidecar_flock_held_elsewhere(db_path: Path) -> Iterator[None]:
# A second open file description of the sidecar: flock makes it contend
# with the manager's own descriptor exactly as another process's would,
# and it never takes this process's gate, like a foreign holder.
import fcntl
fd = os.open(
db_path.with_name(f"{db_path.name}.rekey.lock"),
os.O_RDWR | os.O_CREAT,
0o600,
)
try:
fcntl.flock(fd, fcntl.LOCK_EX)
yield
finally:
os.close(fd)
def _run_bounded(
call: Callable[[], object], timeout: float = 5.0
) -> tuple[threading.Thread, dict[str, object]]:
outcome: dict[str, object] = {}
def target() -> None:
try:
outcome["value"] = call()
except BaseException as error: # noqa: BLE001 - handed to the test
outcome["error"] = error
worker = threading.Thread(target=target, daemon=True)
worker.start()
worker.join(timeout)
return worker, outcome
@pytest.mark.skipif(os.name == "nt", reason="POSIX flock required")
@pytest.mark.parametrize(
("accessor", "expected"),
[
("is_user_connected", False),
("probe_user_connection", UserConnectionStatus.UNAVAILABLE),
("get_session", RekeyLockContentionError),
("open_user_database", RekeyLockContentionError),
],
)
def test_a_sidecar_held_past_the_bound_fails_the_claim_closed(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
accessor: str,
expected: object,
) -> None:
monkeypatch.setattr(
encrypted_db_module,
"_SIDECAR_WAIT_TIMEOUT_SECONDS",
_BOUND,
raising=False,
)
manager = _new_manager(tmp_path)
db_path, engine = _cache_current_engine(manager)
derived = _watch_key_derivation(monkeypatch, db_path)
arguments = ("alice", "password")[
: 2 if accessor == "open_user_database" else 1
]
with _sidecar_flock_held_elsewhere(db_path):
worker, outcome = _run_bounded(
lambda: getattr(manager, accessor)(*arguments)
)
finished = not worker.is_alive()
worker.join(timeout=_JOIN_SECONDS)
assert finished, f"{accessor} waited for the holder instead of the bound"
if isinstance(expected, type):
assert isinstance(outcome.get("error"), expected), outcome
else:
assert outcome.get("value") is expected, outcome
# A retryable fault, not a verdict: nothing derived, nothing evicted,
# and the same engine is handed out once the holder is gone.
assert derived == []
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
assert (
manager.probe_user_connection("alice") is UserConnectionStatus.CONNECTED
)
@pytest.mark.parametrize(
("accessor", "expected"),
[
("probe_user_connection", UserConnectionStatus.UNAVAILABLE),
("get_session", RekeyLockContentionError),
],
)
def test_a_claim_queued_behind_another_claim_shares_the_bound(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
accessor: str,
expected: object,
) -> None:
# Each claim takes the user's lifecycle lock before the sidecar and
# keeps it while it waits for the sidecar. Bounding only the sidecar
# wait would leave the Nth queued request waiting N bounds, so the
# lifecycle lock counts against the same budget.
monkeypatch.setattr(
encrypted_db_module,
"_SIDECAR_WAIT_TIMEOUT_SECONDS",
_BOUND,
raising=False,
)
manager = _new_manager(tmp_path)
_, engine = _cache_current_engine(manager)
lifecycle_lock = manager._get_rekey_lock("alice")
assert lifecycle_lock.acquire(timeout=5)
try:
worker, outcome = _run_bounded(
lambda: getattr(manager, accessor)("alice")
)
finished = not worker.is_alive()
finally:
lifecycle_lock.release()
worker.join(timeout=_JOIN_SECONDS)
assert finished, f"{accessor} queued on the lifecycle lock past the bound"
if isinstance(expected, type):
assert isinstance(outcome.get("error"), expected), outcome
else:
assert outcome.get("value") is expected, outcome
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
assert (
manager.probe_user_connection("alice") is UserConnectionStatus.CONNECTED
)
@pytest.mark.skipif(os.name == "nt", reason="POSIX flock required")
def test_a_cold_open_gives_up_on_a_sidecar_held_past_the_bound(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
encrypted_db_module,
"_SIDECAR_WAIT_TIMEOUT_SECONDS",
_BOUND,
raising=False,
)
manager = _new_manager(tmp_path)
db_path = manager._get_user_db_path("alice")
db_path.touch()
derived = _watch_key_derivation(monkeypatch, db_path)
with (
patch.object(manager, "_open_user_database_cold") as cold_open,
patch.object(manager, "_run_phase2_rekey"),
):
with _sidecar_flock_held_elsewhere(db_path):
worker, outcome = _run_bounded(
lambda: manager.open_user_database("alice", "password")
)
finished = not worker.is_alive()
worker.join(timeout=_JOIN_SECONDS)
assert finished, "the cold open waited for the holder instead of the bound"
# The typed fault the login route answers with 503 (not counted toward
# lockout); the key was derived first, as every cold open does.
assert isinstance(outcome.get("error"), RekeyLockContentionError), outcome
assert derived == [db_path]
cold_open.assert_not_called()
assert "alice" not in manager.connections
def _spy_sidecar_waits(
monkeypatch: pytest.MonkeyPatch, db_path: Path
) -> list[float]:
# The wait each request-path take of this database's sidecar (claim,
# cold open) is granted: its ``timeout``. A held sidecar makes each such
# wait last its full timeout, so the list shows how long a sequence of
# them waited for the holder without timing anything.
real_acquire = encrypted_db_module.acquire_rekey_lock
granted: list[float] = []
def acquire(path, **kwargs):
if Path(path) == db_path and kwargs.get("timeout") is not None:
granted.append(kwargs["timeout"])
return real_acquire(path, **kwargs)
monkeypatch.setattr(encrypted_db_module, "acquire_rekey_lock", acquire)
return granted
@pytest.mark.skipif(os.name == "nt", reason="POSIX flock required")
def test_a_request_waits_out_a_held_sidecar_once(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# One authenticated request claims the user's sidecar up to four times:
# DatabaseMiddleware's probe and reopen, then require_auth's probe and
# its classifying probe. Each used to wait out the bound on its own, so
# a held sidecar cost the request four bounds (about 20 s) before its
# 503. Inside one request's budget the first wait that runs out spends
# it, and each later wait tries once and gives up at once.
monkeypatch.setattr(
encrypted_db_module,
"_SIDECAR_WAIT_TIMEOUT_SECONDS",
_BOUND,
raising=False,
)
manager = _new_manager(tmp_path)
db_path, engine = _cache_current_engine(manager)
derived = _watch_key_derivation(monkeypatch, db_path)
granted = _spy_sidecar_waits(monkeypatch, db_path)
def one_request() -> list[object]:
outcomes: list[object] = []
with encrypted_db_module.sidecar_wait_budget():
outcomes.append(manager.is_user_connected("alice"))
try:
manager.open_user_database("alice", "password")
except RekeyLockContentionError as error:
outcomes.append(type(error))
outcomes.append(manager.is_user_connected("alice"))
outcomes.append(manager.probe_user_connection("alice"))
return outcomes
for _request in range(2):
granted.clear()
with _sidecar_flock_held_elsewhere(db_path):
worker, outcome = _run_bounded(one_request)
finished = not worker.is_alive()
worker.join(timeout=_JOIN_SECONDS)
assert finished, "a request waited for the holder instead of the bound"
assert outcome.get("value") == [
False,
RekeyLockContentionError,
False,
UserConnectionStatus.UNAVAILABLE,
], outcome
# One bound for the request, and a fresh one for the next request.
assert len(granted) == 4, granted
assert 0 < granted[0] <= _BOUND, granted
assert granted[1:] == [0.0, 0.0, 0.0], granted
# Outside every request (the log drain, for one) each wait gets the
# bound: one request's budget does not reach them.
granted.clear()
with _sidecar_flock_held_elsewhere(db_path):
worker, outcome = _run_bounded(
lambda: [manager.is_user_connected("alice") for _ in range(2)]
)
finished = not worker.is_alive()
worker.join(timeout=_JOIN_SECONDS)
assert finished
assert outcome.get("value") == [False, False], outcome
assert len(granted) == 2 and all(0 < wait for wait in granted), granted
# A retryable fault, not a verdict: nothing derived, nothing evicted.
assert derived == []
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
@pytest.mark.skipif(os.name == "nt", reason="POSIX flock required")
@pytest.mark.parametrize("request_spent", [False, True])
@pytest.mark.parametrize("request_raises", [False, True])
def test_background_context_does_not_keep_a_finished_request_wait_budget(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
request_spent: bool,
request_raises: bool,
) -> None:
# Research and indexing workers copy the request context, then keep
# running after its scope exits. Each later background claim must get
# its own bounded wait, whether the request exhausted its budget or
# left it unused, and whether it returned normally or raised.
monkeypatch.setattr(
encrypted_db_module, "_SIDECAR_WAIT_TIMEOUT_SECONDS", _BOUND
)
manager = _new_manager(tmp_path)
db_path, engine = _cache_current_engine(manager)
granted = _spy_sidecar_waits(monkeypatch, db_path)
with _sidecar_flock_held_elsewhere(db_path):
try:
with encrypted_db_module.sidecar_wait_budget():
background_context = copy_context()
if request_spent:
assert not manager.is_user_connected("alice")
if request_raises:
raise RuntimeError("request failed after spawning work")
except RuntimeError as error:
assert request_raises
assert str(error) == "request failed after spawning work"
granted.clear()
worker, outcome = _run_bounded(
lambda: background_context.run(
lambda: [
manager.probe_user_connection("alice") for _ in range(2)
]
)
)
finished = not worker.is_alive()
worker.join(timeout=_JOIN_SECONDS)
assert finished, "a background claim waited beyond its own bound"
assert outcome.get("value") == [UserConnectionStatus.UNAVAILABLE] * 2, (
outcome
)
assert len(granted) == 2 and all(0 < wait <= _BOUND for wait in granted), (
granted
)
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
@pytest.mark.skipif(os.name == "nt", reason="POSIX flock required")
def test_running_background_context_stops_sharing_when_request_finishes(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
encrypted_db_module, "_SIDECAR_WAIT_TIMEOUT_SECONDS", _BOUND
)
manager = _new_manager(tmp_path)
db_path, engine = _cache_current_engine(manager)
granted = _spy_sidecar_waits(monkeypatch, db_path)
first_claim_finished = threading.Event()
request_finished = threading.Event()
def background() -> list[UserConnectionStatus]:
outcomes = [manager.probe_user_connection("alice")]
first_claim_finished.set()
assert request_finished.wait(timeout=_JOIN_SECONDS)
outcomes.extend(
manager.probe_user_connection("alice") for _ in range(2)
)
return outcomes
with _sidecar_flock_held_elsewhere(db_path):
try:
with encrypted_db_module.sidecar_wait_budget():
background_context = copy_context()
worker, outcome = _run_bounded(
lambda: background_context.run(background), timeout=0
)
assert first_claim_finished.wait(timeout=_JOIN_SECONDS)
# While the request lives, the worker's exhausted wait is
# shared with its caller, just like a run_db_sync worker.
assert not manager.is_user_connected("alice")
finally:
request_finished.set()
worker.join(timeout=_JOIN_SECONDS)
finished = not worker.is_alive()
worker.join(timeout=_JOIN_SECONDS)
assert finished, "the background job did not finish its bounded claims"
assert outcome.get("value") == [UserConnectionStatus.UNAVAILABLE] * 3, (
outcome
)
assert len(granted) == 4, granted
assert 0 < granted[0] <= _BOUND and granted[1] == 0.0, granted
assert all(0 < wait <= _BOUND for wait in granted[2:]), granted
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
@pytest.mark.skipif(os.name == "nt", reason="POSIX flock required")
def test_a_spent_request_budget_returns_once_a_wait_gets_the_sidecar(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# Giving up at once is for a sidecar still held past the bound. Once a
# later wait of the same request gets it (the holder let go), the
# request's next wait may wait out the bound again instead of failing
# on the first brief hold of another probe.
monkeypatch.setattr(
encrypted_db_module,
"_SIDECAR_WAIT_TIMEOUT_SECONDS",
_BOUND,
raising=False,
)
manager = _new_manager(tmp_path)
db_path, engine = _cache_current_engine(manager)
granted = _spy_sidecar_waits(monkeypatch, db_path)
holder_seen = threading.Event()
holder_gone = threading.Event()
def one_request() -> list[UserConnectionStatus]:
with encrypted_db_module.sidecar_wait_budget():
outcomes = [
manager.probe_user_connection("alice"),
manager.probe_user_connection("alice"),
]
holder_seen.set()
assert holder_gone.wait(timeout=_JOIN_SECONDS)
outcomes.append(manager.probe_user_connection("alice"))
outcomes.append(manager.probe_user_connection("alice"))
return outcomes
with _sidecar_flock_held_elsewhere(db_path):
worker, outcome = _run_bounded(one_request, timeout=0)
assert holder_seen.wait(timeout=_JOIN_SECONDS)
holder_gone.set()
worker.join(timeout=_JOIN_SECONDS)
assert not worker.is_alive()
assert outcome.get("value") == [
UserConnectionStatus.UNAVAILABLE,
UserConnectionStatus.UNAVAILABLE,
UserConnectionStatus.CONNECTED,
UserConnectionStatus.CONNECTED,
], outcome
assert len(granted) == 4, granted
assert 0 < granted[0] <= _BOUND, granted
assert granted[1:3] == [0.0, 0.0], granted
assert 0 < granted[3] <= _BOUND, granted
assert manager.connections["alice"] is engine
@pytest.mark.skipif(os.name == "nt", reason="POSIX flock required")
def test_a_cold_open_that_runs_out_spends_the_request_budget(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# With no engine cached the probes never take the sidecar, so the
# request's only wait is the reopen's cold open. When it runs out, that
# is the request's wait: a later open in the same request gives up at
# once rather than waiting out a second bound.
monkeypatch.setattr(
encrypted_db_module,
"_SIDECAR_WAIT_TIMEOUT_SECONDS",
_BOUND,
raising=False,
)
manager = _new_manager(tmp_path)
db_path = manager._get_user_db_path("alice")
db_path.touch()
derived = _watch_key_derivation(monkeypatch, db_path)
granted = _spy_sidecar_waits(monkeypatch, db_path)
def one_request() -> list[object]:
outcomes: list[object] = []
with encrypted_db_module.sidecar_wait_budget():
for _attempt in range(2):
try:
manager.open_user_database("alice", "password")
except RekeyLockContentionError as error:
outcomes.append(type(error))
return outcomes
with (
patch.object(manager, "_open_user_database_cold") as cold_open,
patch.object(manager, "_run_phase2_rekey"),
):
with _sidecar_flock_held_elsewhere(db_path):
worker, outcome = _run_bounded(one_request)
finished = not worker.is_alive()
worker.join(timeout=_JOIN_SECONDS)
assert finished, "the cold opens waited for the holder past one bound"
assert outcome.get("value") == [
RekeyLockContentionError,
RekeyLockContentionError,
], outcome
assert len(granted) == 2, granted
assert 0 < granted[0] <= _BOUND, granted
assert granted[1] == 0.0, granted
# Every cold open derives the key before it takes the sidecar.
assert derived == [db_path, db_path]
cold_open.assert_not_called()
assert "alice" not in manager.connections
@pytest.mark.skipif(os.name == "nt", reason="POSIX flock required")
def test_a_sidecar_held_past_the_bound_costs_the_log_drain_one_bound(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# The single log-drain thread claims the user's engine for each entry
# it persists. With the claim bounded, an entry for a user whose
# sidecar is held costs it at most the bound, not the holder's time,
# and is dropped (the file and stderr sinks keep it).
monkeypatch.setattr(
encrypted_db_module,
"_SIDECAR_WAIT_TIMEOUT_SECONDS",
_BOUND,
raising=False,
)
manager = _new_manager(tmp_path)
db_path, engine = _cache_current_engine(manager)
monkeypatch.setattr(encrypted_db_module, "db_manager", manager)
entry = {
"username": "alice",
"timestamp": None,
"message": "queued while the sidecar is held",
"module": "test",
"function": "test",
"line_no": 1,
"level": "INFO",
"research_id": None,
}
with _sidecar_flock_held_elsewhere(db_path):
worker, _ = _run_bounded(
lambda: log_utils._write_log_to_database(entry)
)
finished = not worker.is_alive()
worker.join(timeout=_JOIN_SECONDS)
assert finished, "the log drain waited for one user's sidecar holder"
engine.connect.assert_not_called()
assert manager.connections["alice"] is engine
@pytest.mark.skipif(os.name == "nt", reason="POSIX flock required")
def test_registration_gives_up_on_a_sidecar_held_past_the_bound(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
pytest.importorskip("sqlcipher3")
monkeypatch.setattr(
encrypted_db_module,
"_SIDECAR_WAIT_TIMEOUT_SECONDS",
_BOUND,
raising=False,
)
manager = _new_manager(tmp_path)
db_path = manager._get_user_db_path("bob")
with _sidecar_flock_held_elsewhere(db_path):
worker, outcome = _run_bounded(
lambda: manager.create_user_database("bob", "BobPassword1!"),
timeout=30,
)
finished = not worker.is_alive()
worker.join(timeout=30)
assert finished, "registration waited for the holder instead of the bound"
# Failed clean: the partial database is removed so the username can
# register again, and the stable lock inode stays.
assert isinstance(outcome.get("error"), RekeyLockContentionError), outcome
assert not db_path.exists()
assert not get_salt_file_path(db_path).exists()
assert not db_path.with_name(f"{db_path.name}.rekey.epoch").exists()
assert db_path.with_name(f"{db_path.name}.rekey.lock").exists()
assert "bob" not in manager.connections
def _release_fifo_reader(fifo: Path) -> None:
# A reader blocked opening the FIFO is released by a writer that opens
# and closes it; with no reader waiting the open fails (ENXIO) and
# there is nothing to release.
try:
fd = os.open(fifo, os.O_WRONLY | os.O_NONBLOCK)
except OSError:
return
os.close(fd)
@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="POSIX FIFOs required")
@pytest.mark.parametrize("suffix", [".rekey.epoch", ".rekey.pending"])
def test_a_planted_fifo_cannot_hang_the_probe_or_the_shutdown(
tmp_path: Path, suffix: str
) -> None:
# Every probe reads the epoch files while it holds the user's sidecar
# flock. A FIFO planted at one of their names made that read wait for a
# writer, so the probe, every later probe for the user and shutdown's
# close_all_databases hung behind it.
manager = _new_manager(tmp_path)
db_path, engine = _cache_current_engine(manager)
planted = db_path.with_name(f"{db_path.name}{suffix}")
planted.unlink(missing_ok=True)
os.mkfifo(planted, 0o600)
worker, outcome = _run_bounded(
lambda: manager.probe_user_connection("alice")
)
finished = not worker.is_alive()
_release_fifo_reader(planted)
worker.join(timeout=_JOIN_SECONDS)
assert finished, "the probe waited on a planted FIFO under the flock"
# Nothing was read: a retryable fault, and the engine is kept
assert outcome.get("value") is UserConnectionStatus.UNAVAILABLE, outcome
assert manager.connections["alice"] is engine
engine.dispose.assert_not_called()
assert stat.S_ISFIFO(planted.lstat().st_mode)
# The sidecar was released: the next probe answers too, and shutdown
# disposes the engine instead of waiting on the FIFO.
worker, outcome = _run_bounded(
lambda: manager.probe_user_connection("alice")
)
assert not worker.is_alive()
assert outcome.get("value") is UserConnectionStatus.UNAVAILABLE, outcome
worker, outcome = _run_bounded(manager.close_all_databases)
assert not worker.is_alive()
assert "error" not in outcome, outcome
assert "alice" not in manager.connections
engine.dispose.assert_called_once_with()
@pytest.mark.parametrize(
("error", "expected"),
[
(
rekey_lock_module.RekeySidecarIOError(
Path("/secret/user.db"),
rekey_lock_module.RekeySidecarOperation.ACQUIRE,
errno.EACCES,
),
f"RekeySidecarIOError (operation=acquire, errno={errno.EACCES})",
),
(
rekey_lock_module.RekeySidecarIOError(
Path("/secret/user.db"),
rekey_lock_module.RekeySidecarOperation.OPEN,
None,
),
"RekeySidecarIOError (operation=open, errno=unknown)",
),
(
rekey_lock_module.RekeyEpochUnreadableError(
Path("/secret/user.db"), errno.EIO
),
f"RekeyEpochUnreadableError (errno={errno.EIO})",
),
(
rekey_lock_module.RekeyRecoveryRequiredError(
Path("/secret/user.db")
),
"RekeyRecoveryRequiredError",
),
],
)
def test_describe_rekey_fault_is_scrub_safe(
error: rekey_lock_module.RekeyOperationalError, expected: str
) -> None:
described = rekey_lock_module.describe_rekey_fault(error)
assert described == expected
assert "secret" not in described
assert "user.db" not in described