~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Durable-epoch fencing of server-side auth sessions.
A session is created at login, right after the password proved itself
against the epoch-fenced SQLCipher database (PR #6662). The session
record binds the durable authorization epoch that was live at that
moment. Every validation re-projects the epoch from disk:
* a session whose bound epoch no longer matches (the password was
changed — rekeyed — in ANY worker) is destroyed: its credentials can
no longer open the rekeyed database, so the identity it vouches for
is dead;
* a durable state that was read but cannot vouch for the identity
(recovery required, a rekey pending) fences the session the same way —
fail closed, the record is removed;
* an epoch file that cannot be READ (EMFILE, EIO, ...) fails the check
closed with ``SessionValidationUnavailableError`` but KEEPS the record:
a transient fault must not log every active user out;
* a login whose password was proved against an epoch that is no longer
committed by the time the session is created is refused
(``StaleLoginProofError``): the proof, not the later projection, is
what the session may be bound to.
``require_auth`` (HTTP), ``_enforce_session_revocation``, the socket
connect gate and the subscribe re-validation all funnel through
``validate_session``/``validate_session_identity``, so binding the epoch
here fences each of them. ``has_active_sessions_for`` applies the same
epoch check (read-only); the other idle sweeps are time-based only.
"""
import errno
import os
from pathlib import Path
import pytest
from local_deep_research.database.rekey_epoch import (
read_user_authorization_epoch,
)
from local_deep_research.database.rekey_lock import (
RekeyRecoveryRequiredError,
acquire_rekey_lock,
advance_rekey_epoch,
read_or_initialize_rekey_epoch,
)
from local_deep_research.web.auth import session_manager as sm_module
from local_deep_research.web.auth.session_manager import (
SessionManager,
SessionValidationUnavailableError,
StaleLoginProofError,
)
@pytest.mark.parametrize(
"error_number", [None, errno.EMFILE, errno.ENFILE, errno.EAGAIN, errno.EIO]
)
def test_unavailable_exception_keeps_identity_out_of_rendered_diagnostics(
error_number,
):
identity = "SYNTHETIC_ACCOUNT_MARKER"
failure = SessionValidationUnavailableError(identity, error_number)
status = "unknown" if error_number is None else str(error_number)
assert failure.username == identity
assert failure.errno == error_number
assert failure.args == (
f"session epoch temporarily unreadable (errno={status})",
)
assert identity not in str(failure)
assert identity not in repr(failure)
# Opt out of the autouse _durable_epoch_projection shim: these tests
# exercise the REAL epoch files and failure discipline.
pytestmark = pytest.mark.real_durable_epoch
@pytest.fixture
def data_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
"""An isolated data directory whose user epochs are real files.
``db_manager.data_dir`` is pinned to the SAME directory: another
module in this xdist worker may have pinned the singleton's override
to its own tmp dir, and the epoch projection resolves through the
manager -- env and manager must agree, or the projection reads the
wrong directory and every assertion below fences against nothing.
"""
from local_deep_research.database.encrypted_db import db_manager
monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
# Pin through the override attribute so monkeypatch restores the
# ORIGINAL override (often None = "resolve from env"), instead of
# re-pinning whatever path the getter happened to resolve.
(tmp_path / "encrypted_databases").mkdir(parents=True, exist_ok=True)
monkeypatch.setattr(
db_manager, "_data_dir_override", tmp_path / "encrypted_databases"
)
return tmp_path
def _user_db_path(data_dir: Path, username: str) -> Path:
from local_deep_research.config.paths import get_user_database_filename
return (
data_dir / "encrypted_databases" / get_user_database_filename(username)
)
def _initialize_epoch(data_dir: Path, username: str) -> str:
db_path = _user_db_path(data_dir, username)
db_path.parent.mkdir(parents=True, exist_ok=True)
with acquire_rekey_lock(db_path):
return read_or_initialize_rekey_epoch(db_path)
def _manager() -> SessionManager:
manager = SessionManager()
manager.sessions.clear()
return manager
def test_session_binds_durable_epoch_at_creation(
data_dir: Path,
) -> None:
epoch = _initialize_epoch(data_dir, "alice")
manager = _manager()
session_id = manager.create_session("alice")
record = manager.sessions[session_id]
assert record["rekey_epoch"] == epoch
def test_stale_epoch_removes_exact_local_session(data_dir: Path) -> None:
_initialize_epoch(data_dir, "alice")
manager = _manager()
stale_session = manager.create_session("alice")
with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
advance_rekey_epoch(_user_db_path(data_dir, "alice"))
fresh_session = manager.create_session("alice")
# The stale-epoch session is destroyed, not merely denied...
assert manager.validate_session(stale_session) is None
assert stale_session not in manager.sessions
# ...while the session bound to the current epoch survives.
assert manager.validate_session(fresh_session) == "alice"
def test_validate_session_identity_returns_the_bound_epoch(
data_dir: Path,
) -> None:
epoch = _initialize_epoch(data_dir, "alice")
manager = _manager()
session_id = manager.create_session("alice")
identity = manager.validate_session_identity(session_id)
assert identity is not None
assert identity.username == "alice"
assert identity.rekey_epoch == epoch
def _fail_epoch_reads(
monkeypatch: pytest.MonkeyPatch, error_number: int
) -> None:
"""Inject a real OSError at the epoch-file read itself."""
real_open = os.open
def failing_open(path, *args, **kwargs):
if Path(path).name.endswith((".rekey.epoch", ".rekey.pending")):
raise OSError(error_number, "injected epoch read failure")
return real_open(path, *args, **kwargs)
monkeypatch.setattr(os, "open", failing_open)
@pytest.mark.parametrize(
"error_number",
[errno.EMFILE, errno.ENFILE, errno.EAGAIN, errno.EIO],
)
def test_unreadable_epoch_file_refuses_but_keeps_the_session(
data_dir: Path, monkeypatch: pytest.MonkeyPatch, error_number: int
) -> None:
# A transient I/O error proves nothing about the session: the check
# fails closed (raises, never returns the username) but the record
# survives, so an fd-exhaustion spike does not log everyone out.
epoch = _initialize_epoch(data_dir, "alice")
manager = _manager()
session_id = manager.create_session("alice")
before = dict(manager.sessions[session_id])
with monkeypatch.context() as patched:
_fail_epoch_reads(patched, error_number)
with pytest.raises(SessionValidationUnavailableError) as raised:
manager.validate_session(session_id)
with pytest.raises(SessionValidationUnavailableError):
manager.validate_session_identity(session_id)
assert raised.value.errno == error_number
# Kept, untouched (not even refreshed).
assert manager.sessions[session_id] == before
assert manager.sessions[session_id]["rekey_epoch"] == epoch
# Once the fault clears, the same session validates again.
assert manager.validate_session(session_id) == "alice"
def test_epoch_advanced_during_a_read_fault_still_removes_after_it(
data_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# Keeping the record through a fault is safe only because the next
# successful read re-checks it: a password change committed during
# the fault still ends the session.
_initialize_epoch(data_dir, "alice")
manager = _manager()
session_id = manager.create_session("alice")
with monkeypatch.context() as patched:
_fail_epoch_reads(patched, errno.EMFILE)
with pytest.raises(SessionValidationUnavailableError):
manager.validate_session(session_id)
assert session_id in manager.sessions
with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
advance_rekey_epoch(_user_db_path(data_dir, "alice"))
assert manager.validate_session(session_id) is None
assert session_id not in manager.sessions
def test_malformed_epoch_file_fences_and_removes(data_dir: Path) -> None:
# Read successfully, but the content cannot vouch for anything: that is
# durable state, not a transient fault, so the record goes.
_initialize_epoch(data_dir, "alice")
manager = _manager()
session_id = manager.create_session("alice")
db_path = _user_db_path(data_dir, "alice")
epoch_path = db_path.with_name(f"{db_path.name}.rekey.epoch")
epoch_path.write_text("NOT-A-CANONICAL-UUID", encoding="ascii")
assert manager.validate_session(session_id) is None
assert session_id not in manager.sessions
def test_recovery_required_state_fences_and_removes(
data_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
_initialize_epoch(data_dir, "alice")
manager = _manager()
session_id = manager.create_session("alice")
def ambiguous_state(_username: str) -> str:
raise RekeyRecoveryRequiredError(_user_db_path(data_dir, "alice"))
monkeypatch.setattr(
sm_module,
"read_user_authorization_epoch",
ambiguous_state,
)
# Ambiguous durable state cannot vouch for ANY identity: fail closed
# and remove the record.
assert manager.validate_session(session_id) is None
assert session_id not in manager.sessions
def test_current_epoch_projection_matches_durable_file(
data_dir: Path,
) -> None:
_initialize_epoch(data_dir, "alice")
manager = _manager()
session_id = manager.create_session("alice")
with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
advance_rekey_epoch(_user_db_path(data_dir, "alice"))
assert (
read_user_authorization_epoch("alice")
!= manager.sessions[session_id]["rekey_epoch"]
)
assert manager.validate_session(session_id) is None
def test_create_session_refuses_a_proof_from_a_superseded_epoch(
data_dir: Path,
) -> None:
# The login proved its password against E1; a rekey committed E2
# (another worker's password change) before the session was created.
proven = _initialize_epoch(data_dir, "alice")
with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
advance_rekey_epoch(_user_db_path(data_dir, "alice"))
manager = _manager()
with pytest.raises(StaleLoginProofError):
manager.create_session("alice", proven_epoch=proven)
assert manager.sessions == {}
def test_create_session_binds_a_current_proof(data_dir: Path) -> None:
proven = _initialize_epoch(data_dir, "alice")
manager = _manager()
session_id = manager.create_session("alice", proven_epoch=proven)
assert manager.sessions[session_id]["rekey_epoch"] == proven
assert manager.validate_session(session_id) == "alice"
def test_projection_resolves_through_the_managers_pinned_data_dir(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# The env data dir and the manager's pinned override diverge; the
# epoch lives only beside the manager's databases, where every writer
# puts it. An env-only resolution reads the other directory and fails
# closed on a perfectly healthy identity.
from local_deep_research.config.paths import get_user_database_filename
from local_deep_research.database.encrypted_db import db_manager
env_dir = tmp_path / "env"
pinned_dir = tmp_path / "pinned" / "encrypted_databases"
(env_dir / "encrypted_databases").mkdir(parents=True)
pinned_dir.mkdir(parents=True)
monkeypatch.setenv("LDR_DATA_DIR", str(env_dir))
monkeypatch.setattr(db_manager, "_data_dir_override", pinned_dir)
db_path = pinned_dir / get_user_database_filename("alice")
with acquire_rekey_lock(db_path):
epoch = read_or_initialize_rekey_epoch(db_path)
assert read_user_authorization_epoch("alice") == epoch
def test_has_active_sessions_for_ignores_a_stale_epoch_session(
data_dir: Path,
) -> None:
# A password change in another worker leaves this worker's record in
# place until its next validation; it must not count as active.
_initialize_epoch(data_dir, "alice")
manager = _manager()
stale_session = manager.create_session("alice")
assert manager.has_active_sessions_for("alice") is True
with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
advance_rekey_epoch(_user_db_path(data_dir, "alice"))
assert manager.has_active_sessions_for("alice") is False
# Read-only: the stale record is left for validation to remove.
assert stale_session in manager.sessions
# A session bound to the new epoch does count.
manager.create_session("alice")
assert manager.has_active_sessions_for("alice") is True
def test_has_active_sessions_for_is_unavailable_when_the_epoch_is_unreadable(
data_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# An unavailable check refuses authorization but does not prove that
# shared resources belong to no remaining session.
_initialize_epoch(data_dir, "alice")
manager = _manager()
session_id = manager.create_session("alice")
with monkeypatch.context() as patched:
_fail_epoch_reads(patched, errno.EMFILE)
with pytest.raises(sm_module.SessionValidationUnavailableError):
manager.has_active_sessions_for("alice")
assert session_id in manager.sessions
assert manager.has_active_sessions_for("alice") is True
def test_has_active_sessions_for_is_false_on_recovery_required_state(
data_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
_initialize_epoch(data_dir, "alice")
manager = _manager()
manager.create_session("alice")
def ambiguous_state(_username: str) -> str:
raise RekeyRecoveryRequiredError(_user_db_path(data_dir, "alice"))
monkeypatch.setattr(
sm_module, "read_user_authorization_epoch", ambiguous_state
)
assert manager.has_active_sessions_for("alice") is False
# --- socket teardown when validation removes the record --------------------
#
# A socket is authorised once, at handshake, against a session id recorded
# in ``socketio_asgi._sid_sessions``; ``emit_to_user`` fans out to every sid
# in ``_sid_users`` without re-checking the session. When validation
# REMOVES a record (epoch mismatch after a password change in another
# worker, unvouchable durable state, inline idle expiry) the record is gone
# from every later sweep, so validation itself must sever the sockets that
# handshook on it -- otherwise a socket opened with a stolen cookie keeps
# receiving the user's events (``settings_changed`` carries plaintext
# secrets) after the password change that was meant to fence it.
_SOCKET_MODULE = "local_deep_research.web.services.socketio_asgi"
@pytest.fixture
def severed(monkeypatch: pytest.MonkeyPatch) -> list[str]:
"""Record every session id handed to ``disconnect_session``."""
calls: list[str] = []
monkeypatch.setattr(
f"{_SOCKET_MODULE}.disconnect_session",
lambda session_id: calls.append(session_id),
)
return calls
def test_epoch_mismatch_severs_that_sessions_sockets(
data_dir: Path, severed: list[str]
) -> None:
_initialize_epoch(data_dir, "alice")
manager = _manager()
stale_session = manager.create_session("alice")
with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
advance_rekey_epoch(_user_db_path(data_dir, "alice"))
fresh_session = manager.create_session("alice")
assert manager.validate_session(stale_session) is None
# Only the removed session's sockets go; the new-epoch session (another
# tab that logged in after the change) keeps its own.
assert severed == [stale_session]
assert manager.validate_session(fresh_session) == "alice"
assert severed == [stale_session]
def test_unvouchable_durable_state_severs_that_sessions_sockets(
data_dir: Path, monkeypatch: pytest.MonkeyPatch, severed: list[str]
) -> None:
_initialize_epoch(data_dir, "alice")
manager = _manager()
session_id = manager.create_session("alice")
def ambiguous_state(_username: str) -> str:
raise RekeyRecoveryRequiredError(_user_db_path(data_dir, "alice"))
monkeypatch.setattr(
sm_module, "read_user_authorization_epoch", ambiguous_state
)
assert manager.validate_session(session_id) is None
assert session_id not in manager.sessions
assert severed == [session_id]
def test_inline_idle_expiry_severs_that_sessions_sockets(
data_dir: Path, severed: list[str]
) -> None:
import datetime
_initialize_epoch(data_dir, "alice")
manager = _manager()
session_id = manager.create_session("alice", remember_me=False)
manager.sessions[session_id]["last_access"] -= (
manager.session_timeout + datetime.timedelta(minutes=1)
)
assert manager.validate_session(session_id) is None
assert session_id not in manager.sessions
assert severed == [session_id]
def test_kept_or_unknown_records_sever_nothing(
data_dir: Path, monkeypatch: pytest.MonkeyPatch, severed: list[str]
) -> None:
# Negative controls: a valid session, a transient read fault (record
# KEPT, 503) and an unknown id must not tear down any socket.
_initialize_epoch(data_dir, "alice")
manager = _manager()
session_id = manager.create_session("alice")
assert manager.validate_session(session_id) == "alice"
with monkeypatch.context() as patched:
_fail_epoch_reads(patched, errno.EMFILE)
with pytest.raises(SessionValidationUnavailableError):
manager.validate_session(session_id)
assert manager.validate_session("no-such-session") is None
assert severed == []
assert session_id in manager.sessions
class _CookieRequest:
"""``end_rejected_session`` only touches ``request.session``."""
def __init__(self, session: dict) -> None:
self.session = dict(session)
def test_rejected_session_teardown_survives_an_epoch_read_fault(
data_dir: Path, monkeypatch: pytest.MonkeyPatch, severed: list[str]
) -> None:
# The cookie is cleared whatever happens, so the record and the sockets
# behind it must go too, even while the epoch files cannot be read
# (#7830): otherwise a replayed copy of the cleared cookie keeps a live
# record that nothing can reach to destroy.
from local_deep_research.web.dependencies import auth as auth_dep
_initialize_epoch(data_dir, "alice")
_initialize_epoch(data_dir, "bob")
manager = _manager()
session_id = manager.create_session("alice")
other_tab = manager.create_session("alice")
bobs_session = manager.create_session("bob")
monkeypatch.setattr(auth_dep, "session_manager", manager)
request = _CookieRequest({"username": "alice", "session_id": session_id})
foreign = _CookieRequest({"username": "alice", "session_id": bobs_session})
with monkeypatch.context() as patched:
_fail_epoch_reads(patched, errno.EMFILE)
# The fault is real: validation cannot answer for this record.
with pytest.raises(SessionValidationUnavailableError):
manager.validate_session(session_id)
auth_dep.end_rejected_session(request)
auth_dep.end_rejected_session(foreign)
assert request.session == {}
assert session_id not in manager.sessions
assert severed == [session_id]
# Scoped to that one record: the user's other tab and another user's
# session named by a mismatched cookie are left alone.
assert other_tab in manager.sessions
assert bobs_session in manager.sessions
def test_socket_teardown_failure_never_breaks_validation(
data_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# Best-effort: a socket layer that raises must not turn the removal
# into an exception at the HTTP middleware.
def broken(_session_id: str) -> None:
raise RuntimeError("socket layer down")
monkeypatch.setattr(f"{_SOCKET_MODULE}.disconnect_session", broken)
_initialize_epoch(data_dir, "alice")
manager = _manager()
session_id = manager.create_session("alice")
with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
advance_rekey_epoch(_user_db_path(data_dir, "alice"))
assert manager.validate_session(session_id) is None
assert session_id not in manager.sessions
def test_epoch_mismatch_clears_the_sid_maps_on_a_live_loop(
data_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""End to end through the real socket layer.
``disconnect_session`` -> ``_disconnect_matching`` runs on a live event
loop; the fake ``sio.disconnect`` fires the real ``disconnect`` event
handler, as python-socketio does, so the sid maps are cleaned by the
same code path a logout uses. The other session's socket stays.
"""
import asyncio
import threading
from unittest.mock import Mock
from local_deep_research.web.services import socketio_asgi as sio_mod
_initialize_epoch(data_dir, "alice")
manager = _manager()
stale_session = manager.create_session("alice")
with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
advance_rekey_epoch(_user_db_path(data_dir, "alice"))
fresh_session = manager.create_session("alice")
monkeypatch.setattr(sio_mod, "_lock", asyncio.Lock())
monkeypatch.setattr(sio_mod, "_subscriptions", {})
monkeypatch.setattr(
sio_mod, "_sid_users", {"sid-stolen": "alice", "sid-fresh": "alice"}
)
monkeypatch.setattr(
sio_mod,
"_sid_sessions",
{"sid-stolen": stale_session, "sid-fresh": fresh_session},
)
loop = asyncio.new_event_loop()
thread = threading.Thread(target=loop.run_forever, daemon=True)
thread.start()
disconnected: list[str] = []
async def fake_disconnect(sid: str) -> None:
disconnected.append(sid)
await sio_mod.disconnect(sid)
fake_sio = Mock()
fake_sio.disconnect = fake_disconnect
futures = []
real_schedule = asyncio.run_coroutine_threadsafe
def capture(coro, target_loop):
future = real_schedule(coro, target_loop)
futures.append(future)
return future
try:
with monkeypatch.context() as patched:
patched.setattr(sio_mod, "_get_main_loop", lambda: loop)
patched.setattr(sio_mod, "sio", fake_sio)
patched.setattr(asyncio, "run_coroutine_threadsafe", capture)
assert manager.validate_session(stale_session) is None
assert futures, "validation scheduled no socket teardown"
for future in futures:
future.result(timeout=5)
finally:
loop.call_soon_threadsafe(loop.stop)
thread.join(timeout=5)
loop.close()
assert disconnected == ["sid-stolen"]
assert sio_mod._sid_users == {"sid-fresh": "alice"}
assert sio_mod._sid_sessions == {"sid-fresh": fresh_session}
def test_epoch_read_sharing_violation_is_retried_then_fails_closed(
data_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""A Windows sharing violation on the lock-free epoch read.
A writer's ``os.replace``/``unlink`` in flight makes the reader's open
fail with PermissionError for a few microseconds. A transient one is
retried and the session validates; a persistent one fails closed
(503 for this request, the record KEPT, never a validated identity).
"""
from local_deep_research.database import rekey_lock as rekey_lock_module
epoch = _initialize_epoch(data_dir, "alice")
manager = _manager()
session_id = manager.create_session("alice")
monkeypatch.setattr(
rekey_lock_module, "_SHARING_VIOLATION_DELAY", 0, raising=False
)
real_open = os.open
budget = {"failures": 2}
def sharing_violation(path, *args, **kwargs):
if (
Path(path).name.endswith((".rekey.epoch", ".rekey.pending"))
and budget["failures"] > 0
):
budget["failures"] -= 1
raise PermissionError(errno.EACCES, "sharing violation")
return real_open(path, *args, **kwargs)
monkeypatch.setattr(os, "open", sharing_violation)
identity = manager.validate_session_identity(session_id)
assert identity is not None and identity.rekey_epoch == epoch
assert budget["failures"] == 0
budget["failures"] = 10**6
before = dict(manager.sessions[session_id])
with pytest.raises(SessionValidationUnavailableError) as raised:
manager.validate_session_identity(session_id)
assert raised.value.errno == errno.EACCES
assert manager.sessions[session_id] == before