Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

24.5 KB
Raw Blame
"""Durable-epoch fencing of server-side auth sessions.

A session is created at login, right after the password proved itself
against the epoch-fenced SQLCipher database (PR #6662). The session
record binds the durable authorization epoch that was live at that
moment. Every validation re-projects the epoch from disk:

* a session whose bound epoch no longer matches (the password was
  changed — rekeyed — in ANY worker) is destroyed: its credentials can
  no longer open the rekeyed database, so the identity it vouches for
  is dead;
* a durable state that was read but cannot vouch for the identity
  (recovery required, a rekey pending) fences the session the same way —
  fail closed, the record is removed;
* an epoch file that cannot be READ (EMFILE, EIO, ...) fails the check
  closed with ``SessionValidationUnavailableError`` but KEEPS the record:
  a transient fault must not log every active user out;
* a login whose password was proved against an epoch that is no longer
  committed by the time the session is created is refused
  (``StaleLoginProofError``): the proof, not the later projection, is
  what the session may be bound to.

``require_auth`` (HTTP), ``_enforce_session_revocation``, the socket
connect gate and the subscribe re-validation all funnel through
``validate_session``/``validate_session_identity``, so binding the epoch
here fences each of them. ``has_active_sessions_for`` applies the same
epoch check (read-only); the other idle sweeps are time-based only.
"""

import errno
import os
from pathlib import Path

import pytest

from local_deep_research.database.rekey_epoch import (
    read_user_authorization_epoch,
)
from local_deep_research.database.rekey_lock import (
    RekeyRecoveryRequiredError,
    acquire_rekey_lock,
    advance_rekey_epoch,
    read_or_initialize_rekey_epoch,
)
from local_deep_research.web.auth import session_manager as sm_module
from local_deep_research.web.auth.session_manager import (
    SessionManager,
    SessionValidationUnavailableError,
    StaleLoginProofError,
)


@pytest.mark.parametrize(
    "error_number", [None, errno.EMFILE, errno.ENFILE, errno.EAGAIN, errno.EIO]
)
def test_unavailable_exception_keeps_identity_out_of_rendered_diagnostics(
    error_number,
):
    identity = "SYNTHETIC_ACCOUNT_MARKER"
    failure = SessionValidationUnavailableError(identity, error_number)
    status = "unknown" if error_number is None else str(error_number)
    assert failure.username == identity
    assert failure.errno == error_number
    assert failure.args == (
        f"session epoch temporarily unreadable (errno={status})",
    )
    assert identity not in str(failure)
    assert identity not in repr(failure)


# Opt out of the autouse _durable_epoch_projection shim: these tests
# exercise the REAL epoch files and failure discipline.
pytestmark = pytest.mark.real_durable_epoch


@pytest.fixture
def data_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
    """An isolated data directory whose user epochs are real files.

    ``db_manager.data_dir`` is pinned to the SAME directory: another
    module in this xdist worker may have pinned the singleton's override
    to its own tmp dir, and the epoch projection resolves through the
    manager -- env and manager must agree, or the projection reads the
    wrong directory and every assertion below fences against nothing.
    """
    from local_deep_research.database.encrypted_db import db_manager

    monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
    # Pin through the override attribute so monkeypatch restores the
    # ORIGINAL override (often None = "resolve from env"), instead of
    # re-pinning whatever path the getter happened to resolve.
    (tmp_path / "encrypted_databases").mkdir(parents=True, exist_ok=True)
    monkeypatch.setattr(
        db_manager, "_data_dir_override", tmp_path / "encrypted_databases"
    )
    return tmp_path


def _user_db_path(data_dir: Path, username: str) -> Path:
    from local_deep_research.config.paths import get_user_database_filename

    return (
        data_dir / "encrypted_databases" / get_user_database_filename(username)
    )


def _initialize_epoch(data_dir: Path, username: str) -> str:
    db_path = _user_db_path(data_dir, username)
    db_path.parent.mkdir(parents=True, exist_ok=True)
    with acquire_rekey_lock(db_path):
        return read_or_initialize_rekey_epoch(db_path)


def _manager() -> SessionManager:
    manager = SessionManager()
    manager.sessions.clear()
    return manager


def test_session_binds_durable_epoch_at_creation(
    data_dir: Path,
) -> None:
    epoch = _initialize_epoch(data_dir, "alice")
    manager = _manager()

    session_id = manager.create_session("alice")

    record = manager.sessions[session_id]
    assert record["rekey_epoch"] == epoch


def test_stale_epoch_removes_exact_local_session(data_dir: Path) -> None:
    _initialize_epoch(data_dir, "alice")
    manager = _manager()
    stale_session = manager.create_session("alice")

    with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
        advance_rekey_epoch(_user_db_path(data_dir, "alice"))
    fresh_session = manager.create_session("alice")

    # The stale-epoch session is destroyed, not merely denied...
    assert manager.validate_session(stale_session) is None
    assert stale_session not in manager.sessions
    # ...while the session bound to the current epoch survives.
    assert manager.validate_session(fresh_session) == "alice"


def test_validate_session_identity_returns_the_bound_epoch(
    data_dir: Path,
) -> None:
    epoch = _initialize_epoch(data_dir, "alice")
    manager = _manager()
    session_id = manager.create_session("alice")

    identity = manager.validate_session_identity(session_id)

    assert identity is not None
    assert identity.username == "alice"
    assert identity.rekey_epoch == epoch


def _fail_epoch_reads(
    monkeypatch: pytest.MonkeyPatch, error_number: int
) -> None:
    """Inject a real OSError at the epoch-file read itself."""
    real_open = os.open

    def failing_open(path, *args, **kwargs):
        if Path(path).name.endswith((".rekey.epoch", ".rekey.pending")):
            raise OSError(error_number, "injected epoch read failure")
        return real_open(path, *args, **kwargs)

    monkeypatch.setattr(os, "open", failing_open)


@pytest.mark.parametrize(
    "error_number",
    [errno.EMFILE, errno.ENFILE, errno.EAGAIN, errno.EIO],
)
def test_unreadable_epoch_file_refuses_but_keeps_the_session(
    data_dir: Path, monkeypatch: pytest.MonkeyPatch, error_number: int
) -> None:
    # A transient I/O error proves nothing about the session: the check
    # fails closed (raises, never returns the username) but the record
    # survives, so an fd-exhaustion spike does not log everyone out.
    epoch = _initialize_epoch(data_dir, "alice")
    manager = _manager()
    session_id = manager.create_session("alice")
    before = dict(manager.sessions[session_id])

    with monkeypatch.context() as patched:
        _fail_epoch_reads(patched, error_number)
        with pytest.raises(SessionValidationUnavailableError) as raised:
            manager.validate_session(session_id)
        with pytest.raises(SessionValidationUnavailableError):
            manager.validate_session_identity(session_id)

    assert raised.value.errno == error_number
    # Kept, untouched (not even refreshed).
    assert manager.sessions[session_id] == before
    assert manager.sessions[session_id]["rekey_epoch"] == epoch
    # Once the fault clears, the same session validates again.
    assert manager.validate_session(session_id) == "alice"


def test_epoch_advanced_during_a_read_fault_still_removes_after_it(
    data_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    # Keeping the record through a fault is safe only because the next
    # successful read re-checks it: a password change committed during
    # the fault still ends the session.
    _initialize_epoch(data_dir, "alice")
    manager = _manager()
    session_id = manager.create_session("alice")

    with monkeypatch.context() as patched:
        _fail_epoch_reads(patched, errno.EMFILE)
        with pytest.raises(SessionValidationUnavailableError):
            manager.validate_session(session_id)
    assert session_id in manager.sessions

    with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
        advance_rekey_epoch(_user_db_path(data_dir, "alice"))

    assert manager.validate_session(session_id) is None
    assert session_id not in manager.sessions


def test_malformed_epoch_file_fences_and_removes(data_dir: Path) -> None:
    # Read successfully, but the content cannot vouch for anything: that is
    # durable state, not a transient fault, so the record goes.
    _initialize_epoch(data_dir, "alice")
    manager = _manager()
    session_id = manager.create_session("alice")
    db_path = _user_db_path(data_dir, "alice")
    epoch_path = db_path.with_name(f"{db_path.name}.rekey.epoch")
    epoch_path.write_text("NOT-A-CANONICAL-UUID", encoding="ascii")

    assert manager.validate_session(session_id) is None
    assert session_id not in manager.sessions


def test_recovery_required_state_fences_and_removes(
    data_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    _initialize_epoch(data_dir, "alice")
    manager = _manager()
    session_id = manager.create_session("alice")

    def ambiguous_state(_username: str) -> str:
        raise RekeyRecoveryRequiredError(_user_db_path(data_dir, "alice"))

    monkeypatch.setattr(
        sm_module,
        "read_user_authorization_epoch",
        ambiguous_state,
    )

    # Ambiguous durable state cannot vouch for ANY identity: fail closed
    # and remove the record.
    assert manager.validate_session(session_id) is None
    assert session_id not in manager.sessions


def test_current_epoch_projection_matches_durable_file(
    data_dir: Path,
) -> None:
    _initialize_epoch(data_dir, "alice")
    manager = _manager()
    session_id = manager.create_session("alice")

    with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
        advance_rekey_epoch(_user_db_path(data_dir, "alice"))

    assert (
        read_user_authorization_epoch("alice")
        != manager.sessions[session_id]["rekey_epoch"]
    )
    assert manager.validate_session(session_id) is None


def test_create_session_refuses_a_proof_from_a_superseded_epoch(
    data_dir: Path,
) -> None:
    # The login proved its password against E1; a rekey committed E2
    # (another worker's password change) before the session was created.
    proven = _initialize_epoch(data_dir, "alice")
    with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
        advance_rekey_epoch(_user_db_path(data_dir, "alice"))
    manager = _manager()

    with pytest.raises(StaleLoginProofError):
        manager.create_session("alice", proven_epoch=proven)

    assert manager.sessions == {}


def test_create_session_binds_a_current_proof(data_dir: Path) -> None:
    proven = _initialize_epoch(data_dir, "alice")
    manager = _manager()

    session_id = manager.create_session("alice", proven_epoch=proven)

    assert manager.sessions[session_id]["rekey_epoch"] == proven
    assert manager.validate_session(session_id) == "alice"


def test_projection_resolves_through_the_managers_pinned_data_dir(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    # The env data dir and the manager's pinned override diverge; the
    # epoch lives only beside the manager's databases, where every writer
    # puts it. An env-only resolution reads the other directory and fails
    # closed on a perfectly healthy identity.
    from local_deep_research.config.paths import get_user_database_filename
    from local_deep_research.database.encrypted_db import db_manager

    env_dir = tmp_path / "env"
    pinned_dir = tmp_path / "pinned" / "encrypted_databases"
    (env_dir / "encrypted_databases").mkdir(parents=True)
    pinned_dir.mkdir(parents=True)
    monkeypatch.setenv("LDR_DATA_DIR", str(env_dir))
    monkeypatch.setattr(db_manager, "_data_dir_override", pinned_dir)
    db_path = pinned_dir / get_user_database_filename("alice")
    with acquire_rekey_lock(db_path):
        epoch = read_or_initialize_rekey_epoch(db_path)

    assert read_user_authorization_epoch("alice") == epoch


def test_has_active_sessions_for_ignores_a_stale_epoch_session(
    data_dir: Path,
) -> None:
    # A password change in another worker leaves this worker's record in
    # place until its next validation; it must not count as active.
    _initialize_epoch(data_dir, "alice")
    manager = _manager()
    stale_session = manager.create_session("alice")
    assert manager.has_active_sessions_for("alice") is True

    with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
        advance_rekey_epoch(_user_db_path(data_dir, "alice"))

    assert manager.has_active_sessions_for("alice") is False
    # Read-only: the stale record is left for validation to remove.
    assert stale_session in manager.sessions

    # A session bound to the new epoch does count.
    manager.create_session("alice")
    assert manager.has_active_sessions_for("alice") is True


def test_has_active_sessions_for_is_unavailable_when_the_epoch_is_unreadable(
    data_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    # An unavailable check refuses authorization but does not prove that
    # shared resources belong to no remaining session.
    _initialize_epoch(data_dir, "alice")
    manager = _manager()
    session_id = manager.create_session("alice")

    with monkeypatch.context() as patched:
        _fail_epoch_reads(patched, errno.EMFILE)
        with pytest.raises(sm_module.SessionValidationUnavailableError):
            manager.has_active_sessions_for("alice")

    assert session_id in manager.sessions
    assert manager.has_active_sessions_for("alice") is True


def test_has_active_sessions_for_is_false_on_recovery_required_state(
    data_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    _initialize_epoch(data_dir, "alice")
    manager = _manager()
    manager.create_session("alice")

    def ambiguous_state(_username: str) -> str:
        raise RekeyRecoveryRequiredError(_user_db_path(data_dir, "alice"))

    monkeypatch.setattr(
        sm_module, "read_user_authorization_epoch", ambiguous_state
    )

    assert manager.has_active_sessions_for("alice") is False


# --- socket teardown when validation removes the record --------------------
#
# A socket is authorised once, at handshake, against a session id recorded
# in ``socketio_asgi._sid_sessions``; ``emit_to_user`` fans out to every sid
# in ``_sid_users`` without re-checking the session. When validation
# REMOVES a record (epoch mismatch after a password change in another
# worker, unvouchable durable state, inline idle expiry) the record is gone
# from every later sweep, so validation itself must sever the sockets that
# handshook on it -- otherwise a socket opened with a stolen cookie keeps
# receiving the user's events (``settings_changed`` carries plaintext
# secrets) after the password change that was meant to fence it.


_SOCKET_MODULE = "local_deep_research.web.services.socketio_asgi"


@pytest.fixture
def severed(monkeypatch: pytest.MonkeyPatch) -> list[str]:
    """Record every session id handed to ``disconnect_session``."""
    calls: list[str] = []
    monkeypatch.setattr(
        f"{_SOCKET_MODULE}.disconnect_session",
        lambda session_id: calls.append(session_id),
    )
    return calls


def test_epoch_mismatch_severs_that_sessions_sockets(
    data_dir: Path, severed: list[str]
) -> None:
    _initialize_epoch(data_dir, "alice")
    manager = _manager()
    stale_session = manager.create_session("alice")
    with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
        advance_rekey_epoch(_user_db_path(data_dir, "alice"))
    fresh_session = manager.create_session("alice")

    assert manager.validate_session(stale_session) is None

    # Only the removed session's sockets go; the new-epoch session (another
    # tab that logged in after the change) keeps its own.
    assert severed == [stale_session]
    assert manager.validate_session(fresh_session) == "alice"
    assert severed == [stale_session]


def test_unvouchable_durable_state_severs_that_sessions_sockets(
    data_dir: Path, monkeypatch: pytest.MonkeyPatch, severed: list[str]
) -> None:
    _initialize_epoch(data_dir, "alice")
    manager = _manager()
    session_id = manager.create_session("alice")

    def ambiguous_state(_username: str) -> str:
        raise RekeyRecoveryRequiredError(_user_db_path(data_dir, "alice"))

    monkeypatch.setattr(
        sm_module, "read_user_authorization_epoch", ambiguous_state
    )

    assert manager.validate_session(session_id) is None
    assert session_id not in manager.sessions
    assert severed == [session_id]


def test_inline_idle_expiry_severs_that_sessions_sockets(
    data_dir: Path, severed: list[str]
) -> None:
    import datetime

    _initialize_epoch(data_dir, "alice")
    manager = _manager()
    session_id = manager.create_session("alice", remember_me=False)
    manager.sessions[session_id]["last_access"] -= (
        manager.session_timeout + datetime.timedelta(minutes=1)
    )

    assert manager.validate_session(session_id) is None
    assert session_id not in manager.sessions
    assert severed == [session_id]


def test_kept_or_unknown_records_sever_nothing(
    data_dir: Path, monkeypatch: pytest.MonkeyPatch, severed: list[str]
) -> None:
    # Negative controls: a valid session, a transient read fault (record
    # KEPT, 503) and an unknown id must not tear down any socket.
    _initialize_epoch(data_dir, "alice")
    manager = _manager()
    session_id = manager.create_session("alice")

    assert manager.validate_session(session_id) == "alice"
    with monkeypatch.context() as patched:
        _fail_epoch_reads(patched, errno.EMFILE)
        with pytest.raises(SessionValidationUnavailableError):
            manager.validate_session(session_id)
    assert manager.validate_session("no-such-session") is None

    assert severed == []
    assert session_id in manager.sessions


class _CookieRequest:
    """``end_rejected_session`` only touches ``request.session``."""

    def __init__(self, session: dict) -> None:
        self.session = dict(session)


def test_rejected_session_teardown_survives_an_epoch_read_fault(
    data_dir: Path, monkeypatch: pytest.MonkeyPatch, severed: list[str]
) -> None:
    # The cookie is cleared whatever happens, so the record and the sockets
    # behind it must go too, even while the epoch files cannot be read
    # (#7830): otherwise a replayed copy of the cleared cookie keeps a live
    # record that nothing can reach to destroy.
    from local_deep_research.web.dependencies import auth as auth_dep

    _initialize_epoch(data_dir, "alice")
    _initialize_epoch(data_dir, "bob")
    manager = _manager()
    session_id = manager.create_session("alice")
    other_tab = manager.create_session("alice")
    bobs_session = manager.create_session("bob")
    monkeypatch.setattr(auth_dep, "session_manager", manager)
    request = _CookieRequest({"username": "alice", "session_id": session_id})
    foreign = _CookieRequest({"username": "alice", "session_id": bobs_session})

    with monkeypatch.context() as patched:
        _fail_epoch_reads(patched, errno.EMFILE)
        # The fault is real: validation cannot answer for this record.
        with pytest.raises(SessionValidationUnavailableError):
            manager.validate_session(session_id)
        auth_dep.end_rejected_session(request)
        auth_dep.end_rejected_session(foreign)

    assert request.session == {}
    assert session_id not in manager.sessions
    assert severed == [session_id]
    # Scoped to that one record: the user's other tab and another user's
    # session named by a mismatched cookie are left alone.
    assert other_tab in manager.sessions
    assert bobs_session in manager.sessions


def test_socket_teardown_failure_never_breaks_validation(
    data_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    # Best-effort: a socket layer that raises must not turn the removal
    # into an exception at the HTTP middleware.
    def broken(_session_id: str) -> None:
        raise RuntimeError("socket layer down")

    monkeypatch.setattr(f"{_SOCKET_MODULE}.disconnect_session", broken)
    _initialize_epoch(data_dir, "alice")
    manager = _manager()
    session_id = manager.create_session("alice")
    with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
        advance_rekey_epoch(_user_db_path(data_dir, "alice"))

    assert manager.validate_session(session_id) is None
    assert session_id not in manager.sessions


def test_epoch_mismatch_clears_the_sid_maps_on_a_live_loop(
    data_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    """End to end through the real socket layer.

    ``disconnect_session`` -> ``_disconnect_matching`` runs on a live event
    loop; the fake ``sio.disconnect`` fires the real ``disconnect`` event
    handler, as python-socketio does, so the sid maps are cleaned by the
    same code path a logout uses. The other session's socket stays.
    """
    import asyncio
    import threading
    from unittest.mock import Mock

    from local_deep_research.web.services import socketio_asgi as sio_mod

    _initialize_epoch(data_dir, "alice")
    manager = _manager()
    stale_session = manager.create_session("alice")
    with acquire_rekey_lock(_user_db_path(data_dir, "alice")):
        advance_rekey_epoch(_user_db_path(data_dir, "alice"))
    fresh_session = manager.create_session("alice")

    monkeypatch.setattr(sio_mod, "_lock", asyncio.Lock())
    monkeypatch.setattr(sio_mod, "_subscriptions", {})
    monkeypatch.setattr(
        sio_mod, "_sid_users", {"sid-stolen": "alice", "sid-fresh": "alice"}
    )
    monkeypatch.setattr(
        sio_mod,
        "_sid_sessions",
        {"sid-stolen": stale_session, "sid-fresh": fresh_session},
    )

    loop = asyncio.new_event_loop()
    thread = threading.Thread(target=loop.run_forever, daemon=True)
    thread.start()
    disconnected: list[str] = []

    async def fake_disconnect(sid: str) -> None:
        disconnected.append(sid)
        await sio_mod.disconnect(sid)

    fake_sio = Mock()
    fake_sio.disconnect = fake_disconnect
    futures = []
    real_schedule = asyncio.run_coroutine_threadsafe

    def capture(coro, target_loop):
        future = real_schedule(coro, target_loop)
        futures.append(future)
        return future

    try:
        with monkeypatch.context() as patched:
            patched.setattr(sio_mod, "_get_main_loop", lambda: loop)
            patched.setattr(sio_mod, "sio", fake_sio)
            patched.setattr(asyncio, "run_coroutine_threadsafe", capture)

            assert manager.validate_session(stale_session) is None
            assert futures, "validation scheduled no socket teardown"
            for future in futures:
                future.result(timeout=5)
    finally:
        loop.call_soon_threadsafe(loop.stop)
        thread.join(timeout=5)
        loop.close()

    assert disconnected == ["sid-stolen"]
    assert sio_mod._sid_users == {"sid-fresh": "alice"}
    assert sio_mod._sid_sessions == {"sid-fresh": fresh_session}


def test_epoch_read_sharing_violation_is_retried_then_fails_closed(
    data_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    """A Windows sharing violation on the lock-free epoch read.

    A writer's ``os.replace``/``unlink`` in flight makes the reader's open
    fail with PermissionError for a few microseconds. A transient one is
    retried and the session validates; a persistent one fails closed
    (503 for this request, the record KEPT, never a validated identity).
    """
    from local_deep_research.database import rekey_lock as rekey_lock_module

    epoch = _initialize_epoch(data_dir, "alice")
    manager = _manager()
    session_id = manager.create_session("alice")
    monkeypatch.setattr(
        rekey_lock_module, "_SHARING_VIOLATION_DELAY", 0, raising=False
    )
    real_open = os.open
    budget = {"failures": 2}

    def sharing_violation(path, *args, **kwargs):
        if (
            Path(path).name.endswith((".rekey.epoch", ".rekey.pending"))
            and budget["failures"] > 0
        ):
            budget["failures"] -= 1
            raise PermissionError(errno.EACCES, "sharing violation")
        return real_open(path, *args, **kwargs)

    monkeypatch.setattr(os, "open", sharing_violation)

    identity = manager.validate_session_identity(session_id)
    assert identity is not None and identity.rekey_epoch == epoch
    assert budget["failures"] == 0

    budget["failures"] = 10**6
    before = dict(manager.sessions[session_id])
    with pytest.raises(SessionValidationUnavailableError) as raised:
        manager.validate_session_identity(session_id)
    assert raised.value.errno == errno.EACCES
    assert manager.sessions[session_id] == before