Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

22.4 KB
Raw Blame
"""
Thread-local database session management.
Each thread gets its own database session, reused until its owned cleanup boundary.
"""

import functools
import threading
from contextlib import ContextDecorator
from typing import Optional, Dict, Tuple
from sqlalchemy import text
from sqlalchemy.exc import PendingRollbackError
from sqlalchemy.orm import Session
from loguru import logger

from .encrypted_db import db_manager


class ThreadLocalSessionManager:
    """
    Manages database sessions per thread.
    Each thread gets its own session, reused until the request or task finishes.
    """

    def __init__(self):
        # Thread-local storage for sessions
        self._local = threading.local()
        # Track credentials per thread ID (for cleanup)
        self._thread_credentials: Dict[int, Tuple[str, str]] = {}
        self._lock = threading.Lock()

    def enter_scope(self) -> None:
        """Mark entry into a ``get_user_db_session`` block on this thread.

        ``get_session`` uses the depth to tell a left-over transaction
        from a *completed* block (safe and necessary to roll back — it
        holds a SQLite SHARED lock) apart from an *enclosing* block that
        is still active (rolling back would destroy the caller's
        uncommitted writes)."""
        self._local.scope_depth = getattr(self._local, "scope_depth", 0) + 1

    def exit_scope(self) -> None:
        """Mark exit from a ``get_user_db_session`` block on this thread."""
        self._local.scope_depth = max(
            0, getattr(self._local, "scope_depth", 0) - 1
        )

    def in_scope(self) -> bool:
        """Whether an enclosing ``get_user_db_session`` block is active
        on this thread."""
        return getattr(self._local, "scope_depth", 0) > 0

    def get_session(self, username: str, password: str) -> Optional[Session]:
        """
        Get or create a database session for the current thread.

        The session is created once per thread and reused for all subsequent calls.
        This avoids the expensive SQLCipher decryption on every database access.

        Returns None when the database cannot be opened with ``password``
        (or its schema cannot be initialised). A ``RekeyOperationalError``
        from opening the database or creating the session (the rekey
        sidecar or an epoch file could not be read, or a pending rekey
        needs recovery) is raised instead: it is not a credential verdict,
        and ``get_user_db_session`` turns it into
        ``DatabaseSessionUnavailableError`` so a request can answer a
        retryable 503 rather than log the user out.
        """
        thread_id = threading.get_ident()

        # Check if we already have a session for this thread
        if hasattr(self._local, "session") and self._local.session:
            # SECURITY: ensure cached session belongs to the requesting user
            if getattr(self._local, "username", None) != username:
                logger.warning(
                    f"Thread {thread_id}: Session username mismatch "
                    f"(cached={self._local.username!r}, requested={username!r}), "
                    "clearing stale cross-user session"
                )
                self._cleanup_thread_session()
            else:
                # Verify it's still valid
                try:
                    self._local.session.execute(text("SELECT 1"))
                    if not self.in_scope():
                        # Under DEFERRED isolation the validation SELECT
                        # opens a transaction that holds a SHARED lock on
                        # SQLite until an explicit commit/rollback. A
                        # long-lived thread-local session reused across
                        # requests would keep that lock held and block
                        # the first writer. Roll it back so subsequent
                        # callers start fresh.
                        self._local.session.rollback()
                    # else: re-entrant call from inside an enclosing
                    # get_user_db_session block on this thread — the
                    # validation SELECT joined the caller's open
                    # transaction, and rolling back here would silently
                    # destroy the caller's uncommitted writes (concrete
                    # data loss in library_rag_service.index_document,
                    # whose nested helpers open their own sessions).
                    return self._local.session
                except PendingRollbackError:
                    # Session has a pending rollback (e.g. from a previous database lock error).
                    # Attempt rollback to recover without destroying the session.
                    logger.debug(
                        f"Thread {thread_id}: PendingRollbackError, attempting rollback recovery"
                    )
                    try:
                        self._local.session.rollback()
                        self._local.session.execute(text("SELECT 1"))
                        self._local.session.rollback()
                        return self._local.session
                    except Exception:
                        logger.warning(
                            f"Thread {thread_id}: Rollback recovery failed, creating new session"
                        )
                        self._cleanup_thread_session()
                except Exception:
                    # Session is invalid, will create a new one
                    logger.debug(
                        f"Thread {thread_id}: Existing session invalid, creating new one"
                    )
                    self._cleanup_thread_session()

        # Create new session for this thread
        logger.debug(
            f"Thread {thread_id}: Creating new database session for user {username}"
        )

        # Ensure database is open. open_user_database returns None for
        # credential failures and raises DatabaseInitializationError when
        # the schema can't be initialised; from a worker-thread caller
        # both mean "no usable session right now" so collapse them.
        #
        # A RekeyOperationalError is deliberately NOT collapsed into None,
        # here or from create_thread_safe_session_for_metrics below. It is
        # an operational fault (an fd-exhaustion spike on the rekey
        # sidecar, an unreadable epoch, pending recovery), and a None would
        # reach get_user_db_session as the same DatabaseSessionError a
        # wrong password produces, which the home page answered by logging
        # the user out. get_user_db_session catches it instead.
        from .encrypted_db import DatabaseInitializationError

        try:
            engine = db_manager.open_user_database(username, password)
        except DatabaseInitializationError:
            # ``logger.warning`` (no traceback) rather than
            # ``logger.exception``: ``password`` is a live local in this
            # frame, so rendering a traceback under ``diagnose=True``
            # would dump the plaintext SQLCipher master password
            # (unrecoverable — TRUST.md §5). The redacted failure detail
            # is already logged at the raise site in
            # ``open_user_database`` (#4182).
            logger.warning(
                f"Thread {thread_id}: database init failed for user {username}"
            )
            return None
        if not engine:
            logger.error(
                f"Thread {thread_id}: Failed to open database for user {username}"
            )
            return None

        # Create session for this thread
        session = db_manager.create_thread_safe_session_for_metrics(
            username, password
        )
        if not session:
            logger.error(
                f"Thread {thread_id}: Failed to create session for user {username}"
            )
            return None

        # Store in thread-local storage
        self._local.session = session
        self._local.username = username

        # Track credentials for cleanup
        with self._lock:
            self._thread_credentials[thread_id] = (username, password)

        return session

    def get_current_session(self) -> Optional[Session]:
        """Get the current thread's session if it exists."""
        if hasattr(self._local, "session"):
            return self._local.session
        return None

    def get_scoped_session(self, username: str) -> Optional[Session]:
        """Borrow the active scope's session only when it belongs to username.

        Metrics callbacks must not call ``get_session`` here: its recovery
        path may roll back or replace an enclosing caller-owned transaction.
        A callback for a different research owner simply omits best-effort
        telemetry instead of touching that other user's session.
        """
        if not self.in_scope():
            return None
        if getattr(self._local, "username", None) != username:
            return None
        return self.get_current_session()

    def _cleanup_thread_session(self):
        """Clean up the current thread's session.

        Sessions are bound to the shared per-user QueuePool engine, so
        closing the session returns its connection to the pool — there
        are no per-thread engines to dispose.
        """
        thread_id = threading.get_ident()

        if hasattr(self._local, "session") and self._local.session:
            try:
                self._local.session.rollback()
            except Exception:
                logger.warning(
                    f"Thread {thread_id}: Error rolling back session during cleanup"
                )
            try:
                self._local.session.close()
                logger.debug(f"Thread {thread_id}: Closed database session")
            except Exception:
                logger.warning(f"Thread {thread_id}: Error closing session")
            finally:
                self._local.session = None

        if hasattr(self._local, "username"):
            self._local.username = None

        # Remove from tracking
        with self._lock:
            self._thread_credentials.pop(thread_id, None)

    def reset_session_if_matches(self, session: "Session | None") -> bool:
        """If ``session`` is the current thread's cached session, clear it.

        Used by ``safe_rollback`` in ``database/session_context.py`` to heal
        the thread-local session cache when an error tells us the session
        is structurally unusable (a provisioning race or an invalidated
        cursor). The next caller on this thread then gets a fresh session
        from the shared ``QueuePool`` instead of inheriting the broken
        one.

        Identity-checked: a caller that hands in a session from a
        different thread (or a session that isn't the cached one at all
        — e.g. one borrowed from ``g.db_session``) won't accidentally
        clear someone else's cache. Returns ``True`` if a reset
        actually happened, ``False`` otherwise.
        """
        if session is None:
            return False
        if not hasattr(self._local, "session") or self._local.session is None:
            return False
        if self._local.session is not session:
            return False
        self._cleanup_thread_session()
        return True

    def cleanup_thread(self, thread_id: Optional[int] = None):
        """
        Clean up session for a specific thread or current thread.
        Called when a thread is finishing.
        """
        if thread_id is None:
            thread_id = threading.get_ident()

        # If it's the current thread, we can clean up directly
        if thread_id == threading.get_ident():
            self._cleanup_thread_session()
        else:
            # For other threads, just remove from tracking
            # The thread-local storage will be cleaned up when the thread ends
            with self._lock:
                self._thread_credentials.pop(thread_id, None)

    def cleanup_dead_threads(self):
        """Remove credential entries for threads that are no longer alive.

        Handles the abnormal case of threads that died without triggering
        their cleanup handler. Uses threading.enumerate() to identify
        alive threads and removes credential entries for dead ones.
        Sessions on dead threads are garbage-collected normally and
        their connections return to the shared per-user QueuePool.

        Called from:
        - processor_v2.py: every ~60s in the queue loop
        - fastapi_app.py: in the request middleware's post-request cleanup
        - connection_cleanup.py: in cleanup_idle_connections (every ~300s)
        """
        alive_ids = {t.ident for t in threading.enumerate()}
        with self._lock:
            dead_ids = [
                tid for tid in self._thread_credentials if tid not in alive_ids
            ]
            for tid in dead_ids:
                del self._thread_credentials[tid]
        if dead_ids:
            logger.debug(f"Swept {len(dead_ids)} dead thread credential(s)")

    def clear_credentials_for_user(self, username: str) -> int:
        """Drop cached plaintext credentials for one user, on any thread.

        Entries here hold the user's plaintext SQLCipher password so a
        pooled worker can rebuild its session without a round-trip to the
        password store. Synchronous routes normally drop their owner-thread
        entry through ``WorkerCleanupAPIRoute``. This username-wide cleanup
        is still required on logout because another worker may be executing
        background work for the same user, and one thread cannot clear a
        different thread's ``threading.local`` state.

        Only the tracking dict is cleared. The owning thread's
        ``threading.local`` session is deliberately left alone: one thread
        cannot reach into another's local storage, and the session is
        harmless once the credential is gone — the username re-validation at
        the top of ``get_session()`` evicts it before any reuse. In-flight
        work is unaffected too, since ``get_session()`` is passed the
        password explicitly and never reads it back out of this dict.

        Returns the number of entries dropped.
        """
        with self._lock:
            thread_ids = [
                tid
                for tid, (user, _password) in self._thread_credentials.items()
                if user == username
            ]
            for tid in thread_ids:
                del self._thread_credentials[tid]
        if thread_ids:
            logger.debug(
                f"Cleared cached credentials on {len(thread_ids)} thread(s)"
            )
        return len(thread_ids)

    def cleanup_all(self):
        """Clean up all tracked sessions (for shutdown)."""
        with self._lock:
            thread_ids = list(self._thread_credentials.keys())

        for thread_id in thread_ids:
            self.cleanup_thread(thread_id)


# Global instance
thread_session_manager = ThreadLocalSessionManager()


def get_metrics_session(username: str, password: str) -> Optional[Session]:
    """
    Get a database session for metrics operations in the current thread.
    The session is created once and reused until the thread's owned cleanup
    boundary (request, task, or worker exit).

    Note: This specifically uses create_thread_safe_session_for_metrics internally
    and should only be used for metrics-related database operations.

    Raises ``RekeyOperationalError`` when the database is unavailable for an
    operational reason; see ``ThreadLocalSessionManager.get_session``.
    """
    return thread_session_manager.get_session(username, password)


def get_current_thread_session() -> Optional[Session]:
    """Get the current thread's session if it exists."""
    return thread_session_manager.get_current_session()


def cleanup_current_thread():
    """Clean up every database session owned by the current thread.

    Also clears any passwords cached by ``metrics_writer`` on this thread —
    pooled worker threads must not retain plaintext credentials across tasks.
    """
    try:
        thread_session_manager.cleanup_thread()
    except Exception:
        logger.debug(
            "cleanup_current_thread: error closing thread-local DB session",
            exc_info=True,
        )
    try:
        # Lazy import avoids the encrypted_db -> thread_local_session import
        # cycle during module bootstrap.  This is the second session path:
        # ambient SettingsManager callers use db_utils' per-thread LRU rather
        # than ThreadLocalSessionManager.
        from ..utilities.db_utils import (
            cleanup_cached_user_sessions_current_thread,
        )

        cleanup_cached_user_sessions_current_thread()
    except Exception:
        # ERROR, not debug: this is the #6095 owner-thread registry
        # cleanup that turns a GC-reclaimable session leak into a
        # deterministic release. If it ever breaks (e.g. a cachetools
        # upgrade renames ``cache_lock``), the fix silently reverts to the
        # pre-#6095 behavior (an evicted session waits on the cyclic
        # collector to reclaim its pooled connection) with nothing above
        # debug level to notice.
        #
        # Uses logger.exception(), NOT logger.warning(..., exc_info=True):
        # this module imports raw loguru (see the module imports above),
        # which has no ``exc_info`` parameter. Passing it to warning()
        # makes it an ordinary str.format kwarg that is silently discarded,
        # so no traceback is emitted at all -- verified empirically.
        # ``logger.exception()`` logs at ERROR, louder than WARNING, which
        # suits a silent revert to pre-#6095 behaviour.
        #
        # It DOES attach the traceback: this is plain loguru, not
        # SecureLogger, so nothing gates that on diagnose mode. The
        # traceback still does not leave the process. The encrypted-DB sink
        # persists ``_exception_context(record) + record["message"]`` -- an
        # exception type/value prefix, never the frames -- and the frontend
        # sink forwards only ``record["message"]``; see ``database_sink``,
        # ``_exception_context`` and the frontend sink in
        # ``utilities/log_utils.py``. It reaches the stderr/file sinks,
        # which is exactly where it is wanted.
        logger.exception(
            "cleanup_current_thread: error closing cached DB sessions "
            "(owner-thread registry cleanup failed — falling back to "
            "GC-reclaimed cleanup for this thread's sessions)"
        )
    try:
        from .thread_metrics import metrics_writer

        metrics_writer.clear_passwords()
    except Exception:
        logger.debug(
            "cleanup_current_thread: error clearing metrics_writer passwords",
            exc_info=True,
        )


def clear_user_credentials(username: str) -> int:
    """Drop ``username`` from the session manager's credential registry.

    Called on logout so the plaintext SQLCipher password does not outlive
    the session that authorised it. See
    ``ThreadLocalSessionManager.clear_credentials_for_user`` for why the
    registry cleanup differs from owner-thread cleanup.

    This function clears only ``ThreadLocalSessionManager``'s process-wide
    username-keyed registry. ``thread_metrics.metrics_writer`` storage is
    owner-thread-local; normal owned worker boundaries call
    ``cleanup_current_thread``, and direct integrations must do likewise.
    """
    try:
        return thread_session_manager.clear_credentials_for_user(username)
    except Exception:
        logger.exception("Failed to clear cached credentials on logout")
        return 0


def cleanup_dead_threads():
    """Sweep dead-thread credential entries from the session manager."""
    try:
        thread_session_manager.cleanup_dead_threads()
    except Exception:
        logger.warning("Dead-thread session sweep failed")


class _ThreadCleanup(ContextDecorator):
    """Context manager / decorator for thread-local resource cleanup."""

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        try:
            cleanup_current_thread()
        except Exception:
            logger.debug(
                "thread_cleanup: error during DB session cleanup",
                exc_info=True,
            )
        try:
            from ..config.thread_settings import clear_settings_context

            clear_settings_context()
        except Exception:
            logger.debug(
                "thread_cleanup: error clearing settings context",
                exc_info=True,
            )
        try:
            from ..utilities.thread_context import clear_search_context

            clear_search_context()
        except Exception:
            logger.debug(
                "thread_cleanup: error clearing search context",
                exc_info=True,
            )
        try:
            # Defense-in-depth audit hook (PEP 578). Leaving an active
            # EgressContext on a pooled worker thread would let the
            # next, unrelated task inherit the previous run's scope.
            from ..security.egress.audit_hook import clear_active_context

            clear_active_context()
        except Exception:
            logger.debug(
                "thread_cleanup: error clearing egress audit context",
                exc_info=True,
            )
        return False


def thread_cleanup(func=None):
    """Ensure all thread-local resources are cleaned up when a function or block exits.

    Works as a bare decorator, a decorator factory, or a context manager::

        @thread_cleanup
        def worker(): ...

        @thread_cleanup()
        def worker(): ...

        with thread_cleanup():
            ...

        executor.submit(thread_cleanup(func), arg)
    """
    if func is not None:

        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            with _ThreadCleanup():
                return func(*args, **kwargs)

        return wrapper
    return _ThreadCleanup()


# Context manager for automatic cleanup
class ThreadSessionContext:
    """
    Context manager that ensures thread session is cleaned up.
    Usage:
        with ThreadSessionContext(username, password) as session:
            # Use session
    """

    def __init__(self, username: str, password: str):
        self.username = username
        self.password = password
        self.session = None

    def __enter__(self) -> Optional[Session]:
        self.session = get_metrics_session(self.username, self.password)
        return self.session

    def __exit__(self, exc_type, exc_val, exc_tb):
        # Don't cleanup here - let the thread keep its session
        # Only cleanup when thread ends
        pass