~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Database session context manager and decorator for encrypted databases.
Ensures all database access has proper encryption context.
"""
import functools
from contextlib import contextmanager
from typing import Callable, Iterator, Optional
from loguru import logger
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import Session
from ..security.log_sanitizer import scrub_error
from ..utilities.thread_context import get_search_context
from .encrypted_db import db_manager
from .rekey_lock import RekeyOperationalError, describe_rekey_fault
from .thread_local_session import thread_session_manager
# Placeholder password used when accessing unencrypted databases.
# This should only be used when LDR_ALLOW_UNENCRYPTED=true is set.
UNENCRYPTED_DB_PLACEHOLDER = "unencrypted-mode"
class DatabaseSessionError(Exception):
"""Raised when database session cannot be established."""
pass
class DatabaseSessionUnavailableError(DatabaseSessionError):
"""The session could not be established for an operational reason.
``get_user_db_session`` raises this instead of the base class when
opening or claiming the user's database failed with a
``RekeyOperationalError``: the rekey sidecar could not be opened, locked
or released, an epoch file could not be read or first published (an
fd-exhaustion spike, EACCES, EIO), or the durable rekey state needs
recovery first. None of these is a verdict on the credential or on the
session. A request handler that would end the session on a
``DatabaseSessionError`` answers this one with a retryable 503 and keeps
the session instead (``database_unavailable_exception`` in
``web/dependencies/auth.py``).
``reason`` is the underlying fault's scrub-safe summary, for diagnostics:
its type name plus the sidecar operation and errno when the fault carries
them (``RekeySidecarIOError (operation=open, errno=24)``), the same fields
the login route logs for the same fault class (``describe_rekey_fault``).
It never holds path material. The fault itself is neither chained nor
kept: the frames it was raised through hold the plaintext password.
"""
def __init__(self, reason: str) -> None:
self.reason = reason
super().__init__("Database session unavailable")
def safe_rollback(session: Session, context: str = "") -> None:
"""Roll back the session, swallowing and logging any rollback failure.
SQLAlchemy requires explicit rollback after a failed flush/commit before
the session is usable again. Skipping it leaves the session in
PendingRollbackError state and every subsequent ORM operation cascades.
This helper exists so call sites can recover the session in one line
without repeating the try/except/log boilerplate at every except handler.
``context`` is included in the error log so failed rollbacks can be
traced back to the call site.
Two SQLAlchemy error shapes are treated as "the session is structurally
unusable — give up on it, drop the thread-local cache, and let the next
caller get a fresh one" rather than as loud failures:
* ``InvalidRequestError("...provisioning a new connection; concurrent
operations are not permitted...")`` — the second thread racing on the
per-user QueuePool.
* ``InterfaceError("Cursor needed to be reset because of commit/rollback
and can no longer be fetched from")`` — a cursor was invalidated by a
commit/rollback that fired between the original ``execute()`` and the
lazy-attribute fetch that followed.
In both cases SQLAlchemy has already marked the session unrecoverable, so
a no-op rollback is correct, the thread-local cache is cleared, and the
failure is logged at DEBUG so the production stderr stream stays clean.
Other ``SQLAlchemyError`` failures still hit the loud ``logger.exception``
path (unless their message matches a known broken-session signature) —
the session is normally recoverable via rollback and the operator needs
to see them.
"""
if session is None:
return
log_msg = (
f"Failed to rollback session: {context}"
if context
else "Failed to rollback session"
)
try:
session.rollback()
except SQLAlchemyError as exc:
msg = str(exc)
# Message-substring matching pinned against SQLAlchemy 2.0+ QueuePool error messages
# ('provisioning a new connection', 'concurrent operations are not permitted').
is_provisioning_race = (
"provisioning a new connection" in msg
or "concurrent operations are not permitted" in msg
)
# Requires 'Cursor needed to be reset' message match so other InterfaceErrors
# (e.g., driver/connectivity failures) are not quietly swallowed into the DEBUG path.
is_cursor_invalidated = "Cursor needed to be reset" in msg
if is_provisioning_race or is_cursor_invalidated:
label = f": {context}" if context else ""
logger.debug(f"safe_rollback — resetting broken session{label}")
# Drop the thread-local cache so the next caller on this
# thread gets a fresh session. Identity-checked inside the
# helper, so a caller that hands in a session that ISN'T
# the cached one (e.g. borrowed from ``g.db_session`` or
# owned by a different thread) won't accidentally clear
# someone else's cache. The reset itself is best-effort —
# never let it raise past ``safe_rollback`` (call sites are
# themselves in except handlers).
try:
thread_session_manager.reset_session_if_matches(session)
except Exception:
logger.debug(
f"safe_rollback: reset_session_if_matches raised for{label}"
)
return
logger.exception(log_msg)
except Exception:
logger.exception(log_msg)
@contextmanager
def telemetry_savepoint(session: Session) -> Iterator[None]:
"""Write best-effort telemetry in a SAVEPOINT on a borrowed session.
The telemetry joins the caller's transaction only when the caller
already holds an open write transaction; otherwise it commits at once
in a short transaction of its own. The signal is the DBAPI connection's
``in_transaction`` flag. Production sqlite3/SQLCipher connections use
legacy deferred mode (``isolation_level=""``): the driver opens a
transaction only before INSERT, UPDATE or DELETE and closes it on commit
or rollback, so a SELECT never opens one. ``Session.in_transaction()``
and the pending ``new``/``dirty``/``deleted`` collections cannot make
this distinction: any query autobegins SQLAlchemy's logical transaction,
and a flushed write leaves those collections empty.
* Open (the caller has written, or began a transaction explicitly as
``research_cascade._ensure_outer_db_transaction`` does): the SAVEPOINT
nests inside it, so the telemetry commits or rolls back with the
caller, and no second connection waits on this thread's own writer
lock.
* Idle (the scope has only read, or has already committed): SQLite
begins a transaction for the outermost SAVEPOINT and RELEASE commits
it. The telemetry is durable at once, survives the scope's exit
rollback, and holds the writer lock only while it writes. If the
telemetry fails, that transaction is rolled back here, because
ROLLBACK TO alone would keep it (and any writer lock it took) open
until the caller's scope ends.
Other dialects begin their transaction with the first statement, so the
savepoint always nests there. A connection that does not report a
definite ``False`` counts as open and is never rolled back here.
Failures propagate to the caller, which logs and skips the telemetry.
"""
connection = session.connection()
owned_connection = None
if connection.dialect.name == "sqlite":
driver_connection = connection.connection.driver_connection
if getattr(driver_connection, "in_transaction", None) is False:
owned_connection = driver_connection
try:
with session.begin_nested():
yield
except BaseException:
if owned_connection is not None and owned_connection.in_transaction:
try:
owned_connection.rollback()
except Exception:
logger.debug("Could not end a failed telemetry transaction")
raise
@contextmanager
def get_user_db_session(
username: Optional[str] = None,
password: Optional[str] = None,
session_id: Optional[str] = None,
):
"""
Context manager that ensures proper database session with encryption.
Now uses thread-local sessions for better performance.
Args:
username: Username (required; must be passed explicitly under FastAPI).
password: Password for encrypted database (required for first access).
session_id: Optional session ID for exact per-session password lookup.
Request handlers should pass the current request's session_id so
two concurrent sessions for the same user can't cross-pollinate.
If omitted, the resolver falls back to scanning any active session
for the user — fine for background threads, unsafe for request
handlers.
Yields:
Database session for the user
Raises:
DatabaseSessionUnavailableError: If the session cannot be
established because of a ``RekeyOperationalError`` (a rekey
sidecar or epoch fault, or pending rekey recovery). A subclass
of ``DatabaseSessionError``.
DatabaseSessionError: If session cannot be established
"""
# Import here to avoid circular imports
from .thread_local_session import get_metrics_session
from .session_passwords import session_password_store
if not username:
raise DatabaseSessionError("No authenticated user")
# Resolve password from the provided session_id, then fall back to
# any active session, then the thread context for background workers.
if not password and session_id:
password = session_password_store.get_session_password(
username, session_id
)
if password:
logger.debug(f"Got password from session store for {username}")
if not password:
# Scan active sessions for this user. Safe for background threads;
# request handlers should pass session_id explicitly.
password = session_password_store.get_any_session_password(username)
if not password:
thread_context = get_search_context()
if thread_context and thread_context.get("user_password"):
password = thread_context["user_password"]
logger.debug(f"Got password from thread context for {username}")
if not password and db_manager.has_encryption:
raise DatabaseSessionError(
f"Encrypted database for {username} requires password"
)
if not password:
logger.warning(
f"Accessing unencrypted database for {username} - "
"ensure this is intentional (LDR_ALLOW_UNENCRYPTED=true)"
)
password = UNENCRYPTED_DB_PLACEHOLDER
# A RekeyOperationalError from opening or claiming the database (in
# open_user_database or in create_thread_safe_session_for_metrics, which
# both let it through) is not a credential verdict, so it gets its own
# subclass: request handlers answer it with a retryable 503 that keeps
# the session, where the base class may end it. Raised after the except
# block, so the new error does not even carry the fault as its
# suppressed __context__: the frames it came through hold the password.
session = None
unavailable_reason = None
try:
session = get_metrics_session(username, password)
except RekeyOperationalError as error:
unavailable_reason = describe_rekey_fault(error)
if unavailable_reason is not None:
logger.warning(
f"Database session unavailable for {username}: {unavailable_reason}"
)
raise DatabaseSessionUnavailableError(unavailable_reason)
if not session:
raise DatabaseSessionError(
f"Could not establish session for {username}"
)
# The session is a reused thread-local session. On normal scope exit
# (outermost block) the checked-out connection is released in the
# ``finally`` below, so the pool isn't pinned for the thread's lifetime.
# We MUST wrap the yield in try/except so an exception inside the ``with``
# block doesn't leave a half-committed transaction attached to this
# thread's session: the next caller on the same thread would otherwise
# inherit that dirty state. Thread cleanup (``cleanup_current_thread()``)
# remains as a safety net for paths that never exit a scope.
#
# The scope depth tells get_session's re-validation whether an
# enclosing block is still active on this thread: a nested
# get_user_db_session call (e.g. a helper opening its own session
# inside a caller's with-block) must NOT trigger the stale-lock
# rollback, or the caller's uncommitted writes are destroyed.
from .thread_local_session import thread_session_manager
thread_session_manager.enter_scope()
try:
yield session
except Exception:
# The yielded session is a *reused* thread-local session, not a
# fresh one closed on exit. If the caller's ``with`` block raised
# (most importantly a failed ``session.commit()``/``flush()``),
# the session is left in ``PendingRollbackError`` state and the
# next operation on this thread cascades. Roll it back here so an
# unguarded ``with`` block can't poison the thread, then re-raise
# so the original error still surfaces to the caller.
safe_rollback(session, "get_user_db_session")
raise
finally:
thread_session_manager.exit_scope()
if not thread_session_manager.in_scope():
try:
# If a transaction was left open (e.g. from read queries autobeginning
# a transaction and checking out a connection from QueuePool, or uncommitted writes),
# roll back on outermost exit to return the checked-out connection to QueuePool
# and release SQLite SHARED read locks.
if (
hasattr(session, "in_transaction")
and session.in_transaction()
):
safe_rollback(session, "get_user_db_session exit")
except Exception as exc:
logger.debug(
f"safe_rollback on get_user_db_session exit raised: {scrub_error(exc)}"
)
def with_user_database(func: Callable) -> Callable:
"""
Decorator that ensures function has access to user's database.
Injects 'db_session' as first argument to the decorated function.
Usage:
@with_user_database
def get_user_settings(db_session, setting_key):
return db_session.query(Setting).filter_by(key=setting_key).first()
"""
@functools.wraps(func)
def wrapper(*args, **kwargs):
# Check if username/password provided in kwargs
username = kwargs.pop("_username", None)
password = kwargs.pop("_password", None)
with get_user_db_session(username, password) as db_session:
return func(db_session, *args, **kwargs)
return wrapper
class DatabaseAccessMixin:
"""
Mixin class for services that need database access.
Provides convenient methods for database operations.
"""
def get_db_session(
self, username: Optional[str] = None
) -> Optional[Session]:
"""
DEPRECATED: This method returns a closed session due to context manager exit.
Use `with get_user_db_session(username) as session:` instead.
Raises:
DeprecationWarning: Always raised to prevent usage of broken method.
"""
raise DeprecationWarning(
"get_db_session() is deprecated and returns a closed session. "
"Use `with get_user_db_session(username) as session:` instead."
)
@with_user_database
def execute_with_db(
self, db_session: Session, query_func: Callable, *args, **kwargs
):
"""Execute a function with database session."""
return query_func(db_session, *args, **kwargs)