Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

24.9 KB
Raw Blame
"""
Helpers for running synchronous DB-touching code from async route
handlers without leaking thread-local state.

``DatabaseMiddleware`` cleans up the event-loop thread's DB session at
the end of every request, but it cannot reach into worker threads
spawned by ``asyncio.to_thread``. ThreadPoolExecutor workers are
reused across tasks, so a session opened by one user's task would stay
attached to that worker — the next task that lands on it would inherit
the previous user's session (memory bloat at best, cross-user state
mixing at worst).

``WorkerCleanupAPIRoute`` supplies that owner-thread boundary for plain
``def`` endpoints. ``WorkerCleanupStreamingResponse`` supplies it for each
worker that advances a synchronous response body. ``run_db_sync``,
``run_research_sync`` and ``run_model_discovery_sync`` do the same for
explicit DB work dispatched from asynchronous endpoints. Long research calls
and model discovery each use a separate executor so they cannot fill the
default executor used by authenticated requests.
"""

from __future__ import annotations

import asyncio
import atexit
import functools
import threading
from collections.abc import AsyncIterable, Iterable, Iterator
from concurrent.futures import ThreadPoolExecutor
from contextvars import copy_context
from typing import TYPE_CHECKING, Any, Callable, TypeVar

from fastapi.encoders import jsonable_encoder
from fastapi.routing import APIRoute, APIRouter
from loguru import logger
from starlette.responses import Response, StreamingResponse

if TYPE_CHECKING:
    from fastapi.dependencies.models import Dependant

T = TypeVar("T")

_WORKER_CLEANUP_MARKER = "__ldr_worker_thread_cleanup__"

# Long API v1 research calls must not occupy asyncio's default executor:
# every authenticated request uses that executor to open/check its user DB.
# Keep this pool separate from the short DB dispatches in run_db_sync.
_API_RESEARCH_WORKERS = 8
_api_research_executor: ThreadPoolExecutor | None = None
_api_research_executor_lock = threading.Lock()


def _get_api_research_executor() -> ThreadPoolExecutor:
    global _api_research_executor
    with _api_research_executor_lock:
        if _api_research_executor is None:
            _api_research_executor = ThreadPoolExecutor(
                max_workers=_API_RESEARCH_WORKERS,
                thread_name_prefix="ldr-api-research",
            )
        return _api_research_executor


def _shutdown_api_research_executor() -> None:
    global _api_research_executor
    with _api_research_executor_lock:
        executor = _api_research_executor
        _api_research_executor = None
    if executor is not None:
        executor.shutdown(wait=True, cancel_futures=True)


atexit.register(_shutdown_api_research_executor)

# Model discovery calls each configured provider in turn, and each blocking
# list-models call can run until its own timeout. A client disconnect does
# not cancel the request, so page reloads against a slow provider must not
# fill the default executor or the research workers. A full pool delays only
# other model discovery.
_MODEL_DISCOVERY_WORKERS = 8
_model_discovery_executor: ThreadPoolExecutor | None = None
_model_discovery_executor_lock = threading.Lock()


def _get_model_discovery_executor() -> ThreadPoolExecutor:
    global _model_discovery_executor
    with _model_discovery_executor_lock:
        if _model_discovery_executor is None:
            _model_discovery_executor = ThreadPoolExecutor(
                max_workers=_MODEL_DISCOVERY_WORKERS,
                thread_name_prefix="ldr-model-discovery",
            )
        return _model_discovery_executor


def _shutdown_model_discovery_executor() -> None:
    global _model_discovery_executor
    with _model_discovery_executor_lock:
        executor = _model_discovery_executor
        _model_discovery_executor = None
    if executor is not None:
        executor.shutdown(wait=True, cancel_futures=True)


atexit.register(_shutdown_model_discovery_executor)


def wrap_sync_route_with_cleanup(dependant: Dependant) -> None:
    """Wrap a synchronous FastAPI endpoint in an owner-thread cleanup.

    Mutates ``dependant.call`` in place; does not return the wrapped
    callable.  Takes the ``Dependant`` (rather than the bare callable) so the
    sync/async/generator classification below can use FastAPI's own
    ``is_coroutine_callable`` / ``is_gen_callable`` / ``is_async_gen_callable``
    (``fastapi/dependencies/models.py``) instead of ``inspect.iscoroutinefunction``
    et al. Those ``inspect`` functions do NOT follow a ``functools.wraps``
    ``__wrapped__`` chain; FastAPI's own predicates do (via
    ``inspect.unwrap``), and FastAPI dispatches based on its own predicates
    -- so a hypothetical ``functools.wraps``-decorated sync passthrough over
    an ``async def`` endpoint would be misclassified as sync by ``inspect.*``,
    wrapped here, and then get *awaited* by FastAPI's dispatcher (a 500).
    Unreachable today (no such endpoint exists in this app, and slowapi's
    rate-limit decorator preserves sync/async-ness), but free to close and
    it removes the divergence from FastAPI's own model.

    These three properties are ``cached_property``s on ``Dependant``,
    memoized from whatever ``dependant.call`` is at first access. FastAPI's
    own ``APIRoute.__init__`` (via ``get_route_handler`` ->
    ``get_request_handler``) already reads ``is_coroutine_callable`` while
    building the route, before this function ever runs -- so by the time
    this runs (either from ``WorkerCleanupAPIRoute.__init__`` for direct
    routes, or from the post-``include_router`` sweep in
    ``prepare_router_for_worker_cleanup``), the value is already pinned to
    the real, original endpoint and is unaffected by this function
    subsequently replacing ``dependant.call`` with the cleanup wrapper.

    FastAPI executes plain ``def`` endpoints on an AnyIO worker, while ASGI
    middleware teardown runs on the event-loop thread.  Database sessions are
    thread-local, so middleware cannot release the endpoint worker's sessions.
    This wrapper's ``finally`` runs on that worker and restores Flask's former
    request-teardown ownership model.

    Deliberately NOT a ``yield``-dependency and NOT a ``BackgroundTask``.
    Both would move cleanup to a *second*, separate
    ``anyio.to_thread.run_sync`` dispatch, and anyio gives no thread
    affinity between separate dispatches — the documented hazard in
    ``get_db_session_dep`` (``web/dependencies/auth.py``) for exactly that
    split. Cleanup has to happen inside the SAME worker dispatch that ran
    the endpoint, which is why this wraps ``dependant.call`` itself rather
    than framing the boundary with FastAPI's own request-lifecycle hooks.

    That constraint also decides *where inside this one dispatch* cleanup
    must sit. FastAPI does not serialize a plain (non-``Response``) return
    value inside ``dependant.call``: ``routing.py``'s route handler calls
    ``serialize_response``/``jsonable_encoder`` on the raw return value
    *after* ``dependant.call`` has already returned, back on the event-loop
    thread. Releasing the session before returning — the natural place for
    a ``finally``/``with`` cleanup — would therefore free this worker's DB
    session (and SQLCipher passphrase) before that encoding step runs. Any
    lazily loaded ORM attribute touched only during encoding would then hit
    a closed session (``DetachedInstanceError``) or silently serialize as
    incomplete data instead of raising.
    ``JSONResponse``, ``HTMLResponse``, ``TemplateResponse`` and
    ``RedirectResponse`` are unaffected: they render their body
    synchronously in ``__init__``, i.e. before the endpoint returns them
    here, so the bytes are already materialized. ``FileResponse`` is the
    exception, NOT covered by that claim: its ``__init__`` only sets
    headers (starlette ``responses.py``); the file is opened and streamed
    later, in its async ``__call__``, i.e. AFTER this wrapper's cleanup.
    ``favicon`` and ``serve_static`` (``web/fastapi_app.py``) are
    ``async def``, so this sync-only wrapper never touches them. The
    legacy ``redirect_static`` shim (``web/routers/research.py``) is a
    plain ``def`` that returns one and so does pass through here; that is
    safe only because its body is a static file opened by path, needing no
    DB session or other resource this cleanup releases. Do not extend the
    "renders in ``__init__``" claim to ``FileResponse``: any sync route
    returning one must likewise not depend on request-scoped resources.
    Everything else (a plain dict, list, Pydantic model, ...) gets encoded
    here, on this worker, before cleanup runs. This is NOT, contrary to an
    earlier version of this docstring, simply moving forward a
    ``jsonable_encoder``-only fallback that every route in this app takes.
    ``APIRoute.__init__`` derives ``response_model`` from a route's return
    *type annotation* whenever one isn't given explicitly
    (``fastapi/routing.py``, ~840-905), and a bare ``-> Any`` annotation
    (21 routes in ``news_flask_api.py``, 14 of them synchronous and so
    routed through this wrapper) is truthy, so those routes DO get a
    ``response_field`` -- "no route declares a ``response_model``" is
    false. With a ``response_field`` present, FastAPI's own
    ``serialize_response`` (``routing.py``) takes the
    ``field.validate()`` + ``field.serialize_json()`` branch instead of the
    bare ``jsonable_encoder`` branch -- but that still runs, on the
    event-loop thread, on WHATEVER this wrapper already handed back as this
    worker's return value. Because that value already went through
    ``jsonable_encoder`` here, ``field.validate``/``serialize_json`` sees
    already-plain JSON primitives, not the original rich Python objects, so
    it has nothing left to reformat -- ``jsonable_encoder`` running first
    effectively wins the two encoders' formatting differences rather than
    being replaced by the second one. Measured deltas from running
    ``jsonable_encoder`` first: an aware ``datetime`` serializes as
    ``...+00:00`` (``datetime.isoformat()``, used by ``jsonable_encoder``)
    instead of pydantic's compact ``...Z``; ``Decimal("1.50")`` becomes the
    float ``1.5`` instead of a decimal-preserving pydantic encoding;
    ``timedelta`` becomes a float of total seconds instead of pydantic's
    ISO-8601 duration string. No route in this app hits this today -- every
    datetime returned from one of these routes is already
    ``.isoformat()``'d before it reaches this wrapper -- but nothing pins
    that; see ``tests/web/dependencies/test_threadpool.py`` for a
    regression test on one such route's serialized shape.
    """
    fn = dependant.call
    if fn is None or getattr(fn, _WORKER_CLEANUP_MARKER, False):
        return

    # FastAPI streams generator endpoints after the endpoint call returns.
    # An ordinary ``finally`` would therefore clean up before iteration even
    # starts (and sync generator iterations are not worker-affine). There are
    # no generator endpoints in the production app; leave any future one
    # untouched here rather than apply an incorrect lifetime model.
    #
    # Skipping them silently is only safe because a dedicated assertion
    # watches for them: ``generator_routes == []`` in
    # ``tests/web/dependencies/test_threadpool.py``'s
    # ``test_production_app_wraps_every_sync_api_route``. That test's OTHER
    # census (``missing == []``) cannot cover this -- it classifies routes
    # by the very same ``is_gen_callable``/``is_async_gen_callable`` flags
    # read here, so anything this branch returns early on is excluded from
    # it by construction. Nor can
    # ``test_production_routers_do_not_use_bare_streaming_response``, which
    # AST-scans for a ``StreamingResponse`` construction a generator
    # endpoint never writes. If the ``generator_routes`` assertion is
    # dropped, a new ``def ...: yield`` route streams via a bare
    # ``StreamingResponse``/``iterate_in_threadpool`` with no per-chunk
    # cleanup -- the #6095 leak class -- and nothing says so.
    if (
        dependant.is_coroutine_callable
        or dependant.is_gen_callable
        or dependant.is_async_gen_callable
    ):
        return

    @functools.wraps(fn)
    def _wrapped(*args: Any, **kwargs: Any) -> Any:
        from ...database.thread_local_session import thread_cleanup

        with thread_cleanup():
            result = fn(*args, **kwargs)
            if isinstance(result, Response):
                return result
            # Force JSON encoding now, on the worker thread that still owns
            # the DB session this endpoint used, instead of leaving it for
            # FastAPI to do afterward on the event-loop thread with the
            # session already released.
            return jsonable_encoder(result)

    setattr(_wrapped, _WORKER_CLEANUP_MARKER, True)
    dependant.call = _wrapped


def iterate_sync_with_cleanup(iterable: Iterable[T]) -> Iterator[T]:
    """Iterate a synchronous response body with owner-worker cleanup.

    Starlette advances a synchronous ``StreamingResponse`` body through one
    ``anyio.to_thread.run_sync(next, ...)`` call per chunk.  Those calls happen
    *after* the endpoint has returned, and they are not guaranteed to use the
    endpoint's worker (or even the same worker for consecutive chunks).
    Therefore the endpoint wrapper cannot release sessions opened while the
    body is being produced.

    Wrapping each individual ``next()`` call gives every worker that advances
    the iterator its own deterministic cleanup boundary.  Cleanup happens
    before the chunk is handed back to the event loop, and when iteration
    finishes or raises INSIDE ``next()``.

    Once started (at least one ``next()`` has been pulled), closing or
    finalizing this wrapper throws ``GeneratorExit`` at ``yield item``,
    after the per-chunk cleanup block has exited. The handler below closes
    the wrapped iterator inside another cleanup boundary, covering its
    synchronously executed ``finally``. Work that this ``finally``
    schedules elsewhere is outside that boundary. Closing this wrapper
    before its first ``next()`` has no such boundary to throw into --
    Python marks an unstarted generator closed without ever running the
    handler below.

    That close handler is NOT worker-affine, despite running the same
    ``thread_cleanup()`` the per-chunk block does. This wrapper is a
    generator, and asyncio's async-generator/garbage-collection finalizer
    schedules the closing ``aclose()``/``close()`` on the EVENT LOOP
    thread, so ``thread_cleanup()`` there sweeps the loop thread rather
    than whichever worker last advanced the iterator. That is benign here:
    ``DatabaseMiddleware`` already sweeps the loop thread once per
    request, so the loop thread never accumulates sessions, and the
    per-chunk boundary above -- which does run on the advancing worker --
    is what actually covers the workers.

    This describes what happens WHEN the wrapper is closed, not when a
    client disconnect causes it to close. The pinned Starlette 1.3.1
    ``StreamingResponse`` does not explicitly ``aclose()`` its body
    iterator on a failed send; closure can depend on response/iterator
    finalization. The direct-close regression test therefore does not
    establish deterministic cleanup at disconnect. The synchronous route
    generators close their existing service objects in ``finally``;
    this wrapper protects that work once close/finalization occurs.
    """
    iterator: Iterator[T] | None = None
    try:
        while True:
            try:
                from ...database.thread_local_session import thread_cleanup

                with thread_cleanup():
                    if iterator is None:
                        iterator = iter(iterable)
                    item = next(iterator)
            except StopIteration:
                return
            yield item
    except GeneratorExit:
        # ``iterable`` is typed broadly (``Iterable[T]``); a plain list/tuple
        # iterator has no ``close()``. Only generators (the only thing the
        # five production callers ever pass) do, so guard the call rather
        # than assuming one exists. Those callers are ``library.py:909``
        # and ``:1342``, ``rag.py:1361`` and ``:3508``, and
        # ``research.py:2326``; ``test_threadpool.py``'s
        # ``test_production_routers_do_not_use_bare_streaming_response``
        # pins the count at ``>= 5``.
        close = getattr(iterator, "close", None)
        if close is not None:
            from ...database.thread_local_session import thread_cleanup

            with thread_cleanup():
                close()
        raise


class WorkerCleanupStreamingResponse(StreamingResponse):
    """Streaming response that cleans workers advancing synchronous bodies.

    Async iterables already run on the event-loop task and are covered by the
    request middleware.  Only synchronous iterables are handed to AnyIO
    workers, so only those need the per-iteration wrapper.
    """

    def __init__(self, content: Any, *args: Any, **kwargs: Any) -> None:
        if not isinstance(content, AsyncIterable):
            content = iterate_sync_with_cleanup(content)
        super().__init__(content, *args, **kwargs)


class WorkerCleanupAPIRoute(APIRoute):
    """APIRoute that cleans thread-local state after synchronous endpoints."""

    def __init__(self, path: str, endpoint: Callable[..., Any], **kwargs: Any):
        # Keep ``route.endpoint`` as the exact registered handler.  SlowAPI,
        # route-contract checks, and other FastAPI integrations use that
        # public attribute for metadata and identity.  Request dispatch goes
        # through ``route.dependant.call``, so wrapping that call target gives
        # us the worker-thread boundary without changing route identity.
        super().__init__(path, endpoint, **kwargs)
        if (
            self.dependant.call is None
        ):  # pragma: no cover - APIRoute rejects this first
            raise RuntimeError("FastAPI route has no dispatch callable")
        wrap_sync_route_with_cleanup(self.dependant)


def prepare_router_for_worker_cleanup(router: APIRouter) -> None:
    """Wrap sync dispatch targets on an already-populated router.

    ``FastAPI.include_router`` copies each source route and rebuilds its
    dependency graph, so this must run on the destination application's
    router *after* inclusion.  Mutating ``dependant.call`` is intentional:
    FastAPI dispatches through it while ``route.endpoint`` remains the exact
    handler registered by the module.
    """
    for route in router.routes:
        if isinstance(route, APIRoute):
            wrap_sync_route_with_cleanup(route.dependant)


async def _run_db_sync(
    fn: Callable[..., T],
    args: tuple[Any, ...],
    kwargs: dict[str, Any],
    executor: ThreadPoolExecutor | None,
) -> T:
    def _wrapped() -> T:
        try:
            return fn(*args, **kwargs)
        finally:
            # Avoid importing at module load — cleanup_current_thread
            # imports the SQLCipher engine bootstrap, which is too
            # heavy to run at every web import.
            try:
                from ...database.thread_local_session import (
                    cleanup_current_thread,
                )

                cleanup_current_thread()
            except Exception:
                # Never render a traceback here — this frame holds
                # credentials; see #6223. ``_wrapped`` closes over
                # ``args``, and one production caller passes a plaintext
                # DB password through it
                # (``socketio_asgi.py``: ``run_db_sync(
                # db_manager.open_user_database, username, password)``).
                # ``exc_info=True`` is inert under loguru's ``logger.debug``
                # and stays that way deliberately. Under
                # ``LDR_LOGURU_DIAGNOSE`` loguru renders the value of every
                # identifier on each traced frame's displayed source line,
                # and the stderr sink's default ``backtrace=True`` extends
                # the trace upward through the executor frames. Today no
                # displayed line names ``args`` (this frame's is the
                # ``cleanup_current_thread()`` call), but that is an accident
                # of which line raises, not a property anyone maintains.
                # Both the default and dedicated executors can change their
                # call frames; a traceback from a closure holding a password
                # is one refactor away from rendering it, and the
                # redaction-invariant test cannot catch this call site.
                # So: never ``logger.exception()`` here.
                logger.debug(
                    "run_db_sync: cleanup_current_thread failed",
                    exc_info=True,
                )

            # ``cleanup_current_thread`` only releases the thread-local DB
            # session. The worker thread is POOLED, so any other ambient
            # thread-local state set by ``fn`` outlives this call and is
            # visible to the next user's task on the same thread. Clear the
            # rest too, matching what ``thread_local_session.thread_cleanup``
            # does for dedicated worker threads.
            #
            # This runs inside ``_wrapped``, i.e. ON the worker thread, which
            # is the only place it can work: ``DatabaseMiddleware``'s own
            # cleanup runs in an ``async def`` on the event-loop thread and so
            # cannot reach the thread-locals of a pooled worker at all.
            try:
                from ...config.thread_settings import clear_settings_context

                clear_settings_context()
            except Exception:
                # Never render a traceback here — this frame holds
                # credentials; see #6223 and the handler above.
                logger.debug(
                    "run_db_sync: clear_settings_context failed",
                    exc_info=True,
                )

            # Same reasoning for the egress audit context, which was the one
            # piece of per-thread state this cleanup did not cover. Verified
            # leaking: a context armed inside a run_db_sync task persisted on
            # the pooled worker and was inherited by every later task on it.
            #
            # Not currently exploitable — the only arming site reachable from
            # here (``analyze_topic``) clears in its own ``finally`` — and the
            # leak direction is fail-closed, since the hook only gates
            # PRIVATE_ONLY/STRICT, so a stale context over-blocks a later
            # request rather than permitting egress. Closed by construction
            # anyway, so a future arming site cannot quietly make it matter.
            try:
                from ...security.egress.audit_hook import clear_active_context

                clear_active_context()
            except Exception:
                # Never render a traceback here — this frame holds
                # credentials; see #6223 and the first handler above.
                logger.debug(
                    "run_db_sync: clear_active_context failed",
                    exc_info=True,
                )

    if executor is None:
        return await asyncio.to_thread(_wrapped)
    # run_in_executor does not copy contextvars as asyncio.to_thread does.
    # The research pipeline needs its request owner and egress scope there.
    context = copy_context()
    return await asyncio.get_running_loop().run_in_executor(
        executor, context.run, _wrapped
    )


async def run_db_sync(fn: Callable[..., T], /, *args: Any, **kwargs: Any) -> T:
    """Run short sync DB work on the default executor with worker cleanup.

    Use this in place of ``await asyncio.to_thread(fn, ...)`` when ``fn``
    opens a per-user DB session. Long research calls use
    :func:`run_research_sync` and model discovery uses
    :func:`run_model_discovery_sync`, so they cannot starve authenticated
    requests.
    """
    return await _run_db_sync(fn, args, kwargs, None)


async def run_research_sync(
    fn: Callable[..., T], /, *args: Any, **kwargs: Any
) -> T:
    """Run long DB-touching research on its own eight-worker executor.

    Preserves the caller's contextvars and cleans worker thread-local state.
    """
    return await _run_db_sync(fn, args, kwargs, _get_api_research_executor())


async def run_model_discovery_sync(
    fn: Callable[..., T], /, *args: Any, **kwargs: Any
) -> T:
    """Run model discovery on its own eight-worker executor.

    Discovery can wait out one provider timeout after another, so it stays
    off the default executor and the research workers. Preserves the
    caller's contextvars and cleans worker thread-local state.
    """
    return await _run_db_sync(fn, args, kwargs, _get_model_discovery_executor())


__all__ = [
    "WorkerCleanupAPIRoute",
    "WorkerCleanupStreamingResponse",
    "iterate_sync_with_cleanup",
    "prepare_router_for_worker_cleanup",
    "run_db_sync",
    "run_model_discovery_sync",
    "run_research_sync",
    "wrap_sync_route_with_cleanup",
]