~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Helpers for running synchronous DB-touching code from async route
handlers without leaking thread-local state.
``DatabaseMiddleware`` cleans up the event-loop thread's DB session at
the end of every request, but it cannot reach into worker threads
spawned by ``asyncio.to_thread``. ThreadPoolExecutor workers are
reused across tasks, so a session opened by one user's task would stay
attached to that worker — the next task that lands on it would inherit
the previous user's session (memory bloat at best, cross-user state
mixing at worst).
``WorkerCleanupAPIRoute`` supplies that owner-thread boundary for plain
``def`` endpoints. ``WorkerCleanupStreamingResponse`` supplies it for each
worker that advances a synchronous response body. ``run_db_sync``,
``run_research_sync`` and ``run_model_discovery_sync`` do the same for
explicit DB work dispatched from asynchronous endpoints. Long research calls
and model discovery each use a separate executor so they cannot fill the
default executor used by authenticated requests.
"""
from __future__ import annotations
import asyncio
import atexit
import functools
import threading
from collections.abc import AsyncIterable, Iterable, Iterator
from concurrent.futures import ThreadPoolExecutor
from contextvars import copy_context
from typing import TYPE_CHECKING, Any, Callable, TypeVar
from fastapi.encoders import jsonable_encoder
from fastapi.routing import APIRoute, APIRouter
from loguru import logger
from starlette.responses import Response, StreamingResponse
if TYPE_CHECKING:
from fastapi.dependencies.models import Dependant
T = TypeVar("T")
_WORKER_CLEANUP_MARKER = "__ldr_worker_thread_cleanup__"
# Long API v1 research calls must not occupy asyncio's default executor:
# every authenticated request uses that executor to open/check its user DB.
# Keep this pool separate from the short DB dispatches in run_db_sync.
_API_RESEARCH_WORKERS = 8
_api_research_executor: ThreadPoolExecutor | None = None
_api_research_executor_lock = threading.Lock()
def _get_api_research_executor() -> ThreadPoolExecutor:
global _api_research_executor
with _api_research_executor_lock:
if _api_research_executor is None:
_api_research_executor = ThreadPoolExecutor(
max_workers=_API_RESEARCH_WORKERS,
thread_name_prefix="ldr-api-research",
)
return _api_research_executor
def _shutdown_api_research_executor() -> None:
global _api_research_executor
with _api_research_executor_lock:
executor = _api_research_executor
_api_research_executor = None
if executor is not None:
executor.shutdown(wait=True, cancel_futures=True)
atexit.register(_shutdown_api_research_executor)
# Model discovery calls each configured provider in turn, and each blocking
# list-models call can run until its own timeout. A client disconnect does
# not cancel the request, so page reloads against a slow provider must not
# fill the default executor or the research workers. A full pool delays only
# other model discovery.
_MODEL_DISCOVERY_WORKERS = 8
_model_discovery_executor: ThreadPoolExecutor | None = None
_model_discovery_executor_lock = threading.Lock()
def _get_model_discovery_executor() -> ThreadPoolExecutor:
global _model_discovery_executor
with _model_discovery_executor_lock:
if _model_discovery_executor is None:
_model_discovery_executor = ThreadPoolExecutor(
max_workers=_MODEL_DISCOVERY_WORKERS,
thread_name_prefix="ldr-model-discovery",
)
return _model_discovery_executor
def _shutdown_model_discovery_executor() -> None:
global _model_discovery_executor
with _model_discovery_executor_lock:
executor = _model_discovery_executor
_model_discovery_executor = None
if executor is not None:
executor.shutdown(wait=True, cancel_futures=True)
atexit.register(_shutdown_model_discovery_executor)
def wrap_sync_route_with_cleanup(dependant: Dependant) -> None:
"""Wrap a synchronous FastAPI endpoint in an owner-thread cleanup.
Mutates ``dependant.call`` in place; does not return the wrapped
callable. Takes the ``Dependant`` (rather than the bare callable) so the
sync/async/generator classification below can use FastAPI's own
``is_coroutine_callable`` / ``is_gen_callable`` / ``is_async_gen_callable``
(``fastapi/dependencies/models.py``) instead of ``inspect.iscoroutinefunction``
et al. Those ``inspect`` functions do NOT follow a ``functools.wraps``
``__wrapped__`` chain; FastAPI's own predicates do (via
``inspect.unwrap``), and FastAPI dispatches based on its own predicates
-- so a hypothetical ``functools.wraps``-decorated sync passthrough over
an ``async def`` endpoint would be misclassified as sync by ``inspect.*``,
wrapped here, and then get *awaited* by FastAPI's dispatcher (a 500).
Unreachable today (no such endpoint exists in this app, and slowapi's
rate-limit decorator preserves sync/async-ness), but free to close and
it removes the divergence from FastAPI's own model.
These three properties are ``cached_property``s on ``Dependant``,
memoized from whatever ``dependant.call`` is at first access. FastAPI's
own ``APIRoute.__init__`` (via ``get_route_handler`` ->
``get_request_handler``) already reads ``is_coroutine_callable`` while
building the route, before this function ever runs -- so by the time
this runs (either from ``WorkerCleanupAPIRoute.__init__`` for direct
routes, or from the post-``include_router`` sweep in
``prepare_router_for_worker_cleanup``), the value is already pinned to
the real, original endpoint and is unaffected by this function
subsequently replacing ``dependant.call`` with the cleanup wrapper.
FastAPI executes plain ``def`` endpoints on an AnyIO worker, while ASGI
middleware teardown runs on the event-loop thread. Database sessions are
thread-local, so middleware cannot release the endpoint worker's sessions.
This wrapper's ``finally`` runs on that worker and restores Flask's former
request-teardown ownership model.
Deliberately NOT a ``yield``-dependency and NOT a ``BackgroundTask``.
Both would move cleanup to a *second*, separate
``anyio.to_thread.run_sync`` dispatch, and anyio gives no thread
affinity between separate dispatches — the documented hazard in
``get_db_session_dep`` (``web/dependencies/auth.py``) for exactly that
split. Cleanup has to happen inside the SAME worker dispatch that ran
the endpoint, which is why this wraps ``dependant.call`` itself rather
than framing the boundary with FastAPI's own request-lifecycle hooks.
That constraint also decides *where inside this one dispatch* cleanup
must sit. FastAPI does not serialize a plain (non-``Response``) return
value inside ``dependant.call``: ``routing.py``'s route handler calls
``serialize_response``/``jsonable_encoder`` on the raw return value
*after* ``dependant.call`` has already returned, back on the event-loop
thread. Releasing the session before returning — the natural place for
a ``finally``/``with`` cleanup — would therefore free this worker's DB
session (and SQLCipher passphrase) before that encoding step runs. Any
lazily loaded ORM attribute touched only during encoding would then hit
a closed session (``DetachedInstanceError``) or silently serialize as
incomplete data instead of raising.
``JSONResponse``, ``HTMLResponse``, ``TemplateResponse`` and
``RedirectResponse`` are unaffected: they render their body
synchronously in ``__init__``, i.e. before the endpoint returns them
here, so the bytes are already materialized. ``FileResponse`` is the
exception, NOT covered by that claim: its ``__init__`` only sets
headers (starlette ``responses.py``); the file is opened and streamed
later, in its async ``__call__``, i.e. AFTER this wrapper's cleanup.
``favicon`` and ``serve_static`` (``web/fastapi_app.py``) are
``async def``, so this sync-only wrapper never touches them. The
legacy ``redirect_static`` shim (``web/routers/research.py``) is a
plain ``def`` that returns one and so does pass through here; that is
safe only because its body is a static file opened by path, needing no
DB session or other resource this cleanup releases. Do not extend the
"renders in ``__init__``" claim to ``FileResponse``: any sync route
returning one must likewise not depend on request-scoped resources.
Everything else (a plain dict, list, Pydantic model, ...) gets encoded
here, on this worker, before cleanup runs. This is NOT, contrary to an
earlier version of this docstring, simply moving forward a
``jsonable_encoder``-only fallback that every route in this app takes.
``APIRoute.__init__`` derives ``response_model`` from a route's return
*type annotation* whenever one isn't given explicitly
(``fastapi/routing.py``, ~840-905), and a bare ``-> Any`` annotation
(21 routes in ``news_flask_api.py``, 14 of them synchronous and so
routed through this wrapper) is truthy, so those routes DO get a
``response_field`` -- "no route declares a ``response_model``" is
false. With a ``response_field`` present, FastAPI's own
``serialize_response`` (``routing.py``) takes the
``field.validate()`` + ``field.serialize_json()`` branch instead of the
bare ``jsonable_encoder`` branch -- but that still runs, on the
event-loop thread, on WHATEVER this wrapper already handed back as this
worker's return value. Because that value already went through
``jsonable_encoder`` here, ``field.validate``/``serialize_json`` sees
already-plain JSON primitives, not the original rich Python objects, so
it has nothing left to reformat -- ``jsonable_encoder`` running first
effectively wins the two encoders' formatting differences rather than
being replaced by the second one. Measured deltas from running
``jsonable_encoder`` first: an aware ``datetime`` serializes as
``...+00:00`` (``datetime.isoformat()``, used by ``jsonable_encoder``)
instead of pydantic's compact ``...Z``; ``Decimal("1.50")`` becomes the
float ``1.5`` instead of a decimal-preserving pydantic encoding;
``timedelta`` becomes a float of total seconds instead of pydantic's
ISO-8601 duration string. No route in this app hits this today -- every
datetime returned from one of these routes is already
``.isoformat()``'d before it reaches this wrapper -- but nothing pins
that; see ``tests/web/dependencies/test_threadpool.py`` for a
regression test on one such route's serialized shape.
"""
fn = dependant.call
if fn is None or getattr(fn, _WORKER_CLEANUP_MARKER, False):
return
# FastAPI streams generator endpoints after the endpoint call returns.
# An ordinary ``finally`` would therefore clean up before iteration even
# starts (and sync generator iterations are not worker-affine). There are
# no generator endpoints in the production app; leave any future one
# untouched here rather than apply an incorrect lifetime model.
#
# Skipping them silently is only safe because a dedicated assertion
# watches for them: ``generator_routes == []`` in
# ``tests/web/dependencies/test_threadpool.py``'s
# ``test_production_app_wraps_every_sync_api_route``. That test's OTHER
# census (``missing == []``) cannot cover this -- it classifies routes
# by the very same ``is_gen_callable``/``is_async_gen_callable`` flags
# read here, so anything this branch returns early on is excluded from
# it by construction. Nor can
# ``test_production_routers_do_not_use_bare_streaming_response``, which
# AST-scans for a ``StreamingResponse`` construction a generator
# endpoint never writes. If the ``generator_routes`` assertion is
# dropped, a new ``def ...: yield`` route streams via a bare
# ``StreamingResponse``/``iterate_in_threadpool`` with no per-chunk
# cleanup -- the #6095 leak class -- and nothing says so.
if (
dependant.is_coroutine_callable
or dependant.is_gen_callable
or dependant.is_async_gen_callable
):
return
@functools.wraps(fn)
def _wrapped(*args: Any, **kwargs: Any) -> Any:
from ...database.thread_local_session import thread_cleanup
with thread_cleanup():
result = fn(*args, **kwargs)
if isinstance(result, Response):
return result
# Force JSON encoding now, on the worker thread that still owns
# the DB session this endpoint used, instead of leaving it for
# FastAPI to do afterward on the event-loop thread with the
# session already released.
return jsonable_encoder(result)
setattr(_wrapped, _WORKER_CLEANUP_MARKER, True)
dependant.call = _wrapped
def iterate_sync_with_cleanup(iterable: Iterable[T]) -> Iterator[T]:
"""Iterate a synchronous response body with owner-worker cleanup.
Starlette advances a synchronous ``StreamingResponse`` body through one
``anyio.to_thread.run_sync(next, ...)`` call per chunk. Those calls happen
*after* the endpoint has returned, and they are not guaranteed to use the
endpoint's worker (or even the same worker for consecutive chunks).
Therefore the endpoint wrapper cannot release sessions opened while the
body is being produced.
Wrapping each individual ``next()`` call gives every worker that advances
the iterator its own deterministic cleanup boundary. Cleanup happens
before the chunk is handed back to the event loop, and when iteration
finishes or raises INSIDE ``next()``.
Once started (at least one ``next()`` has been pulled), closing or
finalizing this wrapper throws ``GeneratorExit`` at ``yield item``,
after the per-chunk cleanup block has exited. The handler below closes
the wrapped iterator inside another cleanup boundary, covering its
synchronously executed ``finally``. Work that this ``finally``
schedules elsewhere is outside that boundary. Closing this wrapper
before its first ``next()`` has no such boundary to throw into --
Python marks an unstarted generator closed without ever running the
handler below.
That close handler is NOT worker-affine, despite running the same
``thread_cleanup()`` the per-chunk block does. This wrapper is a
generator, and asyncio's async-generator/garbage-collection finalizer
schedules the closing ``aclose()``/``close()`` on the EVENT LOOP
thread, so ``thread_cleanup()`` there sweeps the loop thread rather
than whichever worker last advanced the iterator. That is benign here:
``DatabaseMiddleware`` already sweeps the loop thread once per
request, so the loop thread never accumulates sessions, and the
per-chunk boundary above -- which does run on the advancing worker --
is what actually covers the workers.
This describes what happens WHEN the wrapper is closed, not when a
client disconnect causes it to close. The pinned Starlette 1.3.1
``StreamingResponse`` does not explicitly ``aclose()`` its body
iterator on a failed send; closure can depend on response/iterator
finalization. The direct-close regression test therefore does not
establish deterministic cleanup at disconnect. The synchronous route
generators close their existing service objects in ``finally``;
this wrapper protects that work once close/finalization occurs.
"""
iterator: Iterator[T] | None = None
try:
while True:
try:
from ...database.thread_local_session import thread_cleanup
with thread_cleanup():
if iterator is None:
iterator = iter(iterable)
item = next(iterator)
except StopIteration:
return
yield item
except GeneratorExit:
# ``iterable`` is typed broadly (``Iterable[T]``); a plain list/tuple
# iterator has no ``close()``. Only generators (the only thing the
# five production callers ever pass) do, so guard the call rather
# than assuming one exists. Those callers are ``library.py:909``
# and ``:1342``, ``rag.py:1361`` and ``:3508``, and
# ``research.py:2326``; ``test_threadpool.py``'s
# ``test_production_routers_do_not_use_bare_streaming_response``
# pins the count at ``>= 5``.
close = getattr(iterator, "close", None)
if close is not None:
from ...database.thread_local_session import thread_cleanup
with thread_cleanup():
close()
raise
class WorkerCleanupStreamingResponse(StreamingResponse):
"""Streaming response that cleans workers advancing synchronous bodies.
Async iterables already run on the event-loop task and are covered by the
request middleware. Only synchronous iterables are handed to AnyIO
workers, so only those need the per-iteration wrapper.
"""
def __init__(self, content: Any, *args: Any, **kwargs: Any) -> None:
if not isinstance(content, AsyncIterable):
content = iterate_sync_with_cleanup(content)
super().__init__(content, *args, **kwargs)
class WorkerCleanupAPIRoute(APIRoute):
"""APIRoute that cleans thread-local state after synchronous endpoints."""
def __init__(self, path: str, endpoint: Callable[..., Any], **kwargs: Any):
# Keep ``route.endpoint`` as the exact registered handler. SlowAPI,
# route-contract checks, and other FastAPI integrations use that
# public attribute for metadata and identity. Request dispatch goes
# through ``route.dependant.call``, so wrapping that call target gives
# us the worker-thread boundary without changing route identity.
super().__init__(path, endpoint, **kwargs)
if (
self.dependant.call is None
): # pragma: no cover - APIRoute rejects this first
raise RuntimeError("FastAPI route has no dispatch callable")
wrap_sync_route_with_cleanup(self.dependant)
def prepare_router_for_worker_cleanup(router: APIRouter) -> None:
"""Wrap sync dispatch targets on an already-populated router.
``FastAPI.include_router`` copies each source route and rebuilds its
dependency graph, so this must run on the destination application's
router *after* inclusion. Mutating ``dependant.call`` is intentional:
FastAPI dispatches through it while ``route.endpoint`` remains the exact
handler registered by the module.
"""
for route in router.routes:
if isinstance(route, APIRoute):
wrap_sync_route_with_cleanup(route.dependant)
async def _run_db_sync(
fn: Callable[..., T],
args: tuple[Any, ...],
kwargs: dict[str, Any],
executor: ThreadPoolExecutor | None,
) -> T:
def _wrapped() -> T:
try:
return fn(*args, **kwargs)
finally:
# Avoid importing at module load — cleanup_current_thread
# imports the SQLCipher engine bootstrap, which is too
# heavy to run at every web import.
try:
from ...database.thread_local_session import (
cleanup_current_thread,
)
cleanup_current_thread()
except Exception:
# Never render a traceback here — this frame holds
# credentials; see #6223. ``_wrapped`` closes over
# ``args``, and one production caller passes a plaintext
# DB password through it
# (``socketio_asgi.py``: ``run_db_sync(
# db_manager.open_user_database, username, password)``).
# ``exc_info=True`` is inert under loguru's ``logger.debug``
# and stays that way deliberately. Under
# ``LDR_LOGURU_DIAGNOSE`` loguru renders the value of every
# identifier on each traced frame's displayed source line,
# and the stderr sink's default ``backtrace=True`` extends
# the trace upward through the executor frames. Today no
# displayed line names ``args`` (this frame's is the
# ``cleanup_current_thread()`` call), but that is an accident
# of which line raises, not a property anyone maintains.
# Both the default and dedicated executors can change their
# call frames; a traceback from a closure holding a password
# is one refactor away from rendering it, and the
# redaction-invariant test cannot catch this call site.
# So: never ``logger.exception()`` here.
logger.debug(
"run_db_sync: cleanup_current_thread failed",
exc_info=True,
)
# ``cleanup_current_thread`` only releases the thread-local DB
# session. The worker thread is POOLED, so any other ambient
# thread-local state set by ``fn`` outlives this call and is
# visible to the next user's task on the same thread. Clear the
# rest too, matching what ``thread_local_session.thread_cleanup``
# does for dedicated worker threads.
#
# This runs inside ``_wrapped``, i.e. ON the worker thread, which
# is the only place it can work: ``DatabaseMiddleware``'s own
# cleanup runs in an ``async def`` on the event-loop thread and so
# cannot reach the thread-locals of a pooled worker at all.
try:
from ...config.thread_settings import clear_settings_context
clear_settings_context()
except Exception:
# Never render a traceback here — this frame holds
# credentials; see #6223 and the handler above.
logger.debug(
"run_db_sync: clear_settings_context failed",
exc_info=True,
)
# Same reasoning for the egress audit context, which was the one
# piece of per-thread state this cleanup did not cover. Verified
# leaking: a context armed inside a run_db_sync task persisted on
# the pooled worker and was inherited by every later task on it.
#
# Not currently exploitable — the only arming site reachable from
# here (``analyze_topic``) clears in its own ``finally`` — and the
# leak direction is fail-closed, since the hook only gates
# PRIVATE_ONLY/STRICT, so a stale context over-blocks a later
# request rather than permitting egress. Closed by construction
# anyway, so a future arming site cannot quietly make it matter.
try:
from ...security.egress.audit_hook import clear_active_context
clear_active_context()
except Exception:
# Never render a traceback here — this frame holds
# credentials; see #6223 and the first handler above.
logger.debug(
"run_db_sync: clear_active_context failed",
exc_info=True,
)
if executor is None:
return await asyncio.to_thread(_wrapped)
# run_in_executor does not copy contextvars as asyncio.to_thread does.
# The research pipeline needs its request owner and egress scope there.
context = copy_context()
return await asyncio.get_running_loop().run_in_executor(
executor, context.run, _wrapped
)
async def run_db_sync(fn: Callable[..., T], /, *args: Any, **kwargs: Any) -> T:
"""Run short sync DB work on the default executor with worker cleanup.
Use this in place of ``await asyncio.to_thread(fn, ...)`` when ``fn``
opens a per-user DB session. Long research calls use
:func:`run_research_sync` and model discovery uses
:func:`run_model_discovery_sync`, so they cannot starve authenticated
requests.
"""
return await _run_db_sync(fn, args, kwargs, None)
async def run_research_sync(
fn: Callable[..., T], /, *args: Any, **kwargs: Any
) -> T:
"""Run long DB-touching research on its own eight-worker executor.
Preserves the caller's contextvars and cleans worker thread-local state.
"""
return await _run_db_sync(fn, args, kwargs, _get_api_research_executor())
async def run_model_discovery_sync(
fn: Callable[..., T], /, *args: Any, **kwargs: Any
) -> T:
"""Run model discovery on its own eight-worker executor.
Discovery can wait out one provider timeout after another, so it stays
off the default executor and the research workers. Preserves the
caller's contextvars and cleans worker thread-local state.
"""
return await _run_db_sync(fn, args, kwargs, _get_model_discovery_executor())
__all__ = [
"WorkerCleanupAPIRoute",
"WorkerCleanupStreamingResponse",
"iterate_sync_with_cleanup",
"prepare_router_for_worker_cleanup",
"run_db_sync",
"run_model_discovery_sync",
"run_research_sync",
"wrap_sync_route_with_cleanup",
]