~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
ASGI Socket.IO integration for FastAPI.
Replaces Flask-SocketIO's SocketIOService with python-socketio's AsyncServer
mounted as an ASGI sub-application. This is the pattern used by Open WebUI
(129k stars) and recommended by the python-socketio documentation.
Usage:
from .socketio_asgi import sio, socket_app, emit_to_subscribers
# Mount in FastAPI app:
app.mount("/ws", socket_app)
# Emit from anywhere (owner is required — see _subscriptions):
emit_to_subscribers("research_progress", research_id, data, owner=username)
"""
import asyncio
import contextvars
import hashlib
from typing import Any
import socketio
from loguru import logger
from ...constants import ResearchStatus
from ...utilities.resource_utils import safe_close
from ..research_state import get_active_research_snapshot
# Determine WebSocket CORS policy from env var
from ...settings.env_registry import get_env_setting
from ...security.log_sanitizer import redact_and_bound_for_log
def _resolve_socketio_cors(env_value: str | None) -> str | list[str] | None:
"""Resolve the WebSocket origin allowlist, never yielding ``[]``.
None (NOT []) is load-bearing: engine.io treats None as "derive the
same-origin whitelist from Host/X-Forwarded-Proto", whereas an empty
list DISABLES origin validation entirely (engineio checks
``if self.cors_allowed_origins != []`` before validating), i.e. [] is
effectively allow-all for WebSocket handshakes. Every degenerate
input (unset, empty, nothing but separators) therefore resolves to
None; ``*`` requires the explicit env opt-in. A ``*`` entry anywhere
inside a comma-separated list is treated as that same opt-in: engine.io
has no "allow-all plus named origins" mode, so a list containing ``*``
already accepts every Origin -- returning it as a list would silently
behave as allow-all while the startup log claims a finite allowlist.
"""
if env_value is None or not env_value.strip():
return None
if env_value.strip() == "*":
return "*"
origins = [o.strip() for o in env_value.split(",") if o.strip()]
if "*" in origins:
return "*"
return origins or None
_ws_origins_env = get_env_setting("security.websocket.allowed_origins")
# Plain assignment (not annotated): the startup-ordering freeze contract
# detects module-scope ast.Assign targets.
_socketio_cors = _resolve_socketio_cors(_ws_origins_env)
def _log_socketio_cors_policy(
cors: str | list[str] | None, env_value: str | None
) -> None:
"""Log the resolved WebSocket origin policy at a default-visible level.
The allow-all posture must never be hidden below the default INFO sink.
A ``*`` mixed with specific origins is logged at WARNING because it
almost certainly is a misconfiguration (the named origins have no
effect). Only the operator's own configured entries (stripped) are
logged, never request-supplied values.
"""
if cors is None:
logger.info(
"Socket.IO CORS: same-origin only (set LDR_SECURITY_WEBSOCKET_ALLOWED_ORIGINS to configure)"
)
elif cors == "*":
named = [
o
for o in (env_value or "").split(",")
if o.strip() and o.strip() != "*"
]
if named:
logger.warning(
"Socket.IO CORS: all origins allowed -- "
"LDR_SECURITY_WEBSOCKET_ALLOWED_ORIGINS contains '*' alongside "
f"specific origins {[o.strip() for o in named]}; engine.io has "
"no allow-all-plus-named mode, so the named origins have no "
"effect. Remove '*' to restrict to them."
)
else:
logger.info("Socket.IO CORS: all origins allowed ('*')")
else:
logger.info(f"Socket.IO CORS: restricted to {cors}")
def log_socketio_cors_policy() -> None:
"""Log the import-time-resolved WebSocket origin policy.
Deliberately NOT called at module import: this module is imported at
the top of ``utilities/log_utils.py``, i.e. while the package's
``logger.disable("local_deep_research")`` (``__init__.py``) is still in
effect and before ``config_logger()`` re-enables it, so an import-time
log line -- including the allow-all INFO/WARNING -- would be silently
dropped. The FastAPI lifespan startup calls this instead, once the
server is starting and logging is configured.
"""
_log_socketio_cors_policy(_socketio_cors, _ws_origins_env)
def _research_id_preview(research_id: Any) -> str:
"""Bounded preview of a client-sent ``research_id`` for log lines.
(#6938: the Socket.IO payload's id is any JSON value of up to the
engine.io buffer size, logged on the rejection paths before any
ownership match. Imported at call time so this module never imports
the routers package at import time.)
"""
from ..routers.notes import _log_value_preview
return _log_value_preview(research_id)
# What engine.io's ``_log_error_once`` receives for a repeat it will not
# emit (see ``_install_origin_rejection_logging``).
_ENGINEIO_SUPPRESSED_TEXT = "Rejected (repeat; text not rendered)."
# ``logging.INFO``: engine.io logs through the standard library, whose
# level numbers are fixed; this module itself logs only through loguru.
_STDLIB_INFO_LEVEL = 20
def _install_origin_rejection_logging(sio: "socketio.AsyncServer") -> bool:
"""Re-emit engine.io's silenced WebSocket origin rejections via loguru.
engine.io validates the Origin at handshake and calls
``_log_error_once('<origin> is not an accepted origin.', 'bad-origin')``,
but the server runs with ``logger=False`` so that message never surfaces —
the only symptom of a misconfigured WebSocket origin is a frozen progress
UI. Wrap that one call to log a WARNING (deduped per origin) pointing at the
fix. An Origin is a scheme+host, not PII. Best-effort: a no-op (returns
False) if engine.io internals change, so it can never break startup.
The dedup set is capped: the handshake is pre-auth and ``Origin`` is
attacker-controlled, so an unbounded set would be a memory-growth + log-
amplification vector. After ``cap`` distinct origins we stop tracking/warning
(an operator has more than enough signal by then).
"""
try:
eio = sio.eio
original = eio._log_error_once
except AttributeError:
logger.debug(
"Socket.IO: origin-rejection logging not installed "
"(engine.io internals changed); handshake rejections stay silent"
)
return False
warned: set[str] = set()
cap = 100
def _log_error_once(message, message_key):
# (#6938: the message embeds the client's Origin header, which
# has no length limit on the polling transport, so this module's
# warning and the text handed on to engine.io's own logger are the
# redacted, bounded form. Redaction still costs a few ms on
# adversarial input (``redact_and_bound_for_log`` scans up to 1,200
# characters for a 300-character result) and this path is pre-auth
# and not rate limited, so it runs only when that text is actually
# logged: for each of the first ``cap`` distinct messages, and when
# engine.io's logger will emit the line (the first time a key is
# seen, unless its level was lowered). The dedup key is a fixed-size SHA-256
# digest of the raw message, so deciding whether to warn needs no
# redaction; once the cap is reached it is not computed at all.)
is_text = isinstance(message, str)
redacted: list[str] = []
def _bounded() -> str:
if not redacted:
redacted.append(
redact_and_bound_for_log(message, 300)
if is_text
else type(message).__name__
)
return redacted[0]
if message_key == "bad-origin" and len(warned) < cap:
dedup_key = (
hashlib.sha256(
message.encode("utf-8", "surrogatepass")
).hexdigest()
if is_text
else type(message).__name__
)
if dedup_key not in warned:
warned.add(dedup_key)
logger.warning(
f"Socket.IO rejected a WebSocket handshake: {_bounded()} "
"Set LDR_SECURITY_WEBSOCKET_ALLOWED_ORIGINS to this origin "
"if it is your front-end; behind a TLS-terminating proxy, "
"also forward X-Forwarded-Proto so the same-origin check "
"sees https."
)
if not is_text:
return original(message, message_key)
if _engineio_will_log(message_key):
return original(_bounded(), message_key)
# engine.io drops this line (a repeat key, logged at INFO under its
# ERROR-level logger); hand it a constant so its bookkeeping runs
# unchanged without paying for a redaction nobody reads.
return original(_ENGINEIO_SUPPRESSED_TEXT, message_key)
def _engineio_will_log(message_key) -> bool:
"""Whether engine.io's ``_log_error_once`` emits for this key.
It logs at ERROR the first time a key is seen and at INFO after
that (``engineio.base_server``). Anything this cannot determine
counts as "will log", so the redacted text is passed on.
"""
try:
if message_key not in eio.log_message_keys:
return True
return bool(eio.logger.isEnabledFor(_STDLIB_INFO_LEVEL))
except Exception:
return True
eio._log_error_once = _log_error_once
return True
# Largest inbound Socket.IO message the server accepts, in bytes.
#
# Clients only ever send ``subscribe_to_research`` /
# ``unsubscribe_from_research`` (``{"research_id": <id>}``) plus engine.io
# ping/pong frames -- well under 1 KiB each. 64 KiB leaves ample headroom
# while bounding what a single connection can make the server buffer.
#
# Enforced twice: uvicorn's ``ws_max_size`` (``web/app.py``) rejects an
# oversized WebSocket frame at the transport before it is fully read, and
# engine.io's ``max_http_buffer_size`` below bounds long-polling bodies and
# remains as the app-level check. ``web/app.py`` repeats the literal rather
# than importing it (it must not import this module before uvicorn starts);
# a test keeps the two in sync.
MAX_INBOUND_MESSAGE_BYTES = 64 * 1024
# Create the async Socket.IO server
sio = socketio.AsyncServer(
async_mode="asgi",
cors_allowed_origins=_socketio_cors,
logger=False,
engineio_logger=False,
ping_timeout=20,
ping_interval=5,
max_http_buffer_size=MAX_INBOUND_MESSAGE_BYTES,
)
# Make a rejected WebSocket origin diagnosable (otherwise it is a silent
# frozen progress UI). Skipped for the allow-all case, which rejects
# nothing.
if _socketio_cors != "*":
_install_origin_rejection_logging(sio)
# Create the ASGI app for mounting
# socketio_path must match how the client connects
socket_app = socketio.ASGIApp(sio, socketio_path="/ws/socket.io")
# Subscription tracking: (username, research_id) -> set of sids.
#
# Keyed by the OWNER as well as the id, never by the id alone. Research ids
# are UUIDs, but the benchmark page subscribes with a numeric
# ``BenchmarkRun.id``, and that id autoincrements inside each user's own
# encrypted database — so every user's first benchmark run is id 1.
#
# The ownership check in ``_owns_research_sync`` cannot catch this, and it is
# not wrong: it asks "does THIS user own this id in THEIR database?", and for
# two different users' run 1 the honest answer is yes for both. Keyed by the
# bare id, both then landed in ``_subscriptions["1"]`` and one user's
# benchmark progress was emitted to the other's browser. Reproduced:
# ``_subscriptions[1] = {'sid-alice', 'sid-bob'}``, with Bob receiving
# Alice's run data.
#
# So the id alone is not a key — a correct per-user check plus a global map
# keyed by a per-user id is still a cross-user leak, and it is invisible in
# either file on its own. See ADR-0009; ``routers/rag.py`` already keys
# ``_active_sse_indexers`` this way.
_subscriptions: dict[tuple[str, str], set[str]] = {}
# Authenticated socket sessions: sid -> username (resolved from session cookie at connect)
_sid_users: dict[str, str] = {}
# sid -> the auth session_id that authorised the handshake. Tracked so
# logout can sever only the sockets of the session being logged out,
# leaving the user's other tabs and devices connected (#5535). Without
# this, single-session logout would either leave every socket alive or
# have to disconnect all of them. Populated in `connect` and popped in
# `disconnect`, under the same `_lock` as `_sid_users`.
_sid_sessions: dict[str, str] = {}
# Eagerly initialised in init_lock() during lifespan startup. We can't
# create asyncio.Lock at import time because that binds it to whatever
# loop (or no loop) is running at import — typically wrong, and on
# Python 3.12+ raises during pre-startup imports. Initialising once in
# the lifespan hook (after the running loop exists) gives us a single
# Lock instance bound to the right loop and avoids the previous lazy
# double-checked-locking race in _get_lock(): two concurrent first-time
# callers could each create a fresh Lock, with the second write
# stomping the first instance and orphaning its awaiters.
_lock: asyncio.Lock | None = None
def init_lock() -> None:
"""Initialise the module-level asyncio.Lock. Call from lifespan
startup, after the event loop is running. Idempotent."""
global _lock
if _lock is None:
_lock = asyncio.Lock()
# Flag suppressing logging during an emit. Set by the log sink itself so an
# emit's own log lines cannot recurse back into the sink that triggered them.
#
# This MUST be a ContextVar, not a threading.local(). The emit no longer runs
# on the caller's thread: `emit_to_subscribers` schedules a coroutine onto the
# event loop via `run_coroutine_threadsafe`, so a thread-local set on the
# calling thread is invisible to the coroutine, which then reads the default
# (True) and logs anyway. Worse, the caller's `finally` restores the previous
# value before the coroutine has even run, so the window closed early too.
#
# Losing suppression is not cosmetic — it is unbounded amplification. The only
# caller that disables logging is `frontend_progress_sink`, a loguru sink: a
# failed emit logs, that record carries the inherited research_id, the sink
# fires again, emits again, fails again. A single research log line was
# measured producing 500+ emits before a hard cap tripped.
#
# A ContextVar is exactly right here: `run_coroutine_threadsafe` snapshots the
# calling context, so the value at scheduling time propagates INTO the
# coroutine, while the caller's own reset stays confined to the caller. It
# also keeps the per-thread isolation the threading.local was introduced for,
# since contexts are per-thread and per-task.
_logging_enabled_var: contextvars.ContextVar[bool] = contextvars.ContextVar(
"socketio_logging_enabled", default=True
)
def _logging_is_enabled() -> bool:
return _logging_enabled_var.get()
# uvicorn's main event loop, captured at app startup. Background threads
# (research workers, log queue, scheduler) need this to schedule emits via
# `asyncio.run_coroutine_threadsafe`. Without it, `asyncio.get_event_loop()`
# from a worker thread either raises (Python 3.12+) or creates a fresh loop
# that the AsyncServer doesn't know about, so emits silently no-op.
_main_loop: asyncio.AbstractEventLoop | None = None
def set_main_loop(loop: asyncio.AbstractEventLoop) -> None:
"""Capture the running event loop so background threads can dispatch to it."""
global _main_loop
_main_loop = loop
def _get_main_loop() -> asyncio.AbstractEventLoop | None:
"""Return the captured main loop, or try to find it as a fallback."""
if _main_loop is not None and not _main_loop.is_closed():
return _main_loop
# Last-resort fallback: try the current running loop (works only if
# called from the main thread inside an async context).
try:
return asyncio.get_running_loop()
except RuntimeError:
return None
def _schedule_coroutine_threadsafe(coro, loop) -> None:
"""Hand *coro* to ``loop`` without leaking it on scheduling failure.
``asyncio.run_coroutine_threadsafe`` owns and eventually closes the
coroutine after it successfully queues the callback. During shutdown,
however, ``loop.call_soon_threadsafe`` can raise synchronously after the
caller has already created the coroutine. In that case ownership never
transfers, so close it here before preserving the original exception for
the public wrapper's best-effort error handling.
"""
try:
asyncio.run_coroutine_threadsafe(coro, loop)
except Exception:
safe_close(coro, "rejected Socket.IO coroutine")
raise
def _decode_session_cookie(cookie_header: str) -> dict | None:
"""Decode a Starlette SessionMiddleware cookie to extract session data.
Uses the same itsdangerous-based scheme Starlette uses, with the
project's SECRET_KEY. Returns None on any failure (no session, bad
signature, expired).
"""
if not cookie_header:
return None
try:
from http.cookies import SimpleCookie
import base64
import json
import itsdangerous
# Look up the session cookie value (same name as SessionMiddleware uses)
jar = SimpleCookie()
jar.load(cookie_header)
morsel = jar.get("session")
if morsel is None:
return None
cookie_value = morsel.value
from ..fastapi_app import SECRET_KEY
from ...security import get_security_default
signer = itsdangerous.TimestampSigner(SECRET_KEY)
# Must mirror SessionMiddleware's max_age so an expired/revoked
# session cookie that the HTTP path would reject does not
# authenticate a WebSocket connection forever.
max_age_seconds = (
get_security_default("security.session_remember_me_days", 30)
* 24
* 3600
)
unsigned = signer.unsign(cookie_value, max_age=max_age_seconds)
return json.loads(base64.b64decode(unsigned))
except Exception:
return None
@sio.event
async def connect(sid, environ, auth=None):
"""Handle client connection.
Parity with main's Flask-SocketIO handler: unauthenticated sockets
are REJECTED at the handshake (returning False refuses the
connection). Accepting them would hand every roomless broadcast —
e.g. parallel_search_started, which carries the searching user's
query text — to clients that never logged in. The JS client simply
reconnects after login, exactly as it did under Flask.
The user's DB engine is lazily opened here (race vs first XHR after
page load, server restart, idle eviction) so subscribe_to_research's
ownership check works immediately after connect.
"""
cookie_header = environ.get("HTTP_COOKIE", "")
session_data = _decode_session_cookie(cookie_header) or {}
username = session_data.get("username")
if not username:
logger.info(f"Rejected unauthenticated WebSocket connection from {sid}")
return False
# The cookie's signature only proves it was issued by us, not that the
# session behind it is still alive. Validate the session id against the
# server-side store, exactly as require_auth does for HTTP.
#
# Without this a captured or stale cookie opens a BRAND NEW authenticated
# socket after logout / password change / session expiry, and can then
# subscribe to that user's live research events. The `is_user_connected`
# check below is username-scoped, so it is true whenever ANY of the
# user's sessions has the database open -- another device, or an
# in-flight research run (logout deliberately leaves the DB open in that
# case). That is a common condition, not an edge case.
#
# This is main's `__session_authorizes` connect-time gate (#5535), which
# had no successor here: DatabaseMiddleware's `_enforce_session_revocation`
# cannot cover it, because that middleware returns early for non-HTTP
# scopes and additionally skips the "/ws/" prefix.
from ..auth.session_manager import (
SessionValidationUnavailableError,
session_manager,
)
handshake_session_id = session_data.get("session_id")
try:
handshake_valid = bool(handshake_session_id) and (
session_manager.validate_session(handshake_session_id) == username
)
except SessionValidationUnavailableError:
# The durable epoch could not be read (transient I/O). Refuse this
# handshake (fail closed) but leave the session intact; the client
# reconnects on its own.
logger.warning(
f"Rejected WebSocket connection for {username} from {sid}: "
"session could not be validated right now"
)
return False
if not handshake_valid:
logger.warning(
f"Rejected WebSocket connection for {username} from {sid}: "
"session is not valid server-side (logged out, expired, or "
"revoked)"
)
return False
from ...database.encrypted_db import db_manager
from ...database.rekey_lock import RekeyOperationalError
from ..dependencies.threadpool import run_db_sync
# The probe is not the microsecond dict lookup it once was: it routes
# through _acquire_current_connection_generation, which takes the
# per-user lifecycle lock and the cross-process rekey sidecar
# (filesystem I/O, and a wait of up to a few seconds for another
# holder). Run it through run_db_sync exactly like the
# open_user_database call below, so a rekey in progress holds up only
# this handshake's worker thread — not the event loop, which this
# single-worker deployment shares with every user's sockets and async
# routes.
try:
user_connected = await run_db_sync(
db_manager.is_user_connected, username
)
except RekeyOperationalError:
# The probe already reports a sidecar that cannot be opened or
# locked (fd exhaustion, EROFS, EACCES, ENOSPC), and an epoch read
# that fails, as a plain False. A failure to release the sidecar
# afterwards (unlock/close) raises RekeySidecarIOError instead:
# degrade that to "not connected" too — ensure_user_database's
# contract — so the handshake fails closed through the normal
# no-connection path instead of raising out of connect(). Either
# way, while an open/lock or epoch-read fault hides a cached
# engine, the lazy open below raises that fault before deriving a
# key, so the refused handshake costs no PBKDF2 derivation.
user_connected = False
if not user_connected:
from ...database.session_passwords import session_password_store
session_id = session_data.get("session_id")
password = (
session_password_store.get_session_password(username, session_id)
if session_id
else None
)
if not password:
logger.info(
f"Rejected WebSocket connection for {username}: "
"no active DB session and no stored password"
)
return False
try:
# SQLCipher open is sync I/O — keep it off the event loop.
#
# Use run_db_sync, not bare asyncio.to_thread: a cold
# open_user_database() call runs the phase-2 rekey path
# (_run_phase2_rekey -> rekey_user_indexes), which opens a
# get_user_db_session() unconditionally, before its own marker
# check (vector_stores/legacy_rekey.py). That leaves a thread-local
# session on whichever asyncio default-executor thread served
# this call, with no cleanup boundary. run_db_sync runs the sync
# call on that same default executor, then cleans up the
# thread-local DB session (and other ambient thread-local
# state) on that worker before returning -- same shape as the
# equivalent fix at fastapi_app.py's DatabaseMiddleware
# (ensure_user_database).
await run_db_sync(db_manager.open_user_database, username, password)
except Exception as e:
# Capture the type name only and log OUTSIDE the handler:
# logger.exception's diagnose=True traceback would render the
# `password` local, so the traceback must never be logged here.
open_failure = type(e).__name__
else:
open_failure = None
if open_failure is not None:
logger.error(
f"Lazy DB open failed for {username} at WebSocket "
f"connect: {open_failure}"
)
return False
async with _lock:
_sid_users[sid] = username
# Remember which auth session authorised this socket so logout can
# sever exactly this session's sockets (#5535). Always present here:
# the connect gate above rejects a cookie without a valid session id.
_sid_sessions[sid] = handshake_session_id
# Close the connect-vs-teardown race (main #5572, ported: main fixed this
# in web/services/socket_service.py, which this branch replaced, so the
# fix had no successor here and would otherwise have been lost).
#
# The gate above and this registration are not atomic with a concurrent
# logout or session expiry. `disconnect_session` severs a session's
# sockets by enumerating `_sid_sessions`; if that enumeration ran BEFORE
# the block above added this sid, the teardown never sees this socket. It
# would then sit registered with a dead session, still receiving that
# user's events -- including `settings_changed` -- until some other
# session ends.
#
# Re-validate after registering and fail closed. Returning False rejects
# the handshake, so python-socketio never fires our `disconnect` handler
# for this sid; the dict entries have to be removed here explicitly.
#
# This is correct only because teardown invalidates the session BEFORE
# enumerating sockets -- logout destroys the session first, then
# disconnects -- so a socket that registers after invalidation is
# guaranteed to observe the failed re-check.
try:
still_valid = (
session_manager.validate_session(handshake_session_id) == username
)
except SessionValidationUnavailableError:
# Cannot prove the session is still alive: fail closed like an
# invalidated one (the session record itself is kept).
still_valid = False
if not still_valid:
async with _lock:
_sid_users.pop(sid, None)
_sid_sessions.pop(sid, None)
logger.warning(
f"WebSocket connect/teardown race for {username}: session was "
f"invalidated while socket {sid} was registering; rejecting it"
)
return False
logger.info(f"Client connected: {sid} (user={username})")
return True
@sio.event
async def disconnect(sid):
"""Handle client disconnection.
The whole body is guarded, as main's ``__handle_disconnect`` was.
``AsyncServer._trigger_event`` does not swallow handler exceptions and
``_handle_disconnect`` calls ``manager.disconnect(sid, ...)`` -- which
removes the sid from engine.io's own room bookkeeping -- only *after*
this handler returns. So a raise from here would leave engine.io still
holding the sid while our maps are half-swept, which is the opposite of
what a disconnect is for.
"""
logger.info(f"Client {sid} disconnected")
try:
# Clean up subscriptions and identity for this client
async with _lock:
_sid_users.pop(sid, None)
_sid_sessions.pop(sid, None)
empty_keys = []
# Keys are (username, research_id) tuples; a disconnecting sid is
# dropped from every subscription set regardless of owner.
for sub_key, sids in _subscriptions.items():
sids.discard(sid)
if not sids:
empty_keys.append(sub_key)
for key in empty_keys:
del _subscriptions[key]
# Clean up thread-local database sessions
try:
from ...database.thread_local_session import (
cleanup_current_thread,
)
cleanup_current_thread()
except Exception:
logger.debug("Error cleaning up thread session on disconnect")
except Exception:
# logger.exception already attaches the traceback; interpolating the
# exception as well is redundant and the repo's custom-code check
# rejects it.
logger.exception(f"Error handling disconnect for {sid}")
def _owns_research_sync(username: str, research_id: str) -> bool:
"""Query the user's encrypted DB for research/benchmark ownership.
Synchronous (SQLAlchemy) — must be called via ``_user_owns_research``
(which offloads to a thread), never directly on the event loop.
Shared by on_subscribe and on_unsubscribe so both handlers apply the
exact same ownership rule.
"""
from ...database.models import ResearchHistory
from ...database.session_context import get_user_db_session
with get_user_db_session(username) as db_session:
if (
db_session.query(ResearchHistory).filter_by(id=research_id).first()
is not None
):
return True
# The benchmark page subscribes with a numeric BenchmarkRun.id,
# which has no matching ResearchHistory row. Without this branch
# benchmark live-progress subscriptions are silently rejected.
if str(research_id).isdigit():
from ...database.models.benchmark import BenchmarkRun
return (
db_session.query(BenchmarkRun.id)
.filter(BenchmarkRun.id == research_id)
.first()
is not None
)
return False
def _subscription_key(owner: str, research_id) -> tuple:
"""Key under which ``research_id``'s subscriptions are stored.
Ported from main's ``SocketIOService.__subscription_key`` (#5600) while
merging that security fix in. This branch already keyed subscriptions by
``(owner, research_id)`` -- and more strictly than main, which composites
only numeric ids -- but it keyed on the RAW id, and main's version also
normalizes a numeric id to ``int``.
That normalization is load-bearing: subscribe-time ids arrive from a JSON
socket payload (so a benchmark run arrives as ``"1"``) while emit-time ids
come from the database (``1``). Keyed raw, those are two different dict
keys, and the event is silently never delivered to a subscriber who is
correctly subscribed. ``isdigit()`` guarantees the ``int()`` is safe.
UUID research ids are left exactly as they are.
"""
if str(research_id).isdigit():
return (owner, int(research_id))
return (owner, research_id)
async def _user_owns_research(username: str, research_id: str) -> bool:
"""Return True if `username` owns `research_id` (research or benchmark).
Offloaded to a thread — SQLAlchemy is sync and must not block the
uvicorn event loop that's serving every other WebSocket + HTTP
request. Used by both on_subscribe and on_unsubscribe as the shared
authorization boundary for the per-research subscription map.
"""
try:
# Use run_db_sync — _owns_research_sync opens get_user_db_session,
# so the worker's thread-local DB session needs cleanup before
# the next task lands on the same worker.
from ..dependencies.threadpool import run_db_sync
return await run_db_sync(_owns_research_sync, username, research_id)
except Exception:
logger.exception(
"Ownership check failed for user={} rid={}",
username,
_research_id_preview(research_id),
)
return False
async def _socket_session_still_valid(
sid: str, username: str
) -> tuple[str | None, bool | None]:
"""Re-check that this socket's originating session is still alive.
Returns ``(session_id, is_valid)``. The id comes back so the caller can
tear down EVERY socket of a dead session, not just the one that asked --
see the rejection path in ``on_subscribe``. ``is_valid`` is ``None``
when the session could not be validated right now (the durable epoch
was unreadable): the caller must refuse the action but NOT tear the
session's sockets down, since nothing is known to be wrong with it.
The connect-time gate is not sufficient on its own: identity is captured
into ``_sid_users`` at handshake and then frozen for the socket's whole
lifetime. A session that expires, is logged out, or has its password
changed WHILE the socket is open leaves that socket able to keep acting --
including subscribing to research it had not subscribed to before.
Logout and password change disconnect sockets actively. Idle expiry has a
bounded revocation-latency window until the periodic sweep (5 minutes in
production). Revalidating on subscribe and unsubscribe closes that window
whenever the socket next performs either action; the sweep remains the
fallback for an otherwise idle socket.
Fails closed: no recorded session id means the socket predates the gate,
which is not a state a current client can reach.
"""
async with _lock:
session_id = _sid_sessions.get(sid)
if not session_id:
return None, False
from ..auth.session_manager import (
SessionValidationUnavailableError,
session_manager,
)
try:
return (
session_id,
session_manager.validate_session(session_id) == username,
)
except SessionValidationUnavailableError:
return session_id, None
@sio.on("subscribe_to_research")
async def on_subscribe(sid, data):
"""Handle client subscription to research updates.
Verifies the requesting socket is authenticated AND that the
requested research_id belongs to that user. Without this check
any logged-in user could spy on any other user's research progress
in real time by guessing/enumerating UUIDs.
"""
research_id = data.get("research_id") if isinstance(data, dict) else None
if not research_id:
return
async with _lock:
username = _sid_users.get(sid)
if not username:
logger.warning(
"Rejected subscribe from unauthenticated sid {} for {}",
sid,
_research_id_preview(research_id),
)
await sio.emit(
"subscribe_error",
{"error": "Authentication required", "research_id": research_id},
room=sid,
)
return
stale_session_id, session_ok = await _socket_session_still_valid(
sid, username
)
if session_ok is None:
# Unverifiable right now (transient epoch-read failure): refuse
# this subscribe, keep the socket and its session.
logger.warning(
f"Rejected subscribe from {sid} ({username}): session could not "
"be validated right now"
)
await sio.emit(
"subscribe_error",
{
"error": "Temporarily unavailable",
"research_id": research_id,
},
room=sid,
)
return
if not session_ok:
logger.warning(
f"Rejected subscribe from {sid} ({username}): originating session "
"is no longer valid; disconnecting that session's sockets"
)
try:
await sio.emit(
"subscribe_error",
{"error": "Session expired", "research_id": research_id},
room=sid,
)
except Exception:
# Revocation is the security boundary; the explanatory frame is
# best-effort. A broken transport must not strand this socket (or
# its siblings) after session validation has already failed.
if _logging_is_enabled():
logger.debug(f"Error notifying stale socket {sid}")
# Sever EVERY socket of that session, not just this one.
#
# `validate_session` DELETES an expired session as a side effect of
# checking it, so by the time we get here the session is already gone
# from the store. Disconnecting only the calling sid would leave a
# sibling socket (a second tab on the same session) connected AND
# permanently un-revocable: the periodic idle sweep can never find
# that session again to tear it down, so the sibling keeps receiving
# the user's events -- including `settings_changed`, which carries
# plaintext secrets -- indefinitely. Main revoked per-session for
# exactly this reason.
if stale_session_id:
disconnect_session(stale_session_id)
else:
try:
await sio.disconnect(sid)
except Exception:
if _logging_is_enabled():
logger.debug(f"Error disconnecting stale socket {sid}")
return
# Verify ownership: the research must exist in this user's encrypted DB.
owns = await _user_owns_research(username, research_id)
if not owns:
logger.warning(
"Rejected subscribe: user {} does not own research {}",
username,
_research_id_preview(research_id),
)
await sio.emit(
"subscribe_error",
{"error": "Not authorized", "research_id": research_id},
room=sid,
)
return
async with _lock:
key = _subscription_key(username, research_id)
if key not in _subscriptions:
_subscriptions[key] = set()
_subscriptions[key].add(sid)
logger.info(
f"Client {sid} (user={username}) subscribed to research {research_id}"
)
# Send current status immediately if available
snapshot = get_active_research_snapshot(research_id)
if snapshot is not None:
progress = snapshot["progress"]
latest_log = snapshot["log"][-1] if snapshot["log"] else None
if latest_log:
await sio.emit(
f"research_progress_{research_id}",
{
"progress": progress,
"message": latest_log.get("message", "Processing..."),
"status": ResearchStatus.IN_PROGRESS,
"log_entry": latest_log,
},
room=sid,
)
@sio.on("unsubscribe_from_research")
async def on_unsubscribe(sid, data):
"""Handle client unsubscribe from research updates.
Applies the same ownership gate as on_subscribe before mutating the
per-research subscription set. This restores authorization-boundary
consistency with on_subscribe (matching main's own rationale for the
check) — it is not closing an active exploit: ``subs.discard(sid)``
can only ever remove the CALLER's own sid, so without this gate an
unauthorized unsubscribe attempt against another user's research_id
was already a silent no-op (that sid was never a member of the set),
and the victim's subscription was never affected.
Also re-validates the originating session, same as on_subscribe. This
reduces idle-expiry revocation latency whenever a socket next sends an
unsubscribe: an expired session is severed immediately instead of waiting
for the bounded periodic sweep. An otherwise idle socket is still handled
by that sweep.
"""
research_id = data.get("research_id") if isinstance(data, dict) else None
if not research_id:
return
async with _lock:
username = _sid_users.get(sid)
if not username:
logger.info(
"Rejected unsubscribe from unauthenticated sid {} for {}",
sid,
_research_id_preview(research_id),
)
return
stale_session_id, session_ok = await _socket_session_still_valid(
sid, username
)
if session_ok is None:
# Unverifiable right now: refuse this unsubscribe only.
logger.warning(
f"Rejected unsubscribe from {sid} ({username}): session could "
"not be validated right now"
)
return
if not session_ok:
logger.warning(
f"Rejected unsubscribe from {sid} ({username}): originating "
"session is no longer valid; disconnecting that session's sockets"
)
# Same rationale as the rejection branch in on_subscribe: sever
# EVERY socket of that session, not just this one. validate_session
# already deleted the session from the store as a side effect of
# checking it, so a sibling socket left connected would be
# permanently un-revocable -- the idle sweep can never find that
# session again to tear it down.
if stale_session_id:
disconnect_session(stale_session_id)
else:
try:
await sio.disconnect(sid)
except Exception:
if _logging_is_enabled():
logger.debug(f"Error disconnecting stale socket {sid}")
return
if not await _user_owns_research(username, research_id):
logger.info(
"Rejected unsubscribe from sid {}: user does not own research {}",
sid,
_research_id_preview(research_id),
)
return
async with _lock:
key = _subscription_key(username, research_id)
subs = _subscriptions.get(key)
if subs:
subs.discard(sid)
if not subs:
_subscriptions.pop(key, None)
def emit_socket_event(event: str, data: Any, room: str | None = None) -> bool:
"""Emit a socket event (sync wrapper for use from background threads).
Schedules the emit on uvicorn's main event loop via
`asyncio.run_coroutine_threadsafe`. Without the captured loop, emits
from worker threads silently no-op because `asyncio.get_event_loop()`
in a non-main thread on Python 3.12+ either raises or creates a new
isolated loop.
"""
loop = _get_main_loop()
if loop is None or not loop.is_running():
if _logging_is_enabled():
logger.debug(f"Cannot emit {event}: main event loop not available")
return False
try:
_schedule_coroutine_threadsafe(_async_emit(event, data, room), loop)
return True
except Exception:
if _logging_is_enabled():
logger.debug(f"Error emitting socket event {event}")
return False
def emit_to_user(event: str, username: str, data: Any) -> bool:
"""Emit a socket event to all sockets authenticated as `username`.
Prevents cross-user leaks for events like settings_changed — each
user's sockets get the event, nobody else's do.
"""
loop = _get_main_loop()
if loop is None or not loop.is_running():
if _logging_is_enabled():
logger.debug(f"Cannot emit {event}: main event loop not available")
return False
async def _emit() -> None:
# Snapshot sids for this user under the lock — connect/disconnect
# mutate _sid_users on this same event loop and could otherwise
# interleave at any await point, raising
# ``RuntimeError: dictionary changed size during iteration``.
async with _lock:
sids = [sid for sid, u in _sid_users.items() if u == username]
for sid in sids:
try:
await sio.emit(event, data, room=sid)
except Exception:
if _logging_is_enabled():
logger.debug(f"Error emitting {event} to {sid}")
try:
_schedule_coroutine_threadsafe(_emit(), loop)
return True
except Exception:
if _logging_is_enabled():
logger.debug(f"Error emitting {event} to user {username}")
return False
def disconnect_user(username: str) -> bool:
"""Disconnect every socket authenticated as ``username``.
A socket is authorised once, at handshake, and never re-checked. So
when a user's session ends — logout, password change, or the idle
sweep closing their database because they have no session left — any
socket they still hold keeps receiving their events indefinitely. This
severs them.
Port of ``SocketIOService.disconnect_user`` (#5535), which grouped
sockets with Socket.IO rooms. This layer already tracks identity
directly in ``_sid_users``, so it filters that instead of maintaining
a parallel per-user room; the snapshot is taken under ``_lock`` for
the same reason ``emit_to_user`` takes it — connect/disconnect mutate
``_sid_users`` on this event loop and would otherwise interleave at an
await point.
Dropping subscriptions is handled for free: ``sio.disconnect(sid)``
fires the ``disconnect`` handler above, which removes the sid from
``_sid_users`` and from every entry in ``_subscriptions``.
Best-effort and non-blocking, like the emit helpers: schedules onto
the main loop and returns whether it could be scheduled, not whether
the sockets are gone. Callers are teardown paths that must not block
or raise.
"""
return _disconnect_matching(
lambda: [sid for sid, u in _sid_users.items() if u == username],
f"user {username}",
)
def disconnect_session(session_id: str) -> bool:
"""Disconnect only the sockets authorised by ``session_id``.
Single-session teardown for logout: the tab being logged out loses its
sockets while the user's other still-valid sessions keep theirs. Port
of ``SocketIOService.disconnect_session`` (#5535), which used a
per-session Socket.IO room; this layer matches on the handshake
session_id recorded in ``_sid_sessions`` instead.
Use ``disconnect_user`` for the all-sessions cases — password change
and the idle sweep — where every session is destroyed anyway.
"""
return _disconnect_matching(
lambda: [sid for sid, s in _sid_sessions.items() if s == session_id],
f"session {session_id[:8]}...",
)
def _disconnect_matching(select_sids, description: str) -> bool:
"""Disconnect every sid returned by ``select_sids``.
``select_sids`` is called while holding ``_lock`` — connect/disconnect
mutate the sid maps on this event loop, so the selection must be a
snapshot taken under the lock rather than a live view, or iteration can
raise ``RuntimeError: dictionary changed size during iteration``.
"""
loop = _get_main_loop()
if loop is None or not loop.is_running():
if _logging_is_enabled():
logger.debug(
f"Cannot disconnect sockets for {description}: "
"main event loop not available"
)
return False
async def _disconnect() -> None:
async with _lock:
sids = select_sids()
for sid in sids:
try:
await sio.disconnect(sid)
except Exception:
if _logging_is_enabled():
logger.debug(f"Error disconnecting {sid}")
if sids:
logger.info(f"Disconnected {len(sids)} socket(s) for {description}")
try:
_schedule_coroutine_threadsafe(_disconnect(), loop)
return True
except Exception:
if _logging_is_enabled():
logger.debug(f"Error disconnecting sockets for {description}")
return False
def emit_to_subscribers(
event_base: str,
research_id: str,
data: Any,
*,
owner: str,
enable_logging: bool = True,
) -> bool:
"""Emit an event to all subscribers of a specific research.
Sync wrapper for use from background threads.
``owner`` is the username whose research this is, and is REQUIRED and
keyword-only on purpose: subscriptions are keyed by ``(owner,
research_id)`` because a benchmark id is only unique within one user's
database (see ``_subscriptions``). Making it required means a call site
that forgets it fails loudly at the call rather than silently reverting
to the id-only lookup that leaked one user's progress to another.
`enable_logging` is carried by a ContextVar so concurrent emits from
different worker threads don't corrupt each other's suppression windows
(as a module global did), AND so the suppression reaches the coroutine
scheduled onto the event loop below — a threading.local could not, since
that coroutine runs on a different thread. See `_logging_enabled_var`.
"""
token = None
if not enable_logging:
token = _logging_enabled_var.set(False)
try:
loop = _get_main_loop()
if loop is None or not loop.is_running():
if _logging_is_enabled():
logger.debug(
f"Cannot emit subscribers for {research_id}: loop unavailable"
)
return False
try:
_schedule_coroutine_threadsafe(
_async_emit_to_subscribers(
event_base, research_id, data, owner
),
loop,
)
return True
except Exception:
if _logging_is_enabled():
logger.debug(
f"Error emitting to subscribers for research {research_id}"
)
return False
finally:
# Reset only this caller's context. The coroutine scheduled above got
# its own snapshot taken while the flag was still False, so it keeps
# the suppression it needs regardless of this reset.
if token is not None:
_logging_enabled_var.reset(token)
async def _async_emit(event: str, data: Any, room: str | None = None) -> None:
"""Async emit implementation."""
try:
if room:
await sio.emit(event, data, room=room)
else:
await sio.emit(event, data)
except Exception:
# The concurrent Future returned by run_coroutine_threadsafe is
# intentionally discarded by the synchronous wrapper, so an
# exception escaping this coroutine would otherwise be silent.
if _logging_is_enabled():
logger.debug(f"Error emitting socket event {event}")
async def _async_emit_to_subscribers(
event_base: str, research_id: str, data: Any, owner: str
) -> None:
"""Async emit to subscribers implementation.
Looks up ``(owner, research_id)``. An unknown or wrong owner simply
matches no entry and the event is dropped — the map cannot be reached
with an id alone, so a caller that loses track of whose research it is
emitting delivers nothing rather than delivering to everyone.
"""
full_event = f"{event_base}_{research_id}"
async with _lock:
subscriptions = _subscriptions.get(
_subscription_key(owner, research_id)
)
if subscriptions:
subscriptions = subscriptions.copy()
else:
subscriptions = None
if subscriptions is not None:
from ..auth.session_manager import session_manager
for sid in subscriptions:
try:
await sio.emit(full_event, data, room=sid)
except Exception:
if _logging_is_enabled():
logger.debug(f"Error emitting to subscriber {sid}")
# Watching a run over the socket IS activity. Without this, a
# user who starts a long research and then just watches the
# progress UI issues no HTTP requests, so nothing refreshes
# their idle timer and the session is reaped mid-run -- the
# idle sweep then disconnects the socket too, so the run
# appears to die in front of them.
#
# This is the second half of main's #5535 (778688295). The
# disconnect_user/disconnect_session half was ported; this half
# was not, which is why touch_session had zero call sites here.
# Own try/except, as main has, so a refresh failure cannot
# strand delivery to the remaining subscribers.
try:
sid_session = _sid_sessions.get(sid)
if sid_session:
session_manager.touch_session(sid_session)
except Exception:
if _logging_is_enabled():
logger.debug(f"Error refreshing session for {sid}")
# No subscribers: drop the event. We must NOT broadcast — that would
# leak one user's research progress to every connected client.
def remove_subscriptions_for_research(research_id: str, owner: str) -> None:
"""Remove all socket subscriptions for a completed research.
``owner`` scopes the removal to the user whose research finished, for
the same reason subscriptions are keyed that way: without it, one
user's benchmark finishing would tear down every other user's
subscription to their own run of the same numeric id.
Sync function safe to call from any thread. Schedules the async
cleanup on the main loop. If the loop is not yet available (very
early startup) or has been closed (shutdown), the cleanup is
silently dropped — disconnect handlers will reap stale entries on
their own, so the worst case is a small temporary leak that does
not race with the async path.
"""
loop = _get_main_loop()
if loop is None or not loop.is_running():
logger.debug(
f"Skipping subscription cleanup for {research_id}: loop unavailable"
)
return
try:
_schedule_coroutine_threadsafe(
_async_remove_subscriptions(research_id, owner), loop
)
except Exception:
logger.debug(
f"Async cleanup scheduling failed for {research_id}", exc_info=True
)
async def _async_remove_subscriptions(research_id: str, owner: str) -> None:
"""Async subscription removal, scoped to the research's owner."""
async with _lock:
removed = _subscriptions.pop(
_subscription_key(owner, research_id), None
)
if removed is not None:
logger.info(
f"Removed {len(removed)} subscription(s) for research {research_id}"
)