~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Queue processor v2 - uses encrypted user databases instead of service.db
Supports both direct execution and queue modes.
"""
import atexit
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from enum import Enum, auto
from typing import Any, Dict, Optional
from loguru import logger
from sqlalchemy import or_, select
from sqlalchemy.orm import Session
from ...constants import ResearchStatus
from ...database.encrypted_db import db_manager
from ...database.models import (
QueuedResearch,
ResearchHistory,
TaskMetadata,
UserActiveResearch,
)
from ...database.queue_service import UserQueueService
from ...database.session_context import get_user_db_session
from ...database.session_passwords import session_password_store
from ...exceptions import (
DuplicateResearchError,
InvalidQueuedResearchOverridesError,
SystemAtCapacityError,
)
from ...security.log_sanitizer import redact_secrets
from ...notifications.queue_helpers import (
send_research_completed_notification_from_session,
send_research_failed_notification_from_session,
)
from .lifecycle_cleanup import (
cleanup_queued_research_state,
reconcile_research_queue_status,
)
from ..routes.research_validation import (
validate_research_query_length,
validate_search_overrides,
)
from ..research_state import get_user_research_start_lock
from ..services.research_service import (
_parse_research_metadata,
clamp_user_max_concurrent,
run_research_process,
start_research_process,
)
_AUTO_PERSIST_EXECUTOR: Optional[ThreadPoolExecutor] = None
_AUTO_PERSIST_LOCK = threading.Lock()
# Admission bound for completion sweeps: 2 workers run at once, and at
# most this many sweeps may be pending in total. Beyond it a sweep is
# skipped with an explicit log instead of queueing unbounded work.
_AUTO_PERSIST_MAX_PENDING = 8
_AUTO_PERSIST_PENDING = threading.Semaphore(_AUTO_PERSIST_MAX_PENDING)
def _auto_persist_enabled_for(username: str) -> bool:
"""Best-effort opt-in check used before enqueueing a sweep."""
try:
from ...research_library.services.download_service import (
_is_auto_persist_enabled,
)
return _is_auto_persist_enabled(None, username)
except Exception: # allow: silent-exception
return False
def _get_auto_persist_executor() -> ThreadPoolExecutor:
global _AUTO_PERSIST_EXECUTOR
if _AUTO_PERSIST_EXECUTOR is None:
with _AUTO_PERSIST_LOCK:
if _AUTO_PERSIST_EXECUTOR is None:
_AUTO_PERSIST_EXECUTOR = ThreadPoolExecutor(
max_workers=2,
thread_name_prefix="auto_persist_sweep",
)
return _AUTO_PERSIST_EXECUTOR
def _shutdown_auto_persist_executor() -> None:
global _AUTO_PERSIST_EXECUTOR
executor = _AUTO_PERSIST_EXECUTOR
_AUTO_PERSIST_EXECUTOR = None
if executor is not None:
try:
executor.shutdown(wait=False, cancel_futures=True)
except Exception: # allow: silent-exception
pass
def _reset_auto_persist_pending() -> None:
"""Reset the admission semaphore for testing."""
global _AUTO_PERSIST_PENDING
_AUTO_PERSIST_PENDING = threading.Semaphore(_AUTO_PERSIST_MAX_PENDING)
atexit.register(_shutdown_auto_persist_executor)
# Retry configuration constants for notification database queries
MAX_RESEARCH_LOOKUP_RETRIES = 3
INITIAL_RESEARCH_LOOKUP_DELAY = 0.5 # seconds
RETRY_BACKOFF_MULTIPLIER = 2
# Give up on a queued research after this many consecutive spawn failures.
# Each failure leaves is_processing=False so the next loop tick retries.
SPAWN_RETRY_LIMIT = 3
@dataclass(frozen=True, slots=True)
class QueueSweepResult:
can_dispatch: bool
cleaned_ids: frozenset[str]
class _DirectStartOutcome(Enum):
"""Outcome of the direct-start attempt used by the queue handoff.
``notify_research_queued`` must know whether it should return because the
research is running/terminal, or fall through to create queue metadata and
register the user for a later dispatch. A bare ``None`` return cannot make
that distinction and previously stranded rows on setup and global-capacity
failures.
"""
STARTED = auto()
ALREADY_RUNNING = auto()
QUEUE_FALLBACK = auto()
TERMINAL_FAILURE = auto()
class QueueProcessorV2:
"""
Processes queued researches using encrypted user databases.
This replaces the service.db approach.
"""
def __init__(self, check_interval=10):
"""
Initialize the queue processor.
Args:
check_interval: How often to check for work (seconds)
"""
self.check_interval = check_interval
self.running = False
self.thread = None
self._loop_iteration = 0
# Wakes the loop out of its inter-iteration wait so stop() returns
# in milliseconds instead of blocking for up to check_interval.
# The test suite stops the processor after every app test
# (tests/conftest.py reset_singletons), so a blocking stop() adds
# ~check_interval seconds of teardown to each of those tests.
self._stop_event = threading.Event()
# Per-user settings will be retrieved from each user's database
# when processing their queue using SettingsManager
logger.info(
"Queue processor v2 initialized - will use per-user settings from SettingsManager"
)
# Track which users we should check
self._users_to_check: set[tuple[str, str]] = set()
self._users_lock = threading.Lock()
# Track pending operations from background threads
self.pending_operations = {}
self._pending_operations_lock = threading.Lock()
# Compatibility view of the shared per-user admission locks. The
# canonical registry lives in research_state so fresh HTTP starts and
# queue replay use the SAME lock for count -> DB claim; a queue-local
# lock would still let those two entry points race each other.
#
# Size is bounded by distinct-usernames-ever-seen, not by session
# count. Entries are tiny (~40 bytes) and deliberately retain stable
# identity for the process lifetime; see pop_user_critical_lock().
self._user_critical_locks: Dict[str, threading.Lock] = {}
self._user_critical_locks_lock = threading.Lock()
# Count consecutive spawn failures per research_id. Entries are
# popped on success or after hitting SPAWN_RETRY_LIMIT (then the
# research is marked FAILED). In-memory is sufficient: a restart
# resets the counter and the research gets a fresh N retries,
# which is the desired behavior if the underlying system issue
# (thread pool, memory) cleared.
# Access is guarded by _spawn_retry_counts_lock because the
# increment path is a read-modify-write and the loop and direct
# request paths can interleave.
self._spawn_retry_counts: dict[str, int] = {}
self._spawn_retry_counts_lock = threading.Lock()
def _get_user_critical_lock(self, username: str) -> threading.Lock:
"""Get (or lazily create) the per-user lock used to serialise the
count-active-and-start-direct critical section for a given user.
"""
lock = get_user_research_start_lock(username)
with self._user_critical_locks_lock:
# Keep the compatibility/introspection view synchronized with the
# canonical research_state registry (tests may snapshot/restore
# that registry between cases).
self._user_critical_locks[username] = lock
return lock
def pop_user_critical_lock(self, username: str) -> None:
"""Compatibility no-op: per-user critical locks keep stable identity.
A caller receives the lock from ``_get_user_critical_lock`` before it
acquires it. Removing even an apparently-unlocked entry therefore has
an unavoidable lookup-to-acquire race: another request can retain the
old lock while a later request creates and acquires a second one. If a
held lock is removed, the race is immediate and the count-to-start
critical section is no longer serialised.
Keep one tiny lock per distinct username for the process lifetime,
matching the deliberately persistent rekey gate in ``research_state``.
The method remains as a no-op so existing user-close cleanup callers do
not need a special-case import path.
"""
del username
def _bump_spawn_retry_count(self, research_id: str) -> int:
"""Atomically increment and return the spawn-retry counter for
``research_id``. Extracted so tests can exercise the real
locked increment path instead of duplicating the lock in the
test worker (which would be a tautology).
"""
with self._spawn_retry_counts_lock:
attempts = self._spawn_retry_counts.get(research_id, 0) + 1
self._spawn_retry_counts[research_id] = attempts
return attempts
@staticmethod
def _commit_with_safe_rollback(db_session: Session, context: str) -> bool:
"""Commit ``db_session`` with best-effort rollback on failure.
Returns ``True`` on success, ``False`` if the commit raised.
The failure path logs via ``logger.exception`` and attempts a
rollback, itself guarded so a subsequent rollback failure is
logged at debug level rather than propagated.
Extracted because the ``try: commit / except: log + try:
rollback`` idiom repeats at ≥5 sites in this module; inlining
hides the defensive structure behind nested ``try`` blocks and
makes each callsite longer than the work it describes.
"""
try:
db_session.commit()
return True
except Exception:
logger.exception(f"Commit failed: {context}")
try:
db_session.rollback()
except Exception:
logger.debug(
f"Rollback after commit failure ({context})",
exc_info=True,
)
return False
def _delete_queue_row_safely(
self, db_session: Session, username: str, research_id: str
) -> None:
"""Best-effort delete of the ``QueuedResearch`` row for
``(username, research_id)``.
Rolls back any pending state first (the session may be in
``PendingRollbackError`` from a failed commit inside
``_start_research``), re-queries the row fresh, deletes it if
present, and commits via ``_commit_with_safe_rollback``.
Use this for ``DuplicateResearchError`` cleanup where the goal
is "drop the queue row regardless of session state." Do NOT
use it for paths that need the delete to be atomic with other
writes (e.g. the terminal FAILED path bundles
``ResearchHistory.status = FAILED`` with the queue-row delete
in a single commit — that stays inline).
"""
try:
db_session.rollback()
except Exception:
logger.debug(
f"Rollback before queue-row delete for {research_id}",
exc_info=True,
)
try:
fresh_queued = (
db_session.query(QueuedResearch)
.filter_by(username=username, research_id=research_id)
.first()
)
if fresh_queued:
db_session.delete(fresh_queued)
self._commit_with_safe_rollback(
db_session,
f"queue-row delete for research {research_id}",
)
except Exception:
logger.exception(
f"Failed to query/delete queue row for {research_id}"
)
try:
db_session.rollback()
except Exception:
logger.debug(
f"Rollback after queue-row delete failure for {research_id}",
exc_info=True,
)
def start(self):
"""Start the queue processor thread."""
if self.running:
logger.warning("Queue processor already running")
return
self.running = True
# Re-arm the wait for restart: stop() leaves the event set, and
# create_app() restarts this singleton after the test-suite
# teardown stops it.
self._stop_event.clear()
self.thread = threading.Thread(
target=self._process_queue_loop, daemon=True
)
self.thread.start()
logger.info("Queue processor v2 started")
def stop(self):
"""Stop the queue processor thread."""
self.running = False
# Wake the loop immediately rather than waiting for the next
# check_interval tick (otherwise shutdown blocks for up to 10s).
self._stop_event.set()
if self.thread:
self.thread.join(timeout=10)
logger.info("Queue processor v2 stopped")
def notify_user_activity(self, username: str, session_id: str):
"""
Notify that a user has activity and their queue should be checked.
Args:
username: The username
session_id: The Flask session ID (for password access)
"""
with self._users_lock:
self._users_to_check.add((username, session_id))
logger.debug(f"User {username} added to queue check list")
def notify_research_queued(self, username: str, research_id: str, **kwargs):
"""
Notify that a research was queued.
In direct mode, this immediately starts the research if slots are available.
In queue mode, it adds to the queue.
Args:
username: The username
research_id: The research ID
**kwargs: Additional parameters for direct execution (query, mode, etc.)
"""
# Pre-declared so the except handlers below can pass it to
# redact_secrets even on paths where it is never assigned.
password = None
# Check user's queue_mode setting when we have database access
if kwargs:
session_id = kwargs.get("session_id")
if session_id:
# Check if we can start it directly
password = session_password_store.get_session_password(
username, session_id
)
if password:
# TOCTOU guard against post-logout DB resurrection.
# Between the read above and open_user_database() below the
# user can log out: logout clears the session password store
# (auth/routes.py step 2) and THEN closes the DB. Re-opening
# here with the captured password would resurrect the user's
# decrypted database — and even resume a queued research —
# after logout. Re-read the store immediately before the
# open; if the credential is gone the user has logged out,
# so abort the direct start (``password`` becomes None and
# the block below is skipped). Keeping this re-check adjacent
# to the open shrinks the residual window to a negligible
# remainder — the store clear and the DB open live in
# different subsystems and are not jointly lockable without a
# fragile cross-subsystem lock. Otherwise adopt the freshest
# credential (never the stale captured one) for both the open
# and _start_research_directly. On abort we fall through to
# the queue-mode path below, which re-reads fresh and no-ops
# cleanly when the user is logged out.
current_password = (
session_password_store.get_session_password(
username, session_id
)
)
if current_password is None:
logger.info(
f"Aborting direct-start DB open for {username}: "
"session credential cleared since read (logout?); "
"leaving research queued"
)
password = current_password
if password:
# The canonical ordering is admission gate -> database.
# Fresh starts, logout/rekey, and library initialisation
# all follow that order. Opening/querying a pooled session
# first and then waiting for the gate can deadlock under
# pool exhaustion with a gate holder waiting for a
# connection. The inner admission section below is
# intentionally retained: the shared gate is reentrant and
# that narrower block documents the count -> claim scope.
admission_lock = self._get_user_critical_lock(username)
admission_lock.acquire()
try:
# Open database and check settings + active count
engine = db_manager.open_user_database(
username, password
)
if engine:
with get_user_db_session(username) as db_session:
# Get user's settings using SettingsManager
from ...settings.manager import SettingsManager
settings_manager = SettingsManager(db_session)
# Get user's queue_mode setting (env > DB > default)
queue_mode = settings_manager.get_setting(
"app.queue_mode", "direct"
)
# Get user's max concurrent setting (env > DB > default),
# clamped to the global semaphore ceiling so a stale or
# user-inflated value can't monopolize it.
max_concurrent = clamp_user_max_concurrent(
settings_manager.get_setting(
"app.max_concurrent_researches", 3
)
)
logger.debug(
f"User {username} settings: queue_mode={queue_mode}, "
f"max_concurrent={max_concurrent}"
)
# Only try direct execution if user has queue_mode="direct"
if queue_mode == "direct":
# Serialise the count→check→start critical
# section at the application layer. Two
# concurrent submissions for the same user
# must not both observe the same active
# count and both start — that would exceed
# max_concurrent. A per-user Python lock
# gives us that atomicity independent of
# the DB isolation level.
with self._get_user_critical_lock(username):
active_count = (
db_session.query(UserActiveResearch)
.filter_by(
username=username,
status=ResearchStatus.IN_PROGRESS,
)
.count()
)
if active_count < max_concurrent:
# We have slots - start directly!
logger.info(
f"Direct mode: Starting research {research_id} immediately "
f"(active: {active_count}/{max_concurrent})"
)
# Start the research directly
outcome = (
self._start_research_directly(
username,
research_id,
password,
**kwargs,
)
)
if (
outcome
is not _DirectStartOutcome.QUEUE_FALLBACK
):
return
logger.info(
"Direct start deferred for "
f"{research_id}; queueing for "
"a later dispatch"
)
else:
logger.info(
"Direct mode: Max concurrent "
f"reached ({active_count}/"
f"{max_concurrent}), queueing "
f"{research_id}"
)
else:
logger.info(
f"User {username} has queue_mode={queue_mode}, "
f"queueing research {research_id}"
)
except Exception as e:
# ``password`` is in scope — drop the traceback
# chain and redact str(e) so the SQLCipher master
# password can't leak via diagnose=True frame
# locals (see the generic handler in
# _start_research_directly for the full rationale).
safe_msg = redact_secrets(str(e), password)
logger.warning(
f"Error in direct execution for {username}: {safe_msg}"
)
finally:
admission_lock.release()
# Fall back to queue mode (or if direct mode failed)
try:
with get_user_db_session(username) as session:
queue_service = UserQueueService(session)
queue_service.add_task_metadata(
task_id=research_id,
task_type="research",
priority=0,
)
logger.info(
f"Research {research_id} queued for user {username}"
)
# Register the user with the processing loop. The loop only
# dispatches users present in _users_to_check; under Flask that
# set was fed by a before_request hook on every authenticated
# request (deleted in the FastAPI migration), so without this
# call a queued research would never start. The loop keeps
# re-checking a registered user every tick until their queue
# drains, so this single registration also covers the
# at-capacity case: when a running research finishes, the next
# tick dispatches the queued one.
session_id = kwargs.get("session_id")
if session_id:
self.notify_user_activity(username, session_id)
else:
logger.warning(
f"Research {research_id} queued for {username} without "
"a session_id — the processing loop cannot pick it up "
"until the user logs in again"
)
except Exception as e:
# ``password`` may be bound above — same redaction rationale.
safe_msg = redact_secrets(str(e), password)
logger.warning(
f"Failed to update queue status for {username}: {safe_msg}"
)
def _start_research_directly(
self, username: str, research_id: str, password: str, **kwargs
) -> _DirectStartOutcome:
"""
Start a research directly without queueing.
Args:
username: The username
research_id: The research ID
password: The user's password
**kwargs: Research parameters (query, mode, settings, etc.)
"""
query = kwargs.get("query")
mode = kwargs.get("mode")
settings_snapshot = kwargs.get("settings_snapshot", {})
# Create active research record
active_record_persisted = False
try:
with get_user_db_session(username) as db_session:
active_record = UserActiveResearch(
username=username,
research_id=research_id,
status=ResearchStatus.IN_PROGRESS,
thread_id="pending",
settings_snapshot=settings_snapshot,
)
db_session.add(active_record)
db_session.commit()
active_record_persisted = True
# Update task status if it exists
queue_service = UserQueueService(db_session)
queue_service.update_task_status(research_id, "processing")
except Exception as e:
# ``password`` is a parameter of this method — drop the
# traceback chain and redact str(e) (full rationale at the
# generic handler below).
safe_msg = redact_secrets(str(e), password)
logger.warning(
f"Failed to create active research record for {research_id}: {safe_msg}"
)
# The first commit lands before TaskMetadata is transitioned. If
# that later update failed, remove the now-stale active row before
# asking the caller to use the normal queue fallback.
if active_record_persisted:
try:
with get_user_db_session(username) as db_session:
active_record = (
db_session.query(UserActiveResearch)
.filter_by(
username=username, research_id=research_id
)
.first()
)
if active_record:
db_session.delete(active_record)
research_row = (
db_session.query(ResearchHistory)
.filter_by(id=research_id)
.first()
)
if research_row:
research_row.status = ResearchStatus.QUEUED
db_session.commit()
except Exception as cleanup_error:
safe_msg = redact_secrets(str(cleanup_error), password)
logger.warning(
"Cleanup after direct-start setup failure failed for "
f"{research_id}: {safe_msg}"
)
return _DirectStartOutcome.QUEUE_FALLBACK
# Extract parameters from kwargs
model_provider = kwargs.get("model_provider")
model = kwargs.get("model")
custom_endpoint = kwargs.get("custom_endpoint")
search_engine = kwargs.get("search_engine")
# Start the research process
try:
research_thread = start_research_process(
research_id,
query,
mode,
run_research_process,
username=username,
user_password=password,
model_provider=model_provider,
model=model,
custom_endpoint=custom_endpoint,
search_engine=search_engine,
max_results=kwargs.get("max_results"),
time_period=kwargs.get("time_period"),
iterations=kwargs.get("iterations"),
questions_per_iteration=kwargs.get("questions_per_iteration"),
strategy=kwargs.get("strategy", "source-based"),
settings_snapshot=settings_snapshot,
)
# Update thread ID
try:
with get_user_db_session(username) as db_session:
active_record = (
db_session.query(UserActiveResearch)
.filter_by(username=username, research_id=research_id)
.first()
)
if active_record:
active_record.thread_id = str(research_thread.ident)
db_session.commit()
except Exception as e:
# ``password`` is in scope — same redaction rationale.
safe_msg = redact_secrets(str(e), password)
logger.warning(
f"Failed to update thread ID for {research_id}: {safe_msg}"
)
logger.info(
f"Direct execution: Started research {research_id} for user {username} "
f"in thread {research_thread.ident}"
)
return _DirectStartOutcome.STARTED
except DuplicateResearchError:
# A live thread already owns this research_id. Do NOT delete
# the UserActiveResearch row or mark ResearchHistory FAILED —
# that state belongs to the live thread, and mutating it
# would terminate a running research from the user's
# perspective while it keeps executing. Same contract as the
# queue processor's dedicated dup branch (#3506).
logger.warning(
f"Duplicate live thread detected for {research_id} "
"in direct mode; leaving state intact"
)
return _DirectStartOutcome.ALREADY_RUNNING
except SystemAtCapacityError:
# System at concurrent-research capacity in the direct-execution
# path. Roll back the IN_PROGRESS active row and mark history
# back to QUEUED so the queue processor can pick it up later.
logger.info(
f"Direct execution hit capacity for {research_id}; re-queueing"
)
try:
with get_user_db_session(username) as db_session:
active_record = (
db_session.query(UserActiveResearch)
.filter_by(username=username, research_id=research_id)
.first()
)
if active_record:
db_session.delete(active_record)
research_row = (
db_session.query(ResearchHistory)
.filter_by(id=research_id)
.first()
)
if research_row:
research_row.status = ResearchStatus.QUEUED
db_session.commit()
except Exception as e:
# ``password`` is in scope — same redaction rationale.
safe_msg = redact_secrets(str(e), password)
logger.warning(
f"Cleanup after capacity reject failed for "
f"{research_id}; the stale UserActiveResearch row is "
f"recovered by reclaim_stale_user_active_research: {safe_msg}"
)
# Let notify_research_queued create TaskMetadata through its one
# canonical queue-fallback path and register (username, session_id)
# with the processing loop. Handling metadata here used to make
# the row look queued while silently omitting that registration.
return _DirectStartOutcome.QUEUE_FALLBACK
except Exception as e:
# ``password`` is in lexical scope (function parameter,
# passed through to ``start_research_process``). A
# SQLAlchemy / requests exception from anywhere in
# ``start_research_process`` could carry frame locals that
# include the SQLCipher master password (which is
# unrecoverable — see TRUST.md §5). Drop the traceback chain
# and redact str(e) defensively.
safe_msg = redact_secrets(str(e), password)
logger.warning(
f"Failed to start research {research_id} directly: {safe_msg}"
)
# Clean up the active record AND mark the research terminal
# FAILED so the user-visible state matches reality (no running
# thread, not IN_PROGRESS). Same contract as the queue
# processor's terminal-failure branch (#3481).
try:
with get_user_db_session(username) as db_session:
active_record = (
db_session.query(UserActiveResearch)
.filter_by(username=username, research_id=research_id)
.first()
)
if active_record:
db_session.delete(active_record)
research_row = (
db_session.query(ResearchHistory)
.filter_by(id=research_id)
.first()
)
if research_row:
research_row.status = ResearchStatus.FAILED
# This direct attempt originated from an already-persisted
# queue row. A terminal spawn failure must consume that row
# and its TaskMetadata atomically with FAILED, otherwise the
# non-QUEUED parent makes it permanently undispatchable
# while queue counters continue to report it.
cleanup_queued_research_state(
db_session,
[research_id],
include_claimed=True,
acting_user=username,
)
db_session.commit()
except Exception as e2:
# ``password`` is in scope — same redaction rationale.
safe_msg = redact_secrets(str(e2), password)
logger.warning(
f"Failed to clean up active research record for {research_id}: {safe_msg}"
)
return _DirectStartOutcome.TERMINAL_FAILURE
@staticmethod
def _drop_active_research_row(session, username: str, research_id: str):
"""Delete the per-user concurrency-cap row for a finished research.
``UserActiveResearch`` exists to count what a user currently has
running. Main deleted finished rows from a ``before_request`` hook
(``web/auth/cleanup_middleware.cleanup_completed_research``) that
sampled ~1% of requests, walked the user's rows, and dropped any
whose thread was no longer active. That hook has no successor
under FastAPI, and nothing replaced its *delete*: the spawn-failure
branches drop their own row, but neither terminal notification
below did, so every normally-completed research left its row
behind permanently.
The visible effect is not a wrong concurrency count —
``reclaim_stale_user_active_research`` re-derives that at the
user's next start, so the cap self-heals — but the rows are never
removed, only flipped to FAILED, and only for a user who starts
another research. A user who finishes and stops accumulates rows
with no purge path anywhere in the codebase, and the ones that do
get reclaimed are mislabelled FAILED despite having succeeded.
Deleting at the terminal notification is strictly better than
main's sampled sweep: it is event-driven, so it cannot miss, and
it costs one statement on a path that already holds an open
session — no extra connection, no per-request overhead.
The caller's session context does not commit on exit, so commit
here rather than relying on a later write to flush this one.
"""
from ...database.models import UserActiveResearch
try:
deleted = (
session.query(UserActiveResearch)
.filter_by(username=username, research_id=research_id)
.delete(synchronize_session=False)
)
if deleted:
session.commit()
logger.debug(
f"Dropped UserActiveResearch row for {research_id}"
)
except Exception:
# Never let bookkeeping break the completion notification the
# user is actually waiting on; the row stays and the next
# reclaim sweep will pick it up.
logger.exception(
f"Could not drop UserActiveResearch row for {research_id}"
)
# Swallowing is not enough: a failed statement leaves the
# session raising PendingRollbackError on everything that
# follows, and the caller runs
# ``send_research_completed_notification_from_session`` on
# *this* session on the very next line. Without the rollback
# the swallow would hand that notification a dead session --
# exactly what the comment above promises not to do. Main did
# the same (web/auth/cleanup_middleware.py, five rollbacks).
try:
session.rollback()
except Exception:
# A genuinely exhausted pool can fail the rollback too;
# this path is user-visible completion, so still no raise.
logger.warning(
"Could not roll back after a failed UserActiveResearch "
f"delete for {research_id}"
)
def notify_research_completed(
self, username: str, research_id: str, user_password: str | None = None
):
"""
Notify that a research completed.
Updates the user's queue status in their database.
Args:
username: The username
research_id: The research ID
user_password: User password for database access. Required for queue
updates and database lookups during notification sending.
Optional only because some callers may not have it
available, in which case only basic updates occur.
"""
try:
# get_user_db_session is already imported at module level (line 19)
# It accepts optional password parameter and returns a context manager
with get_user_db_session(username, user_password) as session:
queue_service = UserQueueService(session)
queue_service.update_task_status(
research_id, ResearchStatus.COMPLETED
)
self._drop_active_research_row(session, username, research_id)
logger.info(
f"Research {research_id} completed for user {username}"
)
# Send notification using helper from notification module
send_research_completed_notification_from_session(
username=username,
research_id=research_id,
db_session=session,
)
except Exception as e:
# ``user_password`` is a parameter of this method — drop the
# traceback chain and redact str(e) so the SQLCipher master
# password can't leak via diagnose=True frame locals.
safe_msg = redact_secrets(str(e), user_password)
logger.warning(
f"Failed to update completion status for {username}: {safe_msg}"
)
# Auto-convert research to document in History collection.
# Documents only — FAISS indexing is triggered separately by the user
# via "Index All" on the History page.
from ...research_library.search.services.research_history_indexer import (
auto_convert_research,
)
auto_convert_research(username, research_id, db_password=user_password)
# Auto-persist research sources (PDFs and text) into the Library.
# Default-off behind research_library.auto_persist_research_sources;
# runs in a daemon thread so source-heavy completions do not hold
# the active-research slot for sequential downloads.
def _auto_persist_async() -> None:
try:
from ...research_library.services.download_service import (
auto_persist_research_sources,
)
auto_persist_research_sources(
username, research_id, db_password=user_password
)
except Exception as exc:
safe_msg = redact_secrets(str(exc), user_password)
logger.debug(
f"auto-persist background failed for {research_id}: {safe_msg}"
)
finally:
_AUTO_PERSIST_PENDING.release()
try:
from ...database.thread_local_session import (
cleanup_current_thread,
)
cleanup_current_thread()
except Exception: # allow: silent-exception
pass
# Check the opt-out before enqueueing so disabled users (the
# default) never occupy a bounded sweep slot.
if not _auto_persist_enabled_for(username):
logger.debug(
f"auto-persist sweep not queued (disabled) for {research_id}"
)
elif not _AUTO_PERSIST_PENDING.acquire(blocking=False):
logger.warning(
f"auto-persist sweep skipped for {research_id}: "
f"{_AUTO_PERSIST_MAX_PENDING} sweeps already pending; "
"already-fetched PDFs remain persisted, and the text "
"sweep can be retriggered by a later library download"
)
else:
try:
executor = _get_auto_persist_executor()
executor.submit(_auto_persist_async)
except Exception as exc:
_AUTO_PERSIST_PENDING.release()
safe_msg = redact_secrets(str(exc), user_password)
logger.debug(
f"auto-persist submission failed for {research_id}: {safe_msg}"
)
def notify_research_failed(
self,
username: str,
research_id: str,
error_message: str | None = None,
user_password: str | None = None,
):
"""
Notify that a research failed.
Updates the user's queue status in their database and sends notification.
Args:
username: The username
research_id: The research ID
error_message: Optional error message
user_password: User password for database access. Required for queue
updates and database lookups during notification sending.
Optional only because some callers may not have it
available, in which case only basic updates occur.
"""
try:
# get_user_db_session is already imported at module level (line 19)
# It accepts optional password parameter and returns a context manager
with get_user_db_session(username, user_password) as session:
queue_service = UserQueueService(session)
queue_service.update_task_status(
research_id,
ResearchStatus.FAILED,
error_message=error_message,
)
self._drop_active_research_row(session, username, research_id)
logger.info(
f"Research {research_id} failed for user {username}: "
f"{error_message}"
)
# Send notification using helper from notification module
send_research_failed_notification_from_session(
username=username,
research_id=research_id,
error_message=error_message or "Unknown error",
db_session=session,
)
except Exception as e:
# ``user_password`` is a parameter of this method — same
# redaction rationale as notify_research_completed.
safe_msg = redact_secrets(str(e), user_password)
logger.warning(
f"Failed to update failure status for {username}: {safe_msg}"
)
def _process_queue_loop(self):
"""Main loop that processes the queue."""
while self.running:
try:
# Get list of users to check (don't clear immediately)
with self._users_lock:
users_to_check = list(self._users_to_check)
# Process each user's queue
users_to_remove = []
for user_session in users_to_check:
try:
username, session_id = user_session
# _process_user_queue returns True if queue is empty
queue_empty = self._process_user_queue(
username, session_id
)
if queue_empty:
users_to_remove.append(user_session)
except Exception:
logger.exception(
f"Error processing queue for user {username}"
)
# Don't remove on error - the _process_user_queue method
# determines whether to keep checking based on error type
# Only remove users whose queues are now empty
with self._users_lock:
for user_session in users_to_remove:
self._users_to_check.discard(user_session)
# Drain pending operations (progress / error updates queued
# by background threads via queue_progress_update /
# queue_error_update). Without this, FAILED status is never
# persisted unless the user happens to make another HTTP
# request that drains it from a request handler.
self._drain_pending_operations()
except Exception:
logger.exception("Error in queue processor loop")
finally:
# Clean up thread-local database session after each iteration.
# The loop opens a new session each iteration via get_user_db_session();
# closing it returns the connection to the shared QueuePool promptly.
try:
from ...database.thread_local_session import (
cleanup_current_thread,
cleanup_dead_threads,
)
cleanup_current_thread()
except Exception:
logger.debug(
"thread-local cleanup on shutdown", exc_info=True
)
# Periodic dead-thread credential sweep (every ~60s).
# One of three sweep trigger points (fastapi_app's
# post-request cleanup, connection_cleanup scheduler,
# and here).
self._loop_iteration += 1
if self._loop_iteration % 6 == 0: # Every ~60s (10s × 6)
try:
cleanup_dead_threads()
except Exception:
logger.debug(
"periodic dead-thread sweep", exc_info=True
)
# Event.wait, not time.sleep: stop() must be able to interrupt
# this pause, otherwise shutdown blocks for up to
# check_interval seconds.
self._stop_event.wait(self.check_interval)
def _drain_pending_operations(self) -> None:
"""Per-iteration drain of pending_operations across users.
Background threads (research workers) write into pending_operations
when they need to mark a research FAILED or update progress but
don't have direct DB access (e.g. password lookup failed). We
opportunistically drain by username here using the queue
processor's own DB access.
"""
with self._pending_operations_lock:
usernames = {
op["username"]
for op in self.pending_operations.values()
if op.get("username")
}
if not usernames:
return
for username in usernames:
try:
with get_user_db_session(username) as db_session:
n = self.process_pending_operations_for_user(
username, db_session
)
if n:
logger.info(
f"Drained {n} pending operation(s) for {username}"
)
except Exception:
# Likely no password available for this user yet — leave the
# ops queued; eviction will reap them after _PENDING_OPS_TTL.
logger.debug(
f"Could not drain pending ops for {username} this tick",
exc_info=True,
)
def _process_user_queue(self, username: str, session_id: str) -> bool:
"""
Process the queue for a specific user.
Args:
username: The username
session_id: The Flask session ID
Returns:
True if the queue is empty, False if there are still items
"""
# Get the user's password from session store
password = session_password_store.get_session_password(
username, session_id
)
if not password:
logger.debug(
f"No password available for user {username}, skipping queue check"
)
return True # Remove from checking - session expired
# Keep the process-wide lock order consistent with fresh starts and
# logout/rekey: per-user admission gate before database/session work.
# Taking a pooled session first and then blocking on the gate creates a
# resource cycle when the gate holder itself needs a connection.
# The acquire blocks without a timeout on purpose. Direct starts,
# follow-ups and chat send/retry hold this gate until their worker is
# registered (or its spawn-failure cleanup finishes), so a slow DB or
# spawn there delays this loop, and with it queue dispatch and pending
# progress/error writes for every user, until that holder returns. The
# wait is bounded by that start. A timeout that then carried on without
# the gate would let this tick reclaim a committed claim before its
# worker is registered.
admission_lock = self._get_user_critical_lock(username)
admission_lock.acquire()
# Open the user's encrypted database
try:
# First ensure the database is open
engine = db_manager.open_user_database(username, password)
if not engine:
logger.error(f"Failed to open database for user {username}")
return False # Keep checking - could be temporary DB issue
# Get a session and process the queue
with get_user_db_session(username, password) as db_session:
queue_service = UserQueueService(db_session)
sweep_result = self._sweep_missing_parent_queue_rows(
db_session, username
)
if not sweep_result.can_dispatch:
return False
# Get user's settings using SettingsManager
from ...settings.manager import SettingsManager
settings_manager = SettingsManager(db_session)
# Get user's max concurrent setting (env > DB > default),
# clamped to the global semaphore ceiling so a stale or
# user-inflated value can't monopolize it.
max_concurrent = clamp_user_max_concurrent(
settings_manager.get_setting(
"app.max_concurrent_researches", 3
)
)
# Serialize the live-count -> dispatch handoff with direct
# starts made through this processor. QueueStatus.active_tasks
# only counts tasks that previously travelled through the
# queue, so it misses already-running direct researches and
# cannot enforce the user's actual concurrency limit.
with self._get_user_critical_lock(username):
# A crashed worker can leave an IN_PROGRESS row behind.
# The fresh-submit route performs this same liveness
# reconciliation before counting; queue replay must do so
# as well or a dead row can block the queue forever.
from ..routes.globals import (
reclaim_stale_user_active_research,
)
if reclaim_stale_user_active_research(
db_session, username, logger=logger
):
db_session.commit()
active_count = (
db_session.query(UserActiveResearch)
.filter_by(
username=username,
status=ResearchStatus.IN_PROGRESS,
)
.count()
)
queue_status = queue_service.get_queue_status() or {
"queued_tasks": 0,
}
available_slots = max_concurrent - active_count
if available_slots <= 0:
# No slots available, but queue might not be empty.
return False # Keep checking
if queue_status["queued_tasks"] == 0 and (
db_session.query(QueuedResearch.id)
.filter_by(username=username)
.first()
is None
):
# A crashed claim still has a queue row but counts
# as processing, not queued. Let dispatch reclaim it
# before deciding this user has no pending work.
return True # Remove from checking
logger.info(
f"Processing queue for {username}: "
f"{active_count} active, "
f"{queue_status['queued_tasks']} queued, "
f"{available_slots} slots available"
)
# Process queued researches while retaining the same lock
# used for the capacity decision. Otherwise a direct
# notification could consume the last slot between the
# count and the queued spawn.
self._start_queued_researches(
db_session,
queue_service,
username,
password,
available_slots,
)
# Check if there are still items in queue.
updated_status = queue_service.get_queue_status() or {
"queued_tasks": 0
}
return bool(
updated_status["queued_tasks"] == 0
and db_session.query(QueuedResearch.id)
.filter_by(username=username)
.first()
is None
)
except Exception as e:
# ``password`` (from the session store above) is in scope —
# drop the traceback chain and redact str(e).
safe_msg = redact_secrets(str(e), password)
logger.warning(
f"Error processing queue for user {username}: {safe_msg}"
)
return False # Keep checking - errors might be temporary
finally:
admission_lock.release()
def _reclaim_stranded_queue_rows(
self, db_session: Session, username: str
) -> int:
"""Reclaim queue rows stranded by a crash or restart.
A row is stranded when ``is_processing=True`` but no live thread
exists in ``_active_research`` for its ``research_id``. This can
happen after a crash/restart between the pre-spawn IN_PROGRESS
commit and the queue-row deletion in ``_start_queued_researches``
— the row is invisible to the normal ``is_processing=False``
query and would never be retried.
Reclaims the ``ResearchHistory`` status and the
``TaskMetadata`` claim (``processing`` -> ``queued``) alongside
``is_processing``, so ``reconcile_research_queue_status`` keeps
reporting the row in ``queued_tasks`` (#6147). Removes leftovers
whose parent is already terminal, including previously released
claims, so they cannot block later queued work. Preserves claims
whose worker is still live. Returns the number of rows recovered.
"""
from ..routes.globals import is_research_active
terminal_statuses = (
ResearchStatus.COMPLETED,
ResearchStatus.SUSPENDED,
ResearchStatus.FAILED,
ResearchStatus.ERROR,
)
terminal_research_ids = select(ResearchHistory.id).where(
ResearchHistory.status.in_(terminal_statuses)
)
stranded = (
db_session.query(QueuedResearch)
.filter_by(username=username)
.filter(
QueuedResearch.is_processing.is_(True)
| QueuedResearch.research_id.in_(terminal_research_ids)
)
.all()
)
reclaimed = 0
finished_ids: set[str] = set()
for row in stranded:
if is_research_active(row.research_id):
# A legitimate in-flight claim; don't touch.
continue
research = (
db_session.query(ResearchHistory)
.filter_by(id=row.research_id)
.first()
)
if research is not None and research.status in terminal_statuses:
finished_ids.add(row.research_id)
continue
row.is_processing = False
status_changed = (
research is not None
and research.status == ResearchStatus.IN_PROGRESS
)
if status_changed:
research.status = ResearchStatus.QUEUED
metadata = (
db_session.query(TaskMetadata)
.filter_by(task_id=row.research_id, task_type="research")
.first()
)
metadata_changed = (
metadata is not None and metadata.status == "processing"
)
if metadata_changed:
# A stranded claim has no live worker, so a 'processing'
# TaskMetadata row is stale by definition. Reverting it
# keeps queued_tasks accurate so the dispatch loop does
# not deregister the user before retrying this row
# (#6147).
metadata.status = "queued"
metadata.started_at = None
reclaimed += 1
logger.warning(
f"Reclaimed stranded queue row for research "
f"{row.research_id} (user {username}): no live thread, "
"resetting is_processing=False"
+ (" and status=QUEUED" if status_changed else "")
+ (
" and TaskMetadata.status=queued"
if metadata_changed
else ""
)
)
if finished_ids:
cleanup_queued_research_state(
db_session,
finished_ids,
include_claimed=True,
acting_user=username,
)
if reclaimed or finished_ids:
reconcile_research_queue_status(db_session, force=True)
if not self._commit_with_safe_rollback(
db_session,
f"reclaim of stranded rows for user {username}",
):
return 0
# Discard retry bookkeeping only after the database cleanup is
# durable. A rollback must leave the whole claim recoverable.
with self._spawn_retry_counts_lock:
for research_id in finished_ids:
self._spawn_retry_counts.pop(research_id, None)
return reclaimed + len(finished_ids)
def _sweep_missing_parent_queue_rows(
self, db_session: Session, username: str
) -> QueueSweepResult:
queued_orphan_ids = {
research_id
for (research_id,) in (
db_session.query(QueuedResearch.research_id)
.outerjoin(
ResearchHistory,
QueuedResearch.research_id == ResearchHistory.id,
)
.filter(
QueuedResearch.username == username,
ResearchHistory.id.is_(None),
)
.all()
)
}
# TaskMetadata has no owner column. A row that another user's queue
# entry points at is theirs, so it is left out here, and
# cleanup_queued_research_state leaves it too when given the acting
# user. The queue row stands in for the owner only because
# queued_researches.research_id is UNIQUE; a test in
# test_queued_research_lifecycle_cleanup.py pins that.
metadata_orphan_ids = {
task_id
for (task_id,) in (
db_session.query(TaskMetadata.task_id)
.outerjoin(
ResearchHistory,
TaskMetadata.task_id == ResearchHistory.id,
)
.outerjoin(
QueuedResearch,
TaskMetadata.task_id == QueuedResearch.research_id,
)
.filter(
TaskMetadata.task_type == "research",
ResearchHistory.id.is_(None),
or_(
QueuedResearch.research_id.is_(None),
QueuedResearch.username == username,
),
)
.all()
)
}
orphaned_research_ids = queued_orphan_ids | metadata_orphan_ids
if not orphaned_research_ids:
if reconcile_research_queue_status(db_session):
if not self._commit_with_safe_rollback(
db_session,
f"queue status reconciliation for user {username}",
):
return QueueSweepResult(False, frozenset())
return QueueSweepResult(True, frozenset())
cleanup_result = cleanup_queued_research_state(
db_session,
orphaned_research_ids,
include_claimed=True,
acting_user=username,
)
if not self._commit_with_safe_rollback(
db_session,
f"missing-parent queue sweep for user {username}",
):
return QueueSweepResult(False, frozenset())
with self._spawn_retry_counts_lock:
for research_id in cleanup_result.cleaned_ids:
self._spawn_retry_counts.pop(research_id, None)
return QueueSweepResult(True, cleanup_result.cleaned_ids)
def _start_queued_researches(
self,
db_session: Session,
queue_service: UserQueueService,
username: str,
password: str,
available_slots: int,
):
"""Start queued researches up to available slots."""
sweep_result = self._sweep_missing_parent_queue_rows(
db_session, username
)
if not sweep_result.can_dispatch:
return
# Before picking work, reclaim any rows stranded by a prior
# crash — otherwise they are invisible to the is_processing=False
# filter below and would never retry.
self._reclaim_stranded_queue_rows(db_session, username)
# Get queued researches
queued = (
db_session.query(QueuedResearch)
.filter_by(username=username, is_processing=False)
.order_by(QueuedResearch.position, QueuedResearch.id)
.limit(available_slots)
.all()
)
for queued_research in queued:
research_id = queued_research.research_id
try:
# Atomically claim this item by flipping is_processing from
# False to True in a single UPDATE. If another worker has
# already claimed it since our SELECT above, the UPDATE will
# match zero rows and we skip. Under non-IMMEDIATE isolation
# the previous SELECT+assign pattern would race and two
# workers could both process the same queued item.
claimed = (
db_session.query(QueuedResearch)
.filter(
QueuedResearch.id == queued_research.id,
QueuedResearch.is_processing.is_(False),
db_session.query(ResearchHistory.id)
.filter(
ResearchHistory.id == research_id,
ResearchHistory.status == ResearchStatus.QUEUED,
)
.exists(),
)
.update(
{QueuedResearch.is_processing: True},
synchronize_session=False,
)
)
db_session.commit()
if not claimed:
logger.debug(
f"Queued research {research_id} "
f"already claimed by another worker; skipping"
)
continue
# Refresh local object state now that we hold the claim
db_session.refresh(queued_research)
# Update task status
queue_service.update_task_status(research_id, "processing")
# Start the research
self._start_research(
db_session,
username,
password,
queued_research,
)
# Success — clear any prior spawn-failure count and
# remove the queue row.
with self._spawn_retry_counts_lock:
self._spawn_retry_counts.pop(research_id, None)
db_session.delete(queued_research)
db_session.commit()
logger.info(
f"Started queued research {research_id} for user {username}"
)
except DuplicateResearchError:
# Raised by _start_research when a prior attempt's thread
# is still live, OR when the ResearchHistory row is in a
# non-QUEUED state (IN_PROGRESS from a prior attempt's
# successful pre-spawn commit; terminal COMPLETED /
# FAILED / SUSPENDED from a thread that already finished
# and cleaned up). In every case the correct behavior is
# the same: clear the stale queue row and the retry
# counter, and do NOT fall through to the FAILED/notify
# path — that would terminate-status a live thread or
# emit a false failure for a completed one.
logger.warning(
f"Research {research_id} is already started "
"(live thread or non-QUEUED status); clearing stale "
"queue row"
)
with self._spawn_retry_counts_lock:
self._spawn_retry_counts.pop(research_id, None)
self._delete_queue_row_safely(db_session, username, research_id)
continue
except SystemAtCapacityError:
# System hit the global concurrent-research capacity while
# dispatching this queued item. _start_research already
# reset the ResearchHistory row back to QUEUED before
# re-raising. This is a transient condition, NOT a spawn
# failure, so it must NOT count toward SPAWN_RETRY_LIMIT —
# otherwise a busy system would wrongly mark a perfectly
# valid queued research FAILED after a few ticks. Just
# release the processing claim so the next tick retries.
# Mirrors the dedicated handler in _start_research_directly.
logger.info(
f"System at capacity dispatching queued research "
f"{research_id}; leaving queued for next tick"
)
# Revert the queued->processing claim from
# update_task_status("processing") above. The research stays
# queued for the next tick, so its slot must return to
# queued_tasks rather than leaking into active_tasks on
# every capacity-rejected retry.
queue_service.update_task_status(research_id, "queued")
fresh_queued = (
db_session.query(QueuedResearch)
.filter_by(username=username, research_id=research_id)
.first()
)
if fresh_queued:
fresh_queued.is_processing = False
self._commit_with_safe_rollback(
db_session,
"is_processing reset after capacity reject for "
f"research {research_id}",
)
continue
except InvalidQueuedResearchOverridesError as error:
logger.warning(
"Invalid persisted search overrides for queued research "
f"{research_id}; marking FAILED"
)
with self._spawn_retry_counts_lock:
self._spawn_retry_counts.pop(research_id, None)
research = (
db_session.query(ResearchHistory)
.filter_by(id=research_id)
.first()
)
fresh_queued = (
db_session.query(QueuedResearch)
.filter_by(username=username, research_id=research_id)
.first()
)
if research:
research.status = ResearchStatus.FAILED
if fresh_queued:
db_session.delete(fresh_queued)
self._commit_with_safe_rollback(
db_session,
"terminal invalid persisted overrides for research "
f"{research_id}",
)
self.notify_research_failed(
username=username,
research_id=research_id,
error_message=str(error),
user_password=password,
)
continue
except Exception as e:
# ``password`` is a parameter of this method — drop the
# traceback chain and redact str(e).
safe_msg = redact_secrets(str(e), password)
logger.warning(
f"Error starting queued research {research_id}: {safe_msg}"
)
# Session may be in PendingRollbackError state after a
# failed commit inside _start_research.
try:
db_session.rollback()
except Exception as rb_err:
# No exc_info: ``password`` is in this frame and a
# rendered traceback could expose it via
# diagnose=True frame locals.
logger.debug(
"Rollback after start failure: "
f"{redact_secrets(str(rb_err), password)}"
)
attempts = self._bump_spawn_retry_count(research_id)
# Re-query in case rollback expired the ORM object.
fresh_queued = (
db_session.query(QueuedResearch)
.filter_by(username=username, research_id=research_id)
.first()
)
if attempts < SPAWN_RETRY_LIMIT:
# Transient failure — allow the next loop tick to
# retry. _start_research rolls back its own
# IN_PROGRESS write on spawn failure, so the fix-ups
# needed here are resetting is_processing and
# reverting the TaskMetadata claim from
# update_task_status("processing") above. Without
# the revert, reconcile_research_queue_status
# under-reports queued_tasks and
# _process_user_queue deregisters the user before
# this row is retried (#6147).
logger.warning(
f"Spawn failed for research {research_id} "
f"(attempt {attempts}/{SPAWN_RETRY_LIMIT}), "
"leaving queued for retry"
)
# Mirror the SystemAtCapacityError branch: revert
# the queued->processing claim so the slot returns
# to queued_tasks instead of leaking into
# active_tasks on every failed attempt.
queue_service.update_task_status(research_id, "queued")
if fresh_queued:
fresh_queued.is_processing = False
reconcile_research_queue_status(db_session, force=True)
self._commit_with_safe_rollback(
db_session,
f"is_processing reset for research {research_id}",
)
continue
# Exhausted retries — mark terminal FAILED, delete the
# queue row to stop re-dispatch, and notify the user.
# The spawn failure was already logged (redacted) at the
# top of this except block; no need to repeat it here.
logger.warning(
f"Spawn failed for research {research_id} "
f"after {attempts} attempts; marking FAILED"
)
with self._spawn_retry_counts_lock:
self._spawn_retry_counts.pop(research_id, None)
try:
research = (
db_session.query(ResearchHistory)
.filter_by(id=research_id)
.first()
)
if research:
research.status = ResearchStatus.FAILED
if fresh_queued:
db_session.delete(fresh_queued)
db_session.commit()
except Exception as e2:
# ``password`` is in scope — same redaction rationale.
safe_msg = redact_secrets(str(e2), password)
logger.warning(
"Failed to persist terminal FAILED state for "
f"research {research_id}: {safe_msg}"
)
try:
db_session.rollback()
except Exception as rb_err:
# No exc_info: same frame-locals rationale as the
# rollback handler above.
logger.debug(
"Rollback after terminal update failure: "
f"{redact_secrets(str(rb_err), password)}"
)
# notify_research_failed opens its own session and
# sends the user notification. Called exactly once
# per research_id because the counter is popped above.
self.notify_research_failed(
username=username,
research_id=research_id,
error_message=(
f"Failed to start research after {attempts} attempts"
),
user_password=password,
)
def _start_research(
self,
db_session: Session,
username: str,
password: str,
queued_research,
):
"""Start a queued research.
Commits ``ResearchHistory.status = IN_PROGRESS`` BEFORE spawning
the thread. If we did this after, a fast-completing thread
(which opens its own DB session) could write ``COMPLETED`` and
then our post-spawn commit would overwrite that with
``IN_PROGRESS``, stranding the research as stuck IN_PROGRESS
after it had already finished.
If ``start_research_process`` raises, reset status back to
``QUEUED`` and re-raise so the caller's 3-strike retry logic
handles it. ``DuplicateResearchError`` is re-raised as-is
because a thread is already running for this research; mutating
status further would be wrong.
"""
research_id = queued_research.research_id
research = (
db_session.query(ResearchHistory).filter_by(id=research_id).first()
)
if not research:
raise ValueError(f"Research {research_id} not found")
# Guard against re-entering _start_research on a retry when a
# prior attempt's post-spawn UserActiveResearch commit failed:
# - IN_PROGRESS means the prior thread is (or was) running.
# - COMPLETED/FAILED means the prior thread already finished
# and cleaned itself up out of _active_research, so a bare
# retry would both overwrite the terminal status with
# IN_PROGRESS and then spawn a *second* thread (because
# check_and_start_research sees no live entry), re-running
# the whole research.
# In all three cases the correct behavior is the same: raise
# DuplicateResearchError so the caller's existing dup branch
# deletes the queue row without mutating status or notifying.
if research.status != ResearchStatus.QUEUED:
raise DuplicateResearchError(
f"Research {research_id} is already started "
f"(status={research.status})"
)
# Extract settings
settings_snapshot = queued_research.settings_snapshot or {}
# Handle new vs legacy structure
if (
isinstance(settings_snapshot, dict)
and "submission" in settings_snapshot
):
submission_params = settings_snapshot.get("submission", {})
complete_settings = settings_snapshot.get("settings_snapshot", {})
allowed_override_keys = (
"model_provider",
"model",
"custom_endpoint",
"search_engine",
"max_results",
"time_period",
"iterations",
"questions_per_iteration",
"strategy",
)
explicit_override_keys = settings_snapshot.get(
"submission_overrides"
)
override_keys = (
explicit_override_keys
if isinstance(explicit_override_keys, list)
else allowed_override_keys
)
runtime_overrides = {
key: submission_params[key]
for key in allowed_override_keys
if key in override_keys and key in submission_params
}
else:
submission_params = settings_snapshot
# A legacy-flat queued row (enqueued by an older version, pre-
# "submission" wrapper) carries no settings_snapshot. Seed the run's
# primary engine from the submitted search_engine so the worker's
# egress build (resolve_run_primary_engine) doesn't fail closed on
# an empty snapshot and refuse the run.
_legacy_engine = submission_params.get("search_engine")
complete_settings = (
{"search.tool": _legacy_engine} if _legacy_engine else {}
)
runtime_overrides = {
"model_provider": submission_params.get("model_provider"),
"model": submission_params.get("model"),
"custom_endpoint": submission_params.get("custom_endpoint"),
"search_engine": submission_params.get("search_engine"),
"max_results": submission_params.get("max_results"),
"time_period": submission_params.get("time_period"),
"iterations": submission_params.get("iterations"),
"questions_per_iteration": submission_params.get(
"questions_per_iteration"
),
"strategy": submission_params.get("strategy", "source-based"),
}
query_validation_error = validate_research_query_length(
queued_research.query
)
if query_validation_error is not None:
raise InvalidQueuedResearchOverridesError(query_validation_error)
validation_error = validate_search_overrides(runtime_overrides)
if validation_error is not None:
raise InvalidQueuedResearchOverridesError(validation_error)
# Claim IN_PROGRESS before spawn to close the
# thread-completes-before-parent-commits race.
research.status = ResearchStatus.IN_PROGRESS
db_session.commit()
try:
research_thread = start_research_process(
research_id,
queued_research.query,
queued_research.mode,
run_research_process,
username=username,
user_password=password, # Pass password for metrics
settings_snapshot=complete_settings,
**runtime_overrides,
)
except DuplicateResearchError:
# A live thread already exists for this research_id (e.g.
# previous attempt's post-spawn commit failed). Do NOT
# reset status — that would contradict the running thread.
raise
except SystemAtCapacityError:
# System at concurrent-research capacity. No thread was
# spawned. Reset to QUEUED so the next dispatch tick can try
# again — this is not a permanent spawn failure and should
# NOT count toward SPAWN_RETRY_LIMIT.
logger.info(
f"System at capacity when dispatching {research_id}; "
"re-queueing for next tick"
)
research.status = ResearchStatus.QUEUED
self._commit_with_safe_rollback(
db_session,
f"status reset to QUEUED after capacity reject for research {research_id}",
)
raise
except Exception:
# Genuine spawn failure: no thread exists. Roll back the
# IN_PROGRESS claim so the retry sees a clean QUEUED row.
research.status = ResearchStatus.QUEUED
self._commit_with_safe_rollback(
db_session,
f"status reset to QUEUED after spawn failure for research {research_id}",
)
raise
# Thread is running. Record the active-research row. If this
# commit fails the live thread is unrecorded but still running.
# Raise DuplicateResearchError instead of letting a generic
# exception propagate, so the caller's dup branch cleans up the
# queue row without bumping the retry counter — if we let this
# count as a spawn failure, three consecutive post-spawn commit
# failures (or one at LIMIT-1) would push the counter to
# SPAWN_RETRY_LIMIT and mark a LIVE thread as terminal FAILED.
active_record = UserActiveResearch(
username=username,
research_id=research_id,
status=ResearchStatus.IN_PROGRESS,
thread_id=str(research_thread.ident),
settings_snapshot=queued_research.settings_snapshot,
)
db_session.add(active_record)
if not self._commit_with_safe_rollback(
db_session,
f"UserActiveResearch persist after spawn for research {research_id}",
):
# Thread is live; the commit failing leaves the UAR row
# unrecorded but the thread running. Raise
# DuplicateResearchError so the caller's dup branch deletes
# the queue row without bumping the retry counter — if we
# let a plain exception count as a spawn failure, a commit
# failure at SPAWN_RETRY_LIMIT - 1 would mark a LIVE thread
# as terminal FAILED.
raise DuplicateResearchError(
f"Research {research_id} thread is live; "
"UserActiveResearch commit failed"
)
# Keep pending_operations bounded: if no request comes in to drain
# them (user never navigates after a crash), they would otherwise
# accumulate forever in memory. We drop oldest entries past this cap
# and entries older than this TTL.
_PENDING_OPS_MAX = 10_000
_PENDING_OPS_TTL_SECONDS = 24 * 60 * 60 # 24 hours
def reconcile_orphan_active_research(
self, username: str, db_session
) -> int:
"""Mark stale UserActiveResearch rows as FAILED.
Called after a user logs in (when their DB is open). If the server
was restarted mid-research, rows with status=IN_PROGRESS have a
thread_id pointing to a now-dead thread. Leaving them as IN_PROGRESS
means the dashboard shows "running forever" with no way to cancel.
Only rows whose worker thread is actually DEAD are reconciled. This
runs on EVERY login, so without the liveness guard a user opening a
second tab / logging in on another device while a research is still
running would wrongly flag that live research FAILED — and
delete_attempt could then hard-delete it out from under the running
worker. Mirrors the is_research_thread_alive() guard chat.py and the
research-start reclaim already use.
Returns the number of rows reconciled.
"""
from datetime import UTC, datetime
from ..research_state import cleanup_research, is_research_thread_alive
try:
stale = (
db_session.query(UserActiveResearch)
.filter_by(username=username, status=ResearchStatus.IN_PROGRESS)
.all()
)
if not stale:
return 0
count = 0
for record in stale:
# Skip rows whose worker thread is still alive — those are
# genuinely running (e.g. a concurrent login), not orphans.
if is_research_thread_alive(record.research_id):
continue
# Also mark the ResearchHistory row as failed
research = (
db_session.query(ResearchHistory)
.filter_by(id=record.research_id)
.first()
)
if research and research.status == ResearchStatus.IN_PROGRESS:
research.status = ResearchStatus.FAILED
research.completed_at = datetime.now(UTC).isoformat()
meta = dict(research.research_meta or {})
meta["failure_reason"] = (
"Server restarted while research was in progress"
)
research.research_meta = meta
record.status = ResearchStatus.FAILED
# Drop the dead thread's in-memory entry too, mirroring
# reclaim_stale_user_active_research. Without this the stale
# _active_research entry outlives the DB reconciliation, so
# capacity checks keep counting a research that is already
# FAILED and its termination flag never clears.
cleanup_research(record.research_id)
count += 1
db_session.commit()
logger.info(
f"Reconciled {count} orphan IN_PROGRESS research records for {username}"
)
return count
except Exception:
logger.exception(
f"Failed to reconcile orphan research records for {username}"
)
return 0
def _evict_stale_pending_operations(self) -> None:
"""Drop expired and over-capacity entries. Caller must hold the lock."""
now = time.time()
# TTL eviction
expired = [
op_id
for op_id, op in self.pending_operations.items()
if now - op.get("timestamp", now) > self._PENDING_OPS_TTL_SECONDS
]
for op_id in expired:
del self.pending_operations[op_id]
# Size cap — drop oldest entries if over
overflow = len(self.pending_operations) - self._PENDING_OPS_MAX
if overflow > 0:
oldest = sorted(
self.pending_operations.items(),
key=lambda kv: kv[1].get("timestamp", 0),
)[:overflow]
for op_id, _ in oldest:
del self.pending_operations[op_id]
def queue_progress_update(
self, username: str, research_id: str, progress: float
):
"""
Queue a progress update that needs database access.
For compatibility with old processor during migration.
Args:
username: The username
research_id: The research ID
progress: The progress value (0-100)
"""
# In processor_v2, we can update directly if we have database access
# or queue it for later processing
operation_id = str(uuid.uuid4())
with self._pending_operations_lock:
self.pending_operations[operation_id] = {
"username": username,
"operation_type": "progress_update",
"research_id": research_id,
"progress": progress,
"timestamp": time.time(),
}
self._evict_stale_pending_operations()
logger.debug(
f"Queued progress update for research {research_id}: {progress}%"
)
def queue_error_update(
self,
username: str,
research_id: str,
status: str,
error_message: str,
metadata: Dict[str, Any],
completed_at: str,
report_path: Optional[str] = None,
):
"""
Queue an error status update that needs database access.
For compatibility with old processor during migration.
Args:
username: The username
research_id: The research ID
status: The status to set (failed, suspended, etc.)
error_message: The error message
metadata: Research metadata
completed_at: Completion timestamp
report_path: Optional path to error report
"""
operation_id = str(uuid.uuid4())
with self._pending_operations_lock:
self.pending_operations[operation_id] = {
"username": username,
"operation_type": "error_update",
"research_id": research_id,
"status": status,
"error_message": error_message,
"metadata": metadata,
"completed_at": completed_at,
"report_path": report_path,
"timestamp": time.time(),
}
self._evict_stale_pending_operations()
logger.info(
f"Queued error update for research {research_id} with status {status}"
)
def process_pending_operations_for_user(
self, username: str, db_session: Session
) -> int:
"""
Process pending operations for a user when we have database access.
Called from request context where encrypted database is accessible.
For compatibility with old processor during migration.
Args:
username: Username to process operations for
db_session: Active database session for the user
Returns:
Number of operations cleared from the queue
"""
# Find pending operations for this user (with lock)
raw_operations = []
with self._pending_operations_lock:
for op_id, op_data in list(self.pending_operations.items()):
if op_data["username"] == username:
raw_operations.append((op_id, op_data))
# Remove immediately to prevent duplicate processing
del self.pending_operations[op_id]
if not raw_operations:
return 0
# Deduplicate and consolidate operations by research_id to minimize DB locks
latest_progress = {}
latest_errors = {}
other_ops = []
for op_id, op_data in raw_operations:
op_type = op_data.get("operation_type")
rid = op_data.get("research_id")
ts = op_data.get("timestamp", 0)
if op_type == "progress_update" and rid:
if rid not in latest_progress or ts >= latest_progress[rid][
1
].get("timestamp", 0):
latest_progress[rid] = (op_id, op_data)
elif op_type == "error_update" and rid:
if rid not in latest_errors or ts >= latest_errors[rid][1].get(
"timestamp", 0
):
latest_errors[rid] = (op_id, op_data)
else:
other_ops.append((op_id, op_data))
operations_to_process = (
list(latest_progress.values())
+ list(latest_errors.values())
+ other_ops
)
from sqlalchemy.exc import (
OperationalError,
PendingRollbackError,
TimeoutError,
)
from ...database.models import ResearchHistory
max_retries = 3
backoff = 0.05
max_requeue_attempts = 10
# Track the raw count of operations being flushed so the returned
# count reflects "operations cleared from the queue" rather than
# "deduplicated batches applied". Set after a successful commit
# so retries don't double-count.
processed_count = 0
for attempt in range(max_retries):
try:
# Consolidate DB queries: fetch all relevant ResearchHistory rows in a single batch
rids = list(
{
op_data["research_id"]
for _, op_data in operations_to_process
if op_data.get("research_id")
}
)
if rids:
researches = (
db_session.query(ResearchHistory)
.filter(ResearchHistory.id.in_(rids))
.all()
)
research_map = {r.id: r for r in researches}
else:
research_map = {}
for op_id, op_data in operations_to_process:
op_type = op_data.get("operation_type")
rid = op_data.get("research_id")
research = research_map.get(rid)
if not research:
continue
if op_type == "progress_update":
research.progress = op_data.get("progress")
elif op_type == "error_update":
research.status = op_data.get("status", "failed")
research.error_message = op_data.get("error_message")
incoming_meta = op_data.get("metadata")
existing_meta = research.research_meta
if isinstance(incoming_meta, dict):
# Merge, never wholesale-replace: a bare
# {"terminated_at": ...} from handle_termination
# must not wipe the row's settings_snapshot (and
# submission block) that view/export rendering
# resolves report.uncited_sources_mode from.
# Callers that already merged (the failure path)
# are unaffected: re-merging is idempotent.
# Read legacy JSON-string metadata using the
# same normalization as completion updates.
merged_meta = _parse_research_metadata(
existing_meta
)
merged_meta.update(incoming_meta)
research.research_meta = merged_meta
else:
research.research_meta = incoming_meta
research.completed_at = op_data.get("completed_at")
report_path = op_data.get("report_path")
if report_path:
research.report_path = report_path
# Single commit for the entire batch of consolidated updates.
# Count is set AFTER the commit succeeds so retries don't
# double-count the raw operations.
db_session.commit()
processed_count = len(raw_operations)
break
except (OperationalError, PendingRollbackError, TimeoutError) as e:
try:
db_session.rollback()
except Exception as rollback_err:
logger.debug(f"Queue flush rollback failed: {rollback_err}")
if attempt < max_retries - 1:
logger.debug(
f"Database locked during queue flush for {username} (attempt {attempt + 1}/{max_retries}); retrying in {backoff * 1000:.0f}ms: {type(e).__name__}"
)
time.sleep(backoff)
backoff *= 2
else:
logger.warning(
f"Failed to commit queue operations for {username} after {max_retries} retries: {type(e).__name__}"
)
# Re-queue raw operations so data is not lost, up to max_requeue_attempts
with self._pending_operations_lock:
for op_id, op_data in raw_operations:
retry_count = op_data.get("retry_count", 0) + 1
if retry_count > max_requeue_attempts:
logger.exception(
f"Dropping queued operation {op_id} ({op_data.get('operation_type')}) for user {username} "
f"after exceeding maximum requeue attempts ({max_requeue_attempts})"
)
elif op_id not in self.pending_operations:
op_data["retry_count"] = retry_count
self.pending_operations[op_id] = op_data
processed_count = 0
except Exception:
logger.exception(
f"Unexpected error committing queue operations for {username}"
)
try:
db_session.rollback()
except Exception as rollback_err:
logger.debug(f"Queue flush rollback failed: {rollback_err}")
# Re-queue raw operations up to max_requeue_attempts
with self._pending_operations_lock:
for op_id, op_data in raw_operations:
retry_count = op_data.get("retry_count", 0) + 1
if retry_count > max_requeue_attempts:
logger.exception(
f"Dropping queued operation {op_id} ({op_data.get('operation_type')}) for user {username} "
f"after exceeding maximum requeue attempts ({max_requeue_attempts})"
)
elif op_id not in self.pending_operations:
op_data["retry_count"] = retry_count
self.pending_operations[op_id] = op_data
processed_count = 0
break
return processed_count
# Global queue processor instance
queue_processor = QueueProcessorV2()