~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for QueueProcessorV2 — real integration tests for core queue orchestration.
These tests exercise the actual class methods with proper mocking of external
dependencies (database, password store, research service), unlike the fake tests
in test_processor_v2_core/operations/research.py that only test Python primitives.
Tests cover:
- notify_research_queued: direct vs queue mode branching
- notify_research_completed: status updates + notifications
- notify_research_failed: error message defaulting, sanitization
- _process_queue_loop: user:session parsing, removal logic
- _process_user_queue: slot calculation, empty queue, no password
- _start_research: new vs legacy settings format
- process_pending_operations_for_user: progress/error updates, rollback
"""
import threading
import time
from unittest.mock import Mock, MagicMock, patch
import pytest
MODULE = "local_deep_research.web.queue.processor_v2"
SETTINGS_MGR = "local_deep_research.settings.manager.SettingsManager"
def _make_processor():
"""Create a QueueProcessorV2 instance without triggering module-level side effects."""
from local_deep_research.web.queue.processor_v2 import QueueProcessorV2
return QueueProcessorV2(check_interval=1)
def _stop_loop_after_call(processor, fn=None):
"""Wrap a ``_process_user_queue`` replacement so the FIRST call it
receives deterministically stops ``_process_queue_loop`` after the
current iteration, without needing any sleep.
``_process_queue_loop`` captures the whole ``_users_to_check`` set at
the top of each iteration and processes all of it before re-checking
``while self.running:`` -- flipping ``running`` mid-iteration does not
cut that iteration short, it only prevents a *second* iteration from
starting. Setting ``_stop_event`` too makes the loop's tail
``self._stop_event.wait(self.check_interval)`` return immediately
instead of blocking for up to ``check_interval`` seconds. Together
that guarantees the loop runs exactly one full iteration and returns
right away, deterministically, regardless of how many users are in
``_users_to_check`` or how the machine happens to be scheduled.
``fn`` is the real per-call behavior (return value or side effect);
it is called first so its result/exception is preserved.
"""
def _wrapped(*args, **kwargs):
try:
if fn is not None:
return fn(*args, **kwargs)
finally:
processor.running = False
processor._stop_event.set()
return _wrapped
# Hard wall-clock bound for one _process_queue_loop run in tests. Must stay
# far below pytest's 180s global timeout (pyproject.toml) so a hung loop
# fails THIS test fast instead of stalling the whole suite/CI run.
LOOP_BOUND_TIMEOUT = 10.0
def _run_queue_loop_bounded(processor, timeout=LOOP_BOUND_TIMEOUT):
"""Run ``_process_queue_loop`` in a daemon thread under a hard
wall-clock bound, mirroring how production ``start()`` runs it.
The bound is a REAL external deadline enforced from the TEST thread:
``Thread.join(timeout)`` expires no matter what the loop does or
fails to do inside, so it does not depend on execution reaching ANY
production seam - not the tail ``self._stop_event.wait(...)``, not
``_process_user_queue``, nor any other call a regression could
skip, move, or block. The shapes that actually hang without this
bound - a ``continue`` or early ``return`` placed before the body's
``_process_user_queue`` call (so the mock stop never fires), a
blocking call in the loop body, or a loop that ignores both stop
signals - become a fast FAILURE via ``pytest.fail`` after
``timeout`` seconds instead of a hang until pytest's 180s global
timeout. Shapes that leave the ``_process_user_queue`` call
reachable (a ``continue`` after the body, a deleted or moved tail
wait) do not need the bound: ``_stop_loop_after_call`` flips
``running = False`` during that call, so ``while self.running:``
ends the loop on its next check regardless.
The happy path is unchanged: ``_stop_loop_after_call`` flips
``running`` off during the first ``_process_user_queue`` call, the
loop's tail wait returns immediately (the event is already set),
and the thread finishes in milliseconds. ``join`` returns only once
the thread is dead, so caller assertions (mock call counts,
``_users_to_check`` contents) run after full thread termination -
no race with loop code still in flight.
A worker exception is never silently lost: ``Thread.join()`` does
NOT propagate exceptions raised by the target, so the loop runs
inside a capturing shim and any ``BaseException`` the worker died
from is re-raised in THIS thread after a completed join. A
production change that makes ``_process_queue_loop`` raise (e.g.
an error handler that stops swallowing ``_process_user_queue``
failures) therefore fails the test, exactly like the direct
synchronous invocation used before the loop moved behind a thread
- instead of killing the worker silently while the caller's
assertions still pass.
On timeout, both production stop signals (``running = False`` and
``_stop_event.set()`` - exactly what ``stop()`` sends) are fired as
a best-effort teardown before failing, plus a short grace join, so
a loop that merely lost its pacing exits cleanly instead of
spinning through the rest of the suite. A loop that ignores even
those signals is daemon: it dies with the test process. That is a
weaker guarantee than it sounds - "the test process" is the whole
xdist worker, and ``pytest.fail`` unwinds the caller's
``patch.object`` block, restoring the REAL ``_process_user_queue``,
so a worker outliving the grace join would run production code
against live singletons for the rest of that worker's life. The
current loop exits within milliseconds of ``running = False`` +
``_stop_event.set()``, so reaching that state needs a second,
simultaneous regression that ignores both stop signals. If the
worker nevertheless died from an exception (during the timeout or
the grace join), that exception is named in the failure message
rather than silently dropped.
"""
worker_error = []
def _loop_target():
# Thread.join() does NOT propagate a target's exception: capture
# the worker's death cause so it can be re-raised in this test
# thread after a completed join, or named in the timeout failure.
try:
processor._process_queue_loop()
except BaseException as exc:
worker_error.append(exc)
processor.running = True
loop_thread = threading.Thread(
target=_loop_target,
name="test-bounded-queue-loop",
daemon=True,
)
loop_thread.start()
loop_thread.join(timeout)
if loop_thread.is_alive():
# Best-effort teardown first: production stop()'s two signals.
processor.running = False
processor._stop_event.set()
loop_thread.join(timeout=2.0)
failure = (
f"_process_queue_loop did not return within {timeout}s "
"wall-clock - loop-hang regression (e.g. an early "
"'continue' bypassing the tail wait, or a blocking call in "
"the loop body). This bound does not depend on any mocked "
"production seam. Stop signals were fired after the "
"timeout; loop thread still alive after 2s grace: "
f"{loop_thread.is_alive()}"
)
if worker_error:
failure += f" Worker thread also raised: {worker_error[0]!r}"
pytest.fail(failure)
if worker_error:
# Completed join and the worker died from an exception: re-raise
# it here so pytest reports the worker failure instead of the
# silently-dead-thread false green the bare join() would give.
raise worker_error[0]
def test_constructor_is_inert_until_explicit_start():
"""Construction initializes state without leaving behind any new
thread that is still alive when this function's second snapshot is
taken.
Regression guard: snapshots the set of live thread objects before
constructing QueueProcessorV2, then asserts no thread present after
construction was absent before. This catches a constructor that spawns
a worker thread that is still running at that point, even if that
thread is never assigned to ``processor.thread`` (a plain
``processor.thread is None`` check alone would miss an
unreferenced/leaked thread).
KNOWN BLIND SPOT (deliberate, not an oversight): ``threading.enumerate()``
lists only CURRENTLY-LIVE threads. A constructor that spawned a
short-lived, fire-and-forget thread which both starts AND finishes
before the ``set(threading.enumerate())`` line below runs would leave
``new_threads`` empty, and this test would pass even though a thread
was spawned. A hook that fires on every thread's bootstrap --
``threading.setprofile``/``settrace`` -- would close that gap:
confirmed against the installed CPython's ``threading.py``
(``Thread._bootstrap_inner``), the profile hook is installed via
``sys.setprofile(_profile_hook)`` *before* ``self.run()`` is invoked,
so it fires even for a thread that starts and dies immediately.
That mechanism was deliberately NOT adopted here: ``setprofile``
installs process-wide for the duration of the constructor call, so it
would also fire for the same unrelated anyio worker threads this test
already has to tolerate (see below) -- turning "starts during the
window" into a new source of exactly the flakiness this test's
one-directional design exists to avoid, to close a gap
(QueueProcessorV2.__init__ spawning a fire-and-forget thread) that is
not a pattern this constructor actually uses. If __init__ ever grows
one, prefer a targeted assertion (e.g. patching ``threading.Thread``
itself for the duration of this test) over broadening a process-wide
hook.
Deliberately a ONE-DIRECTIONAL set difference (``after - before``),
not an equality/count comparison: this test runs inside a shared
pytest process alongside ~100+ modules that use FastAPI's TestClient,
which spins up anyio "AnyIO worker thread" workers that anyio prunes
lazily on a later, unrelated ``run_sync`` call once they've been idle
for a while -- so unrelated threads can legitimately start or die in
the same few lines as the constructor call, for reasons that have
nothing to do with QueueProcessorV2. An equality check on
``active_count()`` or the ``enumerate()`` name set spuriously fails on
that unrelated churn (and a simultaneous start+death can even leave
``active_count()`` unchanged while a new thread genuinely appeared, so
active_count() alone is not a safe substitute either). Threads that
disappeared during the call (``before - after``) are not a defect in
the constructor and must not fail this test -- only a thread the
constructor itself introduced should.
Note: this test does NOT verify that app startup replacing the module
singleton (``processor_v2.queue_processor``) is observable — doing so
would require exercising the startup code path, and an independent
re-derivation of the module attribute (e.g. via importlib.reload) risks
re-running module-level side effects shared with the rest of the test
session. The previous ``processor_v2.queue_processor is singleton``
assertion was dropped because it only re-read the same attribute it had
just cached and could only fail if __init__ itself rebound the module
global -- not the "singleton survives startup" claim it was labeled as.
"""
from local_deep_research.web.queue import processor_v2
threads_before = set(threading.enumerate())
processor = processor_v2.QueueProcessorV2(check_interval=1)
assert processor.running is False
assert processor.thread is None
new_threads = set(threading.enumerate()) - threads_before
assert new_threads == set(), (
f"constructor introduced new thread(s): {new_threads}"
)
# ---------------------------------------------------------------------------
# notify_research_queued — direct vs queue mode branching
# ---------------------------------------------------------------------------
class TestNotifyResearchQueuedDirectMode:
"""Tests for notify_research_queued when queue_mode='direct'."""
@patch(f"{MODULE}.get_user_db_session")
@patch(f"{MODULE}.db_manager")
@patch(f"{MODULE}.session_password_store")
def test_direct_mode_slots_available_starts_directly(
self, mock_pw_store, mock_db_mgr, mock_get_session
):
"""direct mode + slots available -> calls _start_research_directly."""
processor = _make_processor()
mock_pw_store.get_session_password.return_value = "pw"
mock_db_mgr.open_user_database.return_value = Mock()
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_get_session.return_value = mock_session
# SettingsManager says direct mode, max 3 concurrent
mock_settings = Mock()
mock_settings.get_setting.side_effect = lambda key, default: {
"app.queue_mode": "direct",
"app.max_concurrent_researches": 3,
}.get(key, default)
# 0 active researches -> has slots
mock_session.query.return_value.filter_by.return_value.count.return_value = 0
with patch(SETTINGS_MGR, return_value=mock_settings):
with patch.object(
processor, "_start_research_directly"
) as mock_start:
processor.notify_research_queued(
"alice",
"r-001",
session_id="sess1",
query="test query",
)
mock_start.assert_called_once()
@patch(f"{MODULE}.get_user_db_session")
@patch(f"{MODULE}.db_manager")
@patch(f"{MODULE}.session_password_store")
def test_direct_mode_slots_full_falls_back_to_queue(
self, mock_pw_store, mock_db_mgr, mock_get_session
):
"""direct mode + slots full -> queues instead."""
processor = _make_processor()
mock_pw_store.get_session_password.return_value = "pw"
mock_db_mgr.open_user_database.return_value = Mock()
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_get_session.return_value = mock_session
mock_settings = Mock()
mock_settings.get_setting.side_effect = lambda key, default: {
"app.queue_mode": "direct",
"app.max_concurrent_researches": 2,
}.get(key, default)
# 2 active = full at max 2
mock_session.query.return_value.filter_by.return_value.count.return_value = 2
with patch(SETTINGS_MGR, return_value=mock_settings):
with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
mock_qs = Mock()
mock_qs_class.return_value = mock_qs
processor.notify_research_queued(
"alice",
"r-002",
session_id="sess1",
query="test",
)
mock_qs.add_task_metadata.assert_called_once()
@patch(f"{MODULE}.get_user_db_session")
@patch(f"{MODULE}.db_manager")
@patch(f"{MODULE}.session_password_store")
def test_queue_mode_always_queues(
self, mock_pw_store, mock_db_mgr, mock_get_session
):
"""queue_mode='queue' -> always queues regardless of slots."""
processor = _make_processor()
mock_pw_store.get_session_password.return_value = "pw"
mock_db_mgr.open_user_database.return_value = Mock()
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_get_session.return_value = mock_session
mock_settings = Mock()
mock_settings.get_setting.side_effect = lambda key, default: {
"app.queue_mode": "queue",
"app.max_concurrent_researches": 3,
}.get(key, default)
with patch(SETTINGS_MGR, return_value=mock_settings):
with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
mock_qs = Mock()
mock_qs_class.return_value = mock_qs
with patch.object(
processor, "_start_research_directly"
) as mock_start:
processor.notify_research_queued(
"alice",
"r-003",
session_id="sess1",
query="test",
)
mock_start.assert_not_called()
mock_qs.add_task_metadata.assert_called_once()
@patch(f"{MODULE}.get_user_db_session")
def test_missing_session_id_falls_back_to_queue(self, mock_get_session):
"""No session_id kwarg -> falls back to queue mode."""
processor = _make_processor()
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_get_session.return_value = mock_session
with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
mock_qs = Mock()
mock_qs_class.return_value = mock_qs
# No session_id in kwargs
processor.notify_research_queued("alice", "r-004", query="test")
mock_qs.add_task_metadata.assert_called_once()
@patch(f"{MODULE}.get_user_db_session")
@patch(f"{MODULE}.session_password_store")
def test_missing_password_falls_back_to_queue(
self, mock_pw_store, mock_get_session
):
"""No password for session -> falls back to queue mode."""
processor = _make_processor()
mock_pw_store.get_session_password.return_value = None
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_get_session.return_value = mock_session
with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
mock_qs = Mock()
mock_qs_class.return_value = mock_qs
processor.notify_research_queued(
"alice", "r-005", session_id="sess1", query="test"
)
mock_qs.add_task_metadata.assert_called_once()
@patch(f"{MODULE}.get_user_db_session")
@patch(f"{MODULE}.db_manager")
@patch(f"{MODULE}.session_password_store")
def test_exception_in_direct_falls_back_to_queue(
self, mock_pw_store, mock_db_mgr, mock_get_session
):
"""Exception during direct execution -> falls back to queue."""
processor = _make_processor()
mock_pw_store.get_session_password.return_value = "pw"
mock_db_mgr.open_user_database.side_effect = RuntimeError("DB error")
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_get_session.return_value = mock_session
with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
mock_qs = Mock()
mock_qs_class.return_value = mock_qs
processor.notify_research_queued(
"alice", "r-006", session_id="sess1", query="test"
)
mock_qs.add_task_metadata.assert_called_once()
@patch(f"{MODULE}.get_user_db_session")
def test_no_kwargs_queues_directly(self, mock_get_session):
"""No kwargs at all -> goes straight to queue fallback."""
processor = _make_processor()
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_get_session.return_value = mock_session
with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
mock_qs = Mock()
mock_qs_class.return_value = mock_qs
processor.notify_research_queued("alice", "r-007")
mock_qs.add_task_metadata.assert_called_once()
# ---------------------------------------------------------------------------
# notify_research_completed — updates status + sends notification
# ---------------------------------------------------------------------------
class TestNotifyResearchCompleted:
"""Tests for notify_research_completed."""
@patch(f"{MODULE}.send_research_completed_notification_from_session")
@patch(f"{MODULE}.get_user_db_session")
def test_updates_status_and_sends_notification(
self, mock_get_session, mock_send_notif
):
"""Updates queue status to COMPLETED and sends notification."""
processor = _make_processor()
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_get_session.return_value = mock_session
with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
mock_qs = Mock()
mock_qs_class.return_value = mock_qs
processor.notify_research_completed(
"alice", "r-010", user_password="pw"
)
mock_qs.update_task_status.assert_called_once()
mock_send_notif.assert_called_once_with(
username="alice",
research_id="r-010",
db_session=mock_session,
)
@patch(f"{MODULE}.get_user_db_session")
def test_exception_does_not_propagate(self, mock_get_session):
"""Exception in status update is caught silently."""
processor = _make_processor()
mock_get_session.side_effect = RuntimeError("DB unavailable")
# Should not raise
processor.notify_research_completed("alice", "r-011")
@patch(f"{MODULE}.send_research_completed_notification_from_session")
@patch(f"{MODULE}.get_user_db_session")
def test_passes_password_to_session(
self, mock_get_session, mock_send_notif
):
"""User password is passed to get_user_db_session."""
processor = _make_processor()
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_get_session.return_value = mock_session
with patch(f"{MODULE}.UserQueueService"):
processor.notify_research_completed(
"bob", "r-012", user_password="secret"
)
mock_get_session.assert_called_once_with("bob", "secret")
# ---------------------------------------------------------------------------
# notify_research_failed — error message defaulting, sanitization
# ---------------------------------------------------------------------------
class TestNotifyResearchFailed:
"""Tests for notify_research_failed."""
@patch(f"{MODULE}.send_research_failed_notification_from_session")
@patch(f"{MODULE}.get_user_db_session")
def test_sends_error_message_to_queue_service(
self, mock_get_session, mock_send_notif
):
"""Passes error_message to queue service update."""
processor = _make_processor()
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_get_session.return_value = mock_session
with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
mock_qs = Mock()
mock_qs_class.return_value = mock_qs
processor.notify_research_failed(
"alice", "r-020", error_message="OOM", user_password="pw"
)
mock_qs.update_task_status.assert_called_once()
args, kwargs = mock_qs.update_task_status.call_args
assert (
kwargs.get("error_message") == "OOM" or args[2]
if len(args) > 2
else True
)
@patch(f"{MODULE}.send_research_failed_notification_from_session")
@patch(f"{MODULE}.get_user_db_session")
def test_defaults_error_message_to_unknown(
self, mock_get_session, mock_send_notif
):
"""None error_message defaults to 'Unknown error' for notification."""
processor = _make_processor()
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_get_session.return_value = mock_session
with patch(f"{MODULE}.UserQueueService"):
processor.notify_research_failed(
"alice", "r-021", user_password="pw"
)
mock_send_notif.assert_called_once_with(
username="alice",
research_id="r-021",
error_message="Unknown error",
db_session=mock_session,
)
@patch(f"{MODULE}.get_user_db_session")
def test_exception_does_not_propagate(self, mock_get_session):
"""Exception in failure handling is caught."""
processor = _make_processor()
mock_get_session.side_effect = RuntimeError("DB error")
# Should not raise
processor.notify_research_failed("alice", "r-022", error_message="bad")
# ---------------------------------------------------------------------------
# _process_queue_loop — user:session parsing, removal logic
# ---------------------------------------------------------------------------
class TestProcessQueueLoop:
"""Tests for _process_queue_loop."""
def test_parses_user_session_format(self):
"""Correctly parses (username, session_id) tuple entries."""
processor = _make_processor()
processor._users_to_check.add(("alice", "sess-abc"))
with patch.object(
processor,
"_process_user_queue",
side_effect=_stop_loop_after_call(processor, lambda u, s: True),
) as mock_pq:
_run_queue_loop_bounded(processor)
mock_pq.assert_called_with("alice", "sess-abc")
def test_removes_users_with_empty_queues(self):
"""Users whose queues are empty get removed from check set."""
processor = _make_processor()
processor._users_to_check.add(("alice", "sess1"))
processor._users_to_check.add(("bob", "sess2"))
def side_effect(username, session_id):
return username == "alice" # alice's queue is empty
with patch.object(
processor,
"_process_user_queue",
side_effect=_stop_loop_after_call(processor, side_effect),
):
_run_queue_loop_bounded(processor)
assert ("alice", "sess1") not in processor._users_to_check
assert ("bob", "sess2") in processor._users_to_check
def test_keeps_users_on_error(self):
"""Users are kept if _process_user_queue raises."""
processor = _make_processor()
processor._users_to_check.add(("alice", "sess1"))
def raise_transient(username, session_id):
raise RuntimeError("transient")
with patch.object(
processor,
"_process_user_queue",
side_effect=_stop_loop_after_call(processor, raise_transient),
) as mock_pq:
_run_queue_loop_bounded(processor)
# Positive control: the "kept" assertion below is vacuous on its
# own (the test populated the set itself), so pin that the loop
# actually reached the raising call with the real arguments.
mock_pq.assert_called_once_with("alice", "sess1")
assert ("alice", "sess1") in processor._users_to_check
def test_unstoppable_loop_fails_fast_not_global_timeout(self):
"""The wall-clock bound itself must catch a loop that nothing
ever stops - the successor of the old seam-patch test.
With an empty ``_users_to_check`` the real
``_process_queue_loop`` never calls ``_process_user_queue``, so
no mock side effect ever flips ``running``: the loop keeps
tail-waiting forever. For a daemon worker between ``stop()``
calls that is honest production behavior, so what the test must
guarantee is the BOUND: ``_run_queue_loop_bounded`` turns an
unstoppable loop into a fast ``Failed`` (2s budget here, ~90x
under pytest's 180s global timeout), never a suite-stalling
hang. Nothing inside the loop is trusted to end it.
"""
processor = _make_processor()
assert processor._users_to_check == set()
timeout = 2.0
with patch.object(processor, "_process_user_queue") as mock_pq:
started = time.monotonic()
with pytest.raises(pytest.fail.Exception):
_run_queue_loop_bounded(processor, timeout=timeout)
elapsed = time.monotonic() - started
mock_pq.assert_not_called()
# Tied to the timeout this test passes, not a loose constant: a
# bound that silently ignored `timeout=` (falling back to
# LOOP_BOUND_TIMEOUT) or was raised must fail here rather than
# burn extra CI time unnoticed. The mechanism resolves at
# timeout + the 2s grace join; +4.0 is the CI-jitter margin.
assert elapsed < timeout + 4.0
def test_early_continue_regression_fails_fast_not_global_timeout(
self,
):
"""A regression that bypasses every stop seam is still bounded.
TEST-ONLY simulation (production code untouched): the loop is
replaced with a regressed shape that ``continue``s before the
body's ``_process_user_queue`` call, and therefore also before
the tail ``_stop_event.wait(...)``. That loop never reaches the
mock-driven exit - ``_stop_loop_after_call`` can only flip
``running`` from inside the call it wraps - never reaches the
tail wait, and never sleeps: it hot-spins while ``running``
stays True. This is the shape the bound exists for, and the one
no in-loop seam can catch. Only the bound's own deadline and
its ``running = False`` flip can end it, which is precisely
what this test exercises: the failure must arrive in seconds,
not at pytest's 180s global timeout.
"""
processor = _make_processor()
timeout = 2.0
def _loop_with_early_continue():
while processor.running:
# The regression under test: this continue skips the
# tail self._stop_event.wait(self.check_interval).
continue
with patch.object(
processor, "_process_queue_loop", _loop_with_early_continue
):
started = time.monotonic()
with pytest.raises(pytest.fail.Exception):
_run_queue_loop_bounded(processor, timeout=timeout)
elapsed = time.monotonic() - started
# Tied to the timeout this test passes, so a bound that was
# silently weakened (ignoring `timeout=`, or raised) fails here.
assert elapsed < timeout + 4.0
def test_worker_exception_surfaces_to_test_thread(self):
"""An exception raised inside the loop thread fails the test.
``Thread.join()`` does NOT propagate exceptions raised by the
thread target. Without the helper's capture-and-re-raise
wrapper, a production change that makes ``_process_queue_loop``
raise (for example an error handler that stops swallowing
``_process_user_queue`` failures) would kill the worker thread
silently: ``join`` returns normally and the caller's assertions
still run. Before this file moved the loop behind a thread, the
direct synchronous invocation propagated the exception and
failed the test - this guard keeps that regression coverage.
"""
processor = _make_processor()
def _loop_that_raises():
raise RuntimeError("worker-side failure must surface")
with patch.object(processor, "_process_queue_loop", _loop_that_raises):
with pytest.raises(RuntimeError, match="worker-side failure"):
_run_queue_loop_bounded(processor)
# ---------------------------------------------------------------------------
# _process_user_queue — slot calculation, empty queue, no password
# ---------------------------------------------------------------------------
class TestProcessUserQueue:
"""Tests for _process_user_queue."""
@patch(f"{MODULE}.session_password_store")
def test_no_password_returns_true(self, mock_pw_store):
"""No password -> returns True (remove from checking)."""
processor = _make_processor()
mock_pw_store.get_session_password.return_value = None
result = processor._process_user_queue("alice", "sess1")
assert result is True
@patch(f"{MODULE}.get_user_db_session")
@patch(f"{MODULE}.db_manager")
@patch(f"{MODULE}.session_password_store")
def test_empty_queue_returns_true(
self, mock_pw_store, mock_db_mgr, mock_get_session
):
"""Empty queue -> returns True."""
processor = _make_processor()
mock_pw_store.get_session_password.return_value = "pw"
mock_db_mgr.open_user_database.return_value = Mock()
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_session.query.return_value.filter_by.return_value.count.return_value = 0
mock_session.query.return_value.filter_by.return_value.first.return_value = None
mock_get_session.return_value = mock_session
mock_settings = Mock()
mock_settings.get_setting.return_value = 3
mock_qs = Mock()
mock_qs.get_queue_status.return_value = {
"active_tasks": 0,
"queued_tasks": 0,
}
with patch(SETTINGS_MGR, return_value=mock_settings):
with patch(f"{MODULE}.UserQueueService", return_value=mock_qs):
result = processor._process_user_queue("alice", "sess1")
assert result is True
@patch(f"{MODULE}.get_user_db_session")
@patch(f"{MODULE}.db_manager")
@patch(f"{MODULE}.session_password_store")
def test_no_slots_available_returns_false(
self, mock_pw_store, mock_db_mgr, mock_get_session
):
"""No available slots -> returns False (keep checking)."""
processor = _make_processor()
mock_pw_store.get_session_password.return_value = "pw"
mock_db_mgr.open_user_database.return_value = Mock()
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_get_session.return_value = mock_session
mock_settings = Mock()
mock_settings.get_setting.return_value = 2
mock_qs = Mock()
mock_qs.get_queue_status.return_value = {
"active_tasks": 2,
"queued_tasks": 1,
}
with patch(SETTINGS_MGR, return_value=mock_settings):
with patch(f"{MODULE}.UserQueueService", return_value=mock_qs):
result = processor._process_user_queue("alice", "sess1")
assert result is False
@patch(f"{MODULE}.db_manager")
@patch(f"{MODULE}.session_password_store")
def test_failed_db_open_returns_false(self, mock_pw_store, mock_db_mgr):
"""Failed database open -> returns False (keep checking)."""
processor = _make_processor()
mock_pw_store.get_session_password.return_value = "pw"
mock_db_mgr.open_user_database.return_value = None
result = processor._process_user_queue("alice", "sess1")
assert result is False
# ---------------------------------------------------------------------------
# _start_research — new vs legacy settings format
# ---------------------------------------------------------------------------
class TestStartResearch:
"""Tests for _start_research."""
@patch(f"{MODULE}.start_research_process")
def test_new_format_extracts_submission_and_settings(
self, mock_start_research
):
"""New format: {submission: {...}, settings_snapshot: {...}}."""
processor = _make_processor()
mock_thread = Mock()
mock_thread.ident = 12345
mock_start_research.return_value = mock_thread
mock_session = MagicMock()
mock_research = Mock()
mock_research.status = "queued"
mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
queued = Mock()
queued.research_id = "r-100"
queued.query = "test query"
queued.mode = "standard"
queued.settings_snapshot = {
"submission": {
"model_provider": "openai",
"model": "gpt-4",
"strategy": "graph-based",
},
"settings_snapshot": {"setting_a": "value_a"},
}
with patch(f"{MODULE}.UserActiveResearch"):
processor._start_research(mock_session, "alice", "pw", queued)
# Verify start_research_process was called with new-format extracted params
call_kwargs = mock_start_research.call_args.kwargs
assert call_kwargs["model_provider"] == "openai"
assert call_kwargs["model"] == "gpt-4"
assert call_kwargs["strategy"] == "graph-based"
assert call_kwargs["settings_snapshot"] == {"setting_a": "value_a"}
@patch(f"{MODULE}.start_research_process")
def test_new_format_omitted_settings_are_not_forwarded_as_overrides(
self, mock_start_research
):
# Given: a queued row whose request supplied only request-scoped limits.
from local_deep_research.constants import ResearchStatus
processor = _make_processor()
mock_start_research.return_value = Mock(ident=12345)
mock_session = MagicMock()
mock_research = Mock(status=ResearchStatus.QUEUED)
mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
queued = Mock(
research_id="r-deferred",
query="test query",
mode="quick",
settings_snapshot={
"submission": {
"model_provider": "enqueue-provider",
"model": "enqueue-model",
"search_engine": "enqueue-engine",
"max_results": 12,
"time_period": "w",
"iterations": 3,
"questions_per_iteration": 2,
"strategy": "enqueue-strategy",
},
"submission_overrides": ["max_results", "time_period"],
"settings_snapshot": {"search.tool": "snapshot-engine"},
},
)
# When: the queue processor dispatches the row.
with patch(f"{MODULE}.UserActiveResearch"):
processor._start_research(mock_session, "alice", "pw", queued)
# Then: absent configuration keys do not become explicit None/defaults.
call_kwargs = mock_start_research.call_args.kwargs
deferred_keys = {
"model_provider",
"model",
"custom_endpoint",
"search_engine",
"iterations",
"questions_per_iteration",
"strategy",
}
assert deferred_keys.isdisjoint(call_kwargs)
assert call_kwargs["max_results"] == 12
assert call_kwargs["time_period"] == "w"
@patch(f"{MODULE}.start_research_process")
def test_new_format_non_list_submission_overrides_falls_back(
self, mock_start_research
):
"""A malformed (non-list) submission_overrides must not trip the
substring-membership footgun (`key in "somestring"`). The isinstance
guard falls back to treating all present submission keys as overrides,
so a value containing none of the key names still forwards them.
"""
from local_deep_research.constants import ResearchStatus
processor = _make_processor()
mock_start_research.return_value = Mock(ident=12345)
mock_session = MagicMock()
mock_research = Mock(status=ResearchStatus.QUEUED)
mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
queued = Mock(
research_id="r-malformed-overrides",
query="test query",
mode="quick",
settings_snapshot={
"submission": {
"model": "enqueue-model",
"iterations": 4,
},
# Not a list — a corrupted value containing no key names.
# Without the isinstance guard, `"model" in "xyz"` is False
# and every key would be dropped instead.
"submission_overrides": "xyz",
"settings_snapshot": {"search.tool": "snapshot-engine"},
},
)
with patch(f"{MODULE}.UserActiveResearch"):
processor._start_research(mock_session, "alice", "pw", queued)
call_kwargs = mock_start_research.call_args.kwargs
assert call_kwargs["model"] == "enqueue-model"
assert call_kwargs["iterations"] == 4
@patch(f"{MODULE}.start_research_process")
def test_legacy_format_uses_flat_dict(self, mock_start_research):
"""Legacy format: flat dict used as submission_params."""
from local_deep_research.constants import ResearchStatus
processor = _make_processor()
mock_thread = Mock()
mock_thread.ident = 12345
mock_start_research.return_value = mock_thread
mock_session = MagicMock()
mock_research = Mock()
mock_research.status = ResearchStatus.QUEUED
mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
queued = Mock()
queued.research_id = "r-101"
queued.query = "legacy query"
queued.mode = "standard"
queued.settings_snapshot = {
"model_provider": "anthropic",
"model": "claude-3",
}
with patch(f"{MODULE}.UserActiveResearch"):
processor._start_research(mock_session, "alice", "pw", queued)
call_kwargs = mock_start_research.call_args.kwargs
assert call_kwargs["model_provider"] == "anthropic"
assert call_kwargs["model"] == "claude-3"
assert call_kwargs["settings_snapshot"] == {}
@patch(f"{MODULE}.start_research_process")
def test_legacy_format_seeds_search_tool_from_search_engine(
self, mock_start_research
):
"""A legacy-flat row that carries a search_engine seeds the (otherwise
empty) settings_snapshot with search.tool, so the worker's egress build
(resolve_run_primary_engine) doesn't fail closed on an empty snapshot
and refuse a replayed pre-upgrade queued run."""
from local_deep_research.constants import ResearchStatus
processor = _make_processor()
mock_thread = Mock()
mock_thread.ident = 1
mock_start_research.return_value = mock_thread
mock_session = MagicMock()
mock_research = Mock()
mock_research.status = ResearchStatus.QUEUED
mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
queued = Mock()
queued.research_id = "r-102"
queued.query = "legacy query"
queued.mode = "standard"
queued.settings_snapshot = {
"model_provider": "anthropic",
"search_engine": "pubmed",
}
with patch(f"{MODULE}.UserActiveResearch"):
processor._start_research(mock_session, "alice", "pw", queued)
call_kwargs = mock_start_research.call_args.kwargs
assert call_kwargs["settings_snapshot"] == {"search.tool": "pubmed"}
assert call_kwargs["search_engine"] == "pubmed"
@patch(f"{MODULE}.start_research_process")
def test_none_settings_snapshot_handled(self, mock_start_research):
"""None settings_snapshot treated as empty dict."""
from local_deep_research.constants import ResearchStatus
processor = _make_processor()
mock_thread = Mock()
mock_thread.ident = 12345
mock_start_research.return_value = mock_thread
mock_session = MagicMock()
mock_research = Mock()
mock_research.status = ResearchStatus.QUEUED
mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
queued = Mock()
queued.research_id = "r-102"
queued.query = "test"
queued.mode = "standard"
queued.settings_snapshot = None
with patch(f"{MODULE}.UserActiveResearch"):
processor._start_research(mock_session, "alice", "pw", queued)
call_kwargs = mock_start_research.call_args.kwargs
assert call_kwargs["settings_snapshot"] == {}
def test_research_not_found_raises(self):
"""Raises ValueError when research record not found."""
processor = _make_processor()
mock_session = MagicMock()
mock_session.query.return_value.filter_by.return_value.first.return_value = None
queued = Mock()
queued.research_id = "r-999"
queued.settings_snapshot = {}
with pytest.raises(ValueError, match="not found"):
processor._start_research(mock_session, "alice", "pw", queued)
@patch(f"{MODULE}.start_research_process")
def test_status_set_in_progress_before_spawn(self, mock_start_research):
"""IN_PROGRESS must be committed BEFORE start_research_process is
called, so a fast thread cannot race ahead and have its terminal
status overwritten by our post-spawn commit."""
from local_deep_research.constants import ResearchStatus
processor = _make_processor()
mock_thread = Mock()
mock_thread.ident = 999
mock_start_research.return_value = mock_thread
mock_session = MagicMock()
mock_research = Mock()
mock_research.status = ResearchStatus.QUEUED
mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
# Record the status at the moment start_research_process is called.
status_at_spawn = {}
def _capture_status(*args, **kwargs):
status_at_spawn["value"] = mock_research.status
# Capture the commit count at the moment of spawn so we can
# assert the IN_PROGRESS commit already happened, not merely
# that some commit happened at some point in the method.
status_at_spawn["commit_count"] = mock_session.commit.call_count
return mock_thread
mock_start_research.side_effect = _capture_status
queued = Mock()
queued.research_id = "r-race"
queued.query = "q"
queued.mode = "standard"
queued.settings_snapshot = {}
with patch(f"{MODULE}.UserActiveResearch"):
processor._start_research(mock_session, "alice", "pw", queued)
assert status_at_spawn["value"] == ResearchStatus.IN_PROGRESS
# Exactly one commit (the IN_PROGRESS write) must have already
# run by the time start_research_process is called.
assert status_at_spawn["commit_count"] == 1
@patch(f"{MODULE}.start_research_process")
def test_spawn_failure_resets_status_to_queued(self, mock_start_research):
"""On genuine spawn failure (non-duplicate), status must be rolled
back to QUEUED so the next retry sees a clean row and the
research is not stuck IN_PROGRESS if the retry budget is
eventually exhausted."""
from local_deep_research.constants import ResearchStatus
processor = _make_processor()
mock_start_research.side_effect = RuntimeError("spawn failed")
mock_session = MagicMock()
mock_research = Mock()
mock_research.status = ResearchStatus.QUEUED
mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
queued = Mock()
queued.research_id = "r-fail"
queued.query = "q"
queued.mode = "standard"
queued.settings_snapshot = {}
with pytest.raises(RuntimeError, match="spawn failed"):
processor._start_research(mock_session, "alice", "pw", queued)
assert mock_research.status == ResearchStatus.QUEUED
# One commit for IN_PROGRESS before spawn, a second for the QUEUED
# reset after spawn failed. Without the second commit the in-memory
# attribute is rolled back but the DB still shows IN_PROGRESS, so
# assert that both commits actually ran.
assert mock_session.commit.call_count >= 2
@patch(f"{MODULE}.start_research_process")
def test_duplicate_research_error_does_not_reset_status(
self, mock_start_research
):
"""DuplicateResearchError means a live thread already exists for
this research_id (typically a retry after a post-spawn commit
failure). Mutating status in that case would contradict the
running thread — it must be left IN_PROGRESS."""
from local_deep_research.constants import ResearchStatus
from local_deep_research.exceptions import DuplicateResearchError
processor = _make_processor()
mock_start_research.side_effect = DuplicateResearchError(
"thread already live"
)
mock_session = MagicMock()
mock_research = Mock()
mock_research.status = ResearchStatus.QUEUED
mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
queued = Mock()
queued.research_id = "r-dup"
queued.query = "q"
queued.mode = "standard"
queued.settings_snapshot = {}
with pytest.raises(DuplicateResearchError):
processor._start_research(mock_session, "alice", "pw", queued)
# Status must remain IN_PROGRESS (set before spawn, never reset).
assert mock_research.status == ResearchStatus.IN_PROGRESS
@patch(f"{MODULE}.start_research_process")
def test_non_queued_status_raises_duplicate_without_spawning(
self, mock_start_research
):
"""If _start_research is re-entered on a retry and finds the row
already in a non-QUEUED state (IN_PROGRESS from a prior attempt
whose post-spawn commit failed, or terminal COMPLETED/FAILED
because the prior thread already finished and cleaned up), it
must raise DuplicateResearchError *without* mutating status and
*without* calling start_research_process. Otherwise we would
overwrite a terminal status with IN_PROGRESS and spawn a second
thread that re-runs the whole research."""
from local_deep_research.constants import ResearchStatus
from local_deep_research.exceptions import DuplicateResearchError
for starting_status in (
ResearchStatus.IN_PROGRESS,
ResearchStatus.COMPLETED,
ResearchStatus.FAILED,
ResearchStatus.SUSPENDED,
):
processor = _make_processor()
mock_start_research.reset_mock()
mock_session = MagicMock()
mock_research = Mock()
mock_research.status = starting_status
mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
queued = Mock()
queued.research_id = f"r-stale-{starting_status}"
queued.query = "q"
queued.mode = "standard"
queued.settings_snapshot = {}
with pytest.raises(DuplicateResearchError):
processor._start_research(mock_session, "alice", "pw", queued)
# Status unchanged — we did not overwrite a terminal state.
assert mock_research.status == starting_status
# Spawn never attempted — no risk of a second thread.
mock_start_research.assert_not_called()
# No commit for a status we did not change.
mock_session.commit.assert_not_called()
@patch(f"{MODULE}.UserActiveResearch")
@patch(f"{MODULE}.start_research_process")
def test_user_active_research_commit_failure_raises_duplicate(
self, mock_start_research, mock_user_active_research
):
"""If the post-spawn UserActiveResearch commit fails, the thread
is already running. We must raise DuplicateResearchError so the
caller's dup branch deletes the queue row without bumping the
spawn-retry counter — otherwise a commit failure at the retry
limit would mark a LIVE thread as terminal FAILED."""
from local_deep_research.constants import ResearchStatus
from local_deep_research.exceptions import DuplicateResearchError
processor = _make_processor()
mock_thread = Mock()
mock_thread.ident = 42
mock_start_research.return_value = mock_thread
mock_session = MagicMock()
mock_research = Mock()
mock_research.status = ResearchStatus.QUEUED
mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
# Two commits: one for IN_PROGRESS (succeeds), one for
# UserActiveResearch (fails).
commit_results = [None, RuntimeError("UAR commit failed")]
def _commit():
r = commit_results.pop(0)
if isinstance(r, Exception):
raise r
mock_session.commit.side_effect = _commit
queued = Mock()
queued.research_id = "r-uar-fail"
queued.query = "q"
queued.mode = "standard"
queued.settings_snapshot = {}
with pytest.raises(DuplicateResearchError):
processor._start_research(mock_session, "alice", "pw", queued)
# Spawn *was* called (that's how we get into the post-spawn commit).
mock_start_research.assert_called_once()
# Rollback happened after the commit failure.
assert mock_session.rollback.called
# Status must remain IN_PROGRESS — the thread is live, resetting
# would contradict its own writes. This is the key invariant
# that distinguishes this branch from the generic spawn-failure
# branch (which DOES reset to QUEUED).
assert mock_research.status == ResearchStatus.IN_PROGRESS
# Regression guard: _start_research itself must not touch the
# spawn-retry counter on the UAR-commit-failure path. The
# counter is the caller's responsibility (the dup branch in
# _start_queued_researches clears it); if anyone moves bump
# logic into _start_research, this assertion catches it and
# prevents a live thread from being marked terminal FAILED at
# SPAWN_RETRY_LIMIT.
assert "r-uar-fail" not in processor._spawn_retry_counts
# ---------------------------------------------------------------------------
# Queue extraction — what `_start_research` forwards to the worker
# ---------------------------------------------------------------------------
class TestStartResearchExtraction:
"""Queue-to-worker extraction seam: what `_start_research` forwards.
Policy coercion is the WORKER's job — exercised below via
`build_run_egress_context`, not reimplemented here.
"""
@patch(f"{MODULE}.start_research_process")
def test_wrapped_snapshot_subdict_forwarded_verbatim(self, mock_start):
# Wrapped: the inner subdict is forwarded verbatim, including any
# stale policy.egress_scope — coercion is the worker's job.
from local_deep_research.constants import ResearchStatus
processor = _make_processor()
mock_start.return_value = Mock(ident=1)
session = MagicMock()
research = Mock(status=ResearchStatus.QUEUED)
session.query.return_value.filter_by.return_value.first.return_value = (
research
)
queued = Mock(
research_id="r-wrapped",
query="q",
mode="quick",
settings_snapshot={
"submission": {
"model_provider": "ollama",
"model": "llama3",
"search_engine": "pubmed",
},
"settings_snapshot": {
"policy.egress_scope": "unprotected",
"search.tool": "pubmed",
},
},
)
with patch(f"{MODULE}.UserActiveResearch"):
processor._start_research(session, "alice", "pw", queued)
dispatched = mock_start.call_args.kwargs["settings_snapshot"]
assert dispatched.get("policy.egress_scope") == "unprotected"
assert dispatched.get("search.tool") == "pubmed"
@patch(f"{MODULE}.start_research_process")
def test_legacy_flat_drops_policy_egress_scope(self, mock_start):
# Legacy flat: only search.tool is seeded; the top-level
# policy.egress_scope is dropped at extraction.
from local_deep_research.constants import ResearchStatus
processor = _make_processor()
mock_start.return_value = Mock(ident=1)
session = MagicMock()
research = Mock(status=ResearchStatus.QUEUED)
session.query.return_value.filter_by.return_value.first.return_value = (
research
)
queued = Mock(
research_id="r-flat",
query="q",
mode="quick",
settings_snapshot={
"policy.egress_scope": "unprotected",
"search_engine": "pubmed",
"model_provider": "ollama",
"model": "llama3",
},
)
with patch(f"{MODULE}.UserActiveResearch"):
processor._start_research(session, "alice", "pw", queued)
dispatched = mock_start.call_args.kwargs["settings_snapshot"]
assert "policy.egress_scope" not in dispatched
assert dispatched == {"search.tool": "pubmed"}
# ---------------------------------------------------------------------------
# Worker policy setup — `build_run_egress_context` (the production seam
# `run_research_process` invokes at startup)
# ---------------------------------------------------------------------------
class TestWorkerBuildRunEgressContext:
"""Worker policy seam: `build_run_egress_context` is the helper
`run_research_process` calls at startup to overlay current operator
env policy and construct the per-run EgressContext. The read-time
backstop in `context_from_snapshot` coerces a stale operator-disabled
``unprotected`` scope HERE — these tests fail if the helper is weakened
or the backstop is removed.
"""
def test_wrapped_stale_unprotected_coerced_when_gate_off(self, monkeypatch):
# Gate OFF (default): stale unprotected must not survive.
from local_deep_research.security.egress import (
EgressScope,
build_run_egress_context,
)
monkeypatch.delenv("LDR_POLICY_ALLOW_UNPROTECTED_EGRESS", raising=False)
monkeypatch.delenv("LDR_POLICY_EGRESS_SCOPE", raising=False)
snapshot = {
"policy.egress_scope": "unprotected",
"search.tool": "pubmed",
}
ctx = build_run_egress_context(snapshot, "pubmed", username="alice")
assert ctx.scope is not EgressScope.UNPROTECTED
def test_legacy_partial_snapshot_resolves_to_protected_default(
self, monkeypatch
):
# Gate OFF: a near-empty legacy snapshot resolves via the
# registry-default adaptive scope — never UNPROTECTED.
from local_deep_research.security.egress import (
EgressScope,
build_run_egress_context,
)
monkeypatch.delenv("LDR_POLICY_ALLOW_UNPROTECTED_EGRESS", raising=False)
monkeypatch.delenv("LDR_POLICY_EGRESS_SCOPE", raising=False)
snapshot = {"search.tool": "pubmed"} # legacy extraction output
ctx = build_run_egress_context(snapshot, "pubmed", username="alice")
assert ctx.scope is not EgressScope.UNPROTECTED
# ---------------------------------------------------------------------------
# process_pending_operations_for_user — progress/error updates, rollback
# ---------------------------------------------------------------------------
class TestProcessPendingOperationsForUser:
"""Tests for process_pending_operations_for_user."""
def test_progress_update_sets_progress(self):
"""Progress update operation sets research.progress."""
processor = _make_processor()
# Queue a progress update
processor.queue_progress_update("alice", "r-200", 42.5)
mock_session = MagicMock()
mock_research = Mock()
mock_research.id = "r-200"
mock_session.query.return_value.filter.return_value.all.return_value = [
mock_research
]
count = processor.process_pending_operations_for_user(
"alice", mock_session
)
assert count == 1
assert mock_research.progress == 42.5
mock_session.commit.assert_called()
def test_error_update_sets_all_fields(self):
"""Error update sets status, error_message, metadata, completed_at."""
processor = _make_processor()
processor.queue_error_update(
username="alice",
research_id="r-201",
status="failed",
error_message="OOM killed",
metadata={"retries": 3},
completed_at="2026-01-01T00:00:00",
report_path="/tmp/error.html",
)
mock_session = MagicMock()
mock_research = Mock()
mock_research.id = "r-201"
mock_session.query.return_value.filter.return_value.all.return_value = [
mock_research
]
count = processor.process_pending_operations_for_user(
"alice", mock_session
)
assert count == 1
assert mock_research.status == "failed"
assert mock_research.error_message == "OOM killed"
assert mock_research.research_meta == {"retries": 3}
assert mock_research.completed_at == "2026-01-01T00:00:00"
assert mock_research.report_path == "/tmp/error.html"
def test_no_operations_returns_zero(self):
"""No pending operations for user -> returns 0."""
processor = _make_processor()
mock_session = MagicMock()
count = processor.process_pending_operations_for_user(
"alice", mock_session
)
assert count == 0
def test_operations_for_other_users_not_processed(self):
"""Only processes operations for the specified user."""
processor = _make_processor()
processor.queue_progress_update("bob", "r-300", 50.0)
processor.queue_progress_update("alice", "r-301", 75.0)
mock_session = MagicMock()
mock_research = Mock()
mock_research.id = "r-301"
mock_session.query.return_value.filter.return_value.all.return_value = [
mock_research
]
count = processor.process_pending_operations_for_user(
"alice", mock_session
)
assert count == 1
assert mock_research.progress == 75.0
# Bob's operation should still be pending
assert len(processor.pending_operations) == 1
def test_exception_triggers_rollback(self):
"""Exception during processing triggers session rollback."""
processor = _make_processor()
processor.queue_progress_update("alice", "r-400", 50.0)
mock_session = MagicMock()
mock_session.query.side_effect = RuntimeError("DB error")
count = processor.process_pending_operations_for_user(
"alice", mock_session
)
assert count == 0
mock_session.rollback.assert_called_once()
def test_error_update_without_report_path(self):
"""Error update without report_path doesn't set report_path."""
processor = _make_processor()
processor.queue_error_update(
username="alice",
research_id="r-401",
status="suspended",
error_message="Timeout",
metadata={},
completed_at="2026-01-01",
)
mock_session = MagicMock()
mock_research = Mock(
spec=[
"id",
"status",
"error_message",
"research_meta",
"completed_at",
"report_path",
]
)
mock_research.id = "r-401"
mock_session.query.return_value.filter.return_value.all.return_value = [
mock_research
]
count = processor.process_pending_operations_for_user(
"alice", mock_session
)
assert count == 1
assert mock_research.status == "suspended"